From 3cd3328124daeac0cf911ea1a0b9ac128b93ef61 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:36:24 +0000 Subject: [PATCH 001/318] feat(libSceNet): forward sceNetSocketInternal and the Ex socket options to their public twins sceNetSocketInternal, sceNetGetsockoptEx and sceNetSetsockoptEx threw. Their argument lists, restored with the rest of the firmware exports in 4d6942f1e, are the same as sceNetSocket, sceNetGetsockopt and sceNetSetsockopt, so they now call those. Nothing in the tree records how the console tells the entry points apart (for example by the calling process); the debt file says so. Tested: guest_sce_net_tests creates a datagram socket through sceNetSocketInternal, sets SO_REUSEADDR and SO_RCVTIMEO through sceNetSetsockoptEx and reads them back through sceNetGetsockoptEx and sceNetGetsockopt with the same values. On a closed socket and on -1, and for an unsupported family and type, each Ex/Internal call returns the same error and errno as its public twin. --- core/libs/prx/libSceNet/Export.cpp | 9 +++---- core/libs/tests/GuestSceNet.cpp | 39 ++++++++++++++++++++++++++++++ docs/dev/debt/libSceNet.md | 1 + 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index a2185d3ee5..987eff5732 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -2417,8 +2417,7 @@ int APS5_VABI sceNetGetRouteInfo(std::int32_t family, NetRouteInfo* routes, std: } int APS5_VABI sceNetGetsockoptEx(int socketId, int level, int option, void* value, std::uint32_t* valueSize) { - NotImplemented_nid_no_patch(__func__); - return 0; + return sceNetGetsockopt(socketId, level, option, value, valueSize); } void APS5_VABI sceNetGetSystemTime(std::uint64_t* output) { @@ -2484,13 +2483,11 @@ int APS5_VABI sceNetSetDnsInfoToKernel(const NetDnsInfo* info, std::uint32_t fla } int APS5_VABI sceNetSetsockoptEx(int socketId, int level, int option, const void* value, std::uint32_t valueSize) { - NotImplemented_nid_no_patch(__func__); - return 0; + return sceNetSetsockopt(socketId, level, option, value, valueSize); } int APS5_VABI sceNetSocketInternal(const char* name, int domain, int type, int protocol) { - NotImplemented_nid_no_patch(__func__); - return 0; + return sceNetSocket(name, domain, type, protocol); } int APS5_VABI sceNetSyncCreate(NetSyncObject* sync, std::uint32_t signals, const char* name) { diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 6251ef3671..5f2e9c97b7 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -137,6 +137,9 @@ int APS5_VABI sceNetPoolDestroy(int); int APS5_VABI sceNetGetMemoryPoolStats(int, NetMemoryPoolStats*); const char* APS5_VABI sceNetGetIfName(std::uint32_t); int APS5_VABI sceNetGetRandom(std::uint32_t*); +int APS5_VABI sceNetSocketInternal(const char*, int, int, int); +int APS5_VABI sceNetGetsockoptEx(int, int, int, void*, std::uint32_t*); +int APS5_VABI sceNetSetsockoptEx(int, int, int, const void*, std::uint32_t); void* APS5_VABI sceNetMemoryAllocate(std::int32_t, int); void APS5_VABI sceNetMemoryFree(void*); } @@ -302,6 +305,41 @@ static void CheckIfName() { Require(missing != nullptr && missing[0] == '\0' && *sceNetErrnoLoc() == 22); } +static void CheckExForwards() { + const int internal = sceNetSocketInternal("internal", 2, 2, 17); + Require(internal >= 0); + const int enabled = 1; + Require(sceNetSetsockoptEx(internal, 0xffff, 0x4, &enabled, sizeof(enabled)) == 0); + int option = 0; + std::uint32_t optionSize = sizeof(option); + Require(sceNetGetsockoptEx(internal, 0xffff, 0x4, &option, &optionSize) == 0); + int publicOption = 0; + std::uint32_t publicSize = sizeof(publicOption); + Require(sceNetGetsockopt(internal, 0xffff, 0x4, &publicOption, &publicSize) == 0); + Require(option == publicOption && option != 0 && optionSize == publicSize); + const KernelTimeval timeout{0, 250000}; + Require(sceNetSetsockoptEx(internal, 0xffff, 0x1006, &timeout, sizeof(timeout)) == 0); + KernelTimeval read{}; + std::uint32_t readSize = sizeof(read); + Require(sceNetGetsockoptEx(internal, 0xffff, 0x1006, &read, &readSize) == 0); + Require(readSize == sizeof(read) && read.tv_sec == 0 && read.tv_usec == 250000); + Require(sceNetSocketClose(internal) == 0); + for (const int closed : {internal, -1}) { + optionSize = sizeof(option); + const int ex = sceNetGetsockoptEx(closed, 0xffff, 0x4, &option, &optionSize); + const int exError = *sceNetErrnoLoc(); + Require(ex < 0 && ex == sceNetGetsockopt(closed, 0xffff, 0x4, &option, &optionSize) && exError == *sceNetErrnoLoc()); + const int set = sceNetSetsockoptEx(closed, 0xffff, 0x4, &enabled, sizeof(enabled)); + const int setError = *sceNetErrnoLoc(); + Require(set < 0 && set == sceNetSetsockopt(closed, 0xffff, 0x4, &enabled, sizeof(enabled)) && setError == *sceNetErrnoLoc()); + } + for (const auto& [family, type] : {std::pair{1, 2}, std::pair{2, 9}}) { + const int internalResult = sceNetSocketInternal(nullptr, family, type, 0); + const int internalError = *sceNetErrnoLoc(); + Require(internalResult < 0 && internalResult == sceNetSocket(nullptr, family, type, 0) && internalError == *sceNetErrnoLoc()); + } +} + static void CheckRandom() { Require(Failed(sceNetGetRandom(nullptr), 22)); std::array values{}; @@ -1480,6 +1518,7 @@ int main() { CheckSocketHostErrors(); CheckPoolStats(); CheckIfName(); + CheckExForwards(); CheckRandom(); CheckNetMemory(); CheckNetstat(); diff --git a/docs/dev/debt/libSceNet.md b/docs/dev/debt/libSceNet.md index cb6ceff9ff..03ec7eb395 100644 --- a/docs/dev/debt/libSceNet.md +++ b/docs/dev/debt/libSceNet.md @@ -23,6 +23,7 @@ - [sceNetPppoeStart](../../../core/libs/prx/libSceNet/Export.cpp): seven arguments are confirmed, including the parameter pointer on the stack. The 272-byte parameter contains two DNS addresses, two 128-byte credential arrays, MTU and flags. Callback arguments are user data, event and error; the two unused thread-option meanings remain unresolved. - [sceNetConfigRoutingStart](../../../core/libs/prx/libSceNet/Export.cpp): five pointer/scalar arguments are confirmed. Input address data is eight bytes; the mutable server parameter is 20 bytes and duplicates its first address when its count is one. Optional rules contain a count and pointer to 20-byte records with protocol, two address/port pairs and flags. Layouts are defined; rule flag meanings and protocol-specific port interpretation remain unresolved. - [sceNetConfigRoutingShowRoutingConfig](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - the native routine consumes no arguments; the throwing implementation remains unchanged. +- [sceNetSocketInternal, sceNetGetsockoptEx and sceNetSetsockoptEx](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - forward to sceNetSocket, sceNetGetsockopt and sceNetSetsockopt. Their restored argument lists match the public ones, and nothing in the tree records how the console tells these entry points apart from the public ones (for example by the calling process). ### Functional From 7a481079529f3a59ae3aeab944e450c84631175b Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:37:18 +0000 Subject: [PATCH 002/318] feat(libSceNet): forward sceNetSysctl to libkernel's sysctl sceNetSysctl threw. Its restored argument list is FreeBSD's sysctl(3) (name, name length, old value and size, new value and size), so it now calls libkernel's sysctl. It answers exactly the MIBs libkernel models, and a failure is returned as the library's other calls return one (0x80410100 | errno, with the errno set). Tested: guest_sce_net_tests reads hw.pagesize (6.7) through both sysctl and sceNetSysctl and gets the same value and size, including a size-only query. A one-level name gives EINVAL, a null name EFAULT, a write EPERM and a one-byte buffer ENOMEM, each as 0x804101xx with the errno set. --- core/libs/prx/libSceNet/Export.cpp | 5 +++-- core/libs/tests/GuestSceNet.cpp | 22 ++++++++++++++++++++++ docs/dev/debt/libSceNet.md | 1 + 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index 987eff5732..e2b35f2e63 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -484,6 +484,7 @@ int* errno_slot() { } extern "C" int* APS5_VABI __error_nid_postfix(); +extern "C" int APS5_VABI sysctl_nid_postfix(const int* name, std::uint32_t nameLength, void* oldValue, std::size_t* oldLength, const void* newValue, std::size_t newLength); extern "C" int APS5_VABI __inet_pton_nid_postfix(int family, const char* text, void* destination); extern "C" const char* APS5_VABI __inet_ntop_nid_postfix(int family, const void* source, char* destination, std::uint32_t capacity); extern "C" std::size_t NetCtlHostDnsServers_nid_no_patch(std::uint32_t* servers, std::size_t capacity); @@ -2516,8 +2517,8 @@ int APS5_VABI sceNetSyncWait(NetSyncObject* sync, std::uint32_t* timeout) { } int APS5_VABI sceNetSysctl(const int* name, std::uint32_t nameLength, void* oldValue, std::size_t* oldSize, const void* newValue, std::size_t newSize) { - NotImplemented_nid_no_patch(__func__); - return 0; + if (sysctl_nid_postfix(name, nameLength, oldValue, oldSize, newValue, newSize) == 0) return 0; + return fail(*__error_nid_postfix()); } int APS5_VABI sceNetThreadCreate(Pthread* thread, const NetThreadParameter* parameter, const char* name) { diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 5f2e9c97b7..9a43f0f446 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -140,6 +140,8 @@ int APS5_VABI sceNetGetRandom(std::uint32_t*); int APS5_VABI sceNetSocketInternal(const char*, int, int, int); int APS5_VABI sceNetGetsockoptEx(int, int, int, void*, std::uint32_t*); int APS5_VABI sceNetSetsockoptEx(int, int, int, const void*, std::uint32_t); +int APS5_VABI sceNetSysctl(const int*, std::uint32_t, void*, std::size_t*, const void*, std::size_t); +int APS5_VABI sysctl_nid_postfix(const int*, std::uint32_t, void*, std::size_t*, const void*, std::size_t); void* APS5_VABI sceNetMemoryAllocate(std::int32_t, int); void APS5_VABI sceNetMemoryFree(void*); } @@ -340,6 +342,25 @@ static void CheckExForwards() { } } +static void CheckSysctl() { + const std::array pageSize{6, 7}; + int kernelValue = 0; + std::size_t kernelSize = sizeof(kernelValue); + Require(sysctl_nid_postfix(pageSize.data(), 2, &kernelValue, &kernelSize, nullptr, 0) == 0); + int netValue = 0; + std::size_t netSize = sizeof(netValue); + Require(sceNetSysctl(pageSize.data(), 2, &netValue, &netSize, nullptr, 0) == 0); + Require(netValue == kernelValue && netSize == kernelSize && netValue > 0); + std::size_t probe = 0; + Require(sceNetSysctl(pageSize.data(), 2, nullptr, &probe, nullptr, 0) == 0 && probe == sizeof(int)); + Require(Failed(sceNetSysctl(pageSize.data(), 1, &netValue, &netSize, nullptr, 0), 22)); + Require(Failed(sceNetSysctl(nullptr, 2, &netValue, &netSize, nullptr, 0), 14)); + Require(Failed(sceNetSysctl(pageSize.data(), 2, &netValue, &netSize, &netValue, sizeof(netValue)), 1)); + std::uint8_t truncated = 0; + std::size_t truncatedSize = sizeof(truncated); + Require(Failed(sceNetSysctl(pageSize.data(), 2, &truncated, &truncatedSize, nullptr, 0), 12) && truncatedSize == sizeof(truncated)); +} + static void CheckRandom() { Require(Failed(sceNetGetRandom(nullptr), 22)); std::array values{}; @@ -1519,6 +1540,7 @@ int main() { CheckPoolStats(); CheckIfName(); CheckExForwards(); + CheckSysctl(); CheckRandom(); CheckNetMemory(); CheckNetstat(); diff --git a/docs/dev/debt/libSceNet.md b/docs/dev/debt/libSceNet.md index 03ec7eb395..d73599791c 100644 --- a/docs/dev/debt/libSceNet.md +++ b/docs/dev/debt/libSceNet.md @@ -24,6 +24,7 @@ - [sceNetConfigRoutingStart](../../../core/libs/prx/libSceNet/Export.cpp): five pointer/scalar arguments are confirmed. Input address data is eight bytes; the mutable server parameter is 20 bytes and duplicates its first address when its count is one. Optional rules contain a count and pointer to 20-byte records with protocol, two address/port pairs and flags. Layouts are defined; rule flag meanings and protocol-specific port interpretation remain unresolved. - [sceNetConfigRoutingShowRoutingConfig](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - the native routine consumes no arguments; the throwing implementation remains unchanged. - [sceNetSocketInternal, sceNetGetsockoptEx and sceNetSetsockoptEx](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - forward to sceNetSocket, sceNetGetsockopt and sceNetSetsockopt. Their restored argument lists match the public ones, and nothing in the tree records how the console tells these entry points apart from the public ones (for example by the calling process). +- [sceNetSysctl](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - calls libkernel's sysctl with the FreeBSD sysctl(3) argument list, so it answers exactly the MIBs libkernel models, and reports a failure as the library's other calls do (`0x80410100 | errno`, with the errno set). How the console's routine reports errors is not measured. ### Functional From b44ee56f7d8c63801484c34f5fc1bb3cdd69cabc Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:39:06 +0000 Subject: [PATCH 003/318] feat(libSceNet): convert IPv6 text with a numeric scope ID sceNetInetNtopWithScopeId and sceNetInetPtonWithScopeId threw. They now handle AF_INET6 text with an RFC 4007 numeric zone (fe80::1%2): the address part goes through sceNetInetNtop and sceNetInetPton, so it follows the FreeBSD inet text rules, and the zone is printed and parsed as FreeBSD 11.4's getnameinfo (NI_NUMERICSCOPE) and getaddrinfo do. A zero scope ID prints the address alone and text without a zone parses to scope ID 0. An empty zone, one above 2^32 - 1, or an invalid address part returns 0 and leaves both outputs untouched; a buffer too small for the scoped text gives ENOSPC. A zone given as an interface name, and AF_INET, still throw: the tree has no mapping from interface names to the console's interface indexes and no answer for an IPv4 zone. The debt file says so. Tested: guest_sce_net_tests round-trips fe80::1%2 (same address bytes as fe80::1, scope ID 2) and ff02::1%4294967295, checks the exact buffer size and ENOSPC one byte short, the unscoped forms, four rejected texts with untouched outputs, EINVAL for null pointers, and that an interface-name zone and AF_INET throw. --- core/libs/prx/libSceNet/Export.cpp | 41 +++++++++++++++++++++++++--- core/libs/tests/GuestSceNet.cpp | 44 ++++++++++++++++++++++++++++++ docs/dev/debt/libSceNet.md | 1 + 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index e2b35f2e63..32e768a80e 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" @@ -2570,13 +2571,45 @@ int APS5_VABI sceNetShowNetstatExForBuffer(char* buffer, std::int32_t bufferSize } const char* APS5_VABI sceNetInetNtopWithScopeId(std::int32_t family, const void* address, char* text, std::uint32_t size, std::uint32_t scopeId) { - NotImplemented_nid_no_patch(__func__); - return 0; + if (address == nullptr || text == nullptr) { + set_errno(NET_EINVAL); + return nullptr; + } + if (family != NET_AF_INET6) throw std::runtime_error("sceNetInetNtopWithScopeId: family " + std::to_string(family) + " is not implemented"); + std::array unscoped{}; + if (sceNetInetNtop(family, address, unscoped.data(), static_cast(unscoped.size())) == nullptr) return nullptr; + std::string scoped(unscoped.data()); + if (scopeId != 0) scoped += "%" + std::to_string(scopeId); + if (scoped.size() >= size) { + set_errno(NET_ENOSPC); + return nullptr; + } + std::memcpy(text, scoped.c_str(), scoped.size() + 1); + return text; } int APS5_VABI sceNetInetPtonWithScopeId(std::int32_t family, const char* text, void* address, std::uint32_t* scopeId) { - NotImplemented_nid_no_patch(__func__); - return 0; + if (text == nullptr || address == nullptr || scopeId == nullptr) return fail(NET_EINVAL); + if (family != NET_AF_INET6) throw std::runtime_error("sceNetInetPtonWithScopeId: family " + std::to_string(family) + " is not implemented"); + const char* percent = std::strchr(text, '%'); + std::uint64_t zone = 0; + if (percent != nullptr) { + const std::string_view digits(percent + 1); + if (digits.empty()) return 0; + if (!std::all_of(digits.begin(), digits.end(), [](char digit) { return digit >= '0' && digit <= '9'; })) + throw std::runtime_error("sceNetInetPtonWithScopeId: the interface name zone in " + std::string(text) + " is not implemented"); + for (const char digit : digits) { + zone = zone * 10 + static_cast(digit - '0'); + if (zone > 0xffffffffull) return 0; + } + } + const std::string unscoped = percent != nullptr ? std::string(text, percent) : std::string(text); + std::array parsed{}; + const int converted = sceNetInetPton(family, unscoped.c_str(), parsed.data()); + if (converted != 1) return converted; + std::memcpy(address, parsed.data(), parsed.size()); + *scopeId = static_cast(zone); + return 1; } int APS5_VABI sceNetGetDns6Info(NetDns6Info* info, std::uint32_t flags) { diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 9a43f0f446..8f55de7c12 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -142,6 +142,8 @@ int APS5_VABI sceNetGetsockoptEx(int, int, int, void*, std::uint32_t*); int APS5_VABI sceNetSetsockoptEx(int, int, int, const void*, std::uint32_t); int APS5_VABI sceNetSysctl(const int*, std::uint32_t, void*, std::size_t*, const void*, std::size_t); int APS5_VABI sysctl_nid_postfix(const int*, std::uint32_t, void*, std::size_t*, const void*, std::size_t); +const char* APS5_VABI sceNetInetNtopWithScopeId(std::int32_t, const void*, char*, std::uint32_t, std::uint32_t); +int APS5_VABI sceNetInetPtonWithScopeId(std::int32_t, const char*, void*, std::uint32_t*); void* APS5_VABI sceNetMemoryAllocate(std::int32_t, int); void APS5_VABI sceNetMemoryFree(void*); } @@ -361,6 +363,47 @@ static void CheckSysctl() { Require(Failed(sceNetSysctl(pageSize.data(), 2, &truncated, &truncatedSize, nullptr, 0), 12) && truncatedSize == sizeof(truncated)); } +static void CheckScopeIdText() { + std::array address{}; + std::uint32_t scope = 0xa5a5a5a5u; + Require(sceNetInetPtonWithScopeId(28, "fe80::1%2", address.data(), &scope) == 1 && scope == 2); + std::array plain{}; + Require(sceNetInetPton(28, "fe80::1", plain.data()) == 1 && address == plain); + std::array text{}; + Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), text.size(), scope) == text.data()); + Require(std::strcmp(text.data(), "fe80::1%2") == 0); + text.fill('x'); + *sceNetErrnoLoc() = 123; + Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), 9, 2) == nullptr && *sceNetErrnoLoc() == 28 && text[0] == 'x'); + Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), 10, 2) == text.data() && std::strcmp(text.data(), "fe80::1%2") == 0); + Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), text.size(), 0) == text.data() && std::strcmp(text.data(), "fe80::1") == 0); + scope = 7; + Require(sceNetInetPtonWithScopeId(28, "fe80::1", address.data(), &scope) == 1 && scope == 0 && address == plain); + Require(sceNetInetPtonWithScopeId(28, "ff02::1%4294967295", address.data(), &scope) == 1 && scope == 0xffffffffu); + Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), text.size(), scope) == text.data()); + Require(std::strcmp(text.data(), "ff02::1%4294967295") == 0); + for (const char* rejected : {"fe80::1%", "fe80::1%4294967296", "fe80::g%2", "1.2.3.4%2"}) { + address.fill(0xa5); + scope = 0xa5a5a5a5u; + Require(sceNetInetPtonWithScopeId(28, rejected, address.data(), &scope) == 0); + Require(scope == 0xa5a5a5a5u && address[0] == 0xa5 && address[15] == 0xa5); + } + Require(Failed(sceNetInetPtonWithScopeId(28, "fe80::1%2", address.data(), nullptr), 22)); + *sceNetErrnoLoc() = 123; + Require(sceNetInetNtopWithScopeId(28, nullptr, text.data(), text.size(), 2) == nullptr && *sceNetErrnoLoc() == 22); + const auto throws = [](auto&& call) { + try { + call(); + } catch (const std::runtime_error&) { + return true; + } + return false; + }; + Require(throws([&] { sceNetInetPtonWithScopeId(28, "fe80::1%lo0", address.data(), &scope); })); + Require(throws([&] { sceNetInetPtonWithScopeId(2, "127.0.0.1", address.data(), &scope); })); + Require(throws([&] { sceNetInetNtopWithScopeId(2, address.data(), text.data(), text.size(), 0); })); +} + static void CheckRandom() { Require(Failed(sceNetGetRandom(nullptr), 22)); std::array values{}; @@ -1541,6 +1584,7 @@ int main() { CheckIfName(); CheckExForwards(); CheckSysctl(); + CheckScopeIdText(); CheckRandom(); CheckNetMemory(); CheckNetstat(); diff --git a/docs/dev/debt/libSceNet.md b/docs/dev/debt/libSceNet.md index d73599791c..e9e01a3067 100644 --- a/docs/dev/debt/libSceNet.md +++ b/docs/dev/debt/libSceNet.md @@ -25,6 +25,7 @@ - [sceNetConfigRoutingShowRoutingConfig](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - the native routine consumes no arguments; the throwing implementation remains unchanged. - [sceNetSocketInternal, sceNetGetsockoptEx and sceNetSetsockoptEx](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - forward to sceNetSocket, sceNetGetsockopt and sceNetSetsockopt. Their restored argument lists match the public ones, and nothing in the tree records how the console tells these entry points apart from the public ones (for example by the calling process). - [sceNetSysctl](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - calls libkernel's sysctl with the FreeBSD sysctl(3) argument list, so it answers exactly the MIBs libkernel models, and reports a failure as the library's other calls do (`0x80410100 | errno`, with the errno set). How the console's routine reports errors is not measured. +- [sceNetInetNtopWithScopeId and sceNetInetPtonWithScopeId](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - convert IPv6 text with a numeric zone (`fe80::1%2`, [RFC 4007 section 11](https://www.rfc-editor.org/rfc/rfc4007#section-11)) through sceNetInetNtop and sceNetInetPton, as FreeBSD 11.4's `getnameinfo` with `NI_NUMERICSCOPE` prints it and `getaddrinfo` parses it. A zone given as an interface name, and `AF_INET`, throw: the tree has no mapping from interface names to the console's interface indexes, and no answer for an IPv4 zone. ### Functional From a19d1102223115f2c1b1107cb07b5787dfe7d7a5 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:40:08 +0000 Subject: [PATCH 004/318] feat(libSceNet): report the IPv4 default route in sceNetConfigGetDefaultRoute sceNetConfigGetDefaultRoute threw. It now writes the IPv4 gateway of the host interface that libSceNetCtl's sceNetCtlGetInfo already describes as DEFAULT_ROUTE, in network byte order, the order sceNetInetPton writes an AF_INET address in, and returns 0. libSceNetCtl exports NetCtlHostIpv4Gateway_nid_no_patch for it, next to the DNS helper sceNetGetDnsInfo uses. Without a gateway, or for a null pointer, it still throws: the console's answer for a missing route is not measured. Tested: guest_sce_net_tests, after sceNetCtlInit, prints the gateway with sceNetInetNtop and compares it with sceNetCtlGetInfo's DEFAULT_ROUTE text (192.0.2.1 on this host); a null pointer throws. On a host without a default route the case checks that the call throws and leaves the output untouched. --- core/libs/prx/libSceNet/Export.cpp | 6 +++++- core/libs/prx/libSceNetCtl/Export.cpp | 7 +++++++ core/libs/tests/GuestSceNet.cpp | 27 +++++++++++++++++++++++++++ docs/dev/debt/libSceNet.md | 1 + 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index 32e768a80e..8bfb4d3fce 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -489,6 +489,7 @@ extern "C" int APS5_VABI sysctl_nid_postfix(const int* name, std::uint32_t nameL extern "C" int APS5_VABI __inet_pton_nid_postfix(int family, const char* text, void* destination); extern "C" const char* APS5_VABI __inet_ntop_nid_postfix(int family, const void* source, char* destination, std::uint32_t capacity); extern "C" std::size_t NetCtlHostDnsServers_nid_no_patch(std::uint32_t* servers, std::size_t capacity); +extern "C" bool NetCtlHostIpv4Gateway_nid_no_patch(std::uint32_t* gateway); void set_errno(int err) { *errno_slot() = err; @@ -2768,7 +2769,10 @@ int APS5_VABI sceNetConfigDelRoute(const std::uint32_t* destination, const std:: } int APS5_VABI sceNetConfigGetDefaultRoute(std::uint32_t* gateway) { - NotImplemented_nid_no_patch(__func__); + if (!gateway) throw std::runtime_error("sceNetConfigGetDefaultRoute: a null gateway is not supported"); + std::uint32_t found = 0; + if (!NetCtlHostIpv4Gateway_nid_no_patch(&found)) throw std::runtime_error("sceNetConfigGetDefaultRoute: the answer without an IPv4 default route is not implemented"); + *gateway = found; return 0; } diff --git a/core/libs/prx/libSceNetCtl/Export.cpp b/core/libs/prx/libSceNetCtl/Export.cpp index 30be5f6e85..623c98945d 100644 --- a/core/libs/prx/libSceNetCtl/Export.cpp +++ b/core/libs/prx/libSceNetCtl/Export.cpp @@ -378,6 +378,13 @@ extern "C" std::size_t NetCtlHostDnsServers_nid_no_patch(std::uint32_t* servers, return host.dns.size(); } +extern "C" bool NetCtlHostIpv4Gateway_nid_no_patch(std::uint32_t* gateway) { + HostIpv4Interface host; + if (!FindHostIpv4Interface(host) || host.gateway.s_addr == 0) return false; + *gateway = host.gateway.s_addr; + return true; +} + static void WriteInfoWord(NetCtlInfo* info, std::uint32_t value) { std::memcpy(info->opaque, &value, sizeof(value)); } diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 8f55de7c12..ed121fb4e9 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -144,6 +144,7 @@ int APS5_VABI sceNetSysctl(const int*, std::uint32_t, void*, std::size_t*, const int APS5_VABI sysctl_nid_postfix(const int*, std::uint32_t, void*, std::size_t*, const void*, std::size_t); const char* APS5_VABI sceNetInetNtopWithScopeId(std::int32_t, const void*, char*, std::uint32_t, std::uint32_t); int APS5_VABI sceNetInetPtonWithScopeId(std::int32_t, const char*, void*, std::uint32_t*); +int APS5_VABI sceNetConfigGetDefaultRoute(std::uint32_t*); void* APS5_VABI sceNetMemoryAllocate(std::int32_t, int); void APS5_VABI sceNetMemoryFree(void*); } @@ -404,6 +405,31 @@ static void CheckScopeIdText() { Require(throws([&] { sceNetInetNtopWithScopeId(2, address.data(), text.data(), text.size(), 0); })); } +static void CheckDefaultRoute() { + NetCtlInfo info{}; + const int result = sceNetCtlGetInfo(16, &info); + const char* route = reinterpret_cast(info.opaque); + std::uint32_t gateway = 0xa5a5a5a5u; + const auto throws = [](auto&& call) { + try { + call(); + } catch (const std::runtime_error&) { + return true; + } + return false; + }; + Require(throws([] { sceNetConfigGetDefaultRoute(nullptr); })); + if (result != 0 || std::strcmp(route, "0.0.0.0") == 0) { + Require(result == 0 || result == static_cast(0x80412108)); + Require(throws([&] { sceNetConfigGetDefaultRoute(&gateway); }) && gateway == 0xa5a5a5a5u); + return; + } + Require(sceNetConfigGetDefaultRoute(&gateway) == 0); + std::array text{}; + Require(sceNetInetNtop(2, &gateway, text.data(), text.size()) == text.data()); + Require(std::strcmp(text.data(), route) == 0); +} + static void CheckRandom() { Require(Failed(sceNetGetRandom(nullptr), 22)); std::array values{}; @@ -2010,6 +2036,7 @@ int main() { Require(sceNetCtlGetState(&state) == 0); Require(state == 0 || state == 3); Require(sceNetCtlGetIfStat(nullptr) == static_cast(0x80412107)); + CheckDefaultRoute(); { const char* counterPath = "guest_sce_net_byte_counter.txt"; std::FILE* counter = std::fopen(counterPath, "w"); diff --git a/docs/dev/debt/libSceNet.md b/docs/dev/debt/libSceNet.md index e9e01a3067..88d142cd5a 100644 --- a/docs/dev/debt/libSceNet.md +++ b/docs/dev/debt/libSceNet.md @@ -26,6 +26,7 @@ - [sceNetSocketInternal, sceNetGetsockoptEx and sceNetSetsockoptEx](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - forward to sceNetSocket, sceNetGetsockopt and sceNetSetsockopt. Their restored argument lists match the public ones, and nothing in the tree records how the console tells these entry points apart from the public ones (for example by the calling process). - [sceNetSysctl](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - calls libkernel's sysctl with the FreeBSD sysctl(3) argument list, so it answers exactly the MIBs libkernel models, and reports a failure as the library's other calls do (`0x80410100 | errno`, with the errno set). How the console's routine reports errors is not measured. - [sceNetInetNtopWithScopeId and sceNetInetPtonWithScopeId](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - convert IPv6 text with a numeric zone (`fe80::1%2`, [RFC 4007 section 11](https://www.rfc-editor.org/rfc/rfc4007#section-11)) through sceNetInetNtop and sceNetInetPton, as FreeBSD 11.4's `getnameinfo` with `NI_NUMERICSCOPE` prints it and `getaddrinfo` parses it. A zone given as an interface name, and `AF_INET`, throw: the tree has no mapping from interface names to the console's interface indexes, and no answer for an IPv4 zone. +- [sceNetConfigGetDefaultRoute](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - reports the IPv4 gateway of the host interface that sceNetCtlGetInfo describes (`DEFAULT_ROUTE`), in network byte order, the order sceNetInetPton writes an `AF_INET` address in. With no gateway, or a null pointer, it throws: the console's answer for a missing route is not measured. ### Functional From ea2681ca0e0eb602eb6aa4f02cffed8d4057dcfd Mon Sep 17 00:00:00 2001 From: ThatsLeo Date: Mon, 19 Oct 2026 16:59:58 +0200 Subject: [PATCH 005/318] perf(shader): span single-wave wave64 compute by default after a uniformity check A spanned module is checked for barriers inside control flow that a per-invocation value decides, and falls back to two lanes per invocation when it has one, so the spanned layout no longer needs APS5_SPANNED_WAVE; APS5_NO_SPANNED_WAVE turns it off. --- .../Execution/src/VulkanDevice.cpp | 6 +- core/shader/recompiler/CMakeLists.txt | 11 + .../SpirvBackend/SpirvSpannedUniformity.hpp | 15 + .../SpirvBackend/src/SpirvEmitter.cpp | 7 +- .../src/SpirvInstructionOperands.hpp | 231 +++++++++++ .../src/SpirvSpannedUniformity.cpp | 367 ++++++++++++++++++ .../SpirvBackend/src/SpirvSpecialization.cpp | 218 +---------- .../tests/SpannedUniformityTests.cpp | 220 +++++++++++ docs/dev/ENVIRONMENT.md | 2 +- docs/dev/debt/shader.md | 2 +- 10 files changed, 858 insertions(+), 221 deletions(-) create mode 100644 core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvSpannedUniformity.hpp create mode 100644 core/shader/recompiler/SpirvBackend/src/SpirvInstructionOperands.hpp create mode 100644 core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp create mode 100644 core/shader/recompiler/tests/SpannedUniformityTests.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 89ad6e3157..134c4433f7 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1891,10 +1891,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : byteFeatures.pNext = &subgroupSizeFeatures; APS5_LOG_OUT("Wave32 programs run on subgroups of %u: compute %d, mesh %d", 32u, state->computeWave32, state->meshWave32); } - if (std::getenv("APS5_SPANNED_WAVE") != nullptr) { - state->spannedWaves = state->computeFullSubgroups; - APS5_LOG_OUT("APS5_SPANNED_WAVE: single-wave wave64 compute spans two full 32-lane subgroups: %d", state->spannedWaves); - } + state->spannedWaves = state->computeFullSubgroups && std::getenv("APS5_NO_SPANNED_WAVE") == nullptr; + APS5_LOG_OUT("single-wave wave64 compute spans two full 32-lane subgroups: %d", state->spannedWaves); timelineFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR}; timelineFeatures.timelineSemaphore = VK_TRUE; if (state->timelineSemaphores) { diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index ad13332d35..52a2ae6181 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -140,6 +140,7 @@ function(add_shader_recompiler target runtimeTarget) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvBufferFormat.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvRuntimeBuffer.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvSpecialization.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvSpannedUniformity.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvMemory/SpirvConstants.cpp @@ -349,6 +350,16 @@ function(add_shader_recompiler target runtimeTarget) target_link_libraries(${target} PRIVATE SPIRV-Tools-opt SPIRV-Tools-static) endif() target_compile_definitions(${target} PUBLIC ANYPS5_ENABLE_SPIRV_TOOLS=$) + if(BUILD_TESTING AND NOT TARGET spanned_uniformity_tests) + add_test_executable(spanned_uniformity_tests + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/tests/SpannedUniformityTests.cpp" + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvSpannedUniformity.cpp") + target_include_directories(spanned_uniformity_tests PRIVATE + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/include" + ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}) + add_test(NAME spanned_uniformity COMMAND spanned_uniformity_tests) + set_tests_properties(spanned_uniformity PROPERTIES TIMEOUT 20) + endif() if(BUILD_TESTING AND ANYPS5_ENABLE_SPIRV_TOOLS AND NOT ANYPS5_USE_SYSTEM_DEPS AND NOT TARGET spirv_validator_exit_tests) add_test_executable(spirv_validator_exit_tests "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/tests/SpirvValidatorExit.cpp") target_link_libraries(spirv_validator_exit_tests PRIVATE SPIRV-Tools-static) diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvSpannedUniformity.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvSpannedUniformity.hpp new file mode 100644 index 0000000000..23d3be78d7 --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvSpannedUniformity.hpp @@ -0,0 +1,15 @@ +#ifndef CORE_SHADER_RECOMPILER_SPIRVSPANNEDUNIFORMITY_HPP +#define CORE_SHADER_RECOMPILER_SPIRVSPANNEDUNIFORMITY_HPP + +#include +#include +#include +#include + +namespace ShaderRecompiler { + +[[nodiscard]] std::optional SpannedBarrierInDivergentFlow(std::span words); + +} + +#endif diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 17d2db36bf..af989da56c 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -1,5 +1,6 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvEmitter.hpp" +#include "SpirvBackend/SpirvSpannedUniformity.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvEmitterState.hpp" #include "SpirvBackend/SpirvFlowEmitter.hpp" @@ -318,8 +319,12 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh EmitModuleHeader(state, bindings); EmitProgram(state); state.module.EmitEntryPoint(ExecutionModelForStage(state.program.Resources().stage), state.mainFunc, "main", state.interfaceVariables); + auto words = state.module.Finalize(); + if (state.spannedWave) { + if (auto reason = SpannedBarrierInDivergentFlow(words)) throw SpannedWaveUnsupported(*reason); + } if (info != nullptr) info->fullSubgroups = state.spannedWave; - return state.module.Finalize(); + return words; } } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvInstructionOperands.hpp b/core/shader/recompiler/SpirvBackend/src/SpirvInstructionOperands.hpp new file mode 100644 index 0000000000..0109edd90e --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/src/SpirvInstructionOperands.hpp @@ -0,0 +1,231 @@ +#ifndef CORE_SHADER_RECOMPILER_SPIRVINSTRUCTIONOPERANDS_HPP +#define CORE_SHADER_RECOMPILER_SPIRVINSTRUCTIONOPERANDS_HPP + +#define SPV_ENABLE_UTILITY_CODE +#include +#undef SPV_ENABLE_UTILITY_CODE +#include +#include + +namespace ShaderRecompiler::SpirvInstructions { + +template +spv::Op Opcode(const TInstruction& instruction) { + return instruction.empty() ? spv::OpNop : static_cast(instruction[0] & 0xffffu); +} + +template +std::uint32_t Result(const TInstruction& instruction) { + if (instruction.empty()) return 0u; + bool result = false; + bool type = false; + spv::HasResultAndType(Opcode(instruction), &result, &type); + return result ? instruction.at(type ? 2u : 1u) : 0u; +} + +template +void VisitIdOperands(const TInstruction& instruction, std::uint32_t switchLiteralWords, const TVisitor& visit) { + const auto size = instruction.size(); + const auto from = [&](std::size_t first) { + for (auto index = first; index < size; ++index) visit(index); + }; + const auto only = [&](std::initializer_list indices) { + for (const auto index : indices) { + if (index < size) visit(index); + } + }; + constexpr std::uint32_t knownMemoryAccess = static_cast(spv::MemoryAccessVolatileMask) | static_cast(spv::MemoryAccessAlignedMask) | + static_cast(spv::MemoryAccessNontemporalMask) | static_cast(spv::MemoryAccessMakePointerAvailableMask) | + static_cast(spv::MemoryAccessMakePointerVisibleMask) | static_cast(spv::MemoryAccessNonPrivatePointerMask); + const auto memoryOperands = [&](std::size_t index) { + while (index < size) { + const auto mask = instruction[index++]; + if ((mask & ~knownMemoryAccess) != 0u) { + from(index); + return; + } + if ((mask & static_cast(spv::MemoryAccessAlignedMask)) != 0u) ++index; + if ((mask & static_cast(spv::MemoryAccessMakePointerAvailableMask)) != 0u && index < size) visit(index++); + if ((mask & static_cast(spv::MemoryAccessMakePointerVisibleMask)) != 0u && index < size) visit(index++); + } + }; + switch (Opcode(instruction)) { + case spv::OpNop: + case spv::OpCapability: + case spv::OpExtension: + case spv::OpExtInstImport: + case spv::OpMemoryModel: + case spv::OpSource: + case spv::OpSourceContinued: + case spv::OpSourceExtension: + case spv::OpString: + case spv::OpName: + case spv::OpMemberName: + case spv::OpModuleProcessed: + case spv::OpDecorate: + case spv::OpMemberDecorate: + case spv::OpDecorateString: + case spv::OpMemberDecorateString: + case spv::OpTypeVoid: + case spv::OpTypeBool: + case spv::OpTypeInt: + case spv::OpTypeFloat: + case spv::OpTypeSampler: + case spv::OpConstantTrue: + case spv::OpConstantFalse: + case spv::OpConstant: + case spv::OpConstantSampler: + case spv::OpConstantNull: + case spv::OpSpecConstantTrue: + case spv::OpSpecConstantFalse: + case spv::OpSpecConstant: + case spv::OpLabel: + case spv::OpNoLine: + case spv::OpReturn: + case spv::OpUnreachable: + case spv::OpKill: + case spv::OpFunctionEnd: + return; + case spv::OpTypeVector: + case spv::OpTypeMatrix: + case spv::OpTypeImage: + only({2}); + return; + case spv::OpTypePointer: + only({3}); + return; + case spv::OpTypeForwardPointer: + case spv::OpExecutionMode: + case spv::OpLine: + case spv::OpSelectionMerge: + only({1}); + return; + case spv::OpSpecConstantOp: + only({1}); + from(4); + return; + case spv::OpVariable: + case spv::OpFunction: + only({1, 4}); + return; + case spv::OpLoad: + only({1, 3}); + memoryOperands(4); + return; + case spv::OpStore: + case spv::OpCopyMemory: + only({1, 2}); + memoryOperands(3); + return; + case spv::OpCopyMemorySized: + only({1, 2, 3}); + memoryOperands(4); + return; + case spv::OpArrayLength: + case spv::OpCompositeExtract: + only({1, 3}); + return; + case spv::OpCompositeInsert: + case spv::OpVectorShuffle: + only({1, 3, 4}); + return; + case spv::OpExtInst: + only({1, 3}); + from(5); + return; + case spv::OpEntryPoint: { + only({2}); + std::size_t index = 3; + while (index < size) { + const auto word = instruction[index++]; + if ((word & 0xffu) == 0u || (word & 0xff00u) == 0u || (word & 0xff0000u) == 0u || (word & 0xff000000u) == 0u) break; + } + from(index); + return; + } + case spv::OpExecutionModeId: + case spv::OpDecorateId: + only({1}); + from(3); + return; + case spv::OpLoopMerge: + only({1, 2}); + return; + case spv::OpBranchConditional: + only({1, 2, 3}); + return; + case spv::OpSwitch: + only({1, 2}); + for (std::size_t index = 3; index + switchLiteralWords < size; index += switchLiteralWords + 1u) visit(index + switchLiteralWords); + return; + case spv::OpImageSampleImplicitLod: + case spv::OpImageSampleExplicitLod: + case spv::OpImageSampleProjImplicitLod: + case spv::OpImageSampleProjExplicitLod: + case spv::OpImageFetch: + case spv::OpImageRead: + case spv::OpImageSparseSampleImplicitLod: + case spv::OpImageSparseSampleExplicitLod: + case spv::OpImageSparseSampleProjImplicitLod: + case spv::OpImageSparseSampleProjExplicitLod: + case spv::OpImageSparseFetch: + case spv::OpImageSparseRead: + only({1, 3, 4}); + from(6); + return; + case spv::OpImageSampleDrefImplicitLod: + case spv::OpImageSampleDrefExplicitLod: + case spv::OpImageSampleProjDrefImplicitLod: + case spv::OpImageSampleProjDrefExplicitLod: + case spv::OpImageGather: + case spv::OpImageDrefGather: + case spv::OpImageSparseSampleDrefImplicitLod: + case spv::OpImageSparseSampleDrefExplicitLod: + case spv::OpImageSparseSampleProjDrefImplicitLod: + case spv::OpImageSparseSampleProjDrefExplicitLod: + case spv::OpImageSparseGather: + case spv::OpImageSparseDrefGather: + only({1, 3, 4, 5}); + from(7); + return; + case spv::OpImageWrite: + only({1, 2, 3}); + from(5); + return; + case spv::OpGroupIAdd: + case spv::OpGroupFAdd: + case spv::OpGroupFMin: + case spv::OpGroupUMin: + case spv::OpGroupSMin: + case spv::OpGroupFMax: + case spv::OpGroupUMax: + case spv::OpGroupSMax: + case spv::OpGroupNonUniformBallotBitCount: + case spv::OpGroupNonUniformIAdd: + case spv::OpGroupNonUniformFAdd: + case spv::OpGroupNonUniformIMul: + case spv::OpGroupNonUniformFMul: + case spv::OpGroupNonUniformSMin: + case spv::OpGroupNonUniformUMin: + case spv::OpGroupNonUniformFMin: + case spv::OpGroupNonUniformSMax: + case spv::OpGroupNonUniformUMax: + case spv::OpGroupNonUniformFMax: + case spv::OpGroupNonUniformBitwiseAnd: + case spv::OpGroupNonUniformBitwiseOr: + case spv::OpGroupNonUniformBitwiseXor: + case spv::OpGroupNonUniformLogicalAnd: + case spv::OpGroupNonUniformLogicalOr: + case spv::OpGroupNonUniformLogicalXor: + only({1, 3}); + from(5); + return; + default: + from(1); + return; + } +} + +} + +#endif diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp new file mode 100644 index 0000000000..6e495f5ced --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp @@ -0,0 +1,367 @@ +#include "SpirvInstructionOperands.hpp" +#include "SpirvBackend/SpirvSpannedUniformity.hpp" +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { +namespace { + +using Instruction = std::vector; +using SpirvInstructions::Opcode; +using SpirvInstructions::Result; +using SpirvInstructions::VisitIdOperands; + +struct Block { + std::uint32_t label = 0u; + std::vector body; +}; + +struct Function { + std::uint32_t id = 0u; + std::vector parameters; + std::vector blocks; +}; + +struct Module { + std::unordered_map builtins; + std::unordered_map storageClasses; + std::vector functions; +}; + +Module Parse(std::span words) { + if (words.size() < 5u) throw std::runtime_error("SPIR-V module has no header"); + Module module; + Function* function = nullptr; + Block* block = nullptr; + for (std::size_t cursor = 5u; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + if (count == 0u || cursor + count > words.size()) throw std::runtime_error("SPIR-V instruction runs past the module"); + Instruction instruction(words.begin() + static_cast(cursor), words.begin() + static_cast(cursor + count)); + cursor += count; + switch (Opcode(instruction)) { + case spv::OpDecorate: + if (instruction.size() >= 4u && instruction[2] == spv::DecorationBuiltIn) module.builtins[instruction[1]] = instruction[3]; + break; + case spv::OpVariable: + if (instruction.size() >= 4u) module.storageClasses[instruction[2]] = instruction[3]; + if (block != nullptr) block->body.push_back(std::move(instruction)); + break; + case spv::OpFunction: + function = &module.functions.emplace_back(); + function->id = instruction.at(2); + break; + case spv::OpFunctionParameter: + if (function != nullptr) function->parameters.push_back(instruction.at(2)); + break; + case spv::OpLabel: + if (function == nullptr) throw std::runtime_error("SPIR-V label outside a function"); + block = &function->blocks.emplace_back(); + block->label = instruction.at(1); + break; + case spv::OpFunctionEnd: + function = nullptr; + block = nullptr; + break; + default: + if (block != nullptr) block->body.push_back(std::move(instruction)); + break; + } + } + return module; +} + +bool VaryingBuiltin(std::uint32_t builtin) { + switch (builtin) { + case spv::BuiltInLocalInvocationId: + case spv::BuiltInGlobalInvocationId: + case spv::BuiltInLocalInvocationIndex: + case spv::BuiltInSubgroupId: + case spv::BuiltInSubgroupLocalInvocationId: + case spv::BuiltInSubgroupEqMask: + case spv::BuiltInSubgroupGeMask: + case spv::BuiltInSubgroupGtMask: + case spv::BuiltInSubgroupLeMask: + case spv::BuiltInSubgroupLtMask: + return true; + default: + return false; + } +} + +bool PerInvocationResult(spv::Op op) { + const auto raw = static_cast(op); + if (raw >= spv::OpAtomicExchange && raw <= spv::OpAtomicXor) return true; + if (raw >= spv::OpGroupAll && raw <= spv::OpGroupSMax) return true; + if (raw >= spv::OpGroupNonUniformElect && raw <= spv::OpGroupNonUniformQuadSwap) return true; + switch (op) { + case spv::OpAtomicFlagTestAndSet: + case spv::OpAtomicFMinEXT: + case spv::OpAtomicFMaxEXT: + case spv::OpAtomicFAddEXT: + case spv::OpSubgroupBallotKHR: + case spv::OpSubgroupFirstInvocationKHR: + case spv::OpSubgroupAllKHR: + case spv::OpSubgroupAnyKHR: + case spv::OpSubgroupAllEqualKHR: + case spv::OpSubgroupReadInvocationKHR: + case spv::OpGroupNonUniformRotateKHR: + return true; + default: + return false; + } +} + +bool PrivateStorage(std::uint32_t storageClass) { + return storageClass == spv::StorageClassFunction || storageClass == spv::StorageClassPrivate; +} + +class Analysis { +public: + explicit Analysis(const Module& module) : module(module) { + for (const auto& function : module.functions) { + for (const auto& block : function.blocks) { + for (const auto& instruction : block.body) { + if (Opcode(instruction) == spv::OpControlBarrier) barriers.insert(function.id); + } + } + } + for (bool changed = true; changed;) { + changed = false; + for (const auto& function : module.functions) { + if (barriers.contains(function.id)) continue; + for (const auto& block : function.blocks) { + for (const auto& instruction : block.body) { + if (Opcode(instruction) == spv::OpFunctionCall && instruction.size() > 3u && barriers.contains(instruction[3]) && barriers.insert(function.id).second) changed = true; + } + } + } + } + } + + std::optional Run() { + for (std::size_t round = 0u; round < 64u; ++round) { + const auto before = Size(); + report.reset(); + for (const auto& function : module.functions) Visit(function); + if (Size() == before) return report; + } + throw std::runtime_error("spanned wave uniformity did not settle"); + } + +private: + struct Flow { + std::vector order; + std::unordered_map index; + std::unordered_map> successors; + std::unordered_map idom; + std::unordered_map> children; + }; + + std::size_t Size() const { + return nonUniform.size() + storedNonUniform.size() + varyingFunctions.size(); + } + + bool NonUniform(std::uint32_t id) const { + return nonUniform.contains(id); + } + + static std::vector Successors(const Block& block) { + if (block.body.empty()) return {}; + const auto& terminator = block.body.back(); + switch (Opcode(terminator)) { + case spv::OpBranch: return {terminator.at(1)}; + case spv::OpBranchConditional: return {terminator.at(2), terminator.at(3)}; + case spv::OpSwitch: { + std::vector targets{terminator.at(2)}; + for (std::size_t operand = 4u; operand < terminator.size(); operand += 2u) targets.push_back(terminator[operand]); + return targets; + } + default: return {}; + } + } + + static Flow BuildFlow(const Function& function) { + Flow flow; + for (const auto& block : function.blocks) flow.successors[block.label] = Successors(block); + if (function.blocks.empty()) return flow; + std::vector postorder; + std::unordered_set seen{function.blocks.front().label}; + std::vector> stack{{function.blocks.front().label, 0u}}; + while (!stack.empty()) { + auto& [node, next] = stack.back(); + const auto& successors = flow.successors[node]; + if (next == successors.size()) { + postorder.push_back(node); + stack.pop_back(); + continue; + } + const auto successor = successors[next++]; + if (seen.insert(successor).second) stack.emplace_back(successor, 0u); + } + flow.order.assign(postorder.rbegin(), postorder.rend()); + for (std::size_t position = 0u; position < flow.order.size(); ++position) flow.index[flow.order[position]] = position; + std::unordered_map> predecessors; + for (const auto node : flow.order) { + for (const auto successor : flow.successors[node]) predecessors[successor].push_back(node); + } + const auto root = flow.order.front(); + flow.idom[root] = root; + const auto intersect = [&](std::uint32_t left, std::uint32_t right) { + while (left != right) { + while (flow.index[left] > flow.index[right]) left = flow.idom[left]; + while (flow.index[right] > flow.index[left]) right = flow.idom[right]; + } + return left; + }; + for (bool changed = true; changed;) { + changed = false; + for (std::size_t position = 1u; position < flow.order.size(); ++position) { + const auto node = flow.order[position]; + std::uint32_t dominator = 0u; + for (const auto predecessor : predecessors[node]) { + if (!flow.idom.contains(predecessor)) continue; + dominator = dominator == 0u ? predecessor : intersect(predecessor, dominator); + } + if (dominator != 0u && flow.idom[node] != dominator) { + flow.idom[node] = dominator; + changed = true; + } + } + } + for (const auto node : flow.order) { + if (node != root) flow.children[flow.idom[node]].push_back(node); + } + return flow; + } + + static std::vector Construct(const Flow& flow, std::uint32_t header, std::uint32_t merge) { + std::vector blocks; + std::vector pending; + if (const auto found = flow.children.find(header); found != flow.children.end()) pending = found->second; + while (!pending.empty()) { + const auto node = pending.back(); + pending.pop_back(); + if (node == merge) continue; + blocks.push_back(node); + if (const auto found = flow.children.find(node); found != flow.children.end()) pending.insert(pending.end(), found->second.begin(), found->second.end()); + } + return blocks; + } + + std::uint32_t BaseVariable(std::uint32_t pointer, const std::unordered_map& chains) const { + for (std::size_t depth = 0u; depth < 64u; ++depth) { + const auto found = chains.find(pointer); + if (found == chains.end()) break; + pointer = found->second; + } + return pointer; + } + + void Visit(const Function& function) { + if (function.blocks.empty()) return; + const auto flow = BuildFlow(function); + std::unordered_map blocks; + std::unordered_map chains; + for (const auto& block : function.blocks) { + blocks[block.label] = █ + for (const auto& instruction : block.body) { + const auto op = Opcode(instruction); + if ((op == spv::OpAccessChain || op == spv::OpInBoundsAccessChain || op == spv::OpPtrAccessChain) && instruction.size() > 3u) chains[instruction[2]] = instruction[3]; + } + } + for (const auto parameter : function.parameters) nonUniform.insert(parameter); + std::unordered_set divergent; + std::unordered_set divergentMerges; + for (auto position = flow.order.size(); position-- > 0u;) { + const auto header = flow.order[position]; + const auto& body = blocks.at(header)->body; + if (body.size() < 2u) continue; + const auto& mergeInstruction = body[body.size() - 2u]; + const auto mergeOp = Opcode(mergeInstruction); + if (mergeOp != spv::OpSelectionMerge && mergeOp != spv::OpLoopMerge) continue; + const auto merge = mergeInstruction.at(1); + const auto& terminator = body.back(); + const auto terminatorOp = Opcode(terminator); + bool split = (terminatorOp == spv::OpBranchConditional || terminatorOp == spv::OpSwitch) && NonUniform(terminator.at(1)); + const auto construct = Construct(flow, header, merge); + if (mergeOp == spv::OpLoopMerge && !split) { + for (const auto node : construct) { + const auto& successors = flow.successors.at(node); + if (std::find(successors.begin(), successors.end(), merge) == successors.end()) continue; + const auto& exit = blocks.at(node)->body.back(); + const auto exitOp = Opcode(exit); + if (divergent.contains(node) || ((exitOp == spv::OpBranchConditional || exitOp == spv::OpSwitch) && NonUniform(exit.at(1)))) { + split = true; + break; + } + } + } + if (!split) continue; + divergent.insert(construct.begin(), construct.end()); + if (mergeOp == spv::OpLoopMerge) divergent.insert(header); + divergentMerges.insert(merge); + } + for (const auto label : flow.order) { + const bool inDivergentFlow = divergent.contains(label); + for (const auto& instruction : blocks.at(label)->body) { + const auto op = Opcode(instruction); + if (op == spv::OpStore && instruction.size() >= 3u) { + const auto variable = BaseVariable(instruction[1], chains); + const auto storage = module.storageClasses.find(variable); + if (storage != module.storageClasses.end() && PrivateStorage(storage->second) && (inDivergentFlow || NonUniform(instruction[1]) || NonUniform(instruction[2]))) storedNonUniform.insert(variable); + continue; + } + if (inDivergentFlow && !report && (op == spv::OpControlBarrier || (op == spv::OpFunctionCall && instruction.size() > 3u && barriers.contains(instruction[3])))) { + report = std::string(op == spv::OpControlBarrier ? "an OpControlBarrier" : "a call reaching an OpControlBarrier") + " in block %" + std::to_string(label) + " of function %" + std::to_string(function.id) + " sits in control flow a per-invocation value decides"; + } + const auto result = Result(instruction); + if (result == 0u || NonUniform(result)) continue; + bool varying = false; + switch (op) { + case spv::OpLoad: { + const auto variable = BaseVariable(instruction.at(3), chains); + const auto builtin = module.builtins.find(variable); + const auto storage = module.storageClasses.find(variable); + varying = NonUniform(instruction[3]) || (builtin != module.builtins.end() && VaryingBuiltin(builtin->second)) || storedNonUniform.contains(variable) || + (storage != module.storageClasses.end() && storage->second == spv::StorageClassInput && builtin == module.builtins.end()); + break; + } + case spv::OpPhi: + varying = inDivergentFlow || divergentMerges.contains(label); + for (std::size_t operand = 3u; !varying && operand < instruction.size(); operand += 2u) varying = NonUniform(instruction[operand]); + break; + case spv::OpFunctionCall: + varying = instruction.size() > 3u && varyingFunctions.contains(instruction[3]); + for (std::size_t operand = 4u; !varying && operand < instruction.size(); ++operand) varying = NonUniform(instruction[operand]); + break; + default: + varying = PerInvocationResult(op); + if (!varying) VisitIdOperands(instruction, 1u, [&](std::size_t operand) { varying = varying || NonUniform(instruction[operand]); }); + break; + } + if (!varying) continue; + nonUniform.insert(result); + if (op != spv::OpFunctionCall) varyingFunctions.insert(function.id); + } + } + } + + const Module& module; + std::unordered_set barriers; + std::unordered_set nonUniform; + std::unordered_set storedNonUniform; + std::unordered_set varyingFunctions; + std::optional report; +}; + +} + +std::optional SpannedBarrierInDivergentFlow(std::span words) { + const auto module = Parse(words); + return Analysis(module).Run(); +} + +} diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvSpecialization.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvSpecialization.cpp index d77a063d72..53c48e5e87 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvSpecialization.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvSpecialization.cpp @@ -2,6 +2,7 @@ #include #undef SPV_ENABLE_UTILITY_CODE #include "SpirvBackend/SpirvSpecialization.hpp" +#include "SpirvInstructionOperands.hpp" #include #include #include @@ -67,17 +68,9 @@ struct ArenaAllocator { using Instruction = std::vector>; -spv::Op Opcode(const Instruction& instruction) { - return instruction.empty() ? spv::OpNop : static_cast(instruction[0] & 0xffffu); -} - -std::uint32_t Result(const Instruction& instruction) { - if (instruction.empty()) return 0u; - bool result = false; - bool type = false; - spv::HasResultAndType(Opcode(instruction), &result, &type); - return result ? instruction.at(type ? 2u : 1u) : 0u; -} +using SpirvInstructions::Opcode; +using SpirvInstructions::Result; +using SpirvInstructions::VisitIdOperands; Instruction Make(spv::Op op, std::initializer_list operands) { Instruction result{(static_cast(operands.size() + 1u) << 16u) | op}; @@ -272,209 +265,6 @@ bool VisitInputs(const Instruction& instruction, const TVisitor& visit) { return true; } -template -void VisitIdOperands(const Instruction& instruction, std::uint32_t switchLiteralWords, const TVisitor& visit) { - const auto size = instruction.size(); - const auto from = [&](std::size_t first) { - for (auto index = first; index < size; ++index) visit(index); - }; - const auto only = [&](std::initializer_list indices) { - for (const auto index : indices) { - if (index < size) visit(index); - } - }; - constexpr std::uint32_t knownMemoryAccess = static_cast(spv::MemoryAccessVolatileMask) | static_cast(spv::MemoryAccessAlignedMask) | - static_cast(spv::MemoryAccessNontemporalMask) | static_cast(spv::MemoryAccessMakePointerAvailableMask) | - static_cast(spv::MemoryAccessMakePointerVisibleMask) | static_cast(spv::MemoryAccessNonPrivatePointerMask); - const auto memoryOperands = [&](std::size_t index) { - while (index < size) { - const auto mask = instruction[index++]; - if ((mask & ~knownMemoryAccess) != 0u) { - from(index); - return; - } - if ((mask & static_cast(spv::MemoryAccessAlignedMask)) != 0u) ++index; - if ((mask & static_cast(spv::MemoryAccessMakePointerAvailableMask)) != 0u && index < size) visit(index++); - if ((mask & static_cast(spv::MemoryAccessMakePointerVisibleMask)) != 0u && index < size) visit(index++); - } - }; - switch (Opcode(instruction)) { - case spv::OpNop: - case spv::OpCapability: - case spv::OpExtension: - case spv::OpExtInstImport: - case spv::OpMemoryModel: - case spv::OpSource: - case spv::OpSourceContinued: - case spv::OpSourceExtension: - case spv::OpString: - case spv::OpName: - case spv::OpMemberName: - case spv::OpModuleProcessed: - case spv::OpDecorate: - case spv::OpMemberDecorate: - case spv::OpDecorateString: - case spv::OpMemberDecorateString: - case spv::OpTypeVoid: - case spv::OpTypeBool: - case spv::OpTypeInt: - case spv::OpTypeFloat: - case spv::OpTypeSampler: - case spv::OpConstantTrue: - case spv::OpConstantFalse: - case spv::OpConstant: - case spv::OpConstantSampler: - case spv::OpConstantNull: - case spv::OpSpecConstantTrue: - case spv::OpSpecConstantFalse: - case spv::OpSpecConstant: - case spv::OpLabel: - case spv::OpNoLine: - case spv::OpReturn: - case spv::OpUnreachable: - case spv::OpKill: - case spv::OpFunctionEnd: - return; - case spv::OpTypeVector: - case spv::OpTypeMatrix: - case spv::OpTypeImage: - only({2}); - return; - case spv::OpTypePointer: - only({3}); - return; - case spv::OpTypeForwardPointer: - case spv::OpExecutionMode: - case spv::OpLine: - case spv::OpSelectionMerge: - only({1}); - return; - case spv::OpSpecConstantOp: - only({1}); - from(4); - return; - case spv::OpVariable: - case spv::OpFunction: - only({1, 4}); - return; - case spv::OpLoad: - only({1, 3}); - memoryOperands(4); - return; - case spv::OpStore: - case spv::OpCopyMemory: - only({1, 2}); - memoryOperands(3); - return; - case spv::OpCopyMemorySized: - only({1, 2, 3}); - memoryOperands(4); - return; - case spv::OpArrayLength: - case spv::OpCompositeExtract: - only({1, 3}); - return; - case spv::OpCompositeInsert: - case spv::OpVectorShuffle: - only({1, 3, 4}); - return; - case spv::OpExtInst: - only({1, 3}); - from(5); - return; - case spv::OpEntryPoint: { - only({2}); - std::size_t index = 3; - while (index < size) { - const auto word = instruction[index++]; - if ((word & 0xffu) == 0u || (word & 0xff00u) == 0u || (word & 0xff0000u) == 0u || (word & 0xff000000u) == 0u) break; - } - from(index); - return; - } - case spv::OpExecutionModeId: - case spv::OpDecorateId: - only({1}); - from(3); - return; - case spv::OpLoopMerge: - only({1, 2}); - return; - case spv::OpBranchConditional: - only({1, 2, 3}); - return; - case spv::OpSwitch: - only({1, 2}); - for (std::size_t index = 3; index + switchLiteralWords < size; index += switchLiteralWords + 1u) visit(index + switchLiteralWords); - return; - case spv::OpImageSampleImplicitLod: - case spv::OpImageSampleExplicitLod: - case spv::OpImageSampleProjImplicitLod: - case spv::OpImageSampleProjExplicitLod: - case spv::OpImageFetch: - case spv::OpImageRead: - case spv::OpImageSparseSampleImplicitLod: - case spv::OpImageSparseSampleExplicitLod: - case spv::OpImageSparseSampleProjImplicitLod: - case spv::OpImageSparseSampleProjExplicitLod: - case spv::OpImageSparseFetch: - case spv::OpImageSparseRead: - only({1, 3, 4}); - from(6); - return; - case spv::OpImageSampleDrefImplicitLod: - case spv::OpImageSampleDrefExplicitLod: - case spv::OpImageSampleProjDrefImplicitLod: - case spv::OpImageSampleProjDrefExplicitLod: - case spv::OpImageGather: - case spv::OpImageDrefGather: - case spv::OpImageSparseSampleDrefImplicitLod: - case spv::OpImageSparseSampleDrefExplicitLod: - case spv::OpImageSparseSampleProjDrefImplicitLod: - case spv::OpImageSparseSampleProjDrefExplicitLod: - case spv::OpImageSparseGather: - case spv::OpImageSparseDrefGather: - only({1, 3, 4, 5}); - from(7); - return; - case spv::OpImageWrite: - only({1, 2, 3}); - from(5); - return; - case spv::OpGroupIAdd: - case spv::OpGroupFAdd: - case spv::OpGroupFMin: - case spv::OpGroupUMin: - case spv::OpGroupSMin: - case spv::OpGroupFMax: - case spv::OpGroupUMax: - case spv::OpGroupSMax: - case spv::OpGroupNonUniformBallotBitCount: - case spv::OpGroupNonUniformIAdd: - case spv::OpGroupNonUniformFAdd: - case spv::OpGroupNonUniformIMul: - case spv::OpGroupNonUniformFMul: - case spv::OpGroupNonUniformSMin: - case spv::OpGroupNonUniformUMin: - case spv::OpGroupNonUniformFMin: - case spv::OpGroupNonUniformSMax: - case spv::OpGroupNonUniformUMax: - case spv::OpGroupNonUniformFMax: - case spv::OpGroupNonUniformBitwiseAnd: - case spv::OpGroupNonUniformBitwiseOr: - case spv::OpGroupNonUniformBitwiseXor: - case spv::OpGroupNonUniformLogicalAnd: - case spv::OpGroupNonUniformLogicalOr: - case spv::OpGroupNonUniformLogicalXor: - only({1, 3}); - from(5); - return; - default: - from(1); - return; - } -} - struct ScalarType { std::uint32_t width; bool boolean; diff --git a/core/shader/recompiler/tests/SpannedUniformityTests.cpp b/core/shader/recompiler/tests/SpannedUniformityTests.cpp new file mode 100644 index 0000000000..3f96d14f53 --- /dev/null +++ b/core/shader/recompiler/tests/SpannedUniformityTests.cpp @@ -0,0 +1,220 @@ +#include "SpirvBackend/SpirvSpannedUniformity.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +enum Id : std::uint32_t { + Void = 1u, + Bool, + Uint, + FunctionType, + InputPointer, + Index, + Two, + Three, + Semantics, + WorkgroupPointer, + WorkgroupArrayType, + WorkgroupArrayPointer, + Exchange, + FunctionPointer, + True, + Main, + Helper, + First = 100u, +}; + +class Module { +public: + Module() { + words = {spv::MagicNumber, 0x00010300u, 0u, 1000u, 0u}; + emit(spv::OpCapability, {spv::CapabilityShader}); + emit(spv::OpMemoryModel, {spv::AddressingModelLogical, spv::MemoryModelGLSL450}); + emit(spv::OpEntryPoint, {spv::ExecutionModelGLCompute, Main, 0x6e69616du, 0u, Index}); + emit(spv::OpExecutionMode, {Main, spv::ExecutionModeLocalSize, 64u, 1u, 1u}); + emit(spv::OpDecorate, {Index, spv::DecorationBuiltIn, spv::BuiltInLocalInvocationIndex}); + emit(spv::OpTypeVoid, {Void}); + emit(spv::OpTypeBool, {Bool}); + emit(spv::OpTypeInt, {Uint, 32u, 0u}); + emit(spv::OpTypeFunction, {FunctionType, Void}); + emit(spv::OpTypePointer, {InputPointer, spv::StorageClassInput, Uint}); + emit(spv::OpVariable, {InputPointer, Index, spv::StorageClassInput}); + emit(spv::OpConstant, {Uint, Two, 2u}); + emit(spv::OpConstant, {Uint, Three, 3u}); + emit(spv::OpConstant, {Uint, Semantics, 264u}); + emit(spv::OpConstantTrue, {Bool, True}); + emit(spv::OpTypePointer, {WorkgroupPointer, spv::StorageClassWorkgroup, Uint}); + emit(spv::OpTypeArray, {WorkgroupArrayType, Uint, Three}); + emit(spv::OpTypePointer, {WorkgroupArrayPointer, spv::StorageClassWorkgroup, WorkgroupArrayType}); + emit(spv::OpVariable, {WorkgroupArrayPointer, Exchange, spv::StorageClassWorkgroup}); + emit(spv::OpTypePointer, {FunctionPointer, spv::StorageClassFunction, Uint}); + } + + void emit(spv::Op op, std::initializer_list operands) { + words.push_back((static_cast(operands.size() + 1u) << 16u) | op); + words.insert(words.end(), operands); + } + + void function(std::uint32_t id) { + emit(spv::OpFunction, {Void, id, spv::FunctionControlMaskNone, FunctionType}); + } + + void barrier() { + emit(spv::OpControlBarrier, {Two, Two, Semantics}); + } + + void lane(std::uint32_t result) { + emit(spv::OpLoad, {Uint, result, Index}); + } + + void end() { + emit(spv::OpReturn, {}); + emit(spv::OpFunctionEnd, {}); + } + + std::vector words; +}; + +bool failed = false; + +void expect(const char* name, const Module& module, bool rejected) { + const auto reason = ShaderRecompiler::SpannedBarrierInDivergentFlow(module.words); + if (reason.has_value() == rejected) return; + std::fprintf(stderr, "%s: expected %s, got %s\n", name, rejected ? "a rejection" : "no rejection", reason ? reason->c_str() : "none"); + failed = true; +} + +Module selection(bool perLane) { + Module module; + module.function(Main); + module.emit(spv::OpLabel, {First}); + module.lane(First + 1u); + module.emit(spv::OpULessThan, {Bool, First + 2u, perLane ? First + 1u : Two, Three}); + module.emit(spv::OpSelectionMerge, {First + 4u, spv::SelectionControlMaskNone}); + module.emit(spv::OpBranchConditional, {First + 2u, First + 3u, First + 4u}); + module.emit(spv::OpLabel, {First + 3u}); + module.barrier(); + module.emit(spv::OpBranch, {First + 4u}); + module.emit(spv::OpLabel, {First + 4u}); + module.barrier(); + module.end(); + return module; +} + +Module loop(bool laneExit, bool barrierInSelection) { + Module module; + module.function(Main); + module.emit(spv::OpLabel, {First}); + module.lane(First + 1u); + module.emit(spv::OpULessThan, {Bool, First + 2u, First + 1u, Three}); + module.emit(spv::OpBranch, {First + 3u}); + module.emit(spv::OpLabel, {First + 3u}); + module.emit(spv::OpLoopMerge, {First + 6u, First + 5u, spv::LoopControlMaskNone}); + module.emit(spv::OpBranch, {First + 4u}); + module.emit(spv::OpLabel, {First + 4u}); + if (barrierInSelection) { + module.emit(spv::OpSelectionMerge, {First + 8u, spv::SelectionControlMaskNone}); + module.emit(spv::OpBranchConditional, {First + 2u, First + 7u, First + 8u}); + module.emit(spv::OpLabel, {First + 7u}); + module.emit(spv::OpBranch, {First + 8u}); + module.emit(spv::OpLabel, {First + 8u}); + } + module.barrier(); + module.emit(spv::OpBranch, {First + 5u}); + module.emit(spv::OpLabel, {First + 5u}); + module.emit(spv::OpBranchConditional, {laneExit ? First + 2u : True, First + 3u, First + 6u}); + module.emit(spv::OpLabel, {First + 6u}); + module.end(); + return module; +} + +Module call(bool perLane) { + Module module; + module.function(Helper); + module.emit(spv::OpLabel, {First + 10u}); + module.barrier(); + module.end(); + module.function(Main); + module.emit(spv::OpLabel, {First}); + module.lane(First + 1u); + module.emit(spv::OpULessThan, {Bool, First + 2u, perLane ? First + 1u : Two, Three}); + module.emit(spv::OpSelectionMerge, {First + 4u, spv::SelectionControlMaskNone}); + module.emit(spv::OpBranchConditional, {First + 2u, First + 3u, First + 4u}); + module.emit(spv::OpLabel, {First + 3u}); + module.emit(spv::OpFunctionCall, {Void, First + 5u, Helper}); + module.emit(spv::OpBranch, {First + 4u}); + module.emit(spv::OpLabel, {First + 4u}); + module.end(); + return module; +} + +Module joined(const std::function& condition) { + Module module; + module.function(Main); + module.emit(spv::OpLabel, {First}); + module.emit(spv::OpVariable, {FunctionPointer, First + 20u, spv::StorageClassFunction}); + module.lane(First + 1u); + condition(module); + module.emit(spv::OpSelectionMerge, {First + 9u, spv::SelectionControlMaskNone}); + module.emit(spv::OpBranchConditional, {First + 8u, First + 7u, First + 9u}); + module.emit(spv::OpLabel, {First + 7u}); + module.barrier(); + module.emit(spv::OpBranch, {First + 9u}); + module.emit(spv::OpLabel, {First + 9u}); + module.end(); + return module; +} + +void laneSplit(Module& module, std::uint32_t merge) { + module.emit(spv::OpULessThan, {Bool, First + 2u, First + 1u, Three}); + module.emit(spv::OpSelectionMerge, {merge, spv::SelectionControlMaskNone}); + module.emit(spv::OpBranchConditional, {First + 2u, First + 3u, merge}); + module.emit(spv::OpLabel, {First + 3u}); +} + +} + +int main() { + expect("a barrier under a lane comparison", selection(true), true); + expect("a barrier under a constant comparison", selection(false), false); + expect("a barrier in a loop a lane value leaves", loop(true, false), true); + expect("a barrier in a uniform loop after a per-lane selection", loop(false, true), false); + expect("a call reaching a barrier under a lane comparison", call(true), true); + expect("a call reaching a barrier under a constant comparison", call(false), false); + expect("a barrier under a phi joining a per-lane selection", joined([](Module& module) { + laneSplit(module, First + 4u); + module.emit(spv::OpBranch, {First + 4u}); + module.emit(spv::OpLabel, {First + 4u}); + module.emit(spv::OpPhi, {Bool, First + 8u, True, First + 3u, True, First}); + }), true); + expect("a barrier under a variable stored on one side of a per-lane selection", joined([](Module& module) { + module.emit(spv::OpStore, {First + 20u, Two}); + laneSplit(module, First + 4u); + module.emit(spv::OpStore, {First + 20u, Three}); + module.emit(spv::OpBranch, {First + 4u}); + module.emit(spv::OpLabel, {First + 4u}); + module.emit(spv::OpLoad, {Uint, First + 5u, First + 20u}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 5u, Two}); + }), true); + expect("a barrier under an atomic result", joined([](Module& module) { + module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two}); + module.emit(spv::OpAtomicIAdd, {Uint, First + 6u, First + 5u, Two, Semantics, Three}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); + }), true); + expect("a barrier under a workgroup word read at a uniform index", joined([](Module& module) { + module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two}); + module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); + }), false); + expect("a barrier under a workgroup word read at a lane index", joined([](Module& module) { + module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, First + 1u}); + module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); + }), true); + return failed ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index dd75c7f25f..6717fe9c02 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -301,6 +301,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_SNAPSHOT_CHECK` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | skips the live compare that proves a captured snapshot still equals the guest bytes | | `APS5_NO_SORTED_SNAPSHOT_LOOKUP` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | scans the snapshot's regions in AddSnapshot instead of searching them | | `APS5_NO_SPACE_STAMP_ONCE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | stamps every direct import of a cached address space on each hit instead of once per space | +| `APS5_NO_SPANNED_WAVE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | keeps single-wave wave64 compute programs at two guest lanes per invocation instead of spanning two full 32-lane subgroups | | `APS5_NO_SPARSE_HEAP_MIRROR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | disables sparse heap mirrors, mirroring only whole heap ranges | | `APS5_NO_STAGE_A_IMAGES` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | leaves every image element to the stage-B full lookup instead of a stage-A record | | `APS5_NO_STORAGE_DEDUPE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | looks each identical consecutive storage descriptor up, and refreshes its image, separately | @@ -365,7 +366,6 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_SKIP_REQUEST_DUMPS` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderDiagnostics.cpp` | 8 | caps RecompileRequest dumps per shader-skip reason | | `APS5_SKIP_SHADER` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp` | 0 | hex program address of a shader to skip preparing | | `APS5_SNAPSHOT_CHECK_FATAL` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | throws on the first snapshot that differs from registered memory instead of only counting them | -| `APS5_SPANNED_WAVE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | spans a single-wave wave64 compute program over two full 32-lane subgroups | | `APS5_SPIRV_OPT` | string | `core/libs/prx/libSceAgcDriver/tests/SpirvOptimizer.cpp` | | selects the SPIR-V optimization pipeline: `full` restores the performance pipeline, `none` skips optimization | | `APS5_SRGB_ATTACHMENT_PROXY` | string | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | forces the sRGB color-attachment proxy format; `1` forces it | | `APS5_SRGB_SHADER_DECODE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp` | | `1` forces shader sRGB decode for the sRGB 8-bit formats; `copy` does the same, except that 8 and 8_8 sRGB sample an RGBA8 sRGB copy when the device can blit to RGBA8 | diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 551838eb6d..86bc5a73af 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -172,7 +172,7 @@ - [SDWA source selects on `v_cvt_f32_ubyte1`, `v_cvt_f32_ubyte2` and `v_cvt_f32_ubyte3`](../../../core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp) (shader recompiler) - byte N is taken from source 0 after `SRC0_SEL` and `SRC0_SEXT` select and extend it, since SDWA supplies source 0 (RDNA 2 ISA guide 6.8 and 13.3.7) and the opcode reads `S0.u[8N+7:8N]` (12.8). Not measured on hardware. - [Flag structurization](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a graph that every structurizer attempt refuses, irreducible and non-terminating graphs included, is rebuilt with one reach flag per block and one again flag per loop (`structurizeWithFlags`): each block keeps one copy and runs behind a guard on its flag, and each strongly connected component becomes a loop whose single latch repeats it while a back edge set its flag. The output has about 8 times the input's blocks and one route variable per block, and ends in an empty Return that never runs. Only blocks that start the top-level order unconditionally run unguarded, so resource planning also sees the static paths that skip a guarded block and can plan extra descriptor candidates for them. The attempts still run first. Shared merge splitting refuses an attempt once the graph has grown past 16 times its block count at the start of splitting (at least 64 blocks); succeeding attempts grew at most 10.6 times on random graphs of up to 128 blocks, and an attempt that would succeed only beyond the budget leaves the graph to a later attempt or to this construction. Splitting that adds one block per round stays within that budget until its 4 rounds per block run out, so the attempts can still take more than 10 s (9 of 2,000 random graphs of up to 128 blocks, 63 s at most). `APS5_FORCE_STRUCTURIZER_FALLBACK=1` builds every graph this way, for tests - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) use only the base address, stride and record count for resources accessed exclusively by scalar loads; their fourth word is normalized to zero, following AMD RDNA2 ISA section 7.2.2. Vector and mixed accesses retain descriptor validation. Native Vulkan readbacks cover ignored fourth-word fields on Windows/NVIDIA; their console behavior has not been measured. -- [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - `APS5_SPANNED_WAVE=1`, off by default, runs a single-wave wave64 compute program at one lane per invocation on a 32-lane host subgroup and exchanges ballots, shuffles and reductions between the two host subgroups through workgroup memory and barriers. The driver turns it on only where compute subgroups can be pinned at 32 full lanes (`subgroupSizeControl` and `computeFullSubgroups`), passes it as `SpirvTarget::spannedWaves` (part of the cache key and the serialized request), and creates the pipelines of spanned programs with `REQUIRE_FULL_SUBGROUPS`, since the lane number `SubgroupId * 32 + SubgroupLocalInvocationId` needs two full subgroups. Those barriers are correct only where every invocation of the workgroup reaches them, which holds for the guest's wave-uniform control flow. The emulated 64-bit LDS atomics ([LockedLdsUpdate](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp)) run only on the lanes the exec mask keeps, so their lock loop ballots within its own host subgroup, as its elect already does. Any other cross-lane exchange emitted inside per-lane control flow would hang a spanned wave in the same way, and the emitter does not reject one: there is no uniformity check. The only check so far is manual: disassemble the module (`spirv-dis`), list the selection and loop constructs that enclose each `OpControlBarrier`, and read the condition of each enclosing selection. In Ghost of Yōtei's program 0x80003af600 (PPSA26344), 108 of 111 barriers sat only in loops on wave-uniform conditions and 3 inside per-invocation conditionals, all in the lock loop. Other programs and titles have not been scanned. An earlier, cheaper ballot (one barrier instead of three) also lost the device for a reason that is not understood, which may be another barrier placement problem. Measured on an RTX 4090 (Windows): an offline replay of that program and two Yōtei runs that previously lost the device (NVIDIA TDR) now run past it, and [LdsMaskWaterfall.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp) runs both modes and covers a half-inactive wave. Stray (PPSA02101) on Intel ANV and a Demon's Souls (PPSA01342) request replay showed no fault, before this fix +- [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - a spanned wave exchanges ballots, shuffles and reductions between its two host subgroups through workgroup barriers, which hang the device when not every invocation reaches them. [SpannedBarrierInDivergentFlow](../../../core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp) sends a program back to two lanes per invocation when a barrier sits in control flow that a per-invocation value decides. It treats a load of workgroup, storage or uniform memory at a uniform address as uniform, which is unverified for memory another invocation writes without a barrier in between. An earlier one-barrier ballot also lost the device for a reason that is not understood. `APS5_NO_SPANNED_WAVE` turns spanning off. - [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it) those loads keep their selections; LDS loads, stores, atomics, formatted and physical-address (BDA) accesses always do. LDS keeps the branch because NVIDIA's compiler predicates a branched LDS load at no cost, while the select form costs about 2.5 instructions per dword and splits a `ds_read_b128` into four loads (RTX 4060 measurements by @karanvora2599 in #4048); Intel's static counts favour the select only for wide reads, and run time was not measured on either. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. - [Merged hull programs](../../../core/shader/recompiler/Optimization/src/TessellationLowering.cpp) (registered binary type 3) run whole in the hull stage, one invocation per output control point. On the console hull lane i keeps local lane i's registers, which with fewer input than output control points stop matching the patch's control points from i = inputs on; here invocation i holds control point min(i, inputs - 1) of its own patch. A wave-mask branch (`s_cbranch_exec[n]z`, `s_cbranch_vcc[n]z`) tests the invocation's own lane, so it can skip a block, scalar instructions included, that the console runs for the wave; `v_readfirstlane_b32` returns the invocation's own value, not the first active lane's, and another lane's register throws. The register layout (`s3`, `v0`-`v5`, as for the separate halves) is not measured on hardware. s[0:1] hold the hull user-data address (SH 0x102/0x103) when a draw sets it and zero otherwise, so a program reading it unset faults at the null address. - [Wave LDS ordering on AMD's proprietary driver](../../../core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp) (shader recompiler) - there ([SubgroupBarrier.hpp](../../../core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp)) one guest wave's LDS accesses are ordered by a subgroup-scope `OpMemoryBarrier` with no control barrier, relying on a 32-lane subgroup being one hardware wave whose lanes run in lockstep and issue LDS instructions in order, which Vulkan does not guarantee; for LDS kept in device memory the same is assumed of global loads and stores, unmeasured. Driver 32.0.21042.62 (Radeon 0x13c0, RDNA 2) compiled the subgroup-scope `OpControlBarrier` to `s_barrier`, which waves in a branch paired with other waves' barriers (PS5LM Plus). Measured on that driver only ([LdsTreeReduction.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsTreeReduction.cpp)); a Radeon 860M on 32.0.13058.8001 reportedly did not show the fault. From 9bc1e1d2818deae12dc49aa779e5bafe2d21bec8 Mon Sep 17 00:00:00 2001 From: ThatsLeo Date: Tue, 20 Oct 2026 02:45:13 +0200 Subject: [PATCH 006/318] fix(shader): fence shared loads and early exits in the spanned uniformity check A shared load at a uniform address is uniform only from memory the program never writes, or from workgroup memory a barrier fences since the last write. A per-lane return or continue makes the code after its merge divergent. The test links on MinGW. --- core/shader/recompiler/CMakeLists.txt | 7 + .../src/SpirvSpannedUniformity.cpp | 168 +++++++++++++++++- .../tests/SpannedUniformityTests.cpp | 54 ++++++ docs/dev/debt/shader.md | 2 +- 4 files changed, 222 insertions(+), 9 deletions(-) diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 52a2ae6181..25ed2d3a22 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -357,8 +357,15 @@ function(add_shader_recompiler target runtimeTarget) target_include_directories(spanned_uniformity_tests PRIVATE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/include" ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}) + set_target_properties(spanned_uniformity_tests PROPERTIES CXX_EXTENSIONS OFF) + configure_windows_unwind(spanned_uniformity_tests) add_test(NAME spanned_uniformity COMMAND spanned_uniformity_tests) set_tests_properties(spanned_uniformity PROPERTIES TIMEOUT 20) + if(MINGW) + target_link_libraries(spanned_uniformity_tests PRIVATE libc) + set_tests_properties(spanned_uniformity PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() if(BUILD_TESTING AND ANYPS5_ENABLE_SPIRV_TOOLS AND NOT ANYPS5_USE_SYSTEM_DEPS AND NOT TARGET spirv_validator_exit_tests) add_test_executable(spirv_validator_exit_tests "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/tests/SpirvValidatorExit.cpp") diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp index 6e495f5ced..9451bf1e29 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp @@ -28,6 +28,8 @@ struct Function { struct Module { std::unordered_map builtins; std::unordered_map storageClasses; + std::unordered_map pointerClasses; + std::unordered_map constants; std::vector functions; }; @@ -45,6 +47,12 @@ Module Parse(std::span words) { case spv::OpDecorate: if (instruction.size() >= 4u && instruction[2] == spv::DecorationBuiltIn) module.builtins[instruction[1]] = instruction[3]; break; + case spv::OpTypePointer: + if (instruction.size() >= 4u) module.pointerClasses[instruction[1]] = instruction[2]; + break; + case spv::OpConstant: + if (instruction.size() >= 4u) module.constants[instruction[2]] = instruction[3]; + break; case spv::OpVariable: if (instruction.size() >= 4u) module.storageClasses[instruction[2]] = instruction[3]; if (block != nullptr) block->body.push_back(std::move(instruction)); @@ -118,13 +126,37 @@ bool PrivateStorage(std::uint32_t storageClass) { return storageClass == spv::StorageClassFunction || storageClass == spv::StorageClassPrivate; } +bool AtomicWrite(spv::Op op) { + const auto raw = static_cast(op); + return (raw >= spv::OpAtomicExchange && raw <= spv::OpAtomicXor) || op == spv::OpAtomicFlagTestAndSet || op == spv::OpAtomicFMinEXT || op == spv::OpAtomicFMaxEXT || + op == spv::OpAtomicFAddEXT; +} + +bool ImageRead(spv::Op op) { + return op == spv::OpImageRead || op == spv::OpImageFetch || op == spv::OpImageSparseRead || op == spv::OpImageSparseFetch; +} + class Analysis { public: explicit Analysis(const Module& module) : module(module) { for (const auto& function : module.functions) { for (const auto& block : function.blocks) { for (const auto& instruction : block.body) { - if (Opcode(instruction) == spv::OpControlBarrier) barriers.insert(function.id); + const auto op = Opcode(instruction); + if (op == spv::OpControlBarrier) barriers.insert(function.id); + bool hasResult = false; + bool hasType = false; + spv::HasResultAndType(op, &hasResult, &hasType); + if (hasResult && hasType && instruction.size() > 2u) types[instruction[2]] = instruction[1]; + } + } + } + for (const auto& function : module.functions) { + for (const auto& block : function.blocks) { + for (const auto& instruction : block.body) { + const auto pointer = WrittenPointer(instruction); + if (pointer != 0u) written.insert(ClassOf(pointer)); + if (Opcode(instruction) == spv::OpImageWrite) written.insert(spv::StorageClassImage); } } } @@ -142,13 +174,16 @@ public: } std::optional Run() { - for (std::size_t round = 0u; round < 64u; ++round) { + if (barriers.empty()) return std::nullopt; + std::vector flows; + flows.reserve(module.functions.size()); + for (const auto& function : module.functions) flows.push_back(BuildFlow(function)); + for (;;) { const auto before = Size(); report.reset(); - for (const auto& function : module.functions) Visit(function); + for (std::size_t index = 0u; index < module.functions.size(); ++index) Visit(module.functions[index], flows[index]); if (Size() == before) return report; } - throw std::runtime_error("spanned wave uniformity did not settle"); } private: @@ -160,6 +195,74 @@ private: std::unordered_map> children; }; + static std::uint32_t WrittenPointer(const Instruction& instruction) { + const auto op = Opcode(instruction); + if ((op == spv::OpStore || op == spv::OpAtomicStore || op == spv::OpAtomicFlagClear || op == spv::OpCopyMemory) && instruction.size() > 1u) return instruction[1]; + if (AtomicWrite(op) && instruction.size() > 3u) return instruction[3]; + return 0u; + } + + std::uint32_t ClassOf(std::uint32_t pointer) const { + if (const auto variable = module.storageClasses.find(pointer); variable != module.storageClasses.end()) return variable->second; + const auto type = types.find(pointer); + if (type == types.end()) return UnknownClass; + const auto storage = module.pointerClasses.find(type->second); + return storage == module.pointerClasses.end() ? UnknownClass : storage->second; + } + + bool Written(std::uint32_t storageClass) const { + return written.contains(UnknownClass) || written.contains(storageClass); + } + + bool Fence(const Instruction& instruction) const { + if (Opcode(instruction) != spv::OpControlBarrier || instruction.size() < 4u) return false; + const auto scope = module.constants.find(instruction[1]); + const auto semantics = module.constants.find(instruction[3]); + return scope != module.constants.end() && semantics != module.constants.end() && scope->second <= spv::ScopeWorkgroup && + (semantics->second & spv::MemorySemanticsWorkgroupMemoryMask) != 0u; + } + + bool Fenced(bool fenced, const Instruction& instruction) const { + if (Fence(instruction)) return true; + if (Opcode(instruction) == spv::OpFunctionCall) return false; + const auto pointer = WrittenPointer(instruction); + if (pointer == 0u) return fenced; + const auto storageClass = ClassOf(pointer); + return fenced && storageClass != spv::StorageClassWorkgroup && storageClass != UnknownClass; + } + + bool Racy(std::uint32_t pointer, bool fenced) const { + const auto storageClass = ClassOf(pointer); + if (storageClass != UnknownClass && (PrivateStorage(storageClass) || storageClass == spv::StorageClassInput)) return false; + if (!Written(storageClass)) return false; + return !fenced || storageClass != spv::StorageClassWorkgroup; + } + + std::unordered_map FencedEntries(const Function& function, const Flow& flow, const std::unordered_map& blocks) const { + std::unordered_map> predecessors; + for (const auto node : flow.order) { + for (const auto successor : flow.successors.at(node)) predecessors[successor].push_back(node); + } + std::unordered_map entries; + std::unordered_map exits; + for (const auto node : flow.order) exits[node] = true; + const auto root = function.blocks.front().label; + for (bool changed = true; changed;) { + changed = false; + for (const auto node : flow.order) { + bool fenced = node != root; + for (const auto predecessor : predecessors[node]) fenced = fenced && exits.at(predecessor); + entries[node] = fenced; + for (const auto& instruction : blocks.at(node)->body) fenced = Fenced(fenced, instruction); + if (exits[node] != fenced) { + exits[node] = fenced; + changed = true; + } + } + } + return entries; + } + std::size_t Size() const { return nonUniform.size() + storedNonUniform.size() + varyingFunctions.size(); } @@ -237,6 +340,27 @@ private: return flow; } + static bool Exits(spv::Op op) { + return op == spv::OpReturn || op == spv::OpReturnValue || op == spv::OpKill || op == spv::OpTerminateInvocation || op == spv::OpUnreachable; + } + + static std::vector Reachable(const Flow& flow, std::uint32_t from) { + std::vector blocks; + std::unordered_set seen{from}; + std::vector pending{from}; + while (!pending.empty()) { + const auto node = pending.back(); + pending.pop_back(); + blocks.push_back(node); + if (const auto found = flow.successors.find(node); found != flow.successors.end()) { + for (const auto successor : found->second) { + if (seen.insert(successor).second) pending.push_back(successor); + } + } + } + return blocks; + } + static std::vector Construct(const Flow& flow, std::uint32_t header, std::uint32_t merge) { std::vector blocks; std::vector pending; @@ -260,9 +384,8 @@ private: return pointer; } - void Visit(const Function& function) { + void Visit(const Function& function, const Flow& flow) { if (function.blocks.empty()) return; - const auto flow = BuildFlow(function); std::unordered_map blocks; std::unordered_map chains; for (const auto& block : function.blocks) { @@ -272,6 +395,7 @@ private: if ((op == spv::OpAccessChain || op == spv::OpInBoundsAccessChain || op == spv::OpPtrAccessChain) && instruction.size() > 3u) chains[instruction[2]] = instruction[3]; } } + const auto fencedEntries = FencedEntries(function, flow, blocks); for (const auto parameter : function.parameters) nonUniform.insert(parameter); std::unordered_set divergent; std::unordered_set divergentMerges; @@ -303,11 +427,29 @@ private: divergent.insert(construct.begin(), construct.end()); if (mergeOp == spv::OpLoopMerge) divergent.insert(header); divergentMerges.insert(merge); + const std::unordered_set inside(construct.begin(), construct.end()); + bool exits = false; + bool escapes = false; + for (const auto node : construct) { + exits = exits || (!blocks.at(node)->body.empty() && Exits(Opcode(blocks.at(node)->body.back()))); + for (const auto successor : flow.successors.at(node)) escapes = escapes || (successor != merge && successor != header && !inside.contains(successor)); + } + if (exits) { + const auto after = Reachable(flow, merge); + divergent.insert(after.begin(), after.end()); + } else if (escapes) { + const auto after = Construct(flow, merge, 0u); + divergent.insert(merge); + divergent.insert(after.begin(), after.end()); + } } for (const auto label : flow.order) { const bool inDivergentFlow = divergent.contains(label); + bool fenced = fencedEntries.at(label); for (const auto& instruction : blocks.at(label)->body) { const auto op = Opcode(instruction); + const bool fencedBefore = fenced; + fenced = Fenced(fenced, instruction); if (op == spv::OpStore && instruction.size() >= 3u) { const auto variable = BaseVariable(instruction[1], chains); const auto storage = module.storageClasses.find(variable); @@ -326,9 +468,12 @@ private: const auto builtin = module.builtins.find(variable); const auto storage = module.storageClasses.find(variable); varying = NonUniform(instruction[3]) || (builtin != module.builtins.end() && VaryingBuiltin(builtin->second)) || storedNonUniform.contains(variable) || - (storage != module.storageClasses.end() && storage->second == spv::StorageClassInput && builtin == module.builtins.end()); + (storage != module.storageClasses.end() && storage->second == spv::StorageClassInput && builtin == module.builtins.end()) || Racy(instruction[3], fencedBefore); break; } + case spv::OpAtomicLoad: + varying = NonUniform(instruction.at(3)) || Racy(instruction[3], fencedBefore); + break; case spv::OpPhi: varying = inDivergentFlow || divergentMerges.contains(label); for (std::size_t operand = 3u; !varying && operand < instruction.size(); operand += 2u) varying = NonUniform(instruction[operand]); @@ -338,7 +483,7 @@ private: for (std::size_t operand = 4u; !varying && operand < instruction.size(); ++operand) varying = NonUniform(instruction[operand]); break; default: - varying = PerInvocationResult(op); + varying = PerInvocationResult(op) || (ImageRead(op) && Written(spv::StorageClassImage)); if (!varying) VisitIdOperands(instruction, 1u, [&](std::size_t operand) { varying = varying || NonUniform(instruction[operand]); }); break; } @@ -349,7 +494,11 @@ private: } } + static constexpr std::uint32_t UnknownClass = 0xffffffffu; + const Module& module; + std::unordered_map types; + std::unordered_set written; std::unordered_set barriers; std::unordered_set nonUniform; std::unordered_set storedNonUniform; @@ -360,6 +509,9 @@ private: } std::optional SpannedBarrierInDivergentFlow(std::span words) { + bool barrier = false; + for (std::size_t cursor = 5u; !barrier && cursor < words.size(); cursor += std::max(words[cursor] >> 16u, 1u)) barrier = (words[cursor] & 0xffffu) == spv::OpControlBarrier; + if (!barrier) return std::nullopt; const auto module = Parse(words); return Analysis(module).Run(); } diff --git a/core/shader/recompiler/tests/SpannedUniformityTests.cpp b/core/shader/recompiler/tests/SpannedUniformityTests.cpp index 3f96d14f53..80ad5ba1e5 100644 --- a/core/shader/recompiler/tests/SpannedUniformityTests.cpp +++ b/core/shader/recompiler/tests/SpannedUniformityTests.cpp @@ -153,6 +153,32 @@ Module call(bool perLane) { return module; } +Module early(bool perLane, spv::Op exit) { + Module module; + module.function(Main); + module.emit(spv::OpLabel, {First}); + module.lane(First + 1u); + module.emit(spv::OpULessThan, {Bool, First + 2u, perLane ? First + 1u : Two, Three}); + module.emit(spv::OpBranch, {First + 10u}); + module.emit(spv::OpLabel, {First + 10u}); + module.emit(spv::OpLoopMerge, {First + 13u, First + 12u, spv::LoopControlMaskNone}); + module.emit(spv::OpBranch, {First + 11u}); + module.emit(spv::OpLabel, {First + 11u}); + module.emit(spv::OpSelectionMerge, {First + 4u, spv::SelectionControlMaskNone}); + module.emit(spv::OpBranchConditional, {First + 2u, First + 3u, First + 4u}); + module.emit(spv::OpLabel, {First + 3u}); + if (exit == spv::OpReturn) module.emit(spv::OpReturn, {}); + else module.emit(spv::OpBranch, {First + 12u}); + module.emit(spv::OpLabel, {First + 4u}); + module.barrier(); + module.emit(spv::OpBranch, {First + 12u}); + module.emit(spv::OpLabel, {First + 12u}); + module.emit(spv::OpBranchConditional, {True, First + 10u, First + 13u}); + module.emit(spv::OpLabel, {First + 13u}); + module.end(); + return module; +} + Module joined(const std::function& condition) { Module module; module.function(Main); @@ -185,6 +211,9 @@ int main() { expect("a barrier in a loop a lane value leaves", loop(true, false), true); expect("a barrier in a uniform loop after a per-lane selection", loop(false, true), false); expect("a call reaching a barrier under a lane comparison", call(true), true); + expect("a barrier after a per-lane return", early(true, spv::OpReturn), true); + expect("a barrier after a per-lane continue", early(true, spv::OpBranch), true); + expect("a barrier after a uniform return", early(false, spv::OpReturn), false); expect("a call reaching a barrier under a constant comparison", call(false), false); expect("a barrier under a phi joining a per-lane selection", joined([](Module& module) { laneSplit(module, First + 4u); @@ -211,6 +240,31 @@ int main() { module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); }), false); + expect("a barrier under a workgroup word read at a uniform index after a store", joined([](Module& module) { + module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two}); + module.emit(spv::OpStore, {First + 5u, Three}); + module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); + }), true); + expect("a barrier under a workgroup word read at a uniform index after a store and a barrier", joined([](Module& module) { + module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two}); + module.emit(spv::OpStore, {First + 5u, Three}); + module.barrier(); + module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); + }), false); + expect("a barrier under a workgroup word read at a uniform index after a store on one path past the barrier", joined([](Module& module) { + module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two}); + module.barrier(); + module.emit(spv::OpSelectionMerge, {First + 4u, spv::SelectionControlMaskNone}); + module.emit(spv::OpBranchConditional, {True, First + 3u, First + 4u}); + module.emit(spv::OpLabel, {First + 3u}); + module.emit(spv::OpStore, {First + 5u, Three}); + module.emit(spv::OpBranch, {First + 4u}); + module.emit(spv::OpLabel, {First + 4u}); + module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); + }), true); expect("a barrier under a workgroup word read at a lane index", joined([](Module& module) { module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, First + 1u}); module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 86bc5a73af..55071985aa 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -172,7 +172,7 @@ - [SDWA source selects on `v_cvt_f32_ubyte1`, `v_cvt_f32_ubyte2` and `v_cvt_f32_ubyte3`](../../../core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp) (shader recompiler) - byte N is taken from source 0 after `SRC0_SEL` and `SRC0_SEXT` select and extend it, since SDWA supplies source 0 (RDNA 2 ISA guide 6.8 and 13.3.7) and the opcode reads `S0.u[8N+7:8N]` (12.8). Not measured on hardware. - [Flag structurization](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a graph that every structurizer attempt refuses, irreducible and non-terminating graphs included, is rebuilt with one reach flag per block and one again flag per loop (`structurizeWithFlags`): each block keeps one copy and runs behind a guard on its flag, and each strongly connected component becomes a loop whose single latch repeats it while a back edge set its flag. The output has about 8 times the input's blocks and one route variable per block, and ends in an empty Return that never runs. Only blocks that start the top-level order unconditionally run unguarded, so resource planning also sees the static paths that skip a guarded block and can plan extra descriptor candidates for them. The attempts still run first. Shared merge splitting refuses an attempt once the graph has grown past 16 times its block count at the start of splitting (at least 64 blocks); succeeding attempts grew at most 10.6 times on random graphs of up to 128 blocks, and an attempt that would succeed only beyond the budget leaves the graph to a later attempt or to this construction. Splitting that adds one block per round stays within that budget until its 4 rounds per block run out, so the attempts can still take more than 10 s (9 of 2,000 random graphs of up to 128 blocks, 63 s at most). `APS5_FORCE_STRUCTURIZER_FALLBACK=1` builds every graph this way, for tests - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) use only the base address, stride and record count for resources accessed exclusively by scalar loads; their fourth word is normalized to zero, following AMD RDNA2 ISA section 7.2.2. Vector and mixed accesses retain descriptor validation. Native Vulkan readbacks cover ignored fourth-word fields on Windows/NVIDIA; their console behavior has not been measured. -- [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - a spanned wave exchanges ballots, shuffles and reductions between its two host subgroups through workgroup barriers, which hang the device when not every invocation reaches them. [SpannedBarrierInDivergentFlow](../../../core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp) sends a program back to two lanes per invocation when a barrier sits in control flow that a per-invocation value decides. It treats a load of workgroup, storage or uniform memory at a uniform address as uniform, which is unverified for memory another invocation writes without a barrier in between. An earlier one-barrier ballot also lost the device for a reason that is not understood. `APS5_NO_SPANNED_WAVE` turns spanning off. +- [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - a spanned wave exchanges ballots, shuffles and reductions between its two host subgroups through workgroup barriers, which hang the device when not every invocation reaches them. [SpannedBarrierInDivergentFlow](../../../core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp) sends a program back to two lanes per invocation when a barrier sits in control flow that a per-invocation value decides. It counts a load at a uniform address as uniform only from memory the program never writes, or from workgroup memory a barrier fences on every path since the last write; writes by other dispatches or queues are not considered. An earlier one-barrier ballot also lost the device for a reason that is not understood. `APS5_NO_SPANNED_WAVE` turns spanning off. - [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it) those loads keep their selections; LDS loads, stores, atomics, formatted and physical-address (BDA) accesses always do. LDS keeps the branch because NVIDIA's compiler predicates a branched LDS load at no cost, while the select form costs about 2.5 instructions per dword and splits a `ds_read_b128` into four loads (RTX 4060 measurements by @karanvora2599 in #4048); Intel's static counts favour the select only for wide reads, and run time was not measured on either. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. - [Merged hull programs](../../../core/shader/recompiler/Optimization/src/TessellationLowering.cpp) (registered binary type 3) run whole in the hull stage, one invocation per output control point. On the console hull lane i keeps local lane i's registers, which with fewer input than output control points stop matching the patch's control points from i = inputs on; here invocation i holds control point min(i, inputs - 1) of its own patch. A wave-mask branch (`s_cbranch_exec[n]z`, `s_cbranch_vcc[n]z`) tests the invocation's own lane, so it can skip a block, scalar instructions included, that the console runs for the wave; `v_readfirstlane_b32` returns the invocation's own value, not the first active lane's, and another lane's register throws. The register layout (`s3`, `v0`-`v5`, as for the separate halves) is not measured on hardware. s[0:1] hold the hull user-data address (SH 0x102/0x103) when a draw sets it and zero otherwise, so a program reading it unset faults at the null address. - [Wave LDS ordering on AMD's proprietary driver](../../../core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp) (shader recompiler) - there ([SubgroupBarrier.hpp](../../../core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp)) one guest wave's LDS accesses are ordered by a subgroup-scope `OpMemoryBarrier` with no control barrier, relying on a 32-lane subgroup being one hardware wave whose lanes run in lockstep and issue LDS instructions in order, which Vulkan does not guarantee; for LDS kept in device memory the same is assumed of global loads and stores, unmeasured. Driver 32.0.21042.62 (Radeon 0x13c0, RDNA 2) compiled the subgroup-scope `OpControlBarrier` to `s_barrier`, which waves in a branch paired with other waves' barriers (PS5LM Plus). Measured on that driver only ([LdsTreeReduction.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsTreeReduction.cpp)); a Radeon 860M on 32.0.13058.8001 reportedly did not show the fault. From b9362d8ef5b5dc4f5436363da843843274c784c8 Mon Sep 17 00:00:00 2001 From: ThatsLeo Date: Wed, 21 Oct 2026 01:36:57 +0200 Subject: [PATCH 007/318] fix(shader): keep the first guarded SRT fault when invocations do not stop A program whose invocations keep running after a fault, such as a spanned wave, wrote the record of each later guarded read over the first one. It now writes the record only while the word is 0. --- .../src/SpirvMemory/SpirvMemoryInstructions.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index d997a174eb..46c9a12442 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1566,7 +1566,15 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { EmitIfCondition(state, poisoned, [&] { const auto record = Binary(state, spv::OpIAdd, u32, ConstantU32(state, records), Binary(state, spv::OpIMul, u32, Binary(state, spv::OpISub, u32, flag, ConstantU32(state, 1u)), ConstantU32(state, 3u))); const auto stage = ConstantU32(state, static_cast(resources.stage) << BdaAbi::SrtGuardStageShift); - state.module.AddFunction(spv::OpAtomicStore, BdaWord(state, state.faultBufferVariable, ConstantU32(state, BdaAbi::SrtGuardWord)), ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), Binary(state, spv::OpBitwiseOr, u32, record, stage)); + const auto word = BdaWord(state, state.faultBufferVariable, ConstantU32(state, BdaAbi::SrtGuardWord)); + const auto scope = ConstantU32(state, spv::ScopeDevice); + const auto relaxed = ConstantU32(state, spv::MemorySemanticsMaskNone); + const auto value = Binary(state, spv::OpBitwiseOr, u32, record, stage); + if (state.bdaStopsInvocations) { + state.module.AddFunction(spv::OpAtomicStore, word, scope, relaxed, value); + } else { + state.module.AddFunction(spv::OpAtomicCompareExchange, u32, state.module.AllocateId(), word, scope, relaxed, relaxed, value, ConstantU32(state, 0u)); + } }); StopBdaInvocationIf(state, poisoned); } From 5cce035785a1653219c14c5031f6ac38f6e37e6d Mon Sep 17 00:00:00 2001 From: ThatsLeo Date: Wed, 21 Oct 2026 02:04:27 +0200 Subject: [PATCH 008/318] fix(shader): treat guest memory as one class and catch unmerged per-lane continues A store through a device address now makes buffer and uniform loads per-invocation too. A construct is divergent when a per-lane branch without a merge leaves it or continues its loop. --- .../src/SpirvSpannedUniformity.cpp | 30 +++++++++++++- .../tests/SpannedUniformityTests.cpp | 40 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp index 9451bf1e29..1b7a231b6b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp @@ -126,6 +126,16 @@ bool PrivateStorage(std::uint32_t storageClass) { return storageClass == spv::StorageClassFunction || storageClass == spv::StorageClassPrivate; } +std::uint32_t MemoryClass(std::uint32_t storageClass) { + switch (storageClass) { + case spv::StorageClassUniform: + case spv::StorageClassPhysicalStorageBuffer: + return spv::StorageClassStorageBuffer; + default: + return storageClass; + } +} + bool AtomicWrite(spv::Op op) { const auto raw = static_cast(op); return (raw >= spv::OpAtomicExchange && raw <= spv::OpAtomicXor) || op == spv::OpAtomicFlagTestAndSet || op == spv::OpAtomicFMinEXT || op == spv::OpAtomicFMaxEXT || @@ -203,11 +213,11 @@ private: } std::uint32_t ClassOf(std::uint32_t pointer) const { - if (const auto variable = module.storageClasses.find(pointer); variable != module.storageClasses.end()) return variable->second; + if (const auto variable = module.storageClasses.find(pointer); variable != module.storageClasses.end()) return MemoryClass(variable->second); const auto type = types.find(pointer); if (type == types.end()) return UnknownClass; const auto storage = module.pointerClasses.find(type->second); - return storage == module.pointerClasses.end() ? UnknownClass : storage->second; + return storage == module.pointerClasses.end() ? UnknownClass : MemoryClass(storage->second); } bool Written(std::uint32_t storageClass) const { @@ -423,6 +433,22 @@ private: } } } + if (!split) { + const auto continueTarget = mergeOp == spv::OpLoopMerge ? mergeInstruction.at(2) : 0u; + for (const auto node : construct) { + const auto& nodeBody = blocks.at(node)->body; + if (nodeBody.empty()) continue; + const auto& exit = nodeBody.back(); + const auto exitOp = Opcode(exit); + if ((exitOp != spv::OpBranchConditional && exitOp != spv::OpSwitch) || !NonUniform(exit.at(1))) continue; + if (nodeBody.size() >= 2u && (Opcode(nodeBody[nodeBody.size() - 2u]) == spv::OpSelectionMerge || Opcode(nodeBody[nodeBody.size() - 2u]) == spv::OpLoopMerge)) continue; + const auto& successors = flow.successors.at(node); + if (std::find(successors.begin(), successors.end(), merge) != successors.end() || (continueTarget != 0u && std::find(successors.begin(), successors.end(), continueTarget) != successors.end())) { + split = true; + break; + } + } + } if (!split) continue; divergent.insert(construct.begin(), construct.end()); if (mergeOp == spv::OpLoopMerge) divergent.insert(header); diff --git a/core/shader/recompiler/tests/SpannedUniformityTests.cpp b/core/shader/recompiler/tests/SpannedUniformityTests.cpp index 80ad5ba1e5..7e1915714b 100644 --- a/core/shader/recompiler/tests/SpannedUniformityTests.cpp +++ b/core/shader/recompiler/tests/SpannedUniformityTests.cpp @@ -26,6 +26,9 @@ enum Id : std::uint32_t { True, Main, Helper, + StoragePointer, + Storage, + PhysicalPointer, First = 100u, }; @@ -53,6 +56,9 @@ public: emit(spv::OpTypePointer, {WorkgroupArrayPointer, spv::StorageClassWorkgroup, WorkgroupArrayType}); emit(spv::OpVariable, {WorkgroupArrayPointer, Exchange, spv::StorageClassWorkgroup}); emit(spv::OpTypePointer, {FunctionPointer, spv::StorageClassFunction, Uint}); + emit(spv::OpTypePointer, {StoragePointer, spv::StorageClassStorageBuffer, Uint}); + emit(spv::OpVariable, {StoragePointer, Storage, spv::StorageClassStorageBuffer}); + emit(spv::OpTypePointer, {PhysicalPointer, spv::StorageClassPhysicalStorageBuffer, Uint}); } void emit(spv::Op op, std::initializer_list operands) { @@ -179,6 +185,28 @@ Module early(bool perLane, spv::Op exit) { return module; } +Module unmergedContinue(bool perLane) { + Module module; + module.function(Main); + module.emit(spv::OpLabel, {First}); + module.lane(First + 1u); + module.emit(spv::OpULessThan, {Bool, First + 2u, perLane ? First + 1u : Two, Three}); + module.emit(spv::OpBranch, {First + 10u}); + module.emit(spv::OpLabel, {First + 10u}); + module.emit(spv::OpLoopMerge, {First + 13u, First + 12u, spv::LoopControlMaskNone}); + module.emit(spv::OpBranch, {First + 11u}); + module.emit(spv::OpLabel, {First + 11u}); + module.emit(spv::OpBranchConditional, {First + 2u, First + 12u, First + 14u}); + module.emit(spv::OpLabel, {First + 14u}); + module.barrier(); + module.emit(spv::OpBranch, {First + 12u}); + module.emit(spv::OpLabel, {First + 12u}); + module.emit(spv::OpBranchConditional, {True, First + 10u, First + 13u}); + module.emit(spv::OpLabel, {First + 13u}); + module.end(); + return module; +} + Module joined(const std::function& condition) { Module module; module.function(Main); @@ -214,6 +242,8 @@ int main() { expect("a barrier after a per-lane return", early(true, spv::OpReturn), true); expect("a barrier after a per-lane continue", early(true, spv::OpBranch), true); expect("a barrier after a uniform return", early(false, spv::OpReturn), false); + expect("a barrier after a per-lane continue without a selection merge", unmergedContinue(true), true); + expect("a barrier after a uniform continue without a selection merge", unmergedContinue(false), false); expect("a call reaching a barrier under a constant comparison", call(false), false); expect("a barrier under a phi joining a per-lane selection", joined([](Module& module) { laneSplit(module, First + 4u); @@ -265,6 +295,16 @@ int main() { module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); }), true); + expect("a barrier under a buffer word read at a uniform index", joined([](Module& module) { + module.emit(spv::OpLoad, {Uint, First + 6u, Storage}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); + }), false); + expect("a barrier under a buffer word read at a uniform index after a store through a device address", joined([](Module& module) { + module.emit(spv::OpConvertUToPtr, {PhysicalPointer, First + 10u, Two}); + module.emit(spv::OpStore, {First + 10u, Three}); + module.emit(spv::OpLoad, {Uint, First + 6u, Storage}); + module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two}); + }), true); expect("a barrier under a workgroup word read at a lane index", joined([](Module& module) { module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, First + 1u}); module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u}); From e69002935293338c68afa7c42c6598299b3f3071 Mon Sep 17 00:00:00 2001 From: Kyon Date: Thu, 22 Oct 2026 21:36:31 +0200 Subject: [PATCH 009/318] fix(libSceJson2): honor special-float initialization modes --- core/libs/prx/libSceJson2/CMakeLists.txt | 22 ++- core/libs/prx/libSceJson2/Export.cpp | 108 +++++++++++-- core/libs/tests/GuestJson.cpp | 38 +++-- core/libs/tests/GuestJsonFloatFailure.cpp | 121 ++++++++++++++ core/libs/tests/GuestJsonFloatSettings.cpp | 180 +++++++++++++++++++++ core/libs/tests/JsonTestAllocator.hpp | 30 ++++ docs/dev/debt/libSceJson2.md | 8 +- 7 files changed, 470 insertions(+), 37 deletions(-) create mode 100644 core/libs/tests/GuestJsonFloatFailure.cpp create mode 100644 core/libs/tests/GuestJsonFloatSettings.cpp create mode 100644 core/libs/tests/JsonTestAllocator.hpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index cf4780cf80..fa069b5683 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -10,6 +10,11 @@ if(BUILD_TESTING) add_test(NAME guest_json COMMAND guest_json_tests) add_test(NAME guest_json_numeric_locale COMMAND guest_json_tests --numeric-locale) set_tests_properties(guest_json_numeric_locale PROPERTIES SKIP_RETURN_CODE 77) + add_test_executable(guest_json_float_settings_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonFloatSettings.cpp) + target_include_directories(guest_json_float_settings_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_float_settings_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_float_settings_tests) + add_test(NAME guest_json_float_settings COMMAND guest_json_float_settings_tests) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(numeric_locale_dir ${CMAKE_CURRENT_BINARY_DIR}/numeric-locale) set(numeric_locale_data ${LIBS_INCLUDE_DIR}/tests/numeric-locale) @@ -20,7 +25,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() @@ -39,3 +44,18 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + add_test_executable(guest_json_float_failure_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonFloatFailure.cpp Export.cpp) + target_include_directories(guest_json_float_failure_tests PRIVATE ${LIBS_INCLUDE_DIR}) + if(WIN32) + target_link_libraries(guest_json_float_failure_tests PRIVATE libc) + configure_windows_unwind(guest_json_float_failure_tests) + endif() + add_test(NAME guest_json_float_failure COMMAND guest_json_float_failure_tests) + set_tests_properties(guest_json_float_failure PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_json_float_failure PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index a84b073cd9..cc38650d33 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -31,6 +31,10 @@ enum ValueType : std::int32_t { }; constexpr int JsonErrorParse = static_cast(0x80848101); +constexpr int JsonErrorOutOfMemory = static_cast(0x80848102); +constexpr int JsonErrorNotInitialized = static_cast(0x80848110); +constexpr int JsonErrorAlreadyInitialized = static_cast(0x80848111); +constexpr int JsonErrorInvalidArgument = static_cast(0x80848120); struct Node; struct Value { Node* node; }; @@ -88,6 +92,9 @@ struct Globals { std::mutex mutex; NullAccessCallback callback = nullptr; void* context = nullptr; + std::int32_t floatFormat = 2; + bool floatFormatEnabled = false; + bool initialized = false; }; Globals& State() { @@ -277,6 +284,47 @@ void SetHandlers(Node& n, const TSetter& setter) { for (auto& pair : *n.object.items) SetHandlers(NodeOf(pair.value), setter); } +const char* SpecialFloatText(double value) { + return std::isnan(value) ? "nan" : value < 0 ? "-inf" : "inf"; +} + +struct FloatSettings { + std::int32_t format; + bool enabled; +}; + +FloatSettings FloatConfiguration() { + auto& state = State(); + std::lock_guard lock(state.mutex); + return {state.floatFormat, state.floatFormatEnabled}; +} + +void AssignReal(Node& n, double value) { + if (std::isfinite(value)) { + SetType(n, TypeReal); + n.real = value; + return; + } + const auto settings = FloatConfiguration(); + if (settings.enabled && n.handlers.specialFloat) + NotImplemented_nid_no_patch(__func__); + if (settings.enabled && settings.format != 2) { + if (settings.format == 1) { + SetType(n, TypeNull); + } else { + Node pending{}; + pending.type = TypeString; + Construct(pending.string, SpecialFloatText(value)); + Clear(n); + pending.handlers = n.handlers; + n = pending; + } + return; + } + SetType(n, TypeReal); + n.real = value; +} + void AppendEscaped(std::string& out, const std::string& text) { out += '"'; for (const unsigned char c : text) { @@ -315,7 +363,15 @@ void Serialize(std::string& out, const Node& n) { case TypeUInteger: out += std::to_string(n.uinteger); return; case TypeReal: if (!std::isfinite(n.real) && n.handlers.specialFloat) - throw std::runtime_error("sce::Json::Value: special float handler: the FunctionType the console passes is unverified"); + throw std::runtime_error("sce::Json::Value: special float handler: callback dispatch and ownership are unverified"); + if (!std::isfinite(n.real)) { + const auto settings = FloatConfiguration(); + if (settings.enabled) { + if (settings.format == 0) AppendEscaped(out, SpecialFloatText(n.real)); + else out += settings.format == 1 ? "null" : SpecialFloatText(n.real); + return; + } + } out += RealText(n.real); return; case TypeString: AppendEscaped(out, *n.string.text); return; @@ -627,28 +683,39 @@ void* const JsonMemAllocatorVtable[] = { extern "C" { -int APS5_VABI _ZN3sce4Json11InitializerC1Ev(void* self) { - (void)self; - return 0; +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self); + +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void* self) { + *static_cast(self) = 0; } -int APS5_VABI _ZN3sce4Json11InitializerD1Ev(void* self) { - (void)self; - return 0; +void APS5_VABI _ZN3sce4Json11InitializerD1Ev(void* self) { + _ZN3sce4Json11Initializer9terminateEv(self); } int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* self, const void* parameter) { - (void)self; - if (parameter == nullptr) throw std::invalid_argument("sce::Json::Initializer::initialize: null parameter"); + auto& state = State(); + std::lock_guard lock(state.mutex); + if (state.initialized || *static_cast(self)) return JsonErrorAlreadyInitialized; + if (!parameter) return JsonErrorInvalidArgument; + if (!*static_cast(parameter)) return JsonErrorOutOfMemory; + state.floatFormat = 2; + state.floatFormatEnabled = false; + state.initialized = true; + *static_cast(self) = 1; return 0; } int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { - (void)self; auto& state = State(); std::lock_guard lock(state.mutex); + if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; state.callback = nullptr; state.context = nullptr; + state.floatFormat = 2; + state.floatFormatEnabled = false; + state.initialized = false; + *static_cast(self) = 0; return 0; } @@ -871,7 +938,7 @@ void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value* self) { Construct(*self); } void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value* self, bool value) { Construct(*self); self->node->type = TypeBoolean; self->node->boolean = value; } void APS5_VABI _ZN3sce4Json5ValueC1El(Value* self, std::int64_t value) { Construct(*self); self->node->type = TypeInteger; self->node->integer = value; } void APS5_VABI _ZN3sce4Json5ValueC1Em(Value* self, std::uint64_t value) { Construct(*self); self->node->type = TypeUInteger; self->node->uinteger = value; } -void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value* self, double value) { Construct(*self); self->node->type = TypeReal; self->node->real = value; } +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value* self, double value) { Construct(*self, [value](Node& n) { AssignReal(n, value); }); } void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value: null text"); Construct(*self, [text](Node& node) { @@ -920,7 +987,7 @@ int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value* self, ValueType type int APS5_VABI _ZN3sce4Json5Value3setEb(Value* self, bool value) { SetType(NodeOf(*self), TypeBoolean); self->node->boolean = value; return 0; } int APS5_VABI _ZN3sce4Json5Value3setEl(Value* self, std::int64_t value) { SetType(NodeOf(*self), TypeInteger); self->node->integer = value; return 0; } int APS5_VABI _ZN3sce4Json5Value3setEm(Value* self, std::uint64_t value) { SetType(NodeOf(*self), TypeUInteger); self->node->uinteger = value; return 0; } -int APS5_VABI _ZN3sce4Json5Value3setEd(Value* self, double value) { SetType(NodeOf(*self), TypeReal); self->node->real = value; return 0; } +void APS5_VABI _ZN3sce4Json5Value3setEd(Value* self, double value) { AssignReal(NodeOf(*self), value); } int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value::set: null text"); const std::string copy(text); @@ -1049,16 +1116,16 @@ int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value* self, String* ou *out->text = std::move(text); return 0; } -int APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value* self, String* out) { +void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value* self, String* out) { const Node& n = NodeOf(*self); if (n.type == TypeString) { *out->text = *n.string.text; - return 0; + return; } std::string text; Serialize(text, n); *out->text = std::move(text); - return 0; + return; } @@ -1125,8 +1192,15 @@ void APS5_VABI _ZN3sce4Json14InitParameter225setSpecialFloatFormatTypeENS0_22Spe } int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void* self, const InitParameter2* parameter) { - (void)self; - if (parameter == nullptr) throw std::invalid_argument("sce::Json::Initializer::initialize: null parameter"); + auto& state = State(); + std::lock_guard lock(state.mutex); + if (state.initialized || *static_cast(self)) return JsonErrorAlreadyInitialized; + if (!parameter || !parameter->allocator || static_cast(parameter->specialFloatFormatType) > 2) + return JsonErrorInvalidArgument; + state.floatFormat = parameter->specialFloatFormatType; + state.floatFormatEnabled = true; + state.initialized = true; + *static_cast(self) = 1; return 0; } diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index b2951558b6..a83c3602d9 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -1,4 +1,5 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" #include #include #include @@ -29,8 +30,8 @@ using SpecialFloatHandler = Value* (APS5_VABI*)(Value*, std::int32_t, double, co using ElementAccessFailureHandler = void (APS5_VABI*)(std::int32_t, const String*, const std::size_t*, const Value*, const Value*, void*); extern "C" { -int APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); -int APS5_VABI _ZN3sce4Json11InitializerD1Ev(void*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +void APS5_VABI _ZN3sce4Json11InitializerD1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const void*); int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void*, NullAccessCallback, void*); @@ -39,7 +40,7 @@ void APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEP int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, TypeMismatchHandler, void*); int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, SpecialFloatHandler, void*); int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, ElementAccessFailureHandler, void*); -int APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); @@ -857,7 +858,10 @@ static void NullAccess(SetNullAccessCallback setCallback) { lastContext = nullptr; alignas(16) std::uint8_t initializer[16]{}; alignas(16) std::uint8_t parameter[64]{}; - Require(_ZN3sce4Json11InitializerC1Ev(initializer) == 0); + JsonTest::Allocator allocator; + void* allocatorPointer = &allocator; + std::memcpy(parameter, &allocatorPointer, sizeof(allocatorPointer)); + _ZN3sce4Json11InitializerC1Ev(initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(initializer, parameter) == 0); int context = 0; Require(setCallback(initializer, OnNullAccess, &context) == 0); @@ -882,7 +886,7 @@ static void NullAccess(SetNullAccessCallback setCallback) { Require(callbackCalls == 4); _ZN3sce4Json5ValueD1Ev(&root); _ZN3sce4Json5ValueD1Ev(&fallback); - Require(_ZN3sce4Json11InitializerD1Ev(initializer) == 0); + _ZN3sce4Json11InitializerD1Ev(initializer); } static void ValueAccess() { @@ -933,7 +937,7 @@ static void ValueAccess() { alignas(16) std::uint8_t parameter[40]; std::memset(parameter, 0xff, sizeof(parameter)); _ZN3sce4Json14InitParameter2C1Ev(parameter); - int allocator = 0; + JsonTest::Allocator allocator; int userData = 0; _ZN3sce4Json14InitParameter212setAllocatorEPNS0_12MemAllocatorEPv(parameter, &allocator, &userData); _ZN3sce4Json14InitParameter217setFileBufferSizeEm(parameter, 4096); @@ -941,10 +945,10 @@ static void ValueAccess() { std::memcpy(stored, parameter, sizeof(stored)); Require(stored[0] == &allocator && stored[1] == &userData && reinterpret_cast(stored[2]) == 4096); alignas(16) std::uint8_t initializer[16]{}; - Require(_ZN3sce4Json11InitializerC1Ev(initializer) == 0); + _ZN3sce4Json11InitializerC1Ev(initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(initializer, parameter) == 0); Require(_ZN3sce4Json11Initializer9terminateEv(initializer) == 0); - Require(_ZN3sce4Json11InitializerD1Ev(initializer) == 0); + _ZN3sce4Json11InitializerD1Ev(initializer); } static void InitParameter2BaseConstructor() { @@ -963,7 +967,7 @@ static void InitParameter2BaseConstructor() { std::memcpy(&defaultUserData, baseParameter + sizeof(void*), sizeof(defaultUserData)); std::memcpy(&defaultFileBufferSize, baseParameter + 2 * sizeof(void*), sizeof(defaultFileBufferSize)); Require(defaultAllocator == nullptr && defaultUserData == nullptr && defaultFileBufferSize == 0); - int allocator = 0; + JsonTest::Allocator allocator; int userData = 0; _ZN3sce4Json14InitParameter212setAllocatorEPNS0_12MemAllocatorEPv(baseParameter, &allocator, &userData); _ZN3sce4Json14InitParameter217setFileBufferSizeEm(baseParameter, 4096); @@ -978,10 +982,10 @@ static void InitParameter2BaseConstructor() { for (std::size_t i = 16 + sizeof(parameter); i < sizeof(baseStorage); ++i) Require(baseStorage[i] == 0xff); alignas(16) std::uint8_t initializer[16]{}; - Require(_ZN3sce4Json11InitializerC1Ev(initializer) == 0); + _ZN3sce4Json11InitializerC1Ev(initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(initializer, baseParameter) == 0); Require(_ZN3sce4Json11Initializer9terminateEv(initializer) == 0); - Require(_ZN3sce4Json11InitializerD1Ev(initializer) == 0); + _ZN3sce4Json11InitializerD1Ev(initializer); } struct StoredInitParameter2 { @@ -1030,10 +1034,10 @@ static void InitParameters() { Require(Stored(parameter).specialFloatFormatType == 0 && Stored(parameter).fileBufferSize == 8192); alignas(16) std::uint8_t initializer[16]{}; - Require(_ZN3sce4Json11InitializerC1Ev(initializer) == 0); + _ZN3sce4Json11InitializerC1Ev(initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(initializer, parameter) == 0); Require(_ZN3sce4Json11Initializer9terminateEv(initializer) == 0); - Require(_ZN3sce4Json11InitializerD1Ev(initializer) == 0); + _ZN3sce4Json11InitializerD1Ev(initializer); } alignas(16) std::uint8_t rttiParameter[40]; @@ -1246,7 +1250,7 @@ static void ValueTypeMismatchHandler() { RequireMismatch(6, TypeInteger, &root, &context); Require(*_ZNK3sce4Json5Value10getIntegerEv(&other) == 0); Require(mismatchCalls == 6); - Require(_ZN3sce4Json5Value3setEd(&other, 1.5) == 0); + _ZN3sce4Json5Value3setEd(&other, 1.5); Require(_ZN3sce4Json5Value3setERKS1_(&root, &other) == 0); Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 42); RequireMismatch(7, TypeInteger, &root, &context); @@ -1327,10 +1331,10 @@ static void ValueSpecialFloatHandler() { String out{}; _ZN3sce4Json6StringC1Ev(&out); Require(ThrowsUnverified([&] { _ZN3sce4Json5Value9serializeERNS0_6StringE(&real, &out); })); - Require(_ZN3sce4Json5Value3setEd(&real, std::numeric_limits::infinity()) == 0); + _ZN3sce4Json5Value3setEd(&real, std::numeric_limits::infinity()); Require(ThrowsUnverified([&] { _ZN3sce4Json5Value9serializeERNS0_6StringE(&real, &out); })); _ZN3sce4Json6StringD1Ev(&out); - Require(_ZN3sce4Json5Value3setEd(&real, 2.5) == 0); + _ZN3sce4Json5Value3setEd(&real, 2.5); Require(Serialize(real) == "2.5"); Require(specialFloatCalls == 0); _ZN3sce4Json5ValueD1Ev(&real); @@ -1368,7 +1372,7 @@ static void ValueOperatorBool() { Require(_ZNK3sce4Json5ValuecvbEv(&scalar) == (value != 0)); } for (double value : {0.0, -0.0, 1.0, -1.0, std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { - Require(_ZN3sce4Json5Value3setEd(&scalar, value) == 0); + _ZN3sce4Json5Value3setEd(&scalar, value); Require(_ZNK3sce4Json5ValuecvbEv(&scalar) == (value != 0.0)); } for (const char* text : {"", "0", "false", " "}) { diff --git a/core/libs/tests/GuestJsonFloatFailure.cpp b/core/libs/tests/GuestJsonFloatFailure.cpp new file mode 100644 index 0000000000..8b805ae8bc --- /dev/null +++ b/core/libs/tests/GuestJsonFloatFailure.cpp @@ -0,0 +1,121 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { alignas(8) std::array bytes{}; }; +struct Parameter { + void* allocator; + void* userData; + std::size_t bufferSize; + std::int32_t mode; + std::uint32_t reserved[3]; +}; + +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +void APS5_VABI _ZN3sce4Json11InitializerD1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); +const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value*); +} + +#ifndef _WIN32 +extern "C" [[noreturn]] void NotImplemented_nid_no_patch(const char* name) { + std::fprintf(stderr, "Unexpected unsupported Json2 path: %s\n", name); + std::abort(); +} +#endif + +static int allocationBudget = -1; +static std::size_t liveAllocations = 0; + +void* operator new(std::size_t bytes) { + if (allocationBudget == 0) throw std::bad_alloc(); + if (allocationBudget > 0) --allocationBudget; + if (void* pointer = std::malloc(bytes ? bytes : 1)) { + ++liveAllocations; + return pointer; + } + throw std::bad_alloc(); +} + +void operator delete(void* pointer) noexcept { + if (pointer) { + --liveAllocations; + std::free(pointer); + } +} + +void operator delete(void* pointer, std::size_t) noexcept { ::operator delete(pointer); } +void* operator new[](std::size_t bytes) { return ::operator new(bytes); } +void operator delete[](void* pointer) noexcept { ::operator delete(pointer); } +void operator delete[](void* pointer, std::size_t) noexcept { ::operator delete(pointer); } + +static void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "%s\n", message); + std::abort(); + } +} + +int main() { + JsonTest::Allocator allocator; + Parameter parameter{&allocator, nullptr, 0, 0, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, + "Initialize string-format conversion"); + const auto baseline = liveAllocations; + Value value; + _ZN3sce4Json5ValueC1Ed(&value, 0.125); + const auto before = liveAllocations; + allocationBudget = 0; + bool failed = false; + try { + _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + } catch (const std::bad_alloc&) { + failed = true; + } + allocationBudget = -1; + Require(failed, "String conversion must propagate host allocation failure"); + Require(liveAllocations == before && _ZNK3sce4Json5Value7getTypeEv(&value) == 4 && + *_ZNK3sce4Json5Value7getRealEv(&value) == 0.125, + "Failed conversion must retain the original value and ownership"); + allocationBudget = 0; + _ZN3sce4Json5Value3setEd(&value, 0.5); + allocationBudget = -1; + Require(*_ZNK3sce4Json5Value7getRealEv(&value) == 0.5, + "Finite assignment must not require a conversion allocation"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(liveAllocations == baseline, "Destroy must reclaim the retained value"); + for (int budget : {0, 1}) { + Value destination; + destination.bytes.fill(std::byte{0xa5}); + const auto original = destination.bytes; + allocationBudget = budget; + failed = false; + try { + _ZN3sce4Json5ValueC1Ed(&destination, std::numeric_limits::infinity()); + } catch (const std::bad_alloc&) { + failed = true; + } + allocationBudget = -1; + Require(failed && destination.bytes == original && liveAllocations == baseline, + "Failed construction must leave destination storage unchanged and reclaim partial state"); + } + _ZN3sce4Json5ValueC1Ed(&value, std::numeric_limits::infinity()); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == 5, "Conversion must remain usable after allocation failure"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(liveAllocations == baseline, "Successful conversion must not leak"); + _ZN3sce4Json11InitializerD1Ev(&initializer); +} diff --git a/core/libs/tests/GuestJsonFloatSettings.cpp b/core/libs/tests/GuestJsonFloatSettings.cpp new file mode 100644 index 0000000000..25dfd8dee3 --- /dev/null +++ b/core/libs/tests/GuestJsonFloatSettings.cpp @@ -0,0 +1,180 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { alignas(8) std::array bytes{}; }; +struct String { void* text = nullptr; }; +struct Array { void* items = nullptr; }; +struct Parameter { + void* allocator; + void* userData; + std::size_t bufferSize; + std::int32_t mode; + std::uint32_t reserved[3]; +}; +static_assert(sizeof(Parameter) == 40); + +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +void APS5_VABI _ZN3sce4Json11InitializerD1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); +const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value*); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value*, String*); +void APS5_VABI _ZN3sce4Json6StringC1Ev(String*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); +} + +static void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "%s\n", message); + std::abort(); + } +} + +static std::string Serialize(Value& value, bool toString = false) { + String output; + _ZN3sce4Json6StringC1Ev(&output); + if (toString) _ZNK3sce4Json5Value8toStringERNS0_6StringE(&value, &output); + else Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output) == 0, + "Float serialization must succeed"); + const std::string text = _ZNK3sce4Json6String5c_strEv(&output); + _ZN3sce4Json6StringD1Ev(&output); + return text; +} + +static void CheckMode(int mode) { + JsonTest::Allocator allocator; + int userData = 0; + Parameter parameter{&allocator, &userData, 16384, mode, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, + "Initialize a valid float configuration"); + parameter.mode = (mode + 1) % 3; + const std::array numbers{std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN(), + std::bit_cast(std::uint64_t{0x7ff80000cafef00d}), + std::bit_cast(std::uint64_t{0xfff80000a5a55a5a})}; + const std::array tokens{"inf", "-inf", "nan", "nan", "nan"}; + for (unsigned i = 0; i < numbers.size(); ++i) { + Value value; + _ZN3sce4Json5ValueC1Ed(&value, numbers[i]); + const int type = mode == 0 ? 5 : mode == 1 ? 0 : 4; + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == type, + "Non-finite construction must honor the initialization mode"); + const std::string expected = mode == 0 ? '"' + std::string(tokens[i]) + '"' + : mode == 1 ? "null" : tokens[i]; + Require(Serialize(value) == expected, "Non-finite serialization must honor the copied mode"); + Require(Serialize(value, true) == (mode == 0 ? tokens[i] : expected), + "toString must return unquoted string values and serialize other types"); + if (mode == 2) + Require(std::bit_cast(*_ZNK3sce4Json5Value7getRealEv(&value)) == + std::bit_cast(numbers[i]), + "Mode 2 construction and formatting must preserve the real payload"); + _ZN3sce4Json5Value3setEd(&value, 0.125); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == 4 && + *_ZNK3sce4Json5Value7getRealEv(&value) == 0.125 && Serialize(value) == "0.125", + "Finite assignment and formatting must remain unchanged"); + _ZN3sce4Json5Value3setEd(&value, numbers[i]); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == type && Serialize(value) == expected, + "Non-finite assignment must use the same conversion as construction"); + if (mode == 2) + Require(std::bit_cast(*_ZNK3sce4Json5Value7getRealEv(&value)) == + std::bit_cast(numbers[i]), + "Mode 2 assignment must preserve the real payload"); + Array array; + _ZN3sce4Json5ArrayC1Ev(&array); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); + Value nested; + _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&nested, &array); + Require(Serialize(nested) == '[' + expected + ']', "Nested values must retain their converted type"); + _ZN3sce4Json5ValueD1Ev(&nested); + _ZN3sce4Json5ArrayD1Ev(&array); + _ZN3sce4Json5ValueD1Ev(&value); + } + Value zero; + _ZN3sce4Json5ValueC1Ed(&zero, -0.0); + Require(std::signbit(*_ZNK3sce4Json5Value7getRealEv(&zero)), "All modes must preserve finite signed zero"); + _ZN3sce4Json5ValueD1Ev(&zero); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0, + "Termination must reset initializer state"); + _ZN3sce4Json11InitializerD1Ev(&initializer); +} + +static void CheckLifecycle() { + JsonTest::Allocator allocator; + Parameter parameter{&allocator, nullptr, 0, 0, {}}; + std::array storage; + storage.fill(0xa5); + _ZN3sce4Json11InitializerC1Ev(storage.data()); + Require(storage[0] == 0, "The initializer constructor must clear its active byte"); + for (unsigned i = 1; i < storage.size(); ++i) + Require(storage[i] == 0xa5, "The initializer constructor must preserve trailing storage"); + Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(storage.data())) == 0x80848110, + "Termination before initialization must report not initialized"); + Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(storage.data(), nullptr)) == 0x80848120, + "Null InitParameter2 must report invalid argument"); + Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(storage.data(), nullptr)) == 0x80848120, + "Null legacy parameters must report invalid argument"); + Parameter noAllocator{}; + Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(storage.data(), &noAllocator)) == 0x80848120, + "InitParameter2 must reject a null allocator"); + Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(storage.data(), &noAllocator)) == 0x80848102, + "Legacy initialization must report its observed null-allocator error"); + for (int mode : {-1, 3, std::numeric_limits::max()}) { + parameter.mode = mode; + Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(storage.data(), ¶meter)) == 0x80848120, + "Invalid float modes must be rejected"); + Require(storage[0] == 0, "Rejected initialization must preserve the inactive byte"); + } + parameter.mode = 0; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(storage.data(), ¶meter) == 0, + "A valid initialization must work after invalid modes"); + std::uint8_t other = 0; + Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, reinterpret_cast(std::uintptr_t{1}))) == 0x80848111, + "Already initialized must precede parameter access"); + Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(&other)) == 0x80848110, + "An inactive initializer must not terminate active configuration"); + Value value; + _ZN3sce4Json5ValueC1Ed(&value, std::numeric_limits::infinity()); + Require(Serialize(value) == "\"inf\"", "Rejected lifecycle calls must preserve the active mode"); + _ZN3sce4Json5ValueD1Ev(&value); + _ZN3sce4Json11InitializerD1Ev(storage.data()); + Require(storage[0] == 0, "The destructor must terminate an active initializer"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&other, ¶meter) == 0, + "Legacy initialization must work after destructor cleanup"); + _ZN3sce4Json5ValueC1Ed(&value, std::numeric_limits::infinity()); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == 4 && + std::isinf(*_ZNK3sce4Json5Value7getRealEv(&value)), + "Legacy initialization must not apply InitParameter2 conversion rules"); + _ZN3sce4Json5ValueD1Ev(&value); + _ZN3sce4Json11InitializerD1Ev(&other); +} + +int main() { + for (int mode = 0; mode < 3; ++mode) CheckMode(mode); + CheckLifecycle(); + return 0; +} diff --git a/core/libs/tests/JsonTestAllocator.hpp b/core/libs/tests/JsonTestAllocator.hpp new file mode 100644 index 0000000000..ba8a562536 --- /dev/null +++ b/core/libs/tests/JsonTestAllocator.hpp @@ -0,0 +1,30 @@ +#pragma once +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include + +namespace JsonTest { + +inline void* APS5_VABI Allocate(void*, std::size_t bytes, void*) { + return std::malloc(bytes); +} + +inline void APS5_VABI Free(void*, void* pointer, void*) { + std::free(pointer); +} + +inline void APS5_VABI Notify(void*, int error, std::size_t bytes, void*) { + std::fprintf(stderr, "Unexpected Json2 allocation error 0x%x for %zu bytes\n", error, bytes); + std::abort(); +} + +inline void* const AllocatorVtable[] = { + nullptr, nullptr, reinterpret_cast(Allocate), reinterpret_cast(Free), reinterpret_cast(Notify) +}; + +struct Allocator { + void* const* vtable = AllocatorVtable; +}; + +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index b9fac63328..5a15ff042e 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -5,9 +5,13 @@ - [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser -- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - RTTI allocator adaptation and the `InitParameterRtti` layout remain unverified. `setAllocatorRtti` stores the raw allocation-parameter pointer as the allocator; `initialize` ignores allocator, buffer-size and special-float settings. Special-float enum meanings and parser support for non-finite numbers remain unresolved; serialization of non-finite values throws. Setter return types remain unverified. +- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still ignores allocator storage and file-buffer size; allocation admission, allocator notifications and rollback are not implemented. RTTI allocator adaptation and the `InitParameterRtti` layout remain unverified; `setAllocatorRtti` still stores a raw allocation-parameter pointer. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. +- [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the signatures follow the mangled names, with `Array::iterator::operator++(int)` and `Object::iterator::operator++(int)` returning their copy through a result pointer before `this` as `begin` does, since the iterators have non-trivial destructors. `Pair(const String&, const Value&)` and default `Pair()` deep-copy or initialize into the layout `Object` iteration already hands out (key at 0, value at 0x10) and zeroes the 8 bytes between them; what the console keeps there is unknown. -- [sce::Json::Value::setTypeMismatchHandler, setElementAccessFailureHandler and setSpecialFloatHandler](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - members of `Value` that return an int, inferred from a caller that registers them on its root `Value` before parsing into it, with the handler and its context, and accepts 0 or 0x80848104 as success; they return 0. The PS Vita library's `Value::setNullAccessCallback` is such a member too ([vitasdk](https://github.com/vitasdk/vita-headers/blob/15605617cbbb23d266f88af28899fdb573e2355e/include/psp2/json.h#L638)). The handler signatures follow the mangled names. A handler applies to the `Value` it is set on and to the values that `Value` holds then, and `Parser::parse` gives the values it creates the handlers of the `Value` it parses into, since the title reads the errors its handlers record after accessing parsed members; copies, `push_back` and `Object::operator[]` do not carry them, and how the console finds the handler of an accessed value is unknown. A typed getter on a non-null value of another type calls the type mismatch handler with the requested type, the value, null and the context, and returns the value the handler returns if it has the requested type, else the getter's default; passing null as the third handler argument (the second `Value` pointer), and the order of the two `Value` pointers, remain unverified against the console. The element access failure handler is not called: a failed `operator[]` or `getValue` on a value carrying it throws, as the error code the console passes, which the title records as its error, is unknown; without one the access still reaches the null-access callback. The special float handler is not called: serializing a non-finite real throws with or without it, as the `FunctionType` values and the `bool*` output are unknown, and the title's handler returns a different replacement for each of its two types +- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - setter return types are inferred from a title that registers handlers on its root value and accepts 0 or `0x80848104`; the setters return 0. Signatures follow mangled names. The Vita null-access callback is also a member ([vitasdk](https://github.com/vitasdk/vita-headers/blob/15605617cbbb23d266f88af28899fdb573e2355e/include/psp2/json.h#L638)). Applying a handler to current descendants and parser-created values follows that caller; copies, `push_back` and object insertion do not carry handlers. Console ancestor lookup and propagation remain unverified. +- [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - a typed getter on a non-null value of another type calls the registered handler with the requested type, value, null and context. It uses the returned value if its type matches, otherwise the default. The order of the two value pointers and the null second pointer remain unverified against the console. +- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - the registered handler is not called: failed indexing or `getValue` throws because its error code is unknown. Without a handler, access reaches the null-access callback. +- [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - a non-finite real carrying a handler throws during serialization. Assignment under InitParameter2 with a registered handler throws through `NotImplemented_nid_no_patch`, before changing the value. Callback result ownership, ancestor lookup, error propagation and reentrancy remain unimplemented or unverified; the title returns different replacements for its two function types. - [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing. A title's allocator that does not override it calls it through its own vtable; the library never calls the title's allocator, so it never reports an allocation failure. Whether the console's base implementation logs anything is unknown - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - storage uses `std::string`; capacity and allocation growth do not match the system 32-bit capacity and allocator settings. Appending null text with a nonzero length throws instead of dereferencing it. - [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the return type of `advance` remains unverified. Object iterator advancement past the end throws instead of asserting. Dereferencing an empty array iterator throws instead of returning the system null-value pointer or an invalid address. From 6ef99b0edd597af7c1d9e0183c4e05a7dfb1cdf9 Mon Sep 17 00:00:00 2001 From: Kyon Date: Thu, 22 Oct 2026 22:15:21 +0200 Subject: [PATCH 010/318] fix(libSceJson2): adapt RTTI allocator callbacks --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 63 ++++++++- core/libs/tests/GuestJson.cpp | 20 +-- .../libs/tests/GuestJsonAllocatorSettings.cpp | 131 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 3 +- 5 files changed, 203 insertions(+), 21 deletions(-) create mode 100644 core/libs/tests/GuestJsonAllocatorSettings.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index fa069b5683..813027c4c0 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -15,6 +15,11 @@ if(BUILD_TESTING) target_link_libraries(guest_json_float_settings_tests PRIVATE libSceJson2 libc) configure_windows_unwind(guest_json_float_settings_tests) add_test(NAME guest_json_float_settings COMMAND guest_json_float_settings_tests) + add_test_executable(guest_json_allocator_settings_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonAllocatorSettings.cpp) + target_include_directories(guest_json_allocator_settings_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_allocator_settings_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_allocator_settings_tests) + add_test(NAME guest_json_allocator_settings COMMAND guest_json_allocator_settings_tests) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(numeric_locale_dir ${CMAKE_CURRENT_BINARY_DIR}/numeric-locale) set(numeric_locale_data ${LIBS_INCLUDE_DIR}/tests/numeric-locale) @@ -25,7 +30,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index cc38650d33..b5636e821f 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -87,6 +87,15 @@ struct Node { }; using NullAccessCallback = const Value& (APS5_VABI*)(ValueType, const Value*, void*); +using RttiAllocate = void* (APS5_VABI*)(std::size_t, void*); +using RttiFree = void (APS5_VABI*)(void*, void*); +using RttiNotify = void (APS5_VABI*)(int, std::size_t, void*); + +struct AllocParamRtti { + RttiAllocate allocate; + RttiFree free; + RttiNotify notify; +}; struct Globals { std::mutex mutex; @@ -95,6 +104,7 @@ struct Globals { std::int32_t floatFormat = 2; bool floatFormatEnabled = false; bool initialized = false; + AllocParamRtti rtti{}; }; Globals& State() { @@ -102,6 +112,41 @@ Globals& State() { return globals; } +AllocParamRtti RttiCallbacks() { + auto& state = State(); + std::lock_guard lock(state.mutex); + return state.rtti; +} + +void APS5_VABI RttiDestroy(void*) {} + +void APS5_VABI RttiDelete(void* self) { ::operator delete(self); } + +void* APS5_VABI RttiAllocateMemory(void*, std::size_t size, void* context) { + return RttiCallbacks().allocate(size, context); +} + +void APS5_VABI RttiFreeMemory(void*, void* memory, void* context) { + RttiCallbacks().free(memory, context); +} + +void APS5_VABI RttiNotifyError(void*, int error, std::size_t size, void* context) { + const auto callback = RttiCallbacks().notify; + if (callback) callback(error, size, context); + else if (error == JsonErrorOutOfMemory) + std::printf("[libSceJson] Allocation Fail. size = %zu\n", size); + else std::printf("[libSceJson] unknown notification[%#x]\n", static_cast(error)); +} + +void* const RttiAllocatorVtable[] = { + reinterpret_cast(&RttiDestroy), reinterpret_cast(&RttiDelete), + reinterpret_cast(&RttiAllocateMemory), reinterpret_cast(&RttiFreeMemory), + reinterpret_cast(&RttiNotifyError), +}; + +struct RttiAllocator { void* const* vtable = RttiAllocatorVtable; }; +RttiAllocator rttiAllocator; + void Construct(String& s, const std::string& text) { s.text = new std::string(text); } void Construct(Array& a) { a.items = new std::list(); } void Construct(Object& o) { o.items = new std::list(); } @@ -1150,7 +1195,14 @@ struct InitParameterRtti { }; void APS5_VABI _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(InitParameterRtti* self, void* allocParam, void* userData, std::size_t fileBufferSize) { - self->allocParam = allocParam; + *self = {}; + auto& state = State(); + std::lock_guard lock(state.mutex); + if (state.initialized) return; + const auto& callbacks = *static_cast(allocParam); + if (!callbacks.allocate || !callbacks.free) return; + state.rtti = callbacks; + self->allocParam = &rttiAllocator; self->userData = userData; self->fileBufferSize = fileBufferSize; } @@ -1179,7 +1231,14 @@ void APS5_VABI _ZN3sce4Json14InitParameter212setAllocatorEPNS0_12MemAllocatorEPv } void APS5_VABI _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(InitParameter2* self, void* allocParam, void* userData) { - self->allocator = allocParam; + if (!allocParam) return; + auto& state = State(); + std::lock_guard lock(state.mutex); + if (state.initialized) return; + const auto& callbacks = *static_cast(allocParam); + if (!callbacks.allocate || !callbacks.free) return; + state.rtti = callbacks; + self->allocator = &rttiAllocator; self->userData = userData; } diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index a83c3602d9..17a3a96d1d 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -1003,12 +1003,6 @@ static StoredInitParameter2 Stored(const std::uint8_t* parameter) { return stored; } -static bool Untouched(const std::uint8_t* parameter, std::size_t from) { - for (std::size_t i = from; i < 40; ++i) - if (parameter[i] != 0xff) return false; - return true; -} - static void InitParameters() { using Constructor = void (APS5_VABI*)(void*); for (Constructor construct : {_ZN3sce4Json14InitParameter2C1Ev, _ZN3sce4Json14InitParameter2C2Ev}) { @@ -1020,11 +1014,11 @@ static void InitParameters() { Require(stored.specialFloatFormatType == 2); Require(stored.reserved[0] == 0 && stored.reserved[1] == 0 && stored.reserved[2] == 0); - int allocParam = 0; + JsonTest::Allocator allocParam; int userData = 0; _ZN3sce4Json14InitParameter217setFileBufferSizeEm(parameter, 8192); _ZN3sce4Json14InitParameter225setSpecialFloatFormatTypeENS0_22SpecialFloatFormatTypeE(parameter, 2); - _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(parameter, &allocParam, &userData); + _ZN3sce4Json14InitParameter212setAllocatorEPNS0_12MemAllocatorEPv(parameter, &allocParam, &userData); stored = Stored(parameter); Require(stored.allocator == &allocParam && stored.userData == &userData && stored.fileBufferSize == 8192); Require(stored.specialFloatFormatType == 2); @@ -1040,15 +1034,7 @@ static void InitParameters() { _ZN3sce4Json11InitializerD1Ev(initializer); } - alignas(16) std::uint8_t rttiParameter[40]; - std::memset(rttiParameter, 0xff, sizeof(rttiParameter)); - int allocParam = 0; - int userData = 0; - _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(rttiParameter, &allocParam, &userData, 4096); - void* rttiStored[3]{}; - std::memcpy(rttiStored, rttiParameter, sizeof(rttiStored)); - Require(rttiStored[0] == &allocParam && rttiStored[1] == &userData && reinterpret_cast(rttiStored[2]) == 4096); - Require(Untouched(rttiParameter, sizeof(rttiStored))); + } static void ValueClear() { diff --git a/core/libs/tests/GuestJsonAllocatorSettings.cpp b/core/libs/tests/GuestJsonAllocatorSettings.cpp new file mode 100644 index 0000000000..cc2d128218 --- /dev/null +++ b/core/libs/tests/GuestJsonAllocatorSettings.cpp @@ -0,0 +1,131 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include +#include + +using Allocate = void* (APS5_VABI*)(std::size_t, void*); +using Free = void (APS5_VABI*)(void*, void*); +using Notify = void (APS5_VABI*)(int, std::size_t, void*); +struct Callbacks { Allocate allocate; Free free; Notify notify; }; +struct Parameter { + void* allocator; + void* context; + std::size_t bufferSize; + std::int32_t mode; + std::uint32_t reserved[3]; +}; +static_assert(sizeof(Parameter) == 40); +extern "C" { +void APS5_VABI _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(void*, void*, void*); +void APS5_VABI _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(void*, void*, void*, std::size_t); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct Context { std::size_t size = 0; int error = 0; void* memory = nullptr; int calls = 0; }; +static void* APS5_VABI AllocateMemory(std::size_t size, void* context) { + auto& trace = *static_cast(context); + trace.size = size; + ++trace.calls; + trace.memory = std::malloc(size); + return trace.memory; +} +static void APS5_VABI FreeMemory(void* memory, void* context) { + auto& trace = *static_cast(context); + Require(memory == trace.memory, "The adapter must forward the allocation to the matching context"); + ++trace.calls; + std::free(memory); + trace.memory = nullptr; +} +static void APS5_VABI NotifyError(int error, std::size_t size, void* context) { + auto& trace = *static_cast(context); + trace.error = error; + trace.size = size; + ++trace.calls; +} +static void* APS5_VABI AllocateReplacement(std::size_t size, void* context) { + return AllocateMemory(size + 1, context); +} +static void Exercise(Parameter& parameter, Context& trace, bool replacement) { + auto** table = *static_cast(parameter.allocator); + const auto allocate = reinterpret_cast(table[2]); + const auto free = reinterpret_cast(table[3]); + const auto notify = reinterpret_cast(table[4]); + const int before = trace.calls; + void* memory = allocate(parameter.allocator, 127, parameter.context); + Require(memory && trace.size == (replacement ? 128 : 127), "The RTTI adapter must dispatch allocation with the supplied size and context"); + free(parameter.allocator, memory, parameter.context); + notify(parameter.allocator, static_cast(0x80848102), 8193, parameter.context); + Require(trace.calls == before + 3 && trace.error == static_cast(0x80848102) && trace.size == 8193, + "The RTTI adapter must forward free and notification exactly once"); +} +int main() { + Context first, second; + Callbacks callbacks{AllocateMemory, FreeMemory, NotifyError}; + Parameter parameter{nullptr, nullptr, 8192, 1, {11, 12, 13}}; + _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(¶meter, &callbacks, &first); + Require(parameter.allocator && parameter.allocator != &callbacks && parameter.context == &first && + parameter.bufferSize == 8192 && parameter.mode == 1 && parameter.reserved[2] == 13, + "The setter must install an adapter and preserve unrelated settings"); + callbacks = {}; + Exercise(parameter, first, false); + const Parameter saved = parameter; + for (Callbacks invalid : {Callbacks{}, Callbacks{AllocateMemory, nullptr, NotifyError}, + Callbacks{nullptr, FreeMemory, NotifyError}}) { + _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(¶meter, &invalid, &second); + Require(std::memcmp(¶meter, &saved, sizeof(saved)) == 0, "Invalid callbacks must preserve the parameter block"); + alignas(8) std::array legacy; + legacy.fill(std::byte{0xa5}); + _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(legacy.data(), &invalid, &second, 4096); + for (std::size_t i = 0; i < legacy.size(); ++i) + Require(legacy[i] == (i < 24 ? std::byte{} : std::byte{0xa5}), "Rejected legacy construction must clear only its 24-byte base"); + } + _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(¶meter, nullptr, &second); + Require(std::memcmp(¶meter, &saved, sizeof(saved)) == 0, "A null RTTI setter argument must preserve settings"); + Callbacks replacement{AllocateReplacement, FreeMemory, NotifyError}; + Parameter newer{}; + _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(&newer, &replacement, &second); + Require(newer.allocator == parameter.allocator, "RTTI parameter blocks must share the module adapter"); + Exercise(parameter, first, true); + Exercise(newer, second, true); + JsonTest::Allocator allocator; + Parameter regular{&allocator, nullptr, 0, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ®ular) == 0, "Initialize the module"); + callbacks = {AllocateMemory, FreeMemory, NotifyError}; + _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(¶meter, &callbacks, &second); + Require(std::memcmp(¶meter, &saved, sizeof(saved)) == 0, "An active module must refuse RTTI reconfiguration"); + alignas(8) std::array refused; + refused.fill(std::byte{0xa5}); + _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(refused.data(), reinterpret_cast(std::uintptr_t{1}), &second, 4096); + for (std::size_t i = 0; i < refused.size(); ++i) + Require(refused[i] == (i < 24 ? std::byte{} : std::byte{0xa5}), "An active module must reject legacy configuration before reading callbacks"); + Exercise(parameter, first, true); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate the module"); + alignas(8) std::array legacy; + legacy.fill(std::byte{0xa5}); + _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(legacy.data(), &callbacks, &first, 4096); + Parameter stored{}; + std::memcpy(&stored, legacy.data(), 24); + Require(stored.allocator == parameter.allocator && stored.context == &first && stored.bufferSize == 4096, + "Legacy construction must install the adapter, context and buffer size"); + for (std::size_t i = 24; i < legacy.size(); ++i) + Require(legacy[i] == std::byte{0xa5}, "Legacy construction must preserve trailing memory"); + Exercise(stored, first, false); + callbacks.notify = nullptr; + _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(¶meter, &callbacks, &first); + auto** table = *static_cast(parameter.allocator); + const auto notify = reinterpret_cast(table[4]); + const int before = first.calls; + notify(parameter.allocator, static_cast(0x80848102), 512, parameter.context); + Require(first.calls == before, "An optional null notifier must not dispatch a callback"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 5a15ff042e..a7a4ea071a 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -5,7 +5,8 @@ - [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser -- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still ignores allocator storage and file-buffer size; allocation admission, allocator notifications and rollback are not implemented. RTTI allocator adaptation and the `InitParameterRtti` layout remain unverified; `setAllocatorRtti` still stores a raw allocation-parameter pointer. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. +- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still ignores allocator storage and file-buffer size; allocation admission, allocator notifications and rollback are not implemented. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. +- [sce::Json::Value storage and recovery callback ABI](../../../core/libs/prx/libSceJson2/Export.cpp) - values use an 8-byte host-node wrapper rather than the inspected 32-byte inline layout. Special-float recovery at `0x2b90` receives a 32-byte return object through a hidden result pointer; correcting callback dispatch requires compatible payload, parent and auxiliary ownership. Current host allocation counts and request sizes do not follow the inspected library. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the signatures follow the mangled names, with `Array::iterator::operator++(int)` and `Object::iterator::operator++(int)` returning their copy through a result pointer before `this` as `begin` does, since the iterators have non-trivial destructors. `Pair(const String&, const Value&)` and default `Pair()` deep-copy or initialize into the layout `Object` iteration already hands out (key at 0, value at 0x10) and zeroes the 8 bytes between them; what the console keeps there is unknown. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - setter return types are inferred from a title that registers handlers on its root value and accepts 0 or `0x80848104`; the setters return 0. Signatures follow mangled names. The Vita null-access callback is also a member ([vitasdk](https://github.com/vitasdk/vita-headers/blob/15605617cbbb23d266f88af28899fdb573e2355e/include/psp2/json.h#L638)). Applying a handler to current descendants and parser-created values follows that caller; copies, `push_back` and object insertion do not carry handlers. Console ancestor lookup and propagation remain unverified. From 874ec1c50fa14b0edd96cc4a3982a9816d0fc77d Mon Sep 17 00:00:00 2001 From: Kyon Date: Thu, 22 Oct 2026 22:45:59 +0200 Subject: [PATCH 011/318] fix(libSceJson2): use inline values for recovery callbacks --- core/libs/prx/libSceJson2/CMakeLists.txt | 27 +- core/libs/prx/libSceJson2/Export.cpp | 661 +++++++++++++------ core/libs/tests/GuestJson.cpp | 122 ++-- core/libs/tests/GuestJsonElementRecovery.cpp | 129 ++++ core/libs/tests/GuestJsonFloatFailure.cpp | 75 +++ core/libs/tests/GuestJsonNullRecovery.cpp | 128 ++++ core/libs/tests/GuestJsonSpecialFloat.cpp | 164 +++++ core/libs/tests/GuestJsonValueLayout.cpp | 94 +++ core/libs/tests/GuestJsonValueOwnership.cpp | 110 +++ core/libs/tests/JsonCopyFailure.cpp | 4 +- docs/dev/debt/libSceJson2.md | 12 +- 11 files changed, 1237 insertions(+), 289 deletions(-) create mode 100644 core/libs/tests/GuestJsonElementRecovery.cpp create mode 100644 core/libs/tests/GuestJsonNullRecovery.cpp create mode 100644 core/libs/tests/GuestJsonSpecialFloat.cpp create mode 100644 core/libs/tests/GuestJsonValueLayout.cpp create mode 100644 core/libs/tests/GuestJsonValueOwnership.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 813027c4c0..e7b91c444c 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -20,6 +20,31 @@ if(BUILD_TESTING) target_link_libraries(guest_json_allocator_settings_tests PRIVATE libSceJson2 libc) configure_windows_unwind(guest_json_allocator_settings_tests) add_test(NAME guest_json_allocator_settings COMMAND guest_json_allocator_settings_tests) + add_test_executable(guest_json_value_layout_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonValueLayout.cpp) + target_include_directories(guest_json_value_layout_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_value_layout_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_value_layout_tests) + add_test(NAME guest_json_value_layout COMMAND guest_json_value_layout_tests) + add_test_executable(guest_json_value_ownership_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonValueOwnership.cpp) + target_include_directories(guest_json_value_ownership_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_value_ownership_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_value_ownership_tests) + add_test(NAME guest_json_value_ownership COMMAND guest_json_value_ownership_tests) + add_test_executable(guest_json_special_float_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonSpecialFloat.cpp) + target_include_directories(guest_json_special_float_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_special_float_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_special_float_tests) + add_test(NAME guest_json_special_float COMMAND guest_json_special_float_tests) + add_test_executable(guest_json_element_recovery_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonElementRecovery.cpp) + target_include_directories(guest_json_element_recovery_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_element_recovery_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_element_recovery_tests) + add_test(NAME guest_json_element_recovery COMMAND guest_json_element_recovery_tests) + add_test_executable(guest_json_null_recovery_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonNullRecovery.cpp) + target_include_directories(guest_json_null_recovery_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_null_recovery_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_null_recovery_tests) + add_test(NAME guest_json_null_recovery COMMAND guest_json_null_recovery_tests) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(numeric_locale_dir ${CMAKE_CURRENT_BINARY_DIR}/numeric-locale) set(numeric_locale_data ${LIBS_INCLUDE_DIR}/tests/numeric-locale) @@ -30,7 +55,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index b5636e821f..d556ade0df 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -34,14 +34,18 @@ constexpr int JsonErrorParse = static_cast(0x80848101); constexpr int JsonErrorOutOfMemory = static_cast(0x80848102); constexpr int JsonErrorNotInitialized = static_cast(0x80848110); constexpr int JsonErrorAlreadyInitialized = static_cast(0x80848111); +constexpr int JsonErrorSpecialFloat = static_cast(0x80848114); constexpr int JsonErrorInvalidArgument = static_cast(0x80848120); -struct Node; -struct Value { Node* node; }; +struct Value; +struct Handlers; +struct Auxiliary; +template +struct Items : std::list { Value* owner = nullptr; }; struct String { std::string* text; }; -struct Array { std::list* items; }; +struct Array { Items* items; }; struct Pair; -struct Object { std::list* items; }; +struct Object { Items* items; }; struct ArrayIteratorState { std::list::iterator position; std::list::iterator end; @@ -52,15 +56,6 @@ struct ObjectIteratorState { std::list::iterator position{}; }; struct ObjectIterator { ObjectIteratorState* state; }; -struct Pair { - String key; - std::uint64_t reserved; - Value value; -}; -static_assert(sizeof(Value) == 8 && sizeof(String) == 8 && sizeof(Array) == 8 && sizeof(Object) == 8); -static_assert(sizeof(ArrayIterator) == 8 && sizeof(ObjectIterator) == 8); -static_assert(offsetof(Pair, value) == 0x10); - using TypeMismatchHandler = const Value& (APS5_VABI*)(ValueType, const Value*, const Value*, void*); using SpecialFloatHandler = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); using ElementAccessFailureHandler = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); @@ -74,19 +69,53 @@ struct Handlers { void* elementAccessFailureContext = nullptr; }; -struct Node { +struct Value { + Value* parent = nullptr; + Auxiliary* auxiliary = nullptr; + union { + std::uint64_t payload = 0; + bool boolean; + std::int64_t integer; + std::uint64_t uinteger; + double real; + String* string; + Array* array; + Object* object; + }; + std::uint32_t reserved = 0; ValueType type = TypeNull; - bool boolean = false; - std::int64_t integer = 0; - std::uint64_t uinteger = 0; - double real = 0; - String string{}; - Array array{}; - Object object{}; - Handlers handlers{}; }; +using Node = Value; +static_assert(offsetof(Value, parent) == 0 && offsetof(Value, auxiliary) == 8); +static_assert(offsetof(Value, payload) == 16 && offsetof(Value, type) == 28); + +struct Pair { + String key; + std::uint64_t reserved; + Value value; +}; +static_assert(sizeof(Value) == 32 && sizeof(String) == 8 && sizeof(Array) == 8 && sizeof(Object) == 8); +static_assert(sizeof(ArrayIterator) == 8 && sizeof(ObjectIterator) == 8); +static_assert(offsetof(Pair, value) == 0x10); + using NullAccessCallback = const Value& (APS5_VABI*)(ValueType, const Value*, void*); +template +struct HandlerRecord { TCallback callback = nullptr; void* context = nullptr; }; + +struct Auxiliary { + NullAccessCallback nullAccess = nullptr; + void* nullAccessContext = nullptr; + Value* fallback = nullptr; + HandlerRecord* typeMismatch = nullptr; + HandlerRecord* specialFloat = nullptr; + HandlerRecord* elementAccessFailure = nullptr; + std::uint64_t reserved[3]{}; +}; +static_assert(sizeof(Auxiliary) == 72 && offsetof(Auxiliary, fallback) == 16); +static_assert(offsetof(Auxiliary, typeMismatch) == 24 && offsetof(Auxiliary, specialFloat) == 32); +static_assert(offsetof(Auxiliary, elementAccessFailure) == 40); + using RttiAllocate = void* (APS5_VABI*)(std::size_t, void*); using RttiFree = void (APS5_VABI*)(void*, void*); using RttiNotify = void (APS5_VABI*)(int, std::size_t, void*); @@ -105,6 +134,8 @@ struct Globals { bool floatFormatEnabled = false; bool initialized = false; AllocParamRtti rtti{}; + std::recursive_mutex callbackMutex; + bool callbackActive = false; }; Globals& State() { @@ -112,6 +143,14 @@ Globals& State() { return globals; } +struct CallbackScope { + Globals& state = State(); + std::unique_lock lock{state.callbackMutex}; + bool entered = !state.callbackActive; + CallbackScope() { if (entered) state.callbackActive = true; } + ~CallbackScope() { if (entered) state.callbackActive = false; } +}; + AllocParamRtti RttiCallbacks() { auto& state = State(); std::lock_guard lock(state.mutex); @@ -148,8 +187,8 @@ struct RttiAllocator { void* const* vtable = RttiAllocatorVtable; }; RttiAllocator rttiAllocator; void Construct(String& s, const std::string& text) { s.text = new std::string(text); } -void Construct(Array& a) { a.items = new std::list(); } -void Construct(Object& o) { o.items = new std::list(); } +void Construct(Array& a) { a.items = new Items(); } +void Construct(Object& o) { o.items = new Items(); } void Destroy(Value& v); void CopyInto(Value& destination, const Value& source); @@ -170,40 +209,115 @@ void Destroy(Object& o) { o.items = nullptr; } +const Node& RootOf(const Node& n) { + const Node* root = &n; + while (root->parent) root = root->parent; + return *root; +} + +Handlers HandlersOf(const Node& n) { + const auto* auxiliary = RootOf(n).auxiliary; + Handlers handlers; + if (!auxiliary) return handlers; + if (auxiliary->typeMismatch) { + handlers.typeMismatch = auxiliary->typeMismatch->callback; + handlers.typeMismatchContext = auxiliary->typeMismatch->context; + } + if (auxiliary->specialFloat) { + handlers.specialFloat = auxiliary->specialFloat->callback; + handlers.specialFloatContext = auxiliary->specialFloat->context; + } + if (auxiliary->elementAccessFailure) { + handlers.elementAccessFailure = auxiliary->elementAccessFailure->callback; + handlers.elementAccessFailureContext = auxiliary->elementAccessFailure->context; + } + return handlers; +} + +void BindChildren(Node& n) { + if (n.type == TypeArray && n.array) { + n.array->items->owner = &n; + for (auto& item : *n.array->items) item.parent = &n; + } + if (n.type == TypeObject && n.object) { + n.object->items->owner = &n; + for (auto& pair : *n.object->items) pair.value.parent = &n; + } +} + +template +void Destroy(T*& pointer) { + if (pointer) { + Destroy(*pointer); + delete pointer; + pointer = nullptr; + } +} + void Clear(Node& n) { if (n.type == TypeString) Destroy(n.string); if (n.type == TypeArray) Destroy(n.array); if (n.type == TypeObject) Destroy(n.object); - const Handlers handlers = n.handlers; - n = Node{}; - n.handlers = handlers; + n.type = TypeNull; } void Destroy(Value& v) { - if (!v.node) return; - Clear(*v.node); - delete v.node; - v.node = nullptr; + Clear(v); + if (v.auxiliary) { + delete v.auxiliary->typeMismatch; + delete v.auxiliary->specialFloat; + delete v.auxiliary->elementAccessFailure; + if (v.auxiliary->fallback) { + Destroy(*v.auxiliary->fallback); + delete v.auxiliary->fallback; + } + delete v.auxiliary; + v.auxiliary = nullptr; + } } -Node& NodeOf(const Value& v) { - if (!v.node) throw std::runtime_error("sce::Json::Value used before construction"); - return *v.node; +void EnsureAuxiliary(Node& n) { + if (!n.auxiliary) { + n.auxiliary = new Auxiliary(); + n.auxiliary->fallback = new Value(); + n.auxiliary->fallback->parent = &n; + } } -void Construct(Value& v) { v.node = new Node(); } +Node& NodeOf(const Value& v) { return const_cast(v); } + +void Construct(Value& v) { + v.parent = nullptr; + v.auxiliary = nullptr; + v.type = TypeNull; +} + +void TransferPayload(Node& destination, Node& source) { + if (source.type == TypeBoolean) destination.boolean = source.boolean; + else if (source.type != TypeNull) std::memcpy(&destination.payload, &source.payload, sizeof(destination.payload)); + destination.type = source.type; + source.type = TypeNull; + BindChildren(destination); +} template void Construct(Value& destination, TInitialize initialize) { Value pending{}; - Construct(pending); try { - initialize(*pending.node); + initialize(pending); } catch (...) { Destroy(pending); throw; } - destination = pending; + Construct(destination); + TransferPayload(destination, pending); +} + +template +void Construct(T*& destination, const TArgs&... arguments) { + auto pending = std::make_unique(); + Construct(*pending, arguments...); + destination = pending.release(); } void CopyArray(Array& destination, const Array& source) { @@ -236,24 +350,48 @@ void CopyObject(Object& destination, const Object& source) { destination = pending; } +void CopyArray(Array*& destination, const Array* source) { + auto pending = std::make_unique(); + CopyArray(*pending, *source); + destination = pending.release(); +} + +void CopyObject(Object*& destination, const Object* source) { + auto pending = std::make_unique(); + CopyObject(*pending, *source); + destination = pending.release(); +} + +void AssignReal(Node& n, double value); + void AssignNode(Node& destination, const Node& source) { + if (&destination == &source) return; Node copy{}; copy.type = source.type; - copy.boolean = source.boolean; - copy.integer = source.integer; - copy.uinteger = source.uinteger; - copy.real = source.real; try { - if (source.type == TypeString) Construct(copy.string, *source.string.text); - if (source.type == TypeArray) CopyArray(copy.array, source.array); - if (source.type == TypeObject) CopyObject(copy.object, source.object); + if (source.type == TypeString) Construct(copy.string, *source.string->text); + else if (source.type == TypeArray) CopyArray(copy.array, source.array); + else if (source.type == TypeObject) CopyObject(copy.object, source.object); + else if (source.type == TypeReal) AssignReal(copy, source.real); + else std::memcpy(©.payload, &source.payload, sizeof(copy.payload)); } catch (...) { Clear(copy); throw; } Clear(destination); - copy.handlers = destination.handlers; - destination = copy; + TransferPayload(destination, copy); +} + +void RecoverSpecial(const Node& source, const Handlers& handlers, int function, double number, Value& result, bool* error) { + Value returned{}; + try { + handlers.specialFloat(&returned, function, number, source.parent, handlers.specialFloatContext, error); + AssignNode(result, returned); + } catch (...) { + Destroy(returned); + throw; + } + Destroy(returned); } void CopyInto(Value& destination, const Value& source) { @@ -261,6 +399,7 @@ void CopyInto(Value& destination, const Value& source) { } void SetType(Node& n, ValueType type) { + if (n.type == type) return; if (type < TypeNull || type > TypeObject) throw std::invalid_argument("sce::Json::Value::set: invalid value type"); Node pending{}; pending.type = type; @@ -268,65 +407,87 @@ void SetType(Node& n, ValueType type) { if (type == TypeArray) Construct(pending.array); if (type == TypeObject) Construct(pending.object); Clear(n); - pending.handlers = n.handlers; - n = pending; + TransferPayload(n, pending); } const Value& Defaults(ValueType type) { static Value values[8]{}; - static std::once_flag once; - std::call_once(once, [] { - for (int i = 0; i < 8; ++i) { - Construct(values[i]); - SetType(*values[i].node, static_cast(i)); - } + static std::once_flag once[8]; + std::call_once(once[type], [type] { + Construct(values[type]); + SetType(values[type], type); }); return values[type]; } -const Value& NullAccess(ValueType requested, const Value* parent) { - auto& state = State(); - NullAccessCallback callback; - void* context; +const Node& Typed(const Value* self, ValueType requested) { + if (self->type == requested || + ((requested == TypeInteger || requested == TypeUInteger) && + (self->type == TypeInteger || self->type == TypeUInteger))) return *self; + CallbackScope scope; + if (!scope.entered) return Defaults(requested); + NullAccessCallback callback = nullptr; + void* context = nullptr; + bool initialized; { + auto& state = State(); std::lock_guard lock(state.mutex); + initialized = state.initialized; callback = state.callback; context = state.context; } - if (callback) { - const Value& result = callback(requested, parent, context); - if (result.node && (requested == TypeNull || result.node->type == requested)) return result; + if (!initialized) return Defaults(requested); + if (self->type == TypeNull) { + const auto* auxiliary = RootOf(*self).auxiliary; + if (auxiliary && auxiliary->nullAccess) + return Typed(&auxiliary->nullAccess(requested, self->parent, auxiliary->nullAccessContext), requested); } + const auto handlers = HandlersOf(*self); + if (handlers.typeMismatch) + return Typed(&handlers.typeMismatch(requested, self, self->parent, handlers.typeMismatchContext), requested); + if (self->type == TypeNull && callback) + return Typed(&callback(requested, self->parent, context), requested); return Defaults(requested); } -const Value& TypeMismatch(ValueType requested, const Value* self) { - const Handlers& handlers = NodeOf(*self).handlers; - const Value& result = handlers.typeMismatch(requested, self, nullptr, handlers.typeMismatchContext); - if (result.node && result.node->type == requested) return result; - return Defaults(requested); +const Value& MissingElement(const Value* self, int error, const String* key, const std::size_t* index) { + bool initialized; + { + auto& state = State(); + std::lock_guard lock(state.mutex); + initialized = state.initialized; + } + const auto* auxiliary = RootOf(*self).auxiliary; + if (initialized) { + CallbackScope scope; + const auto handlers = HandlersOf(*self); + if (scope.entered && handlers.elementAccessFailure && + (!auxiliary || self != auxiliary->fallback) && self != &Defaults(TypeNull)) + handlers.elementAccessFailure(error, key, index, self->parent, self, handlers.elementAccessFailureContext); + return auxiliary && auxiliary->fallback ? *auxiliary->fallback : Defaults(TypeNull); + } + return Defaults(TypeNull); } -const Node& Typed(const Value* self, ValueType requested) { - const Node& n = NodeOf(*self); - if (n.type == requested) return n; - if (n.type != TypeNull && n.handlers.typeMismatch) return NodeOf(TypeMismatch(requested, self)); - return NodeOf(NullAccess(requested, self)); -} - -const Value& MissingElement(const Value* self) { - if (NodeOf(*self).handlers.elementAccessFailure) - throw std::runtime_error("sce::Json::Value: element access failure handler: the error code the console passes is unverified"); - return NullAccess(TypeNull, self); -} - -template -void SetHandlers(Node& n, const TSetter& setter) { - setter(n.handlers); - if (n.type == TypeArray) - for (auto& item : *n.array.items) SetHandlers(NodeOf(item), setter); - if (n.type == TypeObject) - for (auto& pair : *n.object.items) SetHandlers(NodeOf(pair.value), setter); +template +int SetHandler(Node& n, HandlerRecord* Auxiliary::* member, TCallback callback, void* context, bool modernOnly = false) { + if (n.parent) return static_cast(0x80848104); + { + auto& state = State(); + std::lock_guard lock(state.mutex); + if (!state.initialized) return JsonErrorNotInitialized; + if (modernOnly && !state.floatFormatEnabled) return static_cast(0x80848113); + } + try { + EnsureAuxiliary(n); + auto& record = n.auxiliary->*member; + if (!record) record = new HandlerRecord(); + record->callback = callback; + record->context = context; + } catch (const std::bad_alloc&) { + return JsonErrorOutOfMemory; + } + return 0; } const char* SpecialFloatText(double value) { @@ -351,8 +512,22 @@ void AssignReal(Node& n, double value) { return; } const auto settings = FloatConfiguration(); - if (settings.enabled && n.handlers.specialFloat) - NotImplemented_nid_no_patch(__func__); + if (settings.enabled) { + CallbackScope scope; + const auto handlers = HandlersOf(n); + if (scope.entered && handlers.specialFloat) { + Value result{}; + try { + RecoverSpecial(n, handlers, 1, value, result, nullptr); + AssignNode(n, result); + } catch (...) { + Destroy(result); + throw; + } + Destroy(result); + return; + } + } if (settings.enabled && settings.format != 2) { if (settings.format == 1) { SetType(n, TypeNull); @@ -361,8 +536,7 @@ void AssignReal(Node& n, double value) { pending.type = TypeString; Construct(pending.string, SpecialFloatText(value)); Clear(n); - pending.handlers = n.handlers; - n = pending; + TransferPayload(n, pending); } return; } @@ -400,49 +574,72 @@ std::string RealText(double value) { return std::string(buffer, result.ptr); } -void Serialize(std::string& out, const Node& n) { +void AppendReal(std::string& out, double value) { + if (!std::isfinite(value)) { + const auto settings = FloatConfiguration(); + if (settings.enabled) { + if (settings.format == 0) AppendEscaped(out, SpecialFloatText(value)); + else out += settings.format == 1 ? "null" : SpecialFloatText(value); + return; + } + } + out += RealText(value); +} + +int Serialize(std::string& out, const Node& n) { switch (n.type) { - case TypeNull: out += "null"; return; - case TypeBoolean: out += n.boolean ? "true" : "false"; return; - case TypeInteger: out += std::to_string(n.integer); return; - case TypeUInteger: out += std::to_string(n.uinteger); return; - case TypeReal: - if (!std::isfinite(n.real) && n.handlers.specialFloat) - throw std::runtime_error("sce::Json::Value: special float handler: callback dispatch and ownership are unverified"); - if (!std::isfinite(n.real)) { - const auto settings = FloatConfiguration(); - if (settings.enabled) { - if (settings.format == 0) AppendEscaped(out, SpecialFloatText(n.real)); - else out += settings.format == 1 ? "null" : SpecialFloatText(n.real); - return; + case TypeNull: out += "null"; return 0; + case TypeBoolean: out += n.boolean ? "true" : "false"; return 0; + case TypeInteger: out += std::to_string(n.integer); return 0; + case TypeUInteger: out += std::to_string(n.uinteger); return 0; + case TypeReal: { + if (!std::isfinite(n.real) && FloatConfiguration().enabled) { + CallbackScope scope; + const auto handlers = HandlersOf(n); + if (scope.entered && handlers.specialFloat) { + Value replacement{}; + int result = 0; + try { + bool error = false; + RecoverSpecial(n, handlers, 0, n.real, replacement, &error); + result = error ? JsonErrorSpecialFloat : Serialize(out, replacement); + } catch (...) { + Destroy(replacement); + throw; + } + Destroy(replacement); + return result; } } - out += RealText(n.real); - return; - case TypeString: AppendEscaped(out, *n.string.text); return; + AppendReal(out, n.real); + return 0; + } + case TypeString: AppendEscaped(out, *n.string->text); return 0; case TypeArray: { out += '['; bool first = true; - for (const auto& item : *n.array.items) { + for (const auto& item : *n.array->items) { if (!first) out += ','; first = false; - Serialize(out, NodeOf(item)); + const int result = Serialize(out, item); + if (result < 0) return result; } out += ']'; - return; + return 0; } case TypeObject: { out += '{'; bool first = true; - for (const auto& pair : *n.object.items) { + for (const auto& pair : *n.object->items) { if (!first) out += ','; first = false; AppendEscaped(out, *pair.key.text); out += ':'; - Serialize(out, NodeOf(pair.value)); + const int result = Serialize(out, pair.value); + if (result < 0) return result; } out += '}'; - return; + return 0; } } throw std::runtime_error("sce::Json: invalid value type"); @@ -467,7 +664,10 @@ Value& ObjectEntry(Object& object, const std::string& key) { Construct(pair.key, key); try { Construct(pair.value); - return object.items->insert(position, pair)->value; + auto& value = object.items->insert(position, pair)->value; + value.parent = object.items->owner; + BindChildren(value); + return value; } catch (...) { Destroy(pair.value); Destroy(pair.key); @@ -479,8 +679,8 @@ using ParserReadCallback = int (APS5_VABI*)(char&, void*); class Parser { public: - Parser(const char* text, std::size_t size, const Handlers& handlers) : text(text), end(text + size), handlers(handlers) {} - Parser(ParserReadCallback read, void* context, const Handlers& handlers) : read(read), context(context), handlers(handlers) {} + Parser(const char* text, std::size_t size) : text(text), end(text + size) {} + Parser(ParserReadCallback read, void* context) : read(read), context(context) {} bool Parse(Node& out) { skip(); @@ -496,7 +696,6 @@ private: const char* end = nullptr; ParserReadCallback read = nullptr; void* context = nullptr; - Handlers handlers; std::size_t depth = 0; int current = -2; int readError = 0; @@ -639,7 +838,7 @@ private: std::string text; if (!string(text)) return false; SetType(out, TypeString); - *out.string.text = std::move(text); + *out.string->text = std::move(text); return true; } case '[': { @@ -650,12 +849,11 @@ private: skip(); if (peek() >= 0 && peek() == ']') { take(); --depth; return true; } for (;;) { - Value item{}; + auto& item = out.array->items->emplace_back(); Construct(item); - item.node->handlers = handlers; - out.array.items->push_back(item); + item.parent = &out; skip(); - if (!value(*item.node)) return false; + if (!value(item)) return false; skip(); if (peek() >= 0 && peek() == ',') { take(); continue; } if (peek() >= 0 && peek() == ']') { take(); --depth; return true; } @@ -677,10 +875,9 @@ private: if (peek() < 0 || peek() != ':') return false; take(); skip(); - Value& entry = ObjectEntry(out.object, key); - Clear(*entry.node); - entry.node->handlers = handlers; - if (!value(*entry.node)) return false; + Value& entry = ObjectEntry(*out.object, key); + Clear(entry); + if (!value(entry)) return false; skip(); if (peek() >= 0 && peek() == ',') { take(); continue; } if (peek() >= 0 && peek() == '}') { take(); --depth; return true; } @@ -694,8 +891,8 @@ private: }; int ParseValue(Value* out, Parser& parser) { + if (!out->parent) EnsureAuxiliary(*out); Node parsed{}; - parsed.handlers = NodeOf(*out).handlers; bool parsedOk = false; try { parsedOk = parser.Parse(parsed); @@ -708,7 +905,7 @@ int ParseValue(Value* out, Parser& parser) { return parser.Error(); } Clear(NodeOf(*out)); - *out->node = parsed; + TransferPayload(*out, parsed); return 0; } @@ -765,9 +962,11 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { } int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void* self, NullAccessCallback callback, void* context) { - (void)self; auto& state = State(); std::lock_guard lock(state.mutex); + if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; + if (!callback) return JsonErrorInvalidArgument; + if (state.callback) return static_cast(0x80848112); state.callback = callback; state.context = context; return 0; @@ -854,6 +1053,9 @@ void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array* self, const Val CopyInto(copy, *value); try { self->items->push_back(copy); + auto& inserted = self->items->back(); + inserted.parent = self->items->owner; + BindChildren(inserted); } catch (...) { Destroy(copy); throw; @@ -980,33 +1182,33 @@ void APS5_VABI _ZN3sce4Json6Object5eraseERKNS0_6StringE(Object* self, const Stri } void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value* self) { Construct(*self); } -void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value* self, bool value) { Construct(*self); self->node->type = TypeBoolean; self->node->boolean = value; } -void APS5_VABI _ZN3sce4Json5ValueC1El(Value* self, std::int64_t value) { Construct(*self); self->node->type = TypeInteger; self->node->integer = value; } -void APS5_VABI _ZN3sce4Json5ValueC1Em(Value* self, std::uint64_t value) { Construct(*self); self->node->type = TypeUInteger; self->node->uinteger = value; } +void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value* self, bool value) { Construct(*self); self->type = TypeBoolean; self->boolean = value; } +void APS5_VABI _ZN3sce4Json5ValueC1El(Value* self, std::int64_t value) { Construct(*self); self->type = TypeInteger; self->integer = value; } +void APS5_VABI _ZN3sce4Json5ValueC1Em(Value* self, std::uint64_t value) { Construct(*self); self->type = TypeUInteger; self->uinteger = value; } void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value* self, double value) { Construct(*self, [value](Node& n) { AssignReal(n, value); }); } void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value: null text"); Construct(*self, [text](Node& node) { SetType(node, TypeString); - *node.string.text = text; + *node.string->text = text; }); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6StringE(Value* self, const String* text) { Construct(*self, [text](Node& node) { SetType(node, TypeString); - *node.string.text = *text->text; + *node.string->text = *text->text; }); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value* self, const Array* array) { Construct(*self, [array](Node& node) { node.type = TypeArray; - CopyArray(node.array, *array); + CopyArray(node.array, array); }); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value* self, const Object* object) { Construct(*self, [object](Node& node) { node.type = TypeObject; - CopyObject(node.object, *object); + CopyObject(node.object, object); }); } void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value* self, const Value* other) { CopyInto(*self, *other); } @@ -1029,37 +1231,39 @@ void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value* self, Value* other) { } int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value* self, ValueType type) { SetType(NodeOf(*self), type); return 0; } -int APS5_VABI _ZN3sce4Json5Value3setEb(Value* self, bool value) { SetType(NodeOf(*self), TypeBoolean); self->node->boolean = value; return 0; } -int APS5_VABI _ZN3sce4Json5Value3setEl(Value* self, std::int64_t value) { SetType(NodeOf(*self), TypeInteger); self->node->integer = value; return 0; } -int APS5_VABI _ZN3sce4Json5Value3setEm(Value* self, std::uint64_t value) { SetType(NodeOf(*self), TypeUInteger); self->node->uinteger = value; return 0; } +int APS5_VABI _ZN3sce4Json5Value3setEb(Value* self, bool value) { SetType(NodeOf(*self), TypeBoolean); self->boolean = value; return 0; } +int APS5_VABI _ZN3sce4Json5Value3setEl(Value* self, std::int64_t value) { SetType(NodeOf(*self), TypeInteger); self->integer = value; return 0; } +int APS5_VABI _ZN3sce4Json5Value3setEm(Value* self, std::uint64_t value) { SetType(NodeOf(*self), TypeUInteger); self->uinteger = value; return 0; } void APS5_VABI _ZN3sce4Json5Value3setEd(Value* self, double value) { AssignReal(NodeOf(*self), value); } int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value::set: null text"); const std::string copy(text); SetType(NodeOf(*self), TypeString); - *self->node->string.text = copy; + *self->string->text = copy; return 0; } int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { const std::string copy(*text->text); SetType(NodeOf(*self), TypeString); - *self->node->string.text = copy; + *self->string->text = copy; return 0; } int APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { - Array copy{}; - CopyArray(copy, *array); + Array* copy = nullptr; + CopyArray(copy, array); Clear(NodeOf(*self)); - self->node->type = TypeArray; - self->node->array = copy; + self->type = TypeArray; + self->array = copy; + BindChildren(*self); return 0; } int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value* self, const Object* object) { - Object copy{}; - CopyObject(copy, *object); + Object* copy = nullptr; + CopyObject(copy, object); Clear(NodeOf(*self)); - self->node->type = TypeObject; - self->node->object = copy; + self->type = TypeObject; + self->object = copy; + BindChildren(*self); return 0; } int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value* self, const Value* other) { @@ -1085,38 +1289,49 @@ const std::uint64_t* APS5_VABI _ZNK3sce4Json5Value11getUIntegerEv(const Value* s } return &Typed(self, TypeUInteger).uinteger; } -const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value* self) { - Node& n = const_cast(NodeOf(*self)); - if (n.type == TypeInteger || n.type == TypeUInteger) { - n.real = n.type == TypeInteger ? static_cast(n.integer) : static_cast(n.uinteger); - return &n.real; - } - return &Typed(self, TypeReal).real; -} -const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value* self) { return &Typed(self, TypeString).string; } -const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value* self) { return &Typed(self, TypeArray).array; } -const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value* self) { return &Typed(self, TypeObject).object; } +const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value* self) { return &Typed(self, TypeReal).real; } +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value* self) { return Typed(self, TypeString).string; } +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value* self) { return Typed(self, TypeArray).array; } +const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value* self) { return Typed(self, TypeObject).object; } std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value* self) { const Node& n = NodeOf(*self); - if (n.type == TypeArray) return n.array.items->size(); - if (n.type == TypeObject) return n.object.items->size(); + if (n.type == TypeArray) return n.array->items->size(); + if (n.type == TypeObject) return n.object->items->size(); return 0; } const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value* self, const char* key) { - const Node& n = NodeOf(*self); - if (key != nullptr && n.type == TypeObject) - for (const auto& pair : *n.object.items) - if (std::strcmp(pair.key.text->c_str(), key) == 0) return &pair.value; - return &MissingElement(self); + String text{}; + Construct(text, key ? key : ""); + const Value* result; + try { + if (self->type == TypeObject) { + for (const auto& pair : *self->object->items) + if (std::strcmp(pair.key.text->c_str(), text.text->c_str()) == 0) { + Destroy(text); + return &pair.value; + } + } + result = &MissingElement(self, static_cast(self->type == TypeObject ? 0x80848115 : 0x80848117), &text, nullptr); + } catch (...) { + Destroy(text); + throw; + } + Destroy(text); + return result; } const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value* self, std::size_t index) { - const Node& n = NodeOf(*self); - if (n.type == TypeArray && index < n.array.items->size()) { - auto it = n.array.items->begin(); + if (self->type == TypeArray && index < self->array->items->size()) { + auto it = self->array->items->begin(); std::advance(it, static_cast(index)); return &*it; } - return &MissingElement(self); + if (self->type == TypeObject && index < self->object->items->size()) { + auto it = self->object->items->begin(); + std::advance(it, static_cast(index)); + return &it->value; + } + const auto error = self->type == TypeArray || self->type == TypeObject ? 0x80848116 : 0x80848118; + return &MissingElement(self, static_cast(error), nullptr, &index); } void APS5_VABI _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(Value* self, ValueType type) { Construct(*self, [type](Node& node) { SetType(node, type); }); @@ -1125,21 +1340,27 @@ const Value* APS5_VABI _ZNK3sce4Json5Value8getValueEm(const Value* self, std::si const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value* self, const String* key) { const Node& n = NodeOf(*self); if (key != nullptr && n.type == TypeObject) - for (const auto& pair : *n.object.items) + for (const auto& pair : *n.object->items) if (std::strcmp(pair.key.text->c_str(), key->text->c_str()) == 0) return &pair.value; - return &MissingElement(self); + return &MissingElement(self, static_cast(n.type == TypeObject ? 0x80848115 : 0x80848117), key, nullptr); } Value* APS5_VABI _ZN3sce4Json5Value10referValueEm(Value* self, std::size_t index) { - Node& n = NodeOf(*self); - if (n.type != TypeArray || index >= n.array.items->size()) return nullptr; - auto it = n.array.items->begin(); - std::advance(it, static_cast(index)); - return &*it; + if (self->type == TypeArray && index < self->array->items->size()) { + auto it = self->array->items->begin(); + std::advance(it, static_cast(index)); + return &*it; + } + if (self->type == TypeObject && index < self->object->items->size()) { + auto it = self->object->items->begin(); + std::advance(it, static_cast(index)); + return &it->value; + } + return nullptr; } Value* APS5_VABI _ZN3sce4Json5Value10referValueERKNS0_6StringE(Value* self, const String* key) { Node& n = NodeOf(*self); if (key == nullptr || n.type != TypeObject) return nullptr; - for (auto& pair : *n.object.items) + for (auto& pair : *n.object->items) if (std::strcmp(pair.key.text->c_str(), key->text->c_str()) == 0) return &pair.value; return nullptr; } @@ -1147,43 +1368,39 @@ Array* APS5_VABI _ZN3sce4Json5Value10referArrayEv(Value* self) { Node& n = NodeOf(*self); if (n.type == TypeNull) SetType(n, TypeArray); if (n.type != TypeArray) throw std::runtime_error("sce::Json::Value::referArray: value is not an array"); - return &n.array; + return n.array; } Object* APS5_VABI _ZN3sce4Json5Value11referObjectEv(Value* self) { Node& n = NodeOf(*self); if (n.type == TypeNull) SetType(n, TypeObject); if (n.type != TypeObject) throw std::runtime_error("sce::Json::Value::referObject: value is not an object"); - return &n.object; + return n.object; } int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value* self, String* out) { std::string text; - Serialize(text, NodeOf(*self)); - *out->text = std::move(text); - return 0; + const int result = Serialize(text, *self); + out->text->append(text); + return result; } void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value* self, String* out) { - const Node& n = NodeOf(*self); - if (n.type == TypeString) { - *out->text = *n.string.text; - return; - } - std::string text; - Serialize(text, n); - *out->text = std::move(text); - return; + if (self->type == TypeString) out->text->append(*self->string->text); + else if (self->type == TypeArray) out->text->append("array"); + else if (self->type == TypeObject) out->text->append("object"); + else if (self->type == TypeReal) AppendReal(*out->text, self->real); + else Serialize(*out->text, *self); } int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(Value* out, ParserReadCallback read, void* context) { if (read == nullptr) throw std::invalid_argument("sce::Json::Parser::parse: null read callback"); - Parser parser(read, context, NodeOf(*out).handlers); + Parser parser(read, context); return ParseValue(out, parser); } int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value* out, const char* text, std::size_t size) { if (size == 0) return JsonErrorParse; if (text == nullptr) throw std::invalid_argument("sce::Json::Parser::parse: null text"); - Parser parser(text, size, NodeOf(*out).handlers); + Parser parser(text, size); return ParseValue(out, parser); } @@ -1275,46 +1492,52 @@ void APS5_VABI _ZN3sce4Json12MemAllocator11notifyErrorEimPv(void* self, int erro (void)userData; } -void APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(NullAccessCallback callback, void* context) { - auto& state = State(); - std::lock_guard lock(state.mutex); - state.callback = callback; - state.context = context; +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value* self, NullAccessCallback callback, void* context) { + if (self->parent) return static_cast(0x80848104); + { + auto& state = State(); + std::lock_guard lock(state.mutex); + if (!state.initialized) return JsonErrorNotInitialized; + } + try { + EnsureAuxiliary(*self); + } catch (const std::bad_alloc&) { + return JsonErrorOutOfMemory; + } + self->auxiliary->nullAccess = callback; + self->auxiliary->nullAccessContext = context; + return 0; } int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value* self, ElementAccessFailureHandler handler, void* context) { - SetHandlers(NodeOf(*self), [&](Handlers& handlers) { - handlers.elementAccessFailure = handler; - handlers.elementAccessFailureContext = context; - }); - return 0; + return SetHandler(*self, &Auxiliary::elementAccessFailure, handler, context); } int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value* self, SpecialFloatHandler handler, void* context) { - SetHandlers(NodeOf(*self), [&](Handlers& handlers) { - handlers.specialFloat = handler; - handlers.specialFloatContext = context; - }); - return 0; + return SetHandler(*self, &Auxiliary::specialFloat, handler, context, true); } int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value* self, TypeMismatchHandler handler, void* context) { - SetHandlers(NodeOf(*self), [&](Handlers& handlers) { - handlers.typeMismatch = handler; - handlers.typeMismatchContext = context; - }); - return 0; + return SetHandler(*self, &Auxiliary::typeMismatch, handler, context); } bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value* self) { const Node& n = NodeOf(*self); + if (n.type != TypeBoolean) { + CallbackScope scope; + const auto handlers = HandlersOf(n); + if (scope.entered && handlers.typeMismatch) { + const Value& replacement = handlers.typeMismatch(TypeBoolean, self, self->parent, handlers.typeMismatchContext); + return Typed(&replacement, TypeBoolean).boolean; + } + } switch (n.type) { case TypeNull: return false; case TypeBoolean: return n.boolean; case TypeInteger: return n.integer != 0; case TypeUInteger: return n.uinteger != 0; case TypeReal: return n.real != 0.0; - case TypeString: return n.string.text && !n.string.text->empty(); + case TypeString: return n.string->text && !n.string->text->empty(); case TypeArray: case TypeObject: return true; default: throw std::runtime_error("sce::Json::Value: invalid value type"); @@ -1332,6 +1555,7 @@ void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair* self throw; } *self = pair; + BindChildren(self->value); } void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair* self) { @@ -1352,6 +1576,8 @@ ObjectIterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(ObjectIterator try { CopyInto(copy.value, pair->value); state->position = self->items->insert(it, copy); + state->position->value.parent = self->items->owner; + BindChildren(state->position->value); result->state = state.release(); return result; } catch (...) { @@ -1417,5 +1643,6 @@ void APS5_VABI _ZN3sce4Json6Object4PairC1Ev(Pair* self) { throw; } *self = pair; + BindChildren(self->value); } } diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index 17a3a96d1d..49a64ecff5 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -13,7 +13,7 @@ #include #include -struct Value { void* node; }; +struct Value { alignas(8) std::uint64_t storage[4]{}; }; struct String { void* text; }; struct Array { void* items; }; struct Object { void* items; }; @@ -36,7 +36,7 @@ int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void*, NullAccessCallback, void*); int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void*, NullAccessCallback, void*); -void APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(NullAccessCallback, void*); +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullAccessCallback, void*); int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, TypeMismatchHandler, void*); int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, SpecialFloatHandler, void*); int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, ElementAccessFailureHandler, void*); @@ -206,7 +206,7 @@ static void ParseAndRoundTrip() { Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "count")) == TypeUInteger); Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "count")) == 3); Require(*_ZNK3sce4Json5Value11getUIntegerEv(&Member(root, "count")) == 3); - Require(*_ZNK3sce4Json5Value7getRealEv(&Member(root, "count")) == 3.0); + Require(*_ZNK3sce4Json5Value7getRealEv(&Member(root, "count")) == 0.0); Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "big")) == TypeUInteger); Require(*_ZNK3sce4Json5Value11getUIntegerEv(&Member(root, "big")) == std::numeric_limits::max()); Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "neg")) == -5); @@ -390,7 +390,7 @@ static void ObjectPair() { std::memset(&pair, 0x5a, sizeof(pair)); _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &value); Require(pair.reserved == 0); - Require(pair.key.text != key.text && pair.value.node != value.node); + Require(pair.key.text != key.text && _ZNK3sce4Json5Value9getStringEv(&pair.value) != _ZNK3sce4Json5Value9getStringEv(&value)); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), "answer") == 0); Require(_ZNK3sce4Json5Value7getTypeEv(&pair.value) == TypeString); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&pair.value)), "forty-two") == 0); @@ -401,7 +401,7 @@ static void ObjectPair() { Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), "answer") == 0); Require(_ZNK3sce4Json5Value7getTypeEv(&pair.value) == TypeString); _ZN3sce4Json6Object4PairD1Ev(&pair); - Require(pair.key.text == nullptr && pair.value.node == nullptr); + Require(pair.key.text == nullptr && _ZNK3sce4Json5Value7getTypeEv(&pair.value) == TypeNull); } static void ArrayIteratorCopies() { @@ -867,25 +867,25 @@ static void NullAccess(SetNullAccessCallback setCallback) { Require(setCallback(initializer, OnNullAccess, &context) == 0); _ZN3sce4Json5ValueC1Ev(&fallback); Require(_ZN3sce4Json5Value3setEl(&fallback, 99) == 0); - Value root{}; _ZN3sce4Json5ValueC1Ev(&root); - const char text[] = "{\"present\":\"text\"}"; + const char text[] = "{\"present\":\"text\",\"none\":null}"; Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, text, sizeof(text) - 1) == 0); - Require(&Member(root, "missing") == &fallback); - Require(callbackCalls == 1 && lastRequested == TypeNull && lastParent == &root && lastContext == &context); - Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "present")) == 99); - Require(callbackCalls == 2 && lastRequested == TypeInteger && lastParent == &Member(root, "present")); - Require(!*_ZNK3sce4Json5Value10getBooleanEv(&Member(root, "present"))); - Require(callbackCalls == 3 && lastRequested == TypeBoolean); - Require(_ZNK3sce4Json5Value7getTypeEv(_ZNK3sce4Json5ValueixEm(&root, 0)) == TypeInteger); - - Require(_ZN3sce4Json11Initializer9terminateEv(initializer) == 0); - const Value& unhandled = Member(root, "missing"); - Require(&unhandled != &fallback && _ZNK3sce4Json5Value7getTypeEv(&unhandled) == TypeNull); - Require(callbackCalls == 4); + Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "missing")) == TypeNull && callbackCalls == 0); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "none")) == 99); + Require(callbackCalls == 1 && lastRequested == TypeInteger && lastParent == &root && lastContext == &context); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "present")) == 0 && callbackCalls == 1); + Require(!*_ZNK3sce4Json5Value10getBooleanEv(&Member(root, "none"))); + Require(callbackCalls == 2 && lastRequested == TypeBoolean); + Require(_ZNK3sce4Json5Value7getTypeEv(_ZNK3sce4Json5ValueixEm(&root, 0)) == TypeNull); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "missing")) == 99 && callbackCalls == 3); _ZN3sce4Json5ValueD1Ev(&root); _ZN3sce4Json5ValueD1Ev(&fallback); + Require(_ZN3sce4Json11Initializer9terminateEv(initializer) == 0); + Value after{}; + _ZN3sce4Json5ValueC1Ev(&after); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&after) == 0 && callbackCalls == 3); + _ZN3sce4Json5ValueD1Ev(&after); _ZN3sce4Json11InitializerD1Ev(initializer); } @@ -925,7 +925,7 @@ static void ValueAccess() { Require(_ZN3sce4Json5Value3setEl(element, 15) == 0); Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Value8getValueEm(list, 0)) == 15); Require(_ZN3sce4Json5Value10referValueEm(mutableList, 3) == nullptr); - Require(_ZN3sce4Json5Value10referValueEm(&root, 0) == nullptr); + Require(_ZN3sce4Json5Value10referValueEm(&root, 0) != nullptr); _ZN3sce4Json6StringD1Ev(&key); String missing{}; @@ -1145,29 +1145,27 @@ static void StreamParse() { _ZN3sce4Json5ValueD1Ev(&root); } -static void StaticNullAccess() { +static void ValueNullAccess() { callbackCalls = 0; lastRequested = -1; lastParent = nullptr; lastContext = nullptr; int context = 0; - _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(OnNullAccess, &context); _ZN3sce4Json5ValueC1Ev(&fallback); Require(_ZN3sce4Json5Value3setEl(&fallback, 99) == 0); - Value root{}; _ZN3sce4Json5ValueC1Ev(&root); - const char text[] = "{\"present\":\"text\"}"; + const char text[] = "{\"present\":\"text\",\"none\":null}"; Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, text, sizeof(text) - 1) == 0); - Require(&Member(root, "missing") == &fallback); - Require(callbackCalls == 1 && lastRequested == TypeNull && lastParent == &root && lastContext == &context); - Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "present")) == 99); - Require(callbackCalls == 2 && lastRequested == TypeInteger); - - _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(nullptr, nullptr); - const Value& unhandled = Member(root, "missing"); - Require(&unhandled != &fallback && _ZNK3sce4Json5Value7getTypeEv(&unhandled) == TypeNull); - Require(callbackCalls == 2); + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&root, OnNullAccess, &context) == 0); + Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "missing")) == TypeNull && callbackCalls == 0); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "none")) == 99); + Require(callbackCalls == 1 && lastRequested == TypeInteger && lastParent == &root && lastContext == &context); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "present")) == 0 && callbackCalls == 1); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "missing")) == 99 && callbackCalls == 2); + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&root, nullptr, nullptr) == 0); + Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "missing")) == TypeNull); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "none")) == 0 && callbackCalls == 2); _ZN3sce4Json5ValueD1Ev(&root); _ZN3sce4Json5ValueD1Ev(&fallback); } @@ -1190,7 +1188,7 @@ static const Value& APS5_VABI OnTypeMismatch(std::int32_t requested, const Value static void RequireMismatch(int calls, std::int32_t requested, const Value* value, void* context) { Require(mismatchCalls == calls && mismatchRequested == requested && mismatchValue == value); - Require(mismatchSecond == nullptr && mismatchContext == context); + Require(mismatchSecond == reinterpret_cast(value->storage[0]) && mismatchContext == context); } static void ValueTypeMismatchHandler() { @@ -1215,43 +1213,43 @@ static void ValueTypeMismatchHandler() { const Value* element = _ZNK3sce4Json5ValueixEm(&Member(root, "list"), 0); Require(*_ZNK3sce4Json5Value7getRealEv(element) == 0.0); RequireMismatch(4, TypeReal, element, &context); - Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "none")) == 0); - Require(mismatchCalls == 4); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "none")) == 42); + Require(mismatchCalls == 5); Value later{}; _ZN3sce4Json5ValueC1Ev(&later); Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&later, text, sizeof(text) - 1) == 0); Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(later, "name")) == 0); - Require(mismatchCalls == 4); + Require(mismatchCalls == 5); int laterContext = 0; Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&later, OnTypeMismatch, &laterContext) == 0); Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(later, "name")) == 42); - RequireMismatch(5, TypeInteger, &Member(later, "name"), &laterContext); + RequireMismatch(6, TypeInteger, &Member(later, "name"), &laterContext); Value other{}; _ZN3sce4Json5ValueC1Ev(&other); Require(_ZN3sce4Json5Value3setEPKc(&other, "other") == 0); Require(_ZN3sce4Json5ValueaSERKS1_(&root, &other) == &root); Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 42); - RequireMismatch(6, TypeInteger, &root, &context); + RequireMismatch(7, TypeInteger, &root, &context); Require(*_ZNK3sce4Json5Value10getIntegerEv(&other) == 0); - Require(mismatchCalls == 6); + Require(mismatchCalls == 7); _ZN3sce4Json5Value3setEd(&other, 1.5); Require(_ZN3sce4Json5Value3setERKS1_(&root, &other) == 0); Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 42); - RequireMismatch(7, TypeInteger, &root, &context); + RequireMismatch(8, TypeInteger, &root, &context); _ZN3sce4Json5Value5clearEv(&root); Require(_ZN3sce4Json5Value3setEPKc(&root, "cleared") == 0); Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 42); - RequireMismatch(8, TypeInteger, &root, &context); + RequireMismatch(9, TypeInteger, &root, &context); Value copy{}; _ZN3sce4Json5ValueC1ERKS1_(©, &root); Require(*_ZNK3sce4Json5Value10getIntegerEv(©) == 0); - Require(mismatchCalls == 8); + Require(mismatchCalls == 9); Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&root, nullptr, nullptr) == 0); Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 0); - Require(mismatchCalls == 8); + Require(mismatchCalls == 9); _ZN3sce4Json5ValueD1Ev(©); _ZN3sce4Json5ValueD1Ev(&other); _ZN3sce4Json5ValueD1Ev(&later); @@ -1265,16 +1263,6 @@ static void APS5_VABI OnElementAccessFailure(std::int32_t, const String*, const ++failureCalls; } -template -static bool ThrowsUnverified(const TAccess& access) { - try { - access(); - } catch (const std::runtime_error& error) { - return std::string(error.what()).find("unverified") != std::string::npos; - } - return false; -} - static void ValueElementAccessFailureHandler() { Value root{}; _ZN3sce4Json5ValueC1Ev(&root); @@ -1288,11 +1276,11 @@ static void ValueElementAccessFailureHandler() { Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Value8getValueEm(&list, 0)) == 1); String missing{}; _ZN3sce4Json6StringC1EPKc(&missing, "missing"); - Require(ThrowsUnverified([&] { _ZNK3sce4Json5ValueixEPKc(&root, "missing"); })); - Require(ThrowsUnverified([&] { _ZNK3sce4Json5Value8getValueERKNS0_6StringE(&root, &missing); })); - Require(ThrowsUnverified([&] { _ZNK3sce4Json5ValueixEm(&root, 0); })); - Require(ThrowsUnverified([&] { _ZNK3sce4Json5Value8getValueEm(&list, 1); })); - Require(failureCalls == 0); + Require(_ZNK3sce4Json5Value7getTypeEv(_ZNK3sce4Json5ValueixEPKc(&root, "missing")) == TypeNull); + Require(_ZNK3sce4Json5Value7getTypeEv(_ZNK3sce4Json5Value8getValueERKNS0_6StringE(&root, &missing)) == TypeNull); + Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5ValueixEm(&root, 0)) == 2); + Require(_ZNK3sce4Json5Value7getTypeEv(_ZNK3sce4Json5Value8getValueEm(&list, 1)) == TypeNull); + Require(failureCalls == 3); Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, nullptr, nullptr) == 0); Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "missing")) == TypeNull); @@ -1316,13 +1304,13 @@ static void ValueSpecialFloatHandler() { Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&real, OnSpecialFloat, &context) == 0); String out{}; _ZN3sce4Json6StringC1Ev(&out); - Require(ThrowsUnverified([&] { _ZN3sce4Json5Value9serializeERNS0_6StringE(&real, &out); })); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&real, &out) == 0); _ZN3sce4Json5Value3setEd(&real, std::numeric_limits::infinity()); - Require(ThrowsUnverified([&] { _ZN3sce4Json5Value9serializeERNS0_6StringE(&real, &out); })); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&real, &out) == 0); _ZN3sce4Json6StringD1Ev(&out); _ZN3sce4Json5Value3setEd(&real, 2.5); Require(Serialize(real) == "2.5"); - Require(specialFloatCalls == 0); + Require(specialFloatCalls == 2); _ZN3sce4Json5ValueD1Ev(&real); } @@ -1471,10 +1459,18 @@ int main(int argc, char** argv) { ObjectOperations(); NullAccess(_ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_); NullAccess(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_); - StaticNullAccess(); + JsonTest::Allocator recoveryAllocator; + alignas(8) std::uint8_t recoveryParameter[40]{}; + _ZN3sce4Json14InitParameter2C1Ev(recoveryParameter); + _ZN3sce4Json14InitParameter212setAllocatorEPNS0_12MemAllocatorEPv(recoveryParameter, &recoveryAllocator, nullptr); + std::uint8_t recoveryInitializer = 0; + _ZN3sce4Json11InitializerC1Ev(&recoveryInitializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&recoveryInitializer, recoveryParameter) == 0); + ValueNullAccess(); ValueTypeMismatchHandler(); ValueElementAccessFailureHandler(); ValueSpecialFloatHandler(); + Require(_ZN3sce4Json11Initializer9terminateEv(&recoveryInitializer) == 0); ValueAccess(); InitParameter2BaseConstructor(); InitParameters(); diff --git a/core/libs/tests/GuestJsonElementRecovery.cpp b/core/libs/tests/GuestJsonElementRecovery.cpp new file mode 100644 index 0000000000..cd3894b467 --- /dev/null +++ b/core/libs/tests/GuestJsonElementRecovery.cpp @@ -0,0 +1,129 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uintptr_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* text; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using Handler = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value5clearEv(Value*); +int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, Handler, void*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value*, const String*); +const Value* APS5_VABI _ZNK3sce4Json5Value8getValueEm(const Value*, std::size_t); +Value* APS5_VABI _ZN3sce4Json5Value10referValueEm(Value*, std::size_t); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct Context { + int calls = 0; + unsigned error = 0; + std::string key; + const String* keyPointer = nullptr; + std::size_t index = 0; + bool indexed = false; + const Value* parent = nullptr; + const Value* value = nullptr; + bool reenter = false; + const Value* fallback = nullptr; +}; +static void APS5_VABI Recover(int error, const String* key, const std::size_t* index, + const Value* parent, const Value* value, void* context) { + auto& trace = *static_cast(context); + ++trace.calls; + trace.error = static_cast(error); + trace.key = key ? _ZNK3sce4Json6String5c_strEv(key) : ""; + trace.keyPointer = key; + trace.indexed = index != nullptr; + trace.index = index ? *index : 0; + trace.parent = parent; + trace.value = value; + if (trace.reenter) { + trace.fallback = _ZNK3sce4Json5ValueixEPKc(value, "retry"); + _ZN3sce4Json5Value3setEl(const_cast(trace.fallback), 77); + } +} +static void Event(const Context& trace, int calls, unsigned error, const Value* value, const Value* parent) { + Require(trace.calls == calls && trace.error == error && trace.value == value && trace.parent == parent, + "Recovery must receive the exact error, source value, parent and context"); +} +int main() { + Value root{}; + _ZN3sce4Json5ValueC1Ev(&root); + Context trace; + Require(static_cast(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, Recover, &trace)) == 0x80848110, + "Registration before initialization must report not initialized"); + JsonTest::Allocator allocator; + Parameter parameter{&allocator, nullptr, 0, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize element recovery"); + const char text[] = "{\"list\":[1],\"number\":2}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, text, sizeof(text) - 1) == 0, "Parse recovery tree"); + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, Recover, &trace) == 0, + "Register root recovery"); + const auto* list = _ZNK3sce4Json5ValueixEPKc(&root, "list"); + const auto* number = _ZNK3sce4Json5ValueixEPKc(&root, "number"); + Require(trace.calls == 0 && list->parent == &root && number->parent == &root, "Successful lookups must bypass failure recovery"); + const auto* fallback = _ZNK3sce4Json5ValueixEPKc(&root, "missing"); + Event(trace, 1, 0x80848115, &root, nullptr); + Require(trace.key == "missing" && !trace.indexed && fallback->type == 0 && fallback->parent == &root, + "A missing object key must return its root-owned fallback value"); + String missing{}; + _ZN3sce4Json6StringC1EPKc(&missing, "missing"); + Require(_ZNK3sce4Json5Value8getValueERKNS0_6StringE(&root, &missing) == fallback, + "String and C-string lookup must share the root fallback"); + Event(trace, 2, 0x80848115, &root, nullptr); + Require(trace.keyPointer == &missing, "The String overload must pass the supplied key object"); + _ZNK3sce4Json5Value8getValueERKNS0_6StringE(list, &missing); + Event(trace, 3, 0x80848117, list, &root); + Require(trace.key == "missing" && !trace.indexed, "Wrong-type key lookup must pass the key and no index"); + Require(_ZNK3sce4Json5Value8getValueEm(list, 1) == fallback, "Nested refusal must use the same root fallback"); + Event(trace, 4, 0x80848116, list, &root); + Require(trace.indexed && trace.index == 1 && !trace.keyPointer, "Bounds failure must pass the complete index and no key"); + _ZNK3sce4Json5Value8getValueEm(number, static_cast(-1)); + Event(trace, 5, 0x80848118, number, &root); + Require(trace.indexed && trace.index == static_cast(-1), "Wrong-type numeric lookup must preserve all index bits"); + Require(_ZNK3sce4Json5Value8getValueEm(&root, 0) == list && _ZN3sce4Json5Value10referValueEm(&root, 1) == number && trace.calls == 5, + "Numeric object lookup must use ordered entries without reporting failure"); + Require(_ZN3sce4Json5Value10referValueEm(&root, 2) == nullptr && trace.calls == 5, + "The nullable mutable lookup must not dispatch failure recovery"); + trace.reenter = true; + Require(_ZNK3sce4Json5ValueixEPKc(list, "missing") == fallback, "Reentrant refusal must keep fallback identity"); + Event(trace, 6, 0x80848117, list, &root); + Require(trace.fallback == fallback && fallback->type == 2 && fallback->payload == 77, + "The callback must be able to update the root fallback without recursively dispatching itself"); + trace.reenter = false; + Require(_ZNK3sce4Json5Value8getValueEm(fallback, 9) == fallback && trace.calls == 6, + "A refusal on the fallback itself must suppress recovery"); + _ZN3sce4Json5Value5clearEv(const_cast(fallback)); + Require(_ZNK3sce4Json5ValueixEPKc(&root, "missing") == fallback && trace.calls == 7 && fallback->type == 0, + "Recovery must resume after reentry and retain fallback ownership"); + Require(static_cast(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(const_cast(list), Recover, &trace)) == 0x80848104, + "An owned child must reject handler registration"); + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, nullptr, nullptr) == 0, + "Unregister element recovery"); + Require(_ZNK3sce4Json5ValueixEPKc(&root, "missing") == fallback && trace.calls == 7, + "Unregistration must preserve fallback identity and stop dispatch"); + _ZN3sce4Json6StringD1Ev(&missing); + _ZN3sce4Json5ValueD1Ev(&root); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate element recovery"); + return 0; +} diff --git a/core/libs/tests/GuestJsonFloatFailure.cpp b/core/libs/tests/GuestJsonFloatFailure.cpp index 8b805ae8bc..c10c8c2808 100644 --- a/core/libs/tests/GuestJsonFloatFailure.cpp +++ b/core/libs/tests/GuestJsonFloatFailure.cpp @@ -10,6 +10,8 @@ #include struct Value { alignas(8) std::array bytes{}; }; +struct String { void* text; }; +using Handler = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); struct Parameter { void* allocator; void* userData; @@ -23,10 +25,14 @@ void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); void APS5_VABI _ZN3sce4Json11InitializerD1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value*); +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, Handler, void*); } #ifndef _WIN32 @@ -68,6 +74,74 @@ static void Require(bool condition, const char* message) { } } +struct CallbackFailure {}; +struct Context { bool throwAfterConstruction = false; int calls = 0; }; +static constexpr const char* Replacement = "A callback replacement long enough to require separately allocated character storage"; + +static Value* APS5_VABI Recover(Value* result, std::int32_t, double, const Value*, void* context, bool*) { + auto& trace = *static_cast(context); + ++trace.calls; + _ZN3sce4Json5ValueC1EPKc(result, Replacement); + if (trace.throwAfterConstruction) throw CallbackFailure{}; + return result; +} + +static void CheckRecoveryFailures() { + const auto baseline = liveAllocations; + Context context; + bool succeeded = false; + unsigned failures = 0; + for (int budget = 0; budget < 64 && !succeeded; ++budget) { + Value value; + _ZN3sce4Json5ValueC1Ed(&value, 0.125); + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, Recover, &context) == 0, + "Register allocation-failure recovery"); + const auto before = liveAllocations; + allocationBudget = budget; + try { + _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + succeeded = true; + } catch (const std::bad_alloc&) { + ++failures; + } + allocationBudget = -1; + if (succeeded) { + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == 5 && + std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&value)), Replacement) == 0, + "Recovery must remain usable after every earlier allocation failure"); + } else { + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == 4 && + *_ZNK3sce4Json5Value7getRealEv(&value) == 0.125 && liveAllocations == before, + "Failed callback construction or cloning must retain the destination and reclaim temporary ownership"); + } + _ZN3sce4Json5ValueD1Ev(&value); + Require(liveAllocations == baseline, "Recovery teardown must reclaim metadata, replacement and temporary allocations"); + } + Require(succeeded && failures > 0, "Recovery must exercise failed and successful host allocations"); + Value value; + _ZN3sce4Json5ValueC1Ed(&value, 0.125); + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, Recover, &context) == 0, + "Register throwing recovery"); + const auto before = liveAllocations; + context.throwAfterConstruction = true; + bool threw = false; + try { + _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + } catch (const CallbackFailure&) { + threw = true; + } + Require(threw && liveAllocations == before && _ZNK3sce4Json5Value7getTypeEv(&value) == 4 && + *_ZNK3sce4Json5Value7getRealEv(&value) == 0.125, + "A throwing host callback must release its constructed return object and preserve the destination"); + context.throwAfterConstruction = false; + const int calls = context.calls; + _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + Require(context.calls == calls + 1 && _ZNK3sce4Json5Value7getTypeEv(&value) == 5, + "Callback exceptions must release the reentry guard before retry"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(liveAllocations == baseline, "A retried callback must not leak"); +} + int main() { JsonTest::Allocator allocator; Parameter parameter{&allocator, nullptr, 0, 0, {}}; @@ -117,5 +191,6 @@ int main() { Require(_ZNK3sce4Json5Value7getTypeEv(&value) == 5, "Conversion must remain usable after allocation failure"); _ZN3sce4Json5ValueD1Ev(&value); Require(liveAllocations == baseline, "Successful conversion must not leak"); + CheckRecoveryFailures(); _ZN3sce4Json11InitializerD1Ev(&initializer); } diff --git a/core/libs/tests/GuestJsonNullRecovery.cpp b/core/libs/tests/GuestJsonNullRecovery.cpp new file mode 100644 index 0000000000..2486eb9bc3 --- /dev/null +++ b/core/libs/tests/GuestJsonNullRecovery.cpp @@ -0,0 +1,128 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uintptr_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using Handler = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using MismatchHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void*, Handler, void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, Handler, void*); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, MismatchHandler, void*); +bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +const bool* APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct Context { int calls = 0; int requested = -1; const Value* parent = nullptr; const Value* replacement; const Value* reenter = nullptr; }; +static const Value* mismatchSource = nullptr; +static const Value& APS5_VABI Mismatch(std::int32_t requested, const Value* self, const Value* parent, void* context) { + mismatchSource = self; + auto& trace = *static_cast(context); + ++trace.calls; + trace.requested = requested; + trace.parent = parent; + if (trace.reenter) + Require(*_ZNK3sce4Json5Value10getIntegerEv(trace.reenter) == 0, "Reentrant mismatch recovery must return its typed default"); + return *trace.replacement; +} +static const Value& APS5_VABI Recover(std::int32_t requested, const Value* parent, void* context) { + auto& trace = *static_cast(context); + ++trace.calls; + trace.requested = requested; + trace.parent = parent; + if (trace.reenter) + Require(*_ZNK3sce4Json5Value10getIntegerEv(trace.reenter) == 0, "Reentrant null access must return its typed default"); + return *trace.replacement; +} +int main() { + std::uint8_t initializer = 0, inactive = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Value replacement{}, localReplacement{}; + _ZN3sce4Json5ValueC1El(&replacement, 99); + _ZN3sce4Json5ValueC1El(&localReplacement, 77); + Context global{0, -1, nullptr, &replacement, nullptr}; + Context local{0, -1, nullptr, &localReplacement, nullptr}; + Require(static_cast(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(&initializer, Recover, &global)) == 0x80848110, + "Global callback registration must require an active initializer"); + JsonTest::Allocator allocator; + Parameter parameter{&allocator, nullptr, 0, 2, {}}; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize null recovery"); + Require(static_cast(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(&inactive, Recover, &global)) == 0x80848110, + "An inactive object cannot register a module-global callback"); + Require(static_cast(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(&initializer, nullptr, nullptr)) == 0x80848120, + "Global registration must reject a null callback"); + Require(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(&initializer, Recover, &global) == 0, "Register global null recovery"); + Require(static_cast(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(&initializer, Recover, &local)) == 0x80848112, + "Duplicate global registration must preserve its original callback and context"); + Value root{}; + _ZN3sce4Json5ValueC1Ev(&root); + const char text[] = "{\"n\":null,\"s\":\"text\"}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, text, sizeof(text) - 1) == 0, "Parse null recovery tree"); + const auto* null = _ZNK3sce4Json5ValueixEPKc(&root, "n"); + const auto* string = _ZNK3sce4Json5ValueixEPKc(&root, "s"); + const auto* missing = _ZNK3sce4Json5ValueixEPKc(&root, "missing"); + Require(missing->type == 0 && global.calls == 0, "Missing lookup must not invoke a typed null-access callback"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(null) == 99 && global.calls == 1 && global.requested == 2 && global.parent == &root, + "A typed null getter must call global recovery with its immediate parent"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(string) == 0 && global.calls == 1, + "A non-null type mismatch must not use null-access recovery"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(missing) == 99 && global.calls == 2 && global.parent == &root, + "The parser-owned fallback must retain its root parent"); + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&root, Recover, &local) == 0, + "The value callback setter must take this, callback and context"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(null) == 77 && local.calls == 1 && local.parent == &root && global.calls == 2, + "Root-local null recovery must take precedence over global recovery"); + local.reenter = null; + Require(!*_ZNK3sce4Json5Value10getBooleanEv(null) && local.calls == 2 && local.requested == 1, + "A mismatched callback result and reentrant getter must not recursively dispatch"); + local.reenter = nullptr; + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&root, nullptr, nullptr) == 0, + "A value-local callback may be unregistered"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(null) == 99 && global.calls == 3 && local.calls == 2, + "Unregistration must restore global recovery"); + Require(static_cast(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(const_cast(null), Recover, &local)) == 0x80848104, + "Owned children must reject local null callback registration"); + Context mismatch{0, -1, nullptr, &localReplacement, string}; + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&root, Mismatch, &mismatch) == 0, + "Register ancestor mismatch recovery"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(string) == 77 && mismatch.calls == 1 && mismatchSource == string && + mismatch.parent == &root && mismatch.requested == 2, + "Mismatch recovery must pass the requested type, source and immediate parent"); + Require(!*_ZNK3sce4Json5Value10getBooleanEv(string) && mismatch.calls == 2 && mismatch.requested == 1, + "A wrong-type recovery result must return the requested default without recursively dispatching"); + Require(!_ZNK3sce4Json5ValuecvbEv(string) && mismatch.calls == 3, + "Boolean conversion must consult mismatch recovery before ordinary truth conversion"); + Require(!_ZNK3sce4Json5ValuecvbEv(&root) && mismatch.calls == 4 && mismatchSource == &root && !mismatch.parent, + "A root boolean conversion must pass a null parent to mismatch recovery"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(null) == 77 && global.calls == 3 && mismatch.calls == 5, + "Root mismatch recovery must take precedence over module-global null recovery"); + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&root, Recover, &local) == 0, + "Restore root-local null recovery"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(null) == 77 && local.calls == 3 && mismatch.calls == 5, + "Root-local null recovery must take precedence over mismatch recovery"); + _ZN3sce4Json5ValueD1Ev(&root); + _ZN3sce4Json5ValueD1Ev(&replacement); + _ZN3sce4Json5ValueD1Ev(&localReplacement); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate null recovery"); + Value after{}; + _ZN3sce4Json5ValueC1Ev(&after); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&after) == 0 && global.calls == 3, "Termination must clear global recovery"); + _ZN3sce4Json5ValueD1Ev(&after); + return 0; +} diff --git a/core/libs/tests/GuestJsonSpecialFloat.cpp b/core/libs/tests/GuestJsonSpecialFloat.cpp new file mode 100644 index 0000000000..3eb871ebbe --- /dev/null +++ b/core/libs/tests/GuestJsonSpecialFloat.cpp @@ -0,0 +1,164 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uintptr_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* text; }; +struct Array { void* items; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using Handler = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, Handler, void*); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value*, String*); +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value*); +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct Context { + int calls = 0; + int function = -1; + const Value* parent = nullptr; + bool sawErrorPointer = false; + bool reject = false; + bool returnReal = false; + Value* reenter = nullptr; +}; +static std::string Serialize(Value& value, int expectedError = 0, const char* prefix = "") { + String output{}; + _ZN3sce4Json6StringC1EPKc(&output, prefix); + const int result = _ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output); + Require(result == expectedError, "Serialization must propagate the callback's error flag"); + std::string text = _ZNK3sce4Json6String5c_strEv(&output); + _ZN3sce4Json6StringD1Ev(&output); + return text; +} +static Value* APS5_VABI Recover(Value* result, std::int32_t function, double number, + const Value* parent, void* context, bool* error) { + auto& trace = *static_cast(context); + ++trace.calls; + trace.function = function; + trace.parent = parent; + trace.sawErrorPointer = error != nullptr; + Require(number == std::numeric_limits::infinity(), "The handler must receive the original non-finite value"); + if (trace.reenter) { + Require(Serialize(*trace.reenter) == "inf", "Reentrant serialization must use the configured fallback"); + _ZN3sce4Json5Value3setEd(trace.reenter, number); + } + if (trace.returnReal) _ZN3sce4Json5ValueC1Ed(result, number); + else _ZN3sce4Json5ValueC1EPKc(result, function == 0 ? "format" : "assign"); + if (error) *error = trace.reject; + return result; +} +static void CheckMode(int mode) { + Value value{}; + _ZN3sce4Json5ValueC1Ed(&value, std::numeric_limits::infinity()); + JsonTest::Allocator allocator; + Parameter parameter{&allocator, nullptr, 0, mode, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize special-float recovery"); + Context trace; + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, Recover, &trace) == 0, + "Register a special-float handler"); + Require(Serialize(value, 0, "prefix:") == "prefix:\"format\"" && value.type == 4, + "Serialization must append a separately owned replacement without mutating the original real"); + Require(trace.calls == 1 && trace.function == 0 && !trace.parent && trace.sawErrorPointer, + "Serialization must supply function 0, the parent and an error pointer"); + String direct{}; + _ZN3sce4Json6StringC1EPKc(&direct, "prefix:"); + _ZNK3sce4Json5Value8toStringERNS0_6StringE(&value, &direct); + const char* directExpected = mode == 0 ? "prefix:\"inf\"" : mode == 1 ? "prefix:null" : "prefix:inf"; + Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&direct), directExpected) == 0 && trace.calls == 1, + "toString must append the configured scalar format without dispatching recovery"); + _ZN3sce4Json6StringD1Ev(&direct); + trace.reject = true; + Require(Serialize(value, static_cast(0x80848114), "keep") == "keep" && value.type == 4, + "A rejected replacement must not overwrite existing output or the source"); + trace.reject = false; + _ZN3sce4Json5Value3setEd(&value, 0.125); + Require(trace.calls == 2 && value.type == 4, "Finite assignment must bypass recovery"); + _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + Require(trace.calls == 3 && trace.function == 1 && !trace.parent && !trace.sawErrorPointer, + "Assignment must supply function 1 and no error pointer for the void setter"); + Require(value.type == 5 && std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&value)), "assign") == 0, + "Assignment must retain the replacement after the callback temporary is destroyed"); + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, nullptr, nullptr) == 0, + "Unregister recovery"); + _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + Require(value.type == (mode == 0 ? 5 : mode == 1 ? 0 : 4) && trace.calls == 3, + "Unregistering must restore the configured fallback conversion"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate recovery tests"); +} +static void CheckParentsAndReentry() { + JsonTest::Allocator allocator; + Parameter parameter{&allocator, nullptr, 0, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize parent recovery"); + Array array{}; + _ZN3sce4Json5ArrayC1Ev(&array); + Value leaf{}, root{}, other{}; + _ZN3sce4Json5ValueC1Ed(&leaf, std::numeric_limits::infinity()); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &leaf); + _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&root, &array); + _ZN3sce4Json5ValueC1Ed(&other, std::numeric_limits::infinity()); + Context trace; + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&root, Recover, &trace) == 0, "Register ancestor recovery"); + auto* child = const_cast(_ZNK3sce4Json5Array5frontEv(_ZNK3sce4Json5Value8getArrayEv(&root))); + Require(static_cast(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(child, Recover, &trace)) == 0x80848104, + "An owned child must reject local handler registration"); + Require(Serialize(root) == "[\"format\"]" && trace.calls == 1 && trace.parent == &root, + "Nested serialization must use the root handler and pass the immediate parent"); + trace.reject = true; + Require(Serialize(root, static_cast(0x80848114), "keep") == "keep[" && trace.calls == 2, + "A nested rejection must retain output already emitted before the refused element"); + trace.reject = false; + _ZN3sce4Json5Value3setEd(child, std::numeric_limits::infinity()); + Require(trace.calls == 3 && trace.function == 1 && trace.parent == &root && child->type == 5, + "Nested assignment must use ancestor recovery"); + trace.reenter = &other; + trace.returnReal = true; + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&other, Recover, &trace) == 0, "Register reentry target"); + const int before = trace.calls; + Require(Serialize(other) == "inf" && trace.calls == before + 1, + "A returned non-finite real and reentrant operations must not recursively dispatch recovery"); + trace.reenter = nullptr; + trace.returnReal = false; + Require(Serialize(other) == "\"format\"" && trace.calls == before + 2, + "The callback guard must be released after recovery"); + _ZN3sce4Json5ValueD1Ev(&root); + _ZN3sce4Json5ValueD1Ev(&leaf); + _ZN3sce4Json5ValueD1Ev(&other); + _ZN3sce4Json5ArrayD1Ev(&array); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate parent recovery"); +} +int main() { + for (int mode = 0; mode < 3; ++mode) CheckMode(mode); + CheckParentsAndReentry(); + return 0; +} diff --git a/core/libs/tests/GuestJsonValueLayout.cpp b/core/libs/tests/GuestJsonValueLayout.cpp new file mode 100644 index 0000000000..02b619bc67 --- /dev/null +++ b/core/libs/tests/GuestJsonValueLayout.cpp @@ -0,0 +1,94 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Value { + void* parent; + void* auxiliary; + std::uint64_t payload; + std::uint32_t reserved; + std::int32_t type; +}; +static_assert(sizeof(Value) == 32 && offsetof(Value, payload) == 16 && offsetof(Value, type) == 28); +extern "C" { +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value*, bool); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueC1Em(Value*, std::uint64_t); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value5clearEv(Value*); +int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +const std::uint64_t* APS5_VABI _ZNK3sce4Json5Value11getUIntegerEv(const Value*); +const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value*); +std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +static void Prefix(const Value& value, int type) { + Require(!value.parent && !value.auxiliary && value.type == type, "Value must expose its native parent, auxiliary and inline type fields"); + Require(value.reserved == 0xa5a5a5a5U, "Value construction must preserve the reserved word"); +} +int main() { + alignas(8) std::array storage; + std::memset(storage.data(), 0xa5, sizeof(storage)); + auto& value = storage[1]; + _ZN3sce4Json5ValueC1Ev(&value); + Prefix(value, 0); + Require(value.payload == 0xa5a5a5a5a5a5a5a5ULL, "Null construction must preserve inactive payload bytes"); + _ZN3sce4Json5ValueD1Ev(&value); + _ZN3sce4Json5ValueC1Eb(&value, true); + Prefix(value, 1); + Require(value.payload == 0xa5a5a5a5a5a5a501ULL, "Boolean construction must write only its one-byte payload"); + _ZN3sce4Json5ValueD1Ev(&value); + _ZN3sce4Json5ValueC1El(&value, -73); + Prefix(value, 2); + Require(std::bit_cast(value.payload) == -73 && + _ZNK3sce4Json5Value10getIntegerEv(&value) == reinterpret_cast(&value.payload), + "Signed integer storage must be inline and returned by reference"); + Require(_ZNK3sce4Json5Value11getUIntegerEv(&value) == &value.payload, + "Both integer getters must refer to the same inline bits"); + Require(*_ZNK3sce4Json5Value7getRealEv(&value) == 0.0 && value.type == 2 && + std::bit_cast(value.payload) == -73, + "An unmatched real getter must leave integer storage unchanged and return the default real"); + Value copy; + std::memset(©, 0xa5, sizeof(copy)); + _ZN3sce4Json5ValueC1ERKS1_(©, &value); + Prefix(copy, 2); + Require(copy.payload == value.payload && _ZNK3sce4Json5Value10getIntegerEv(©) != _ZNK3sce4Json5Value10getIntegerEv(&value), + "A scalar copy must own independent inline storage"); + _ZN3sce4Json5Value3setEl(©, 19); + Require(std::bit_cast(value.payload) == -73 && copy.payload == 19, "Assignment must not alias a scalar source"); + _ZN3sce4Json5Value5clearEv(©); + Require(copy.type == 0 && copy.payload == 19 && copy.reserved == 0xa5a5a5a5U, + "Clear must reset type while preserving inactive scalar bytes"); + _ZN3sce4Json5ValueD1Ev(©); + _ZN3sce4Json5ValueD1Ev(&value); + _ZN3sce4Json5ValueC1Em(&value, 0xfedcba9876543210ULL); + Prefix(value, 3); + Require(value.payload == 0xfedcba9876543210ULL && *_ZNK3sce4Json5Value11getUIntegerEv(&value) == value.payload, + "Unsigned construction must preserve all 64 payload bits"); + _ZN3sce4Json5ValueD1Ev(&value); + for (std::uint64_t bits : {0x3fc0000000000000ULL, 0x8000000000000000ULL, 0x7ff80000cafef00dULL}) { + _ZN3sce4Json5ValueC1Ed(&value, std::bit_cast(bits)); + Prefix(value, 4); + Require(value.payload == bits && _ZNK3sce4Json5Value7getRealEv(&value) == reinterpret_cast(&value.payload), + "Real construction must preserve inline payload bits and reference identity"); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == value.type, "The getter must read the inline type"); + _ZN3sce4Json5ValueD1Ev(&value); + } + for (unsigned index : {0U, 2U}) { + const auto* bytes = reinterpret_cast(&storage[index]); + for (std::size_t offset = 0; offset < sizeof(Value); ++offset) + Require(bytes[offset] == 0xa5, "Value operations must not overwrite neighboring objects"); + } + return 0; +} diff --git a/core/libs/tests/GuestJsonValueOwnership.cpp b/core/libs/tests/GuestJsonValueOwnership.cpp new file mode 100644 index 0000000000..e03f3e0edf --- /dev/null +++ b/core/libs/tests/GuestJsonValueOwnership.cpp @@ -0,0 +1,110 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uintptr_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* items; }; +struct String { void* text; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); +int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value*, Value*); +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +static const Value& Front(const Value& value) { + return *_ZNK3sce4Json5Array5frontEv(_ZNK3sce4Json5Value8getArrayEv(&value)); +} +static void Tree(const Value& root) { + Require(root.type == 6 && !root.parent, "A copied root must own an independent root value"); + const auto& child = Front(root); + Require(child.parent == &root && child.type == 6, "An array element must refer to its owning value"); + const auto& leaf = Front(child); + Require(leaf.parent == &child && leaf.type == 2 && leaf.payload == 42, + "Nested children must refer to their immediate owner and preserve inline payloads"); +} +int main() { + JsonTest::Allocator allocator; + Parameter parameter{&allocator, nullptr, 0, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize ownership tests"); + Value text{}; + _ZN3sce4Json5ValueC1EPKc(&text, "keep"); + const auto payload = text.payload; + const auto* string = _ZNK3sce4Json5Value9getStringEv(&text); + Require(reinterpret_cast(string) == payload, "A string payload must hold its String object address"); + _ZN3sce4Json5Value3setENS0_9ValueTypeE(&text, 5); + Require(text.payload == payload && std::strcmp(_ZNK3sce4Json6String5c_strEv(string), "keep") == 0, + "Setting an existing type must preserve its contents and identity"); + Value leaf{}; + _ZN3sce4Json5ValueC1El(&leaf, 42); + Array inner{}; + _ZN3sce4Json5ArrayC1Ev(&inner); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&inner, &leaf); + Require(!_ZNK3sce4Json5Array5frontEv(&inner)->parent, "A standalone array must not invent a value owner"); + Value child{}; + _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&child, &inner); + Array outer{}; + _ZN3sce4Json5ArrayC1Ev(&outer); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&outer, &child); + Value root{}; + _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&root, &outer); + Tree(root); + Require(reinterpret_cast(_ZNK3sce4Json5Value8getArrayEv(&root)) == root.payload, + "An array payload must hold its Array object address"); + Value copy{}; + _ZN3sce4Json5ValueC1ERKS1_(©, &root); + Tree(copy); + Require(copy.payload != root.payload && Front(copy).payload != Front(root).payload, + "Copies must not share owned containers"); + const auto rootPayload = root.payload; + _ZN3sce4Json5Value3setENS0_9ValueTypeE(&root, 6); + Require(root.payload == rootPayload && _ZNK3sce4Json5Array4sizeEv(_ZNK3sce4Json5Value8getArrayEv(&root)) == 1, + "Setting the existing array type must retain elements and storage"); + auto* ownedArray = const_cast(_ZNK3sce4Json5Value8getArrayEv(©)); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(ownedArray, &child); + const auto* appended = _ZNK3sce4Json5Array4backEv(ownedArray); + Require(appended->parent == © && Front(*appended).parent == appended, + "Appending to an owned array must bind both new and nested children"); + _ZN3sce4Json5Value3setERKS1_(©, &Front(copy)); + Require(!copy.parent && Front(copy).parent == © && Front(copy).payload == 42, + "Assignment from a descendant must finish copying before releasing the old tree"); + _ZN3sce4Json5Value4swapERS1_(&root, &text); + Require(root.type == 5 && text.type == 6, "Swap must exchange payloads"); + Tree(text); + Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&root)), "keep") == 0, + "Swap must preserve string contents"); + _ZN3sce4Json5ValueD1Ev(&root); + _ZN3sce4Json5ValueD1Ev(©); + _ZN3sce4Json5ValueD1Ev(&child); + _ZN3sce4Json5ValueD1Ev(&leaf); + _ZN3sce4Json5ValueD1Ev(&text); + _ZN3sce4Json5ArrayD1Ev(&inner); + _ZN3sce4Json5ArrayD1Ev(&outer); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate ownership tests"); + return 0; +} diff --git a/core/libs/tests/JsonCopyFailure.cpp b/core/libs/tests/JsonCopyFailure.cpp index 6f95b142e3..90f4ab421b 100644 --- a/core/libs/tests/JsonCopyFailure.cpp +++ b/core/libs/tests/JsonCopyFailure.cpp @@ -7,7 +7,7 @@ #include #include -struct Value { void* node; }; +struct Value { alignas(8) std::uint64_t storage[4]{}; }; struct String { void* text; }; struct Array { void* items; }; struct Object { void* items; }; @@ -203,7 +203,7 @@ int main() { {"set object", Operation::SetObject, &objectValue, objectJson}, {"set empty array", Operation::SetTypeArray, &objectValue, "[]"}, {"set empty object", Operation::SetTypeObject, &objectValue, "{}"}, - {"set empty string", Operation::SetTypeString, &objectValue, "\"\""}, + {"retain string type", Operation::SetTypeString, &objectValue, "\"previous\""}, {"pair copy", Operation::PairCopy, &objectValue, objectJson}, }; unsigned failures = 0; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index a7a4ea071a..e8e83a1f18 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -6,17 +6,17 @@ - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still ignores allocator storage and file-buffer size; allocation admission, allocator notifications and rollback are not implemented. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. -- [sce::Json::Value storage and recovery callback ABI](../../../core/libs/prx/libSceJson2/Export.cpp) - values use an 8-byte host-node wrapper rather than the inspected 32-byte inline layout. Special-float recovery at `0x2b90` receives a 32-byte return object through a hidden result pointer; correcting callback dispatch requires compatible payload, parent and auxiliary ownership. Current host allocation counts and request sizes do not follow the inspected library. +- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records, strings and containers still use host allocations. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the signatures follow the mangled names, with `Array::iterator::operator++(int)` and `Object::iterator::operator++(int)` returning their copy through a result pointer before `this` as `begin` does, since the iterators have non-trivial destructors. `Pair(const String&, const Value&)` and default `Pair()` deep-copy or initialize into the layout `Object` iteration already hands out (key at 0, value at 0x10) and zeroes the 8 bytes between them; what the console keeps there is unknown. -- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - setter return types are inferred from a title that registers handlers on its root value and accepts 0 or `0x80848104`; the setters return 0. Signatures follow mangled names. The Vita null-access callback is also a member ([vitasdk](https://github.com/vitasdk/vita-headers/blob/15605617cbbb23d266f88af28899fdb573e2355e/include/psp2/json.h#L638)). Applying a handler to current descendants and parser-created values follows that caller; copies, `push_back` and object insertion do not carry handlers. Console ancestor lookup and propagation remain unverified. -- [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - a typed getter on a non-null value of another type calls the registered handler with the requested type, value, null and context. It uses the returned value if its type matches, otherwise the default. The order of the two value pointers and the null second pointer remain unverified against the console. -- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - the registered handler is not called: failed indexing or `getValue` throws because its error code is unknown. Without a handler, access reaches the null-access callback. -- [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - a non-finite real carrying a handler throws during serialization. Assignment under InitParameter2 with a registered handler throws through `NotImplemented_nid_no_patch`, before changing the value. Callback result ownership, ancestor lookup, error propagation and reentrancy remain unimplemented or unverified; the title returns different replacements for its two function types. +- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed admission remains incomplete. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`; the host auxiliary allocation path is not the supplied allocator. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. +- [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent legacy handler maps and module-global mismatch registration remain incomplete. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Compatibility below SDK 6 is unverified. +- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - module-global recovery and SDK-dependent dispatch remain incomplete. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Host allocation failure behavior still differs; compatibility below SDK 6 is unverified. +- [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - module-global special-float registration and allocator-backed failures remain incomplete. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. - [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing. A title's allocator that does not override it calls it through its own vtable; the library never calls the title's allocator, so it never reports an allocation failure. Whether the console's base implementation logs anything is unknown - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - storage uses `std::string`; capacity and allocation growth do not match the system 32-bit capacity and allocator settings. Appending null text with a nonzero length throws instead of dereferencing it. - [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the return type of `advance` remains unverified. Object iterator advancement past the end throws instead of asserting. Dereferencing an empty array iterator throws instead of returning the system null-value pointer or an invalid address. -- [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - callback dispatch before conversion is not implemented. The system library invokes the applicable type mismatch handler before converting a non-boolean value, with SDK-dependent global callback handling. +- [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - module-global mismatch fallback and legacy SDK behavior remain incomplete. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. ### Unknown 3.20 import declarations From bf9ad74a2a6d833a60f78182595e1f5e46cef853 Mon Sep 17 00:00:00 2001 From: Kyon Date: Thu, 22 Oct 2026 23:24:42 +0200 Subject: [PATCH 012/318] fix(libSceJson2): honor string allocation and global recovery settings --- core/libs/prx/libSceJson2/CMakeLists.txt | 33 +- core/libs/prx/libSceJson2/Export.cpp | 426 +++++++++++++++---- core/libs/tests/GuestJson.cpp | 66 ++- core/libs/tests/GuestJsonFile.cpp | 134 ++++++ core/libs/tests/GuestJsonFloatFailure.cpp | 18 +- core/libs/tests/GuestJsonGlobalRecovery.cpp | 122 ++++++ core/libs/tests/GuestJsonStringAllocator.cpp | 124 ++++++ core/libs/tests/JsonCopyFailure.cpp | 9 + docs/dev/debt/libSceJson2.md | 39 +- 9 files changed, 841 insertions(+), 130 deletions(-) create mode 100644 core/libs/tests/GuestJsonFile.cpp create mode 100644 core/libs/tests/GuestJsonGlobalRecovery.cpp create mode 100644 core/libs/tests/GuestJsonStringAllocator.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index e7b91c444c..412298132d 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -45,6 +45,16 @@ if(BUILD_TESTING) target_link_libraries(guest_json_null_recovery_tests PRIVATE libSceJson2 libc) configure_windows_unwind(guest_json_null_recovery_tests) add_test(NAME guest_json_null_recovery COMMAND guest_json_null_recovery_tests) + add_test_executable(guest_json_global_recovery_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonGlobalRecovery.cpp) + target_include_directories(guest_json_global_recovery_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_global_recovery_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_global_recovery_tests) + add_test(NAME guest_json_global_recovery COMMAND guest_json_global_recovery_tests) + add_test_executable(guest_json_string_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonStringAllocator.cpp) + target_include_directories(guest_json_string_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_string_allocator_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_string_allocator_tests) + add_test(NAME guest_json_string_allocator COMMAND guest_json_string_allocator_tests) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(numeric_locale_dir ${CMAKE_CURRENT_BINARY_DIR}/numeric-locale) set(numeric_locale_data ${LIBS_INCLUDE_DIR}/tests/numeric-locale) @@ -55,37 +65,50 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery PROPERTIES ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:$") + set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() if(BUILD_TESTING) add_test_executable(guest_json_copy_failure_tests ${LIBS_INCLUDE_DIR}/tests/JsonCopyFailure.cpp Export.cpp) target_include_directories(guest_json_copy_failure_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_copy_failure_tests PRIVATE libkernel libc) if(WIN32) - target_link_libraries(guest_json_copy_failure_tests PRIVATE libc) configure_windows_unwind(guest_json_copy_failure_tests) endif() add_test(NAME guest_json_copy_failure COMMAND guest_json_copy_failure_tests) set_tests_properties(guest_json_copy_failure PROPERTIES TIMEOUT 30) if(WIN32) set_tests_properties(guest_json_copy_failure PROPERTIES ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:$") + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() if(BUILD_TESTING) add_test_executable(guest_json_float_failure_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonFloatFailure.cpp Export.cpp) target_include_directories(guest_json_float_failure_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_float_failure_tests PRIVATE libkernel libc) if(WIN32) - target_link_libraries(guest_json_float_failure_tests PRIVATE libc) configure_windows_unwind(guest_json_float_failure_tests) endif() add_test(NAME guest_json_float_failure COMMAND guest_json_float_failure_tests) set_tests_properties(guest_json_float_failure PROPERTIES TIMEOUT 30) if(WIN32) set_tests_properties(guest_json_float_failure PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() +endif() + +if(BUILD_TESTING) + add_test_executable(guest_json_file_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonFile.cpp Export.cpp) + target_include_directories(guest_json_file_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_file_tests PRIVATE libc) + configure_windows_unwind(guest_json_file_tests) + add_test(NAME guest_json_file COMMAND guest_json_file_tests) + set_tests_properties(guest_json_file PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_json_file PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index d556ade0df..6bf35145ca 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -14,8 +14,12 @@ #include #include #include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libkernel/File/include/File.hpp" namespace { @@ -42,7 +46,9 @@ struct Handlers; struct Auxiliary; template struct Items : std::list { Value* owner = nullptr; }; -struct String { std::string* text; }; +struct Text { char* characters; std::uint32_t length; std::uint32_t capacity; std::uint64_t reserved; }; +static_assert(sizeof(Text) == 24); +struct String { Text* text; }; struct Array { Items* items; }; struct Pair; struct Object { Items* items; }; @@ -126,14 +132,23 @@ struct AllocParamRtti { RttiNotify notify; }; +struct AllocationContext { + void* allocator = nullptr; + void* userData = nullptr; + std::size_t fileBufferSize = 0; +}; +static_assert(sizeof(AllocationContext) == 24); + struct Globals { - std::mutex mutex; + std::recursive_mutex mutex; NullAccessCallback callback = nullptr; void* context = nullptr; std::int32_t floatFormat = 2; bool floatFormatEnabled = false; bool initialized = false; AllocParamRtti rtti{}; + Handlers handlers{}; + AllocationContext allocation{}; std::recursive_mutex callbackMutex; bool callbackActive = false; }; @@ -157,6 +172,54 @@ AllocParamRtti RttiCallbacks() { return state.rtti; } +void* Allocate(const AllocationContext& context, std::size_t size) { + auto* const* table = *static_cast(context.allocator); + const auto allocate = reinterpret_cast(table[2]); + void* memory = allocate(context.allocator, size, context.userData); + if (!memory) { + const auto notify = reinterpret_cast(table[4]); + notify(context.allocator, JsonErrorOutOfMemory, size, context.userData); + } + return memory; +} + +void Release(const AllocationContext& context, void* memory) { + auto* const* table = *static_cast(context.allocator); + const auto free = reinterpret_cast(table[3]); + free(context.allocator, memory, context.userData); +} + +struct FileReader { + AllocationContext allocation; + void* buffer = nullptr; + int descriptor = -1; + std::size_t available = 0; + std::size_t position = 0; + + ~FileReader() { + if (descriptor >= 0) sceKernelClose(descriptor); + Release(allocation, buffer); + } + + static int APS5_VABI Read(char& character, void* context) { + auto& reader = *static_cast(context); + if (reader.descriptor < 0) { + if (reader.descriptor == -1) return JsonErrorNotInitialized; + if (reader.descriptor == static_cast(0x80020002)) return static_cast(0x80848103); + return reader.descriptor; + } + if (reader.position == reader.available) { + const auto count = sceKernelRead(reader.descriptor, reader.buffer, reader.allocation.fileBufferSize); + if (count < 0) return static_cast(count); + if (!count) return JsonErrorParse; + reader.available = static_cast(count); + reader.position = 0; + } + character = static_cast(reader.buffer)[reader.position++]; + return 0; + } +}; + void APS5_VABI RttiDestroy(void*) {} void APS5_VABI RttiDelete(void* self) { ::operator delete(self); } @@ -186,13 +249,123 @@ void* const RttiAllocatorVtable[] = { struct RttiAllocator { void* const* vtable = RttiAllocatorVtable; }; RttiAllocator rttiAllocator; -void Construct(String& s, const std::string& text) { s.text = new std::string(text); } +char emptyCharacters[1]{}; + +AllocationContext ActiveAllocation() { + auto& state = State(); + std::lock_guard lock(state.mutex); + return state.allocation; +} + +void* Allocate(std::size_t size) { + const auto context = ActiveAllocation(); + return context.allocator ? Allocate(context, size) : nullptr; +} + +void Release(void* memory) { + const auto context = ActiveAllocation(); + if (context.allocator) Release(context, memory); +} + +const char* Characters(const String& string) { + return string.text ? string.text->characters : emptyCharacters; +} + +std::string_view View(const String& string) { + return {Characters(string), string.text ? string.text->length : 0}; +} + +bool Grow(Text& text, std::size_t requested) { + if (requested > std::numeric_limits::max()) + throw std::length_error("sce::Json::String: capacity exceeds 32 bits"); + if (requested <= text.capacity) return true; + std::uint32_t capacity = static_cast(requested); + std::uint32_t sdk = std::numeric_limits::max(); + try { + const auto* process = static_cast(ApplicationProcessParameters_nid_no_patch()); + if (process) std::memcpy(&sdk, process + 0x10, sizeof(sdk)); + } catch (const std::runtime_error&) { + } + if (sdk > 0x044fffff) { + capacity |= 0x10; + if (capacity / 3 < text.capacity / 2) { + const auto grown = std::uint64_t{text.capacity} + text.capacity / 2; + capacity = static_cast(std::min(grown, std::uint64_t{std::numeric_limits::max()})); + } + } + auto* memory = static_cast(Allocate(capacity)); + if (!memory) return false; + std::memcpy(memory, text.characters, text.length); + memory[text.length] = 0; + if (text.characters != emptyCharacters) Release(text.characters); + text.characters = memory; + text.capacity = capacity; + return true; +} + +void ClearText(String& string) { + if (!string.text) return; + if (string.text->characters != emptyCharacters) Release(string.text->characters); + string.text->characters = emptyCharacters; + string.text->length = 0; + string.text->capacity = 0; +} + +void AssignText(String& string, std::string_view bytes, bool preserveBinary = false) { + if (!string.text || bytes.data() == string.text->characters) return; + if (bytes.size() >= std::numeric_limits::max()) + throw std::length_error("sce::Json::String: length exceeds 32 bits"); + ClearText(string); + if (bytes.empty() || (!preserveBinary && !bytes.front())) return; + auto* memory = static_cast(Allocate(bytes.size() + 1)); + if (!memory) return; + std::memcpy(memory, bytes.data(), bytes.size()); + memory[bytes.size()] = 0; + string.text->characters = memory; + string.text->length = static_cast(bytes.size()); + string.text->capacity = static_cast(bytes.size() + 1); +} + +void AppendText(String& string, std::string_view bytes) { + if (!string.text) return; + if (bytes.size() >= std::numeric_limits::max() - string.text->length) + throw std::length_error("sce::Json::String: length exceeds 32 bits"); + const auto source = reinterpret_cast(bytes.data()); + const auto destination = reinterpret_cast(string.text->characters); + const bool aliases = source >= destination && source - destination < string.text->capacity; + const auto offset = source - destination; + if (!Grow(*string.text, string.text->length + bytes.size() + 1)) return; + const char* data = aliases ? string.text->characters + offset : bytes.data(); + std::memmove(string.text->characters + string.text->length, data, bytes.size()); + string.text->length += static_cast(bytes.size()); + string.text->characters[string.text->length] = 0; +} + +void Construct(String& string, std::string_view bytes) { + string.text = static_cast(Allocate(sizeof(Text))); + if (!string.text) return; + string.text->characters = emptyCharacters; + string.text->length = 0; + string.text->capacity = 0; + AssignText(string, bytes); +} + +void Construct(String& string, const String& source) { + if (!source.text) { string.text = nullptr; return; } + Construct(string, std::string_view{}); + AssignText(string, View(source), true); +} void Construct(Array& a) { a.items = new Items(); } void Construct(Object& o) { o.items = new Items(); } void Destroy(Value& v); void CopyInto(Value& destination, const Value& source); -void Destroy(String& s) { delete s.text; s.text = nullptr; } +void Destroy(String& s) { + if (!s.text) return; + ClearText(s); + Release(s.text); + s.text = nullptr; +} void Destroy(Array& a) { if (!a.items) return; for (auto& item : *a.items) Destroy(item); @@ -218,16 +391,21 @@ const Node& RootOf(const Node& n) { Handlers HandlersOf(const Node& n) { const auto* auxiliary = RootOf(n).auxiliary; Handlers handlers; + { + auto& state = State(); + std::lock_guard lock(state.mutex); + if (state.initialized) handlers = state.handlers; + } if (!auxiliary) return handlers; - if (auxiliary->typeMismatch) { + if (auxiliary->typeMismatch && auxiliary->typeMismatch->callback) { handlers.typeMismatch = auxiliary->typeMismatch->callback; handlers.typeMismatchContext = auxiliary->typeMismatch->context; } - if (auxiliary->specialFloat) { + if (auxiliary->specialFloat && auxiliary->specialFloat->callback) { handlers.specialFloat = auxiliary->specialFloat->callback; handlers.specialFloatContext = auxiliary->specialFloat->context; } - if (auxiliary->elementAccessFailure) { + if (auxiliary->elementAccessFailure && auxiliary->elementAccessFailure->callback) { handlers.elementAccessFailure = auxiliary->elementAccessFailure->callback; handlers.elementAccessFailureContext = auxiliary->elementAccessFailure->context; } @@ -340,7 +518,7 @@ void CopyObject(Object& destination, const Object& source) { try { for (const auto& pair : *source.items) { auto& copy = pending.items->emplace_back(); - Construct(copy.key, *pair.key.text); + Construct(copy.key, pair.key); CopyInto(copy.value, pair.value); } } catch (...) { @@ -369,7 +547,7 @@ void AssignNode(Node& destination, const Node& source) { Node copy{}; copy.type = source.type; try { - if (source.type == TypeString) Construct(copy.string, *source.string->text); + if (source.type == TypeString) Construct(copy.string, *source.string); else if (source.type == TypeArray) CopyArray(copy.array, source.array); else if (source.type == TypeObject) CopyObject(copy.object, source.object); else if (source.type == TypeReal) AssignReal(copy, source.real); @@ -410,14 +588,35 @@ void SetType(Node& n, ValueType type) { TransferPayload(n, pending); } +struct DefaultStorage { + Value values[8]{}; + bool ready[8]{}; +}; + +DefaultStorage& SharedDefaults() { + static DefaultStorage defaults; + return defaults; +} + const Value& Defaults(ValueType type) { - static Value values[8]{}; - static std::once_flag once[8]; - std::call_once(once[type], [type] { - Construct(values[type]); - SetType(values[type], type); - }); - return values[type]; + auto& state = State(); + std::lock_guard lock(state.mutex); + auto& defaults = SharedDefaults(); + if (!defaults.ready[type]) { + Construct(defaults.values[type]); + SetType(defaults.values[type], type); + defaults.ready[type] = true; + } + return defaults.values[type]; +} + +void ResetDefaults() { + auto& defaults = SharedDefaults(); + for (unsigned type = 0; type < 8; ++type) { + if (defaults.ready[type]) Destroy(defaults.values[type]); + defaults.values[type] = {}; + defaults.ready[type] = false; + } } const Node& Typed(const Value* self, ValueType requested) { @@ -442,11 +641,14 @@ const Node& Typed(const Value* self, ValueType requested) { if (auxiliary && auxiliary->nullAccess) return Typed(&auxiliary->nullAccess(requested, self->parent, auxiliary->nullAccessContext), requested); } + const auto* auxiliary = RootOf(*self).auxiliary; + if (auxiliary && auxiliary->typeMismatch && auxiliary->typeMismatch->callback) + return Typed(&auxiliary->typeMismatch->callback(requested, self, self->parent, auxiliary->typeMismatch->context), requested); + if (self->type == TypeNull && callback) + return Typed(&callback(requested, self->parent, context), requested); const auto handlers = HandlersOf(*self); if (handlers.typeMismatch) return Typed(&handlers.typeMismatch(requested, self, self->parent, handlers.typeMismatchContext), requested); - if (self->type == TypeNull && callback) - return Typed(&callback(requested, self->parent, context), requested); return Defaults(requested); } @@ -490,6 +692,20 @@ int SetHandler(Node& n, HandlerRecord* Auxiliary::* member, TCallback return 0; } +template +int SetGlobalHandler(void* self, TCallback Handlers::* callbackMember, void* Handlers::* contextMember, + TCallback callback, void* context, bool modernOnly = false) { + auto& state = State(); + std::lock_guard lock(state.mutex); + if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; + if (modernOnly && !state.floatFormatEnabled) return static_cast(0x80848113); + if (!callback) return JsonErrorInvalidArgument; + if (state.handlers.*callbackMember) return static_cast(0x80848112); + state.handlers.*callbackMember = callback; + state.handlers.*contextMember = context; + return 0; +} + const char* SpecialFloatText(double value) { return std::isnan(value) ? "nan" : value < 0 ? "-inf" : "inf"; } @@ -544,7 +760,7 @@ void AssignReal(Node& n, double value) { n.real = value; } -void AppendEscaped(std::string& out, const std::string& text) { +void AppendEscaped(std::string& out, std::string_view text) { out += '"'; for (const unsigned char c : text) { switch (c) { @@ -614,7 +830,7 @@ int Serialize(std::string& out, const Node& n) { AppendReal(out, n.real); return 0; } - case TypeString: AppendEscaped(out, *n.string->text); return 0; + case TypeString: AppendEscaped(out, View(*n.string)); return 0; case TypeArray: { out += '['; bool first = true; @@ -633,7 +849,7 @@ int Serialize(std::string& out, const Node& n) { for (const auto& pair : *n.object->items) { if (!first) out += ','; first = false; - AppendEscaped(out, *pair.key.text); + AppendEscaped(out, View(pair.key)); out += ':'; const int result = Serialize(out, pair.value); if (result < 0) return result; @@ -653,13 +869,13 @@ int CompareText(const char* left, const char* right) { return static_cast(*left) - static_cast(*right); } -std::list::iterator ObjectPosition(const Object& object, const std::string& key) { - return std::find_if(object.items->begin(), object.items->end(), [&](const Pair& pair) { return std::strcmp(pair.key.text->c_str(), key.c_str()) >= 0; }); +std::list::iterator ObjectPosition(const Object& object, std::string_view key) { + return std::find_if(object.items->begin(), object.items->end(), [&](const Pair& pair) { return std::strcmp(Characters(pair.key), key.data()) >= 0; }); } -Value& ObjectEntry(Object& object, const std::string& key) { +Value& ObjectEntry(Object& object, std::string_view key) { auto position = ObjectPosition(object, key); - if (position != object.items->end() && std::strcmp(position->key.text->c_str(), key.c_str()) == 0) return position->value; + if (position != object.items->end() && std::strcmp(Characters(position->key), key.data()) == 0) return position->value; Pair pair{}; Construct(pair.key, key); try { @@ -838,7 +1054,7 @@ private: std::string text; if (!string(text)) return false; SetType(out, TypeString); - *out.string->text = std::move(text); + AssignText(*out.string, text); return true; } case '[': { @@ -941,6 +1157,8 @@ int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* if (state.initialized || *static_cast(self)) return JsonErrorAlreadyInitialized; if (!parameter) return JsonErrorInvalidArgument; if (!*static_cast(parameter)) return JsonErrorOutOfMemory; + ResetDefaults(); + std::memcpy(&state.allocation, parameter, sizeof(state.allocation)); state.floatFormat = 2; state.floatFormatEnabled = false; state.initialized = true; @@ -954,6 +1172,9 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; state.callback = nullptr; state.context = nullptr; + state.handlers = {}; + ResetDefaults(); + state.allocation = {}; state.floatFormat = 2; state.floatFormatEnabled = false; state.initialized = false; @@ -976,6 +1197,18 @@ int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5Va return _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(self, callback, context); } +int APS5_VABI _ZN3sce4Json11Initializer28setGlobalTypeMismatchHandlerEPFRKNS0_5ValueENS0_9ValueTypeEPS3_S6_PvES7_(void* self, TypeMismatchHandler callback, void* context) { + return SetGlobalHandler(self, &Handlers::typeMismatch, &Handlers::typeMismatchContext, callback, context); +} + +int APS5_VABI _ZN3sce4Json11Initializer28setGlobalSpecialFloatHandlerEPFKNS0_5ValueENS0_12FunctionTypeEdPS3_PvPbES6_(void* self, SpecialFloatHandler callback, void* context) { + return SetGlobalHandler(self, &Handlers::specialFloat, &Handlers::specialFloatContext, callback, context, true); +} + +int APS5_VABI _ZN3sce4Json11Initializer36setGlobalElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKNS0_5ValueES9_PvESA_(void* self, ElementAccessFailureHandler callback, void* context) { + return SetGlobalHandler(self, &Handlers::elementAccessFailure, &Handlers::elementAccessFailureContext, callback, context); +} + void APS5_VABI _ZN3sce4Json12MemAllocatorC2Ev(void* self) { *static_cast(self) = &JsonMemAllocatorVtable[2]; } @@ -986,63 +1219,68 @@ void APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void* self) { std::size_t _ZN3sce4Json6String4nposE = static_cast(-1); -void APS5_VABI _ZN3sce4Json6StringC1Ev(String* self) { Construct(*self, std::string()); } -void APS5_VABI _ZN3sce4Json6StringC1EPKc(String* self, const char* text) { - if (text == nullptr) throw std::invalid_argument("sce::Json::String: null text"); - Construct(*self, text); -} -void APS5_VABI _ZN3sce4Json6StringC1ERKS1_(String* self, const String* other) { Construct(*self, *other->text); } +void APS5_VABI _ZN3sce4Json6StringC1Ev(String* self) { Construct(*self, std::string_view{}); } +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String* self, const char* text) { Construct(*self, text ? text : ""); } +void APS5_VABI _ZN3sce4Json6StringC1ERKS1_(String* self, const String* other) { Construct(*self, *other); } void APS5_VABI _ZN3sce4Json6StringD1Ev(String* self) { Destroy(*self); } String* APS5_VABI _ZN3sce4Json6StringaSERKS1_(String* self, const String* other) { - if (self != other) *self->text = *other->text; + if (self->text != other->text && other->text) AssignText(*self, View(*other), true); return self; } -const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String* self) { return self->text->c_str(); } -bool APS5_VABI _ZNK3sce4Json6String5emptyEv(const String* self) { return self->text->empty(); } -std::size_t APS5_VABI _ZNK3sce4Json6String6lengthEv(const String* self) { return self->text->size(); } -std::size_t APS5_VABI _ZNK3sce4Json6String4sizeEv(const String* self) { return self->text->size(); } -std::size_t APS5_VABI _ZNK3sce4Json6String8capacityEv(const String* self) { return self->text->capacity(); } -bool APS5_VABI _ZNK3sce4Json6StringeqEPKc(const String* self, const char* text) { return text != nullptr && CompareText(self->text->c_str(), text) == 0; } -bool APS5_VABI _ZNK3sce4Json6StringeqERKS1_(const String* self, const String* other) { return other != nullptr && other->text != nullptr && CompareText(self->text->c_str(), other->text->c_str()) == 0; } -void APS5_VABI _ZN3sce4Json6String5clearEv(String* self) { self->text->clear(); } -void APS5_VABI _ZN3sce4Json6String6resizeEm(String* self, std::size_t count) { self->text->resize(count); } -void APS5_VABI _ZN3sce4Json6String7reserveEm(String* self, std::size_t capacity) { self->text->reserve(capacity); } +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String* self) { return Characters(*self); } +bool APS5_VABI _ZNK3sce4Json6String5emptyEv(const String* self) { return View(*self).empty(); } +std::size_t APS5_VABI _ZNK3sce4Json6String6lengthEv(const String* self) { return View(*self).size(); } +std::size_t APS5_VABI _ZNK3sce4Json6String4sizeEv(const String* self) { return View(*self).size(); } +std::size_t APS5_VABI _ZNK3sce4Json6String8capacityEv(const String* self) { return self->text ? self->text->capacity : 0; } +bool APS5_VABI _ZNK3sce4Json6StringeqEPKc(const String* self, const char* text) { return text != nullptr && CompareText(Characters(*self), text) == 0; } +bool APS5_VABI _ZNK3sce4Json6StringeqERKS1_(const String* self, const String* other) { return other != nullptr && CompareText(Characters(*self), Characters(*other)) == 0; } +void APS5_VABI _ZN3sce4Json6String5clearEv(String* self) { ClearText(*self); } +void APS5_VABI _ZN3sce4Json6String6resizeEm(String* self, std::size_t count) { + if (!self->text || count == self->text->length) return; + auto& text = *self->text; + if (count < text.length) { text.characters[count] = 0; text.length = static_cast(count); } + else if (count < std::numeric_limits::max() && Grow(text, count + 1)) { + std::memset(text.characters + text.length, 0, count - text.length + 1); + text.length = static_cast(count); + } else if (count >= std::numeric_limits::max()) { + throw std::length_error("sce::Json::String: length exceeds 32 bits"); + } +} +void APS5_VABI _ZN3sce4Json6String7reserveEm(String* self, std::size_t capacity) { if (self->text) Grow(*self->text, capacity); } String* APS5_VABI _ZN3sce4Json6String6appendERKS1_(String* self, const String* other) { - const std::string copy = *other->text; - self->text->append(copy); + AppendText(*self, Characters(*other)); return self; } String* APS5_VABI _ZN3sce4Json6String6appendEPKc(String* self, const char* text) { - if (text == nullptr) throw std::invalid_argument("sce::Json::String::append: null text"); - self->text->append(text); + if (!text) throw std::invalid_argument("sce::Json::String::append: null text"); + AppendText(*self, text); return self; } String* APS5_VABI _ZN3sce4Json6String6appendEPKcm(String* self, const char* text, std::size_t length) { - if (text == nullptr && length > 0) throw std::invalid_argument("sce::Json::String::append: null text"); - self->text->append(text, length); + if (!text && length) throw std::invalid_argument("sce::Json::String::append: null text"); + AppendText(*self, {text ? text : "", length}); return self; } -String* APS5_VABI _ZN3sce4Json6StringpLEPKc(String* self, const char* text) { - return _ZN3sce4Json6String6appendEPKc(self, text); -} +String* APS5_VABI _ZN3sce4Json6StringpLEPKc(String* self, const char* text) { return _ZN3sce4Json6String6appendEPKc(self, text); } String* APS5_VABI _ZN3sce4Json6StringpLEh(String* self, unsigned char ch) { - self->text->push_back(static_cast(ch)); + const char text[]{static_cast(ch), 0}; + AppendText(*self, text); return self; } String* APS5_VABI _ZNK3sce4Json6String6substrEmm(String* result, const String* self, std::size_t position, std::size_t length) { - if (position > self->text->size()) throw std::out_of_range("sce::Json::String::substr: position past the end"); - Construct(*result, self->text->substr(position, length)); + if (position > View(*self).size()) throw std::out_of_range("sce::Json::String::substr: position past the end"); + Construct(*result, View(*self).substr(position, length)); return result; } -std::size_t APS5_VABI _ZNK3sce4Json6String4findEcm(const String* self, char character, std::size_t position) { return self->text->find(character, position); } -std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEcm(const String* self, char character, std::size_t position) { return self->text->rfind(character, position); } +std::size_t APS5_VABI _ZNK3sce4Json6String4findEcm(const String* self, char character, std::size_t position) { return View(*self).find(character, position); } +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEcm(const String* self, char character, std::size_t position) { return View(*self).rfind(character, position); } int APS5_VABI _ZNK3sce4Json6String7compareEPKc(const String* self, const char* text) { - if (text == nullptr) throw std::invalid_argument("sce::Json::String::compare: null text"); - return CompareText(self->text->c_str(), text); + if (!text) throw std::invalid_argument("sce::Json::String::compare: null text"); + return CompareText(Characters(*self), text); } int APS5_VABI _ZNK3sce4Json6String7compareERKS1_(const String* self, const String* other) { - if (other == nullptr || other->text == nullptr) throw std::invalid_argument("sce::Json::String::compare: null other"); - return CompareText(self->text->c_str(), other->text->c_str()); + if (!other) throw std::invalid_argument("sce::Json::String::compare: null other"); + return CompareText(Characters(*self), Characters(*other)); } void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array* self) { Construct(*self); } @@ -1129,7 +1367,7 @@ void APS5_VABI _ZN3sce4Json6Object5clearEv(Object* self) { } self->items->clear(); } -Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object* self, const String* key) { return &ObjectEntry(*self, *key->text); } +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object* self, const String* key) { return &ObjectEntry(*self, View(*key)); } ObjectIterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(ObjectIterator* result, const Object* self) { result->state = new ObjectIteratorState{self->items, self->items->begin()}; return result; } ObjectIterator* APS5_VABI _ZNK3sce4Json6Object3endEv(ObjectIterator* result, const Object* self) { result->state = new ObjectIteratorState{self->items, self->items->end()}; return result; } void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(ObjectIterator* self) { delete self->state; self->state = nullptr; } @@ -1146,7 +1384,7 @@ Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const ObjectIterator* self) { } Pair* APS5_VABI _ZNK3sce4Json6Object8iteratorptEv(const ObjectIterator* self) { return _ZNK3sce4Json6Object8iteratordeEv(self); } ObjectIterator* APS5_VABI _ZNK3sce4Json6Object4findERKNS0_6StringE(ObjectIterator* result, const Object* self, const String* key) { - const auto position = std::find_if(self->items->begin(), self->items->end(), [&](const Pair& pair) { return std::strcmp(pair.key.text->c_str(), key->text->c_str()) == 0; }); + const auto position = std::find_if(self->items->begin(), self->items->end(), [&](const Pair& pair) { return std::strcmp(Characters(pair.key), Characters(*key)) == 0; }); result->state = new ObjectIteratorState{self->items, position}; return result; } @@ -1174,7 +1412,7 @@ bool APS5_VABI _ZNK3sce4Json6Object5emptyEv(const Object* self) { return self->i std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object* self) { return self->items->size(); } void APS5_VABI _ZN3sce4Json6Object5eraseERKNS0_6StringE(Object* self, const String* key) { if (key == nullptr || key->text == nullptr) throw std::invalid_argument("sce::Json::Object::erase: null key"); - auto it = std::find_if(self->items->begin(), self->items->end(), [&](const Pair& pair) { return std::strcmp(pair.key.text->c_str(), key->text->c_str()) == 0; }); + auto it = std::find_if(self->items->begin(), self->items->end(), [&](const Pair& pair) { return std::strcmp(Characters(pair.key), Characters(*key)) == 0; }); if (it == self->items->end()) return; Destroy(it->key); Destroy(it->value); @@ -1190,13 +1428,13 @@ void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value: null text"); Construct(*self, [text](Node& node) { SetType(node, TypeString); - *node.string->text = text; + AssignText(*node.string, text); }); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6StringE(Value* self, const String* text) { Construct(*self, [text](Node& node) { SetType(node, TypeString); - *node.string->text = *text->text; + AssignText(*node.string, View(*text), true); }); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value* self, const Array* array) { @@ -1239,13 +1477,13 @@ int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value::set: null text"); const std::string copy(text); SetType(NodeOf(*self), TypeString); - *self->string->text = copy; + AssignText(*self->string, copy); return 0; } int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { - const std::string copy(*text->text); + const std::string copy(View(*text)); SetType(NodeOf(*self), TypeString); - *self->string->text = copy; + AssignText(*self->string, copy, true); return 0; } int APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { @@ -1306,7 +1544,7 @@ const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value* self, const char* try { if (self->type == TypeObject) { for (const auto& pair : *self->object->items) - if (std::strcmp(pair.key.text->c_str(), text.text->c_str()) == 0) { + if (std::strcmp(Characters(pair.key), Characters(text)) == 0) { Destroy(text); return &pair.value; } @@ -1341,7 +1579,7 @@ const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value* const Node& n = NodeOf(*self); if (key != nullptr && n.type == TypeObject) for (const auto& pair : *n.object->items) - if (std::strcmp(pair.key.text->c_str(), key->text->c_str()) == 0) return &pair.value; + if (std::strcmp(Characters(pair.key), Characters(*key)) == 0) return &pair.value; return &MissingElement(self, static_cast(n.type == TypeObject ? 0x80848115 : 0x80848117), key, nullptr); } Value* APS5_VABI _ZN3sce4Json5Value10referValueEm(Value* self, std::size_t index) { @@ -1361,7 +1599,7 @@ Value* APS5_VABI _ZN3sce4Json5Value10referValueERKNS0_6StringE(Value* self, cons Node& n = NodeOf(*self); if (key == nullptr || n.type != TypeObject) return nullptr; for (auto& pair : *n.object->items) - if (std::strcmp(pair.key.text->c_str(), key->text->c_str()) == 0) return &pair.value; + if (std::strcmp(Characters(pair.key), Characters(*key)) == 0) return &pair.value; return nullptr; } Array* APS5_VABI _ZN3sce4Json5Value10referArrayEv(Value* self) { @@ -1379,15 +1617,19 @@ Object* APS5_VABI _ZN3sce4Json5Value11referObjectEv(Value* self) { int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value* self, String* out) { std::string text; const int result = Serialize(text, *self); - out->text->append(text); + AppendText(*out, text); return result; } void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value* self, String* out) { - if (self->type == TypeString) out->text->append(*self->string->text); - else if (self->type == TypeArray) out->text->append("array"); - else if (self->type == TypeObject) out->text->append("object"); - else if (self->type == TypeReal) AppendReal(*out->text, self->real); - else Serialize(*out->text, *self); + std::string text; + if (self->type == TypeString) AppendText(*out, Characters(*self->string)); + else { + if (self->type == TypeArray) text = "array"; + else if (self->type == TypeObject) text = "object"; + else if (self->type == TypeReal) AppendReal(text, self->real); + else Serialize(text, *self); + AppendText(*out, text); + } } @@ -1405,6 +1647,22 @@ int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value* out, const char* } +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKc(Value* out, const char* path) { + AllocationContext allocation; + { + auto& state = State(); + std::lock_guard lock(state.mutex); + if (!state.initialized) return JsonErrorNotInitialized; + allocation = state.allocation; + } + if (!allocation.fileBufferSize) return static_cast(0x80848105); + FileReader reader{allocation}; + reader.buffer = Allocate(allocation, allocation.fileBufferSize); + if (reader.buffer) reader.descriptor = sceKernelOpen(path, 0, 0555); + Parser parser(FileReader::Read, &reader); + return ParseValue(out, parser); +} + struct InitParameterRtti { void* allocParam; void* userData; @@ -1473,6 +1731,8 @@ int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void if (state.initialized || *static_cast(self)) return JsonErrorAlreadyInitialized; if (!parameter || !parameter->allocator || static_cast(parameter->specialFloatFormatType) > 2) return JsonErrorInvalidArgument; + ResetDefaults(); + state.allocation = {parameter->allocator, parameter->userData, parameter->fileBufferSize}; state.floatFormat = parameter->specialFloatFormatType; state.floatFormatEnabled = true; state.initialized = true; @@ -1537,7 +1797,7 @@ bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value* self) { case TypeInteger: return n.integer != 0; case TypeUInteger: return n.uinteger != 0; case TypeReal: return n.real != 0.0; - case TypeString: return n.string->text && !n.string->text->empty(); + case TypeString: return !View(*n.string).empty(); case TypeArray: case TypeObject: return true; default: throw std::runtime_error("sce::Json::Value: invalid value type"); @@ -1547,7 +1807,7 @@ bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value* self) { void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair* self, const String* key, const Value* value) { Pair pair{}; pair.reserved = 0; - Construct(pair.key, *key->text); + Construct(pair.key, *key); try { CopyInto(pair.value, *value); } catch (...) { @@ -1564,15 +1824,15 @@ void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair* self) { } ObjectIterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(ObjectIterator* result, Object* self, const Pair* pair) { - auto it = ObjectPosition(*self, *pair->key.text); + auto it = ObjectPosition(*self, View(pair->key)); auto state = std::make_unique(ObjectIteratorState{self->items, it}); - if (it != self->items->end() && std::strcmp(it->key.text->c_str(), pair->key.text->c_str()) == 0) { + if (it != self->items->end() && std::strcmp(Characters(it->key), Characters(pair->key)) == 0) { result->state = state.release(); return result; } Pair copy{}; copy.reserved = 0; - Construct(copy.key, *pair->key.text); + Construct(copy.key, pair->key); try { CopyInto(copy.value, pair->value); state->position = self->items->insert(it, copy); diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index 49a64ecff5..d6d5c04ae5 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -934,6 +934,9 @@ static void ValueAccess() { _ZN3sce4Json6StringD1Ev(&missing); _ZN3sce4Json5ValueD1Ev(&root); +} + +static void ValueAccessParameters() { alignas(16) std::uint8_t parameter[40]; std::memset(parameter, 0xff, sizeof(parameter)); _ZN3sce4Json14InitParameter2C1Ev(parameter); @@ -1369,7 +1372,21 @@ static void ValueOperatorBool() { _ZN3sce4Json5ValueD1Ev(&cleared); } +struct ActiveModule { + JsonTest::Allocator allocator; + alignas(8) std::uint8_t parameter[40]{}; + std::uint8_t initializer = 0; + ActiveModule() { + _ZN3sce4Json14InitParameter2C1Ev(parameter); + _ZN3sce4Json14InitParameter212setAllocatorEPNS0_12MemAllocatorEPv(parameter, &allocator, nullptr); + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, parameter) == 0); + } + ~ActiveModule() { Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0); } +}; + static int NumericLocale() { + ActiveModule module; const std::string previous = std::setlocale(LC_NUMERIC, nullptr); const char* selected = nullptr; for (const char* name : {"xx_COMMA", "de_DE.UTF-8", "de_DE.utf8", "de-DE", "German_Germany.1252", "fr_FR.UTF-8"}) { @@ -1441,22 +1458,25 @@ static int NumericLocale() { int main(int argc, char** argv) { if (argc == 2 && std::strcmp(argv[1], "--numeric-locale") == 0) return NumericLocale(); - ParseAndRoundTrip(); - NestingDepth(); - ObjectsAndArrays(); - ObjectFind(); - ObjectPair(); - ObjectInsert(); - ObjectKeyOrder(); - ArrayIteratorCopies(); - ObjectIteratorCopies(); - AllocatorNotifyError(); - StringMethods(); - StringComparison(); - ParseDuplicateKeys(); - ValueSwap(); - ArrayIteratorAdvance(); - ObjectOperations(); + { + ActiveModule module; + ParseAndRoundTrip(); + NestingDepth(); + ObjectsAndArrays(); + ObjectFind(); + ObjectPair(); + ObjectInsert(); + ObjectKeyOrder(); + ArrayIteratorCopies(); + ObjectIteratorCopies(); + AllocatorNotifyError(); + StringMethods(); + StringComparison(); + ParseDuplicateKeys(); + ValueSwap(); + ArrayIteratorAdvance(); + ObjectOperations(); + } NullAccess(_ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_); NullAccess(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_); JsonTest::Allocator recoveryAllocator; @@ -1471,11 +1491,15 @@ int main(int argc, char** argv) { ValueElementAccessFailureHandler(); ValueSpecialFloatHandler(); Require(_ZN3sce4Json11Initializer9terminateEv(&recoveryInitializer) == 0); - ValueAccess(); InitParameter2BaseConstructor(); InitParameters(); - ValueClear(); - ValueOperatorBool(); - IntegerRepresentation(); - StreamParse(); + ValueAccessParameters(); + { + ActiveModule module; + ValueAccess(); + ValueClear(); + ValueOperatorBool(); + IntegerRepresentation(); + StreamParse(); + } } diff --git a/core/libs/tests/GuestJsonFile.cpp b/core/libs/tests/GuestJsonFile.cpp new file mode 100644 index 0000000000..f0e39867c7 --- /dev/null +++ b/core/libs/tests/GuestJsonFile.cpp @@ -0,0 +1,134 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Value { alignas(8) std::uint64_t words[4]{}; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKc(Value*, const char*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct FileTrace { + std::string_view input; + std::size_t position = 0; + std::size_t chunk = 1; + std::size_t bufferSize = 997; + int descriptor = 99; + std::int64_t readError = 0; + int opens = 0; + int reads = 0; + int closes = 0; + int bufferAllocations = 0; + int bufferFrees = 0; + int notifications = 0; + void* buffer = nullptr; + bool failBuffer = false; +}; +static FileTrace* activeFile = nullptr; +extern "C" int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) { + auto& trace = *activeFile; + Require(std::strcmp(path, "input.json") == 0 && flags == 0 && mode == 0555, "File parser must pass its filename and native open arguments"); + Require(trace.buffer != nullptr, "Open must follow successful file-buffer allocation"); + ++trace.opens; + return trace.descriptor; +} +extern "C" std::int64_t APS5_VABI sceKernelRead(int descriptor, void* output, std::size_t bytes) { + auto& trace = *activeFile; + Require(descriptor == trace.descriptor && output == trace.buffer && bytes == trace.bufferSize, "Read must use the configured buffer and descriptor"); + ++trace.reads; + if (trace.readError) return trace.readError; + const auto count = std::min({bytes, trace.chunk, trace.input.size() - trace.position}); + std::memcpy(output, trace.input.data() + trace.position, count); + trace.position += count; + return static_cast(count); +} +extern "C" int APS5_VABI sceKernelClose(int descriptor) { + auto& trace = *activeFile; + Require(descriptor == trace.descriptor && trace.buffer != nullptr, "Close must precede release of the file buffer"); + ++trace.closes; + return 0; +} +static void* APS5_VABI AllocateFile(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + if (bytes == trace.bufferSize) { + ++trace.bufferAllocations; + if (trace.failBuffer) return nullptr; + Require(!trace.buffer, "Only one live file buffer per parse"); + trace.buffer = std::malloc(bytes); + return trace.buffer; + } + return std::malloc(bytes); +} +static void APS5_VABI FreeFile(void*, void* memory, void* context) { + auto& trace = *static_cast(context); + if (memory && memory == trace.buffer) { ++trace.bufferFrees; trace.buffer = nullptr; } + std::free(memory); +} +static void APS5_VABI NotifyFile(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && bytes == trace.bufferSize, "File allocation failure must notify its exact size and OOM status"); + ++trace.notifications; +} +static void* const FileVtable[]{nullptr, nullptr, reinterpret_cast(AllocateFile), reinterpret_cast(FreeFile), reinterpret_cast(NotifyFile)}; +struct FileAllocator { void* const* vtable = FileVtable; }; +static void CheckParse(FileTrace& trace, std::string_view input, int descriptor, std::int64_t error, int expected) { + trace.input = input; trace.position = 0; trace.descriptor = descriptor; trace.readError = error; + const int opens = trace.opens, closes = trace.closes, frees = trace.bufferFrees; + Value value{}; + _ZN3sce4Json5ValueC1Ev(&value); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKc(&value, "input.json") == expected, "File parse must preserve success, syntax, open and read results"); + Require(trace.opens == opens + 1 && trace.closes == closes + (descriptor >= 0 ? 1 : 0) && trace.bufferFrees == frees + 1 && !trace.buffer, + "Every completed file parse must close its owned descriptor and release its buffer exactly once"); + if (!expected) Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5ValueixEPKc(&value, "answer")) == 42, "Short reads must assemble the parsed result"); + _ZN3sce4Json5ValueD1Ev(&value); +} +int main() { + FileTrace trace; activeFile = &trace; + FileAllocator allocator; + Parameter parameter{&allocator, &trace, trace.bufferSize, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + Require(static_cast(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKc(&value, "input.json")) == 0x80848110 && !trace.opens && !trace.bufferAllocations, "Inactive file parser must refuse before allocation or open"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize file parser"); + parameter.bufferSize = 3; + for (std::size_t chunk : {std::size_t{1}, std::size_t{7}, trace.bufferSize}) { + trace.chunk = chunk; + CheckParse(trace, "{\"answer\":42}", 99, 0, 0); + } + CheckParse(trace, "", 99, 0, static_cast(0x80848101)); + CheckParse(trace, "{\"answer\":", 99, 0, static_cast(0x80848101)); + CheckParse(trace, "", static_cast(0x80020002), 0, static_cast(0x80848103)); + CheckParse(trace, "", static_cast(0x8002000d), 0, static_cast(0x8002000d)); + CheckParse(trace, "", -1, 0, static_cast(0x80848110)); + CheckParse(trace, "", 99, static_cast(0x80020005), static_cast(0x80020005)); + trace.failBuffer = true; + const int opens = trace.opens, closes = trace.closes, frees = trace.bufferFrees; + Require(static_cast(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKc(&value, "input.json")) == 0x80848110 && trace.notifications == 1 && trace.opens == opens && trace.closes == closes && trace.bufferFrees == frees, + "Failed file-buffer allocation must notify, avoid open and preserve the native parser status"); + trace.failBuffer = false; + CheckParse(trace, "{\"answer\":42}", 99, 0, 0); + _ZN3sce4Json5ValueD1Ev(&value); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate file parser"); + parameter.bufferSize = 0; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize without a file buffer"); + _ZN3sce4Json5ValueC1Ev(&value); + Require(static_cast(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKc(&value, "input.json")) == 0x80848105 && trace.opens == opens + 1, "Zero configured file-buffer size must refuse before open"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate zero-buffer configuration"); + return 0; +} diff --git a/core/libs/tests/GuestJsonFloatFailure.cpp b/core/libs/tests/GuestJsonFloatFailure.cpp index c10c8c2808..51472898ba 100644 --- a/core/libs/tests/GuestJsonFloatFailure.cpp +++ b/core/libs/tests/GuestJsonFloatFailure.cpp @@ -172,7 +172,9 @@ int main() { "Finite assignment must not require a conversion allocation"); _ZN3sce4Json5ValueD1Ev(&value); Require(liveAllocations == baseline, "Destroy must reclaim the retained value"); - for (int budget : {0, 1}) { + bool constructed = false; + unsigned constructorFailures = 0; + for (int budget = 0; budget < 64 && !constructed; ++budget) { Value destination; destination.bytes.fill(std::byte{0xa5}); const auto original = destination.bytes; @@ -184,9 +186,19 @@ int main() { failed = true; } allocationBudget = -1; - Require(failed && destination.bytes == original && liveAllocations == baseline, - "Failed construction must leave destination storage unchanged and reclaim partial state"); + if (failed) { + ++constructorFailures; + Require(destination.bytes == original && liveAllocations == baseline, + "Failed construction must leave destination storage unchanged and reclaim partial state"); + } else { + constructed = true; + Require(_ZNK3sce4Json5Value7getTypeEv(&destination) == 5, + "Construction must remain usable after earlier host allocation failures"); + _ZN3sce4Json5ValueD1Ev(&destination); + Require(liveAllocations == baseline, "Successful construction must release its host ownership"); + } } + Require(constructed && constructorFailures > 0, "Exercise failed and successful host construction"); _ZN3sce4Json5ValueC1Ed(&value, std::numeric_limits::infinity()); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == 5, "Conversion must remain usable after allocation failure"); _ZN3sce4Json5ValueD1Ev(&value); diff --git a/core/libs/tests/GuestJsonGlobalRecovery.cpp b/core/libs/tests/GuestJsonGlobalRecovery.cpp new file mode 100644 index 0000000000..4609355981 --- /dev/null +++ b/core/libs/tests/GuestJsonGlobalRecovery.cpp @@ -0,0 +1,122 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using Mismatch = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +using Null = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using Special = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +using Element = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +int APS5_VABI _ZN3sce4Json11Initializer28setGlobalTypeMismatchHandlerEPFRKNS0_5ValueENS0_9ValueTypeEPS3_S6_PvES7_(void*, Mismatch, void*); +int APS5_VABI _ZN3sce4Json11Initializer28setGlobalSpecialFloatHandlerEPFKNS0_5ValueENS0_12FunctionTypeEdPS3_PvPbES6_(void*, Special, void*); +int APS5_VABI _ZN3sce4Json11Initializer36setGlobalElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKNS0_5ValueES9_PvESA_(void*, Element, void*); +int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void*, Null, void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, Mismatch, void*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, Special, void*); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, Element, void*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct CallbackTrace { int calls = 0; const Value* source = nullptr; const Value* parent = nullptr; const Value* replacement = nullptr; bool reenter = false; int function = -1; }; +static const Value& APS5_VABI RecoverMismatch(std::int32_t requested, const Value* source, const Value* parent, void* context) { + auto& trace = *static_cast(context); + ++trace.calls; trace.source = source; trace.parent = parent; + if (trace.reenter) Require(*_ZNK3sce4Json5Value10getIntegerEv(source) == 0, "Global mismatch reentry must use the typed default"); + Require(requested == 1 || requested == 2, "Preserve requested type"); + return *trace.replacement; +} +static const Value& APS5_VABI RecoverNull(std::int32_t, const Value*, void* context) { + auto& trace = *static_cast(context); ++trace.calls; return *trace.replacement; +} +static Value* APS5_VABI RecoverFloat(Value* result, std::int32_t function, double, const Value* parent, void* context, bool* error) { + auto& trace = *static_cast(context); + ++trace.calls; trace.function = function; trace.parent = parent; + Require(function == 1 && !error, "Assignment passes its function kind and no error output"); + _ZN3sce4Json5ValueC1El(result, trace.calls + 100); + return result; +} +static void APS5_VABI RecoverElement(int error, const String* key, const std::size_t* index, const Value* parent, const Value* source, void* context) { + auto& trace = *static_cast(context); + ++trace.calls; trace.source = source; trace.parent = parent; + Require(static_cast(error) == 0x80848118 && !key && index && *index == 9, "Global element recovery receives lookup details"); + if (trace.reenter) _ZNK3sce4Json5ValueixEm(source, 9); +} +int main() { + JsonTest::Allocator allocator; + Parameter parameter{&allocator, nullptr, 16384, 2, {}}; + std::uint8_t initializer = 0, inactive = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Value root{}, replacement{}, falseValue{}; + _ZN3sce4Json5ValueC1Ev(&root); + _ZN3sce4Json5ValueC1El(&replacement, 77); + _ZN3sce4Json5ValueC1Ev(&falseValue); + CallbackTrace mismatch{0, nullptr, nullptr, &replacement, true}; + CallbackTrace local{0, nullptr, nullptr, &falseValue, false}; + CallbackTrace null{0, nullptr, nullptr, &replacement, false}; + CallbackTrace special{}, element{}; element.reenter = true; + auto registerMismatch = [&](void* self, Mismatch callback, void* context) { return _ZN3sce4Json11Initializer28setGlobalTypeMismatchHandlerEPFRKNS0_5ValueENS0_9ValueTypeEPS3_S6_PvES7_(self, callback, context); }; + auto registerSpecial = [&](void* self, Special callback, void* context) { return _ZN3sce4Json11Initializer28setGlobalSpecialFloatHandlerEPFKNS0_5ValueENS0_12FunctionTypeEdPS3_PvPbES6_(self, callback, context); }; + auto registerElement = [&](void* self, Element callback, void* context) { return _ZN3sce4Json11Initializer36setGlobalElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKNS0_5ValueES9_PvESA_(self, callback, context); }; + Require(static_cast(registerMismatch(&initializer, RecoverMismatch, &mismatch)) == 0x80848110 && + static_cast(registerSpecial(&initializer, RecoverFloat, &special)) == 0x80848110 && + static_cast(registerElement(&initializer, RecoverElement, &element)) == 0x80848110, "Global handlers require initialization"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize global recovery"); + Require(static_cast(registerMismatch(&inactive, RecoverMismatch, &mismatch)) == 0x80848110 && + static_cast(registerSpecial(&inactive, RecoverFloat, &special)) == 0x80848110 && + static_cast(registerElement(&inactive, RecoverElement, &element)) == 0x80848110, "Inactive initializer cannot register handlers"); + Require(static_cast(registerMismatch(&initializer, nullptr, nullptr)) == 0x80848120 && + static_cast(registerSpecial(&initializer, nullptr, nullptr)) == 0x80848120 && + static_cast(registerElement(&initializer, nullptr, nullptr)) == 0x80848120, "Global handlers reject null callbacks"); + Require(registerMismatch(&initializer, RecoverMismatch, &mismatch) == 0 && registerSpecial(&initializer, RecoverFloat, &special) == 0 && registerElement(&initializer, RecoverElement, &element) == 0, "Register all global handlers"); + Require(static_cast(registerMismatch(&initializer, RecoverMismatch, &local)) == 0x80848112 && + static_cast(registerSpecial(&initializer, RecoverFloat, &local)) == 0x80848112 && + static_cast(registerElement(&initializer, RecoverElement, &local)) == 0x80848112, "Duplicate registration preserves original contexts"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 77 && mismatch.calls == 1 && mismatch.source == &root && !mismatch.parent, "Global mismatch recovers an unowned null value"); + Require(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(&initializer, RecoverNull, &null) == 0, "Register global null callback"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 77 && null.calls == 1 && mismatch.calls == 1, "Global null callback precedes global mismatch for null getters"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&root, RecoverMismatch, &local) == 0, "Register local mismatch"); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 0 && local.calls == 1 && null.calls == 1 && mismatch.calls == 1, "Local mismatch overrides both global callbacks"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&root, nullptr, nullptr) == 0, "Unregister local mismatch"); + _ZN3sce4Json5Value3setEd(&root, std::numeric_limits::infinity()); + Require(special.calls == 1 && * _ZNK3sce4Json5Value10getIntegerEv(&root) == 101, "Global float handler supplies assignment replacement"); + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&root, RecoverFloat, &local) == 0, "Register local float handler"); + _ZN3sce4Json5Value3setEd(&root, std::numeric_limits::infinity()); + Require(special.calls == 1 && local.calls == 2, "Local float handler overrides global recovery"); + const auto* fallback = _ZNK3sce4Json5ValueixEm(&root, 9); + Require(element.calls == 1 && element.source == &root && !element.parent && fallback->type == 0, "Global element handler receives failed access and suppresses reentry"); + _ZNK3sce4Json5ValueixEm(fallback, 9); + Require(element.calls == 1, "Fallback access must not recursively invoke global element recovery"); + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, RecoverElement, &local) == 0, "Register local element handler"); + _ZNK3sce4Json5ValueixEm(&root, 9); + Require(element.calls == 1 && local.calls == 3, "Local element handler overrides global recovery"); + _ZN3sce4Json5ValueD1Ev(&root); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate global recovery"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Reinitialize after global recovery"); + Require(registerMismatch(&initializer, RecoverMismatch, &mismatch) == 0 && registerSpecial(&initializer, RecoverFloat, &special) == 0 && registerElement(&initializer, RecoverElement, &element) == 0, "Termination clears all global registration slots"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate second initialization"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&initializer, ¶meter) == 0, "Legacy initialization"); + Require(static_cast(registerSpecial(&initializer, nullptr, nullptr)) == 0x80848113 && static_cast(registerSpecial(&initializer, RecoverFloat, &special)) == 0x80848113, "Legacy initialization rejects global float registration before callback validation"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate legacy initialization"); + _ZN3sce4Json5ValueD1Ev(&replacement); + _ZN3sce4Json5ValueD1Ev(&falseValue); + return 0; +} diff --git a/core/libs/tests/GuestJsonStringAllocator.cpp b/core/libs/tests/GuestJsonStringAllocator.cpp new file mode 100644 index 0000000000..4571dcf67e --- /dev/null +++ b/core/libs/tests/GuestJsonStringAllocator.cpp @@ -0,0 +1,124 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct String { void* storage; }; +struct Value { alignas(8) std::uint64_t words[4]{}; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringC1ERKS1_(String*, const String*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +std::size_t APS5_VABI _ZNK3sce4Json6String4sizeEv(const String*); +std::size_t APS5_VABI _ZNK3sce4Json6String8capacityEv(const String*); +void APS5_VABI _ZN3sce4Json6String7reserveEm(String*, std::size_t); +void APS5_VABI _ZN3sce4Json6String6resizeEm(String*, std::size_t); +void APS5_VABI _ZN3sce4Json6String5clearEv(String*); +String* APS5_VABI _ZN3sce4Json6StringaSERKS1_(String*, const String*); +String* APS5_VABI _ZN3sce4Json6String6appendEPKcm(String*, const char*, std::size_t); +String* APS5_VABI _ZN3sce4Json6String6appendERKS1_(String*, const String*); +String* APS5_VABI _ZN3sce4Json6StringpLEh(String*, unsigned char); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct StringTrace { + std::map live; + std::vector requests; + int budget = -1; + int notifications = 0; +}; +static void* APS5_VABI AllocateString(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) return nullptr; + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeString(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + Require(trace.live.erase(memory) == 1, "String must free through its owning allocation context"); std::free(memory); +} +static void APS5_VABI NotifyString(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "String failure notification must identify the failed request"); ++trace.notifications; +} +static void* const StringVtable[]{nullptr, nullptr, reinterpret_cast(AllocateString), reinterpret_cast(FreeString), reinterpret_cast(NotifyString)}; +struct StringAllocator { void* const* vtable = StringVtable; }; +static void CheckText(const String& text, const char* bytes, std::size_t length, std::size_t capacity) { + Require(_ZNK3sce4Json6String4sizeEv(&text) == length && _ZNK3sce4Json6String8capacityEv(&text) == capacity && + std::memcmp(_ZNK3sce4Json6String5c_strEv(&text), bytes, length + 1) == 0, "String bytes, length and capacity must match the reference"); +} +int main() { + Value beforeInitialization{}; + _ZN3sce4Json5ValueC1Ev(&beforeInitialization); + _ZNK3sce4Json5Value9getStringEv(&beforeInitialization); + _ZN3sce4Json5ValueD1Ev(&beforeInitialization); + StringTrace trace; + StringAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize string allocation contract"); + const auto baseline = trace.live.size(); + for (int budget : {0, 1}) { + trace.budget = budget; + String failed{}; _ZN3sce4Json6StringC1EPKc(&failed, "abc"); + Require(budget ? trace.live.count(failed.storage) == 1 : failed.storage == nullptr, "A header failure and a character failure retain distinct partial ownership"); + CheckText(failed, "", 0, 0); + trace.budget = -1; + _ZN3sce4Json6StringD1Ev(&failed); + Require(!failed.storage && trace.live.size() == baseline, "Failed string construction must be destructible without leaking"); + } + String text{}; _ZN3sce4Json6StringC1EPKc(&text, "abc"); + Require(trace.live.at(text.storage) == 24 && trace.live.at(*static_cast(text.storage)) == 4, "Actual string storage must use native allocator-backed header and characters"); + for (unsigned i = 16; i < 24; ++i) Require(static_cast(text.storage)[i] == 0xa5, "String construction must preserve reserved header bytes"); + CheckText(text, "abc", 3, 4); + trace.budget = 0; + _ZN3sce4Json6String7reserveEm(&text, 17); CheckText(text, "abc", 3, 4); + _ZN3sce4Json6String6resizeEm(&text, 18); CheckText(text, "abc", 3, 4); + trace.budget = -1; + _ZN3sce4Json6String7reserveEm(&text, 17); CheckText(text, "abc", 3, 17); + _ZN3sce4Json6String6resizeEm(&text, 18); + char grown[19]{}; std::memcpy(grown, "abc", 3); CheckText(text, grown, 18, 25); + _ZN3sce4Json6String6resizeEm(&text, 2); CheckText(text, "ab", 2, 25); + _ZN3sce4Json6String5clearEv(&text); CheckText(text, "", 0, 0); + const char binary[]{'a', 0, 'b', 0}; + _ZN3sce4Json6String6appendEPKcm(&text, binary, 3); CheckText(text, binary, 3, 20); + String copy{}; _ZN3sce4Json6StringC1ERKS1_(©, &text); CheckText(copy, binary, 3, 4); + String appended{}; _ZN3sce4Json6StringC1EPKc(&appended, nullptr); + _ZN3sce4Json6String6appendERKS1_(&appended, &text); CheckText(appended, "a", 1, 18); + _ZN3sce4Json6StringpLEh(&appended, 0); CheckText(appended, "a", 1, 18); + _ZN3sce4Json6StringaSERKS1_(&appended, &text); CheckText(appended, binary, 3, 4); + trace.budget = 0; + _ZN3sce4Json6StringaSERKS1_(&appended, ©); CheckText(appended, "", 0, 0); + trace.budget = -1; + Require(trace.notifications == 5, "Each header, character, growth and assignment failure must notify once"); + _ZN3sce4Json6StringD1Ev(&appended); _ZN3sce4Json6StringD1Ev(©); _ZN3sce4Json6StringD1Ev(&text); + Require(trace.live.size() == baseline, "String operations must release all owned resources"); + Value null{}; _ZN3sce4Json5ValueC1Ev(&null); + const auto* defaultString = _ZNK3sce4Json5Value9getStringEv(&null); + Require(defaultString && defaultString->storage && trace.live.at(defaultString->storage) == 24, "The default string header must belong to the active allocator"); + _ZN3sce4Json5ValueD1Ev(&null); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate string allocation contract"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Reinitialize default string ownership"); + _ZN3sce4Json5ValueC1Ev(&null); + defaultString = _ZNK3sce4Json5Value9getStringEv(&null); + Require(defaultString && defaultString->storage && trace.live.at(defaultString->storage) == 24, "A new initialization must own its own default string header"); + _ZN3sce4Json5ValueD1Ev(&null); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Release defaults from repeated initialization"); + return 0; +} diff --git a/core/libs/tests/JsonCopyFailure.cpp b/core/libs/tests/JsonCopyFailure.cpp index 90f4ab421b..a7ea79512e 100644 --- a/core/libs/tests/JsonCopyFailure.cpp +++ b/core/libs/tests/JsonCopyFailure.cpp @@ -1,4 +1,5 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" #include #include #include @@ -15,6 +16,9 @@ struct Iterator { void* state; }; struct Pair { String key; std::uint64_t reserved; Value value; }; extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const void*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); @@ -170,6 +174,11 @@ static bool Run(Operation operation, const Value& source, const char* expected, } int main() { + JsonTest::Allocator allocator; + struct { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; } parameter{&allocator, nullptr, 0, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize copy failure contract"); constexpr const char* objectJson = "{\"long\":\"a sufficiently long string to require its own allocation\",\"nested\":[{\"leaf\":\"another sufficiently long string\"},[true,null,17]]}"; constexpr const char* arrayJson = "[1,{\"long\":\"a sufficiently long string to require its own allocation\"},[false,null]]"; constexpr const char* stringJson = "\"a sufficiently long string to require its own allocation\""; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index e8e83a1f18..34ee7c72de 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -1,23 +1,5 @@ # libSceJson2 technical debt -### Unknown function info - -- [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. -- [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. -- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser -- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still ignores allocator storage and file-buffer size; allocation admission, allocator notifications and rollback are not implemented. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. -- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records, strings and containers still use host allocations. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. -- [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. -- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the signatures follow the mangled names, with `Array::iterator::operator++(int)` and `Object::iterator::operator++(int)` returning their copy through a result pointer before `this` as `begin` does, since the iterators have non-trivial destructors. `Pair(const String&, const Value&)` and default `Pair()` deep-copy or initialize into the layout `Object` iteration already hands out (key at 0, value at 0x10) and zeroes the 8 bytes between them; what the console keeps there is unknown. -- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed admission remains incomplete. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`; the host auxiliary allocation path is not the supplied allocator. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. -- [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent legacy handler maps and module-global mismatch registration remain incomplete. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Compatibility below SDK 6 is unverified. -- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - module-global recovery and SDK-dependent dispatch remain incomplete. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Host allocation failure behavior still differs; compatibility below SDK 6 is unverified. -- [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - module-global special-float registration and allocator-backed failures remain incomplete. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. -- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing. A title's allocator that does not override it calls it through its own vtable; the library never calls the title's allocator, so it never reports an allocation failure. Whether the console's base implementation logs anything is unknown -- [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - storage uses `std::string`; capacity and allocation growth do not match the system 32-bit capacity and allocator settings. Appending null text with a nonzero length throws instead of dereferencing it. -- [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the return type of `advance` remains unverified. Object iterator advancement past the end throws instead of asserting. Dereferencing an empty array iterator throws instead of returning the system null-value pointer or an invalid address. -- [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - module-global mismatch fallback and legacy SDK behavior remain incomplete. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. - ### Unknown 3.20 import declarations - [_ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R996H0YIhiQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. @@ -94,3 +76,24 @@ - [_ZNK3sce4Json6String5rfindEPKcm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `VpQBXbaHXOo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json6String5rfindEPKcmm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `A29RHrlfTM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json6String5rfindERKS1_m](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `-R9mdXBspeQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. + +### Unknown function info + +- [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. +- [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. +- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser +- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still skips the allocator-backed context, default container resources and mutex handles. Allocator admission and rollback remain incomplete; file-buffer size is snapshotted and used by file parsing. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. +- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records, outer value handles and containers still use host allocations. String headers and characters use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. +- [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. +- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the signatures follow the mangled names, with `Array::iterator::operator++(int)` and `Object::iterator::operator++(int)` returning their copy through a result pointer before `this` as `begin` does, since the iterators have non-trivial destructors. `Pair(const String&, const Value&)` and default `Pair()` deep-copy or initialize into the layout `Object` iteration already hands out (key at 0, value at 0x10) and zeroes the 8 bytes between them; what the console keeps there is unknown. +- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed admission remains incomplete. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`; the host auxiliary allocation path is not the supplied allocator. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. +- [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent legacy handler maps and dispatch remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Compatibility below SDK 6 is unverified. +- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Host allocation failure behavior still differs; compatibility below SDK 6 is unverified. +- [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. +- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String and file-buffer allocation failures dispatch the supplied allocator's notifier, but containers and value metadata still bypass it +- [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. +- [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the return type of `advance` remains unverified. Object iterator advancement past the end throws instead of asserting. Dereferencing an empty array iterator throws instead of returning the system null-value pointer or an invalid address. +- [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy SDK behavior remains incomplete. Root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. + +- [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - destination state after parse failure and allocator failures outside the file buffer remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. +- [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - host list headers, nodes and iterator state still differ from allocator-backed 24/32-byte headers, 48-byte nodes and 8-byte iterator state observed at `0xb2d0`, `0xb6f0` and `0xb130`. The host clamps advancement at the end; the inspected implementation follows circular links. Allocation failure admission and rollback remain incomplete. From bba30ca315246489162012eea4a2e8fcef6ac120 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 00:04:59 +0200 Subject: [PATCH 013/318] fix(libSceJson2): use allocator-backed array storage and iterator state --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 290 +++++++++++++++----- core/libs/tests/GuestJson.cpp | 42 +-- core/libs/tests/GuestJsonArrayAllocator.cpp | 167 +++++++++++ core/libs/tests/JsonCopyFailure.cpp | 2 +- docs/dev/debt/libSceJson2.md | 10 +- 6 files changed, 429 insertions(+), 89 deletions(-) create mode 100644 core/libs/tests/GuestJsonArrayAllocator.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 412298132d..d0e30543f5 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -55,6 +55,11 @@ if(BUILD_TESTING) target_link_libraries(guest_json_string_allocator_tests PRIVATE libSceJson2 libc) configure_windows_unwind(guest_json_string_allocator_tests) add_test(NAME guest_json_string_allocator COMMAND guest_json_string_allocator_tests) + add_test_executable(guest_json_array_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonArrayAllocator.cpp) + target_include_directories(guest_json_array_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_array_allocator_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_array_allocator_tests) + add_test(NAME guest_json_array_allocator COMMAND guest_json_array_allocator_tests) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(numeric_locale_dir ${CMAKE_CURRENT_BINARY_DIR}/numeric-locale) set(numeric_locale_data ${LIBS_INCLUDE_DIR}/tests/numeric-locale) @@ -65,7 +70,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 6bf35145ca..b8f8ffdb8b 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -49,12 +50,13 @@ struct Items : std::list { Value* owner = nullptr; }; struct Text { char* characters; std::uint32_t length; std::uint32_t capacity; std::uint64_t reserved; }; static_assert(sizeof(Text) == 24); struct String { Text* text; }; -struct Array { Items* items; }; +struct ArrayStorage; +struct ArrayLink; +struct Array { ArrayStorage* items; }; struct Pair; struct Object { Items* items; }; struct ArrayIteratorState { - std::list::iterator position; - std::list::iterator end; + ArrayLink* position; }; struct ArrayIterator { ArrayIteratorState* state; }; struct ObjectIteratorState { @@ -151,6 +153,7 @@ struct Globals { AllocationContext allocation{}; std::recursive_mutex callbackMutex; bool callbackActive = false; + std::uint64_t allocationFailures = 0; }; Globals& State() { @@ -179,6 +182,9 @@ void* Allocate(const AllocationContext& context, std::size_t size) { if (!memory) { const auto notify = reinterpret_cast(table[4]); notify(context.allocator, JsonErrorOutOfMemory, size, context.userData); + auto& state = State(); + std::lock_guard lock(state.mutex); + ++state.allocationFailures; } return memory; } @@ -275,17 +281,33 @@ std::string_view View(const String& string) { return {Characters(string), string.text ? string.text->length : 0}; } -bool Grow(Text& text, std::size_t requested) { - if (requested > std::numeric_limits::max()) - throw std::length_error("sce::Json::String: capacity exceeds 32 bits"); - if (requested <= text.capacity) return true; - std::uint32_t capacity = static_cast(requested); +std::uint32_t CompiledSdk() { std::uint32_t sdk = std::numeric_limits::max(); try { const auto* process = static_cast(ApplicationProcessParameters_nid_no_patch()); if (process) std::memcpy(&sdk, process + 0x10, sizeof(sdk)); } catch (const std::runtime_error&) { } + return sdk; +} + +bool ModernValues() { + const auto sdk = CompiledSdk(); + return sdk >= 0x06000000 || (sdk >= 0x05508700 && sdk < 0x05510000); +} + +std::uint64_t AllocationFailures() { + auto& state = State(); + std::lock_guard lock(state.mutex); + return state.allocationFailures; +} + +bool Grow(Text& text, std::size_t requested) { + if (requested > std::numeric_limits::max()) + throw std::length_error("sce::Json::String: capacity exceeds 32 bits"); + if (requested <= text.capacity) return true; + std::uint32_t capacity = static_cast(requested); + const auto sdk = CompiledSdk(); if (sdk > 0x044fffff) { capacity |= 0x10; if (capacity / 3 < text.capacity / 2) { @@ -355,7 +377,108 @@ void Construct(String& string, const String& source) { Construct(string, std::string_view{}); AssignText(string, View(source), true); } -void Construct(Array& a) { a.items = new Items(); } +struct GuestOutOfMemory {}; + +void Construct(Value& value); +void Destroy(Value& value); +void CopyInto(Value& destination, const Value& source); + +struct ArrayLink { + ArrayLink* previous; + ArrayLink* next; + Value value; +}; +static_assert(sizeof(ArrayLink) == 48 && offsetof(ArrayLink, value) == 16); + +struct ArrayStorage { + ArrayLink* head; + std::size_t count; + std::uint64_t reserved; + + struct iterator { + using value_type = Value; + using difference_type = std::ptrdiff_t; + using pointer = Value*; + using reference = Value&; + using iterator_category = std::bidirectional_iterator_tag; + ArrayLink* node = nullptr; + Value& operator*() const { return node->value; } + Value* operator->() const { return &node->value; } + iterator& operator++() { node = node->next; return *this; } + iterator operator++(int) { const auto previous = *this; ++*this; return previous; } + iterator& operator--() { node = node->previous; return *this; } + iterator operator--(int) { const auto previous = *this; --*this; return previous; } + bool operator==(const iterator&) const = default; + }; + + Value* Owner() const { + Value* owner = nullptr; + if (ModernValues()) std::memcpy(&owner, reinterpret_cast(this) + 24, sizeof(owner)); + return owner; + } + void SetOwner(Value* owner) { + if (ModernValues()) std::memcpy(reinterpret_cast(this) + 24, &owner, sizeof(owner)); + } + iterator begin() const { return {head ? head->next : nullptr}; } + iterator end() const { return {head}; } + bool empty() const { return !count; } + std::size_t size() const { return count; } + Value& front() { return head->next->value; } + Value& back() { return head->previous->value; } + const Value& front() const { return head->next->value; } + const Value& back() const { return head->previous->value; } + Value& emplace_back() { + if (!head) throw GuestOutOfMemory{}; + auto* link = static_cast(Allocate(sizeof(ArrayLink))); + if (!link) throw GuestOutOfMemory{}; + Construct(link->value); + link->previous = head->previous; + link->next = head; + head->previous->next = link; + head->previous = link; + ++count; + return link->value; + } + void ReleaseContents() { + if (!head) return; + auto* link = head->next; + while (link != head) { + auto* next = link->next; + Destroy(link->value); + Release(link); + link = next; + } + } + void clear() { + ReleaseContents(); + if (head) { head->previous = head; head->next = head; } + count = 0; + } + void PopBack() { + if (!head || !count) return; + auto* link = head->previous; + link->previous->next = head; + head->previous = link->previous; + Destroy(link->value); + Release(link); + --count; + } +}; +static_assert(sizeof(ArrayStorage) == 24 && sizeof(ArrayIteratorState) == 8); + +ArrayLink* NewArrayHead() { + auto* head = static_cast(Allocate(sizeof(ArrayLink))); + if (head) { head->previous = head; head->next = head; } + return head; +} + +void Construct(Array& array) { + array.items = static_cast(Allocate(ModernValues() ? 32 : 24)); + if (!array.items) return; + array.items->count = 0; + array.items->head = NewArrayHead(); + array.items->SetOwner(nullptr); +} void Construct(Object& o) { o.items = new Items(); } void Destroy(Value& v); void CopyInto(Value& destination, const Value& source); @@ -368,8 +491,11 @@ void Destroy(String& s) { } void Destroy(Array& a) { if (!a.items) return; - for (auto& item : *a.items) Destroy(item); - delete a.items; + a.items->ReleaseContents(); + Release(a.items->head); + a.items->head = reinterpret_cast(std::uintptr_t{0xffffffff}); + a.items->count = std::numeric_limits::max(); + Release(a.items); a.items = nullptr; } void Destroy(Object& o) { @@ -413,8 +539,8 @@ Handlers HandlersOf(const Node& n) { } void BindChildren(Node& n) { - if (n.type == TypeArray && n.array) { - n.array->items->owner = &n; + if (n.type == TypeArray && n.array && n.array->items) { + n.array->items->SetOwner(&n); for (auto& item : *n.array->items) item.parent = &n; } if (n.type == TypeObject && n.object) { @@ -500,15 +626,27 @@ void Construct(T*& destination, const TArgs&... arguments) { void CopyArray(Array& destination, const Array& source) { Array pending{}; + if (!source.items) { destination = pending; return; } Construct(pending); + if (!pending.items) { destination = pending; return; } + auto* originalHead = pending.items->head; + pending.items->head = NewArrayHead(); try { - for (const auto& item : *source.items) { - CopyInto(pending.items->emplace_back(), item); + if (pending.items->head) { + for (const auto& item : *source.items) { + try { + CopyInto(pending.items->emplace_back(), item); + } catch (const GuestOutOfMemory&) { + break; + } + } } } catch (...) { + Release(originalHead); Destroy(pending); throw; } + Release(originalHead); destination = pending; } @@ -1065,6 +1203,7 @@ private: skip(); if (peek() >= 0 && peek() == ']') { take(); --depth; return true; } for (;;) { + if (!out.array->items) throw GuestOutOfMemory{}; auto& item = out.array->items->emplace_back(); Construct(item); item.parent = &out; @@ -1112,6 +1251,9 @@ int ParseValue(Value* out, Parser& parser) { bool parsedOk = false; try { parsedOk = parser.Parse(parsed); + } catch (const GuestOutOfMemory&) { + Clear(parsed); + return JsonErrorOutOfMemory; } catch (...) { Clear(parsed); throw; @@ -1287,66 +1429,98 @@ void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array* self) { Construct(*self); } void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array* self, const Array* other) { CopyArray(*self, *other); } void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array* self) { Destroy(*self); } void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array* self, const Value* value) { + if (!self->items || !self->items->head) return; + const auto failures = AllocationFailures(); Value copy{}; - CopyInto(copy, *value); + copy.parent = self->items->Owner(); try { - self->items->push_back(copy); - auto& inserted = self->items->back(); - inserted.parent = self->items->owner; - BindChildren(inserted); + AssignNode(copy, *value); + auto& inserted = self->items->emplace_back(); + inserted.parent = self->items->Owner(); + TransferPayload(inserted, copy); + } catch (const GuestOutOfMemory&) { } catch (...) { Destroy(copy); throw; } + Destroy(copy); + if (AllocationFailures() != failures) self->items->PopBack(); } -ArrayIterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(ArrayIterator* result, const Array* self) { result->state = new ArrayIteratorState{self->items->begin(), self->items->end()}; return result; } -ArrayIterator* APS5_VABI _ZNK3sce4Json5Array3endEv(ArrayIterator* result, const Array* self) { result->state = new ArrayIteratorState{self->items->end(), self->items->end()}; return result; } -bool APS5_VABI _ZNK3sce4Json5Array5emptyEv(const Array* self) { return self->items->empty(); } -std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array* self) { return self->items->size(); } + +ArrayIterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(ArrayIterator* result, const Array* self) { + result->state = static_cast(Allocate(sizeof(ArrayIteratorState))); + if (result->state) result->state->position = self->items && self->items->head ? self->items->head->next : nullptr; + if (self->items && (!self->items->head || !self->items->head->next)) { + Release(result->state); + result->state = nullptr; + } + return result; +} +ArrayIterator* APS5_VABI _ZNK3sce4Json5Array3endEv(ArrayIterator* result, const Array* self) { + result->state = static_cast(Allocate(sizeof(ArrayIteratorState))); + if (result->state) result->state->position = self->items ? self->items->head : nullptr; + if (self->items && !self->items->head) { + Release(result->state); + result->state = nullptr; + } + return result; +} +bool APS5_VABI _ZNK3sce4Json5Array5emptyEv(const Array* self) { return !self->items || self->items->empty(); } +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array* self) { return self->items ? self->items->size() : 0; } void APS5_VABI _ZN3sce4Json5Array5clearEv(Array* self) { - for (auto& item : *self->items) Destroy(item); + if (!self->items) return; self->items->clear(); } const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array* self) { - if (self->items->empty()) throw std::out_of_range("sce::Json::Array::front: empty array"); + if (!self->items) return &Defaults(TypeNull); + if (!self->items->head) NotImplemented_nid_no_patch(__func__); return &self->items->front(); } const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array* self) { - if (self->items->empty()) throw std::out_of_range("sce::Json::Array::back: empty array"); + if (!self->items) return &Defaults(TypeNull); + if (!self->items->head) return reinterpret_cast(16); return &self->items->back(); } void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array* self) { + if (!self->items || !self->items->head) NotImplemented_nid_no_patch(__func__); if (self->items->empty()) throw std::out_of_range("sce::Json::Array::pop_front: empty array"); - Destroy(self->items->front()); - self->items->pop_front(); + auto* link = self->items->head->next; + self->items->head->next = link->next; + link->next->previous = self->items->head; + Destroy(link->value); + Release(link); + --self->items->count; } -void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(ArrayIterator* self) { delete self->state; self->state = nullptr; } +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(ArrayIterator* self) { Release(self->state); self->state = nullptr; } ArrayIterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(ArrayIterator* self) { - if (self->state && self->state->position != self->state->end) ++self->state->position; + if (self->state && self->state->position) self->state->position = self->state->position->next; return self; } Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const ArrayIterator* self) { - if (!self->state || self->state->position == self->state->end) throw std::out_of_range("sce::Json::Array::iterator: no current element"); - return &*self->state->position; + if (!self->state) return const_cast(&Defaults(TypeNull)); + return reinterpret_cast(reinterpret_cast(self->state->position) + 16); } Value* APS5_VABI _ZNK3sce4Json5Array8iteratorptEv(const ArrayIterator* self) { return _ZNK3sce4Json5Array8iteratordeEv(self); } bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const ArrayIterator* self, const ArrayIterator* other) { - if (!self->state || !other->state) return false; - const Value* left = self->state && self->state->position != self->state->end ? &*self->state->position : nullptr; - const Value* right = other->state && other->state->position != other->state->end ? &*other->state->position : nullptr; - return left != right; + return self->state && other->state && self->state->position != other->state->position; } void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(ArrayIterator* self, const ArrayIterator* other) { - self->state = other->state ? new ArrayIteratorState(*other->state) : nullptr; + self->state = other->state ? static_cast(Allocate(sizeof(ArrayIteratorState))) : nullptr; + if (self->state) *self->state = *other->state; } -ArrayIterator* APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(ArrayIterator* self, std::size_t count) { - if (self->state) { - while (count && self->state->position != self->state->end) { - ++self->state->position; - --count; - } - } - return self; +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(ArrayIterator* self, std::size_t count) { + if (!self->state || !self->state->position || !count) return; + auto* start = self->state->position; + auto* node = start; + std::size_t traversed = 0; + do { + node = node->next; + ++traversed; + } while (node && node != start && traversed < count); + if (node == start) count %= traversed; + else { self->state->position = node; return; } + while (count--) node = node->next; + self->state->position = node; } void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object* self) { Construct(*self); } @@ -1847,33 +2021,25 @@ ObjectIterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(ObjectIterator } } -ArrayIterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(ArrayIterator* result, ArrayIterator* self, int) { - _ZN3sce4Json5Array8iteratorC1ERKS2_(result, self); - _ZN3sce4Json5Array8iteratorppEv(self); - return result; +ArrayIterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(ArrayIterator* self, int) { + return _ZN3sce4Json5Array8iteratorppEv(self); } ArrayIterator* APS5_VABI _ZN3sce4Json5Array8iteratoraSERKS2_(ArrayIterator* self, const ArrayIterator* other) { if (self->state) { - if (other->state && other->state->position != other->state->end) { - *self->state = *other->state; - } else { - delete self->state; - self->state = nullptr; - } + if (other->state && other->state->position) *self->state = *other->state; + else { Release(self->state); self->state = nullptr; } } return self; } void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(ArrayIterator* self) { - self->state = new ArrayIteratorState{}; + self->state = static_cast(Allocate(sizeof(ArrayIteratorState))); + if (self->state) self->state->position = nullptr; } -ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(ObjectIterator* result, ObjectIterator* self, int) { - auto state = self->state ? std::make_unique(*self->state) : nullptr; - _ZN3sce4Json6Object8iteratorppEv(self); - result->state = state.release(); - return result; +ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(ObjectIterator* self, int) { + return _ZN3sce4Json6Object8iteratorppEv(self); } ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iteratoraSERKS2_(ObjectIterator* self, const ObjectIterator* other) { diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index d6d5c04ae5..b03e1c6eda 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -141,15 +141,15 @@ void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair*); Iterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(Iterator*, Object*, const Pair*); void APS5_VABI _ZN3sce4Json6Object8iteratorC1Ev(Iterator*); Iterator* APS5_VABI _ZN3sce4Json6Object8iteratoraSERKS2_(Iterator*, const Iterator*); -Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, Iterator*, int); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); Iterator* APS5_VABI _ZN3sce4Json5Array8iteratoraSERKS2_(Iterator*, const Iterator*); -Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, Iterator*, int); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); void APS5_VABI _ZN3sce4Json12MemAllocator11notifyErrorEimPv(void*, int, std::size_t, void*); bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value*); void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value*, Value*); void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); -Iterator* APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); void APS5_VABI _ZN3sce4Json6Object8iteratorC1ERKS2_(Iterator*, const Iterator*); Iterator* APS5_VABI _ZN3sce4Json6Object8iterator7advanceEm(Iterator*, std::size_t); void APS5_VABI _ZN3sce4Json6Object5eraseERKNS0_6StringE(Object*, const String*); @@ -347,19 +347,14 @@ static void ObjectsAndArrays() { for (; _ZNK3sce4Json5Array8iteratorneERKS2_(&it, &end); _ZN3sce4Json5Array8iteratorppEv(&it)) ++visited; Require(visited == 2); Require(*_ZNK3sce4Json5Value10getBooleanEv(_ZNK3sce4Json5Array5frontEv(&array))); + _ZN3sce4Json5Array8iteratorD1Ev(&it); _ZNK3sce4Json5Array5beginEv(&it, &array); Require(_ZNK3sce4Json5Array8iteratorptEv(&it) == _ZNK3sce4Json5Array8iteratordeEv(&it)); Array copy{}; _ZN3sce4Json5ArrayC1ERKS1_(©, &array); _ZN3sce4Json5Array5clearEv(&array); Require(_ZNK3sce4Json5Array4sizeEv(&array) == 0); - bool threw = false; - try { - _ZNK3sce4Json5Array5frontEv(&array); - } catch (const std::out_of_range&) { - threw = true; - } - Require(threw); + Require(_ZNK3sce4Json5Array5frontEv(&array) == _ZNK3sce4Json5Array4backEv(&array)); Require(_ZNK3sce4Json5Array4sizeEv(©) == 2); Require(*_ZNK3sce4Json5Value10getBooleanEv(_ZNK3sce4Json5Array5frontEv(©))); Require(*_ZNK3sce4Json5Value7getRealEv(_ZNK3sce4Json5Array4backEv(©)) == 2.5); @@ -368,13 +363,15 @@ static void ObjectsAndArrays() { Require(*_ZNK3sce4Json5Value7getRealEv(_ZNK3sce4Json5Array5frontEv(©)) == 2.5); _ZN3sce4Json5Array9pop_frontEv(©); Require(_ZNK3sce4Json5Array5emptyEv(©)); - threw = false; + bool threw = false; try { _ZN3sce4Json5Array9pop_frontEv(©); } catch (const std::out_of_range&) { threw = true; } Require(threw && _ZNK3sce4Json5Array5emptyEv(©)); + _ZN3sce4Json5Array8iteratorD1Ev(&it); + _ZN3sce4Json5Array8iteratorD1Ev(&end); _ZN3sce4Json5ArrayD1Ev(©); _ZN3sce4Json5ArrayD1Ev(&array); } @@ -430,7 +427,8 @@ static void ArrayIteratorCopies() { Iterator previous{}; while (_ZNK3sce4Json5Array8iteratorneERKS2_(&it, &end)) { _ZN3sce4Json5Array8iteratorD1Ev(&previous); - Require(_ZN3sce4Json5Array8iteratorppEi(&previous, &it, 0) == &previous); + _ZN3sce4Json5Array8iteratorC1ERKS2_(&previous, &it); + Require(_ZN3sce4Json5Array8iteratorppEi(&it, 0) == &it); seen.push_back(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array8iteratordeEv(&previous))); Require(_ZNK3sce4Json5Array8iteratorneERKS2_(&previous, &it)); } @@ -438,9 +436,14 @@ static void ArrayIteratorCopies() { Require(!_ZNK3sce4Json5Array8iteratorneERKS2_(&it, &end)); Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array8iteratordeEv(&previous)) == 3); Require(_ZN3sce4Json5Array8iteratoraSERKS2_(&it, &end) == &it); - Require(it.position == nullptr); + Require(it.position != nullptr && !_ZNK3sce4Json5Array8iteratorneERKS2_(&it, &end)); Require(_ZN3sce4Json5Array8iteratoraSERKS2_(&it, &begin) == &it); - Require(it.position == nullptr); + Require(it.position != nullptr); + Iterator empty{}; + _ZN3sce4Json5Array8iteratorC1Ev(&empty); + Require(_ZN3sce4Json5Array8iteratoraSERKS2_(&it, &empty) == &it && it.position == nullptr); + Require(_ZN3sce4Json5Array8iteratoraSERKS2_(&it, &begin) == &it && it.position == nullptr); + _ZN3sce4Json5Array8iteratorD1Ev(&empty); Require(!_ZNK3sce4Json5Array8iteratorneERKS2_(&it, &begin)); _ZN3sce4Json5Array8iteratorD1Ev(&previous); @@ -542,11 +545,8 @@ static void ObjectIteratorCopies() { Require(!_ZNK3sce4Json6Object8iteratoreqERKS2_(&begin, &end)); Require(_ZNK3sce4Json6Object8iteratorneERKS2_(&begin, &end)); - Iterator post{}; - Require(_ZN3sce4Json6Object8iteratorppEi(&post, &begin, 0) == &post); - Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&_ZNK3sce4Json6Object8iteratordeEv(&post)->key), "one") == 0); + Require(_ZN3sce4Json6Object8iteratorppEi(&begin, 0) == &begin); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&_ZNK3sce4Json6Object8iteratordeEv(&begin)->key), "two") == 0); - _ZN3sce4Json6Object8iteratorD1Ev(&post); Iterator assigned{}; _ZN3sce4Json6Object8iteratorC1Ev(&assigned); @@ -768,14 +768,16 @@ static void ArrayIteratorAdvance() { _ZNK3sce4Json5Array5beginEv(&begin, &array); Iterator copy{}; _ZN3sce4Json5Array8iteratorC1ERKS2_(©, &begin); - Require(_ZN3sce4Json5Array8iterator7advanceEm(©, 3) == ©); + _ZN3sce4Json5Array8iterator7advanceEm(©, 3); Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array8iteratordeEv(©)) == 30); Iterator end{}; _ZNK3sce4Json5Array3endEv(&end, &array); _ZN3sce4Json5Array8iterator7advanceEm(©, std::numeric_limits::max()); + Require(!_ZNK3sce4Json5Array8iteratorneERKS2_(©, &begin)); + _ZN3sce4Json5Array8iterator7advanceEm(©, 5); Require(!_ZNK3sce4Json5Array8iteratorneERKS2_(©, &end)); _ZN3sce4Json5Array8iteratorppEv(©); - Require(!_ZNK3sce4Json5Array8iteratorneERKS2_(©, &end)); + Require(!_ZNK3sce4Json5Array8iteratorneERKS2_(©, &begin)); Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array8iteratordeEv(&begin)) == 0); _ZN3sce4Json5Array8iteratorD1Ev(&end); _ZN3sce4Json5Array8iteratorD1Ev(&begin); diff --git a/core/libs/tests/GuestJsonArrayAllocator.cpp b/core/libs/tests/GuestJsonArrayAllocator.cpp new file mode 100644 index 0000000000..bf8e5dc6b7 --- /dev/null +++ b/core/libs/tests/GuestJsonArrayAllocator.cpp @@ -0,0 +1,167 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Iterator { void* state; }; +struct Object { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ArrayTrace { + std::map live; + std::vector requests; + int budget = -1; + unsigned notifications = 0; + std::set headers; +}; +static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) return nullptr; + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeArray(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); + } + Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Array notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; +struct ArrayAllocator { void* const* vtable = ArrayVtable; }; +static void CheckLinks(const Array& array, std::size_t count) { + const auto* header = static_cast(array.storage); + Require(header && header[1] == count, "Array header must expose its native element count"); + const auto* head = reinterpret_cast(header[0]); + const auto* previous = head; + std::size_t seen = 0; + for (auto* node = reinterpret_cast(head[1]); node != head; node = reinterpret_cast(node[1])) { + Require(node[0] == reinterpret_cast(previous), "Every array link must retain its predecessor"); + const auto* value = reinterpret_cast(node + 2); + Require(!value->parent && !value->auxiliary && value->type == 2 && value->payload == 10 * (++seen), "Array nodes must expose their inline values and ownership"); + previous = node; + Require(seen <= count, "Array links must close at the sentinel"); + } + Require(seen == count && head[0] == reinterpret_cast(previous), "Array sentinel must close both list directions"); +} +int main() { + ArrayTrace trace; ArrayAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize array allocation contract"); + const auto baseline = trace.live.size(); + for (int budget : {0, 1}) { + trace.budget = budget; Array partial{}; _ZN3sce4Json5ArrayC1Ev(&partial); + Require(budget ? partial.storage && trace.live.at(partial.storage) == 32 && !*static_cast(partial.storage) : !partial.storage, + "Header and sentinel failures retain distinct partial array ownership"); + Require(_ZNK3sce4Json5Array4sizeEv(&partial) == 0, "Partial arrays remain empty"); + if (partial.storage) trace.headers.insert(partial.storage); + trace.budget = -1; _ZN3sce4Json5ArrayD1Ev(&partial); + Require(!partial.storage && trace.live.size() == baseline, "Partial arrays must be destructible without leaks"); + } + Array array{}; _ZN3sce4Json5ArrayC1Ev(&array); + trace.headers.insert(array.storage); + auto* header = static_cast(array.storage); + Require(trace.live.at(array.storage) == 32 && trace.live.at(reinterpret_cast(header[0])) == 48 && !header[3], "Actual array header and sentinel must use the configured allocator"); + Require(_ZNK3sce4Json5Array5frontEv(&array) == reinterpret_cast(header[0] + 16) && _ZNK3sce4Json5Array4backEv(&array) == reinterpret_cast(header[0] + 16), "Empty array endpoints expose the sentinel payload address"); + Value value{}; + for (int number : {10, 20, 30}) { _ZN3sce4Json5ValueC1El(&value, number); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); _ZN3sce4Json5ValueD1Ev(&value); } + CheckLinks(array, 3); + Iterator begin{}, end{}, copy{}; + _ZNK3sce4Json5Array5beginEv(&begin, &array); _ZNK3sce4Json5Array3endEv(&end, &array); + Require(trace.live.at(begin.state) == 8 && trace.live.at(end.state) == 8 && *static_cast(end.state) == reinterpret_cast(header[0]), "Iterator state must own one allocator-backed native node pointer"); + Require(_ZN3sce4Json5Array8iteratorppEi(&begin, 0) == &begin && *_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array8iteratordeEv(&begin)) == 20, "Postfix increment must update this iterator without a hidden result object"); + _ZN3sce4Json5Array8iteratorC1ERKS2_(©, &begin); + Require(trace.live.at(copy.state) == 8 && copy.state != begin.state, "Iterator copy must own independent state"); + _ZN3sce4Json5Array8iterator7advanceEm(&begin, 2); + Require(!_ZNK3sce4Json5Array8iteratorneERKS2_(&begin, &end), "Advance must reach the sentinel"); + _ZN3sce4Json5Array8iteratorppEv(&begin); + Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array8iteratordeEv(&begin)) == 10, "Array increment must follow the circular link after the sentinel"); + _ZN3sce4Json5Array8iterator7advanceEm(&begin, 6); + Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array8iteratordeEv(&begin)) == 30, "Advance must preserve circular traversal"); + trace.budget = 0; Iterator failed{}; _ZNK3sce4Json5Array5beginEv(&failed, &array); + Require(!failed.state, "Iterator allocation failure must leave an empty outer handle"); + trace.budget = -1; + _ZN3sce4Json5Array8iteratorD1Ev(©); _ZN3sce4Json5Array8iteratorD1Ev(&begin); _ZN3sce4Json5Array8iteratorD1Ev(&end); + const auto beforeCopies = trace.live.size(); + for (int budget = 0; budget <= 6; ++budget) { + trace.budget = budget; Array duplicate{}; _ZN3sce4Json5ArrayC1ERKS1_(&duplicate, &array); + const auto expected = budget < 4 ? 0 : static_cast(budget - 3); + Require(_ZNK3sce4Json5Array4sizeEv(&duplicate) == expected, "Array copy allocation failures retain the observed copied prefix"); + if (expected) CheckLinks(duplicate, expected); + if (duplicate.storage) trace.headers.insert(duplicate.storage); + trace.budget = -1; _ZN3sce4Json5ArrayD1Ev(&duplicate); + Require(trace.live.size() == beforeCopies, "Partial array copies must release all resources"); + } + trace.budget = 0; _ZN3sce4Json5ValueC1El(&value, 40); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); trace.budget = -1; + Require(_ZNK3sce4Json5Array4sizeEv(&array) == 2 && *_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array4backEv(&array)) == 20, "Failed append must retain the reference module's removal of the existing tail"); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); + Require(_ZNK3sce4Json5Array4sizeEv(&array) == 3 && *_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array4backEv(&array)) == 40, "Append remains usable after failure"); + _ZN3sce4Json5ValueD1Ev(&value); _ZN3sce4Json5Array5clearEv(&array); + Require(_ZNK3sce4Json5Array4sizeEv(&array) == 0 && trace.live.size() == baseline + 2, "Clear must retain only the array header and sentinel"); + _ZN3sce4Json5ArrayD1Ev(&array); + Object object{}; _ZN3sce4Json6ObjectC1Ev(&object); + for (const char* key : {"a", "b"}) { String string{}; _ZN3sce4Json6StringC1EPKc(&string, key); _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &string); _ZN3sce4Json6StringD1Ev(&string); } + Iterator member{}; _ZNK3sce4Json6Object5beginEv(&member, &object); + Require(_ZN3sce4Json6Object8iteratorppEi(&member, 0) == &member && std::strcmp(_ZNK3sce4Json6String5c_strEv(&_ZNK3sce4Json6Object8iteratordeEv(&member)->key), "b") == 0, + "Object postfix increment must also update this iterator without a hidden result object"); + _ZN3sce4Json6Object8iteratorD1Ev(&member); _ZN3sce4Json6ObjectD1Ev(&object); + Require(trace.live.size() == baseline, "Array and iterator operations must release their allocated resources"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate array allocation contract"); + return 0; +} diff --git a/core/libs/tests/JsonCopyFailure.cpp b/core/libs/tests/JsonCopyFailure.cpp index a7ea79512e..07c5701410 100644 --- a/core/libs/tests/JsonCopyFailure.cpp +++ b/core/libs/tests/JsonCopyFailure.cpp @@ -248,7 +248,7 @@ int main() { ++failures; } allocationBudget = -1; - Require(succeeded == (activeBudget == 1), "iterator allocation failure ignored"); + Require(succeeded == (!objectIterator || activeBudget == 1), "Host iterator allocation boundary changed"); if (succeeded) Require(copy.state != source.state, "iterator copy shares owned storage"); if (objectIterator) _ZN3sce4Json6Object8iteratorD1Ev(©); else _ZN3sce4Json5Array8iteratorD1Ev(©); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 34ee7c72de..b1f296ed5c 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -83,17 +83,17 @@ - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still skips the allocator-backed context, default container resources and mutex handles. Allocator admission and rollback remain incomplete; file-buffer size is snapshotted and used by file parsing. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. -- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records, outer value handles and containers still use host allocations. String headers and characters use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. +- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records, outer value handles and object storage still use host allocations. String headers, characters, array storage and array iterator states use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. -- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the signatures follow the mangled names, with `Array::iterator::operator++(int)` and `Object::iterator::operator++(int)` returning their copy through a result pointer before `this` as `begin` does, since the iterators have non-trivial destructors. `Pair(const String&, const Value&)` and default `Pair()` deep-copy or initialize into the layout `Object` iteration already hands out (key at 0, value at 0x10) and zeroes the 8 bytes between them; what the console keeps there is unknown. +- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - `Pair(const String&, const Value&)` and default `Pair()` zero the 8 bytes between the key and value; what the console keeps there is unknown. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed admission remains incomplete. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`; the host auxiliary allocation path is not the supplied allocator. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent legacy handler maps and dispatch remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Compatibility below SDK 6 is unverified. - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Host allocation failure behavior still differs; compatibility below SDK 6 is unverified. - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. -- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String and file-buffer allocation failures dispatch the supplied allocator's notifier, but containers and value metadata still bypass it +- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array and file-buffer allocation failures dispatch the supplied allocator's notifier, but object storage and value metadata still bypass it - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. -- [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the return type of `advance` remains unverified. Object iterator advancement past the end throws instead of asserting. Dereferencing an empty array iterator throws instead of returning the system null-value pointer or an invalid address. +- [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; a missing iterator state uses the module default value, whose allocator-backed identity remains incomplete. - [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy SDK behavior remains incomplete. Root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. - [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - destination state after parse failure and allocator failures outside the file buffer remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. -- [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - host list headers, nodes and iterator state still differ from allocator-backed 24/32-byte headers, 48-byte nodes and 8-byte iterator state observed at `0xb2d0`, `0xb6f0` and `0xb130`. The host clamps advancement at the end; the inspected implementation follows circular links. Allocation failure admission and rollback remain incomplete. +- [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, module-default identity, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. From 1969970386d01359561f207422ed6d9377162e8d Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 00:35:30 +0200 Subject: [PATCH 014/318] fix(libSceJson2): use allocator-backed object trees and iterators --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 655 +++++++++++++++---- core/libs/tests/GuestJson.cpp | 6 +- core/libs/tests/GuestJsonObjectAllocator.cpp | 510 +++++++++++++++ core/libs/tests/JsonCopyFailure.cpp | 4 +- docs/dev/debt/libSceJson2.md | 7 +- 6 files changed, 1055 insertions(+), 134 deletions(-) create mode 100644 core/libs/tests/GuestJsonObjectAllocator.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index d0e30543f5..7760eb7e6c 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -60,6 +60,11 @@ if(BUILD_TESTING) target_link_libraries(guest_json_array_allocator_tests PRIVATE libSceJson2 libc) configure_windows_unwind(guest_json_array_allocator_tests) add_test(NAME guest_json_array_allocator COMMAND guest_json_array_allocator_tests) + add_test_executable(guest_json_object_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonObjectAllocator.cpp) + target_include_directories(guest_json_object_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_object_allocator_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_object_allocator_tests) + add_test(NAME guest_json_object_allocator COMMAND guest_json_object_allocator_tests) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(numeric_locale_dir ${CMAKE_CURRENT_BINARY_DIR}/numeric-locale) set(numeric_locale_data ${LIBS_INCLUDE_DIR}/tests/numeric-locale) @@ -70,7 +75,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index b8f8ffdb8b..94b2b1ec48 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -45,8 +44,6 @@ constexpr int JsonErrorInvalidArgument = static_cast(0x80848120); struct Value; struct Handlers; struct Auxiliary; -template -struct Items : std::list { Value* owner = nullptr; }; struct Text { char* characters; std::uint32_t length; std::uint32_t capacity; std::uint64_t reserved; }; static_assert(sizeof(Text) == 24); struct String { Text* text; }; @@ -54,14 +51,15 @@ struct ArrayStorage; struct ArrayLink; struct Array { ArrayStorage* items; }; struct Pair; -struct Object { Items* items; }; +struct ObjectStorage; +struct ObjectLink; +struct Object { ObjectStorage* items; }; struct ArrayIteratorState { ArrayLink* position; }; struct ArrayIterator { ArrayIteratorState* state; }; struct ObjectIteratorState { - std::list* owner = nullptr; - std::list::iterator position{}; + ObjectLink* position; }; struct ObjectIterator { ObjectIteratorState* state; }; using TypeMismatchHandler = const Value& (APS5_VABI*)(ValueType, const Value*, const Value*, void*); @@ -479,13 +477,340 @@ void Construct(Array& array) { array.items->head = NewArrayHead(); array.items->SetOwner(nullptr); } -void Construct(Object& o) { o.items = new Items(); } +void Destroy(String& string); +void BindChildren(Value& value); +void AssignNode(Value& destination, const Value& source); +void TransferPayload(Value& destination, Value& source); + +void PoisonText(Text& text) { + text.characters = reinterpret_cast(std::uintptr_t{0xffffffff}); + text.length = std::numeric_limits::max(); + text.capacity = std::numeric_limits::max(); +} + +struct InlineText { + Text text; + InlineText() { text.characters = emptyCharacters; text.length = 0; text.capacity = 0; } + ~InlineText() { String string{&text}; ClearText(string); PoisonText(text); } + InlineText(const InlineText&) = delete; + InlineText& operator=(const InlineText&) = delete; + void Assign(std::string_view bytes) { String string{&text}; AssignText(string, bytes, true); } +}; + +struct ObjectLink { + ObjectLink* parent; + ObjectLink* left; + ObjectLink* right; + Text key; + Pair pair; + std::uint8_t color; + std::uint8_t nil; + std::uint8_t reserved[6]; +}; +static_assert(sizeof(ObjectLink) == 104 && offsetof(ObjectLink, key) == 24); +static_assert(offsetof(ObjectLink, pair) == 48 && offsetof(ObjectLink, color) == 96); + +ObjectLink* NextObjectLink(ObjectLink* node) { + if (!node) return nullptr; + if (node->nil) throw std::out_of_range("sce::Json::Object::iterator: advance past the end"); + if (!node->right->nil) { + node = node->right; + while (!node->left->nil) node = node->left; + return node; + } + auto* parent = node->parent; + while (!parent->nil && node == parent->right) { node = parent; parent = parent->parent; } + return parent; +} + +void DestroyObjectLink(ObjectLink* node) { + Destroy(node->pair.value); + Destroy(node->pair.key); + String key{&node->key}; + ClearText(key); + PoisonText(node->key); + Release(node); +} + +void ReleaseObjectTree(ObjectLink* node) { + if (!node || node->nil) return; + ReleaseObjectTree(node->left); + ReleaseObjectTree(node->right); + if (!node->parent->nil) { + if (node->parent->left == node) node->parent->left = node->left; + else node->parent->right = node->right; + } + DestroyObjectLink(node); +} + +ObjectLink* NewObjectHead() { + auto* head = static_cast(Allocate(sizeof(ObjectLink))); + if (head) { head->parent = head; head->left = head; head->right = head; head->color = 0; head->nil = 1; } + return head; +} + +struct ObjectStorage { + ObjectLink* head; + std::size_t count; + std::uint64_t reserved; + + struct iterator { + using value_type = Pair; + using difference_type = std::ptrdiff_t; + using pointer = Pair*; + using reference = Pair&; + using iterator_category = std::bidirectional_iterator_tag; + ObjectLink* node = nullptr; + Pair& operator*() const { return node->pair; } + Pair* operator->() const { return &node->pair; } + iterator& operator++() { node = NextObjectLink(node); return *this; } + iterator operator++(int) { const auto previous = *this; ++*this; return previous; } + iterator& operator--() { + if (node->nil) node = node->right; + else if (!node->left->nil) { node = node->left; while (!node->right->nil) node = node->right; } + else { auto* parent = node->parent; while (!parent->nil && node == parent->left) { node = parent; parent = parent->parent; } node = parent; } + return *this; + } + bool operator==(const iterator&) const = default; + }; + + Value* Owner() const { + Value* owner = nullptr; + if (ModernValues()) std::memcpy(&owner, reinterpret_cast(this) + 24, sizeof(owner)); + return owner; + } + void SetOwner(Value* owner) { if (ModernValues()) std::memcpy(reinterpret_cast(this) + 24, &owner, sizeof(owner)); } + iterator begin() const { return {head ? head->left : nullptr}; } + iterator end() const { return {head}; } + bool empty() const { return !count; } + std::size_t size() const { return count; } + ObjectLink* Find(const char* key) const { + if (!head) return nullptr; + auto* node = head->parent; + while (!node->nil) { + const int comparison = std::strcmp(key, node->key.characters); + if (!comparison) return node; + node = comparison < 0 ? node->left : node->right; + } + return head; + } + void RefreshEndpoints() { + auto* first = head->parent; + auto* last = first; + if (!first->nil) { + while (!first->left->nil) first = first->left; + while (!last->right->nil) last = last->right; + } + head->left = first; + head->right = last; + } + void RotateLeft(ObjectLink* node) { + auto* pivot = node->right; + node->right = pivot->left; + if (!pivot->left->nil) pivot->left->parent = node; + pivot->parent = node->parent; + if (node->parent->nil) head->parent = pivot; + else if (node == node->parent->left) node->parent->left = pivot; + else node->parent->right = pivot; + pivot->left = node; + node->parent = pivot; + } + void RotateRight(ObjectLink* node) { + auto* pivot = node->left; + node->left = pivot->right; + if (!pivot->right->nil) pivot->right->parent = node; + pivot->parent = node->parent; + if (node->parent->nil) head->parent = pivot; + else if (node == node->parent->right) node->parent->right = pivot; + else node->parent->left = pivot; + pivot->right = node; + node->parent = pivot; + } + void BalanceInsertion(ObjectLink* node) { + while (node->parent->color == 1) { + auto* parent = node->parent; + auto* grandparent = parent->parent; + if (parent == grandparent->left) { + auto* uncle = grandparent->right; + if (uncle->color == 1) { parent->color = 0; uncle->color = 0; grandparent->color = 1; node = grandparent; } + else { + if (node == parent->right) { node = parent; RotateLeft(node); parent = node->parent; grandparent = parent->parent; } + parent->color = 0; grandparent->color = 1; RotateRight(grandparent); + } + } else { + auto* uncle = grandparent->left; + if (uncle->color == 1) { parent->color = 0; uncle->color = 0; grandparent->color = 1; node = grandparent; } + else { + if (node == parent->left) { node = parent; RotateRight(node); parent = node->parent; grandparent = parent->parent; } + parent->color = 0; grandparent->color = 1; RotateLeft(grandparent); + } + } + } + head->parent->color = 0; + } + ObjectLink* Insert(const Text& key, const Pair& pair, bool& inserted) { + inserted = false; + if (!head) return nullptr; + auto* parent = head; + auto* position = head->parent; + int comparison = 0; + while (!position->nil) { + parent = position; + comparison = std::strcmp(key.characters, position->key.characters); + if (!comparison) return position; + position = comparison < 0 ? position->left : position->right; + } + auto* node = static_cast(Allocate(sizeof(ObjectLink))); + if (!node) return head; + node->parent = parent; node->left = head; node->right = head; + node->key.characters = emptyCharacters; node->key.length = 0; node->key.capacity = 0; + node->pair.key.text = nullptr; + Construct(node->pair.value); + node->color = 1; node->nil = 0; + try { + String keyStorage{&node->key}; AssignText(keyStorage, {key.characters, key.length}, true); + Construct(node->pair.key, pair.key); + std::memcpy(&node->pair.reserved, &pair.reserved, sizeof(std::uint32_t)); + CopyInto(node->pair.value, pair.value); + } catch (...) { DestroyObjectLink(node); throw; } + if (parent == head) head->parent = node; + else if (comparison < 0) parent->left = node; + else parent->right = node; + ++count; + BalanceInsertion(node); + RefreshEndpoints(); + inserted = true; + return node; + } + void Replace(ObjectLink* node, ObjectLink* replacement) { + if (node->parent->nil) head->parent = replacement; + else if (node == node->parent->left) node->parent->left = replacement; + else node->parent->right = replacement; + if (!replacement->nil) replacement->parent = node->parent; + } + void BalanceErasure(ObjectLink* node, ObjectLink* parent) { + while (node != head->parent && node->color == 0) { + if (node == parent->left) { + auto* sibling = parent->right; + if (sibling->color == 1) { sibling->color = 0; parent->color = 1; RotateLeft(parent); sibling = parent->right; } + if (sibling->left->color == 0 && sibling->right->color == 0) { + if (!sibling->nil) sibling->color = 1; + node = parent; parent = node->parent; + } else { + if (sibling->right->color == 0) { sibling->left->color = 0; sibling->color = 1; RotateRight(sibling); sibling = parent->right; } + sibling->color = parent->color; parent->color = 0; sibling->right->color = 0; + RotateLeft(parent); node = head->parent; + } + } else { + auto* sibling = parent->left; + if (sibling->color == 1) { sibling->color = 0; parent->color = 1; RotateRight(parent); sibling = parent->left; } + if (sibling->right->color == 0 && sibling->left->color == 0) { + if (!sibling->nil) sibling->color = 1; + node = parent; parent = node->parent; + } else { + if (sibling->left->color == 0) { sibling->right->color = 0; sibling->color = 1; RotateLeft(sibling); sibling = parent->left; } + sibling->color = parent->color; parent->color = 0; sibling->left->color = 0; + RotateRight(parent); node = head->parent; + } + } + } + node->color = 0; + } + void Erase(ObjectLink* removed) { + if (!removed || removed->nil) return; + auto* successor = removed; + auto color = successor->color; + ObjectLink* child; + ObjectLink* parent; + if (removed->left->nil) { child = removed->right; parent = removed->parent; Replace(removed, child); } + else if (removed->right->nil) { child = removed->left; parent = removed->parent; Replace(removed, child); } + else { + successor = removed->right; + while (!successor->left->nil) successor = successor->left; + color = successor->color; + child = successor->right; + if (successor->parent == removed) parent = successor; + else { + parent = successor->parent; + Replace(successor, child); + successor->right = removed->right; + successor->right->parent = successor; + } + Replace(removed, successor); + successor->left = removed->left; + successor->left->parent = successor; + successor->color = removed->color; + } + if (color == 0) BalanceErasure(child, parent); + RefreshEndpoints(); + DestroyObjectLink(removed); + --count; + } + void clear() { + if (!head) return; + ReleaseObjectTree(head->parent); + head->parent = head; head->left = head; head->right = head; + count = 0; + } +}; +static_assert(sizeof(ObjectStorage) == 24 && sizeof(ObjectIteratorState) == 8); + +void Construct(Object& object) { + object.items = static_cast(Allocate(ModernValues() ? 32 : 24)); + if (!object.items) return; + object.items->count = 0; + object.items->head = NewObjectHead(); + object.items->SetOwner(nullptr); + if (!object.items->head) { + object.items->head = reinterpret_cast(std::uintptr_t{0xffffffff}); + object.items->count = std::numeric_limits::max(); + Release(object.items); + object.items = nullptr; + } +} + +ObjectLink* CloneObjectTree(ObjectLink* source, ObjectLink* parent, ObjectLink* head) { + auto* node = static_cast(Allocate(sizeof(ObjectLink))); + if (!node) return head; + node->parent = parent; node->left = head; node->right = head; + node->key.characters = emptyCharacters; node->key.length = 0; node->key.capacity = 0; + node->pair.key.text = nullptr; + Construct(node->pair.value); + node->color = source->color; node->nil = 0; + try { + String key{&node->key}; AssignText(key, {source->key.characters, source->key.length}, true); + Construct(node->pair.key, source->pair.key); + std::memcpy(&node->pair.reserved, &source->pair.reserved, sizeof(std::uint32_t)); + CopyInto(node->pair.value, source->pair.value); + if (!source->left->nil) node->left = CloneObjectTree(source->left, node, head); + if (!source->right->nil) node->right = CloneObjectTree(source->right, node, head); + } catch (...) { + ReleaseObjectTree(node->left); ReleaseObjectTree(node->right); DestroyObjectLink(node); throw; + } + return node; +} + +void CopyObjectStorage(ObjectStorage& destination, const ObjectStorage& source) { + auto* replacement = NewObjectHead(); + ObjectStorage pending{replacement, 0, 0}; + try { + if (replacement && source.head && !source.head->parent->nil) { + replacement->parent = CloneObjectTree(source.head->parent, replacement, replacement); + if (replacement->parent != replacement) { pending.count = source.count; pending.RefreshEndpoints(); } + } + } catch (...) { pending.clear(); Release(replacement); throw; } + destination.clear(); Release(destination.head); + destination.head = replacement; + destination.count = pending.count; +} + void Destroy(Value& v); void CopyInto(Value& destination, const Value& source); void Destroy(String& s) { if (!s.text) return; ClearText(s); + PoisonText(*s.text); Release(s.text); s.text = nullptr; } @@ -498,14 +823,14 @@ void Destroy(Array& a) { Release(a.items); a.items = nullptr; } -void Destroy(Object& o) { - if (!o.items) return; - for (auto& pair : *o.items) { - Destroy(pair.key); - Destroy(pair.value); - } - delete o.items; - o.items = nullptr; +void Destroy(Object& object) { + if (!object.items) return; + object.items->clear(); + Release(object.items->head); + object.items->head = reinterpret_cast(std::uintptr_t{0xffffffff}); + object.items->count = std::numeric_limits::max(); + Release(object.items); + object.items = nullptr; } const Node& RootOf(const Node& n) { @@ -543,8 +868,8 @@ void BindChildren(Node& n) { n.array->items->SetOwner(&n); for (auto& item : *n.array->items) item.parent = &n; } - if (n.type == TypeObject && n.object) { - n.object->items->owner = &n; + if (n.type == TypeObject && n.object && n.object->items) { + n.object->items->SetOwner(&n); for (auto& pair : *n.object->items) pair.value.parent = &n; } } @@ -652,17 +977,14 @@ void CopyArray(Array& destination, const Array& source) { void CopyObject(Object& destination, const Object& source) { Object pending{}; - Construct(pending); - try { - for (const auto& pair : *source.items) { - auto& copy = pending.items->emplace_back(); - Construct(copy.key, pair.key); - CopyInto(copy.value, pair.value); - } - } catch (...) { - Destroy(pending); - throw; - } + if (!source.items) { destination = pending; return; } + pending.items = static_cast(Allocate(ModernValues() ? 32 : 24)); + if (!pending.items) { destination = pending; return; } + pending.items->count = 0; + pending.items->head = NewObjectHead(); + pending.items->SetOwner(nullptr); + try { CopyObjectStorage(*pending.items, *source.items); } + catch (...) { Destroy(pending); throw; } destination = pending; } @@ -987,7 +1309,7 @@ int Serialize(std::string& out, const Node& n) { for (const auto& pair : *n.object->items) { if (!first) out += ','; first = false; - AppendEscaped(out, View(pair.key)); + AppendEscaped(out, Characters(pair.key)); out += ':'; const int result = Serialize(out, pair.value); if (result < 0) return result; @@ -1007,26 +1329,102 @@ int CompareText(const char* left, const char* right) { return static_cast(*left) - static_cast(*right); } -std::list::iterator ObjectPosition(const Object& object, std::string_view key) { - return std::find_if(object.items->begin(), object.items->end(), [&](const Pair& pair) { return std::strcmp(Characters(pair.key), key.data()) >= 0; }); +struct ScopedString { + String string{}; + explicit ScopedString(std::string_view text) { Construct(string, text); } + explicit ScopedString(const String& source) { Construct(string, source); } + ~ScopedString() { Destroy(string); } +}; + +struct ScopedValue { + Value value{}; + ~ScopedValue() { Destroy(value); } +}; + +struct TemporaryTreeEntry { + InlineText key; + Pair pair{}; + TemporaryTreeEntry(const Text& sourceKey, const Pair& source) { + try { + key.Assign({sourceKey.characters, sourceKey.length}); + Construct(pair.key, source.key); + std::memcpy(&pair.reserved, &source.reserved, sizeof(std::uint32_t)); + CopyInto(pair.value, source.value); + } catch (...) { Destroy(pair.value); Destroy(pair.key); throw; } + } + ~TemporaryTreeEntry() { Destroy(pair.value); Destroy(pair.key); } +}; + +struct ScopedPair { + Pair pair{}; + ~ScopedPair() { Destroy(pair.value); Destroy(pair.key); } +}; + +void AssignObject(Object& destination, const Object& source) { + if (!destination.items || !source.items) return; + if (!ModernValues()) { + CopyObjectStorage(*destination.items, *source.items); + return; + } + Object pending{}; + Construct(pending); + if (!pending.items) return; + pending.items->SetOwner(destination.items->Owner()); + try { + for (const auto& pair : *source.items) { + ScopedValue placeholder; + SetType(placeholder.value, pair.value.type); + ScopedPair copied; + { + ScopedString key{std::string_view{Characters(pair.key)}}; + Construct(copied.pair.key, key.string); + CopyInto(copied.pair.value, placeholder.value); + } + const auto failures = AllocationFailures(); + ObjectLink* node; + bool inserted; + { + InlineText key; key.Assign(Characters(pair.key)); + TemporaryTreeEntry entry(key.text, copied.pair); + node = pending.items->Insert(entry.key.text, entry.pair, inserted); + } + if (inserted && AllocationFailures() == failures) { + node->pair.value.parent = destination.items->Owner(); + AssignNode(node->pair.value, pair.value); + } + } + } catch (...) { Destroy(pending); throw; } + auto previous = destination; + destination = pending; + Destroy(previous); } -Value& ObjectEntry(Object& object, std::string_view key) { - auto position = ObjectPosition(object, key); - if (position != object.items->end() && std::strcmp(Characters(position->key), key.data()) == 0) return position->value; - Pair pair{}; - Construct(pair.key, key); - try { - Construct(pair.value); - auto& value = object.items->insert(position, pair)->value; - value.parent = object.items->owner; - BindChildren(value); - return value; - } catch (...) { - Destroy(pair.value); - Destroy(pair.key); - throw; +Value* ObjectAccess(Object& object, const String& input) { + if (!object.items || !input.text) return const_cast(&Defaults(TypeNull)); + if (!object.items->head) NotImplemented_nid_no_patch(__func__); + const auto failures = AllocationFailures(); + ObjectLink* node; + { + InlineText key; key.Assign(Characters(input)); + ScopedString defaultKey{std::string_view{}}; + Pair source{}; source.key = defaultKey.string; + TemporaryTreeEntry entry(key.text, source); + bool inserted; + node = object.items->Insert(entry.key.text, entry.pair, inserted); } + if (AllocationFailures() != failures) return const_cast(&Defaults(TypeNull)); + if (!node || node->nil) NotImplemented_nid_no_patch(__func__); + AssignText(node->pair.key, View(input), true); + node->pair.value.parent = object.items->Owner(); + return &node->pair.value; +} + +Value& ObjectEntry(Object& object, std::string_view input) { + const auto failures = AllocationFailures(); + ScopedString key(input); + auto* value = ObjectAccess(object, key.string); + if (AllocationFailures() != failures) throw GuestOutOfMemory{}; + return *value; } using ParserReadCallback = int (APS5_VABI*)(char&, void*); @@ -1082,6 +1480,7 @@ private: return true; } static void appendUtf8(std::string& out, std::uint32_t code) { + if (!code) return; if (code < 0x80) out += static_cast(code); else if (code < 0x800) { out += static_cast(0xc0 | (code >> 6)); out += static_cast(0x80 | (code & 0x3f)); } else if (code < 0x10000) { out += static_cast(0xe0 | (code >> 12)); out += static_cast(0x80 | ((code >> 6) & 0x3f)); out += static_cast(0x80 | (code & 0x3f)); } @@ -1527,70 +1926,64 @@ void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object* self) { Construct(*self); } void APS5_VABI _ZN3sce4Json6ObjectC1ERKS1_(Object* self, const Object* other) { CopyObject(*self, *other); } void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object* self) { Destroy(*self); } Object* APS5_VABI _ZN3sce4Json6ObjectaSERKS1_(Object* self, const Object* other) { - if (self == other) return self; - Object copy{}; - CopyObject(copy, *other); - Destroy(*self); - *self = copy; + AssignObject(*self, *other); return self; } -void APS5_VABI _ZN3sce4Json6Object5clearEv(Object* self) { - for (auto& pair : *self->items) { - Destroy(pair.key); - Destroy(pair.value); - } - self->items->clear(); +void APS5_VABI _ZN3sce4Json6Object5clearEv(Object* self) { if (self->items) self->items->clear(); } +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object* self, const String* key) { return ObjectAccess(*self, *key); } +ObjectIterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(ObjectIterator* result, const Object* self) { + result->state = static_cast(Allocate(sizeof(ObjectIteratorState))); + if (result->state) result->state->position = self->items && self->items->head ? self->items->head->left : nullptr; + if (self->items && (!self->items->head || !self->items->head->left)) { Release(result->state); result->state = nullptr; } + return result; } -Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object* self, const String* key) { return &ObjectEntry(*self, View(*key)); } -ObjectIterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(ObjectIterator* result, const Object* self) { result->state = new ObjectIteratorState{self->items, self->items->begin()}; return result; } -ObjectIterator* APS5_VABI _ZNK3sce4Json6Object3endEv(ObjectIterator* result, const Object* self) { result->state = new ObjectIteratorState{self->items, self->items->end()}; return result; } -void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(ObjectIterator* self) { delete self->state; self->state = nullptr; } +ObjectIterator* APS5_VABI _ZNK3sce4Json6Object3endEv(ObjectIterator* result, const Object* self) { + result->state = static_cast(Allocate(sizeof(ObjectIteratorState))); + if (result->state) result->state->position = self->items ? self->items->head : nullptr; + if (self->items && !self->items->head) { Release(result->state); result->state = nullptr; } + return result; +} +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(ObjectIterator* self) { Release(self->state); self->state = nullptr; } ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEv(ObjectIterator* self) { - if (self->state && self->state->owner) { - if (self->state->position == self->state->owner->end()) throw std::out_of_range("sce::Json::Object::iterator: advance past the end"); - ++self->state->position; - } + if (self->state) self->state->position = NextObjectLink(self->state->position); return self; } Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const ObjectIterator* self) { - if (!self->state || !self->state->owner || self->state->position == self->state->owner->end()) throw std::out_of_range("sce::Json::Object::iterator: no current element"); - return &*self->state->position; + if (self->state) return reinterpret_cast(reinterpret_cast(self->state->position) + 48); + const auto* object = Defaults(TypeObject).object; + if (!object || !object->items) NotImplemented_nid_no_patch(__func__); + ObjectIterator fallback{}; + _ZNK3sce4Json6Object5beginEv(&fallback, object); + if (!fallback.state) NotImplemented_nid_no_patch(__func__); + auto* pair = reinterpret_cast(reinterpret_cast(fallback.state->position) + 48); + Release(fallback.state); + return pair; } Pair* APS5_VABI _ZNK3sce4Json6Object8iteratorptEv(const ObjectIterator* self) { return _ZNK3sce4Json6Object8iteratordeEv(self); } ObjectIterator* APS5_VABI _ZNK3sce4Json6Object4findERKNS0_6StringE(ObjectIterator* result, const Object* self, const String* key) { - const auto position = std::find_if(self->items->begin(), self->items->end(), [&](const Pair& pair) { return std::strcmp(Characters(pair.key), Characters(*key)) == 0; }); - result->state = new ObjectIteratorState{self->items, position}; + result->state = static_cast(Allocate(sizeof(ObjectIteratorState))); + if (result->state) result->state->position = self->items && key->text ? self->items->Find(Characters(*key)) : nullptr; + if (self->items && key->text && !self->items->head) { Release(result->state); result->state = nullptr; } return result; } bool APS5_VABI _ZNK3sce4Json6Object8iteratoreqERKS2_(const ObjectIterator* self, const ObjectIterator* other) { - if (!self->state || !other->state) return false; - return self->state->owner == other->state->owner && self->state->position == other->state->position; + return self->state && other->state && self->state->position == other->state->position; } bool APS5_VABI _ZNK3sce4Json6Object8iteratorneERKS2_(const ObjectIterator* self, const ObjectIterator* other) { - if (!self->state || !other->state) return false; - return !_ZNK3sce4Json6Object8iteratoreqERKS2_(self, other); + return self->state && other->state && self->state->position != other->state->position; } void APS5_VABI _ZN3sce4Json6Object8iteratorC1ERKS2_(ObjectIterator* self, const ObjectIterator* other) { - self->state = other->state ? new ObjectIteratorState(*other->state) : nullptr; + self->state = other->state ? static_cast(Allocate(sizeof(ObjectIteratorState))) : nullptr; + if (self->state) *self->state = *other->state; } -ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iterator7advanceEm(ObjectIterator* self, std::size_t count) { - if (self->state && self->state->owner) { - while (count) { - _ZN3sce4Json6Object8iteratorppEv(self); - --count; - } - } - return self; +void APS5_VABI _ZN3sce4Json6Object8iterator7advanceEm(ObjectIterator* self, std::size_t count) { + if (self->state) while (count--) self->state->position = NextObjectLink(self->state->position); } -bool APS5_VABI _ZNK3sce4Json6Object5emptyEv(const Object* self) { return self->items->empty(); } -std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object* self) { return self->items->size(); } +bool APS5_VABI _ZNK3sce4Json6Object5emptyEv(const Object* self) { return !self->items || self->items->empty(); } +std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object* self) { return self->items ? self->items->size() : 0; } void APS5_VABI _ZN3sce4Json6Object5eraseERKNS0_6StringE(Object* self, const String* key) { - if (key == nullptr || key->text == nullptr) throw std::invalid_argument("sce::Json::Object::erase: null key"); - auto it = std::find_if(self->items->begin(), self->items->end(), [&](const Pair& pair) { return std::strcmp(Characters(pair.key), Characters(*key)) == 0; }); - if (it == self->items->end()) return; - Destroy(it->key); - Destroy(it->value); - self->items->erase(it); + if (!self->items || !key->text) return; + self->items->Erase(self->items->Find(Characters(*key))); } void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value* self) { Construct(*self); } @@ -1717,11 +2110,8 @@ const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value* self, const char* const Value* result; try { if (self->type == TypeObject) { - for (const auto& pair : *self->object->items) - if (std::strcmp(Characters(pair.key), Characters(text)) == 0) { - Destroy(text); - return &pair.value; - } + const auto* node = self->object->items ? self->object->items->Find(Characters(text)) : nullptr; + if (node && !node->nil) { Destroy(text); return &node->pair.value; } } result = &MissingElement(self, static_cast(self->type == TypeObject ? 0x80848115 : 0x80848117), &text, nullptr); } catch (...) { @@ -1751,9 +2141,10 @@ void APS5_VABI _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(Value* self, ValueType type) const Value* APS5_VABI _ZNK3sce4Json5Value8getValueEm(const Value* self, std::size_t index) { return _ZNK3sce4Json5ValueixEm(self, index); } const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value* self, const String* key) { const Node& n = NodeOf(*self); - if (key != nullptr && n.type == TypeObject) - for (const auto& pair : *n.object->items) - if (std::strcmp(Characters(pair.key), Characters(*key)) == 0) return &pair.value; + if (key && n.type == TypeObject && n.object->items) { + const auto* node = n.object->items->Find(Characters(*key)); + if (node && !node->nil) return &node->pair.value; + } return &MissingElement(self, static_cast(n.type == TypeObject ? 0x80848115 : 0x80848117), key, nullptr); } Value* APS5_VABI _ZN3sce4Json5Value10referValueEm(Value* self, std::size_t index) { @@ -1772,9 +2163,9 @@ Value* APS5_VABI _ZN3sce4Json5Value10referValueEm(Value* self, std::size_t index Value* APS5_VABI _ZN3sce4Json5Value10referValueERKNS0_6StringE(Value* self, const String* key) { Node& n = NodeOf(*self); if (key == nullptr || n.type != TypeObject) return nullptr; - for (auto& pair : *n.object->items) - if (std::strcmp(Characters(pair.key), Characters(*key)) == 0) return &pair.value; - return nullptr; + if (!n.object->items) return nullptr; + auto* node = n.object->items->Find(Characters(*key)); + return node && !node->nil ? &node->pair.value : nullptr; } Array* APS5_VABI _ZN3sce4Json5Value10referArrayEv(Value* self) { Node& n = NodeOf(*self); @@ -1998,27 +2389,42 @@ void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair* self) { } ObjectIterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(ObjectIterator* result, Object* self, const Pair* pair) { - auto it = ObjectPosition(*self, View(pair->key)); - auto state = std::make_unique(ObjectIteratorState{self->items, it}); - if (it != self->items->end() && std::strcmp(Characters(it->key), Characters(pair->key)) == 0) { - result->state = state.release(); + result->state = static_cast(Allocate(sizeof(ObjectIteratorState))); + if (result->state) result->state->position = nullptr; + if (!self->items) return result; + if (!ModernValues()) { + ObjectLink* node; + { + InlineText key; key.Assign(Characters(pair->key)); + TemporaryTreeEntry entry(key.text, *pair); + bool inserted; + node = self->items->Insert(entry.key.text, entry.pair, inserted); + } + if (!node) { Release(result->state); result->state = nullptr; } + else if (result->state) result->state->position = node; return result; } - Pair copy{}; - copy.reserved = 0; - Construct(copy.key, pair->key); - try { - CopyInto(copy.value, pair->value); - state->position = self->items->insert(it, copy); - state->position->value.parent = self->items->owner; - BindChildren(state->position->value); - result->state = state.release(); - return result; - } catch (...) { - Destroy(copy.value); - Destroy(copy.key); - throw; + ScopedValue value; + value.value.parent = self->items->Owner(); + AssignNode(value.value, pair->value); + const auto failures = AllocationFailures(); + ObjectLink* node; + bool inserted; + { + InlineText key; key.Assign(Characters(pair->key)); + ScopedString copiedKey(pair->key); + Pair source{}; source.key = copiedKey.string; + TemporaryTreeEntry entry(key.text, source); + node = self->items->Insert(entry.key.text, entry.pair, inserted); } + if (inserted && AllocationFailures() == failures) { + TransferPayload(node->pair.value, value.value); + node->pair.value.parent = self->items->Owner(); + BindChildren(node->pair.value); + } + if (!node) { Release(result->state); result->state = nullptr; } + else if (result->state) result->state->position = node; + return result; } ArrayIterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(ArrayIterator* self, int) { @@ -2044,18 +2450,15 @@ ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(ObjectIterator* self, ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iteratoraSERKS2_(ObjectIterator* self, const ObjectIterator* other) { if (self->state) { - if (other->state && other->state->owner) { - *self->state = *other->state; - } else { - delete self->state; - self->state = nullptr; - } + if (other->state && other->state->position) *self->state = *other->state; + else { Release(self->state); self->state = nullptr; } } return self; } void APS5_VABI _ZN3sce4Json6Object8iteratorC1Ev(ObjectIterator* self) { - self->state = new ObjectIteratorState{}; + self->state = static_cast(Allocate(sizeof(ObjectIteratorState))); + if (self->state) self->state->position = nullptr; } void APS5_VABI _ZN3sce4Json6Object4PairC1Ev(Pair* self) { diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index b03e1c6eda..dbd0af06a3 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -151,7 +151,7 @@ void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value*, Value*); void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); void APS5_VABI _ZN3sce4Json6Object8iteratorC1ERKS2_(Iterator*, const Iterator*); -Iterator* APS5_VABI _ZN3sce4Json6Object8iterator7advanceEm(Iterator*, std::size_t); +void APS5_VABI _ZN3sce4Json6Object8iterator7advanceEm(Iterator*, std::size_t); void APS5_VABI _ZN3sce4Json6Object5eraseERKNS0_6StringE(Object*, const String*); } @@ -657,7 +657,7 @@ static void ObjectKeyOrder() { static void ParseDuplicateKeys() { Value root{}; _ZN3sce4Json5ValueC1Ev(&root); - for (const char* text : {R"({"a":1,"a":2})", R"({"a":[1,2],"a":2})", R"({"a":{"old":true},"a":2})", R"({"a":1,"a\u0000suffix":2})"}) { + for (const char* text : {R"({"a":1,"a":2})", R"({"a":[1,2],"a":2})", R"({"a":{"old":true},"a":2})"}) { Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, text, std::strlen(text)) == 0); Require(_ZNK3sce4Json5Value5countEv(&root) == 1); Require(Serialize(root) == R"({"a":2})"); @@ -815,7 +815,7 @@ static void ObjectOperations() { _ZNK3sce4Json6Object5beginEv(&begin, &object); Iterator copy{}; _ZN3sce4Json6Object8iteratorC1ERKS2_(©, &begin); - Require(_ZN3sce4Json6Object8iterator7advanceEm(©, 2) == ©); + _ZN3sce4Json6Object8iterator7advanceEm(©, 2); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&_ZNK3sce4Json6Object8iteratordeEv(©)->key), "gamma") == 0); _ZN3sce4Json6Object8iterator7advanceEm(©, 1); bool endRejected = false; diff --git a/core/libs/tests/GuestJsonObjectAllocator.cpp b/core/libs/tests/GuestJsonObjectAllocator.cpp new file mode 100644 index 0000000000..d0d72b2f7d --- /dev/null +++ b/core/libs/tests/GuestJsonObjectAllocator.cpp @@ -0,0 +1,510 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Object { void* storage; }; +struct Iterator { void* state; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct StoredText { const char* characters; std::uint32_t length; std::uint32_t capacity; std::uint64_t reserved; }; +struct TreeNode { TreeNode* parent; TreeNode* left; TreeNode* right; StoredText key; Pair pair; std::uint8_t color; std::uint8_t nil; std::uint8_t reserved[6]; }; +struct TreeHeader { TreeNode* head; std::size_t count; std::uint64_t reserved; Value* owner; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +static_assert(sizeof(TreeNode) == 104 && offsetof(TreeNode, pair) == 48 && offsetof(TreeNode, color) == 96); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6StringE(Value*, const String*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value*, const Object*); +std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value*); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value*, std::size_t); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectC1ERKS1_(Object*, const Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Object* APS5_VABI _ZN3sce4Json6ObjectaSERKS1_(Object*, const Object*); +void APS5_VABI _ZN3sce4Json6Object5clearEv(Object*); +std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +void APS5_VABI _ZN3sce4Json6Object5eraseERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(Iterator*, Object*, const Pair*); +void APS5_VABI _ZN3sce4Json6Object8iteratorC1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratoraSERKS2_(Iterator*, const Iterator*); +Iterator* APS5_VABI _ZNK3sce4Json6Object3endEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZNK3sce4Json6Object4findERKNS0_6StringE(Iterator*, const Object*, const String*); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEv(Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringC1Ev(String*); +String* APS5_VABI _ZN3sce4Json6String6appendEPKcm(String*, const char*, std::size_t); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ObjectAllocationTrace { + std::map live; + std::vector requests; + std::set headers; + int budget = -1; + bool failOnce = false; + unsigned notifications = 0; +}; +static void* APS5_VABI AllocateObject(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.failOnce) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeObject(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Object teardown must poison the header before freeing it"); + } + Require(trace.live.erase(memory) == 1, "Object resources must return through their owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyObject(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Object notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ObjectVtable[]{nullptr, nullptr, reinterpret_cast(AllocateObject), reinterpret_cast(FreeObject), reinterpret_cast(NotifyObject)}; +struct ObjectAllocator { void* const* vtable = ObjectVtable; }; +static bool Owned(const ObjectAllocationTrace& trace, const void* memory, std::size_t bytes) { + const auto found = trace.live.find(const_cast(memory)); return found != trace.live.end() && found->second == bytes; +} +static Value* Entry(Object& object, const char* key) { + String text{}; _ZN3sce4Json6StringC1EPKc(&text, key); + auto* result = _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &text); + _ZN3sce4Json6StringD1Ev(&text); return result; +} +static void CheckTree(const Object& object, ObjectAllocationTrace& trace, const std::map& entries) { + const auto* header = static_cast(object.storage); + Require(Owned(trace, header, 32) && Owned(trace, header->head, 104) && !header->owner && header->count == entries.size(), "Object header and sentinel must be actual allocator-owned resources"); + trace.headers.insert(object.storage); + auto* head = header->head; + Require(head->nil == 1 && head->color == 0, "Sentinel flags must mark a black nil node"); + std::vector nodes; + std::function visit = [&](TreeNode* node, TreeNode* parent) -> unsigned { + if (node == head) return 1; + Require(Owned(trace, node, 104) && !node->nil && node->color <= 1 && node->parent == parent, "Every tree node must retain its native allocation and parent link"); + Require(node->color == 0 || (node->left->color == 0 && node->right->color == 0), "Red nodes must have black children"); + const auto left = visit(node->left, node); nodes.push_back(node); const auto right = visit(node->right, node); + Require(left == right, "Every path must retain the same black height"); + Require(Owned(trace, node->pair.key.storage, 24) && !node->pair.value.parent && !node->pair.value.auxiliary, "Node pairs must expose native string and value ownership"); + const auto found = entries.find(node->key.characters); + Require(found != entries.end() && found->second == &node->pair.value && std::strcmp(node->key.characters, _ZNK3sce4Json6String5c_strEv(&node->pair.key)) == 0, "Rotations must preserve entry identity and both key representations"); + return left + (node->color == 0); + }; + Require(head->parent->color == 0, "Tree root must remain black"); visit(head->parent, head); + Require(nodes.size() == entries.size() && (nodes.empty() ? head->left == head && head->right == head : head->left == nodes.front() && head->right == nodes.back()), "Sentinel endpoints must match the ordered entries"); + auto expected = entries.begin(); + for (const auto* node : nodes) { Require(expected->first == node->key.characters, "Tree iteration must preserve key order"); ++expected; } +} +static std::vector Keys(const Object& object) { + Iterator current{}, end{}; _ZNK3sce4Json6Object5beginEv(¤t, &object); _ZNK3sce4Json6Object3endEv(&end, &object); + std::vector keys; + while (*static_cast(current.state) != *static_cast(end.state)) { + keys.emplace_back(_ZNK3sce4Json6String5c_strEv(&_ZNK3sce4Json6Object8iteratordeEv(¤t)->key)); + _ZN3sce4Json6Object8iteratorppEv(¤t); + } + _ZN3sce4Json6Object8iteratorD1Ev(¤t); _ZN3sce4Json6Object8iteratorD1Ev(&end); return keys; +} +static void CheckSerialized(Value& value, const char* expected) { + String output{}; _ZN3sce4Json6StringC1Ev(&output); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output) == 0 && std::strcmp(_ZNK3sce4Json6String5c_strEv(&output), expected) == 0, + "Object keys and string values must retain their distinct binary serialization behavior"); + _ZN3sce4Json6StringD1Ev(&output); +} +static void CheckBinaryKeys() { + String key{}; _ZN3sce4Json6StringC1EPKc(&key, "a"); + const char suffix[]{0, 'b'}; _ZN3sce4Json6String6appendEPKcm(&key, suffix, sizeof(suffix)); + Value value{}; _ZN3sce4Json5ValueC1ERKNS0_6StringE(&value, &key); + CheckSerialized(value, R"("a\u0000b")"); _ZN3sce4Json5ValueD1Ev(&value); + Object object{}; _ZN3sce4Json6ObjectC1Ev(&object); _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &key); + _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(&value, &object); CheckSerialized(value, R"({"a":null})"); + _ZN3sce4Json5ValueD1Ev(&value); _ZN3sce4Json6ObjectD1Ev(&object); _ZN3sce4Json6StringD1Ev(&key); + _ZN3sce4Json5ValueC1Ev(&value); + const char input[] = R"({"a":1,"a\u0000suffix":2})"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, input, sizeof(input) - 1) == 0 && _ZNK3sce4Json5Value5countEv(&value) == 2, + "The inspected parser drops the decoded zero byte without treating the two keys as duplicates"); + CheckSerialized(value, R"({"a":1,"asuffix":2})"); _ZN3sce4Json5ValueD1Ev(&value); + _ZN3sce4Json5ValueC1Ev(&value); + const char string[] = R"("a\u0000b")"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, string, sizeof(string) - 1) == 0, "Parse a zero-byte escape in a string value"); + CheckSerialized(value, R"("ab")"); _ZN3sce4Json5ValueD1Ev(&value); +} +static void CheckFailures(ObjectAllocationTrace& trace, const Value* moduleFallback) { + const auto baseline = trace.live.size(); + for (bool once : {false, true}) for (int budget = 0; budget <= 8; ++budget) { + Object object{}; _ZN3sce4Json6ObjectC1Ev(&object); Entry(object, "a"); Entry(object, "c"); + trace.headers.insert(object.storage); + String text{}; _ZN3sce4Json6StringC1EPKc(&text, "b"); + trace.budget = budget; trace.failOnce = once; + const auto notifications = trace.notifications; + auto* result = _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &text); + trace.budget = -1; + const bool inserted = once ? budget != 4 : budget >= 5; + const auto expected = inserted ? std::size_t{3} : std::size_t{2}; + Require(_ZNK3sce4Json6Object4sizeEv(&object) == expected, "Failed object insertion must retain the observed partial entry count"); + const bool fallback = budget < 7; + Require((result == moduleFallback) == fallback, "Insertion must preserve its operation-level versus late-assignment failure result"); + std::vector expectedKeys{"a", "c"}; + if (inserted) { + if (once && (budget == 0 || budget == 2)) expectedKeys = {"", "a", "c"}; + else expectedKeys = {"a", budget == 8 ? "b" : "", "c"}; + } + Require(Keys(object) == expectedKeys, "Partial allocation failures must preserve the observed public keys and ordering"); + static constexpr unsigned persistentNotifications[]{3, 3, 3, 2, 1, 2, 1, 1, 0}; + Require(trace.notifications - notifications == (once ? static_cast(budget < 8) : persistentNotifications[budget]), "Object failures must notify for each real failed request"); + _ZN3sce4Json6StringD1Ev(&text); _ZN3sce4Json6ObjectD1Ev(&object); + Require(trace.live.size() == baseline, "All partial insertion states must be destructible without leaks"); + } + trace.failOnce = false; +} +static std::string AssignmentContents(const Object& object) { + std::string result; + auto* head = static_cast(object.storage)->head; + auto* node = head->left; + while (node != head) { + result += node->pair.key.storage ? static_cast(node->pair.key.storage)->characters : "~"; + result += ":" + std::to_string(node->pair.value.type) + ":" + std::to_string(node->pair.value.payload) + ";"; + if (!node->right->nil) { node = node->right; while (!node->left->nil) node = node->left; } + else { auto* parent = node->parent; while (!parent->nil && node == parent->right) { node = parent; parent = parent->parent; } node = parent; } + } + return result; +} +static void CheckInsertions(ObjectAllocationTrace& trace) { + struct Case { bool duplicate; bool once; int budget; std::size_t count; unsigned notifications; bool state; const char* result; const char* contents; }; + const Case cases[]{ + {0, 0, -1, 3, 0, 1, "b", "a:2:11;b:2:22;c:2:33;"}, + {0, 0, 0, 2, 4, 0, "NONE", "a:2:11;c:2:33;"}, + {0, 0, 1, 2, 3, 1, "END", "a:2:11;c:2:33;"}, + {0, 0, 2, 2, 3, 1, "END", "a:2:11;c:2:33;"}, + {0, 0, 3, 2, 4, 1, "END", "a:2:11;c:2:33;"}, + {0, 0, 4, 2, 3, 1, "END", "a:2:11;c:2:33;"}, + {0, 0, 5, 2, 2, 1, "END", "a:2:11;c:2:33;"}, + {0, 0, 6, 2, 2, 1, "END", "a:2:11;c:2:33;"}, + {0, 0, 7, 2, 1, 1, "END", "a:2:11;c:2:33;"}, + {0, 0, 8, 3, 2, 1, "NULL", "a:2:11;~:0:11936128518282651045;c:2:33;"}, + {0, 0, 9, 3, 1, 1, "NULL", "a:2:11;~:0:11936128518282651045;c:2:33;"}, + {0, 0, 10, 3, 1, 1, "", "a:2:11;:0:11936128518282651045;c:2:33;"}, + {0, 0, 11, 3, 0, 1, "b", "a:2:11;b:2:22;c:2:33;"}, + {0, 0, 12, 3, 0, 1, "b", "a:2:11;b:2:22;c:2:33;"}, + {0, 1, -1, 3, 0, 1, "b", "a:2:11;b:2:22;c:2:33;"}, + {0, 1, 0, 3, 1, 0, "NONE", "a:2:11;b:2:22;c:2:33;"}, + {0, 1, 1, 3, 1, 1, "b", "b:0:11936128518282651045;a:2:11;c:2:33;"}, + {0, 1, 2, 3, 1, 1, "NULL", "a:2:11;~:0:11936128518282651045;c:2:33;"}, + {0, 1, 3, 3, 1, 1, "", "a:2:11;:0:11936128518282651045;c:2:33;"}, + {0, 1, 4, 3, 1, 1, "b", "b:0:11936128518282651045;a:2:11;c:2:33;"}, + {0, 1, 5, 3, 1, 1, "NULL", "a:2:11;~:0:11936128518282651045;c:2:33;"}, + {0, 1, 6, 3, 1, 1, "", "a:2:11;:0:11936128518282651045;c:2:33;"}, + {0, 1, 7, 2, 1, 1, "END", "a:2:11;c:2:33;"}, + {0, 1, 8, 3, 1, 1, "b", "a:2:11;b:0:11936128518282651045;c:2:33;"}, + {0, 1, 9, 3, 1, 1, "NULL", "a:2:11;~:0:11936128518282651045;c:2:33;"}, + {0, 1, 10, 3, 1, 1, "", "a:2:11;:0:11936128518282651045;c:2:33;"}, + {0, 1, 11, 3, 0, 1, "b", "a:2:11;b:2:22;c:2:33;"}, + {0, 1, 12, 3, 0, 1, "b", "a:2:11;b:2:22;c:2:33;"}, + {1, 0, -1, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 0, 0, 2, 4, 0, "NONE", "a:2:11;c:2:33;"}, + {1, 0, 1, 2, 3, 1, "END", "a:2:11;c:2:33;"}, + {1, 0, 2, 2, 3, 1, "END", "a:2:11;c:2:33;"}, + {1, 0, 3, 2, 4, 1, "END", "a:2:11;c:2:33;"}, + {1, 0, 4, 2, 3, 1, "END", "a:2:11;c:2:33;"}, + {1, 0, 5, 2, 1, 1, "a", "a:2:11;c:2:33;"}, + {1, 0, 6, 2, 1, 1, "a", "a:2:11;c:2:33;"}, + {1, 0, 7, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 0, 8, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 0, 9, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 0, 10, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 0, 11, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 0, 12, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, -1, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 0, 2, 1, 0, "NONE", "a:2:11;c:2:33;"}, + {1, 1, 1, 3, 1, 1, "a", "a:0:11936128518282651045;a:2:11;c:2:33;"}, + {1, 1, 2, 2, 1, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 3, 2, 1, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 4, 3, 1, 1, "a", "a:0:11936128518282651045;a:2:11;c:2:33;"}, + {1, 1, 5, 2, 1, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 6, 2, 1, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 7, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 8, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 9, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 10, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 11, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + {1, 1, 12, 2, 0, 1, "a", "a:2:11;c:2:33;"}, + }; + const auto baseline = trace.live.size(); + for (const auto& test : cases) { + Object object{}; _ZN3sce4Json6ObjectC1Ev(&object); + auto* first = Entry(object, "a"); _ZN3sce4Json5Value3setEl(first, 11); + _ZN3sce4Json5Value3setEl(Entry(object, "c"), 33); + Pair pair{}; _ZN3sce4Json6StringC1EPKc(&pair.key, test.duplicate ? "a" : "b"); + _ZN3sce4Json5ValueC1Ev(&pair.value); _ZN3sce4Json5Value3setEl(&pair.value, 22); + const auto notifications = trace.notifications; + trace.budget = test.budget; trace.failOnce = test.once; + Iterator result{}; + Require(_ZN3sce4Json6Object6insertERKNS1_4PairE(&result, &object, &pair) == &result, "Insertion must return its result iterator"); + trace.budget = -1; trace.failOnce = false; + Require((result.state != nullptr) == test.state, "Insertion must preserve independent iterator allocation failure"); + auto* node = result.state ? *static_cast(result.state) : nullptr; + const char* key = !node ? "NONE" : node->nil ? "END" : node->pair.key.storage ? static_cast(node->pair.key.storage)->characters : "NULL"; + Require(std::strcmp(key, test.result) == 0, "Insertion must return the reference entry or end node on failure"); + Require(_ZNK3sce4Json6Object4sizeEv(&object) == test.count && AssignmentContents(object) == test.contents, "Insertion must preserve partial keys, inactive payloads and duplicate values"); + Require(trace.notifications - notifications == test.notifications, "Insertion must notify every failed owned allocation"); + Require(first->type == 2 && first->payload == 11 && pair.value.type == 2 && pair.value.payload == 22, "Insertion must preserve existing entries and its input value"); + _ZN3sce4Json6Object8iteratorD1Ev(&result); _ZN3sce4Json6StringD1Ev(&pair.key); _ZN3sce4Json5ValueD1Ev(&pair.value); _ZN3sce4Json6ObjectD1Ev(&object); + Require(trace.live.size() == baseline, "Insertion failure must release temporary allocations and iterator state"); + } +} +static void CheckIteratorFailures(ObjectAllocationTrace& trace) { + const auto baseline = trace.live.size(); + Object object{}; _ZN3sce4Json6ObjectC1Ev(&object); + Entry(object, "a"); + String key{}; _ZN3sce4Json6StringC1EPKc(&key, "a"); + Iterator source{}; _ZNK3sce4Json6Object5beginEv(&source, &object); + const auto retained = trace.live.size(); + for (unsigned operation = 0; operation < 5; ++operation) { + const auto requests = trace.requests.size(); + const auto notifications = trace.notifications; + Iterator failed{}; trace.budget = 0; + switch (operation) { + case 0: _ZN3sce4Json6Object8iteratorC1Ev(&failed); break; + case 1: _ZN3sce4Json6Object8iteratorC1ERKS2_(&failed, &source); break; + case 2: _ZNK3sce4Json6Object5beginEv(&failed, &object); break; + case 3: _ZNK3sce4Json6Object3endEv(&failed, &object); break; + case 4: _ZNK3sce4Json6Object4findERKNS0_6StringE(&failed, &object, &key); break; + } + trace.budget = -1; + Require(!failed.state && trace.live.size() == retained, "Failed iterator allocation must retain no state or borrowed resources"); + Require(trace.requests.size() == requests + 1 && trace.requests.back() == 8 && trace.notifications == notifications + 1, "Iterator failure must report its actual eight-byte allocation"); + _ZN3sce4Json6Object8iteratorD1Ev(&failed); + } + Iterator absent{}, copy{}, target{}, end{}; + const auto requests = trace.requests.size(); + trace.budget = 0; + _ZN3sce4Json6Object8iteratorC1ERKS2_(©, &absent); + Require(_ZN3sce4Json6Object8iteratoraSERKS2_(&absent, &source) == &absent && !absent.state && !copy.state && trace.requests.size() == requests, "Copying absent state and assigning into absent state must not allocate"); + trace.budget = -1; + _ZN3sce4Json6Object8iteratorC1ERKS2_(&target, &source); + _ZNK3sce4Json6Object3endEv(&end, &object); + auto* storage = target.state; + trace.budget = 0; + _ZN3sce4Json6Object8iteratoraSERKS2_(&target, &end); + Require(target.state == storage && *static_cast(target.state) == static_cast(object.storage)->head, "Assignment from end must preserve owned iterator state and copy the sentinel"); + _ZN3sce4Json6Object8iteratoraSERKS2_(&target, &absent); + Require(!target.state && trace.live.size() == retained + 1, "Assignment from absent state must release the destination iterator state"); + trace.budget = -1; + _ZN3sce4Json6Object8iteratorD1Ev(&end); _ZN3sce4Json6Object8iteratorD1Ev(&target); + Object missing{}; + _ZNK3sce4Json6Object5beginEv(&target, &missing); + Require(Owned(trace, target.state, 8) && !*static_cast(target.state), "A missing object must produce an allocated iterator with no position"); + _ZN3sce4Json6Object8iteratorD1Ev(&target); + _ZN3sce4Json6Object8iteratorD1Ev(&source); _ZN3sce4Json6StringD1Ev(&key); _ZN3sce4Json6ObjectD1Ev(&object); + Require(trace.live.size() == baseline, "Iterator failure and assignment paths must release all owned state"); +} +static void CheckNestedAssignment(ObjectAllocationTrace& trace) { + const auto baseline = trace.live.size(); + Value source{}, destination{}; + _ZN3sce4Json5ValueC1Ev(&source); _ZN3sce4Json5ValueC1Ev(&destination); + const char input[] = "{\"nested\":{\"x\":1},\"array\":[2],\"text\":\"abc\",\"bool\":true,\"real\":1.25,\"nil\":null}"; + const char previous[] = "{\"old\":0}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&source, input, sizeof(input) - 1) == 0, "Parse nested assignment source"); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&destination, previous, sizeof(previous) - 1) == 0, "Parse nested assignment destination"); + auto* target = reinterpret_cast(destination.payload); + _ZN3sce4Json6ObjectaSERKS1_(target, reinterpret_cast(source.payload)); + String expected{}, actual{}; + _ZN3sce4Json6StringC1Ev(&expected); _ZN3sce4Json6StringC1Ev(&actual); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&source, &expected) == 0 && _ZN3sce4Json5Value9serializeERNS0_6StringE(&destination, &actual) == 0, "Serialize nested assignment results"); + Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&expected), _ZNK3sce4Json6String5c_strEv(&actual)) == 0, "Assignment must preserve nested collections, strings and scalar types"); + auto* header = static_cast(target->storage); + Require(header->owner == &destination, "Assigned root object must retain its owner"); + Iterator iterator{}; _ZNK3sce4Json6Object5beginEv(&iterator, target); + for (std::size_t index = 0; index < header->count; ++index) { + const auto* pair = _ZNK3sce4Json6Object8iteratordeEv(&iterator); + Require(pair->value.parent == &destination, "Assigned entries must bind to the destination value"); + if (pair->value.type == 6 || pair->value.type == 7) { + auto* nestedStorage = *reinterpret_cast(pair->value.payload); + Value* owner = nullptr; std::memcpy(&owner, static_cast(nestedStorage) + 24, sizeof(owner)); + Require(owner == &pair->value, "Nested collection ownership must bind to the assigned entry"); + } + _ZN3sce4Json6Object8iteratorppEv(&iterator); + } + _ZN3sce4Json6Object8iteratorD1Ev(&iterator); + _ZN3sce4Json6StringD1Ev(&actual); _ZN3sce4Json6StringD1Ev(&expected); + _ZN3sce4Json5ValueD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); + Require(trace.live.size() == baseline, "Nested assignment must release all resources"); +} +static void CheckAssignments(ObjectAllocationTrace& trace) { + struct Case { bool once; int budget; std::size_t count; unsigned notifications; const char* contents; }; + const Case cases[]{ + {0, -1, 2, 0, "a:2:11;c:2:33;"}, + {0, 0, 1, 1, "z:2:99;"}, + {0, 1, 1, 1, "z:2:99;"}, + {0, 2, 0, 6, ""}, + {0, 3, 0, 7, ""}, + {0, 4, 0, 6, ""}, + {0, 5, 0, 7, ""}, + {0, 6, 0, 6, ""}, + {0, 7, 0, 6, ""}, + {0, 8, 0, 5, ""}, + {0, 9, 0, 5, ""}, + {0, 10, 0, 4, ""}, + {0, 11, 1, 4, "~:2:0;"}, + {0, 12, 1, 4, "~:2:0;"}, + {0, 13, 1, 4, ":2:0;"}, + {0, 14, 1, 3, "a:2:11;"}, + {0, 15, 1, 4, "a:2:11;"}, + {0, 16, 1, 3, "a:2:11;"}, + {0, 17, 1, 4, "a:2:11;"}, + {0, 18, 1, 3, "a:2:11;"}, + {0, 19, 1, 3, "a:2:11;"}, + {0, 20, 1, 2, "a:2:11;"}, + {0, 21, 1, 2, "a:2:11;"}, + {0, 22, 1, 1, "a:2:11;"}, + {0, 23, 2, 2, "a:2:11;~:2:0;"}, + {0, 24, 2, 1, "a:2:11;~:2:0;"}, + {0, 25, 2, 1, "a:2:11;:2:0;"}, + {0, 26, 2, 0, "a:2:11;c:2:33;"}, + {1, -1, 2, 0, "a:2:11;c:2:33;"}, + {1, 0, 1, 1, "z:2:99;"}, + {1, 1, 1, 1, "z:2:99;"}, + {1, 2, 2, 1, "~:2:11;c:2:33;"}, + {1, 3, 2, 1, ":2:11;c:2:33;"}, + {1, 4, 2, 1, "~:2:11;c:2:33;"}, + {1, 5, 2, 1, ":2:11;c:2:33;"}, + {1, 6, 2, 1, "a:2:0;c:2:33;"}, + {1, 7, 2, 1, "a:2:0;c:2:33;"}, + {1, 8, 2, 1, "~:2:0;c:2:33;"}, + {1, 9, 2, 1, ":2:0;c:2:33;"}, + {1, 10, 1, 1, "c:2:33;"}, + {1, 11, 2, 1, "a:2:0;c:2:33;"}, + {1, 12, 2, 1, "~:2:0;c:2:33;"}, + {1, 13, 2, 1, ":2:0;c:2:33;"}, + {1, 14, 2, 1, "a:2:11;~:2:33;"}, + {1, 15, 2, 1, "a:2:11;:2:33;"}, + {1, 16, 2, 1, "a:2:11;~:2:33;"}, + {1, 17, 2, 1, "a:2:11;:2:33;"}, + {1, 18, 2, 1, "c:2:0;a:2:11;"}, + {1, 19, 2, 1, "c:2:0;a:2:11;"}, + {1, 20, 2, 1, "a:2:11;~:2:0;"}, + {1, 21, 2, 1, "a:2:11;:2:0;"}, + {1, 22, 1, 1, "a:2:11;"}, + {1, 23, 2, 1, "a:2:11;c:2:0;"}, + {1, 24, 2, 1, "a:2:11;~:2:0;"}, + {1, 25, 2, 1, "a:2:11;:2:0;"}, + {1, 26, 2, 0, "a:2:11;c:2:33;"}, + }; + const auto baseline = trace.live.size(); + for (const auto& test : cases) { + Object source{}, destination{}; + _ZN3sce4Json6ObjectC1Ev(&source); _ZN3sce4Json6ObjectC1Ev(&destination); + _ZN3sce4Json5Value3setEl(Entry(source, "a"), 11); + _ZN3sce4Json5Value3setEl(Entry(source, "c"), 33); + _ZN3sce4Json5Value3setEl(Entry(destination, "z"), 99); + Value owner{}; + static_cast(destination.storage)->owner = &owner; + auto* previous = destination.storage; + trace.headers.insert(previous); + const auto notifications = trace.notifications; + trace.budget = test.budget; trace.failOnce = test.once; + Require(_ZN3sce4Json6ObjectaSERKS1_(&destination, &source) == &destination, "Object assignment must return its destination"); + trace.budget = -1; trace.failOnce = false; + Require((destination.storage == previous) == (test.budget == 0 || test.budget == 1), "Only initial replacement allocation failures preserve the old object header"); + Require(static_cast(destination.storage)->owner == &owner, "Replacement trees must retain their destination owner"); + Require(_ZNK3sce4Json6Object4sizeEv(&destination) == test.count && AssignmentContents(destination) == test.contents, "Assignment failure must retain the reference partial entries and values"); + Require(trace.notifications - notifications == test.notifications, "Assignment must report each failed owned allocation"); + Require(AssignmentContents(source) == "a:2:11;c:2:33;", "Assignment must preserve the source"); + trace.headers.insert(destination.storage); + _ZN3sce4Json6ObjectD1Ev(&destination); _ZN3sce4Json6ObjectD1Ev(&source); + Require(trace.live.size() == baseline, "Assignment failure must reclaim temporary and replaced resources"); + } + Object object{}; _ZN3sce4Json6ObjectC1Ev(&object); + _ZN3sce4Json5Value3setEl(Entry(object, "a"), 11); + auto* previous = object.storage; + trace.headers.insert(previous); + _ZN3sce4Json6ObjectaSERKS1_(&object, &object); + Require(object.storage != previous && AssignmentContents(object) == "a:2:11;", "Modern self-assignment must rebuild while preserving values"); + trace.headers.insert(object.storage); + _ZN3sce4Json6ObjectD1Ev(&object); + Require(trace.live.size() == baseline, "Self-assignment must release its previous tree"); +} +int main() { + ObjectAllocationTrace trace; ObjectAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize object allocation contract"); + const auto baseline = trace.live.size(); + for (int budget : {0, 1}) { + Object partial{}; trace.budget = budget; _ZN3sce4Json6ObjectC1Ev(&partial); trace.budget = -1; + Require(!partial.storage && _ZNK3sce4Json6Object4sizeEv(&partial) == 0 && trace.live.size() == baseline, "Failed object construction must reclaim partial header ownership"); + _ZN3sce4Json6ObjectD1Ev(&partial); + } + Object object{}; _ZN3sce4Json6ObjectC1Ev(&object); + std::map entries; + CheckTree(object, trace, entries); + for (unsigned index = 0; index < 32; ++index) { + char key[3]; std::snprintf(key, sizeof(key), "%02u", index * 13 % 32); + auto* value = Entry(object, key); _ZN3sce4Json5Value3setEl(value, index); + entries.emplace(key, value); CheckTree(object, trace, entries); + } + Iterator current{}, copy{}; _ZNK3sce4Json6Object5beginEv(¤t, &object); _ZN3sce4Json6Object8iteratorC1ERKS2_(©, ¤t); + Require(Owned(trace, current.state, 8) && Owned(trace, copy.state, 8) && current.state != copy.state && *static_cast(current.state) == *static_cast(copy.state), "Object iterator copies must own one independent node pointer"); + _ZN3sce4Json6Object8iteratorD1Ev(©); _ZN3sce4Json6Object8iteratorD1Ev(¤t); + const auto beforeCopies = trace.live.size(); + for (int budget = 0; budget <= 10; ++budget) { + Object duplicate{}; trace.budget = budget; _ZN3sce4Json6ObjectC1ERKS1_(&duplicate, &object); trace.budget = -1; + Require(_ZNK3sce4Json6Object4sizeEv(&duplicate) == (budget >= 4 ? 32 : 0), "Partial tree copies retain the source count once the root allocation succeeds"); + if (duplicate.storage) trace.headers.insert(duplicate.storage); + _ZN3sce4Json6ObjectD1Ev(&duplicate); Require(trace.live.size() == beforeCopies, "Partial tree copies must release their actual nodes"); + } + for (unsigned index = 0; index < 32; ++index) { + char key[3]; std::snprintf(key, sizeof(key), "%02u", index * 7 % 32); + String text{}; _ZN3sce4Json6StringC1EPKc(&text, key); + const auto requests = trace.requests.size(); + const auto notifications = trace.notifications; + trace.budget = 0; + _ZN3sce4Json6Object5eraseERKNS0_6StringE(&object, &text); + _ZN3sce4Json6Object5eraseERKNS0_6StringE(&object, &text); + trace.budget = -1; + Require(trace.requests.size() == requests && trace.notifications == notifications, "Existing and missing key erasure must work without allocating or reporting allocation failure"); + _ZN3sce4Json6StringD1Ev(&text); + entries.erase(key); CheckTree(object, trace, entries); + } + _ZN3sce4Json6Object5clearEv(&object); _ZN3sce4Json6ObjectD1Ev(&object); + Require(trace.live.size() == baseline, "Tree operations must release all owned resources"); + Value null{}; _ZN3sce4Json5ValueC1Ev(&null); + const auto* moduleFallback = _ZNK3sce4Json5ValueixEm(&null, 0); + _ZN3sce4Json5ValueD1Ev(&null); + CheckFailures(trace, moduleFallback); + CheckIteratorFailures(trace); + CheckInsertions(trace); + CheckAssignments(trace); + CheckNestedAssignment(trace); + CheckBinaryKeys(); + Require(trace.live.size() == baseline, "Binary-key operations must release their storage"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate object allocation contract"); + return 0; +} diff --git a/core/libs/tests/JsonCopyFailure.cpp b/core/libs/tests/JsonCopyFailure.cpp index 07c5701410..233ed22ac8 100644 --- a/core/libs/tests/JsonCopyFailure.cpp +++ b/core/libs/tests/JsonCopyFailure.cpp @@ -236,7 +236,7 @@ int main() { Iterator source{}; if (objectIterator) _ZNK3sce4Json6Object5beginEv(&source, object); else _ZNK3sce4Json5Array5beginEv(&source, array); - for (activeBudget = 0; activeBudget <= 1; ++activeBudget) { + for (activeBudget = 0; activeBudget < 1; ++activeBudget) { Iterator copy{}; bool succeeded = false; allocationBudget = activeBudget; @@ -248,7 +248,7 @@ int main() { ++failures; } allocationBudget = -1; - Require(succeeded == (!objectIterator || activeBudget == 1), "Host iterator allocation boundary changed"); + Require(succeeded, "Iterator copies must use the guest allocator without host allocations"); if (succeeded) Require(copy.state != source.state, "iterator copy shares owned storage"); if (objectIterator) _ZN3sce4Json6Object8iteratorD1Ev(©); else _ZN3sce4Json5Array8iteratorD1Ev(©); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index b1f296ed5c..c1929fbfda 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -83,17 +83,20 @@ - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still skips the allocator-backed context, default container resources and mutex handles. Allocator admission and rollback remain incomplete; file-buffer size is snapshotted and used by file parsing. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. -- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records, outer value handles and object storage still use host allocations. String headers, characters, array storage and array iterator states use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. +- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records and outer value handles still use host allocations. String headers, characters, array and object storage and their iterator states use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - `Pair(const String&, const Value&)` and default `Pair()` zero the 8 bytes between the key and value; what the console keeps there is unknown. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed admission remains incomplete. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`; the host auxiliary allocation path is not the supplied allocator. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent legacy handler maps and dispatch remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Compatibility below SDK 6 is unverified. - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Host allocation failure behavior still differs; compatibility below SDK 6 is unverified. - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. -- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array and file-buffer allocation failures dispatch the supplied allocator's notifier, but object storage and value metadata still bypass it +- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object and file-buffer allocation failures dispatch the supplied allocator's notifier, but value metadata still bypasses it - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. - [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; a missing iterator state uses the module default value, whose allocator-backed identity remains incomplete. - [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy SDK behavior remains incomplete. Root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. - [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - destination state after parse failure and allocator failures outside the file buffer remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. - [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, module-default identity, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. + +- [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocation failures, legacy SDK ownership, concurrent allocation-operation accounting and default-object identity remain incomplete. Headers, red-black tree nodes, key storage and iterator states use the supplied allocator. Scalar insertion, duplicate-entry and copy failure contracts follow `0xda40`, `0xca40` and `0xe000`; partial trees can retain incomplete entries or a source count exceeding their actual nodes. Modern assignment follows `0xd400`: initial replacement allocation failure preserves the old tree, while later failures can replace it with partial entries; self-assignment rebuilds the tree. Legacy insertion copies the pair directly; SDK 5 metadata boundary tests match the inspected module, without establishing cross-firmware compatibility. Existing and missing key erasure allocates nothing, following `0xd350`. Object iterator dereference with no state throws if the temporary default iterator cannot be allocated; the reference can recurse in that case. Full Linux/Windows execution remains unverified. +- [sce::Json::Parser embedded zero characters](../../../core/libs/prx/libSceJson2/Export.cpp) - decoded Unicode zero is discarded through the inspected character-append behavior. Binary string values serialize using their stored length, while object keys serialize their C-string prefix. Broader malformed Unicode handling remains incomplete. From fc3428f03ba4bd0fddbf976a1b8eeda02563fa1e Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 00:41:21 +0200 Subject: [PATCH 015/318] fix(libSceJson2): allocate value handles through the guest allocator --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 110 ++++++++++------- core/libs/tests/GuestJsonFloatFailure.cpp | 12 +- core/libs/tests/GuestJsonValueAllocator.cpp | 126 ++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 6 +- 5 files changed, 209 insertions(+), 52 deletions(-) create mode 100644 core/libs/tests/GuestJsonValueAllocator.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 7760eb7e6c..a7cd7404e1 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -60,6 +60,11 @@ if(BUILD_TESTING) target_link_libraries(guest_json_array_allocator_tests PRIVATE libSceJson2 libc) configure_windows_unwind(guest_json_array_allocator_tests) add_test(NAME guest_json_array_allocator COMMAND guest_json_array_allocator_tests) + add_test_executable(guest_json_value_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonValueAllocator.cpp) + target_include_directories(guest_json_value_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_value_allocator_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_value_allocator_tests) + add_test(NAME guest_json_value_allocator COMMAND guest_json_value_allocator_tests) add_test_executable(guest_json_object_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonObjectAllocator.cpp) target_include_directories(guest_json_object_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_object_allocator_tests PRIVATE libSceJson2 libc) @@ -75,7 +80,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 94b2b1ec48..ac417bf029 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -878,7 +878,7 @@ template void Destroy(T*& pointer) { if (pointer) { Destroy(*pointer); - delete pointer; + Release(pointer); pointer = nullptr; } } @@ -942,11 +942,23 @@ void Construct(Value& destination, TInitialize initialize) { TransferPayload(destination, pending); } +bool HasStorage(const String& value) { return value.text != nullptr; } +bool HasStorage(const Array& value) { return value.items != nullptr; } +bool HasStorage(const Object& value) { return value.items != nullptr; } + +template +void ConstructHandle(T*& destination, TInitialize initialize) { + destination = static_cast(Allocate(sizeof(T))); + if (!destination) return; + *destination = {}; + try { initialize(*destination); } + catch (...) { Destroy(destination); throw; } + if (!HasStorage(*destination)) { Release(destination); destination = nullptr; } +} + template void Construct(T*& destination, const TArgs&... arguments) { - auto pending = std::make_unique(); - Construct(*pending, arguments...); - destination = pending.release(); + ConstructHandle(destination, [&](T& value) { Construct(value, arguments...); }); } void CopyArray(Array& destination, const Array& source) { @@ -989,15 +1001,11 @@ void CopyObject(Object& destination, const Object& source) { } void CopyArray(Array*& destination, const Array* source) { - auto pending = std::make_unique(); - CopyArray(*pending, *source); - destination = pending.release(); + ConstructHandle(destination, [&](Array& value) { CopyArray(value, *source); }); } void CopyObject(Object*& destination, const Object* source) { - auto pending = std::make_unique(); - CopyObject(*pending, *source); - destination = pending.release(); + ConstructHandle(destination, [&](Object& value) { CopyObject(value, *source); }); } void AssignReal(Node& n, double value); @@ -1012,6 +1020,8 @@ void AssignNode(Node& destination, const Node& source) { else if (source.type == TypeObject) CopyObject(copy.object, source.object); else if (source.type == TypeReal) AssignReal(copy, source.real); else std::memcpy(©.payload, &source.payload, sizeof(copy.payload)); + if ((copy.type == TypeString && !copy.string) || (copy.type == TypeArray && !copy.array) || + (copy.type == TypeObject && !copy.object)) copy.type = TypeNull; } catch (...) { Clear(copy); throw; @@ -1039,13 +1049,21 @@ void CopyInto(Value& destination, const Value& source) { void SetType(Node& n, ValueType type) { if (n.type == type) return; if (type < TypeNull || type > TypeObject) throw std::invalid_argument("sce::Json::Value::set: invalid value type"); - Node pending{}; - pending.type = type; - if (type == TypeString) Construct(pending.string, std::string()); - if (type == TypeArray) Construct(pending.array); - if (type == TypeObject) Construct(pending.object); Clear(n); - TransferPayload(n, pending); + if (type == TypeBoolean) n.boolean = false; + else if (type == TypeInteger || type == TypeUInteger || type == TypeReal) n.payload = 0; + else if (type == TypeString) { + Construct(n.string, std::string_view{}); + if (!n.string) return; + } else if (type == TypeArray) { + Construct(n.array); + if (!n.array) return; + } else if (type == TypeObject) { + Construct(n.object); + if (!n.object) return; + } + n.type = type; + BindChildren(n); } struct DefaultStorage { @@ -1208,11 +1226,15 @@ void AssignReal(Node& n, double value) { if (settings.format == 1) { SetType(n, TypeNull); } else { - Node pending{}; - pending.type = TypeString; - Construct(pending.string, SpecialFloatText(value)); - Clear(n); - TransferPayload(n, pending); + String temporary{}; + Construct(temporary, SpecialFloatText(value)); + try { + if (temporary.text) { + SetType(n, TypeString); + if (n.string) AssignText(*n.string, View(temporary), true); + } + } catch (...) { Destroy(temporary); throw; } + Destroy(temporary); } return; } @@ -1591,6 +1613,7 @@ private: std::string text; if (!string(text)) return false; SetType(out, TypeString); + if (out.type != TypeString) throw GuestOutOfMemory{}; AssignText(*out.string, text); return true; } @@ -1599,6 +1622,7 @@ private: take(); ++depth; SetType(out, TypeArray); + if (out.type != TypeArray) throw GuestOutOfMemory{}; skip(); if (peek() >= 0 && peek() == ']') { take(); --depth; return true; } for (;;) { @@ -1619,6 +1643,7 @@ private: take(); ++depth; SetType(out, TypeObject); + if (out.type != TypeObject) throw GuestOutOfMemory{}; skip(); if (peek() >= 0 && peek() == '}') { take(); --depth; return true; } for (;;) { @@ -1990,31 +2015,29 @@ void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value* self) { Construct(*self); } void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value* self, bool value) { Construct(*self); self->type = TypeBoolean; self->boolean = value; } void APS5_VABI _ZN3sce4Json5ValueC1El(Value* self, std::int64_t value) { Construct(*self); self->type = TypeInteger; self->integer = value; } void APS5_VABI _ZN3sce4Json5ValueC1Em(Value* self, std::uint64_t value) { Construct(*self); self->type = TypeUInteger; self->uinteger = value; } -void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value* self, double value) { Construct(*self, [value](Node& n) { AssignReal(n, value); }); } +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value* self, double value) { Construct(*self); AssignReal(*self, value); } void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value: null text"); - Construct(*self, [text](Node& node) { - SetType(node, TypeString); - AssignText(*node.string, text); - }); + Construct(*self); + SetType(*self, TypeString); + if (self->type == TypeString) AssignText(*self->string, text); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6StringE(Value* self, const String* text) { - Construct(*self, [text](Node& node) { - SetType(node, TypeString); - AssignText(*node.string, View(*text), true); - }); + Construct(*self); + SetType(*self, TypeString); + if (self->type == TypeString) AssignText(*self->string, View(*text), true); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value* self, const Array* array) { - Construct(*self, [array](Node& node) { - node.type = TypeArray; - CopyArray(node.array, array); - }); + Construct(*self); + CopyArray(self->array, array); + self->type = self->array ? TypeArray : TypeNull; + BindChildren(*self); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value* self, const Object* object) { - Construct(*self, [object](Node& node) { - node.type = TypeObject; - CopyObject(node.object, object); - }); + Construct(*self); + CopyObject(self->object, object); + self->type = self->object ? TypeObject : TypeNull; + BindChildren(*self); } void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value* self, const Value* other) { CopyInto(*self, *other); } void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value* self) { Destroy(*self); } @@ -2044,20 +2067,20 @@ int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value::set: null text"); const std::string copy(text); SetType(NodeOf(*self), TypeString); - AssignText(*self->string, copy); + if (self->type == TypeString) AssignText(*self->string, copy); return 0; } int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { const std::string copy(View(*text)); SetType(NodeOf(*self), TypeString); - AssignText(*self->string, copy, true); + if (self->type == TypeString) AssignText(*self->string, copy, true); return 0; } int APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { Array* copy = nullptr; CopyArray(copy, array); Clear(NodeOf(*self)); - self->type = TypeArray; + self->type = copy ? TypeArray : TypeNull; self->array = copy; BindChildren(*self); return 0; @@ -2066,7 +2089,7 @@ int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value* self, const Object* o Object* copy = nullptr; CopyObject(copy, object); Clear(NodeOf(*self)); - self->type = TypeObject; + self->type = copy ? TypeObject : TypeNull; self->object = copy; BindChildren(*self); return 0; @@ -2136,7 +2159,8 @@ const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value* self, std::size_t in return &MissingElement(self, static_cast(error), nullptr, &index); } void APS5_VABI _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(Value* self, ValueType type) { - Construct(*self, [type](Node& node) { SetType(node, type); }); + Construct(*self); + SetType(*self, type); } const Value* APS5_VABI _ZNK3sce4Json5Value8getValueEm(const Value* self, std::size_t index) { return _ZNK3sce4Json5ValueixEm(self, index); } const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value* self, const String* key) { diff --git a/core/libs/tests/GuestJsonFloatFailure.cpp b/core/libs/tests/GuestJsonFloatFailure.cpp index 51472898ba..90dd2b2dfc 100644 --- a/core/libs/tests/GuestJsonFloatFailure.cpp +++ b/core/libs/tests/GuestJsonFloatFailure.cpp @@ -117,7 +117,7 @@ static void CheckRecoveryFailures() { _ZN3sce4Json5ValueD1Ev(&value); Require(liveAllocations == baseline, "Recovery teardown must reclaim metadata, replacement and temporary allocations"); } - Require(succeeded && failures > 0, "Recovery must exercise failed and successful host allocations"); + Require(succeeded && failures == 0, "Recovery value storage must use the guest allocator without host allocations"); Value value; _ZN3sce4Json5ValueC1Ed(&value, 0.125); Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, Recover, &context) == 0, @@ -161,10 +161,10 @@ int main() { failed = true; } allocationBudget = -1; - Require(failed, "String conversion must propagate host allocation failure"); - Require(liveAllocations == before && _ZNK3sce4Json5Value7getTypeEv(&value) == 4 && - *_ZNK3sce4Json5Value7getRealEv(&value) == 0.125, - "Failed conversion must retain the original value and ownership"); + Require(!failed, "String conversion must use the guest allocator without host allocations"); + Require(liveAllocations == before && _ZNK3sce4Json5Value7getTypeEv(&value) == 5 && + std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&value)), "inf") == 0, + "Conversion must own its string through the guest allocator"); allocationBudget = 0; _ZN3sce4Json5Value3setEd(&value, 0.5); allocationBudget = -1; @@ -198,7 +198,7 @@ int main() { Require(liveAllocations == baseline, "Successful construction must release its host ownership"); } } - Require(constructed && constructorFailures > 0, "Exercise failed and successful host construction"); + Require(constructed && constructorFailures == 0, "Float construction must not depend on host allocation"); _ZN3sce4Json5ValueC1Ed(&value, std::numeric_limits::infinity()); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == 5, "Conversion must remain usable after allocation failure"); _ZN3sce4Json5ValueD1Ev(&value); diff --git a/core/libs/tests/GuestJsonValueAllocator.cpp b/core/libs/tests/GuestJsonValueAllocator.cpp new file mode 100644 index 0000000000..de423ef794 --- /dev/null +++ b/core/libs/tests/GuestJsonValueAllocator.cpp @@ -0,0 +1,126 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(Value*, std::int32_t); +void APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ValueAllocationTrace { + std::map live; + std::vector requests; + int budget = -1; + unsigned notifications = 0; +}; +static void* APS5_VABI ValueAllocationAllocate(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) return nullptr; + if (trace.budget > 0) --trace.budget; + auto* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI ValueAllocationFree(void*, void* memory, void* context) { + if (!memory) return; + Require(static_cast(context)->live.erase(memory) == 1, "Value resources must return through their owning allocator"); + std::free(memory); +} +static void APS5_VABI ValueAllocationNotify(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Value failure notification must identify its allocation"); + ++trace.notifications; +} +static void* const ValueAllocationVtable[]{nullptr, nullptr, reinterpret_cast(ValueAllocationAllocate), reinterpret_cast(ValueAllocationFree), reinterpret_cast(ValueAllocationNotify)}; +struct ValueAllocationAllocator { void* const* vtable = ValueAllocationVtable; }; +static bool Owned(const ValueAllocationTrace& trace, const void* pointer, std::size_t bytes) { + const auto found = trace.live.find(const_cast(pointer)); return found != trace.live.end() && found->second == bytes; +} +static void Check(const Value& value, int expected, const ValueAllocationTrace& trace) { + Require(value.type == expected && value.reserved == 0xa5a5a5a5, "Value type admission must preserve inactive metadata"); + if (expected < 5) return; + const auto* handle = reinterpret_cast(value.payload); + Require(Owned(trace, handle, 8) && Owned(trace, *handle, expected == 5 ? 24 : 32), "Complex values must own an actual allocator-backed handle and header"); + if (expected != 5) { + Value* owner = nullptr; std::memcpy(&owner, static_cast(*handle) + 24, sizeof(owner)); + Require(owner == &value, "Collection headers must bind to their enclosing value"); + } +} +int main() { + ValueAllocationTrace trace; ValueAllocationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 0, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize value allocation contract"); + const auto baseline = trace.live.size(); + for (int target = 0; target <= 7; ++target) for (int budget = 0; budget <= 4; ++budget) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + trace.budget = budget; + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&value, target); + trace.budget = -1; + const auto expected = target < 5 ? target : budget < (target == 7 ? 3 : 2) ? 0 : target; + Check(value, expected, trace); + Require(!value.parent && !value.auxiliary, "Typed construction must clear ownership metadata"); + if (target >= 5 && expected == 0) Require(value.payload == 0, "Failed complex construction must clear the payload pointer"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Typed construction must release partial owned resources"); + } + for (int previous = 0; previous <= 7; ++previous) for (int target = 0; target <= 7; ++target) for (int budget = 0; budget <= 4; ++budget) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&value, previous); + Value owner{}; value.parent = &owner; + const auto payload = value.payload; + const auto requests = trace.requests.size(); + const auto notifications = trace.notifications; + trace.budget = budget; + _ZN3sce4Json5Value3setENS0_9ValueTypeE(&value, target); + trace.budget = -1; + const auto expected = previous == target || target < 5 ? target : budget < (target == 7 ? 3 : 2) ? 0 : target; + Check(value, expected, trace); + Require(value.parent == &owner && !value.auxiliary, "Type changes must preserve parent and auxiliary ownership"); + if (previous == target) Require(value.payload == payload && trace.requests.size() == requests && trace.notifications == notifications, "Retaining the same type must preserve storage without allocating"); + else if (target >= 5 && expected == 0) Require(value.payload == 0 && trace.notifications > notifications, "Failed type changes must clear the previous payload and notify failure"); + else if (target == 0) Require(value.payload == (previous >= 5 ? 0 : payload), "Clearing a scalar must preserve its inactive payload while complex values clear their handle"); + else if (target == 1) Require(value.payload == ((previous >= 5 ? 0 : payload) & ~UINT64_C(255)), "Boolean type changes must clear only their active byte"); + else if (target < 5) Require(value.payload == 0, "Numeric type changes must initialize their active payload"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Type changes must reclaim the old payload even when replacement fails"); + } + for (bool constructor : {false, true}) for (int budget = 0; budget <= 5; ++budget) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + if (!constructor) _ZN3sce4Json5ValueC1Ed(&value, 0.125); + const auto payload = value.payload; + const auto notifications = trace.notifications; + trace.budget = budget; + if (constructor) _ZN3sce4Json5ValueC1Ed(&value, std::numeric_limits::infinity()); + else _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + trace.budget = -1; + const auto expected = budget == 0 ? (constructor ? 0 : 4) : budget < 4 ? 0 : 5; + Check(value, expected, trace); + Require(trace.notifications - notifications == (budget == 5 ? 0u : budget == 1 ? 2u : 1u), "Float conversion must report temporary and destination allocation failures"); + if (budget == 0) Require(value.payload == payload, "Failure to allocate the conversion temporary must preserve the prior payload"); + else if (expected == 0) Require(value.payload == 0, "Failed destination conversion must leave a null handle"); + else { + const auto* header = *reinterpret_cast(value.payload); + const auto* characters = *static_cast(header); + Require(std::strcmp(characters, budget == 4 ? "" : "inf") == 0, "Float conversion must retain the admitted string and character failure state"); + } + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Float conversion must release temporary and destination allocations"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate value allocation contract"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index c1929fbfda..8869c11ee4 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -83,14 +83,14 @@ - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still skips the allocator-backed context, default container resources and mutex handles. Allocator admission and rollback remain incomplete; file-buffer size is snapshotted and used by file parsing. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. -- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records and outer value handles still use host allocations. String headers, characters, array and object storage and their iterator states use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. +- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records still use host allocations. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - `Pair(const String&, const Value&)` and default `Pair()` zero the 8 bytes between the key and value; what the console keeps there is unknown. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed admission remains incomplete. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`; the host auxiliary allocation path is not the supplied allocator. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent legacy handler maps and dispatch remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Compatibility below SDK 6 is unverified. - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Host allocation failure behavior still differs; compatibility below SDK 6 is unverified. - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. -- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object and file-buffer allocation failures dispatch the supplied allocator's notifier, but value metadata still bypasses it +- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object and file-buffer allocation failures dispatch the supplied allocator's notifier, but callback metadata still bypasses it - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. - [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; a missing iterator state uses the module default value, whose allocator-backed identity remains incomplete. - [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy SDK behavior remains incomplete. Root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. @@ -100,3 +100,5 @@ - [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocation failures, legacy SDK ownership, concurrent allocation-operation accounting and default-object identity remain incomplete. Headers, red-black tree nodes, key storage and iterator states use the supplied allocator. Scalar insertion, duplicate-entry and copy failure contracts follow `0xda40`, `0xca40` and `0xe000`; partial trees can retain incomplete entries or a source count exceeding their actual nodes. Modern assignment follows `0xd400`: initial replacement allocation failure preserves the old tree, while later failures can replace it with partial entries; self-assignment rebuilds the tree. Legacy insertion copies the pair directly; SDK 5 metadata boundary tests match the inspected module, without establishing cross-firmware compatibility. Existing and missing key erasure allocates nothing, following `0xd350`. Object iterator dereference with no state throws if the temporary default iterator cannot be allocated; the reference can recurse in that case. Full Linux/Windows execution remains unverified. - [sce::Json::Parser embedded zero characters](../../../core/libs/prx/libSceJson2/Export.cpp) - decoded Unicode zero is discarded through the inspected character-append behavior. Binary string values serialize using their stored length, while object keys serialize their C-string prefix. Broader malformed Unicode handling remains incomplete. + +- [sce::Json::Value handle admission and type changes](../../../core/libs/prx/libSceJson2/Export.cpp) - full copy/assignment failure ordering, nested recovery failures and older SDK compatibility remain incomplete. The supplied allocator owns each eight-byte complex-value handle. Typed construction and type changes follow `0x580`, `0x10d0`, `0x1170` and `0x1210`: the old payload is destroyed before replacement admission; string/header failure leaves null, an array can retain a header without its sentinel, and object construction reclaims a failed sentinel. Scalar changes preserve inactive payload bytes. Special-float string conversion follows the temporary string at `0x40a0`; failure of its header preserves the prior value, while later failures can leave null or an empty string. Local tests cover SDK 6 process metadata, not every firmware version. From 3a55d54c1f802c0d78435cceb0ceb02c71b240e2 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 00:46:27 +0200 Subject: [PATCH 016/318] fix(libSceJson2): use guest allocations for callback metadata --- core/libs/prx/libSceJson2/Export.cpp | 71 ++++++++++------- core/libs/tests/GuestJsonValueAllocator.cpp | 84 ++++++++++++++++++++- docs/dev/debt/libSceJson2.md | 10 ++- 3 files changed, 132 insertions(+), 33 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index ac417bf029..3ccb93fad6 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -847,7 +846,7 @@ Handlers HandlersOf(const Node& n) { std::lock_guard lock(state.mutex); if (state.initialized) handlers = state.handlers; } - if (!auxiliary) return handlers; + if (!auxiliary || !ModernValues()) return handlers; if (auxiliary->typeMismatch && auxiliary->typeMismatch->callback) { handlers.typeMismatch = auxiliary->typeMismatch->callback; handlers.typeMismatchContext = auxiliary->typeMismatch->context; @@ -893,24 +892,38 @@ void Clear(Node& n) { void Destroy(Value& v) { Clear(v); if (v.auxiliary) { - delete v.auxiliary->typeMismatch; - delete v.auxiliary->specialFloat; - delete v.auxiliary->elementAccessFailure; + if (ModernValues()) { + if (v.auxiliary->typeMismatch) Release(v.auxiliary->typeMismatch); + v.auxiliary->typeMismatch = nullptr; + if (v.auxiliary->specialFloat) Release(v.auxiliary->specialFloat); + v.auxiliary->specialFloat = nullptr; + if (v.auxiliary->elementAccessFailure) Release(v.auxiliary->elementAccessFailure); + v.auxiliary->elementAccessFailure = nullptr; + } if (v.auxiliary->fallback) { Destroy(*v.auxiliary->fallback); - delete v.auxiliary->fallback; + Release(v.auxiliary->fallback); + v.auxiliary->fallback = nullptr; } - delete v.auxiliary; + Release(v.auxiliary); v.auxiliary = nullptr; } } -void EnsureAuxiliary(Node& n) { - if (!n.auxiliary) { - n.auxiliary = new Auxiliary(); - n.auxiliary->fallback = new Value(); - n.auxiliary->fallback->parent = &n; - } +int EnsureAuxiliary(Node& n) { + if (n.auxiliary) return 0; + const auto bytes = ModernValues() ? sizeof(Auxiliary) : std::size_t{24}; + auto* auxiliary = static_cast(Allocate(bytes)); + if (!auxiliary) return JsonErrorOutOfMemory; + auxiliary->nullAccess = nullptr; + auxiliary->nullAccessContext = nullptr; + auxiliary->fallback = static_cast(Allocate(sizeof(Value))); + if (auxiliary->fallback) Construct(*auxiliary->fallback); + if (ModernValues()) std::memset(reinterpret_cast(auxiliary) + 24, 0, sizeof(Auxiliary) - 24); + n.auxiliary = auxiliary; + if (!auxiliary->fallback) return JsonErrorOutOfMemory; + auxiliary->fallback->parent = &n; + return 0; } Node& NodeOf(const Value& v) { return const_cast(v); } @@ -1120,7 +1133,7 @@ const Node& Typed(const Value* self, ValueType requested) { return Typed(&auxiliary->nullAccess(requested, self->parent, auxiliary->nullAccessContext), requested); } const auto* auxiliary = RootOf(*self).auxiliary; - if (auxiliary && auxiliary->typeMismatch && auxiliary->typeMismatch->callback) + if (ModernValues() && auxiliary && auxiliary->typeMismatch && auxiliary->typeMismatch->callback) return Typed(&auxiliary->typeMismatch->callback(requested, self, self->parent, auxiliary->typeMismatch->context), requested); if (self->type == TypeNull && callback) return Typed(&callback(requested, self->parent, context), requested); @@ -1158,15 +1171,16 @@ int SetHandler(Node& n, HandlerRecord* Auxiliary::* member, TCallback if (!state.initialized) return JsonErrorNotInitialized; if (modernOnly && !state.floatFormatEnabled) return static_cast(0x80848113); } - try { - EnsureAuxiliary(n); - auto& record = n.auxiliary->*member; - if (!record) record = new HandlerRecord(); - record->callback = callback; - record->context = context; - } catch (const std::bad_alloc&) { - return JsonErrorOutOfMemory; + if (!ModernValues()) NotImplemented_nid_no_patch(__func__); + const auto status = EnsureAuxiliary(n); + if (status) return status; + auto& record = n.auxiliary->*member; + if (!record) { + record = static_cast*>(Allocate(sizeof(HandlerRecord))); + if (!record) return JsonErrorOutOfMemory; } + record->callback = callback; + record->context = context; return 0; } @@ -1670,7 +1684,10 @@ private: }; int ParseValue(Value* out, Parser& parser) { - if (!out->parent) EnsureAuxiliary(*out); + if (!out->parent) { + const auto status = EnsureAuxiliary(*out); + if (status) return status; + } Node parsed{}; bool parsedOk = false; try { @@ -2348,11 +2365,9 @@ int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS std::lock_guard lock(state.mutex); if (!state.initialized) return JsonErrorNotInitialized; } - try { - EnsureAuxiliary(*self); - } catch (const std::bad_alloc&) { - return JsonErrorOutOfMemory; - } + if (!ModernValues() && !self->auxiliary) return static_cast(0x80848104); + const auto status = EnsureAuxiliary(*self); + if (status) return status; self->auxiliary->nullAccess = callback; self->auxiliary->nullAccessContext = context; return 0; diff --git a/core/libs/tests/GuestJsonValueAllocator.cpp b/core/libs/tests/GuestJsonValueAllocator.cpp index de423ef794..a959062c6d 100644 --- a/core/libs/tests/GuestJsonValueAllocator.cpp +++ b/core/libs/tests/GuestJsonValueAllocator.cpp @@ -1,4 +1,5 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include #include #include #include @@ -10,6 +11,10 @@ struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using NullCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using MismatchCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +using FloatCallback = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +using ElementCallback = void (APS5_VABI*)(int, const void*, const std::size_t*, const Value*, const Value*, void*); extern "C" { void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); @@ -19,6 +24,10 @@ void APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullCallback, void*); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, MismatchCallback, void*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatCallback, void*); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, ElementCallback, void*); } static void Require(bool valid, const char* message) { if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } @@ -26,8 +35,11 @@ static void Require(bool valid, const char* message) { struct ValueAllocationTrace { std::map live; std::vector requests; + std::vector frees; int budget = -1; unsigned notifications = 0; + unsigned nullFrees = 0; + const Value* registering = nullptr; }; static void* APS5_VABI ValueAllocationAllocate(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); trace.requests.push_back(bytes); @@ -37,13 +49,15 @@ static void* APS5_VABI ValueAllocationAllocate(void*, std::size_t bytes, void* c std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; } static void APS5_VABI ValueAllocationFree(void*, void* memory, void* context) { - if (!memory) return; + if (!memory) { ++static_cast(context)->nullFrees; return; } Require(static_cast(context)->live.erase(memory) == 1, "Value resources must return through their owning allocator"); + static_cast(context)->frees.push_back(memory); std::free(memory); } static void APS5_VABI ValueAllocationNotify(void*, int error, std::size_t bytes, void* context) { auto& trace = *static_cast(context); Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Value failure notification must identify its allocation"); + if (trace.registering && bytes != 16) Require(!trace.registering->auxiliary, "Auxiliary admission must remain unpublished during allocation failure notification"); ++trace.notifications; } static void* const ValueAllocationVtable[]{nullptr, nullptr, reinterpret_cast(ValueAllocationAllocate), reinterpret_cast(ValueAllocationFree), reinterpret_cast(ValueAllocationNotify)}; @@ -61,11 +75,79 @@ static void Check(const Value& value, int expected, const ValueAllocationTrace& Require(owner == &value, "Collection headers must bind to their enclosing value"); } } +static const Value& APS5_VABI NullRecovery(std::int32_t, const Value* value, void*) { return *value; } +static const Value& APS5_VABI MismatchRecovery(std::int32_t, const Value* value, const Value*, void*) { return *value; } +static Value* APS5_VABI FloatRecovery(Value* value, std::int32_t, double, const Value*, void*, bool*) { return value; } +static void APS5_VABI ElementRecovery(int, const void*, const std::size_t*, const Value*, const Value*, void*) {} +static int Register(Value& value, unsigned kind, void* context, bool empty = false) { + switch (kind) { + case 0: return _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&value, empty ? nullptr : NullRecovery, context); + case 1: return _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&value, empty ? nullptr : MismatchRecovery, context); + case 2: return _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, empty ? nullptr : FloatRecovery, context); + default: return _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&value, empty ? nullptr : ElementRecovery, context); + } +} +static void CheckMetadata(ValueAllocationTrace& trace) { + const auto baseline = trace.live.size(); + for (unsigned kind = 0; kind < 4; ++kind) for (int budget = 0; budget <= 3; ++budget) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&value, 0); + const auto payload = value.payload; + const auto notifications = trace.notifications; + trace.budget = budget; + trace.registering = &value; + const auto status = Register(value, kind, &trace); + trace.registering = nullptr; + trace.budget = -1; + const bool admitted = budget >= (kind == 0 ? 2 : 3); + Require(static_cast(status) == (admitted ? 0u : 0x80848102u) && trace.notifications - notifications == (admitted ? 0u : 1u), "Callback metadata admission must return and notify its actual allocation failure"); + Require(value.payload == payload && value.reserved == 0xa5a5a5a5 && value.type == 0, "Callback registration must preserve the value payload"); + Require((value.auxiliary != nullptr) == (budget >= 1), "Failed fallback allocation must retain its auxiliary record"); + if (value.auxiliary) { + Require(Owned(trace, value.auxiliary, 72), "Modern auxiliary records must use the supplied allocator"); + auto* slots = static_cast(value.auxiliary); + Require((slots[2] != nullptr) == (budget >= 2), "Fallback allocation failure must preserve a null fallback pointer"); + if (slots[2]) { + auto* fallback = static_cast(slots[2]); + Require(Owned(trace, fallback, 32) && fallback->parent == &value && !fallback->auxiliary && fallback->type == 0 && fallback->payload == UINT64_C(0xa5a5a5a5a5a5a5a5), "Fallback values must retain native ownership and inactive payload bytes"); + } + for (unsigned index = 6; index < 9; ++index) Require(!slots[index], "Auxiliary reserved fields must initialize to zero"); + if (kind && admitted) Require(Owned(trace, slots[kind + 2], 16), "Callback records must own their actual sixteen-byte allocation"); + } + auto* auxiliary = value.auxiliary; + auto* fallback = auxiliary ? static_cast(auxiliary)[2] : nullptr; + const auto requests = trace.requests.size(); + Require(Register(value, kind, &value) == 0, "Callback registration must remain usable after allocation failure"); + Require(!auxiliary || (value.auxiliary == auxiliary && static_cast(auxiliary)[2] == fallback), "Registration retry must reuse partial auxiliary state without recreating a missing fallback"); + if (admitted) Require(trace.requests.size() == requests, "Replacing an existing callback must not allocate"); + auto* slots = static_cast(value.auxiliary); + auto* record = kind ? static_cast(slots[kind + 2]) : slots; + Require(record[0] && record[1] == &value, "Registration must retain its callback and replacement context"); + const auto retained = trace.requests.size(); + Require(Register(value, kind, &trace, true) == 0 && !record[0] && record[1] == &trace && trace.requests.size() == retained, "A null callback must replace its record without deleting or reallocating metadata"); + _ZN3sce4Json5Value3setENS0_9ValueTypeE(&value, 2); + Require(value.auxiliary == slots, "Payload type changes must preserve root callback metadata"); + Value parent{}; value.parent = &parent; + Require(static_cast(Register(value, kind, &trace)) == 0x80848104 && trace.requests.size() == retained, "Owned children must reject callback registration without allocating"); + value.parent = nullptr; + std::vector expectedFrees; + if (kind) expectedFrees.push_back(record); + if (slots[2]) expectedFrees.push_back(slots[2]); + expectedFrees.push_back(slots); + const auto freed = trace.frees.size(); + const auto nullFrees = trace.nullFrees; + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.nullFrees == nullFrees, "Metadata destruction must not free absent callback records"); + Require(trace.frees.size() == freed + expectedFrees.size() && std::equal(expectedFrees.begin(), expectedFrees.end(), trace.frees.begin() + freed), "Metadata teardown must release callback records, fallback and auxiliary in ownership order"); + Require(trace.live.size() == baseline, "Callback metadata destruction must release all admitted and partial resources"); + } +} int main() { ValueAllocationTrace trace; ValueAllocationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 0, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize value allocation contract"); const auto baseline = trace.live.size(); + CheckMetadata(trace); for (int target = 0; target <= 7; ++target) for (int budget = 0; budget <= 4; ++budget) { Value value; std::memset(&value, 0xa5, sizeof(value)); trace.budget = budget; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 8869c11ee4..8295b9fcd8 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -83,14 +83,14 @@ - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still skips the allocator-backed context, default container resources and mutex handles. Allocator admission and rollback remain incomplete; file-buffer size is snapshotted and used by file parsing. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. -- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, but auxiliary records still use host allocations. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. +- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - `Pair(const String&, const Value&)` and default `Pair()` zero the 8 bytes between the key and value; what the console keeps there is unknown. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. -- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed admission remains incomplete. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`; the host auxiliary allocation path is not the supplied allocator. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. +- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy registration remains incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent legacy handler maps and dispatch remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Compatibility below SDK 6 is unverified. -- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Host allocation failure behavior still differs; compatibility below SDK 6 is unverified. +- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback-value failure behavior and compatibility below SDK 6 remain incomplete. - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. -- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object and file-buffer allocation failures dispatch the supplied allocator's notifier, but callback metadata still bypasses it +- [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object, callback metadata and file-buffer allocation failures dispatch the supplied allocator's notifier. - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. - [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; a missing iterator state uses the module default value, whose allocator-backed identity remains incomplete. - [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy SDK behavior remains incomplete. Root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. @@ -102,3 +102,5 @@ - [sce::Json::Parser embedded zero characters](../../../core/libs/prx/libSceJson2/Export.cpp) - decoded Unicode zero is discarded through the inspected character-append behavior. Binary string values serialize using their stored length, while object keys serialize their C-string prefix. Broader malformed Unicode handling remains incomplete. - [sce::Json::Value handle admission and type changes](../../../core/libs/prx/libSceJson2/Export.cpp) - full copy/assignment failure ordering, nested recovery failures and older SDK compatibility remain incomplete. The supplied allocator owns each eight-byte complex-value handle. Typed construction and type changes follow `0x580`, `0x10d0`, `0x1170` and `0x1210`: the old payload is destroyed before replacement admission; string/header failure leaves null, an array can retain a header without its sentinel, and object construction reclaims a failed sentinel. Scalar changes preserve inactive payload bytes. Special-float string conversion follows the temporary string at `0x40a0`; failure of its header preserves the prior value, while later failures can leave null or an empty string. Local tests cover SDK 6 process metadata, not every firmware version. + +- [sce::Json::Value auxiliary admission and retry](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy per-root mismatch, float and element handler maps remain unsupported and throw. Modern admission follows `0x3280`: a failed fallback allocation retains the 72-byte auxiliary record; later registration reuses it without recreating the missing fallback. Local tests cover all four callback setters, their allocation failures, replacement and null callbacks, child rejection, parent preservation and teardown through the original allocator. Legacy auxiliary records use 24 bytes, but their external handler maps and dispatch remain incomplete. From 63ad6527a455035b4054d90f005e4d0092f35279 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 00:50:58 +0200 Subject: [PATCH 017/318] fix(libSceJson2): match scalar and string value assignment --- core/libs/prx/libSceJson2/Export.cpp | 48 ++++++++++------- core/libs/tests/GuestJsonValueAllocator.cpp | 57 +++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 89 insertions(+), 20 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 3ccb93fad6..f0a0ad593e 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -942,18 +942,6 @@ void TransferPayload(Node& destination, Node& source) { BindChildren(destination); } -template -void Construct(Value& destination, TInitialize initialize) { - Value pending{}; - try { - initialize(pending); - } catch (...) { - Destroy(pending); - throw; - } - Construct(destination); - TransferPayload(destination, pending); -} bool HasStorage(const String& value) { return value.text != nullptr; } bool HasStorage(const Array& value) { return value.items != nullptr; } @@ -1023,18 +1011,39 @@ void CopyObject(Object*& destination, const Object* source) { void AssignReal(Node& n, double value); +void SetType(Node& n, ValueType type); + void AssignNode(Node& destination, const Node& source) { + if (source.type == TypeNull) { SetType(destination, TypeNull); return; } + if (source.type == TypeBoolean) { + const auto value = source.boolean; + SetType(destination, TypeBoolean); + destination.boolean = value; + return; + } + if (source.type == TypeInteger || source.type == TypeUInteger) { + const auto value = source.payload; + SetType(destination, source.type); + destination.payload = value; + return; + } + if (source.type == TypeReal) { AssignReal(destination, source.real); return; } + if (source.type == TypeString) { + const auto* string = source.string; + if (!string) NotImplemented_nid_no_patch(__func__); + if (!string->text) return; + SetType(destination, TypeString); + if (destination.string && destination.string->text != string->text) AssignText(*destination.string, View(*string), true); + return; + } if (&destination == &source) return; Node copy{}; copy.type = source.type; try { - if (source.type == TypeString) Construct(copy.string, *source.string); - else if (source.type == TypeArray) CopyArray(copy.array, source.array); + if (source.type == TypeArray) CopyArray(copy.array, source.array); else if (source.type == TypeObject) CopyObject(copy.object, source.object); - else if (source.type == TypeReal) AssignReal(copy, source.real); - else std::memcpy(©.payload, &source.payload, sizeof(copy.payload)); - if ((copy.type == TypeString && !copy.string) || (copy.type == TypeArray && !copy.array) || - (copy.type == TypeObject && !copy.object)) copy.type = TypeNull; + else NotImplemented_nid_no_patch(__func__); + if ((copy.type == TypeArray && !copy.array) || (copy.type == TypeObject && !copy.object)) copy.type = TypeNull; } catch (...) { Clear(copy); throw; @@ -1056,7 +1065,8 @@ void RecoverSpecial(const Node& source, const Handlers& handlers, int function, } void CopyInto(Value& destination, const Value& source) { - Construct(destination, [&](Node& node) { AssignNode(node, NodeOf(source)); }); + Construct(destination); + AssignNode(destination, source); } void SetType(Node& n, ValueType type) { diff --git a/core/libs/tests/GuestJsonValueAllocator.cpp b/core/libs/tests/GuestJsonValueAllocator.cpp index a959062c6d..ba6a21538f 100644 --- a/core/libs/tests/GuestJsonValueAllocator.cpp +++ b/core/libs/tests/GuestJsonValueAllocator.cpp @@ -24,6 +24,9 @@ void APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); +Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullCallback, void*); int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, MismatchCallback, void*); int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatCallback, void*); @@ -142,12 +145,66 @@ static void CheckMetadata(ValueAllocationTrace& trace) { Require(trace.live.size() == baseline, "Callback metadata destruction must release all admitted and partial resources"); } } +static void CheckScalarCopies(ValueAllocationTrace& trace) { + const auto baseline = trace.live.size(); + constexpr const char* text = "a string requiring independently allocated storage"; + for (int previous = -1; previous <= 7; ++previous) for (int type = 0; type <= 5; ++type) for (int budget = 0; budget <= 3; ++budget) { + Value source; std::memset(&source, 0xa5, sizeof(source)); + if (type == 5) _ZN3sce4Json5ValueC1EPKc(&source, text); + else { + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&source, type); + if (type == 1) source.payload = UINT64_C(0xa5a5a5a5a5a5a501); + if (type == 2) source.payload = 42; + if (type == 3) source.payload = UINT64_C(0xfedcba9876543210); + if (type == 4) source.payload = UINT64_C(0x3fe8000000000000); + } + Value destination; std::memset(&destination, 0xa5, sizeof(destination)); + void* auxiliary = nullptr; + std::uint64_t payload = destination.payload; + void* header = nullptr; + if (previous >= 0) { + if (previous == 5) _ZN3sce4Json5ValueC1EPKc(&destination, "previous"); + else _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&destination, previous); + Require(Register(destination, 0, &trace, true) == 0, "Prepare assignment metadata"); + auxiliary = destination.auxiliary; + payload = destination.payload; + if (previous == 5) header = *reinterpret_cast(payload); + } + const auto requests = trace.requests.size(); + trace.budget = budget; + if (previous < 0) _ZN3sce4Json5ValueC1ERKS1_(&destination, &source); + else Require(_ZN3sce4Json5ValueaSERKS1_(&destination, &source) == &destination, "Value assignment must return its destination"); + trace.budget = -1; + const auto expected = type == 5 && previous != 5 && budget < 2 ? 0 : type; + Check(destination, expected, trace); + Require(!destination.parent && destination.auxiliary == auxiliary, "Copy and assignment must preserve their metadata contract"); + if (type == 0) Require(destination.payload == (previous >= 5 ? 0 : payload), "Null copies must preserve inactive scalar payload bytes"); + else if (type == 1) Require(destination.payload == (((previous >= 5 ? 0 : payload) & ~UINT64_C(255)) | 1), "Boolean copies must preserve inactive destination bytes"); + else if (type < 5) Require(destination.payload == source.payload, "Numeric copies must retain the exact active payload"); + else if (!expected) Require(destination.payload == 0, "Failed string admission must leave a null payload pointer"); + else { + const auto* stored = *reinterpret_cast(destination.payload); + const auto* characters = *static_cast(stored); + const bool complete = budget >= (previous == 5 ? 1 : 3); + Require(std::strcmp(characters, complete ? text : "") == 0, "String copy failure must retain its admitted empty header"); + if (previous == 5) Require(stored == header, "String assignment must reuse the existing header"); + } + if (type < 5) Require(trace.requests.size() == requests, "Scalar copies must not allocate"); + if (type == 5) { + const auto* stored = *reinterpret_cast(source.payload); + Require(std::strcmp(*static_cast(stored), text) == 0, "String copy failures must preserve the source"); + } + _ZN3sce4Json5ValueD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); + Require(trace.live.size() == baseline, "Copy failures must reclaim destination and source resources"); + } +} int main() { ValueAllocationTrace trace; ValueAllocationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 0, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize value allocation contract"); const auto baseline = trace.live.size(); CheckMetadata(trace); + CheckScalarCopies(trace); for (int target = 0; target <= 7; ++target) for (int budget = 0; budget <= 4; ++budget) { Value value; std::memset(&value, 0xa5, sizeof(value)); trace.budget = budget; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 8295b9fcd8..8e31da9563 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -101,6 +101,8 @@ - [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocation failures, legacy SDK ownership, concurrent allocation-operation accounting and default-object identity remain incomplete. Headers, red-black tree nodes, key storage and iterator states use the supplied allocator. Scalar insertion, duplicate-entry and copy failure contracts follow `0xda40`, `0xca40` and `0xe000`; partial trees can retain incomplete entries or a source count exceeding their actual nodes. Modern assignment follows `0xd400`: initial replacement allocation failure preserves the old tree, while later failures can replace it with partial entries; self-assignment rebuilds the tree. Legacy insertion copies the pair directly; SDK 5 metadata boundary tests match the inspected module, without establishing cross-firmware compatibility. Existing and missing key erasure allocates nothing, following `0xd350`. Object iterator dereference with no state throws if the temporary default iterator cannot be allocated; the reference can recurse in that case. Full Linux/Windows execution remains unverified. - [sce::Json::Parser embedded zero characters](../../../core/libs/prx/libSceJson2/Export.cpp) - decoded Unicode zero is discarded through the inspected character-append behavior. Binary string values serialize using their stored length, while object keys serialize their C-string prefix. Broader malformed Unicode handling remains incomplete. -- [sce::Json::Value handle admission and type changes](../../../core/libs/prx/libSceJson2/Export.cpp) - full copy/assignment failure ordering, nested recovery failures and older SDK compatibility remain incomplete. The supplied allocator owns each eight-byte complex-value handle. Typed construction and type changes follow `0x580`, `0x10d0`, `0x1170` and `0x1210`: the old payload is destroyed before replacement admission; string/header failure leaves null, an array can retain a header without its sentinel, and object construction reclaims a failed sentinel. Scalar changes preserve inactive payload bytes. Special-float string conversion follows the temporary string at `0x40a0`; failure of its header preserves the prior value, while later failures can leave null or an empty string. Local tests cover SDK 6 process metadata, not every firmware version. +- [sce::Json::Value handle admission and type changes](../../../core/libs/prx/libSceJson2/Export.cpp) - collection copy/assignment failure ordering, nested recovery failures and older SDK compatibility remain incomplete. The supplied allocator owns each eight-byte complex-value handle. Typed construction and type changes follow `0x580`, `0x10d0`, `0x1170` and `0x1210`: the old payload is destroyed before replacement admission; string/header failure leaves null, an array can retain a header without its sentinel, and object construction reclaims a failed sentinel. Scalar changes preserve inactive payload bytes. Special-float string conversion follows the temporary string at `0x40a0`; failure of its header preserves the prior value, while later failures can leave null or an empty string. Local tests cover SDK 6 process metadata, not every firmware version. - [sce::Json::Value auxiliary admission and retry](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy per-root mismatch, float and element handler maps remain unsupported and throw. Modern admission follows `0x3280`: a failed fallback allocation retains the 72-byte auxiliary record; later registration reuses it without recreating the missing fallback. Local tests cover all four callback setters, their allocation failures, replacement and null callbacks, child rejection, parent preservation and teardown through the original allocator. Legacy auxiliary records use 24 bytes, but their external handler maps and dispatch remain incomplete. + +- [sce::Json::Value scalar and string copying](../../../core/libs/prx/libSceJson2/Export.cpp) - collection assignment and legacy callback-map propagation remain incomplete. Scalar and string dispatch follow `0x1e40`: null and boolean copies preserve inactive destination bytes, numeric values retain their exact payloads, and string assignment changes type before copying characters into the existing header. A character-allocation failure can empty an existing string; it does not restore its old contents. Copy construction initializes ownership metadata directly, following `0x1e20`. Local allocator tests cover null, boolean, signed, unsigned, finite real and string sources across all destination types and failure budgets. From 4943df4928d472e65854f907d3981813d5903e32 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 00:56:32 +0200 Subject: [PATCH 018/318] fix(libSceJson2): match collection value assignment protocols --- core/libs/prx/libSceJson2/Export.cpp | 102 +++++++++++++------- core/libs/tests/GuestJsonArrayAllocator.cpp | 51 +++++++++- core/libs/tests/GuestJsonValueAllocator.cpp | 39 ++++++++ docs/dev/debt/libSceJson2.md | 2 + 4 files changed, 160 insertions(+), 34 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index f0a0ad593e..27041e0856 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -816,7 +816,7 @@ void Destroy(String& s) { void Destroy(Array& a) { if (!a.items) return; a.items->ReleaseContents(); - Release(a.items->head); + if (a.items->head) Release(a.items->head); a.items->head = reinterpret_cast(std::uintptr_t{0xffffffff}); a.items->count = std::numeric_limits::max(); Release(a.items); @@ -825,7 +825,7 @@ void Destroy(Array& a) { void Destroy(Object& object) { if (!object.items) return; object.items->clear(); - Release(object.items->head); + if (object.items->head) Release(object.items->head); object.items->head = reinterpret_cast(std::uintptr_t{0xffffffff}); object.items->count = std::numeric_limits::max(); Release(object.items); @@ -962,29 +962,29 @@ void Construct(T*& destination, const TArgs&... arguments) { ConstructHandle(destination, [&](T& value) { Construct(value, arguments...); }); } +void CopyArrayStorage(ArrayStorage& destination, const ArrayStorage& source) { + ArrayStorage pending{NewArrayHead(), 0, 0}; + try { + if (pending.head) { + for (const auto& item : source) { + try { CopyInto(pending.emplace_back(), item); } + catch (const GuestOutOfMemory&) { break; } + } + } + } catch (...) { pending.clear(); if (pending.head) Release(pending.head); throw; } + destination.clear(); + if (destination.head) Release(destination.head); + destination.head = pending.head; + destination.count = pending.count; +} + void CopyArray(Array& destination, const Array& source) { Array pending{}; if (!source.items) { destination = pending; return; } Construct(pending); if (!pending.items) { destination = pending; return; } - auto* originalHead = pending.items->head; - pending.items->head = NewArrayHead(); - try { - if (pending.items->head) { - for (const auto& item : *source.items) { - try { - CopyInto(pending.items->emplace_back(), item); - } catch (const GuestOutOfMemory&) { - break; - } - } - } - } catch (...) { - Release(originalHead); - Destroy(pending); - throw; - } - Release(originalHead); + try { CopyArrayStorage(*pending.items, *source.items); } + catch (...) { Destroy(pending); throw; } destination = pending; } @@ -1012,6 +1012,8 @@ void CopyObject(Object*& destination, const Object* source) { void AssignReal(Node& n, double value); void SetType(Node& n, ValueType type); +void AssignArray(Array& destination, const Array& source); +void AssignObject(Object& destination, const Object& source); void AssignNode(Node& destination, const Node& source) { if (source.type == TypeNull) { SetType(destination, TypeNull); return; } @@ -1036,20 +1038,23 @@ void AssignNode(Node& destination, const Node& source) { if (destination.string && destination.string->text != string->text) AssignText(*destination.string, View(*string), true); return; } - if (&destination == &source) return; - Node copy{}; - copy.type = source.type; - try { - if (source.type == TypeArray) CopyArray(copy.array, source.array); - else if (source.type == TypeObject) CopyObject(copy.object, source.object); - else NotImplemented_nid_no_patch(__func__); - if ((copy.type == TypeArray && !copy.array) || (copy.type == TypeObject && !copy.object)) copy.type = TypeNull; - } catch (...) { - Clear(copy); - throw; + if (source.type == TypeArray) { + const auto* array = source.array; + if (!array) NotImplemented_nid_no_patch(__func__); + if (!array->items) return; + SetType(destination, TypeArray); + if (destination.array) AssignArray(*destination.array, *array); + return; } - Clear(destination); - TransferPayload(destination, copy); + if (source.type == TypeObject) { + const auto* object = source.object; + if (!object) NotImplemented_nid_no_patch(__func__); + if (!object->items) return; + SetType(destination, TypeObject); + if (destination.object) AssignObject(*destination.object, *object); + return; + } + NotImplemented_nid_no_patch(__func__); } void RecoverSpecial(const Node& source, const Handlers& handlers, int function, double number, Value& result, bool* error) { @@ -1401,6 +1406,36 @@ struct TemporaryTreeEntry { ~TemporaryTreeEntry() { Destroy(pair.value); Destroy(pair.key); } }; +void AssignArray(Array& destination, const Array& source) { + if (!destination.items || !source.items) return; + if (!ModernValues()) { + CopyArrayStorage(*destination.items, *source.items); + return; + } + Array pending{}; + Construct(pending); + if (!pending.items) return; + if (!pending.items->head) { Destroy(pending); return; } + pending.items->SetOwner(destination.items->Owner()); + try { + for (const auto& item : *source.items) { + ScopedValue placeholder; + SetType(placeholder.value, item.type); + const auto failures = AllocationFailures(); + try { CopyInto(pending.items->emplace_back(), placeholder.value); } + catch (const GuestOutOfMemory&) {} + if (AllocationFailures() == failures) { + auto& value = pending.items->back(); + value.parent = destination.items->Owner(); + AssignNode(value, item); + } + } + } catch (...) { Destroy(pending); throw; } + auto previous = destination; + destination = pending; + Destroy(previous); +} + struct ScopedPair { Pair pair{}; ~ScopedPair() { Destroy(pair.value); Destroy(pair.key); } @@ -1879,6 +1914,7 @@ int APS5_VABI _ZNK3sce4Json6String7compareERKS1_(const String* self, const Strin void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array* self) { Construct(*self); } void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array* self, const Array* other) { CopyArray(*self, *other); } void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array* self) { Destroy(*self); } +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array* self, const Array* other) { AssignArray(*self, *other); return self; } void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array* self, const Value* value) { if (!self->items || !self->items->head) return; const auto failures = AllocationFailures(); diff --git a/core/libs/tests/GuestJsonArrayAllocator.cpp b/core/libs/tests/GuestJsonArrayAllocator.cpp index bf8e5dc6b7..15954c0f90 100644 --- a/core/libs/tests/GuestJsonArrayAllocator.cpp +++ b/core/libs/tests/GuestJsonArrayAllocator.cpp @@ -25,6 +25,7 @@ const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); @@ -58,12 +59,13 @@ struct ArrayTrace { std::map live; std::vector requests; int budget = -1; + bool once = false; unsigned notifications = 0; std::set headers; }; static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); trace.requests.push_back(bytes); - if (trace.budget == 0) return nullptr; + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } if (trace.budget > 0) --trace.budget; void* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); @@ -99,12 +101,59 @@ static void CheckLinks(const Array& array, std::size_t count) { } Require(seen == count && head[0] == reinterpret_cast(previous), "Array sentinel must close both list directions"); } +static void CheckAssignments(ArrayTrace& trace) { + const auto baseline = trace.live.size(); + for (bool once : {false, true}) for (int budget = 0; budget <= 5; ++budget) { + Array source{}, destination{}; + _ZN3sce4Json5ArrayC1Ev(&source); _ZN3sce4Json5ArrayC1Ev(&destination); + Value value{}; + for (int number : {10, 20, 30}) { _ZN3sce4Json5ValueC1El(&value, number); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&source, &value); _ZN3sce4Json5ValueD1Ev(&value); } + _ZN3sce4Json5ValueC1El(&value, 99); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&destination, &value); _ZN3sce4Json5ValueD1Ev(&value); + Value owner{}; + static_cast(destination.storage)[3] = reinterpret_cast(&owner); + auto* previous = destination.storage; + trace.headers.insert(previous); + const auto notifications = trace.notifications; + trace.budget = budget; trace.once = once; + Require(_ZN3sce4Json5ArrayaSERKS1_(&destination, &source) == &destination, "Array assignment must return its destination"); + trace.budget = -1; trace.once = false; + auto* header = static_cast(destination.storage); + Require((destination.storage == previous) == (budget < 2) && header[3] == reinterpret_cast(&owner), "Array assignment must preserve the original on initial failure and carry its owner into a replacement"); + const auto expected = budget < 2 ? 1 : budget == 5 ? 3 : once ? 2 : budget - 2; + Require(header[1] == static_cast(expected), "Array assignment must retain the observed partial list"); + Require(trace.notifications - notifications == (budget < 2 ? 1u : budget == 5 ? 0u : once ? 1u : static_cast(5 - budget)), "Array assignment must report failed node admission while continuing through the source"); + auto* head = reinterpret_cast(header[0]); + auto* node = reinterpret_cast(head[1]); + for (int index = 0, sourceIndex = 0; index < expected; ++index, ++sourceIndex) { + if (once && budget >= 2 && budget < 5 && sourceIndex == budget - 2) ++sourceIndex; + const auto* entry = reinterpret_cast(node + 2); + Require(entry->type == 2 && entry->payload == static_cast(budget < 2 ? 99 : (sourceIndex + 1) * 10), "Array assignment must preserve retained entry values"); + if (budget >= 2) Require(entry->parent == &owner && !entry->auxiliary, "Replacement array entries must bind to the destination owner"); + node = reinterpret_cast(node[1]); + } + Require(node == head, "Replacement array traversal must close at its sentinel"); + CheckLinks(source, 3); + trace.headers.insert(destination.storage); + _ZN3sce4Json5ArrayD1Ev(&destination); _ZN3sce4Json5ArrayD1Ev(&source); + Require(trace.live.size() == baseline, "Assignment must release failed replacement resources and its previous list"); + } + Array array{}; _ZN3sce4Json5ArrayC1Ev(&array); + Value value{}; + for (int number : {10, 20, 30}) { _ZN3sce4Json5ValueC1El(&value, number); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); _ZN3sce4Json5ValueD1Ev(&value); } + auto* previous = array.storage; trace.headers.insert(previous); + _ZN3sce4Json5ArrayaSERKS1_(&array, &array); + Require(array.storage != previous, "Modern array self-assignment must replace its header"); + CheckLinks(array, 3); + trace.headers.insert(array.storage); _ZN3sce4Json5ArrayD1Ev(&array); + Require(trace.live.size() == baseline, "Array self-assignment must release its old list"); +} int main() { ArrayTrace trace; ArrayAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 2, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize array allocation contract"); const auto baseline = trace.live.size(); + CheckAssignments(trace); for (int budget : {0, 1}) { trace.budget = budget; Array partial{}; _ZN3sce4Json5ArrayC1Ev(&partial); Require(budget ? partial.storage && trace.live.at(partial.storage) == 32 && !*static_cast(partial.storage) : !partial.storage, diff --git a/core/libs/tests/GuestJsonValueAllocator.cpp b/core/libs/tests/GuestJsonValueAllocator.cpp index ba6a21538f..e256662183 100644 --- a/core/libs/tests/GuestJsonValueAllocator.cpp +++ b/core/libs/tests/GuestJsonValueAllocator.cpp @@ -198,6 +198,44 @@ static void CheckScalarCopies(ValueAllocationTrace& trace) { Require(trace.live.size() == baseline, "Copy failures must reclaim destination and source resources"); } } +static void CheckCollectionCopies(ValueAllocationTrace& trace) { + const auto baseline = trace.live.size(); + for (int previous = -1; previous <= 7; ++previous) for (int type : {6, 7}) for (int budget = 0; budget <= 5; ++budget) { + Value source; std::memset(&source, 0xa5, sizeof(source)); + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&source, type); + const auto sourcePayload = source.payload; + Value destination; std::memset(&destination, 0xa5, sizeof(destination)); + void* auxiliary = nullptr; + void* originalHeader = nullptr; + if (previous >= 0) { + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&destination, previous); + Require(Register(destination, 0, &trace, true) == 0, "Prepare collection assignment metadata"); + auxiliary = destination.auxiliary; + if (previous == type) originalHeader = *reinterpret_cast(destination.payload); + } + const auto notifications = trace.notifications; + trace.budget = budget; + if (previous < 0) _ZN3sce4Json5ValueC1ERKS1_(&destination, &source); + else _ZN3sce4Json5ValueaSERKS1_(&destination, &source); + trace.budget = -1; + const auto threshold = type == 6 ? 2 : 3; + const auto expected = previous == type || budget >= threshold ? type : 0; + Check(destination, expected, trace); + Require(!destination.parent && destination.auxiliary == auxiliary, "Collection copies must preserve destination metadata"); + if (!expected) Require(destination.payload == 0, "Failed collection admission must leave a null handle"); + else { + auto* header = *reinterpret_cast(destination.payload); + const auto* words = static_cast(header); + Require(words[1] == 0 && (words[0] != 0) == !(type == 6 && previous != type && budget == 2), "Collection assignment must preserve the reference partial-header state"); + if (previous == type) Require((header == originalHeader) == (budget < 2), "Same-type assignment must preserve the header only when replacement admission fails"); + } + const auto expectedNotifications = budget == 5 || (previous == type && budget >= 2) ? 0u : type == 6 && previous != type && budget == 2 ? 2u : 1u; + Require(trace.notifications - notifications == expectedNotifications, "Collection copies must notify both type-admission and replacement allocation failures"); + Require(source.payload == sourcePayload && source.type == type, "Collection assignment must preserve source ownership"); + _ZN3sce4Json5ValueD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); + Require(trace.live.size() == baseline, "Collection copies must reclaim partial handles and replaced storage"); + } +} int main() { ValueAllocationTrace trace; ValueAllocationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 0, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); @@ -205,6 +243,7 @@ int main() { const auto baseline = trace.live.size(); CheckMetadata(trace); CheckScalarCopies(trace); + CheckCollectionCopies(trace); for (int target = 0; target <= 7; ++target) for (int budget = 0; budget <= 4; ++budget) { Value value; std::memset(&value, 0xa5, sizeof(value)); trace.budget = budget; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 8e31da9563..384a83aa00 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -106,3 +106,5 @@ - [sce::Json::Value auxiliary admission and retry](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy per-root mismatch, float and element handler maps remain unsupported and throw. Modern admission follows `0x3280`: a failed fallback allocation retains the 72-byte auxiliary record; later registration reuses it without recreating the missing fallback. Local tests cover all four callback setters, their allocation failures, replacement and null callbacks, child rejection, parent preservation and teardown through the original allocator. Legacy auxiliary records use 24 bytes, but their external handler maps and dispatch remain incomplete. - [sce::Json::Value scalar and string copying](../../../core/libs/prx/libSceJson2/Export.cpp) - collection assignment and legacy callback-map propagation remain incomplete. Scalar and string dispatch follow `0x1e40`: null and boolean copies preserve inactive destination bytes, numeric values retain their exact payloads, and string assignment changes type before copying characters into the existing header. A character-allocation failure can empty an existing string; it does not restore its old contents. Copy construction initializes ownership metadata directly, following `0x1e20`. Local allocator tests cover null, boolean, signed, unsigned, finite real and string sources across all destination types and failure budgets. + +- [sce::Json::Array and Value collection assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - collection-specific Value constructors/setters, nested allocator failures, legacy callback propagation and concurrent operation accounting remain incomplete. Modern array assignment follows `0xbc80`: initial header/sentinel failure preserves the original, node failures retain a partial replacement and iteration continues, self-assignment replaces the header, and entries bind to the destination owner. Legacy assignment uses replacement list storage inside the existing header, following `0xb540`. Value collection dispatch follows `0x1e40`, changing type before invoking Array/Object assignment rather than cloning through their copy constructors. Local tests cover scalar array assignment, self-assignment and empty collection Value admission across destination types. From 6df805d9f6a66ad4e1beeb841cb6395543998855 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 00:59:53 +0200 Subject: [PATCH 019/318] fix(libSceJson2): honor collection constructor and setter admission --- core/libs/prx/libSceJson2/Export.cpp | 43 +++++---------- core/libs/tests/GuestJsonValueAllocator.cpp | 61 ++++++++++++++++++++- core/libs/tests/JsonCopyFailure.cpp | 4 +- docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 78 insertions(+), 34 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 27041e0856..0f4860b0d8 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1001,13 +1001,6 @@ void CopyObject(Object& destination, const Object& source) { destination = pending; } -void CopyArray(Array*& destination, const Array* source) { - ConstructHandle(destination, [&](Array& value) { CopyArray(value, *source); }); -} - -void CopyObject(Object*& destination, const Object* source) { - ConstructHandle(destination, [&](Object& value) { CopyObject(value, *source); }); -} void AssignReal(Node& n, double value); @@ -2092,15 +2085,15 @@ void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6StringE(Value* self, const String* te } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value* self, const Array* array) { Construct(*self); - CopyArray(self->array, array); - self->type = self->array ? TypeArray : TypeNull; - BindChildren(*self); + if (!array->items) return; + SetType(*self, TypeArray); + if (self->array) AssignArray(*self->array, *array); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value* self, const Object* object) { Construct(*self); - CopyObject(self->object, object); - self->type = self->object ? TypeObject : TypeNull; - BindChildren(*self); + if (!object->items) return; + SetType(*self, TypeObject); + if (self->object) AssignObject(*self->object, *object); } void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value* self, const Value* other) { CopyInto(*self, *other); } void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value* self) { Destroy(*self); } @@ -2139,23 +2132,15 @@ int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* t if (self->type == TypeString) AssignText(*self->string, copy, true); return 0; } -int APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { - Array* copy = nullptr; - CopyArray(copy, array); - Clear(NodeOf(*self)); - self->type = copy ? TypeArray : TypeNull; - self->array = copy; - BindChildren(*self); - return 0; +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { + if (!array->items) return; + SetType(*self, TypeArray); + if (self->array) AssignArray(*self->array, *array); } -int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value* self, const Object* object) { - Object* copy = nullptr; - CopyObject(copy, object); - Clear(NodeOf(*self)); - self->type = copy ? TypeObject : TypeNull; - self->object = copy; - BindChildren(*self); - return 0; +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value* self, const Object* object) { + if (!object->items) return; + SetType(*self, TypeObject); + if (self->object) AssignObject(*self->object, *object); } int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value* self, const Value* other) { if (self != other) AssignNode(NodeOf(*self), NodeOf(*other)); diff --git a/core/libs/tests/GuestJsonValueAllocator.cpp b/core/libs/tests/GuestJsonValueAllocator.cpp index e256662183..85645df985 100644 --- a/core/libs/tests/GuestJsonValueAllocator.cpp +++ b/core/libs/tests/GuestJsonValueAllocator.cpp @@ -11,6 +11,8 @@ struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +struct Array { void* storage; }; +struct Object { void* storage; }; using NullCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); using MismatchCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); using FloatCallback = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); @@ -27,6 +29,10 @@ void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value*, const Object*); +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value*, const Object*); int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullCallback, void*); int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, MismatchCallback, void*); int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatCallback, void*); @@ -200,7 +206,7 @@ static void CheckScalarCopies(ValueAllocationTrace& trace) { } static void CheckCollectionCopies(ValueAllocationTrace& trace) { const auto baseline = trace.live.size(); - for (int previous = -1; previous <= 7; ++previous) for (int type : {6, 7}) for (int budget = 0; budget <= 5; ++budget) { + for (bool specialized : {false, true}) for (int previous = -1; previous <= 7; ++previous) for (int type : {6, 7}) for (int budget = 0; budget <= 5; ++budget) { Value source; std::memset(&source, 0xa5, sizeof(source)); _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&source, type); const auto sourcePayload = source.payload; @@ -215,7 +221,15 @@ static void CheckCollectionCopies(ValueAllocationTrace& trace) { } const auto notifications = trace.notifications; trace.budget = budget; - if (previous < 0) _ZN3sce4Json5ValueC1ERKS1_(&destination, &source); + if (specialized && type == 6) { + const auto* array = reinterpret_cast(source.payload); + if (previous < 0) _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&destination, array); + else _ZN3sce4Json5Value3setERKNS0_5ArrayE(&destination, array); + } else if (specialized) { + const auto* object = reinterpret_cast(source.payload); + if (previous < 0) _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(&destination, object); + else _ZN3sce4Json5Value3setERKNS0_6ObjectE(&destination, object); + } else if (previous < 0) _ZN3sce4Json5ValueC1ERKS1_(&destination, &source); else _ZN3sce4Json5ValueaSERKS1_(&destination, &source); trace.budget = -1; const auto threshold = type == 6 ? 2 : 3; @@ -236,6 +250,48 @@ static void CheckCollectionCopies(ValueAllocationTrace& trace) { Require(trace.live.size() == baseline, "Collection copies must reclaim partial handles and replaced storage"); } } +static void CheckMissingCollections(ValueAllocationTrace& trace) { + const auto baseline = trace.live.size(); + const Array array{}; const Object object{}; + for (int previous = -1; previous <= 7; ++previous) for (bool useArray : {false, true}) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + if (previous >= 0) { + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&value, previous); + Require(Register(value, 0, &trace, true) == 0, "Prepare missing-source collection metadata"); + } + const auto original = value; + const auto requests = trace.requests.size(); + const auto notifications = trace.notifications; + trace.budget = 0; + if (previous < 0) { + if (useArray) _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&value, &array); + else _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(&value, &object); + } else { + if (useArray) _ZN3sce4Json5Value3setERKNS0_5ArrayE(&value, &array); + else _ZN3sce4Json5Value3setERKNS0_6ObjectE(&value, &object); + } + trace.budget = -1; + Require(trace.requests.size() == requests && trace.notifications == notifications, "A missing input header must not allocate or notify"); + if (previous >= 0) Require(std::memcmp(&value, &original, sizeof(value)) == 0, "A missing input header must preserve the entire destination"); + else Require(value.type == 0 && !value.parent && !value.auxiliary && value.payload == original.payload && value.reserved == original.reserved, "Construction from a missing header must initialize null metadata and preserve inactive bytes"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Missing collection inputs must not leak ownership"); + } + for (int type : {6, 7}) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&value, type); + Require(Register(value, 0, &trace, true) == 0, "Prepare collection self-setting metadata"); + const auto handle = value.payload; + const auto* header = *reinterpret_cast(handle); + auto* auxiliary = value.auxiliary; + if (type == 6) _ZN3sce4Json5Value3setERKNS0_5ArrayE(&value, reinterpret_cast(handle)); + else _ZN3sce4Json5Value3setERKNS0_6ObjectE(&value, reinterpret_cast(handle)); + Check(value, type, trace); + Require(value.payload == handle && *reinterpret_cast(handle) != header && value.auxiliary == auxiliary, "Collection self-setting must replace storage while retaining the handle and callback metadata"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Collection self-setting must release replaced storage"); + } +} int main() { ValueAllocationTrace trace; ValueAllocationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 0, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); @@ -244,6 +300,7 @@ int main() { CheckMetadata(trace); CheckScalarCopies(trace); CheckCollectionCopies(trace); + CheckMissingCollections(trace); for (int target = 0; target <= 7; ++target) for (int budget = 0; budget <= 4; ++budget) { Value value; std::memset(&value, 0xa5, sizeof(value)); trace.budget = budget; diff --git a/core/libs/tests/JsonCopyFailure.cpp b/core/libs/tests/JsonCopyFailure.cpp index 233ed22ac8..2b19fd9d47 100644 --- a/core/libs/tests/JsonCopyFailure.cpp +++ b/core/libs/tests/JsonCopyFailure.cpp @@ -35,8 +35,8 @@ void APS5_VABI _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(Value*, std::int32_t); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); -int APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value*, const Array*); -int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value*, const Object*); +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value*, const Object*); int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 384a83aa00..af83da0e39 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -107,4 +107,6 @@ - [sce::Json::Value scalar and string copying](../../../core/libs/prx/libSceJson2/Export.cpp) - collection assignment and legacy callback-map propagation remain incomplete. Scalar and string dispatch follow `0x1e40`: null and boolean copies preserve inactive destination bytes, numeric values retain their exact payloads, and string assignment changes type before copying characters into the existing header. A character-allocation failure can empty an existing string; it does not restore its old contents. Copy construction initializes ownership metadata directly, following `0x1e20`. Local allocator tests cover null, boolean, signed, unsigned, finite real and string sources across all destination types and failure budgets. -- [sce::Json::Array and Value collection assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - collection-specific Value constructors/setters, nested allocator failures, legacy callback propagation and concurrent operation accounting remain incomplete. Modern array assignment follows `0xbc80`: initial header/sentinel failure preserves the original, node failures retain a partial replacement and iteration continues, self-assignment replaces the header, and entries bind to the destination owner. Legacy assignment uses replacement list storage inside the existing header, following `0xb540`. Value collection dispatch follows `0x1e40`, changing type before invoking Array/Object assignment rather than cloning through their copy constructors. Local tests cover scalar array assignment, self-assignment and empty collection Value admission across destination types. +- [sce::Json::Array and Value collection assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocator failures, legacy callback propagation and concurrent operation accounting remain incomplete. Modern array assignment follows `0xbc80`: initial header/sentinel failure preserves the original, node failures retain a partial replacement and iteration continues, self-assignment replaces the header, and entries bind to the destination owner. Legacy assignment uses replacement list storage inside the existing header, following `0xb540`. Value collection dispatch follows `0x1e40`, changing type before invoking Array/Object assignment rather than cloning through their copy constructors. Local tests cover scalar array assignment, self-assignment and empty collection Value admission across destination types. + +- [sce::Json::Value constructors and setters taking Array/Object](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocation failures and older SDK compatibility remain incomplete. These paths follow `0x1d00`, `0x1d50`, `0x1d90` and `0x1de0`: a missing source header is ignored, while admitted inputs change the destination type before collection assignment. Missing-header construction initializes null ownership metadata without changing inactive payload bytes. Self-setting retains the Value handle and callback metadata while replacing collection storage. The setters expose no consistent status result and are declared `void`; local tests verify side effects rather than a fabricated success return. From 2f203559a95ea23f7704e01a4a9b7c1118742cca Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 01:05:35 +0200 Subject: [PATCH 020/318] test(libSceJson2): cover nested array copy allocation failures --- core/libs/tests/GuestJsonValueAllocator.cpp | 138 +++++++++++++++++++- docs/dev/debt/libSceJson2.md | 2 + 2 files changed, 139 insertions(+), 1 deletion(-) diff --git a/core/libs/tests/GuestJsonValueAllocator.cpp b/core/libs/tests/GuestJsonValueAllocator.cpp index 85645df985..7b8d40de52 100644 --- a/core/libs/tests/GuestJsonValueAllocator.cpp +++ b/core/libs/tests/GuestJsonValueAllocator.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; @@ -26,6 +27,8 @@ void APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); @@ -46,13 +49,14 @@ struct ValueAllocationTrace { std::vector requests; std::vector frees; int budget = -1; + bool once = false; unsigned notifications = 0; unsigned nullFrees = 0; const Value* registering = nullptr; }; static void* APS5_VABI ValueAllocationAllocate(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); trace.requests.push_back(bytes); - if (trace.budget == 0) return nullptr; + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } if (trace.budget > 0) --trace.budget; auto* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; @@ -292,6 +296,137 @@ static void CheckMissingCollections(ValueAllocationTrace& trace) { Require(trace.live.size() == baseline, "Collection self-setting must release replaced storage"); } } +static void PrepareNestedSource(Value& value) { + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&value, 6); + auto* array = reinterpret_cast(value.payload); + Value item{}; + _ZN3sce4Json5ValueC1EPKc(&item, "abc"); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(array, &item); + _ZN3sce4Json5ValueD1Ev(&item); + _ZN3sce4Json5ValueC1El(&item, 42); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(array, &item); + _ZN3sce4Json5ValueD1Ev(&item); + Value nested{}; + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&nested, 6); + _ZN3sce4Json5ValueC1EPKc(&item, "def"); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(reinterpret_cast(nested.payload), &item); + _ZN3sce4Json5ValueD1Ev(&item); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(array, &nested); + _ZN3sce4Json5ValueD1Ev(&nested); +} +static std::string NestedState(const Value& value, const Value* parent = nullptr) { + std::string state = std::to_string(value.type) + (value.parent == parent ? "p" : value.parent ? "x" : "-"); + if (value.type == 0) return state; + if (value.type == 2) return state + std::to_string(value.payload); + if (value.type != 5 && value.type != 6) { Require(false, "Unexpected nested test value type"); return {}; } + if (!value.payload) return state + "!"; + const auto* header = *reinterpret_cast(value.payload); + if (!header) return state + "?"; + if (value.type == 5) return state + ":" + *static_cast(header); + const auto* words = static_cast(header); + state += words[3] == reinterpret_cast(&value) ? "o" : "!"; + state += std::to_string(words[1]) + "["; + if (!words[0]) return state + "?]"; + const auto* head = reinterpret_cast(words[0]); + auto* node = reinterpret_cast(head[1]); + std::size_t count = 0; + const auto* previous = head; + while (node != head) { + Require(node[0] == reinterpret_cast(previous), "Nested array predecessor links must preserve ownership order"); + Require(++count <= words[1], "Nested array nodes must close at their sentinel"); + state += NestedState(*reinterpret_cast(node + 2), &value) + ";"; + previous = node; + node = reinterpret_cast(node[1]); + } + Require(head[0] == reinterpret_cast(previous), "Nested array sentinel must retain its last live entry"); + Require(count == words[1], "Nested array count must match its live links"); + return state + "]"; +} +static void CheckNestedCopies(ValueAllocationTrace& trace) { + struct Case { bool once; int budget; unsigned notifications; std::size_t requests; const char* state; }; + const Case cases[]{ + {0, 0, 1, 1, "0p"}, + {0, 1, 1, 2, "0p"}, + {0, 2, 2, 4, "6po0[?]"}, + {0, 3, 1, 4, "6po0[]"}, + {0, 4, 1, 5, "6po0[]"}, + {0, 5, 5, 10, "6po0[]"}, + {0, 6, 5, 11, "6po0[]"}, + {0, 7, 4, 11, "6po0[]"}, + {0, 8, 4, 12, "6po1[0-;]"}, + {0, 9, 4, 13, "6po1[0-;]"}, + {0, 10, 4, 14, "6po1[5p:;]"}, + {0, 11, 3, 14, "6po1[5p:abc;]"}, + {0, 12, 2, 14, "6po2[5p:abc;2p42;]"}, + {0, 13, 2, 15, "6po2[5p:abc;2p42;]"}, + {0, 14, 2, 16, "6po2[5p:abc;2p42;]"}, + {0, 15, 1, 16, "6po2[5p:abc;2p42;]"}, + {0, 16, 1, 17, "6po3[5p:abc;2p42;0-;]"}, + {0, 17, 1, 18, "6po3[5p:abc;2p42;0-;]"}, + {0, 18, 2, 20, "6po3[5p:abc;2p42;6-o0[?];]"}, + {0, 19, 1, 20, "6po3[5p:abc;2p42;6-o0[];]"}, + {0, 20, 1, 21, "6po3[5p:abc;2p42;6-o0[];]"}, + {0, 21, 1, 22, "6po3[5p:abc;2p42;6po0[];]"}, + {0, 22, 1, 23, "6po3[5p:abc;2p42;6po0[];]"}, + {0, 23, 2, 25, "6po3[5p:abc;2p42;6po0[];]"}, + {0, 24, 2, 26, "6po3[5p:abc;2p42;6po0[];]"}, + {0, 25, 1, 26, "6po3[5p:abc;2p42;6po0[];]"}, + {0, 26, 1, 27, "6po3[5p:abc;2p42;6po1[0-;];]"}, + {0, 27, 1, 28, "6po3[5p:abc;2p42;6po1[0-;];]"}, + {0, 28, 1, 29, "6po3[5p:abc;2p42;6po1[5p:;];]"}, + {0, 29, 0, 29, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 0, 1, 1, "0p"}, + {1, 1, 1, 2, "0p"}, + {1, 2, 1, 29, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 3, 1, 4, "6po0[]"}, + {1, 4, 1, 5, "6po0[]"}, + {1, 5, 1, 28, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 6, 1, 29, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 7, 1, 26, "6po2[2p42;6po1[5p:def;];]"}, + {1, 8, 1, 27, "6po3[0-;2p42;6po1[5p:def;];]"}, + {1, 9, 1, 28, "6po3[0-;2p42;6po1[5p:def;];]"}, + {1, 10, 1, 29, "6po3[5p:;2p42;6po1[5p:def;];]"}, + {1, 11, 1, 29, "6po2[5p:abc;6po1[5p:def;];]"}, + {1, 12, 1, 25, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 13, 1, 26, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 14, 1, 29, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 15, 1, 16, "6po2[5p:abc;2p42;]"}, + {1, 16, 1, 17, "6po3[5p:abc;2p42;0-;]"}, + {1, 17, 1, 18, "6po3[5p:abc;2p42;0-;]"}, + {1, 18, 1, 21, "6po3[5p:abc;2p42;6-o0[];]"}, + {1, 19, 1, 20, "6po3[5p:abc;2p42;6-o0[];]"}, + {1, 20, 1, 21, "6po3[5p:abc;2p42;6-o0[];]"}, + {1, 21, 1, 22, "6po3[5p:abc;2p42;6po0[];]"}, + {1, 22, 1, 23, "6po3[5p:abc;2p42;6po0[];]"}, + {1, 23, 1, 28, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 24, 1, 29, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + {1, 25, 1, 26, "6po3[5p:abc;2p42;6po0[];]"}, + {1, 26, 1, 27, "6po3[5p:abc;2p42;6po1[0-;];]"}, + {1, 27, 1, 28, "6po3[5p:abc;2p42;6po1[0-;];]"}, + {1, 28, 1, 29, "6po3[5p:abc;2p42;6po1[5p:;];]"}, + {1, 29, 0, 29, "6po3[5p:abc;2p42;6po1[5p:def;];]"}, + }; + const auto baseline = trace.live.size(); + for (bool constructor : {false, true}) for (const auto& test : cases) { + Value source{}; PrepareNestedSource(source); + const auto sourceState = NestedState(source); + Value destination{}; + if (!constructor) _ZN3sce4Json5ValueC1EPKc(&destination, "previous"); + const auto requests = trace.requests.size(); + const auto notifications = trace.notifications; + trace.budget = test.budget; trace.once = test.once; + if (constructor) _ZN3sce4Json5ValueC1ERKS1_(&destination, &source); + else _ZN3sce4Json5ValueaSERKS1_(&destination, &source); + trace.budget = -1; trace.once = false; + const auto state = NestedState(destination); + const bool matches = state == test.state && trace.requests.size() - requests == test.requests && trace.notifications - notifications == test.notifications; + if (!matches) std::fprintf(stderr, "Nested copy constructor=%d once=%d budget=%d expected=%s actual=%s requests=%zu notifications=%u\n", constructor, test.once, test.budget, test.state, state.c_str(), trace.requests.size() - requests, trace.notifications - notifications); + Require(matches, "Nested copy must retain the reference partial state and allocation failure accounting"); + Require(NestedState(source) == sourceState, "Nested allocation failure must preserve its source"); + _ZN3sce4Json5ValueD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); + Require(trace.live.size() == baseline, "Nested partial-state teardown must release all admitted ownership"); + } +} int main() { ValueAllocationTrace trace; ValueAllocationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 0, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); @@ -301,6 +436,7 @@ int main() { CheckScalarCopies(trace); CheckCollectionCopies(trace); CheckMissingCollections(trace); + CheckNestedCopies(trace); for (int target = 0; target <= 7; ++target) for (int budget = 0; budget <= 4; ++budget) { Value value; std::memset(&value, 0xa5, sizeof(value)); trace.budget = budget; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index af83da0e39..b585ddfe78 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -110,3 +110,5 @@ - [sce::Json::Array and Value collection assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocator failures, legacy callback propagation and concurrent operation accounting remain incomplete. Modern array assignment follows `0xbc80`: initial header/sentinel failure preserves the original, node failures retain a partial replacement and iteration continues, self-assignment replaces the header, and entries bind to the destination owner. Legacy assignment uses replacement list storage inside the existing header, following `0xb540`. Value collection dispatch follows `0x1e40`, changing type before invoking Array/Object assignment rather than cloning through their copy constructors. Local tests cover scalar array assignment, self-assignment and empty collection Value admission across destination types. - [sce::Json::Value constructors and setters taking Array/Object](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocation failures and older SDK compatibility remain incomplete. These paths follow `0x1d00`, `0x1d50`, `0x1d90` and `0x1de0`: a missing source header is ignored, while admitted inputs change the destination type before collection assignment. Missing-header construction initializes null ownership metadata without changing inactive payload bytes. Self-setting retains the Value handle and callback metadata while replacing collection storage. The setters expose no consistent status result and are declared `void`; local tests verify side effects rather than a fabricated success return. + +- [sce::Json::Value nested array allocation regressions](../../../core/libs/tests/GuestJsonValueAllocator.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. From 6e34c2544a365ad3814cbe1b29f5c984908d9fdc Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 01:20:04 +0200 Subject: [PATCH 021/318] fix(libSceJson2): own initialization resources and roll back allocation failures --- core/libs/prx/libSceJson2/CMakeLists.txt | 12 +- core/libs/prx/libSceJson2/Export.cpp | 142 ++++++++++++--- core/libs/tests/GuestJsonAllocator.cpp | 221 +++++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 8 +- 4 files changed, 351 insertions(+), 32 deletions(-) create mode 100644 core/libs/tests/GuestJsonAllocator.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index a7cd7404e1..5f6d721f77 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(libSceJson2 SHARED EXCLUDE_FROM_ALL Export.cpp InventoryExports.cpp) +target_link_libraries(libSceJson2 PRIVATE libkernel libc) target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) @@ -60,6 +61,11 @@ if(BUILD_TESTING) target_link_libraries(guest_json_array_allocator_tests PRIVATE libSceJson2 libc) configure_windows_unwind(guest_json_array_allocator_tests) add_test(NAME guest_json_array_allocator COMMAND guest_json_array_allocator_tests) + add_test_executable(guest_json_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonAllocator.cpp) + target_include_directories(guest_json_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_allocator_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_allocator_tests) + add_test(NAME guest_json_allocator COMMAND guest_json_allocator_tests) add_test_executable(guest_json_value_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonValueAllocator.cpp) target_include_directories(guest_json_value_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_value_allocator_tests PRIVATE libSceJson2 libc) @@ -80,7 +86,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() @@ -118,12 +124,12 @@ endif() if(BUILD_TESTING) add_test_executable(guest_json_file_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonFile.cpp Export.cpp) target_include_directories(guest_json_file_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_json_file_tests PRIVATE libc) + target_link_libraries(guest_json_file_tests PRIVATE libkernel libc) configure_windows_unwind(guest_json_file_tests) add_test(NAME guest_json_file COMMAND guest_json_file_tests) set_tests_properties(guest_json_file PROPERTIES TIMEOUT 30) if(WIN32) set_tests_properties(guest_json_file PROPERTIES ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:$") + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 0f4860b0d8..089814fe22 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include "prx/libc/include/General.hpp" #include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libkernel/File/include/File.hpp" +#include "prx/libkernel/Pthread/include/Mutex.hpp" namespace { @@ -147,7 +149,10 @@ struct Globals { bool initialized = false; AllocParamRtti rtti{}; Handlers handlers{}; - AllocationContext allocation{}; + AllocationContext* allocation = nullptr; + Value* fallback = nullptr; + PthreadMutex* internalMutex = nullptr; + PthreadMutex* recoveryMutex = nullptr; std::recursive_mutex callbackMutex; bool callbackActive = false; std::uint64_t allocationFailures = 0; @@ -158,9 +163,20 @@ Globals& State() { return globals; } +struct KernelMutexLock { + PthreadMutex* mutex; + explicit KernelMutexLock(PthreadMutex* value) : mutex(value) { + if (mutex && scePthreadMutexLock(mutex)) NotImplemented_nid_no_patch(__func__); + } + ~KernelMutexLock() { if (mutex) scePthreadMutexUnlock(mutex); } + KernelMutexLock(const KernelMutexLock&) = delete; + KernelMutexLock& operator=(const KernelMutexLock&) = delete; +}; + struct CallbackScope { Globals& state = State(); std::unique_lock lock{state.callbackMutex}; + KernelMutexLock kernelLock{state.recoveryMutex}; bool entered = !state.callbackActive; CallbackScope() { if (entered) state.callbackActive = true; } ~CallbackScope() { if (entered) state.callbackActive = false; } @@ -181,6 +197,7 @@ void* Allocate(const AllocationContext& context, std::size_t size) { notify(context.allocator, JsonErrorOutOfMemory, size, context.userData); auto& state = State(); std::lock_guard lock(state.mutex); + KernelMutexLock kernelLock(state.internalMutex); ++state.allocationFailures; } return memory; @@ -257,7 +274,7 @@ char emptyCharacters[1]{}; AllocationContext ActiveAllocation() { auto& state = State(); std::lock_guard lock(state.mutex); - return state.allocation; + return state.allocation ? *state.allocation : AllocationContext{}; } void* Allocate(std::size_t size) { @@ -296,6 +313,7 @@ bool ModernValues() { std::uint64_t AllocationFailures() { auto& state = State(); std::lock_guard lock(state.mutex); + KernelMutexLock kernelLock(state.internalMutex); return state.allocationFailures; } @@ -1100,6 +1118,7 @@ DefaultStorage& SharedDefaults() { const Value& Defaults(ValueType type) { auto& state = State(); std::lock_guard lock(state.mutex); + if (type == TypeNull && state.fallback) return *state.fallback; auto& defaults = SharedDefaults(); if (!defaults.ready[type]) { Construct(defaults.values[type]); @@ -1118,6 +1137,87 @@ void ResetDefaults() { } } +PthreadMutex* CreateKernelMutex() { + auto* mutex = static_cast(Allocate(sizeof(PthreadMutex))); + if (!mutex) return nullptr; + *mutex = nullptr; + PthreadMutexattr attribute = nullptr; + if (scePthreadMutexattrInit(&attribute)) { Release(mutex); NotImplemented_nid_no_patch(__func__); } + const auto typeStatus = scePthreadMutexattrSettype(&attribute, 2); + const auto status = typeStatus ? typeStatus : scePthreadMutexInit(mutex, &attribute, "SceJson_internal"); + scePthreadMutexattrDestroy(&attribute); + if (status) { Release(mutex); NotImplemented_nid_no_patch(__func__); } + return mutex; +} + +void ReleaseInitialization() { + auto& state = State(); + if (state.fallback) { Destroy(*state.fallback); Release(state.fallback); state.fallback = nullptr; } + if (state.recoveryMutex && state.recoveryMutex != state.internalMutex) { + scePthreadMutexDestroy(state.recoveryMutex); + Release(state.recoveryMutex); + } + state.recoveryMutex = nullptr; + if (state.internalMutex) { scePthreadMutexDestroy(state.internalMutex); Release(state.internalMutex); state.internalMutex = nullptr; } + ResetDefaults(); + if (state.allocation) { + auto* allocation = state.allocation; + const auto context = *allocation; + Release(context, allocation); + state.allocation = nullptr; + } + state.floatFormat = 2; + state.floatFormatEnabled = false; + state.initialized = false; +} + +template +void CreateDefault(ValueType type) { + auto& defaults = SharedDefaults(); + auto& value = defaults.values[type]; + Construct(value); + auto* handle = static_cast(Allocate(sizeof(T))); + value.payload = reinterpret_cast(handle); + value.type = type; + defaults.ready[type] = true; + if (handle) { + *handle = {}; + if constexpr (std::is_same_v) Construct(*handle, std::string_view{}); + else Construct(*handle); + } +} + +int InitializeResources(void* self, const void* parameter, std::int32_t format, bool modern) { + auto& state = State(); + AllocationContext context; + std::memcpy(&context, parameter, offsetof(AllocationContext, fileBufferSize)); + auto* const* table = *static_cast(context.allocator); + const auto allocate = reinterpret_cast(table[2]); + auto* allocation = static_cast(allocate(context.allocator, sizeof(AllocationContext), context.userData)); + if (!allocation) return JsonErrorOutOfMemory; + std::memcpy(&context.fileBufferSize, static_cast(parameter) + offsetof(AllocationContext, fileBufferSize), sizeof(context.fileBufferSize)); + *allocation = context; + state.allocation = allocation; + state.floatFormat = format; + state.floatFormatEnabled = modern; + try { + CreateDefault(TypeString); + CreateDefault(TypeArray); + CreateDefault(TypeObject); + state.fallback = static_cast(Allocate(sizeof(Value))); + if (state.fallback) Construct(*state.fallback); + state.internalMutex = CreateKernelMutex(); + const auto& defaults = SharedDefaults(); + const bool admitted = defaults.values[TypeString].string && defaults.values[TypeArray].array && + defaults.values[TypeObject].object && state.fallback && state.internalMutex; + if (admitted) state.recoveryMutex = ModernValues() ? CreateKernelMutex() : state.internalMutex; + if (!admitted || !state.recoveryMutex) { ReleaseInitialization(); return JsonErrorOutOfMemory; } + } catch (...) { ReleaseInitialization(); throw; } + state.initialized = true; + *static_cast(self) = 1; + return 0; +} + const Node& Typed(const Value* self, ValueType requested) { if (self->type == requested || ((requested == TypeInteger || requested == TypeUInteger) && @@ -1176,7 +1276,7 @@ int SetHandler(Node& n, HandlerRecord* Auxiliary::* member, TCallback { auto& state = State(); std::lock_guard lock(state.mutex); - if (!state.initialized) return JsonErrorNotInitialized; + if (!state.allocation) return JsonErrorNotInitialized; if (modernOnly && !state.floatFormatEnabled) return static_cast(0x80848113); } if (!ModernValues()) NotImplemented_nid_no_patch(__func__); @@ -1774,31 +1874,25 @@ void APS5_VABI _ZN3sce4Json11InitializerD1Ev(void* self) { int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* self, const void* parameter) { auto& state = State(); + std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); - if (state.initialized || *static_cast(self)) return JsonErrorAlreadyInitialized; + if (state.allocation || state.initialized || *static_cast(self)) return JsonErrorAlreadyInitialized; if (!parameter) return JsonErrorInvalidArgument; if (!*static_cast(parameter)) return JsonErrorOutOfMemory; ResetDefaults(); - std::memcpy(&state.allocation, parameter, sizeof(state.allocation)); - state.floatFormat = 2; - state.floatFormatEnabled = false; - state.initialized = true; - *static_cast(self) = 1; - return 0; + return InitializeResources(self, parameter, 2, false); } int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { auto& state = State(); + std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; + if (state.callbackActive) NotImplemented_nid_no_patch(__func__); state.callback = nullptr; state.context = nullptr; state.handlers = {}; - ResetDefaults(); - state.allocation = {}; - state.floatFormat = 2; - state.floatFormatEnabled = false; - state.initialized = false; + ReleaseInitialization(); *static_cast(self) = 0; return 0; } @@ -2290,7 +2384,7 @@ int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKc(Value* out, const char* auto& state = State(); std::lock_guard lock(state.mutex); if (!state.initialized) return JsonErrorNotInitialized; - allocation = state.allocation; + allocation = state.allocation ? *state.allocation : AllocationContext{}; } if (!allocation.fileBufferSize) return static_cast(0x80848105); FileReader reader{allocation}; @@ -2310,7 +2404,7 @@ void APS5_VABI _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(InitP *self = {}; auto& state = State(); std::lock_guard lock(state.mutex); - if (state.initialized) return; + if (state.allocation) return; const auto& callbacks = *static_cast(allocParam); if (!callbacks.allocate || !callbacks.free) return; state.rtti = callbacks; @@ -2346,7 +2440,7 @@ void APS5_VABI _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocPa if (!allocParam) return; auto& state = State(); std::lock_guard lock(state.mutex); - if (state.initialized) return; + if (state.allocation) return; const auto& callbacks = *static_cast(allocParam); if (!callbacks.allocate || !callbacks.free) return; state.rtti = callbacks; @@ -2364,17 +2458,13 @@ void APS5_VABI _ZN3sce4Json14InitParameter225setSpecialFloatFormatTypeENS0_22Spe int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void* self, const InitParameter2* parameter) { auto& state = State(); + std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); - if (state.initialized || *static_cast(self)) return JsonErrorAlreadyInitialized; + if (state.allocation || state.initialized || *static_cast(self)) return JsonErrorAlreadyInitialized; if (!parameter || !parameter->allocator || static_cast(parameter->specialFloatFormatType) > 2) return JsonErrorInvalidArgument; ResetDefaults(); - state.allocation = {parameter->allocator, parameter->userData, parameter->fileBufferSize}; - state.floatFormat = parameter->specialFloatFormatType; - state.floatFormatEnabled = true; - state.initialized = true; - *static_cast(self) = 1; - return 0; + return InitializeResources(self, parameter, parameter->specialFloatFormatType, true); } void APS5_VABI _ZN3sce4Json5Value5clearEv(Value* self) { @@ -2394,7 +2484,7 @@ int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS { auto& state = State(); std::lock_guard lock(state.mutex); - if (!state.initialized) return JsonErrorNotInitialized; + if (!state.allocation) return JsonErrorNotInitialized; } if (!ModernValues() && !self->auxiliary) return static_cast(0x80848104); const auto status = EnsureAuxiliary(*self); diff --git a/core/libs/tests/GuestJsonAllocator.cpp b/core/libs/tests/GuestJsonAllocator.cpp new file mode 100644 index 0000000000..57e17b89ea --- /dev/null +++ b/core/libs/tests/GuestJsonAllocator.cpp @@ -0,0 +1,221 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { alignas(8) std::uint64_t words[4]{}; }; +struct String { void* storage; }; +struct Array { void* storage; }; +struct Object { void* storage; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value*); +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); +const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +std::size_t APS5_VABI _ZNK3sce4Json6String4sizeEv(const String*); +std::size_t APS5_VABI _ZNK3sce4Json6String8capacityEv(const String*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object*); +void APS5_VABI _ZN3sce4Json6String7reserveEm(String*, std::size_t); +void APS5_VABI _ZN3sce4Json6String6resizeEm(String*, std::size_t); +void APS5_VABI _ZN3sce4Json6String5clearEv(String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct AllocationTrace { + int budget = -1; + bool failOnce = false; + Parameter* changeInput = nullptr; + void* replacementContext = nullptr; + std::map live; + std::vector requests; + std::vector notifications; +}; +static void* APS5_VABI AllocateTracked(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + trace.requests.push_back(bytes); + if (trace.changeInput && trace.requests.size() == 1) { + trace.changeInput->context = trace.replacementContext; + trace.changeInput->bufferSize = 32768; + } + if (trace.budget == 0) { + if (trace.failOnce) trace.budget = -1; + return nullptr; + } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes ? bytes : 1); + Require(memory != nullptr, "The tracking allocator must have host memory"); + std::memset(memory, 0xa5, bytes); + trace.live.emplace(memory, bytes); + return memory; +} +static void APS5_VABI FreeTracked(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + Require(trace.live.erase(memory) == 1, "Free must use the allocator and context that own the resource"); + std::free(memory); +} +static void APS5_VABI NotifyTracked(void*, int error, std::size_t bytes, void* context) { + Require(static_cast(error) == 0x80848102, "Allocation notification must carry the Json2 out-of-memory error"); + auto& trace = *static_cast(context); + Require(!trace.requests.empty() && trace.requests.back() == bytes, "Notification size must match the failed allocation request"); + trace.notifications.push_back(bytes); +} +static void* const Vtable[]{nullptr, nullptr, reinterpret_cast(AllocateTracked), reinterpret_cast(FreeTracked), reinterpret_cast(NotifyTracked)}; +struct TrackingAllocator { void* const* vtable = Vtable; }; +static bool Owned(const AllocationTrace& trace, const void* memory, std::size_t bytes) { + const auto found = trace.live.find(const_cast(memory)); + return found != trace.live.end() && found->second == bytes; +} +static void CheckResources(AllocationTrace& trace) { + Require(trace.live.size() == 12, "SDK 6 initialization must own its actual twelve allocated resources"); + const std::vector requests{24, 8, 24, 8, 32, 48, 8, 32, 104, 32, 8, 8}; + Require(trace.requests.size() >= requests.size() && std::equal(requests.begin(), requests.end(), trace.requests.end() - requests.size()), + "Initialization must allocate its actual context, defaults, fallback and synchronization handles in reference order"); + bool contextFound = false; + for (const auto& [memory, bytes] : trace.live) { + if (bytes != 24) continue; + std::uintptr_t words[3]; std::memcpy(words, memory, sizeof(words)); + if (words[1] == reinterpret_cast(&trace) && words[2] == 16384) contextFound = true; + } + Require(contextFound, "Initialization must retain the supplied context and buffer setting in its owned context record"); + Value null; + _ZN3sce4Json5ValueC1Ev(&null); + const auto* string = _ZNK3sce4Json5Value9getStringEv(&null); + const auto* array = _ZNK3sce4Json5Value8getArrayEv(&null); + const auto* object = _ZNK3sce4Json5Value9getObjectEv(&null); + Require(Owned(trace, string, 8) && Owned(trace, string->storage, 24), "The default string must use the configured allocator"); + Require(Owned(trace, array, 8) && Owned(trace, array->storage, 32) && Owned(trace, *static_cast(array->storage), 48), + "The default array must own its allocator-backed header and sentinel"); + Require(Owned(trace, object, 8) && Owned(trace, object->storage, 32) && Owned(trace, *static_cast(object->storage), 104), + "The default object must own its allocator-backed header and tree sentinel"); + Require(Owned(trace, _ZNK3sce4Json5ValueixEPKc(&null, "missing"), 32), "Missing access must return the allocated module fallback"); + _ZN3sce4Json5ValueD1Ev(&null); + String text{}; + _ZN3sce4Json6StringC1EPKc(&text, "abc"); + Require(Owned(trace, text.storage, 24) && Owned(trace, *static_cast(text.storage), 4) && + _ZNK3sce4Json6String4sizeEv(&text) == 3 && _ZNK3sce4Json6String8capacityEv(&text) == 4, + "String construction must allocate the native header and exact initial character capacity"); + _ZN3sce4Json6String7reserveEm(&text, 17); + Require(Owned(trace, *static_cast(text.storage), 17) && _ZNK3sce4Json6String8capacityEv(&text) == 17, + "Reserve must apply the SDK-dependent buffer growth rule"); + _ZN3sce4Json6String6resizeEm(&text, 18); + const char* data = _ZNK3sce4Json6String5c_strEv(&text); + Require(Owned(trace, data, 25) && _ZNK3sce4Json6String4sizeEv(&text) == 18 && std::memcmp(data, "abc", 3) == 0, + "Resize must retain the prefix and allocate the configured character capacity"); + for (unsigned index = 3; index <= 18; ++index) + Require(data[index] == 0, "Growing a string must zero the added bytes and terminator"); + _ZN3sce4Json6String5clearEv(&text); + Require(Owned(trace, text.storage, 24) && _ZNK3sce4Json6String4sizeEv(&text) == 0 && _ZNK3sce4Json6String8capacityEv(&text) == 0, + "Clear must release characters while retaining the string header"); + _ZN3sce4Json6StringD1Ev(&text); + Require(trace.live.size() == 12, "Temporary string operations must leave only initialization resources alive"); +} +static void CheckSingleFailures() { + for (int point = 0; point < 12; ++point) { + AllocationTrace trace; + trace.budget = point; + trace.failOnce = true; + TrackingAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + const int result = _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter); + const bool optional = point == 2 || point == 4 || point == 5 || point == 7 || point == 8; + Require(result == (optional ? 0 : static_cast(0x80848102)) && initializer == (optional ? 1 : 0), + "Single inner-default allocation failures must retain the observed admission result"); + Require(trace.notifications.size() == (point == 0 ? 0 : 1), "Each failed request must produce its observed notification count"); + if (optional) { + Value null; + _ZN3sce4Json5ValueC1Ev(&null); + if (point == 2) { + const auto* string = _ZNK3sce4Json5Value9getStringEv(&null); + Require(Owned(trace, string, 8) && !string->storage && _ZNK3sce4Json6String4sizeEv(string) == 0 && + _ZNK3sce4Json6String8capacityEv(string) == 0 && _ZNK3sce4Json6String5c_strEv(string)[0] == 0, + "A failed default string header must retain its usable empty outer handle"); + } else if (point == 4 || point == 5) { + const auto* array = _ZNK3sce4Json5Value8getArrayEv(&null); + Require(Owned(trace, array, 8) && _ZNK3sce4Json5Array4sizeEv(array) == 0, + "A failed default array resource must retain an empty outer handle"); + Require(point == 4 ? !array->storage : Owned(trace, array->storage, 32), + "Array admission must preserve the observed partial header ownership"); + } else { + const auto* object = _ZNK3sce4Json5Value9getObjectEv(&null); + Require(Owned(trace, object, 8) && !object->storage && _ZNK3sce4Json6Object4sizeEv(object) == 0, + "A failed default object resource must retain its empty outer handle and reclaim partial tree storage"); + } + _ZN3sce4Json5ValueD1Ev(&null); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate partial-default initialization"); + } + Require(trace.live.empty(), "Both admitted partial defaults and refused initialization must release all allocated resources"); + trace.budget = -1; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, + "A subsequent full initialization must work after every single failure schedule"); + CheckResources(trace); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Release resources after retry"); + } +} + +static void CheckContextSnapshot() { + AllocationTrace trace, replacement; + TrackingAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + trace.changeInput = ¶meter; + trace.replacementContext = &replacement; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize with a reconfigured input during context allocation"); + bool found = false; + for (const auto& [memory, bytes] : trace.live) { + if (bytes != 24) continue; + std::uintptr_t words[3]; std::memcpy(words, memory, sizeof(words)); + if (words[0] == reinterpret_cast(&allocator) && words[1] == reinterpret_cast(&trace) && words[2] == 32768) found = true; + } + Require(found && trace.live.size() == 12 && replacement.requests.empty(), "Context admission must cache allocator and user data before allocation and read the buffer setting afterward"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty() && replacement.live.empty(), "Termination must release the admitted context snapshot"); +} +int main() { + for (int budget = 0; budget <= 12; ++budget) { + AllocationTrace trace; + trace.budget = budget; + TrackingAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + const int result = _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter); + if (budget < 12) { + Require(static_cast(result) == 0x80848102 && initializer == 0 && trace.live.empty(), + "Every initialization allocation failure must report OOM, roll back ownership and leave retry legal"); + Require((budget == 0) == trace.notifications.empty(), "Only the first context-allocation failure omits notification"); + trace.budget = -1; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, + "Initialization must succeed after failure rollback without reconstructing the initializer"); + } else { + Require(result == 0 && initializer == 1, "Initialization must succeed after its final resource allocation"); + trace.budget = -1; + } + CheckResources(trace); + AllocationTrace unrelated; + parameter.context = &unrelated; + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty() && unrelated.live.empty(), + "Termination must release the snapshotted allocation context and every resource"); + } + CheckSingleFailures(); + CheckContextSnapshot(); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index b585ddfe78..de1e299790 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -82,8 +82,8 @@ - [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser -- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization still skips the allocator-backed context, default container resources and mutex handles. Allocator admission and rollback remain incomplete; file-buffer size is snapshotted and used by file parsing. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, but allocator failures can still differ. The module firmware version is unverified; these are library observations, not console measurements. -- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are still created lazily rather than through native initialization admission. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. +- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. +- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - `Pair(const String&, const Value&)` and default `Pair()` zero the 8 bytes between the key and value; what the console keeps there is unknown. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy registration remains incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. @@ -96,7 +96,7 @@ - [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy SDK behavior remains incomplete. Root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. - [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - destination state after parse failure and allocator failures outside the file buffer remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. -- [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, module-default identity, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. +- [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. - [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocation failures, legacy SDK ownership, concurrent allocation-operation accounting and default-object identity remain incomplete. Headers, red-black tree nodes, key storage and iterator states use the supplied allocator. Scalar insertion, duplicate-entry and copy failure contracts follow `0xda40`, `0xca40` and `0xe000`; partial trees can retain incomplete entries or a source count exceeding their actual nodes. Modern assignment follows `0xd400`: initial replacement allocation failure preserves the old tree, while later failures can replace it with partial entries; self-assignment rebuilds the tree. Legacy insertion copies the pair directly; SDK 5 metadata boundary tests match the inspected module, without establishing cross-firmware compatibility. Existing and missing key erasure allocates nothing, following `0xd350`. Object iterator dereference with no state throws if the temporary default iterator cannot be allocated; the reference can recurse in that case. Full Linux/Windows execution remains unverified. - [sce::Json::Parser embedded zero characters](../../../core/libs/prx/libSceJson2/Export.cpp) - decoded Unicode zero is discarded through the inspected character-append behavior. Binary string values serialize using their stored length, while object keys serialize their C-string prefix. Broader malformed Unicode handling remains incomplete. @@ -112,3 +112,5 @@ - [sce::Json::Value constructors and setters taking Array/Object](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocation failures and older SDK compatibility remain incomplete. These paths follow `0x1d00`, `0x1d50`, `0x1d90` and `0x1de0`: a missing source header is ignored, while admitted inputs change the destination type before collection assignment. Missing-header construction initializes null ownership metadata without changing inactive payload bytes. Self-setting retains the Value handle and callback metadata while replacing collection storage. The setters expose no consistent status result and are declared `void`; local tests verify side effects rather than a fabricated success return. - [sce::Json::Value nested array allocation regressions](../../../core/libs/tests/GuestJsonValueAllocator.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. + +- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - initialization follows `0x1340`, `0xda0` and `0x940`: a 24-byte allocation context owns the original allocator, user context and file-buffer setting; default handles/storage, a 32-byte fallback and kernel mutex handles are allocated before admission. Inner default failures can retain usable outer handles; required-resource failure reclaims ownership and leaves retry legal. SDK 6 uses two recursive kernel mutex handles, while the legacy path shares one. Local tests substitute the kernel boundary and cover all twelve SDK 6 allocation positions, partial admission and retry. Failed kernel mutex setup or locking throws rather than fabricating success; terminating from an active callback remains unsupported. These checks do not prove Linux/Windows integration or every reentrant/concurrent lifecycle. From 686c978e29a76812cae9abe3e5a9bcb21bf2a7ff Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 01:25:30 +0200 Subject: [PATCH 022/318] docs(libSceJson2): scope debt entries to the library --- docs/dev/debt/libSceJson2.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index de1e299790..e85c8c70f7 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -98,7 +98,7 @@ - [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - destination state after parse failure and allocator failures outside the file buffer remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. - [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. -- [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocation failures, legacy SDK ownership, concurrent allocation-operation accounting and default-object identity remain incomplete. Headers, red-black tree nodes, key storage and iterator states use the supplied allocator. Scalar insertion, duplicate-entry and copy failure contracts follow `0xda40`, `0xca40` and `0xe000`; partial trees can retain incomplete entries or a source count exceeding their actual nodes. Modern assignment follows `0xd400`: initial replacement allocation failure preserves the old tree, while later failures can replace it with partial entries; self-assignment rebuilds the tree. Legacy insertion copies the pair directly; SDK 5 metadata boundary tests match the inspected module, without establishing cross-firmware compatibility. Existing and missing key erasure allocates nothing, following `0xd350`. Object iterator dereference with no state throws if the temporary default iterator cannot be allocated; the reference can recurse in that case. Full Linux/Windows execution remains unverified. +- [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocation failures, legacy ownership and concurrent operation accounting remain incomplete. Allocator-backed tree nodes can retain incomplete entries or a source count exceeding their live nodes after copy failure (`0xca40`, `0xe000`). Modern assignment (`0xd400`) preserves the original on initial allocation failure but can replace it with partial entries after later failures; self-assignment rebuilds the tree. Legacy insertion copies the pair directly. SDK 5 boundary tests do not establish cross-firmware compatibility. Erasure allocates nothing (`0xd350`). Missing-state iterator dereference throws if its temporary default iterator cannot be allocated; the reference can recurse. Full Linux/Windows execution remains unverified. - [sce::Json::Parser embedded zero characters](../../../core/libs/prx/libSceJson2/Export.cpp) - decoded Unicode zero is discarded through the inspected character-append behavior. Binary string values serialize using their stored length, while object keys serialize their C-string prefix. Broader malformed Unicode handling remains incomplete. - [sce::Json::Value handle admission and type changes](../../../core/libs/prx/libSceJson2/Export.cpp) - collection copy/assignment failure ordering, nested recovery failures and older SDK compatibility remain incomplete. The supplied allocator owns each eight-byte complex-value handle. Typed construction and type changes follow `0x580`, `0x10d0`, `0x1170` and `0x1210`: the old payload is destroyed before replacement admission; string/header failure leaves null, an array can retain a header without its sentinel, and object construction reclaims a failed sentinel. Scalar changes preserve inactive payload bytes. Special-float string conversion follows the temporary string at `0x40a0`; failure of its header preserves the prior value, while later failures can leave null or an empty string. Local tests cover SDK 6 process metadata, not every firmware version. @@ -111,6 +111,6 @@ - [sce::Json::Value constructors and setters taking Array/Object](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocation failures and older SDK compatibility remain incomplete. These paths follow `0x1d00`, `0x1d50`, `0x1d90` and `0x1de0`: a missing source header is ignored, while admitted inputs change the destination type before collection assignment. Missing-header construction initializes null ownership metadata without changing inactive payload bytes. Self-setting retains the Value handle and callback metadata while replacing collection storage. The setters expose no consistent status result and are declared `void`; local tests verify side effects rather than a fabricated success return. -- [sce::Json::Value nested array allocation regressions](../../../core/libs/tests/GuestJsonValueAllocator.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. +- [sce::Json::Value nested array allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. - [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - initialization follows `0x1340`, `0xda0` and `0x940`: a 24-byte allocation context owns the original allocator, user context and file-buffer setting; default handles/storage, a 32-byte fallback and kernel mutex handles are allocated before admission. Inner default failures can retain usable outer handles; required-resource failure reclaims ownership and leaves retry legal. SDK 6 uses two recursive kernel mutex handles, while the legacy path shares one. Local tests substitute the kernel boundary and cover all twelve SDK 6 allocation positions, partial admission and retry. Failed kernel mutex setup or locking throws rather than fabricating success; terminating from an active callback remains unsupported. These checks do not prove Linux/Windows integration or every reentrant/concurrent lifecycle. From 22a712040bcf8638dbcb4ede13f4b88e0dcf68aa Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 05:39:07 +0200 Subject: [PATCH 023/318] fix(libSceJson2): preserve legacy mismatch callback ownership --- core/libs/prx/libSceJson2/CMakeLists.txt | 12 + core/libs/prx/libSceJson2/Export.cpp | 431 ++++++++++++++------ core/libs/tests/GuestJsonLegacyHandlers.cpp | 195 +++++++++ docs/dev/debt/libSceJson2.md | 12 +- 4 files changed, 520 insertions(+), 130 deletions(-) create mode 100644 core/libs/tests/GuestJsonLegacyHandlers.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 5f6d721f77..27b581f352 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -133,3 +133,15 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + add_test_executable(guest_json_legacy_handlers_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonLegacyHandlers.cpp Export.cpp) + target_include_directories(guest_json_legacy_handlers_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_legacy_handlers_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_legacy_handlers_tests) + add_test(NAME guest_json_legacy_handlers COMMAND guest_json_legacy_handlers_tests) + if(WIN32) + set_tests_properties(guest_json_legacy_handlers PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 089814fe22..31e7c11850 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ constexpr int JsonErrorInvalidArgument = static_cast(0x80848120); struct Value; struct Handlers; struct Auxiliary; +struct LegacyHandlerMap; struct Text { char* characters; std::uint32_t length; std::uint32_t capacity; std::uint64_t reserved; }; static_assert(sizeof(Text) == 24); struct String { Text* text; }; @@ -105,7 +107,6 @@ static_assert(sizeof(Value) == 32 && sizeof(String) == 8 && sizeof(Array) == 8 & static_assert(sizeof(ArrayIterator) == 8 && sizeof(ObjectIterator) == 8); static_assert(offsetof(Pair, value) == 0x10); - using NullAccessCallback = const Value& (APS5_VABI*)(ValueType, const Value*, void*); template struct HandlerRecord { TCallback callback = nullptr; void* context = nullptr; }; @@ -151,6 +152,7 @@ struct Globals { Handlers handlers{}; AllocationContext* allocation = nullptr; Value* fallback = nullptr; + LegacyHandlerMap* legacyHandlers = nullptr; PthreadMutex* internalMutex = nullptr; PthreadMutex* recoveryMutex = nullptr; std::recursive_mutex callbackMutex; @@ -514,6 +516,136 @@ struct InlineText { void Assign(std::string_view bytes) { String string{&text}; AssignText(string, bytes, true); } }; +template +void RefreshTreeEndpoints(Node* head) { + auto* first = head->parent; + auto* last = first; + if (!first->nil) { + while (!first->left->nil) first = first->left; + while (!last->right->nil) last = last->right; + } + head->left = first; + head->right = last; +} + +template +void RotateTreeLeft(Node* head, Node* node) { + auto* pivot = node->right; + node->right = pivot->left; + if (!pivot->left->nil) pivot->left->parent = node; + pivot->parent = node->parent; + if (node->parent->nil) head->parent = pivot; + else if (node == node->parent->left) node->parent->left = pivot; + else node->parent->right = pivot; + pivot->left = node; + node->parent = pivot; +} + +template +void RotateTreeRight(Node* head, Node* node) { + auto* pivot = node->left; + node->left = pivot->right; + if (!pivot->right->nil) pivot->right->parent = node; + pivot->parent = node->parent; + if (node->parent->nil) head->parent = pivot; + else if (node == node->parent->right) node->parent->right = pivot; + else node->parent->left = pivot; + pivot->right = node; + node->parent = pivot; +} + +template +void BalanceTreeInsertion(Node* head, Node* node) { + while (node->parent->color == 1) { + auto* parent = node->parent; + auto* grandparent = parent->parent; + if (parent == grandparent->left) { + auto* uncle = grandparent->right; + if (uncle->color == 1) { parent->color = 0; uncle->color = 0; grandparent->color = 1; node = grandparent; } + else { + if (node == parent->right) { node = parent; RotateTreeLeft(head, node); parent = node->parent; grandparent = parent->parent; } + parent->color = 0; grandparent->color = 1; RotateTreeRight(head, grandparent); + } + } else { + auto* uncle = grandparent->left; + if (uncle->color == 1) { parent->color = 0; uncle->color = 0; grandparent->color = 1; node = grandparent; } + else { + if (node == parent->left) { node = parent; RotateTreeRight(head, node); parent = node->parent; grandparent = parent->parent; } + parent->color = 0; grandparent->color = 1; RotateTreeLeft(head, grandparent); + } + } + } + head->parent->color = 0; +} + +template +void ReplaceTreeNode(Node* head, Node* node, Node* replacement) { + if (node->parent->nil) head->parent = replacement; + else if (node == node->parent->left) node->parent->left = replacement; + else node->parent->right = replacement; + if (!replacement->nil) replacement->parent = node->parent; +} + +template +void BalanceTreeErasure(Node* head, Node* node, Node* parent) { + while (node != head->parent && node->color == 0) { + if (node == parent->left) { + auto* sibling = parent->right; + if (sibling->color == 1) { sibling->color = 0; parent->color = 1; RotateTreeLeft(head, parent); sibling = parent->right; } + if (sibling->left->color == 0 && sibling->right->color == 0) { + if (!sibling->nil) sibling->color = 1; + node = parent; parent = node->parent; + } else { + if (sibling->right->color == 0) { sibling->left->color = 0; sibling->color = 1; RotateTreeRight(head, sibling); sibling = parent->right; } + sibling->color = parent->color; parent->color = 0; sibling->right->color = 0; + RotateTreeLeft(head, parent); node = head->parent; + } + } else { + auto* sibling = parent->left; + if (sibling->color == 1) { sibling->color = 0; parent->color = 1; RotateTreeRight(head, parent); sibling = parent->left; } + if (sibling->right->color == 0 && sibling->left->color == 0) { + if (!sibling->nil) sibling->color = 1; + node = parent; parent = node->parent; + } else { + if (sibling->left->color == 0) { sibling->right->color = 0; sibling->color = 1; RotateTreeLeft(head, sibling); sibling = parent->left; } + sibling->color = parent->color; parent->color = 0; sibling->left->color = 0; + RotateTreeRight(head, parent); node = head->parent; + } + } + } + node->color = 0; +} + +template +void UnlinkTreeNode(Node* head, Node* removed) { + if (!removed || removed->nil) return; + auto* successor = removed; + auto color = successor->color; + Node* child; + Node* parent; + if (removed->left->nil) { child = removed->right; parent = removed->parent; ReplaceTreeNode(head, removed, child); } + else if (removed->right->nil) { child = removed->left; parent = removed->parent; ReplaceTreeNode(head, removed, child); } + else { + successor = removed->right; + while (!successor->left->nil) successor = successor->left; + color = successor->color; + child = successor->right; + if (successor->parent == removed) parent = successor; + else { + parent = successor->parent; + ReplaceTreeNode(head, successor, child); + successor->right = removed->right; + successor->right->parent = successor; + } + ReplaceTreeNode(head, removed, successor); + successor->left = removed->left; + successor->left->parent = successor; + successor->color = removed->color; + } + if (color == 0) BalanceTreeErasure(head, child, parent); + RefreshTreeEndpoints(head); +} + struct ObjectLink { ObjectLink* parent; ObjectLink* left; @@ -611,60 +743,9 @@ struct ObjectStorage { } return head; } - void RefreshEndpoints() { - auto* first = head->parent; - auto* last = first; - if (!first->nil) { - while (!first->left->nil) first = first->left; - while (!last->right->nil) last = last->right; - } - head->left = first; - head->right = last; - } - void RotateLeft(ObjectLink* node) { - auto* pivot = node->right; - node->right = pivot->left; - if (!pivot->left->nil) pivot->left->parent = node; - pivot->parent = node->parent; - if (node->parent->nil) head->parent = pivot; - else if (node == node->parent->left) node->parent->left = pivot; - else node->parent->right = pivot; - pivot->left = node; - node->parent = pivot; - } - void RotateRight(ObjectLink* node) { - auto* pivot = node->left; - node->left = pivot->right; - if (!pivot->right->nil) pivot->right->parent = node; - pivot->parent = node->parent; - if (node->parent->nil) head->parent = pivot; - else if (node == node->parent->right) node->parent->right = pivot; - else node->parent->left = pivot; - pivot->right = node; - node->parent = pivot; - } - void BalanceInsertion(ObjectLink* node) { - while (node->parent->color == 1) { - auto* parent = node->parent; - auto* grandparent = parent->parent; - if (parent == grandparent->left) { - auto* uncle = grandparent->right; - if (uncle->color == 1) { parent->color = 0; uncle->color = 0; grandparent->color = 1; node = grandparent; } - else { - if (node == parent->right) { node = parent; RotateLeft(node); parent = node->parent; grandparent = parent->parent; } - parent->color = 0; grandparent->color = 1; RotateRight(grandparent); - } - } else { - auto* uncle = grandparent->left; - if (uncle->color == 1) { parent->color = 0; uncle->color = 0; grandparent->color = 1; node = grandparent; } - else { - if (node == parent->left) { node = parent; RotateRight(node); parent = node->parent; grandparent = parent->parent; } - parent->color = 0; grandparent->color = 1; RotateLeft(grandparent); - } - } - } - head->parent->color = 0; - } + void RefreshEndpoints() { RefreshTreeEndpoints(head); } + + void BalanceInsertion(ObjectLink* node) { BalanceTreeInsertion(head, node); } ObjectLink* Insert(const Text& key, const Pair& pair, bool& inserted) { inserted = false; if (!head) return nullptr; @@ -699,67 +780,10 @@ struct ObjectStorage { inserted = true; return node; } - void Replace(ObjectLink* node, ObjectLink* replacement) { - if (node->parent->nil) head->parent = replacement; - else if (node == node->parent->left) node->parent->left = replacement; - else node->parent->right = replacement; - if (!replacement->nil) replacement->parent = node->parent; - } - void BalanceErasure(ObjectLink* node, ObjectLink* parent) { - while (node != head->parent && node->color == 0) { - if (node == parent->left) { - auto* sibling = parent->right; - if (sibling->color == 1) { sibling->color = 0; parent->color = 1; RotateLeft(parent); sibling = parent->right; } - if (sibling->left->color == 0 && sibling->right->color == 0) { - if (!sibling->nil) sibling->color = 1; - node = parent; parent = node->parent; - } else { - if (sibling->right->color == 0) { sibling->left->color = 0; sibling->color = 1; RotateRight(sibling); sibling = parent->right; } - sibling->color = parent->color; parent->color = 0; sibling->right->color = 0; - RotateLeft(parent); node = head->parent; - } - } else { - auto* sibling = parent->left; - if (sibling->color == 1) { sibling->color = 0; parent->color = 1; RotateRight(parent); sibling = parent->left; } - if (sibling->right->color == 0 && sibling->left->color == 0) { - if (!sibling->nil) sibling->color = 1; - node = parent; parent = node->parent; - } else { - if (sibling->left->color == 0) { sibling->right->color = 0; sibling->color = 1; RotateLeft(sibling); sibling = parent->left; } - sibling->color = parent->color; parent->color = 0; sibling->left->color = 0; - RotateRight(parent); node = head->parent; - } - } - } - node->color = 0; - } + void Erase(ObjectLink* removed) { if (!removed || removed->nil) return; - auto* successor = removed; - auto color = successor->color; - ObjectLink* child; - ObjectLink* parent; - if (removed->left->nil) { child = removed->right; parent = removed->parent; Replace(removed, child); } - else if (removed->right->nil) { child = removed->left; parent = removed->parent; Replace(removed, child); } - else { - successor = removed->right; - while (!successor->left->nil) successor = successor->left; - color = successor->color; - child = successor->right; - if (successor->parent == removed) parent = successor; - else { - parent = successor->parent; - Replace(successor, child); - successor->right = removed->right; - successor->right->parent = successor; - } - Replace(removed, successor); - successor->left = removed->left; - successor->left->parent = successor; - successor->color = removed->color; - } - if (color == 0) BalanceErasure(child, parent); - RefreshEndpoints(); + UnlinkTreeNode(head, removed); DestroyObjectLink(removed); --count; } @@ -772,6 +796,134 @@ struct ObjectStorage { }; static_assert(sizeof(ObjectStorage) == 24 && sizeof(ObjectIteratorState) == 8); +extern "C" void* APS5_VABI _Znwm_nid_postfix(std::size_t bytes); +extern "C" void APS5_VABI _ZdlPv_nid_postfix(void* memory); + +struct LegacyHandlerNode { + LegacyHandlerNode* parent; + LegacyHandlerNode* left; + LegacyHandlerNode* right; + const Value* key; + HandlerRecord record; + std::uint8_t color; + std::uint8_t nil; + std::uint8_t reserved[6]; +}; +static_assert(sizeof(LegacyHandlerNode) == 56 && offsetof(LegacyHandlerNode, record) == 32 && offsetof(LegacyHandlerNode, nil) == 49); + +struct LegacyHandlerMap { + LegacyHandlerNode* head; + std::size_t count; + std::uint64_t reserved; + LegacyHandlerNode* Find(const Value* key) const { + auto* node = head->parent; + while (!node->nil) { + const auto requested = reinterpret_cast(key); + const auto stored = reinterpret_cast(node->key); + if (requested == stored) return node; + node = requested < stored ? node->left : node->right; + } + return head; + } + void RefreshEndpoints() { RefreshTreeEndpoints(head); } + + void BalanceInsertion(LegacyHandlerNode* node) { BalanceTreeInsertion(head, node); } + LegacyHandlerNode* Insert(const Value* key) { + auto* parent = head; + auto* position = head->parent; + bool left = false; + while (!position->nil) { + parent = position; + const auto requested = reinterpret_cast(key); + const auto stored = reinterpret_cast(position->key); + if (requested == stored) return position; + left = requested < stored; + position = left ? position->left : position->right; + } + auto* node = static_cast(_Znwm_nid_postfix(sizeof(LegacyHandlerNode))); + node->parent = parent; node->left = head; node->right = head; + node->key = key; node->record = {}; + node->color = 1; node->nil = 0; + if (parent->nil) head->parent = node; + else if (left) parent->left = node; + else parent->right = node; + ++count; + BalanceInsertion(node); + RefreshEndpoints(); + return node; + } + + void Erase(LegacyHandlerNode* removed) { + if (!removed || removed->nil) return; + UnlinkTreeNode(head, removed); + _ZdlPv_nid_postfix(removed); + --count; + } + void ReleaseTree(LegacyHandlerNode* node) { + if (node->nil) return; + ReleaseTree(node->left); ReleaseTree(node->right); + _ZdlPv_nid_postfix(node); + } +}; +static_assert(sizeof(LegacyHandlerMap) == 24); + +HandlerRecord LegacyHandlerOf(const Value& value) { + auto& state = State(); + KernelMutexLock lock(state.recoveryMutex); + if (!state.legacyHandlers) return {}; + const auto* node = state.legacyHandlers->Find(&value); + return node->nil ? HandlerRecord{} : node->record; +} + +int RegisterLegacyHandler(Value& value, TypeMismatchHandler callback, void* context) { + auto& state = State(); + KernelMutexLock lock(state.recoveryMutex); + if (!state.legacyHandlers) { + auto* map = static_cast(Allocate(sizeof(LegacyHandlerMap))); + if (!map) return JsonErrorOutOfMemory; + map->count = 0; + try { map->head = static_cast(_Znwm_nid_postfix(sizeof(LegacyHandlerNode))); } + catch (const std::bad_alloc&) { Release(map); NotImplemented_nid_no_patch(__func__); } + map->head->parent = map->head; map->head->left = map->head; map->head->right = map->head; + map->head->color = 0; map->head->nil = 1; + state.legacyHandlers = map; + } + const auto failures = AllocationFailures(); + auto* node = state.legacyHandlers->Insert(&value); + if (AllocationFailures() != failures) return JsonErrorOutOfMemory; + node->record = {callback, context}; + return 0; +} + +void EraseLegacyHandler(const Value& value) { + auto& state = State(); + KernelMutexLock lock(state.recoveryMutex); + if (state.legacyHandlers) state.legacyHandlers->Erase(state.legacyHandlers->Find(&value)); +} + +void CopyLegacyHandler(Value& destination, const Value& source) { + auto& state = State(); + KernelMutexLock lock(state.recoveryMutex); + if (!state.legacyHandlers) return; + const auto* sourceNode = state.legacyHandlers->Find(&source); + if (!sourceNode->nil) { + const auto record = sourceNode->record; + RegisterLegacyHandler(destination, record.callback, record.context); + } +} + +void ReleaseLegacyHandlers() { + auto& state = State(); + auto* map = state.legacyHandlers; + if (!map) return; + map->ReleaseTree(map->head->parent); + _ZdlPv_nid_postfix(map->head); + map->head = reinterpret_cast(std::uintptr_t{0xffffffff}); + map->count = std::numeric_limits::max(); + Release(map); + state.legacyHandlers = nullptr; +} + void Construct(Object& object) { object.items = static_cast(Allocate(ModernValues() ? 32 : 24)); if (!object.items) return; @@ -864,7 +1016,13 @@ Handlers HandlersOf(const Node& n) { std::lock_guard lock(state.mutex); if (state.initialized) handlers = state.handlers; } - if (!auxiliary || !ModernValues()) return handlers; + if (!ModernValues()) { + const auto record = LegacyHandlerOf(RootOf(n)); + handlers.typeMismatch = record.callback; + handlers.typeMismatchContext = record.context; + return handlers; + } + if (!auxiliary) return handlers; if (auxiliary->typeMismatch && auxiliary->typeMismatch->callback) { handlers.typeMismatch = auxiliary->typeMismatch->callback; handlers.typeMismatchContext = auxiliary->typeMismatch->context; @@ -926,6 +1084,7 @@ void Destroy(Value& v) { Release(v.auxiliary); v.auxiliary = nullptr; } + if (!ModernValues()) EraseLegacyHandler(v); } int EnsureAuxiliary(Node& n) { @@ -960,7 +1119,6 @@ void TransferPayload(Node& destination, Node& source) { BindChildren(destination); } - bool HasStorage(const String& value) { return value.text != nullptr; } bool HasStorage(const Array& value) { return value.items != nullptr; } bool HasStorage(const Object& value) { return value.items != nullptr; } @@ -1019,14 +1177,13 @@ void CopyObject(Object& destination, const Object& source) { destination = pending; } - void AssignReal(Node& n, double value); void SetType(Node& n, ValueType type); void AssignArray(Array& destination, const Array& source); void AssignObject(Object& destination, const Object& source); -void AssignNode(Node& destination, const Node& source) { +void AssignPayload(Node& destination, const Node& source) { if (source.type == TypeNull) { SetType(destination, TypeNull); return; } if (source.type == TypeBoolean) { const auto value = source.boolean; @@ -1068,6 +1225,11 @@ void AssignNode(Node& destination, const Node& source) { NotImplemented_nid_no_patch(__func__); } +void AssignNode(Node& destination, const Node& source) { + AssignPayload(destination, source); + if (!ModernValues()) CopyLegacyHandler(destination, source); +} + void RecoverSpecial(const Node& source, const Handlers& handlers, int function, double number, Value& result, bool* error) { Value returned{}; try { @@ -1152,6 +1314,7 @@ PthreadMutex* CreateKernelMutex() { void ReleaseInitialization() { auto& state = State(); + ReleaseLegacyHandlers(); if (state.fallback) { Destroy(*state.fallback); Release(state.fallback); state.fallback = nullptr; } if (state.recoveryMutex && state.recoveryMutex != state.internalMutex) { scePthreadMutexDestroy(state.recoveryMutex); @@ -1222,6 +1385,11 @@ const Node& Typed(const Value* self, ValueType requested) { if (self->type == requested || ((requested == TypeInteger || requested == TypeUInteger) && (self->type == TypeInteger || self->type == TypeUInteger))) return *self; + if (!ModernValues() && self->type == TypeNull) { + const auto* auxiliary = RootOf(*self).auxiliary; + if (auxiliary && auxiliary->nullAccess) + return Typed(&auxiliary->nullAccess(requested, self->parent, auxiliary->nullAccessContext), requested); + } CallbackScope scope; if (!scope.entered) return Defaults(requested); NullAccessCallback callback = nullptr; @@ -1235,6 +1403,11 @@ const Node& Typed(const Value* self, ValueType requested) { context = state.context; } if (!initialized) return Defaults(requested); + if (!ModernValues()) { + const auto record = LegacyHandlerOf(RootOf(*self)); + if (record.callback) return Typed(&record.callback(requested, self, self->parent, record.context), requested); + return Defaults(requested); + } if (self->type == TypeNull) { const auto* auxiliary = RootOf(*self).auxiliary; if (auxiliary && auxiliary->nullAccess) @@ -1273,13 +1446,18 @@ const Value& MissingElement(const Value* self, int error, const String* key, con template int SetHandler(Node& n, HandlerRecord* Auxiliary::* member, TCallback callback, void* context, bool modernOnly = false) { if (n.parent) return static_cast(0x80848104); + const bool legacyMismatch = !ModernValues() && std::is_same_v; + if (legacyMismatch && !n.auxiliary) return static_cast(0x80848104); { auto& state = State(); std::lock_guard lock(state.mutex); if (!state.allocation) return JsonErrorNotInitialized; if (modernOnly && !state.floatFormatEnabled) return static_cast(0x80848113); } - if (!ModernValues()) NotImplemented_nid_no_patch(__func__); + if (!ModernValues()) { + if constexpr (std::is_same_v) return RegisterLegacyHandler(n, callback, context); + else NotImplemented_nid_no_patch(__func__); + } const auto status = EnsureAuxiliary(n); if (status) return status; auto& record = n.auxiliary->*member; @@ -1857,7 +2035,6 @@ void* const JsonMemAllocatorVtable[] = { reinterpret_cast(&JsonMemAllocatorPureVirtual), reinterpret_cast(&JsonMemAllocatorPureVirtual), }; - } extern "C" { @@ -2005,6 +2182,12 @@ Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array* self, const Array* other) { A void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array* self, const Value* value) { if (!self->items || !self->items->head) return; const auto failures = AllocationFailures(); + if (!ModernValues()) { + try { CopyInto(self->items->emplace_back(), *value); } + catch (const GuestOutOfMemory&) {} + if (AllocationFailures() != failures) self->items->PopBack(); + return; + } Value copy{}; copy.parent = self->items->Owner(); try { @@ -2202,10 +2385,10 @@ void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value* self, Value* other) { AssignNode(NodeOf(*self), NodeOf(*other)); AssignNode(NodeOf(*other), temporary); } catch (...) { - Clear(temporary); + Destroy(temporary); throw; } - Clear(temporary); + Destroy(temporary); } int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value* self, ValueType type) { SetType(NodeOf(*self), type); return 0; } @@ -2363,7 +2546,6 @@ void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value* self, Str } } - int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(Value* out, ParserReadCallback read, void* context) { if (read == nullptr) throw std::invalid_argument("sce::Json::Parser::parse: null read callback"); Parser parser(read, context); @@ -2377,7 +2559,6 @@ int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value* out, const char* return ParseValue(out, parser); } - int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKc(Value* out, const char* path) { AllocationContext allocation; { @@ -2484,7 +2665,7 @@ int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS { auto& state = State(); std::lock_guard lock(state.mutex); - if (!state.allocation) return JsonErrorNotInitialized; + if (ModernValues() && !state.allocation) return JsonErrorNotInitialized; } if (!ModernValues() && !self->auxiliary) return static_cast(0x80848104); const auto status = EnsureAuxiliary(*self); diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp new file mode 100644 index 0000000000..567063856d --- /dev/null +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -0,0 +1,195 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; }; +using Handler = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +using NullHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value*, bool); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); +Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value*, Value*); +const bool* APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value*); +bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value*); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, Handler, void*); +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullHandler, void*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct LegacyTrace { + std::map guest; + std::map heap; + int budget = -1; + unsigned notifications = 0; +}; +static LegacyTrace* HeapTrace = nullptr; +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + alignas(8) static const std::uint32_t parameters[]{0, 0, 0, 0, 0x05000000, 0}; + return parameters; +} +extern "C" void* APS5_VABI _Znwm_nid_postfix(std::size_t bytes) { + void* memory = std::malloc(bytes); + if (!memory) throw std::bad_alloc(); + std::memset(memory, 0xa5, bytes); + Require(HeapTrace && bytes == 56, "Legacy map nodes must use the actual libc new allocation boundary"); + HeapTrace->heap.emplace(memory, bytes); + return memory; +} +extern "C" void APS5_VABI _ZdlPv_nid_postfix(void* memory) { + Require(HeapTrace && HeapTrace->heap.erase(memory) == 1, "Legacy map nodes must return through libc delete"); + std::free(memory); +} +static void* APS5_VABI AllocateLegacy(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + if (trace.budget == 0) return nullptr; + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.guest.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeLegacy(void*, void* memory, void* context) { + if (!memory) return; + Require(static_cast(context)->guest.erase(memory) == 1, "Legacy metadata must return through its owning allocator"); + std::free(memory); +} +static void APS5_VABI NotifyLegacy(void*, int error, std::size_t bytes, void* context) { + Require(static_cast(error) == 0x80848102 && (bytes == 24 || bytes == 48), "Legacy allocation failure must identify its map header or array node"); + ++static_cast(context)->notifications; +} +static void* const LegacyVtable[]{nullptr, nullptr, reinterpret_cast(AllocateLegacy), reinterpret_cast(FreeLegacy), reinterpret_cast(NotifyLegacy)}; +struct LegacyAllocator { void* const* vtable = LegacyVtable; }; +struct LegacyNode { LegacyNode* parent; LegacyNode* left; LegacyNode* right; const Value* key; Handler callback; void* context; std::uint8_t color; std::uint8_t nil; std::uint8_t reserved[6]; }; +struct LegacyHeader { LegacyNode* head; std::size_t count; std::uint64_t reserved; }; +static_assert(sizeof(LegacyNode) == 56 && sizeof(LegacyHeader) == 24); +struct Context { Value replacement{}; unsigned calls = 0; const Value* source = nullptr; const Value* parent = nullptr; }; +static const Value& APS5_VABI RecoverLegacy(std::int32_t type, const Value* source, const Value* parent, void* context) { + auto& result = *static_cast(context); + Require(type == 1, "Legacy mismatch dispatch must identify the requested type"); + ++result.calls; result.source = source; result.parent = parent; return result.replacement; +} +static void ParseLegacy(Value& value, const char* text = "9") { + _ZN3sce4Json5ValueC1Ev(&value); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, text, std::strlen(text)) == 0, "Prepare legacy root auxiliary metadata"); +} +static void CheckMap(const LegacyTrace& trace, const std::map& entries) { + const LegacyHeader* header = nullptr; + for (const auto& [memory, size] : trace.guest) { + if (size != 24) continue; + void* head = nullptr; std::memcpy(&head, memory, sizeof(head)); + if (trace.heap.contains(head)) { Require(!header, "Legacy registration must use one shared map header"); header = static_cast(memory); } + } + Require(header && header->count == entries.size() && header->reserved == UINT64_C(0xa5a5a5a5a5a5a5a5), "Legacy map header must preserve count and reserved bytes"); + auto* head = header->head; + Require(head->nil == 1 && head->color == 0 && head->parent->color == 0, "Legacy sentinel and root must be black"); + auto expected = entries.begin(); + std::function visit = [&](const LegacyNode* node, const LegacyNode* parent) -> unsigned { + if (node == head) return 1; + Require(trace.heap.contains(const_cast(node)) && node->parent == parent && !node->nil && node->color <= 1, "Legacy nodes must retain their libc ownership and parent link"); + Require(!node->color || (!node->left->color && !node->right->color), "Legacy red nodes must have black children"); + const auto left = visit(node->left, node); + Require(expected != entries.end() && node->key == expected->first && node->callback == RecoverLegacy && node->context == expected->second, "Legacy map entries must preserve sorted keys and callback identity"); + ++expected; + const auto right = visit(node->right, node); + Require(left == right, "Legacy tree paths must preserve black height"); + return left + !node->color; + }; + visit(head->parent, head); + Require(expected == entries.end() && trace.heap.size() == entries.size() + 1, "Legacy map must retain only its actual sentinel and entry nodes"); +} +struct NullContext { const Value* other; Context* replacement; }; +static const Value& APS5_VABI RecoverLegacyNull(std::int32_t requested, const Value* parent, void* context) { + auto& value = *static_cast(context); + Require(requested == 1 && !parent && *_ZNK3sce4Json5Value10getBooleanEv(value.other), "Legacy null recovery must allow independent mismatch recovery before the reentry guard"); + return value.replacement->replacement; +} +int main() { + LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; + Parameter parameter{&allocator, &trace, 16384}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&initializer, ¶meter) == 0 && trace.guest.size() == 11, "Initialize legacy allocation and shared mutex resources"); + const auto baseline = trace.guest.size(); + Context first, second; _ZN3sce4Json5ValueC1Eb(&first.replacement, true); _ZN3sce4Json5ValueC1Eb(&second.replacement, false); + Value source{}; _ZN3sce4Json5ValueC1El(&source, 9); + Require(static_cast(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &first)) == 0x80848104 && trace.guest.size() == baseline && trace.heap.empty(), "Legacy mismatch registration must reject a root without auxiliary metadata"); + _ZN3sce4Json5ValueD1Ev(&source); ParseLegacy(source); + const auto before = trace.guest.size(); trace.budget = 0; + Require(static_cast(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &first)) == 0x80848102 && trace.notifications == 1 && trace.guest.size() == before && trace.heap.empty(), "Failed map admission must retain the root and permit retry"); + trace.budget = -1; + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &first) == 0, "Register legacy mismatch after failure"); + CheckMap(trace, {{&source, &first}}); + Require(*_ZNK3sce4Json5Value10getBooleanEv(&source) && first.source == &source && !first.parent, "Legacy getters must dispatch the registered borrowed replacement"); + const auto guest = trace.guest.size(); const auto heap = trace.heap.size(); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &second) == 0 && !_ZNK3sce4Json5ValuecvbEv(&source) && second.calls == 1 && trace.guest.size() == guest && trace.heap.size() == heap, "Legacy callback replacement must reuse the map entry and govern boolean conversion"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, nullptr, nullptr) == 0 && !*_ZNK3sce4Json5Value10getBooleanEv(&source), "A null legacy mismatch callback must disable dispatch without removing the entry"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &first) == 0, "Restore legacy callback"); + Value copy{}; _ZN3sce4Json5ValueC1ERKS1_(©, &source); + Require(!copy.auxiliary && *_ZNK3sce4Json5Value10getBooleanEv(©) && first.source == ©, "Legacy value copies must propagate map callbacks independently of auxiliary ownership"); + Value assigned{}; ParseLegacy(assigned); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&assigned, RecoverLegacy, &second) == 0, "Prepare legacy assignment callback"); + auto* auxiliary = assigned.auxiliary; + _ZN3sce4Json5ValueaSERKS1_(&assigned, &source); + Require(assigned.auxiliary == auxiliary && *_ZNK3sce4Json5Value10getBooleanEv(&assigned) && first.source == &assigned, "Legacy assignment must copy the source map record while retaining destination metadata"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&assigned, RecoverLegacy, &second) == 0, "Prepare distinct swap callbacks"); + _ZN3sce4Json5Value4swapERS1_(&source, &assigned); + Require(!*_ZNK3sce4Json5Value10getBooleanEv(&source) && *_ZNK3sce4Json5Value10getBooleanEv(&assigned) && assigned.auxiliary == auxiliary, "Legacy swap must exchange callback records and retain destination auxiliary ownership"); + CheckMap(trace, {{&source, &second}, {©, &first}, {&assigned, &first}}); + _ZN3sce4Json5Value4swapERS1_(&source, &assigned); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&assigned, RecoverLegacy, &first) == 0, "Restore assignment callback after swap"); + Array array{}; _ZN3sce4Json5ArrayC1Ev(&array); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &source); + const auto* element = _ZNK3sce4Json5Array5frontEv(&array); + Require(*_ZNK3sce4Json5Value10getBooleanEv(element) && first.source == element && !first.parent, "Legacy array append must copy the handler map entry without binding a modern parent"); + CheckMap(trace, {{&source, &first}, {©, &first}, {&assigned, &first}, {element, &first}}); + trace.budget = 0; + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &source); + trace.budget = -1; + Require(_ZNK3sce4Json5Array4sizeEv(&array) == 0, "A failed legacy append must remove the existing tail and its handler record"); + CheckMap(trace, {{&source, &first}, {©, &first}, {&assigned, &first}}); + _ZN3sce4Json5ArrayD1Ev(&array); + Value parsed{}; ParseLegacy(parsed, "[9]"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&parsed, RecoverLegacy, &first) == 0, "Register legacy parsed-container recovery"); + const auto* child = _ZNK3sce4Json5Array5frontEv(reinterpret_cast(parsed.payload)); + Require(*_ZNK3sce4Json5Value10getBooleanEv(child) && first.source == child && first.parent == &parsed, "Legacy descendant mismatch recovery must use the root map and immediate parent"); + _ZN3sce4Json5ValueD1Ev(&parsed); + Value null{}; ParseLegacy(null, "null"); NullContext nullContext{&source, &first}; + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&null, RecoverLegacyNull, &nullContext) == 0 && *_ZNK3sce4Json5Value10getBooleanEv(&null), "Legacy local null recovery must preserve its pre-guard dispatch"); + _ZN3sce4Json5ValueD1Ev(&null); _ZN3sce4Json5ValueD1Ev(&assigned); _ZN3sce4Json5ValueD1Ev(©); _ZN3sce4Json5ValueD1Ev(&source); + CheckMap(trace, {}); + std::array roots{}; std::map entries; + for (unsigned index = 0; index < roots.size(); ++index) { + auto& root = roots[index * 13 % roots.size()]; ParseLegacy(root); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&root, RecoverLegacy, &first) == 0, "Register legacy tree entry"); + entries.emplace(&root, &first); CheckMap(trace, entries); + } + for (unsigned index = 0; index < roots.size(); ++index) { + auto& root = roots[index * 7 % roots.size()]; + _ZN3sce4Json5ValueD1Ev(&root); entries.erase(&root); CheckMap(trace, entries); + } + _ZN3sce4Json5ValueD1Ev(&second.replacement); _ZN3sce4Json5ValueD1Ev(&first.replacement); + Require(trace.guest.size() == baseline + 1 && trace.heap.size() == 1, "Empty legacy map storage must remain owned until termination"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.guest.empty() && trace.heap.empty(), "Termination must release both allocator domains and all legacy map resources"); + HeapTrace = nullptr; + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index e85c8c70f7..3455cffcb5 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -86,14 +86,14 @@ - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - `Pair(const String&, const Value&)` and default `Pair()` zero the 8 bytes between the key and value; what the console keeps there is unknown. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. -- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy registration remains incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. SDK-dependent legacy registration and storage remain unverified. -- [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent legacy handler maps and dispatch remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Compatibility below SDK 6 is unverified. +- [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. +- [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - libc-heap failures and untested legacy callback combinations remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Additional legacy compatibility remains unverified. - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback-value failure behavior and compatibility below SDK 6 remain incomplete. - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. - [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object, callback metadata and file-buffer allocation failures dispatch the supplied allocator's notifier. - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. - [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; a missing iterator state uses the module default value, whose allocator-backed identity remains incomplete. -- [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy SDK behavior remains incomplete. Root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. +- [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - untested legacy callback combinations remain incomplete. Modern root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. - [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - destination state after parse failure and allocator failures outside the file buffer remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. - [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. @@ -103,9 +103,9 @@ - [sce::Json::Value handle admission and type changes](../../../core/libs/prx/libSceJson2/Export.cpp) - collection copy/assignment failure ordering, nested recovery failures and older SDK compatibility remain incomplete. The supplied allocator owns each eight-byte complex-value handle. Typed construction and type changes follow `0x580`, `0x10d0`, `0x1170` and `0x1210`: the old payload is destroyed before replacement admission; string/header failure leaves null, an array can retain a header without its sentinel, and object construction reclaims a failed sentinel. Scalar changes preserve inactive payload bytes. Special-float string conversion follows the temporary string at `0x40a0`; failure of its header preserves the prior value, while later failures can leave null or an empty string. Local tests cover SDK 6 process metadata, not every firmware version. -- [sce::Json::Value auxiliary admission and retry](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy per-root mismatch, float and element handler maps remain unsupported and throw. Modern admission follows `0x3280`: a failed fallback allocation retains the 72-byte auxiliary record; later registration reuses it without recreating the missing fallback. Local tests cover all four callback setters, their allocation failures, replacement and null callbacks, child rejection, parent preservation and teardown through the original allocator. Legacy auxiliary records use 24 bytes, but their external handler maps and dispatch remain incomplete. +- [sce::Json::Value auxiliary admission and retry](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float and element registration remain unsupported and throw. Modern admission follows `0x3280`: a failed fallback allocation retains the 72-byte auxiliary record; later registration reuses it without recreating the missing fallback. Local tests cover all four callback setters, their allocation failures, replacement and null callbacks, child rejection, parent preservation and teardown through the original allocator. Legacy auxiliary records use 24 bytes; mismatch callbacks use a separate map whose header uses the supplied allocator and whose 56-byte nodes use libc new/delete. -- [sce::Json::Value scalar and string copying](../../../core/libs/prx/libSceJson2/Export.cpp) - collection assignment and legacy callback-map propagation remain incomplete. Scalar and string dispatch follow `0x1e40`: null and boolean copies preserve inactive destination bytes, numeric values retain their exact payloads, and string assignment changes type before copying characters into the existing header. A character-allocation failure can empty an existing string; it does not restore its old contents. Copy construction initializes ownership metadata directly, following `0x1e20`. Local allocator tests cover null, boolean, signed, unsigned, finite real and string sources across all destination types and failure budgets. +- [sce::Json::Value scalar and string copying](../../../core/libs/prx/libSceJson2/Export.cpp) - nested collection assignment failures remain incomplete. Scalar and string dispatch follow `0x1e40`: null and boolean copies preserve inactive destination bytes, numeric values retain their exact payloads, and string assignment changes type before copying characters into the existing header. A character-allocation failure can empty an existing string; it does not restore its old contents. Copy construction initializes ownership metadata directly, following `0x1e20`. Local allocator tests cover null, boolean, signed, unsigned, finite real and string sources across all destination types and failure budgets. - [sce::Json::Array and Value collection assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocator failures, legacy callback propagation and concurrent operation accounting remain incomplete. Modern array assignment follows `0xbc80`: initial header/sentinel failure preserves the original, node failures retain a partial replacement and iteration continues, self-assignment replaces the header, and entries bind to the destination owner. Legacy assignment uses replacement list storage inside the existing header, following `0xb540`. Value collection dispatch follows `0x1e40`, changing type before invoking Array/Object assignment rather than cloning through their copy constructors. Local tests cover scalar array assignment, self-assignment and empty collection Value admission across destination types. @@ -114,3 +114,5 @@ - [sce::Json::Value nested array allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. - [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - initialization follows `0x1340`, `0xda0` and `0x940`: a 24-byte allocation context owns the original allocator, user context and file-buffer setting; default handles/storage, a 32-byte fallback and kernel mutex handles are allocated before admission. Inner default failures can retain usable outer handles; required-resource failure reclaims ownership and leaves retry legal. SDK 6 uses two recursive kernel mutex handles, while the legacy path shares one. Local tests substitute the kernel boundary and cover all twelve SDK 6 allocation positions, partial admission and retry. Failed kernel mutex setup or locking throws rather than fabricating success; terminating from an active callback remains unsupported. These checks do not prove Linux/Windows integration or every reentrant/concurrent lifecycle. + +- [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. From 296017fa95cf879cbaa3d9e9aee8f44440cea1f8 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 05:41:10 +0200 Subject: [PATCH 024/318] docs(libSceJson2): record verified parser failure state mismatch --- docs/dev/debt/libSceJson2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 3455cffcb5..c022855bea 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -81,7 +81,7 @@ - [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. -- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser +- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser. Reference entry points `0x6940` and `0x6840` mutate the supplied destination directly: parsing `[1,` over integer 77 leaves a two-entry array, `{"a":1,` leaves a one-entry object, and an unterminated string retains its decoded prefix. Empty input, an unknown token and an incomplete boolean literal preserve the previous integer. The current transactional parser and its existing destination-preservation assertions contradict these reference results; they require a coordinated repair, including nested ownership and allocation failures. - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. From 74784daf13ee4ef930a178b7d50aefd6dcb7cf29 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 05:45:27 +0200 Subject: [PATCH 025/318] fix(libSceJson2): retain parser destination state on failure --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 42 +++++----- core/libs/tests/GuestJson.cpp | 16 ++-- core/libs/tests/GuestJsonParserState.cpp | 99 ++++++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 7 +- 5 files changed, 138 insertions(+), 33 deletions(-) create mode 100644 core/libs/tests/GuestJsonParserState.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 27b581f352..722cd23d32 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -21,6 +21,11 @@ if(BUILD_TESTING) target_link_libraries(guest_json_allocator_settings_tests PRIVATE libSceJson2 libc) configure_windows_unwind(guest_json_allocator_settings_tests) add_test(NAME guest_json_allocator_settings COMMAND guest_json_allocator_settings_tests) + add_test_executable(guest_json_parser_state_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonParserState.cpp) + target_include_directories(guest_json_parser_state_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_parser_state_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_parser_state_tests) + add_test(NAME guest_json_parser_state COMMAND guest_json_parser_state_tests) add_test_executable(guest_json_value_layout_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonValueLayout.cpp) target_include_directories(guest_json_value_layout_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_value_layout_tests PRIVATE libSceJson2 libc) @@ -86,7 +91,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 31e7c11850..e358c24e14 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1831,7 +1831,8 @@ private: } return true; } - static void appendUtf8(std::string& out, std::uint32_t code) { + template + static void appendUtf8(Output& out, std::uint32_t code) { if (!code) return; if (code < 0x80) out += static_cast(code); else if (code < 0x800) { out += static_cast(0xc0 | (code >> 6)); out += static_cast(0x80 | (code & 0x3f)); } @@ -1850,7 +1851,12 @@ private: } return true; } - bool string(std::string& out) { + struct ParsedText { + String& string; + void operator+=(char character) { AppendText(string, {&character, 1}); } + }; + template + bool string(Output& out) { if (peek() < 0 || peek() != '"') return false; take(); while (peek() >= 0 && peek() != '"') { @@ -1909,11 +1915,11 @@ private: errno = 0; if (text[0] == '-') { const long long value = std::strtoll(text.c_str(), nullptr, 10); - if (errno == 0) { out.type = TypeInteger; out.integer = value; return true; } + if (errno == 0) { SetType(out, TypeInteger); out.integer = value; return true; } } else { const unsigned long long value = std::strtoull(text.c_str(), nullptr, 10); if (errno == 0) { - out.type = TypeUInteger; + SetType(out, TypeUInteger); out.uinteger = value; return true; } @@ -1929,7 +1935,7 @@ private: converted = !stream.fail() && stream.eof(); } if (!converted || !std::isfinite(real)) throw std::runtime_error("sce::Json::Parser::parse: number " + text + " overflows a double; the console's behaviour is unverified"); - out.type = TypeReal; + SetType(out, TypeReal); out.real = real; return true; } @@ -1938,14 +1944,12 @@ private: switch (peek()) { case 'n': if (!literal("null")) return false; SetType(out, TypeNull); return true; case 't': if (!literal("true")) return false; SetType(out, TypeBoolean); out.boolean = true; return true; - case 'f': if (!literal("false")) return false; SetType(out, TypeBoolean); return true; + case 'f': if (!literal("false")) return false; SetType(out, TypeBoolean); out.boolean = false; return true; case '"': { - std::string text; - if (!string(text)) return false; SetType(out, TypeString); if (out.type != TypeString) throw GuestOutOfMemory{}; - AssignText(*out.string, text); - return true; + ParsedText text{*out.string}; + return string(text); } case '[': { if (depth == MaxNestingDepth) return false; @@ -1985,7 +1989,6 @@ private: take(); skip(); Value& entry = ObjectEntry(*out.object, key); - Clear(entry); if (!value(entry)) return false; skip(); if (peek() >= 0 && peek() == ',') { take(); continue; } @@ -2004,24 +2007,15 @@ int ParseValue(Value* out, Parser& parser) { const auto status = EnsureAuxiliary(*out); if (status) return status; } - Node parsed{}; + const auto failures = AllocationFailures(); bool parsedOk = false; try { - parsedOk = parser.Parse(parsed); + parsedOk = parser.Parse(NodeOf(*out)); } catch (const GuestOutOfMemory&) { - Clear(parsed); return JsonErrorOutOfMemory; - } catch (...) { - Clear(parsed); - throw; } - if (!parsedOk) { - Clear(parsed); - return parser.Error(); - } - Clear(NodeOf(*out)); - TransferPayload(*out, parsed); - return 0; + if (AllocationFailures() != failures) return JsonErrorOutOfMemory; + return parsedOk ? 0 : parser.Error(); } [[noreturn]] void APS5_VABI JsonMemAllocatorPureVirtual() { diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index dbd0af06a3..9f8f8ee842 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -246,9 +246,13 @@ static void ParseAndRoundTrip() { Require(Serialize(reparsed) == serialized); _ZN3sce4Json5ValueD1Ev(&reparsed); - for (const char* invalid : {"{\"a\":}", "[1,]", "\"open", "tru"}) { - Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, invalid, std::strlen(invalid)) == JsonErrorParse); - Require(_ZNK3sce4Json5Value7getTypeEv(&root) == TypeObject && _ZNK3sce4Json5Value5countEv(&root) == 8); + for (const auto& [invalid, expected] : std::vector>{ + {"{\"a\":}", "{\"a\":null}"}, {"[1,]", "[1,null]"}, {"\"open", "\"open\""}, {"tru", "null"}}) { + Value partial{}; + _ZN3sce4Json5ValueC1Ev(&partial); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&partial, invalid, std::strlen(invalid)) == JsonErrorParse); + Require(Serialize(partial) == expected); + _ZN3sce4Json5ValueD1Ev(&partial); } Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, nullptr, 0) == JsonErrorParse); bool nullTextRejected = false; @@ -261,14 +265,16 @@ static void ParseAndRoundTrip() { Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, "", 0) == JsonErrorParse); Require(_ZNK3sce4Json5Value7getTypeEv(&root) == TypeObject && _ZNK3sce4Json5Value5countEv(&root) == 8); for (const char* overflow : {"1e309", "[-1e309]", "{\"a\":[1,1e400]}"}) { + Value partial{}; + _ZN3sce4Json5ValueC1Ev(&partial); bool unverified = false; try { - _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, overflow, std::strlen(overflow)); + _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&partial, overflow, std::strlen(overflow)); } catch (const std::runtime_error& error) { unverified = std::string(error.what()).find("unverified") != std::string::npos; } Require(unverified); - Require(_ZNK3sce4Json5Value7getTypeEv(&root) == TypeObject && _ZNK3sce4Json5Value5countEv(&root) == 8); + _ZN3sce4Json5ValueD1Ev(&partial); } _ZN3sce4Json5ValueD1Ev(&root); } diff --git a/core/libs/tests/GuestJsonParserState.cpp b/core/libs/tests/GuestJsonParserState.cpp new file mode 100644 index 0000000000..892db3a6a9 --- /dev/null +++ b/core/libs/tests/GuestJsonParserState.cpp @@ -0,0 +1,99 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "JsonTestAllocator.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Array { void* storage; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +static void* APS5_VABI AllocateTracked(void*, std::size_t size, void* context) { + void* memory = std::malloc(size); + Require(memory != nullptr, "Tracking allocator requires host memory"); + static_cast*>(context)->emplace(memory, size); + return memory; +} +static void APS5_VABI FreeTracked(void*, void* memory, void* context) { + if (!memory) return; + Require(static_cast*>(context)->erase(memory) == 1, "Parser allocations must be freed exactly once by their owning allocator"); + std::free(memory); +} +static int Parse(Value& value, const char* text) { + return _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, text, std::strlen(text)); +} +int main() { + JsonTest::Allocator allocator; + void* const vtable[]{nullptr, nullptr, reinterpret_cast(AllocateTracked), reinterpret_cast(FreeTracked), reinterpret_cast(JsonTest::Notify)}; + allocator.vtable = vtable; + std::map live; + Parameter parameter{&allocator, &live, 512}; + std::uint64_t initializer[2]{}; + _ZN3sce4Json11InitializerC1Ev(initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(initializer, ¶meter) == 0, "Initialize parser state tests"); + const auto baseline = live.size(); + for (const char* text : {"", "?", "tru"}) { + Value value{}; _ZN3sce4Json5ValueC1El(&value, 77); + Require(static_cast(Parse(value, text)) == 0x80848101 && value.type == 2 && value.payload == 77, "Unrecognized or incomplete scalar input must preserve the previous payload"); + _ZN3sce4Json5ValueD1Ev(&value); + } + for (const char* text : {"[1,", "[1,?]"}) { + Value value{}; _ZN3sce4Json5ValueC1El(&value, 77); + Require(static_cast(Parse(value, text)) == 0x80848101 && value.type == 6 && _ZNK3sce4Json5Value5countEv(&value) == 2, "Failed array parsing must retain admitted entries instead of restoring the destination"); + auto* array = reinterpret_cast(value.payload); + const auto* first = _ZNK3sce4Json5Array5frontEv(array); + const auto* last = _ZNK3sce4Json5Array4backEv(array); + Require(first->type == 3 && first->payload == 1 && first->parent == &value && last->type == 0, "Partial array entries must retain their payload and destination parent"); + Require(Parse(value, "[]") == 0 && _ZNK3sce4Json5Value5countEv(&value) == 2, "Parsing an empty array into an existing array must retain its entries"); + _ZN3sce4Json5ValueD1Ev(&value); + } + for (const char* text : {"{\"a\":1,", "{\"a\":?}"}) { + Value value{}; _ZN3sce4Json5ValueC1El(&value, 77); + Require(static_cast(Parse(value, text)) == 0x80848101 && value.type == 7 && _ZNK3sce4Json5Value5countEv(&value) == 1, "Failed object parsing must retain the admitted member"); + Require(Parse(value, "{}") == 0 && _ZNK3sce4Json5Value5countEv(&value) == 1, "Parsing an empty object into an existing object must retain its members"); + _ZN3sce4Json5ValueD1Ev(&value); + } + Value nested{}; _ZN3sce4Json5ValueC1El(&nested, 77); + Require(static_cast(Parse(nested, "[[1,")) == 0x80848101 && nested.type == 6, "Nested failures must retain the outer collection"); + const auto* inner = _ZNK3sce4Json5Array5frontEv(reinterpret_cast(nested.payload)); + Require(inner->type == 6 && inner->parent == &nested && _ZNK3sce4Json5Value5countEv(inner) == 2, "Nested partial collections must retain their immediate owner"); + const auto* leaf = _ZNK3sce4Json5Array5frontEv(reinterpret_cast(inner->payload)); + Require(leaf->parent == inner && leaf->payload == 1, "Nested leaf parents must point into the retained destination tree"); + Require(Parse(nested, "42") == 0 && nested.type == 3 && nested.payload == 42, "Numeric parsing must replace an existing collection"); + _ZN3sce4Json5ValueD1Ev(&nested); + Value object{}; _ZN3sce4Json5ValueC1El(&object, 77); + Require(Parse(object, "{\"a\":9}") == 0 && static_cast(Parse(object, "{\"a\":?}")) == 0x80848101, "Prepare duplicate member failure"); + const auto* retained = _ZNK3sce4Json5ValueixEPKc(&object, "a"); + Require(retained->type == 3 && retained->payload == 9 && retained->parent == &object, "An unrecognized duplicate member value must preserve the prior member"); + _ZN3sce4Json5ValueD1Ev(&object); + Value value{}; _ZN3sce4Json5ValueC1El(&value, 77); + Require(static_cast(Parse(value, "\"unterminated")) == 0x80848101 && value.type == 5, "Unterminated strings must change the destination type"); + Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(value.payload)), "unterminated") == 0, "Unterminated strings must retain their decoded prefix"); + Require(Parse(value, "\"!\"") == 0 && std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(value.payload)), "unterminated!") == 0, "Parsing another string must append to existing string storage"); + Require(Parse(value, "true") == 0 && Parse(value, "false") == 0 && value.type == 1 && !(value.payload & 255), "Parsing false must replace an existing true boolean"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(live.size() == baseline, "Partial parsing and numeric replacement must reclaim all value allocations"); + Require(_ZN3sce4Json11Initializer9terminateEv(initializer) == 0, "Terminate parser state tests"); + Require(live.empty(), "Termination must reclaim the remaining parser context resources"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index c022855bea..5bf46bd5f1 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -79,9 +79,10 @@ ### Unknown function info -- [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. -- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a null text pointer with nonzero size throws instead of being dereferenced. A number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown. Syntax validation for number tokens and destination state after a failed parse still differ from the system parser. Reference entry points `0x6940` and `0x6840` mutate the supplied destination directly: parsing `[1,` over integer 77 leaves a two-entry array, `{"a":1,` leaves a one-entry object, and an unterminated string retains its decoded prefix. Empty input, an unknown token and an incomplete boolean literal preserve the previous integer. The current transactional parser and its existing destination-preservation assertions contradict these reference results; they require a coordinated repair, including nested ownership and allocation failures. + +- [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. +- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size and double overflow throw rather than inventing a result. Numeric-token validation, parser allocation failures and nested operation accounting remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. @@ -95,7 +96,7 @@ - [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; a missing iterator state uses the module default value, whose allocator-backed identity remains incomplete. - [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - untested legacy callback combinations remain incomplete. Modern root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. -- [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - destination state after parse failure and allocator failures outside the file buffer remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. +- [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator failures outside the file buffer and callback-read failure combinations remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. - [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. - [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocation failures, legacy ownership and concurrent operation accounting remain incomplete. Allocator-backed tree nodes can retain incomplete entries or a source count exceeding their live nodes after copy failure (`0xca40`, `0xe000`). Modern assignment (`0xd400`) preserves the original on initial allocation failure but can replace it with partial entries after later failures; self-assignment rebuilds the tree. Legacy insertion copies the pair directly. SDK 5 boundary tests do not establish cross-firmware compatibility. Erasure allocates nothing (`0xd350`). Missing-state iterator dereference throws if its temporary default iterator cannot be allocated; the reference can recurse. Full Linux/Windows execution remains unverified. From 9c781b90ba7d7e6618f424b54052e16d641085fc Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 05:50:01 +0200 Subject: [PATCH 026/318] test(libSceJson2): verify nested object allocation failures --- core/libs/tests/GuestJsonObjectAllocator.cpp | 493 +++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 2 files changed, 496 insertions(+), 1 deletion(-) diff --git a/core/libs/tests/GuestJsonObjectAllocator.cpp b/core/libs/tests/GuestJsonObjectAllocator.cpp index d0d72b2f7d..0495c98c1e 100644 --- a/core/libs/tests/GuestJsonObjectAllocator.cpp +++ b/core/libs/tests/GuestJsonObjectAllocator.cpp @@ -357,6 +357,498 @@ static void CheckNestedAssignment(ObjectAllocationTrace& trace) { _ZN3sce4Json5ValueD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); Require(trace.live.size() == baseline, "Nested assignment must release all resources"); } +static std::string NestedObjectFailureState(const Value& value, const Value* parent = nullptr) { + std::string state = std::to_string(value.type) + (value.parent == parent ? "p" : value.parent ? "x" : "-"); + if (value.type == 0) return state; + if (value.type < 5) return state + ":" + std::to_string(value.payload); + if (!value.payload) return state + "!"; + const auto* header = *reinterpret_cast(value.payload); + if (!header) return state + "?"; + if (value.type == 5) { + const auto* text = static_cast(header); + return state + ":" + std::string(text->characters, text->length); + } + const auto* words = static_cast(header); + state += words[3] == reinterpret_cast(&value) ? "o" : "!"; + state += std::to_string(words[1]) + "["; + if (!words[0]) return state + "?]"; + if (value.type == 6) { + const auto* head = reinterpret_cast(words[0]); + const auto* node = reinterpret_cast(head[1]); + unsigned steps = 0; + while (node != head) { + Require(++steps < 32, "Nested test array links must reach their sentinel"); + state += NestedObjectFailureState(*reinterpret_cast(node + 2), &value) + ";"; + node = reinterpret_cast(node[1]); + } + } else { + Require(value.type == 7, "Nested test collection must be an array or object"); + const auto* head = reinterpret_cast(words[0]); + unsigned steps = 0; + std::function visit = [&](const TreeNode* node) { + if (node == head) return; + Require(node && ++steps < 32, "Nested test tree links must reach their sentinel"); + visit(node->left); + state += node->pair.key.storage ? _ZNK3sce4Json6String5c_strEv(&node->pair.key) : "~"; + state += "=" + NestedObjectFailureState(node->pair.value, &value) + ";"; + visit(node->right); + }; + visit(head->parent); + } + return state + "]"; +} +static void CheckNestedFailureCases(ObjectAllocationTrace& trace) { + static const char* const states[]{ + "7po1[z=3p:99;]", + "7po0[]", + "7po1[~=0-;]", + "7po1[=0-;]", + "7po1[a=0-;]", + "7po1[a=5p:;]", + "7po1[a=5p:abc;]", + "7po2[a=5p:abc;~=0-;]", + "7po2[a=5p:abc;=0-;]", + "7po2[a=5p:abc;b=0-;]", + "7po2[a=5p:abc;b=7-o0[];]", + "7po2[a=5p:abc;b=7po0[];]", + "7po2[a=5p:abc;b=7po1[~=0-;];]", + "7po2[a=5p:abc;b=7po1[=0-;];]", + "7po2[a=5p:abc;b=7po1[x=0-;];]", + "7po2[a=5p:abc;b=7po1[x=5p:;];]", + "7po2[a=5p:abc;b=7po1[x=5p:def;];]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];~=0-;]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];=0-;]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];c=0-;]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];c=6-o0[?];]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];c=6-o0[];]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];c=6po0[];]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[~=5p:abc;b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[=5p:abc;b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[a=5-:;b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[~=5-:;b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[=5-:;b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[a=0-;b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po2[b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[a=5p:;b=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;~=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;=7po1[x=5p:def;];c=6po1[3p:42;];]", + "7po3[b=7-o0[];a=5p:abc;c=6po1[3p:42;];]", + "7po3[a=5p:abc;~=7-o0[];c=6po1[3p:42;];]", + "7po3[a=5p:abc;=7-o0[];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=0-;c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7-o0[];c=6po1[3p:42;];]", + "7po2[a=5p:abc;c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po0[];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[~=5p:def;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[=5p:def;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[x=5-:;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[~=5-:;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[=5-:;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[x=0-;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[x=5p:;];c=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];~=6po1[3p:42;];]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];=6po1[3p:42;];]", + "7po3[c=6-o0[];a=5p:abc;b=7po1[x=5p:def;];]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];~=6-o0[];]", + "7po3[a=5p:abc;b=7po1[x=5p:def;];=6-o0[];]", + "7p?", + "7p!0[?]", + "7p!0[]", + "7p!3[~=0-;]", + "7p!3[=0-;]", + "7p!3[b=0-;]", + "7p!3[b=7-o0[];]", + "7p!3[b=7-o1[~=0-;];]", + "7p!3[b=7-o1[=0-;];]", + "7p!3[b=7-o1[x=0-;];]", + "7p!3[b=7-o1[x=5p:;];]", + "7p!3[b=7-o1[x=5p:def;];]", + "7p!3[~=0-;b=7-o1[x=5p:def;];]", + "7p!3[=0-;b=7-o1[x=5p:def;];]", + "7p!3[a=0-;b=7-o1[x=5p:def;];]", + "7p!3[a=5-:;b=7-o1[x=5p:def;];]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];~=0-;]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];=0-;]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];c=0-;]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];c=6-o0[?];]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];c=6-o0[];]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;~=7-o1[x=5p:def;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;=7-o1[x=5p:def;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=0-;c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o0[];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[~=5p:def;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[=5p:def;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[x=5-:;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[~=5-:;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[=5-:;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[x=0-;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[x=5p:;];c=6-o1[3p:42;];]", + "7p!3[b=7-o1[x=5p:def;];c=6-o1[3p:42;];]", + "7p!3[~=5-:abc;b=7-o1[x=5p:def;];c=6-o1[3p:42;];]", + "7p!3[=5-:abc;b=7-o1[x=5p:def;];c=6-o1[3p:42;];]", + "7p!3[a=0-;b=7-o1[x=5p:def;];c=6-o1[3p:42;];]", + "7p!3[a=5-:;b=7-o1[x=5p:def;];c=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];~=6-o1[3p:42;];]", + "7p!3[a=5-:abc;b=7-o1[x=5p:def;];=6-o1[3p:42;];]", + }; + struct Case { bool constructor; bool once; int budget; unsigned notifications; std::size_t requests; unsigned state; }; + const Case cases[]{ + {0, 0, 0, 1, 1, 0}, + {0, 0, 1, 1, 2, 0}, + {0, 0, 2, 12, 14, 1}, + {0, 0, 3, 12, 15, 1}, + {0, 0, 4, 12, 16, 1}, + {0, 0, 5, 13, 18, 1}, + {0, 0, 6, 12, 18, 1}, + {0, 0, 7, 13, 20, 1}, + {0, 0, 8, 12, 20, 1}, + {0, 0, 9, 12, 21, 1}, + {0, 0, 10, 12, 22, 1}, + {0, 0, 11, 12, 23, 1}, + {0, 0, 12, 11, 23, 1}, + {0, 0, 13, 11, 24, 1}, + {0, 0, 14, 10, 24, 1}, + {0, 0, 15, 10, 25, 1}, + {0, 0, 16, 9, 25, 1}, + {0, 0, 17, 9, 26, 2}, + {0, 0, 18, 10, 28, 2}, + {0, 0, 19, 10, 29, 3}, + {0, 0, 20, 9, 29, 4}, + {0, 0, 21, 9, 30, 4}, + {0, 0, 22, 9, 31, 5}, + {0, 0, 23, 8, 31, 6}, + {0, 0, 24, 8, 32, 6}, + {0, 0, 25, 8, 33, 6}, + {0, 0, 26, 8, 34, 6}, + {0, 0, 27, 9, 36, 6}, + {0, 0, 28, 8, 36, 6}, + {0, 0, 29, 9, 38, 6}, + {0, 0, 30, 8, 38, 6}, + {0, 0, 31, 8, 39, 6}, + {0, 0, 32, 8, 40, 6}, + {0, 0, 33, 9, 42, 6}, + {0, 0, 34, 9, 43, 6}, + {0, 0, 35, 8, 43, 6}, + {0, 0, 36, 8, 44, 6}, + {0, 0, 37, 7, 44, 6}, + {0, 0, 38, 7, 45, 6}, + {0, 0, 39, 6, 45, 6}, + {0, 0, 40, 6, 46, 6}, + {0, 0, 41, 6, 47, 6}, + {0, 0, 42, 6, 48, 6}, + {0, 0, 43, 6, 49, 6}, + {0, 0, 44, 5, 49, 6}, + {0, 0, 45, 7, 52, 7}, + {0, 0, 46, 6, 52, 7}, + {0, 0, 47, 6, 53, 8}, + {0, 0, 48, 5, 53, 9}, + {0, 0, 49, 5, 54, 9}, + {0, 0, 50, 5, 55, 9}, + {0, 0, 51, 5, 56, 10}, + {0, 0, 52, 5, 57, 10}, + {0, 0, 53, 5, 58, 11}, + {0, 0, 54, 5, 59, 11}, + {0, 0, 55, 8, 63, 11}, + {0, 0, 56, 8, 64, 11}, + {0, 0, 57, 8, 65, 11}, + {0, 0, 58, 9, 67, 11}, + {0, 0, 59, 8, 67, 11}, + {0, 0, 60, 9, 69, 11}, + {0, 0, 61, 8, 69, 11}, + {0, 0, 62, 8, 70, 11}, + {0, 0, 63, 8, 71, 11}, + {0, 0, 64, 8, 72, 11}, + {0, 0, 65, 7, 72, 11}, + {0, 0, 66, 7, 73, 11}, + {0, 0, 67, 6, 73, 11}, + {0, 0, 68, 6, 74, 11}, + {0, 0, 69, 5, 74, 11}, + {0, 0, 70, 7, 77, 12}, + {0, 0, 71, 6, 77, 12}, + {0, 0, 72, 6, 78, 13}, + {0, 0, 73, 5, 78, 14}, + {0, 0, 74, 5, 79, 14}, + {0, 0, 75, 5, 80, 15}, + {0, 0, 76, 4, 80, 16}, + {0, 0, 77, 4, 81, 16}, + {0, 0, 78, 5, 83, 16}, + {0, 0, 79, 4, 83, 16}, + {0, 0, 80, 5, 85, 16}, + {0, 0, 81, 4, 85, 16}, + {0, 0, 82, 5, 87, 16}, + {0, 0, 83, 4, 87, 16}, + {0, 0, 84, 4, 88, 16}, + {0, 0, 85, 6, 91, 16}, + {0, 0, 86, 5, 91, 16}, + {0, 0, 87, 5, 92, 16}, + {0, 0, 88, 4, 92, 16}, + {0, 0, 89, 4, 93, 16}, + {0, 0, 90, 3, 93, 16}, + {0, 0, 91, 3, 94, 16}, + {0, 0, 92, 2, 94, 16}, + {0, 0, 93, 2, 95, 16}, + {0, 0, 94, 3, 97, 16}, + {0, 0, 95, 2, 97, 16}, + {0, 0, 96, 2, 98, 16}, + {0, 0, 97, 1, 98, 16}, + {0, 0, 98, 3, 101, 17}, + {0, 0, 99, 2, 101, 17}, + {0, 0, 100, 2, 102, 18}, + {0, 0, 101, 1, 102, 19}, + {0, 0, 102, 1, 103, 19}, + {0, 0, 103, 2, 105, 20}, + {0, 0, 104, 1, 105, 21}, + {0, 0, 105, 1, 106, 21}, + {0, 0, 106, 1, 107, 22}, + {0, 0, 107, 1, 108, 22}, + {0, 0, 108, 1, 109, 22}, + {0, 0, 109, 0, 109, 23}, + {0, 1, 0, 1, 1, 0}, + {0, 1, 1, 1, 2, 0}, + {0, 1, 2, 1, 104, 23}, + {0, 1, 3, 1, 105, 23}, + {0, 1, 4, 1, 102, 24}, + {0, 1, 5, 1, 106, 25}, + {0, 1, 6, 1, 104, 24}, + {0, 1, 7, 1, 107, 25}, + {0, 1, 8, 1, 106, 23}, + {0, 1, 9, 1, 107, 23}, + {0, 1, 10, 1, 106, 26}, + {0, 1, 11, 1, 107, 26}, + {0, 1, 12, 1, 105, 27}, + {0, 1, 13, 1, 107, 28}, + {0, 1, 14, 1, 105, 29}, + {0, 1, 15, 1, 106, 29}, + {0, 1, 16, 1, 103, 30}, + {0, 1, 17, 1, 108, 26}, + {0, 1, 18, 1, 107, 27}, + {0, 1, 19, 1, 108, 28}, + {0, 1, 20, 1, 107, 29}, + {0, 1, 21, 1, 108, 29}, + {0, 1, 22, 1, 109, 31}, + {0, 1, 23, 1, 95, 23}, + {0, 1, 24, 1, 96, 23}, + {0, 1, 25, 1, 97, 23}, + {0, 1, 26, 1, 102, 32}, + {0, 1, 27, 1, 106, 33}, + {0, 1, 28, 1, 104, 32}, + {0, 1, 29, 1, 107, 33}, + {0, 1, 30, 1, 98, 23}, + {0, 1, 31, 1, 99, 23}, + {0, 1, 32, 1, 100, 23}, + {0, 1, 33, 1, 108, 23}, + {0, 1, 34, 1, 109, 23}, + {0, 1, 35, 1, 84, 34}, + {0, 1, 36, 1, 85, 34}, + {0, 1, 37, 1, 83, 35}, + {0, 1, 38, 1, 85, 36}, + {0, 1, 39, 1, 77, 37}, + {0, 1, 40, 1, 78, 37}, + {0, 1, 41, 1, 79, 37}, + {0, 1, 42, 1, 85, 38}, + {0, 1, 43, 1, 86, 38}, + {0, 1, 44, 1, 78, 39}, + {0, 1, 45, 1, 86, 38}, + {0, 1, 46, 1, 85, 35}, + {0, 1, 47, 1, 86, 36}, + {0, 1, 48, 1, 82, 37}, + {0, 1, 49, 1, 83, 37}, + {0, 1, 50, 1, 84, 37}, + {0, 1, 51, 1, 85, 38}, + {0, 1, 52, 1, 86, 38}, + {0, 1, 53, 1, 87, 40}, + {0, 1, 54, 1, 88, 40}, + {0, 1, 55, 1, 104, 23}, + {0, 1, 56, 1, 105, 23}, + {0, 1, 57, 1, 102, 41}, + {0, 1, 58, 1, 106, 42}, + {0, 1, 59, 1, 104, 41}, + {0, 1, 60, 1, 107, 42}, + {0, 1, 61, 1, 106, 23}, + {0, 1, 62, 1, 107, 23}, + {0, 1, 63, 1, 106, 43}, + {0, 1, 64, 1, 107, 43}, + {0, 1, 65, 1, 105, 44}, + {0, 1, 66, 1, 107, 45}, + {0, 1, 67, 1, 105, 46}, + {0, 1, 68, 1, 106, 46}, + {0, 1, 69, 1, 103, 40}, + {0, 1, 70, 1, 108, 43}, + {0, 1, 71, 1, 107, 44}, + {0, 1, 72, 1, 108, 45}, + {0, 1, 73, 1, 107, 46}, + {0, 1, 74, 1, 108, 46}, + {0, 1, 75, 1, 109, 47}, + {0, 1, 76, 1, 95, 23}, + {0, 1, 77, 1, 96, 23}, + {0, 1, 78, 1, 109, 23}, + {0, 1, 79, 1, 102, 48}, + {0, 1, 80, 1, 106, 49}, + {0, 1, 81, 1, 104, 48}, + {0, 1, 82, 1, 107, 49}, + {0, 1, 83, 1, 98, 23}, + {0, 1, 84, 1, 99, 23}, + {0, 1, 85, 1, 109, 23}, + {0, 1, 86, 1, 108, 23}, + {0, 1, 87, 1, 109, 23}, + {0, 1, 88, 1, 104, 50}, + {0, 1, 89, 1, 105, 50}, + {0, 1, 90, 1, 103, 51}, + {0, 1, 91, 1, 105, 52}, + {0, 1, 92, 1, 97, 19}, + {0, 1, 93, 1, 98, 19}, + {0, 1, 94, 1, 106, 21}, + {0, 1, 95, 1, 105, 21}, + {0, 1, 96, 1, 106, 21}, + {0, 1, 97, 1, 98, 16}, + {0, 1, 98, 1, 106, 21}, + {0, 1, 99, 1, 105, 51}, + {0, 1, 100, 1, 106, 52}, + {0, 1, 101, 1, 102, 19}, + {0, 1, 102, 1, 103, 19}, + {0, 1, 103, 1, 106, 21}, + {0, 1, 104, 1, 105, 21}, + {0, 1, 105, 1, 106, 21}, + {0, 1, 106, 1, 107, 22}, + {0, 1, 107, 1, 108, 22}, + {0, 1, 108, 1, 109, 22}, + {0, 1, 109, 0, 109, 23}, + {1, 0, 0, 1, 1, 53}, + {1, 0, 1, 2, 3, 54}, + {1, 0, 2, 1, 3, 54}, + {1, 0, 3, 1, 4, 55}, + {1, 0, 4, 5, 9, 56}, + {1, 0, 5, 4, 9, 56}, + {1, 0, 6, 4, 10, 57}, + {1, 0, 7, 3, 10, 58}, + {1, 0, 8, 3, 11, 58}, + {1, 0, 9, 3, 12, 58}, + {1, 0, 10, 3, 13, 59}, + {1, 0, 11, 3, 14, 59}, + {1, 0, 12, 6, 18, 59}, + {1, 0, 13, 6, 19, 59}, + {1, 0, 14, 6, 20, 59}, + {1, 0, 15, 7, 22, 59}, + {1, 0, 16, 6, 22, 59}, + {1, 0, 17, 7, 24, 59}, + {1, 0, 18, 6, 24, 59}, + {1, 0, 19, 6, 25, 59}, + {1, 0, 20, 6, 26, 59}, + {1, 0, 21, 6, 27, 59}, + {1, 0, 22, 5, 27, 59}, + {1, 0, 23, 5, 28, 59}, + {1, 0, 24, 4, 28, 59}, + {1, 0, 25, 4, 29, 59}, + {1, 0, 26, 3, 29, 59}, + {1, 0, 27, 5, 32, 60}, + {1, 0, 28, 4, 32, 60}, + {1, 0, 29, 4, 33, 61}, + {1, 0, 30, 3, 33, 62}, + {1, 0, 31, 3, 34, 62}, + {1, 0, 32, 3, 35, 63}, + {1, 0, 33, 2, 35, 64}, + {1, 0, 34, 4, 38, 65}, + {1, 0, 35, 3, 38, 65}, + {1, 0, 36, 3, 39, 66}, + {1, 0, 37, 2, 39, 67}, + {1, 0, 38, 2, 40, 67}, + {1, 0, 39, 2, 41, 68}, + {1, 0, 40, 1, 41, 69}, + {1, 0, 41, 3, 44, 70}, + {1, 0, 42, 2, 44, 70}, + {1, 0, 43, 2, 45, 71}, + {1, 0, 44, 1, 45, 72}, + {1, 0, 45, 1, 46, 72}, + {1, 0, 46, 2, 48, 73}, + {1, 0, 47, 1, 48, 74}, + {1, 0, 48, 1, 49, 74}, + {1, 0, 49, 1, 50, 74}, + {1, 0, 50, 0, 50, 75}, + {1, 1, 0, 1, 1, 53}, + {1, 1, 1, 1, 50, 75}, + {1, 1, 2, 1, 3, 54}, + {1, 1, 3, 1, 4, 55}, + {1, 1, 4, 1, 50, 75}, + {1, 1, 5, 1, 49, 76}, + {1, 1, 6, 1, 50, 77}, + {1, 1, 7, 1, 25, 78}, + {1, 1, 8, 1, 26, 78}, + {1, 1, 9, 1, 27, 78}, + {1, 1, 10, 1, 28, 79}, + {1, 1, 11, 1, 29, 79}, + {1, 1, 12, 1, 45, 75}, + {1, 1, 13, 1, 46, 75}, + {1, 1, 14, 1, 43, 80}, + {1, 1, 15, 1, 47, 81}, + {1, 1, 16, 1, 45, 80}, + {1, 1, 17, 1, 48, 81}, + {1, 1, 18, 1, 47, 75}, + {1, 1, 19, 1, 48, 75}, + {1, 1, 20, 1, 47, 82}, + {1, 1, 21, 1, 48, 82}, + {1, 1, 22, 1, 46, 83}, + {1, 1, 23, 1, 48, 84}, + {1, 1, 24, 1, 46, 85}, + {1, 1, 25, 1, 47, 85}, + {1, 1, 26, 1, 44, 79}, + {1, 1, 27, 1, 49, 82}, + {1, 1, 28, 1, 48, 83}, + {1, 1, 29, 1, 49, 84}, + {1, 1, 30, 1, 48, 85}, + {1, 1, 31, 1, 49, 85}, + {1, 1, 32, 1, 50, 86}, + {1, 1, 33, 1, 44, 87}, + {1, 1, 34, 1, 50, 75}, + {1, 1, 35, 1, 49, 88}, + {1, 1, 36, 1, 50, 89}, + {1, 1, 37, 1, 48, 90}, + {1, 1, 38, 1, 49, 90}, + {1, 1, 39, 1, 50, 91}, + {1, 1, 40, 1, 41, 69}, + {1, 1, 41, 1, 50, 75}, + {1, 1, 42, 1, 49, 92}, + {1, 1, 43, 1, 50, 93}, + {1, 1, 44, 1, 45, 72}, + {1, 1, 45, 1, 46, 72}, + {1, 1, 46, 1, 50, 75}, + {1, 1, 47, 1, 48, 74}, + {1, 1, 48, 1, 49, 74}, + {1, 1, 49, 1, 50, 74}, + {1, 1, 50, 0, 50, 75}, + }; + const auto baseline = trace.live.size(); + for (const auto& test : cases) { + Value source{}, destination{}; + _ZN3sce4Json5ValueC1Ev(&source); _ZN3sce4Json5ValueC1Ev(&destination); + const char input[] = "{\"a\":\"abc\",\"b\":{\"x\":\"def\"},\"c\":[42]}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&source, input, sizeof(input) - 1) == 0, "Prepare nested failure source"); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&destination, "{\"z\":99}", 8) == 0, "Prepare nested failure destination"); + const auto sourceState = NestedObjectFailureState(source); + Object copy{}; + auto* sourceObject = reinterpret_cast(source.payload); + const auto notifications = trace.notifications; + const auto requests = trace.requests.size(); + trace.budget = test.budget; trace.failOnce = test.once; + if (test.constructor) _ZN3sce4Json6ObjectC1ERKS1_(©, sourceObject); + else _ZN3sce4Json6ObjectaSERKS1_(reinterpret_cast(destination.payload), sourceObject); + trace.budget = -1; trace.failOnce = false; + std::string actual; + if (test.constructor) { + Value proxy{}; proxy.type = 7; proxy.payload = reinterpret_cast(©); + actual = NestedObjectFailureState(proxy); + } else actual = NestedObjectFailureState(destination); + const bool matches = actual == states[test.state] && trace.notifications - notifications == test.notifications && trace.requests.size() - requests == test.requests; + if (!matches) std::fprintf(stderr, "Nested object constructor=%d once=%d budget=%d expected=%s actual=%s requests=%zu notifications=%u\n", test.constructor, test.once, test.budget, states[test.state], actual.c_str(), trace.requests.size() - requests, trace.notifications - notifications); + Require(matches, "Nested object failure must retain the reference state, ownership and allocation accounting"); + Require(NestedObjectFailureState(source) == sourceState, "Nested allocation failures must preserve source contents and ownership"); + if (test.constructor) _ZN3sce4Json6ObjectD1Ev(©); + _ZN3sce4Json5ValueD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); + Require(trace.live.size() == baseline, "Nested object failure must reclaim every admitted resource"); + } +} static void CheckAssignments(ObjectAllocationTrace& trace) { struct Case { bool once; int budget; std::size_t count; unsigned notifications; const char* contents; }; const Case cases[]{ @@ -503,6 +995,7 @@ int main() { CheckInsertions(trace); CheckAssignments(trace); CheckNestedAssignment(trace); + CheckNestedFailureCases(trace); CheckBinaryKeys(); Require(trace.live.size() == baseline, "Binary-key operations must release their storage"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate object allocation contract"); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 5bf46bd5f1..6504410711 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -99,7 +99,7 @@ - [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator failures outside the file buffer and callback-read failure combinations remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. - [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. -- [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested allocation failures, legacy ownership and concurrent operation accounting remain incomplete. Allocator-backed tree nodes can retain incomplete entries or a source count exceeding their live nodes after copy failure (`0xca40`, `0xe000`). Modern assignment (`0xd400`) preserves the original on initial allocation failure but can replace it with partial entries after later failures; self-assignment rebuilds the tree. Legacy insertion copies the pair directly. SDK 5 boundary tests do not establish cross-firmware compatibility. Erasure allocates nothing (`0xd350`). Missing-state iterator dereference throws if its temporary default iterator cannot be allocated; the reference can recurse. Full Linux/Windows execution remains unverified. +- [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy ownership, callback-bearing copies and concurrent operation accounting remain incomplete. Allocator-backed tree nodes can retain incomplete entries or a source count exceeding their live nodes after copy failure (`0xca40`, `0xe000`). Modern assignment (`0xd400`) preserves the original on initial allocation failure but can replace it with partial entries after later failures; self-assignment rebuilds the tree. Legacy insertion copies the pair directly. SDK 5 boundary tests do not establish cross-firmware compatibility. Erasure allocates nothing (`0xd350`). Missing-state iterator dereference throws if its temporary default iterator cannot be allocated; the reference can recurse. Full Linux/Windows execution remains unverified. - [sce::Json::Parser embedded zero characters](../../../core/libs/prx/libSceJson2/Export.cpp) - decoded Unicode zero is discarded through the inspected character-append behavior. Binary string values serialize using their stored length, while object keys serialize their C-string prefix. Broader malformed Unicode handling remains incomplete. - [sce::Json::Value handle admission and type changes](../../../core/libs/prx/libSceJson2/Export.cpp) - collection copy/assignment failure ordering, nested recovery failures and older SDK compatibility remain incomplete. The supplied allocator owns each eight-byte complex-value handle. Typed construction and type changes follow `0x580`, `0x10d0`, `0x1170` and `0x1210`: the old payload is destroyed before replacement admission; string/header failure leaves null, an array can retain a header without its sentinel, and object construction reclaims a failed sentinel. Scalar changes preserve inactive payload bytes. Special-float string conversion follows the temporary string at `0x40a0`; failure of its header preserves the prior value, while later failures can leave null or an empty string. Local tests cover SDK 6 process metadata, not every firmware version. @@ -117,3 +117,5 @@ - [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - initialization follows `0x1340`, `0xda0` and `0x940`: a 24-byte allocation context owns the original allocator, user context and file-buffer setting; default handles/storage, a 32-byte fallback and kernel mutex handles are allocated before admission. Inner default failures can retain usable outer handles; required-resource failure reclaims ownership and leaves retry legal. SDK 6 uses two recursive kernel mutex handles, while the legacy path shares one. Local tests substitute the kernel boundary and cover all twelve SDK 6 allocation positions, partial admission and retry. Failed kernel mutex setup or locking throws rather than fabricating success; terminating from an active callback remains unsupported. These checks do not prove Linux/Windows integration or every reentrant/concurrent lifecycle. - [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. + +- [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copies, alternate SDK behavior and concurrent failure accounting remain unverified. From 27da7edacea9af36e0e6cf4bdc5973410025fdb5 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 05:54:12 +0200 Subject: [PATCH 027/318] fix(libSceJson2): admit parser metadata after value storage --- core/libs/prx/libSceJson2/Export.cpp | 23 +++++++----- core/libs/tests/GuestJsonParserState.cpp | 46 +++++++++++++++++++++--- docs/dev/debt/libSceJson2.md | 2 ++ 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index e358c24e14..aa59aa3480 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1787,6 +1787,7 @@ public: Parser(ParserReadCallback read, void* context) : read(read), context(context) {} bool Parse(Node& out) { + failures = AllocationFailures(); skip(); return value(out); } @@ -1801,6 +1802,7 @@ private: ParserReadCallback read = nullptr; void* context = nullptr; std::size_t depth = 0; + std::uint64_t failures = 0; int current = -2; int readError = 0; @@ -1891,6 +1893,9 @@ private: take(); return true; } + void Admit(Node& out) { + if (!out.parent) EnsureAuxiliary(out); + } bool number(Node& out) { std::string token; if (peek() >= 0 && peek() == '-') token.push_back(static_cast(take())); @@ -1915,12 +1920,13 @@ private: errno = 0; if (text[0] == '-') { const long long value = std::strtoll(text.c_str(), nullptr, 10); - if (errno == 0) { SetType(out, TypeInteger); out.integer = value; return true; } + if (errno == 0) { SetType(out, TypeInteger); out.integer = value; Admit(out); return true; } } else { const unsigned long long value = std::strtoull(text.c_str(), nullptr, 10); if (errno == 0) { SetType(out, TypeUInteger); out.uinteger = value; + Admit(out); return true; } } @@ -1937,17 +1943,20 @@ private: if (!converted || !std::isfinite(real)) throw std::runtime_error("sce::Json::Parser::parse: number " + text + " overflows a double; the console's behaviour is unverified"); SetType(out, TypeReal); out.real = real; + Admit(out); return true; } bool value(Node& out) { if (peek() < 0) return false; switch (peek()) { - case 'n': if (!literal("null")) return false; SetType(out, TypeNull); return true; - case 't': if (!literal("true")) return false; SetType(out, TypeBoolean); out.boolean = true; return true; - case 'f': if (!literal("false")) return false; SetType(out, TypeBoolean); out.boolean = false; return true; + case 'n': if (!literal("null")) return false; SetType(out, TypeNull); Admit(out); return true; + case 't': if (!literal("true")) return false; SetType(out, TypeBoolean); out.boolean = true; Admit(out); return true; + case 'f': if (!literal("false")) return false; SetType(out, TypeBoolean); out.boolean = false; Admit(out); return true; case '"': { SetType(out, TypeString); + Admit(out); if (out.type != TypeString) throw GuestOutOfMemory{}; + if (AllocationFailures() != failures) return false; ParsedText text{*out.string}; return string(text); } @@ -1956,6 +1965,7 @@ private: take(); ++depth; SetType(out, TypeArray); + Admit(out); if (out.type != TypeArray) throw GuestOutOfMemory{}; skip(); if (peek() >= 0 && peek() == ']') { take(); --depth; return true; } @@ -1977,6 +1987,7 @@ private: take(); ++depth; SetType(out, TypeObject); + Admit(out); if (out.type != TypeObject) throw GuestOutOfMemory{}; skip(); if (peek() >= 0 && peek() == '}') { take(); --depth; return true; } @@ -2003,10 +2014,6 @@ private: }; int ParseValue(Value* out, Parser& parser) { - if (!out->parent) { - const auto status = EnsureAuxiliary(*out); - if (status) return status; - } const auto failures = AllocationFailures(); bool parsedOk = false; try { diff --git a/core/libs/tests/GuestJsonParserState.cpp b/core/libs/tests/GuestJsonParserState.cpp index 892db3a6a9..db5079181d 100644 --- a/core/libs/tests/GuestJsonParserState.cpp +++ b/core/libs/tests/GuestJsonParserState.cpp @@ -28,33 +28,69 @@ const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); static void Require(bool valid, const char* message) { if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } } +struct ParserTrace : std::map { + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::size_t requests = 0; +}; static void* APS5_VABI AllocateTracked(void*, std::size_t size, void* context) { + auto& trace = *static_cast(context); + ++trace.requests; + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; void* memory = std::malloc(size); Require(memory != nullptr, "Tracking allocator requires host memory"); - static_cast*>(context)->emplace(memory, size); + static_cast(context)->emplace(memory, size); return memory; } static void APS5_VABI FreeTracked(void*, void* memory, void* context) { if (!memory) return; - Require(static_cast*>(context)->erase(memory) == 1, "Parser allocations must be freed exactly once by their owning allocator"); + Require(static_cast(context)->erase(memory) == 1, "Parser allocations must be freed exactly once by their owning allocator"); std::free(memory); } +static void APS5_VABI NotifyTracked(void*, int error, std::size_t, void* context) { + Require(static_cast(error) == 0x80848102, "Parser allocation failures must report the Json2 out-of-memory error"); + ++static_cast(context)->notifications; +} static int Parse(Value& value, const char* text) { return _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, text, std::strlen(text)); } int main() { JsonTest::Allocator allocator; - void* const vtable[]{nullptr, nullptr, reinterpret_cast(AllocateTracked), reinterpret_cast(FreeTracked), reinterpret_cast(JsonTest::Notify)}; + void* const vtable[]{nullptr, nullptr, reinterpret_cast(AllocateTracked), reinterpret_cast(FreeTracked), reinterpret_cast(NotifyTracked)}; allocator.vtable = vtable; - std::map live; + ParserTrace live; Parameter parameter{&allocator, &live, 512}; std::uint64_t initializer[2]{}; _ZN3sce4Json11InitializerC1Ev(initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(initializer, ¶meter) == 0, "Initialize parser state tests"); const auto baseline = live.size(); + struct FailureCase { bool once; int budget; int type; unsigned notifications; std::size_t requests; const char* text; }; + const FailureCase failures[]{ + {false, 0, 0, 2, 2, ""}, {false, 1, 0, 2, 3, ""}, + {false, 2, 5, 1, 3, ""}, {false, 3, 5, 1, 4, ""}, + {false, 4, 5, 9, 13, ""}, {false, 5, 5, 0, 5, "abcdefghi"}, + {true, 0, 0, 1, 3, ""}, {true, 1, 0, 1, 4, ""}, + {true, 2, 5, 1, 3, ""}, {true, 3, 5, 1, 4, ""}, + {true, 4, 5, 1, 6, "bcdefghi"}, {true, 5, 5, 0, 5, "abcdefghi"}, + }; + for (const auto& failure : failures) { + Value partial{}; _ZN3sce4Json5ValueC1El(&partial, 77); + const auto notifications = live.notifications; + const auto requests = live.requests; + live.budget = failure.budget; live.once = failure.once; + const auto status = static_cast(Parse(partial, "\"abcdefghi\"")); + live.budget = -1; live.once = false; + Require(status == (failure.notifications ? 0x80848102u : 0u) && partial.type == failure.type, "String parser allocation failures must retain the reference type and status"); + Require(live.notifications - notifications == failure.notifications && live.requests - requests == failure.requests, "String parser failures must follow storage-before-metadata admission and character retries"); + if (partial.type == 5) Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(partial.payload)), failure.text) == 0, "Failed string parsing must retain the admitted character suffix"); + _ZN3sce4Json5ValueD1Ev(&partial); + Require(live.size() == baseline, "String parser failure must reclaim partial storage and metadata"); + } for (const char* text : {"", "?", "tru"}) { Value value{}; _ZN3sce4Json5ValueC1El(&value, 77); - Require(static_cast(Parse(value, text)) == 0x80848101 && value.type == 2 && value.payload == 77, "Unrecognized or incomplete scalar input must preserve the previous payload"); + Require(static_cast(Parse(value, text)) == 0x80848101 && value.type == 2 && value.payload == 77 && !value.auxiliary, "Unrecognized or incomplete scalar input must preserve the previous payload"); _ZN3sce4Json5ValueD1Ev(&value); } for (const char* text : {"[1,", "[1,?]"}) { diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 6504410711..a793901534 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -119,3 +119,5 @@ - [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. - [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copies, alternate SDK behavior and concurrent failure accounting remain unverified. + +- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. Parser array/object low-memory paths, inherited allocation-error scopes and child-metadata rebinding remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. From 9c5c3de534ee098d342296d0e4d16176172ca19f Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:00:06 +0200 Subject: [PATCH 028/318] fix(libSceJson2): preserve nested parser allocation failures --- core/libs/prx/libSceJson2/Export.cpp | 135 +++++++++++++++-------- core/libs/tests/GuestJsonParserState.cpp | 52 +++++++++ docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 145 insertions(+), 46 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index aa59aa3480..6d939d85f5 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -158,6 +158,8 @@ struct Globals { std::recursive_mutex callbackMutex; bool callbackActive = false; std::uint64_t allocationFailures = 0; + std::uint32_t allocationDepth = 0; + bool allocationFailed = false; }; Globals& State() { @@ -175,6 +177,36 @@ struct KernelMutexLock { KernelMutexLock& operator=(const KernelMutexLock&) = delete; }; +class AllocationOperation { +public: + AllocationOperation() { + auto& state = State(); + std::lock_guard lock(state.mutex); + KernelMutexLock kernelLock(state.internalMutex); + ++state.allocationDepth; + } + ~AllocationOperation() { if (active) Finish(); } + AllocationOperation(const AllocationOperation&) = delete; + AllocationOperation& operator=(const AllocationOperation&) = delete; + static bool Failed() { + auto& state = State(); + std::lock_guard lock(state.mutex); + KernelMutexLock kernelLock(state.internalMutex); + return state.allocationFailed; + } + bool Finish() { + auto& state = State(); + std::lock_guard lock(state.mutex); + KernelMutexLock kernelLock(state.internalMutex); + const bool failed = state.allocationFailed; + if (--state.allocationDepth == 0) state.allocationFailed = false; + active = false; + return failed; + } +private: + bool active = true; +}; + struct CallbackScope { Globals& state = State(); std::unique_lock lock{state.callbackMutex}; @@ -201,6 +233,7 @@ void* Allocate(const AllocationContext& context, std::size_t size) { std::lock_guard lock(state.mutex); KernelMutexLock kernelLock(state.internalMutex); ++state.allocationFailures; + if (state.allocationDepth) state.allocationFailed = true; } return memory; } @@ -1779,6 +1812,31 @@ Value& ObjectEntry(Object& object, std::string_view input) { return *value; } +void AppendArray(Array* self, const Value* value) { + if (!self->items || !self->items->head) return; + AllocationOperation operation; + if (!ModernValues()) { + try { CopyInto(self->items->emplace_back(), *value); } + catch (const GuestOutOfMemory&) {} + if (operation.Finish()) self->items->PopBack(); + return; + } + Value copy{}; + copy.parent = self->items->Owner(); + try { + AssignNode(copy, *value); + auto& inserted = self->items->emplace_back(); + inserted.parent = self->items->Owner(); + TransferPayload(inserted, copy); + } catch (const GuestOutOfMemory&) { + } catch (...) { + Destroy(copy); + throw; + } + Destroy(copy); + if (operation.Finish()) self->items->PopBack(); +} + using ParserReadCallback = int (APS5_VABI*)(char&, void*); class Parser { @@ -1787,7 +1845,6 @@ public: Parser(ParserReadCallback read, void* context) : read(read), context(context) {} bool Parse(Node& out) { - failures = AllocationFailures(); skip(); return value(out); } @@ -1802,7 +1859,6 @@ private: ParserReadCallback read = nullptr; void* context = nullptr; std::size_t depth = 0; - std::uint64_t failures = 0; int current = -2; int readError = 0; @@ -1897,32 +1953,39 @@ private: if (!out.parent) EnsureAuxiliary(out); } bool number(Node& out) { - std::string token; - if (peek() >= 0 && peek() == '-') token.push_back(static_cast(take())); + ScopedString token{std::string_view{}}; + auto append = [&](char character) { AppendText(token.string, {&character, 1}); }; + if (peek() >= 0 && peek() == '-') append(static_cast(take())); if (peek() < 0 || peek() < '0' || peek() > '9') return false; - while (peek() >= '0' && peek() <= '9') token.push_back(static_cast(take())); + while (peek() >= '0' && peek() <= '9') append(static_cast(take())); bool integral = true; if (peek() >= 0 && peek() == '.') { integral = false; - token.push_back(static_cast(take())); + append(static_cast(take())); if (peek() < 0 || peek() < '0' || peek() > '9') return false; - while (peek() >= 0 && peek() >= '0' && peek() <= '9') token.push_back(static_cast(take())); + while (peek() >= 0 && peek() >= '0' && peek() <= '9') append(static_cast(take())); } if (peek() >= 0 && (peek() == 'e' || peek() == 'E')) { integral = false; - token.push_back(static_cast(take())); - if (peek() >= 0 && (peek() == '+' || peek() == '-')) token.push_back(static_cast(take())); + append(static_cast(take())); + if (peek() >= 0 && (peek() == '+' || peek() == '-')) append(static_cast(take())); if (peek() < 0 || peek() < '0' || peek() > '9') return false; - while (peek() >= 0 && peek() >= '0' && peek() <= '9') token.push_back(static_cast(take())); + while (peek() >= 0 && peek() >= '0' && peek() <= '9') append(static_cast(take())); + } + const auto text = View(token.string); + if (text.empty()) { + SetType(out, TypeUInteger); + out.uinteger = 0; + Admit(out); + return true; } - const std::string& text = token; if (integral) { errno = 0; if (text[0] == '-') { - const long long value = std::strtoll(text.c_str(), nullptr, 10); + const long long value = std::strtoll(Characters(token.string), nullptr, 10); if (errno == 0) { SetType(out, TypeInteger); out.integer = value; Admit(out); return true; } } else { - const unsigned long long value = std::strtoull(text.c_str(), nullptr, 10); + const unsigned long long value = std::strtoull(Characters(token.string), nullptr, 10); if (errno == 0) { SetType(out, TypeUInteger); out.uinteger = value; @@ -1935,19 +1998,19 @@ private: const auto parsed = std::from_chars(text.data(), text.data() + text.size(), real); bool converted = parsed.ec == std::errc{} && parsed.ptr == text.data() + text.size(); if (parsed.ec == std::errc::result_out_of_range) { - std::istringstream stream(text); + std::istringstream stream(std::string{text}); stream.imbue(std::locale::classic()); stream >> real; converted = !stream.fail() && stream.eof(); } - if (!converted || !std::isfinite(real)) throw std::runtime_error("sce::Json::Parser::parse: number " + text + " overflows a double; the console's behaviour is unverified"); + if (!converted || !std::isfinite(real)) throw std::runtime_error("sce::Json::Parser::parse: number " + std::string{text} + " overflows a double; the console's behaviour is unverified"); SetType(out, TypeReal); out.real = real; Admit(out); return true; } bool value(Node& out) { - if (peek() < 0) return false; + if (AllocationOperation::Failed() || peek() < 0) return false; switch (peek()) { case 'n': if (!literal("null")) return false; SetType(out, TypeNull); Admit(out); return true; case 't': if (!literal("true")) return false; SetType(out, TypeBoolean); out.boolean = true; Admit(out); return true; @@ -1956,7 +2019,7 @@ private: SetType(out, TypeString); Admit(out); if (out.type != TypeString) throw GuestOutOfMemory{}; - if (AllocationFailures() != failures) return false; + if (AllocationOperation::Failed()) return false; ParsedText text{*out.string}; return string(text); } @@ -1971,8 +2034,11 @@ private: if (peek() >= 0 && peek() == ']') { take(); --depth; return true; } for (;;) { if (!out.array->items) throw GuestOutOfMemory{}; - auto& item = out.array->items->emplace_back(); - Construct(item); + if (!out.array->items->head) NotImplemented_nid_no_patch(__func__); + Value placeholder{}; + AppendArray(out.array, &placeholder); + if (AllocationOperation::Failed()) return false; + auto& item = out.array->items->back(); item.parent = &out; skip(); if (!value(item)) return false; @@ -2008,20 +2074,22 @@ private: } } default: + if (peek() != '-' && (peek() < '0' || peek() > '9')) return false; return number(out); } } }; int ParseValue(Value* out, Parser& parser) { - const auto failures = AllocationFailures(); + AllocationOperation operation; bool parsedOk = false; try { parsedOk = parser.Parse(NodeOf(*out)); } catch (const GuestOutOfMemory&) { + operation.Finish(); return JsonErrorOutOfMemory; } - if (AllocationFailures() != failures) return JsonErrorOutOfMemory; + if (operation.Finish()) return JsonErrorOutOfMemory; return parsedOk ? 0 : parser.Error(); } @@ -2180,30 +2248,7 @@ void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array* self) { Construct(*self); } void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array* self, const Array* other) { CopyArray(*self, *other); } void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array* self) { Destroy(*self); } Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array* self, const Array* other) { AssignArray(*self, *other); return self; } -void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array* self, const Value* value) { - if (!self->items || !self->items->head) return; - const auto failures = AllocationFailures(); - if (!ModernValues()) { - try { CopyInto(self->items->emplace_back(), *value); } - catch (const GuestOutOfMemory&) {} - if (AllocationFailures() != failures) self->items->PopBack(); - return; - } - Value copy{}; - copy.parent = self->items->Owner(); - try { - AssignNode(copy, *value); - auto& inserted = self->items->emplace_back(); - inserted.parent = self->items->Owner(); - TransferPayload(inserted, copy); - } catch (const GuestOutOfMemory&) { - } catch (...) { - Destroy(copy); - throw; - } - Destroy(copy); - if (AllocationFailures() != failures) self->items->PopBack(); -} +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array* self, const Value* value) { AppendArray(self, value); } ArrayIterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(ArrayIterator* result, const Array* self) { result->state = static_cast(Allocate(sizeof(ArrayIteratorState))); diff --git a/core/libs/tests/GuestJsonParserState.cpp b/core/libs/tests/GuestJsonParserState.cpp index db5079181d..e6b6edb5bb 100644 --- a/core/libs/tests/GuestJsonParserState.cpp +++ b/core/libs/tests/GuestJsonParserState.cpp @@ -88,6 +88,58 @@ int main() { _ZN3sce4Json5ValueD1Ev(&partial); Require(live.size() == baseline, "String parser failure must reclaim partial storage and metadata"); } + struct ArrayFailure { bool once; int budget; int type; std::size_t count; std::uint64_t number; int lastType; const char* text; unsigned notifications; std::size_t requests; }; + const ArrayFailure arrayFailures[]{ + {0, 0, 0, 0, 0, 0, "", 2, 2}, + {0, 1, 0, 0, 0, 0, "", 2, 3}, + {0, 3, 6, 0, 0, 0, "", 2, 5}, + {0, 4, 6, 0, 0, 0, "", 2, 6}, + {0, 5, 6, 0, 0, 0, "", 1, 6}, + {0, 6, 6, 0, 0, 0, "", 2, 8}, + {0, 7, 6, 0, 0, 0, "", 2, 9}, + {0, 8, 6, 0, 0, 0, "", 1, 9}, + {0, 9, 6, 2, 1, 0, "", 1, 10}, + {0, 10, 6, 2, 1, 0, "", 1, 11}, + {0, 11, 6, 2, 1, 5, "", 3, 14}, + {0, 12, 6, 2, 1, 5, "abc", 0, 12}, + {1, 0, 0, 0, 0, 0, "", 1, 3}, + {1, 1, 0, 0, 0, 0, "", 1, 4}, + {1, 3, 6, 0, 0, 0, "", 1, 5}, + {1, 4, 6, 0, 0, 0, "", 1, 6}, + {1, 5, 6, 0, 0, 0, "", 1, 6}, + {1, 6, 6, 1, 0, 0, "", 1, 8}, + {1, 7, 6, 1, 0, 0, "", 1, 9}, + {1, 8, 6, 0, 0, 0, "", 1, 9}, + {1, 9, 6, 2, 1, 0, "", 1, 10}, + {1, 10, 6, 2, 1, 0, "", 1, 11}, + {1, 11, 6, 2, 1, 5, "bc", 1, 13}, + {1, 12, 6, 2, 1, 5, "abc", 0, 12}, + }; + for (const auto& failure : arrayFailures) { + Value partial{}; _ZN3sce4Json5ValueC1El(&partial, 77); + const auto notifications = live.notifications; + const auto requests = live.requests; + live.budget = failure.budget; live.once = failure.once; + const auto status = static_cast(Parse(partial, "[1,\"abc\"]")); + live.budget = -1; live.once = false; + Require(status == (failure.notifications ? 0x80848102u : 0u) && partial.type == failure.type, "Array parser allocation failures must retain the reference status and type"); + Require(live.notifications - notifications == failure.notifications && live.requests - requests == failure.requests, "Array parser failures must account for numeric-token storage and inherited append errors"); + if (partial.type == 6) { + Require(_ZNK3sce4Json5Value5countEv(&partial) == failure.count, "Array parser failures must retain the reference partial count"); + if (failure.count) { + const auto* array = reinterpret_cast(partial.payload); + const auto* first = _ZNK3sce4Json5Array5frontEv(array); + const auto* last = _ZNK3sce4Json5Array4backEv(array); + Require(first->type == 3 && first->payload == failure.number && first->parent == &partial && last->parent == &partial, "Array parser failures must retain numeric values and admitted parent links"); + if (failure.count == 2) { + Require(last->type == failure.lastType, "Array parser failures must preserve the final admitted value type"); + if (last->type == 5) Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(last->payload)), failure.text) == 0, "Array parser failures must retain the admitted string suffix"); + } + } + } + _ZN3sce4Json5ValueD1Ev(&partial); + Require(live.size() == baseline, "Array parser failure must reclaim all admitted resources"); + } for (const char* text : {"", "?", "tru"}) { Value value{}; _ZN3sce4Json5ValueC1El(&value, 77); Require(static_cast(Parse(value, text)) == 0x80848101 && value.type == 2 && value.payload == 77 && !value.auxiliary, "Unrecognized or incomplete scalar input must preserve the previous payload"); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index a793901534..246b757fee 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -120,4 +120,6 @@ - [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copies, alternate SDK behavior and concurrent failure accounting remain unverified. -- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. Parser array/object low-memory paths, inherited allocation-error scopes and child-metadata rebinding remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. +- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. Parser object low-memory paths, remaining inherited allocation-error scopes and child-metadata rebinding remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. + +- [sce::Json::Parser array allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - numeric token storage uses the supplied allocator; array parsing uses the public append implementation. Parser and append operations share the depth/error flag observed at `0x6840`, `0x310`, `0x340` and `0xb880`, retaining inherited errors until the outer operation finishes. Twenty-four reference-backed cases cover persistent and one-shot failures through successful parsing, partial counts, numeric-zero fallback after token-allocation failure, string suffixes, parent links, notifications and cleanup. Missing-sentinel parsing throws unsupported because reference execution crashes; other collection/callback operation scopes and concurrent behavior remain incomplete. From d7d06f79ace5814f03ff3e2f53206f0d26fffae3 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:03:31 +0200 Subject: [PATCH 029/318] fix(libSceJson2): preserve object parser allocation failures --- core/libs/prx/libSceJson2/Export.cpp | 21 ++-- core/libs/tests/GuestJsonObjectAllocator.cpp | 107 +++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 118 insertions(+), 14 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 6d939d85f5..33dd51372f 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1787,7 +1787,7 @@ void AssignObject(Object& destination, const Object& source) { Value* ObjectAccess(Object& object, const String& input) { if (!object.items || !input.text) return const_cast(&Defaults(TypeNull)); if (!object.items->head) NotImplemented_nid_no_patch(__func__); - const auto failures = AllocationFailures(); + AllocationOperation operation; ObjectLink* node; { InlineText key; key.Assign(Characters(input)); @@ -1797,21 +1797,13 @@ Value* ObjectAccess(Object& object, const String& input) { bool inserted; node = object.items->Insert(entry.key.text, entry.pair, inserted); } - if (AllocationFailures() != failures) return const_cast(&Defaults(TypeNull)); + if (operation.Finish()) return const_cast(&Defaults(TypeNull)); if (!node || node->nil) NotImplemented_nid_no_patch(__func__); AssignText(node->pair.key, View(input), true); node->pair.value.parent = object.items->Owner(); return &node->pair.value; } -Value& ObjectEntry(Object& object, std::string_view input) { - const auto failures = AllocationFailures(); - ScopedString key(input); - auto* value = ObjectAccess(object, key.string); - if (AllocationFailures() != failures) throw GuestOutOfMemory{}; - return *value; -} - void AppendArray(Array* self, const Value* value) { if (!self->items || !self->items->head) return; AllocationOperation operation; @@ -1915,6 +1907,7 @@ private: }; template bool string(Output& out) { + if (AllocationOperation::Failed()) return false; if (peek() < 0 || peek() != '"') return false; take(); while (peek() >= 0 && peek() != '"') { @@ -2057,16 +2050,18 @@ private: if (out.type != TypeObject) throw GuestOutOfMemory{}; skip(); if (peek() >= 0 && peek() == '}') { take(); --depth; return true; } + ScopedString key{std::string_view{}}; for (;;) { skip(); - std::string key; - if (!string(key)) return false; + ParsedText text{key.string}; + if (!string(text)) return false; skip(); if (peek() < 0 || peek() != ':') return false; take(); skip(); - Value& entry = ObjectEntry(*out.object, key); + Value& entry = *ObjectAccess(*out.object, key.string); if (!value(entry)) return false; + ClearText(key.string); skip(); if (peek() >= 0 && peek() == ',') { take(); continue; } if (peek() >= 0 && peek() == '}') { take(); --depth; return true; } diff --git a/core/libs/tests/GuestJsonObjectAllocator.cpp b/core/libs/tests/GuestJsonObjectAllocator.cpp index 0495c98c1e..281fb28e12 100644 --- a/core/libs/tests/GuestJsonObjectAllocator.cpp +++ b/core/libs/tests/GuestJsonObjectAllocator.cpp @@ -397,6 +397,112 @@ static std::string NestedObjectFailureState(const Value& value, const Value* par } return state + "]"; } +static void CheckParserObjectFailures(ObjectAllocationTrace& trace) { + static const char* const states[]{ + "0p", + "7po0[]", + "7po1[~=0-;]", + "7po1[=0p;]", + "7po1[a=0p;]", + "7po1[a=5p:;]", + "7po1[a=5p:abc;]", + "7po2[a=5p:abc;~=0-;]", + "7po2[a=5p:abc;=0p;]", + "7po2[a=5p:abc;b=0p;]", + "7po2[a=5p:abc;b=6po0[];]", + "7po2[a=5p:abc;b=6po1[3p:0;];]", + "7po2[a=5p:abc;b=6po1[3p:2;];]", + "7po1[=0-;]", + "7po1[a=5p:bc;]", + "7po2[=0-;a=5p:abc;]", + "7po2[a=5p:abc;=0-;]", + }; + struct Case { bool once; int budget; unsigned notifications; std::size_t requests; unsigned state; }; + const Case cases[]{ + {0, 0, 2, 2, 0}, + {0, 1, 2, 3, 0}, + {0, 2, 2, 4, 0}, + {0, 3, 2, 5, 1}, + {0, 4, 2, 6, 1}, + {0, 5, 1, 6, 1}, + {0, 6, 3, 9, 1}, + {0, 7, 3, 10, 1}, + {0, 8, 3, 11, 1}, + {0, 9, 3, 12, 1}, + {0, 10, 2, 12, 1}, + {0, 11, 1, 12, 1}, + {0, 12, 2, 14, 2}, + {0, 13, 1, 14, 2}, + {0, 14, 1, 15, 3}, + {0, 15, 1, 16, 4}, + {0, 16, 1, 17, 4}, + {0, 17, 3, 20, 5}, + {0, 18, 3, 21, 6}, + {0, 19, 3, 22, 6}, + {0, 20, 3, 23, 6}, + {0, 21, 3, 24, 6}, + {0, 22, 2, 24, 6}, + {0, 23, 1, 24, 6}, + {0, 24, 2, 26, 7}, + {0, 25, 1, 26, 7}, + {0, 26, 1, 27, 8}, + {0, 27, 1, 28, 9}, + {0, 28, 1, 29, 9}, + {0, 30, 1, 31, 10}, + {0, 31, 1, 32, 11}, + {0, 32, 1, 33, 11}, + {0, 33, 0, 33, 12}, + {1, 0, 1, 3, 0}, + {1, 1, 1, 4, 0}, + {1, 2, 1, 5, 0}, + {1, 3, 1, 5, 1}, + {1, 4, 1, 6, 1}, + {1, 5, 1, 6, 1}, + {1, 6, 1, 11, 13}, + {1, 7, 1, 12, 13}, + {1, 8, 1, 12, 2}, + {1, 9, 1, 13, 13}, + {1, 10, 1, 13, 2}, + {1, 11, 1, 12, 1}, + {1, 12, 1, 14, 13}, + {1, 13, 1, 14, 2}, + {1, 14, 1, 15, 3}, + {1, 15, 1, 16, 4}, + {1, 16, 1, 17, 4}, + {1, 17, 1, 19, 14}, + {1, 18, 1, 23, 15}, + {1, 19, 1, 24, 15}, + {1, 20, 1, 24, 7}, + {1, 21, 1, 25, 15}, + {1, 22, 1, 25, 7}, + {1, 23, 1, 24, 6}, + {1, 24, 1, 26, 16}, + {1, 25, 1, 26, 7}, + {1, 26, 1, 27, 8}, + {1, 27, 1, 28, 9}, + {1, 28, 1, 29, 9}, + {1, 30, 1, 31, 10}, + {1, 31, 1, 32, 11}, + {1, 32, 1, 33, 11}, + {1, 33, 0, 33, 12}, + }; + const auto baseline = trace.live.size(); + for (const auto& test : cases) { + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); _ZN3sce4Json5Value3setEl(&value, 77); + const auto notifications = trace.notifications; + const auto requests = trace.requests.size(); + const char input[] = "{\"a\":\"abc\",\"b\":[2]}"; + trace.budget = test.budget; trace.failOnce = test.once; + const auto status = static_cast(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, input, sizeof(input) - 1)); + trace.budget = -1; trace.failOnce = false; + const auto actual = NestedObjectFailureState(value); + const bool matches = status == (test.notifications ? 0x80848102u : 0u) && actual == states[test.state] && trace.notifications - notifications == test.notifications && trace.requests.size() - requests == test.requests; + if (!matches) std::fprintf(stderr, "Object parse once=%d budget=%d expected=%s actual=%s status=%x requests=%zu notifications=%u\n", test.once, test.budget, states[test.state], actual.c_str(), status, trace.requests.size() - requests, trace.notifications - notifications); + Require(matches, "Object parser failures must retain reference partial members, ownership and allocation accounting"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Object parser failures must reclaim every partial key, member and metadata allocation"); + } +} static void CheckNestedFailureCases(ObjectAllocationTrace& trace) { static const char* const states[]{ "7po1[z=3p:99;]", @@ -996,6 +1102,7 @@ int main() { CheckAssignments(trace); CheckNestedAssignment(trace); CheckNestedFailureCases(trace); + CheckParserObjectFailures(trace); CheckBinaryKeys(); Require(trace.live.size() == baseline, "Binary-key operations must release their storage"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate object allocation contract"); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 246b757fee..b375048db4 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -120,6 +120,8 @@ - [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copies, alternate SDK behavior and concurrent failure accounting remain unverified. -- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. Parser object low-memory paths, remaining inherited allocation-error scopes and child-metadata rebinding remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. +- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. More complex parser low-memory paths, remaining inherited allocation-error scopes and child-metadata rebinding remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. - [sce::Json::Parser array allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - numeric token storage uses the supplied allocator; array parsing uses the public append implementation. Parser and append operations share the depth/error flag observed at `0x6840`, `0x310`, `0x340` and `0xb880`, retaining inherited errors until the outer operation finishes. Twenty-four reference-backed cases cover persistent and one-shot failures through successful parsing, partial counts, numeric-zero fallback after token-allocation failure, string suffixes, parent links, notifications and cleanup. Missing-sentinel parsing throws unsupported because reference execution crashes; other collection/callback operation scopes and concurrent behavior remain incomplete. + +- [sce::Json::Parser object allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - object keys use one allocator-backed string reused between members, following `0x6250`; lookup closes its shared operation scope before assigning the exposed key and owner, following `0xda40`. Sixty-six reference-backed cases cover persistent and one-shot failure at each returning allocation position through a 33-allocation successful parse of an object containing a string and array. Checks include partial keys/members, numeric and string fallback values, parent links, error status, request/notification counts and complete allocator cleanup. The nested array sentinel failure is excluded because reference execution crashes; more complex nested objects, callback-driven allocation failures and concurrency remain unverified. From e0f06a04771d1c52bfe22a2607d6a950d7e281a9 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:08:38 +0200 Subject: [PATCH 030/318] fix(libSceJson2): preserve inherited collection allocation errors --- core/libs/prx/libSceJson2/Export.cpp | 27 ++++----- core/libs/tests/GuestJsonObjectAllocator.cpp | 59 ++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 8 ++- 3 files changed, 74 insertions(+), 20 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 33dd51372f..8be4b4b071 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -157,7 +157,6 @@ struct Globals { PthreadMutex* recoveryMutex = nullptr; std::recursive_mutex callbackMutex; bool callbackActive = false; - std::uint64_t allocationFailures = 0; std::uint32_t allocationDepth = 0; bool allocationFailed = false; }; @@ -232,7 +231,6 @@ void* Allocate(const AllocationContext& context, std::size_t size) { auto& state = State(); std::lock_guard lock(state.mutex); KernelMutexLock kernelLock(state.internalMutex); - ++state.allocationFailures; if (state.allocationDepth) state.allocationFailed = true; } return memory; @@ -345,13 +343,6 @@ bool ModernValues() { return sdk >= 0x06000000 || (sdk >= 0x05508700 && sdk < 0x05510000); } -std::uint64_t AllocationFailures() { - auto& state = State(); - std::lock_guard lock(state.mutex); - KernelMutexLock kernelLock(state.internalMutex); - return state.allocationFailures; -} - bool Grow(Text& text, std::size_t requested) { if (requested > std::numeric_limits::max()) throw std::length_error("sce::Json::String: capacity exceeds 32 bits"); @@ -921,9 +912,9 @@ int RegisterLegacyHandler(Value& value, TypeMismatchHandler callback, void* cont map->head->color = 0; map->head->nil = 1; state.legacyHandlers = map; } - const auto failures = AllocationFailures(); + AllocationOperation operation; auto* node = state.legacyHandlers->Insert(&value); - if (AllocationFailures() != failures) return JsonErrorOutOfMemory; + if (operation.Finish()) return JsonErrorOutOfMemory; node->record = {callback, context}; return 0; } @@ -1725,10 +1716,10 @@ void AssignArray(Array& destination, const Array& source) { for (const auto& item : *source.items) { ScopedValue placeholder; SetType(placeholder.value, item.type); - const auto failures = AllocationFailures(); + AllocationOperation operation; try { CopyInto(pending.items->emplace_back(), placeholder.value); } catch (const GuestOutOfMemory&) {} - if (AllocationFailures() == failures) { + if (!operation.Finish()) { auto& value = pending.items->back(); value.parent = destination.items->Owner(); AssignNode(value, item); @@ -1765,7 +1756,7 @@ void AssignObject(Object& destination, const Object& source) { Construct(copied.pair.key, key.string); CopyInto(copied.pair.value, placeholder.value); } - const auto failures = AllocationFailures(); + AllocationOperation operation; ObjectLink* node; bool inserted; { @@ -1773,7 +1764,8 @@ void AssignObject(Object& destination, const Object& source) { TemporaryTreeEntry entry(key.text, copied.pair); node = pending.items->Insert(entry.key.text, entry.pair, inserted); } - if (inserted && AllocationFailures() == failures) { + const bool failed = operation.Finish(); + if (inserted && !failed) { node->pair.value.parent = destination.items->Owner(); AssignNode(node->pair.value, pair.value); } @@ -2789,7 +2781,7 @@ ObjectIterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(ObjectIterator ScopedValue value; value.value.parent = self->items->Owner(); AssignNode(value.value, pair->value); - const auto failures = AllocationFailures(); + AllocationOperation operation; ObjectLink* node; bool inserted; { @@ -2799,7 +2791,8 @@ ObjectIterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(ObjectIterator TemporaryTreeEntry entry(key.text, source); node = self->items->Insert(entry.key.text, entry.pair, inserted); } - if (inserted && AllocationFailures() == failures) { + const bool failed = operation.Finish(); + if (inserted && !failed) { TransferPayload(node->pair.value, value.value); node->pair.value.parent = self->items->Owner(); BindChildren(node->pair.value); diff --git a/core/libs/tests/GuestJsonObjectAllocator.cpp b/core/libs/tests/GuestJsonObjectAllocator.cpp index 281fb28e12..411221a40a 100644 --- a/core/libs/tests/GuestJsonObjectAllocator.cpp +++ b/core/libs/tests/GuestJsonObjectAllocator.cpp @@ -14,6 +14,7 @@ struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; struct String { void* storage; }; struct Object { void* storage; }; +struct Array { void* storage; }; struct Iterator { void* state; }; struct Pair { String key; std::uint64_t reserved; Value value; }; struct StoredText { const char* characters; std::uint32_t length; std::uint32_t capacity; std::uint64_t reserved; }; @@ -21,6 +22,7 @@ struct TreeNode { TreeNode* parent; TreeNode* left; TreeNode* right; StoredText struct TreeHeader { TreeNode* head; std::size_t count; std::uint64_t reserved; Value* owner; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; static_assert(sizeof(TreeNode) == 104 && offsetof(TreeNode, pair) == 48 && offsetof(TreeNode, color) == 96); +using ReadCallback = int (APS5_VABI*)(char&, void*); extern "C" { void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); @@ -34,6 +36,8 @@ std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value*); int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value*, std::size_t); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(Value*, ReadCallback, void*); void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); void APS5_VABI _ZN3sce4Json6ObjectC1ERKS1_(Object*, const Object*); void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); @@ -397,6 +401,60 @@ static std::string NestedObjectFailureState(const Value& value, const Value* par } return state + "]"; } +struct NestedReader { + unsigned operation; + Value* source; + Value* destination; + Pair* pair; + Iterator* iterator; + std::size_t offset = 0; + unsigned calls = 0; +}; +static int APS5_VABI ReadWithCollectionOperation(char& character, void* context) { + auto& reader = *static_cast(context); + if (reader.offset == 2) { + ++reader.calls; + if (reader.operation == 0) _ZN3sce4Json5ArrayaSERKS1_(reinterpret_cast(reader.destination->payload), reinterpret_cast(reader.source->payload)); + else if (reader.operation == 1) _ZN3sce4Json6ObjectaSERKS1_(reinterpret_cast(reader.destination->payload), reinterpret_cast(reader.source->payload)); + else _ZN3sce4Json6Object6insertERKNS1_4PairE(reader.iterator, reinterpret_cast(reader.destination->payload), reader.pair); + } + const char text[] = "\"abc\""; + if (reader.offset == sizeof(text) - 1) return static_cast(0x80848101u); + character = text[reader.offset++]; + return 0; +} +static void CheckInheritedCollectionErrors(ObjectAllocationTrace& trace) { + const char* const failedStates[]{"6po2[3-:0;3-:0;]", "7po2[a=3-:0;c=3-:0;]", "7po2[a=0-;z=3p:99;]"}; + const char* const healthyStates[]{"6po2[3p:10;3p:20;]", "7po2[a=3p:11;c=3p:33;]", "7po2[a=2p:11;z=3p:99;]"}; + const auto baseline = trace.live.size(); + for (unsigned operation = 0; operation < 3; ++operation) for (bool fail : {false, true}) { + Value source{}, destination{}, parsed{}; + _ZN3sce4Json5ValueC1Ev(&source); _ZN3sce4Json5ValueC1Ev(&destination); _ZN3sce4Json5ValueC1Ev(&parsed); + const auto* input = operation == 0 ? "[10,20]" : "{\"a\":11,\"c\":33}"; + const auto* previous = operation == 0 ? "[99]" : "{\"z\":99}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&source, input, std::strlen(input)) == 0 && _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&destination, previous, std::strlen(previous)) == 0, "Prepare nested operation source and destination"); + const auto sourceState = NestedObjectFailureState(source); + Pair pair{}; _ZN3sce4Json6StringC1EPKc(&pair.key, "a"); _ZN3sce4Json5Value3setEl(&pair.value, 11); + Iterator iterator{}; + NestedReader reader{operation, &source, &destination, &pair, &iterator}; + const auto notifications = trace.notifications; + trace.budget = fail ? 4 : -1; trace.failOnce = fail; + const auto status = static_cast(_ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(&parsed, ReadWithCollectionOperation, &reader)); + trace.budget = -1; trace.failOnce = false; + const auto actual = NestedObjectFailureState(destination); + const auto* expected = fail ? failedStates[operation] : healthyStates[operation]; + if (actual != expected) std::fprintf(stderr, "Nested operation=%u fail=%d expected=%s actual=%s\n", operation, fail, expected, actual.c_str()); + Require(reader.calls == 1 && actual == expected, "Nested collection operations must retain inherited errors and preserve healthy behavior"); + Require(status == (fail ? 0x80848102u : 0u) && trace.notifications - notifications == (fail ? 1u : 0u), "Nested operation completion must retain the parser failure until outer completion"); + Require(NestedObjectFailureState(source) == sourceState, "Nested failure propagation must preserve its source"); + Require(parsed.type == 5 && std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(parsed.payload)), fail ? "bc" : "abc") == 0, "Nested calls must preserve the parser character-retry result"); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&parsed, "true", 4) == 0, "Outer completion must clear inherited errors before the next independent parse"); + _ZN3sce4Json6Object8iteratorD1Ev(&iterator); + _ZN3sce4Json6StringD1Ev(&pair.key); _ZN3sce4Json5ValueD1Ev(&pair.value); + _ZN3sce4Json5ValueD1Ev(&parsed); _ZN3sce4Json5ValueD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); + Require(trace.live.size() == baseline, "Nested operation completion must reclaim parser, collection and iterator resources"); + } +} static void CheckParserObjectFailures(ObjectAllocationTrace& trace) { static const char* const states[]{ "0p", @@ -1103,6 +1161,7 @@ int main() { CheckNestedAssignment(trace); CheckNestedFailureCases(trace); CheckParserObjectFailures(trace); + CheckInheritedCollectionErrors(trace); CheckBinaryKeys(); Require(trace.live.size() == baseline, "Binary-key operations must release their storage"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate object allocation contract"); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index b375048db4..dc24c23f70 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -82,7 +82,7 @@ - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. -- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size and double overflow throw rather than inventing a result. Numeric-token validation, parser allocation failures and nested operation accounting remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. +- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size and double overflow throw rather than inventing a result. Numeric-token validation, more complex parser allocation failures and cross-thread operation behavior remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. @@ -120,8 +120,10 @@ - [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copies, alternate SDK behavior and concurrent failure accounting remain unverified. -- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. More complex parser low-memory paths, remaining inherited allocation-error scopes and child-metadata rebinding remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. +- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. More complex parser low-memory paths, callback-driven allocation failures and child-metadata rebinding remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. -- [sce::Json::Parser array allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - numeric token storage uses the supplied allocator; array parsing uses the public append implementation. Parser and append operations share the depth/error flag observed at `0x6840`, `0x310`, `0x340` and `0xb880`, retaining inherited errors until the outer operation finishes. Twenty-four reference-backed cases cover persistent and one-shot failures through successful parsing, partial counts, numeric-zero fallback after token-allocation failure, string suffixes, parent links, notifications and cleanup. Missing-sentinel parsing throws unsupported because reference execution crashes; other collection/callback operation scopes and concurrent behavior remain incomplete. +- [sce::Json::Parser array allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - numeric token storage uses the supplied allocator; array parsing uses the public append implementation. Parser and append operations share the depth/error flag observed at `0x6840`, `0x310`, `0x340` and `0xb880`, retaining inherited errors until the outer operation finishes. Twenty-four reference-backed cases cover persistent and one-shot failures through successful parsing, partial counts, numeric-zero fallback after token-allocation failure, string suffixes, parent links, notifications and cleanup. Missing-sentinel parsing throws unsupported because reference execution crashes; callback-bearing collections and concurrent behavior remain incomplete. - [sce::Json::Parser object allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - object keys use one allocator-backed string reused between members, following `0x6250`; lookup closes its shared operation scope before assigning the exposed key and owner, following `0xda40`. Sixty-six reference-backed cases cover persistent and one-shot failure at each returning allocation position through a 33-allocation successful parse of an object containing a string and array. Checks include partial keys/members, numeric and string fallback values, parent links, error status, request/notification counts and complete allocator cleanup. The nested array sentinel failure is excluded because reference execution crashes; more complex nested objects, callback-driven allocation failures and concurrency remain unverified. + +- [sce::Json::Collection inherited allocation errors](../../../core/libs/prx/libSceJson2/Export.cpp) - array assignment, object assignment/insertion and legacy mismatch-map updates now use the shared depth/error protocol instead of counter snapshots, following `0xbc80`, `0xd400`, `0xcd30` and `0x3b00`. Six reference-backed public parser/read-callback cases invoke array assignment, object assignment or insertion after a one-shot character allocation failure, plus healthy controls. Checks cover placeholder retention, skipped payload binding, unchanged source, outer error persistence, recovery on the next independent parse and complete cleanup. Baseline and targeted assignment/insertion mutations fail these checks. Nested legacy map updates, callback-return cloning and cross-thread failure propagation remain unverified. From 9b53fff8dd6c6e2be0c9ae21eee1e5580771ad76 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:16:52 +0200 Subject: [PATCH 031/318] fix(libSceJson2): integrate merged pending export inventory --- .../libs/prx/libSceJson2/InventoryExports.cpp | 240 +++++------------- core/libs/tests/GuestJsonArrayAllocator.cpp | 23 +- docs/dev/debt/libSceJson2.md | 151 ++++++----- 3 files changed, 160 insertions(+), 254 deletions(-) diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index 5c89743172..20cbf74dd5 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -5,448 +5,344 @@ extern "C" { -int APS5_VABI sceJson2Unknown_001_nid_no_patch() { +int APS5_VABI _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_", sceJson2Unknown_001_nid_no_patch); -int APS5_VABI sceJson2Unknown_002_nid_no_patch() { +int APS5_VABI _ZN3sce4Json11InitializerC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json11Initializer28setGlobalSpecialFloatHandlerEPFKNS0_5ValueENS0_12FunctionTypeEdPS3_PvPbES6_", sceJson2Unknown_002_nid_no_patch); -int APS5_VABI sceJson2Unknown_003_nid_no_patch() { +int APS5_VABI _ZN3sce4Json11InitializerD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json11Initializer28setGlobalTypeMismatchHandlerEPFRKNS0_5ValueENS0_9ValueTypeEPS3_S6_PvES7_", sceJson2Unknown_003_nid_no_patch); -int APS5_VABI sceJson2Unknown_004_nid_no_patch() { +int APS5_VABI _ZN3sce4Json12MemAllocatorD0Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json11Initializer36setGlobalElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKNS0_5ValueES9_PvESA_", sceJson2Unknown_004_nid_no_patch); -int APS5_VABI sceJson2Unknown_005_nid_no_patch() { +int APS5_VABI _ZN3sce4Json12MemAllocatorD1Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json11InitializerC2Ev", sceJson2Unknown_005_nid_no_patch); -int APS5_VABI sceJson2Unknown_006_nid_no_patch() { +int APS5_VABI _ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json11InitializerD2Ev", sceJson2Unknown_006_nid_no_patch); -int APS5_VABI sceJson2Unknown_007_nid_no_patch() { +int APS5_VABI _ZN3sce4Json19InternalInitializer19terminate_staticlibEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json12MemAllocatorD0Ev", sceJson2Unknown_007_nid_no_patch); -int APS5_VABI sceJson2Unknown_008_nid_no_patch() { +int APS5_VABI _ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json12MemAllocatorD1Ev", sceJson2Unknown_008_nid_no_patch); -int APS5_VABI sceJson2Unknown_009_nid_no_patch() { +int APS5_VABI _ZN3sce4Json19InternalInitializerC1Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm", sceJson2Unknown_009_nid_no_patch); -int APS5_VABI sceJson2Unknown_010_nid_no_patch() { +int APS5_VABI _ZN3sce4Json19InternalInitializerC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json19InternalInitializer19terminate_staticlibEv", sceJson2Unknown_010_nid_no_patch); -int APS5_VABI sceJson2Unknown_011_nid_no_patch() { +int APS5_VABI _ZN3sce4Json19InternalInitializerD1Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE", sceJson2Unknown_011_nid_no_patch); -int APS5_VABI sceJson2Unknown_012_nid_no_patch() { +int APS5_VABI _ZN3sce4Json19InternalInitializerD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json19InternalInitializerC1Ev", sceJson2Unknown_012_nid_no_patch); -int APS5_VABI sceJson2Unknown_013_nid_no_patch() { +int APS5_VABI _ZN3sce4Json4FreeEPv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json19InternalInitializerC2Ev", sceJson2Unknown_013_nid_no_patch); -int APS5_VABI sceJson2Unknown_014_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json19InternalInitializerD1Ev", sceJson2Unknown_014_nid_no_patch); -int APS5_VABI sceJson2Unknown_015_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json19InternalInitializerD2Ev", sceJson2Unknown_015_nid_no_patch); -int APS5_VABI sceJson2Unknown_016_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json4FreeEPv", sceJson2Unknown_016_nid_no_patch); -int APS5_VABI sceJson2Unknown_017_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Array8iteratorC2ERKS2_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Array10push_frontERKNS0_5ValueE", sceJson2Unknown_017_nid_no_patch); -int APS5_VABI sceJson2Unknown_018_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Array8iteratorC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Array5eraseERKNS1_8iteratorE", sceJson2Unknown_018_nid_no_patch); -int APS5_VABI sceJson2Unknown_019_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Array8iteratorD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE", sceJson2Unknown_019_nid_no_patch); -int APS5_VABI sceJson2Unknown_020_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Array8pop_backEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Array8iteratorC2ERKS2_", sceJson2Unknown_020_nid_no_patch); -int APS5_VABI sceJson2Unknown_021_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ArrayC2ERKS1_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Array8iteratorC2Ev", sceJson2Unknown_021_nid_no_patch); -int APS5_VABI sceJson2Unknown_022_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ArrayC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Array8iteratorD2Ev", sceJson2Unknown_022_nid_no_patch); -int APS5_VABI sceJson2Unknown_023_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ArrayD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Array8pop_backEv", sceJson2Unknown_023_nid_no_patch); -int APS5_VABI sceJson2Unknown_024_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Value11referStringEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Array9pop_frontEv", sceJson2Unknown_024_nid_no_patch); -int APS5_VABI sceJson2Unknown_025_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Value12referBooleanEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ArrayC2ERKS1_", sceJson2Unknown_025_nid_no_patch); -int APS5_VABI sceJson2Unknown_026_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Value12referIntegerEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ArrayC2Ev", sceJson2Unknown_026_nid_no_patch); -int APS5_VABI sceJson2Unknown_027_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Value13referUIntegerEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ArrayD2Ev", sceJson2Unknown_027_nid_no_patch); -int APS5_VABI sceJson2Unknown_028_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Value9referRealEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ArrayaSERKS1_", sceJson2Unknown_028_nid_no_patch); -int APS5_VABI sceJson2Unknown_029_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Value11referStringEv", sceJson2Unknown_029_nid_no_patch); -int APS5_VABI sceJson2Unknown_030_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Value12referBooleanEv", sceJson2Unknown_030_nid_no_patch); -int APS5_VABI sceJson2Unknown_031_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2ENS0_9ValueTypeE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Value12referIntegerEv", sceJson2Unknown_031_nid_no_patch); -int APS5_VABI sceJson2Unknown_032_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2EPKc() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Value13referUIntegerEv", sceJson2Unknown_032_nid_no_patch); -int APS5_VABI sceJson2Unknown_033_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_5ArrayE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Value9referRealEv", sceJson2Unknown_033_nid_no_patch); -int APS5_VABI sceJson2Unknown_034_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6ObjectE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_", sceJson2Unknown_034_nid_no_patch); -int APS5_VABI sceJson2Unknown_035_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6StringE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_", sceJson2Unknown_035_nid_no_patch); -int APS5_VABI sceJson2Unknown_036_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2ERKS1_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2ENS0_9ValueTypeE", sceJson2Unknown_036_nid_no_patch); -int APS5_VABI sceJson2Unknown_037_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2Eb() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2EPKc", sceJson2Unknown_037_nid_no_patch); -int APS5_VABI sceJson2Unknown_038_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2Ed() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2ERKNS0_5ArrayE", sceJson2Unknown_038_nid_no_patch); -int APS5_VABI sceJson2Unknown_039_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2El() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2ERKNS0_6ObjectE", sceJson2Unknown_039_nid_no_patch); -int APS5_VABI sceJson2Unknown_040_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2Em() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2ERKNS0_6StringE", sceJson2Unknown_040_nid_no_patch); -int APS5_VABI sceJson2Unknown_041_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2ERKS1_", sceJson2Unknown_041_nid_no_patch); -int APS5_VABI sceJson2Unknown_042_nid_no_patch() { +int APS5_VABI _ZN3sce4Json5ValueD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2Eb", sceJson2Unknown_042_nid_no_patch); -int APS5_VABI sceJson2Unknown_043_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6MallocEm() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2Ed", sceJson2Unknown_043_nid_no_patch); -int APS5_VABI sceJson2Unknown_044_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2El", sceJson2Unknown_044_nid_no_patch); -int APS5_VABI sceJson2Unknown_045_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6Object4PairC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2Em", sceJson2Unknown_045_nid_no_patch); -int APS5_VABI sceJson2Unknown_046_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6Object4PairD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueC2Ev", sceJson2Unknown_046_nid_no_patch); -int APS5_VABI sceJson2Unknown_047_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6Object8iteratorC2ERKS2_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json5ValueD2Ev", sceJson2Unknown_047_nid_no_patch); -int APS5_VABI sceJson2Unknown_048_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6Object8iteratorC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6MallocEm", sceJson2Unknown_048_nid_no_patch); -int APS5_VABI sceJson2Unknown_049_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6Object8iteratorD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE", sceJson2Unknown_049_nid_no_patch); -int APS5_VABI sceJson2Unknown_050_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6ObjectC2ERKS1_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6Object4PairC2Ev", sceJson2Unknown_050_nid_no_patch); -int APS5_VABI sceJson2Unknown_051_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6ObjectC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6Object4PairD2Ev", sceJson2Unknown_051_nid_no_patch); -int APS5_VABI sceJson2Unknown_052_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6ObjectD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6Object8iteratorC2ERKS2_", sceJson2Unknown_052_nid_no_patch); -int APS5_VABI sceJson2Unknown_053_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6StringC2EPKc() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6Object8iteratorC2Ev", sceJson2Unknown_053_nid_no_patch); -int APS5_VABI sceJson2Unknown_054_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6StringC2ERKS1_() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6Object8iteratorD2Ev", sceJson2Unknown_054_nid_no_patch); -int APS5_VABI sceJson2Unknown_055_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6StringC2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6ObjectC2ERKS1_", sceJson2Unknown_055_nid_no_patch); -int APS5_VABI sceJson2Unknown_056_nid_no_patch() { +int APS5_VABI _ZN3sce4Json6StringD2Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6ObjectC2Ev", sceJson2Unknown_056_nid_no_patch); -int APS5_VABI sceJson2Unknown_057_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json14InitParameter211getUserDataEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6ObjectD2Ev", sceJson2Unknown_057_nid_no_patch); -int APS5_VABI sceJson2Unknown_058_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json14InitParameter212getAllocatorEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6Parser5parseERNS0_5ValueEPKc", sceJson2Unknown_058_nid_no_patch); -int APS5_VABI sceJson2Unknown_059_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json14InitParameter217getFileBufferSizeEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6StringC2EPKc", sceJson2Unknown_059_nid_no_patch); -int APS5_VABI sceJson2Unknown_060_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6StringC2ERKS1_", sceJson2Unknown_060_nid_no_patch); -int APS5_VABI sceJson2Unknown_061_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6StringC2Ev", sceJson2Unknown_061_nid_no_patch); -int APS5_VABI sceJson2Unknown_062_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json6String2atEm() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZN3sce4Json6StringD2Ev", sceJson2Unknown_062_nid_no_patch); -int APS5_VABI sceJson2Unknown_063_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json6String4findEPKcm() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZNK3sce4Json14InitParameter211getUserDataEv", sceJson2Unknown_063_nid_no_patch); -int APS5_VABI sceJson2Unknown_064_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json6String4findEPKcmm() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZNK3sce4Json14InitParameter212getAllocatorEv", sceJson2Unknown_064_nid_no_patch); -int APS5_VABI sceJson2Unknown_065_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json6String4findERKS1_m() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZNK3sce4Json14InitParameter217getFileBufferSizeEv", sceJson2Unknown_065_nid_no_patch); -int APS5_VABI sceJson2Unknown_066_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json6String5rfindEPKcm() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv", sceJson2Unknown_066_nid_no_patch); -int APS5_VABI sceJson2Unknown_067_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json6String5rfindEPKcmm() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZNK3sce4Json5ValueixERKNS0_6StringE", sceJson2Unknown_067_nid_no_patch); -int APS5_VABI sceJson2Unknown_068_nid_no_patch() { +int APS5_VABI _ZNK3sce4Json6String5rfindERKS1_m() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("_ZNK3sce4Json6String2atEm", sceJson2Unknown_068_nid_no_patch); - -int APS5_VABI sceJson2Unknown_069_nid_no_patch() { - NotImplemented_nid_no_patch(__func__); - return 0; -} -APS5_EXPORT("_ZNK3sce4Json6String4findEPKcm", sceJson2Unknown_069_nid_no_patch); - -int APS5_VABI sceJson2Unknown_070_nid_no_patch() { - NotImplemented_nid_no_patch(__func__); - return 0; -} -APS5_EXPORT("_ZNK3sce4Json6String4findEPKcmm", sceJson2Unknown_070_nid_no_patch); - -int APS5_VABI sceJson2Unknown_071_nid_no_patch() { - NotImplemented_nid_no_patch(__func__); - return 0; -} -APS5_EXPORT("_ZNK3sce4Json6String4findERKS1_m", sceJson2Unknown_071_nid_no_patch); - -int APS5_VABI sceJson2Unknown_072_nid_no_patch() { - NotImplemented_nid_no_patch(__func__); - return 0; -} -APS5_EXPORT("_ZNK3sce4Json6String5rfindEPKcm", sceJson2Unknown_072_nid_no_patch); - -int APS5_VABI sceJson2Unknown_073_nid_no_patch() { - NotImplemented_nid_no_patch(__func__); - return 0; -} -APS5_EXPORT("_ZNK3sce4Json6String5rfindEPKcmm", sceJson2Unknown_073_nid_no_patch); - -int APS5_VABI sceJson2Unknown_074_nid_no_patch() { - NotImplemented_nid_no_patch(__func__); - return 0; -} -APS5_EXPORT("_ZNK3sce4Json6String5rfindERKS1_m", sceJson2Unknown_074_nid_no_patch); } diff --git a/core/libs/tests/GuestJsonArrayAllocator.cpp b/core/libs/tests/GuestJsonArrayAllocator.cpp index 15954c0f90..8fddc2acd8 100644 --- a/core/libs/tests/GuestJsonArrayAllocator.cpp +++ b/core/libs/tests/GuestJsonArrayAllocator.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,7 @@ std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array*); Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); @@ -86,7 +88,8 @@ static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* con } static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; struct ArrayAllocator { void* const* vtable = ArrayVtable; }; -static void CheckLinks(const Array& array, std::size_t count) { +static void CheckLinks(const Array& array, std::size_t count, std::initializer_list values = {}) { + Require(values.size() == 0 || values.size() == count, "Expected array values must cover every live node"); const auto* header = static_cast(array.storage); Require(header && header[1] == count, "Array header must expose its native element count"); const auto* head = reinterpret_cast(header[0]); @@ -95,9 +98,10 @@ static void CheckLinks(const Array& array, std::size_t count) { for (auto* node = reinterpret_cast(head[1]); node != head; node = reinterpret_cast(node[1])) { Require(node[0] == reinterpret_cast(previous), "Every array link must retain its predecessor"); const auto* value = reinterpret_cast(node + 2); - Require(!value->parent && !value->auxiliary && value->type == 2 && value->payload == 10 * (++seen), "Array nodes must expose their inline values and ownership"); + Require(++seen <= count, "Array links must close at the sentinel"); + const auto expected = values.size() ? *(values.begin() + seen - 1) : 10 * seen; + Require(!value->parent && !value->auxiliary && value->type == 2 && value->payload == expected, "Array nodes must expose their inline values and ownership"); previous = node; - Require(seen <= count, "Array links must close at the sentinel"); } Require(seen == count && head[0] == reinterpret_cast(previous), "Array sentinel must close both list directions"); } @@ -201,6 +205,19 @@ int main() { Require(_ZNK3sce4Json5Array4sizeEv(&array) == 2 && *_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array4backEv(&array)) == 20, "Failed append must retain the reference module's removal of the existing tail"); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); Require(_ZNK3sce4Json5Array4sizeEv(&array) == 3 && *_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array4backEv(&array)) == 40, "Append remains usable after failure"); + Iterator retained{}; _ZNK3sce4Json5Array5beginEv(&retained, &array); _ZN3sce4Json5Array8iteratorppEv(&retained); + const auto* survivor = _ZNK3sce4Json5Array8iteratordeEv(&retained); + const auto requests = trace.requests.size(); + const auto notifications = trace.notifications; + trace.budget = 0; + _ZN3sce4Json5Array9pop_frontEv(&array); + Require(_ZNK3sce4Json5Array4sizeEv(&array) == 2 && _ZNK3sce4Json5Array5frontEv(&array) == survivor, "Removing the front must preserve the next element and its iterator identity"); + CheckLinks(array, 2, {20, 40}); + _ZN3sce4Json5Array9pop_frontEv(&array); _ZN3sce4Json5Array9pop_frontEv(&array); + Require(trace.requests.size() == requests && trace.notifications == notifications, "Front removal must reclaim allocator-owned nodes without allocating"); + CheckLinks(array, 0); + trace.budget = -1; + _ZN3sce4Json5Array8iteratorD1Ev(&retained); _ZN3sce4Json5ValueD1Ev(&value); _ZN3sce4Json5Array5clearEv(&array); Require(_ZNK3sce4Json5Array4sizeEv(&array) == 0 && trace.live.size() == baseline + 2, "Clear must retain only the array header and sentinel"); _ZN3sce4Json5ArrayD1Ev(&array); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index dc24c23f70..fd51f78b21 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -1,87 +1,9 @@ # libSceJson2 technical debt -### Unknown 3.20 import declarations - -- [_ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R996H0YIhiQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json11Initializer28setGlobalSpecialFloatHandlerEPFKNS0_5ValueENS0_12FunctionTypeEdPS3_PvPbES6_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `i1393UBWu1U` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json11Initializer28setGlobalTypeMismatchHandlerEPFRKNS0_5ValueENS0_9ValueTypeEPS3_S6_PvES7_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `tck9g-yOqPg` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json11Initializer36setGlobalElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKNS0_5ValueES9_PvESA_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `j8Q7SBgMm28` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json11InitializerC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `6qFqND4iwPA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json11InitializerD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `qQZGdYkEytk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json12MemAllocatorD1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `O1uwoqazS2A` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Mp8DPb3M4I0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializer19terminate_staticlibEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `dvp2zGSQzMs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BPSPc3pUMJo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializerC1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `WntRKhYTmZw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializerC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2n+fxNdSZfs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializerD1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `yudT4-bglko` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializerD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `caP-y4fH-zQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json4FreeEPv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `bnk+edDbqMk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array10push_frontERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MQxDxBzWz40` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array5eraseERKNS1_8iteratorE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2mbVAtshWRI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ixJAz92H1uE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array8iteratorC2ERKS2_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Hdc3DSIsG2I` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array8iteratorC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `JSg1MSo7NKE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array8iteratorD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `9niNrXsF3Bc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array8pop_backEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `woec1toi4Xw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array9pop_frontEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `pPs3qZTZQvQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ArrayC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `6NlbpyIfUuY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ArrayC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `t6AHhkWu4mY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ArrayD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MIyMPw7YTGM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ArrayaSERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `w5UbvPLGye0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value11referStringEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `m5J28iw-QPs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value12referBooleanEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `QxjfcfBhEdc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value12referIntegerEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R0ac5QOdlpo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value13referUIntegerEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Nx5tez4siL4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value9referRealEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `t7UEJrhojhk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ENS0_9ValueTypeE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PWm9MyJJVqU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2EPKc](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `+hHa4QYtLY8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ERKNS0_5ArrayE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `VH9dsKpQTmk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ERKNS0_6ObjectE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `JXlrCLLCLmc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `veNviXt6wC0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `TZyuFeGuw9Y` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2Eb](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `OK4Ot0ue7J0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2Ed](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `23nm4oXHlfI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2El](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `EjxUpzR2Yx8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2Em](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `cqucBPSeVDA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `-wa17B7TGnw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `0eUrW9JAxM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6MallocEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Xzy0onDzSAc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `lt3J4dPvEtM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object4PairC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Wu-urs6MMpQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object4PairD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `TM0Sj+Ebins` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object8iteratorC2ERKS2_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `r5CShmcua+M` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object8iteratorC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `OxO+V0ouoS4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object8iteratorD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `patTzc7zBjw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6ObjectC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `tSRqQ-+Y4PE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6ObjectC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `SkKczWHdklc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6ObjectD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `1siRBursJjw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Parser5parseERNS0_5ValueEPKc](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `LB3jxppxyKU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6StringC2EPKc](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `N78gkQzXRL4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6StringC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `gALFgNfF+9I` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6StringC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `eG9E9M6XvTM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6StringD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Ui7YFnSTCBw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json14InitParameter211getUserDataEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `GWtzpTKbZ+0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json14InitParameter212getAllocatorEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BykPm1ajcM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json14InitParameter217getFileBufferSizeEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Pcg6xQUhq9E` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PDkib0zyABg` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json5ValueixERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `clF7J7N9xXE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String2atEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `KrKM-5SdQrA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String4findEPKcm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `yjsf6SvXA08` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String4findEPKcmm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `wwDnDDsQVc0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String4findERKS1_m](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `awkeNR6Ukp8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String5rfindEPKcm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `VpQBXbaHXOo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String5rfindEPKcmm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `A29RHrlfTM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String5rfindERKS1_m](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `-R9mdXBspeQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - ### Unknown function info -- [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - removes and destroys the first element; on an empty array it throws `std::out_of_range`, as `front` and `back` do here. The console's result for an empty array is unverified. - - [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. +- [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) - the console result for an empty array remains unverified; the existing host behavior throws `std::out_of_range`. Missing storage throws unsupported. Nonempty removal releases the allocator-owned node without allocating and preserves surviving element/iterator identity. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size and double overflow throw rather than inventing a result. Numeric-token validation, more complex parser allocation failures and cross-thread operation behavior remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. @@ -127,3 +49,74 @@ - [sce::Json::Parser object allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - object keys use one allocator-backed string reused between members, following `0x6250`; lookup closes its shared operation scope before assigning the exposed key and owner, following `0xda40`. Sixty-six reference-backed cases cover persistent and one-shot failure at each returning allocation position through a 33-allocation successful parse of an object containing a string and array. Checks include partial keys/members, numeric and string fallback values, parent links, error status, request/notification counts and complete allocator cleanup. The nested array sentinel failure is excluded because reference execution crashes; more complex nested objects, callback-driven allocation failures and concurrency remain unverified. - [sce::Json::Collection inherited allocation errors](../../../core/libs/prx/libSceJson2/Export.cpp) - array assignment, object assignment/insertion and legacy mismatch-map updates now use the shared depth/error protocol instead of counter snapshots, following `0xbc80`, `0xd400`, `0xcd30` and `0x3b00`. Six reference-backed public parser/read-callback cases invoke array assignment, object assignment or insertion after a one-shot character allocation failure, plus healthy controls. Checks cover placeholder retention, skipped payload binding, unchanged source, outer error persistence, recovery on the next independent parse and complete cleanup. Baseline and targeted assignment/insertion mutations fail these checks. Nested legacy map updates, callback-return cloning and cross-thread failure propagation remain unverified. + +### Unknown 3.20 import declarations + +- [_ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R996H0YIhiQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json11InitializerC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `6qFqND4iwPA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json11InitializerD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `qQZGdYkEytk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json12MemAllocatorD1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `O1uwoqazS2A` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Mp8DPb3M4I0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json19InternalInitializer19terminate_staticlibEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `dvp2zGSQzMs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BPSPc3pUMJo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json19InternalInitializerC1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `WntRKhYTmZw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json19InternalInitializerC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2n+fxNdSZfs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json19InternalInitializerD1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `yudT4-bglko` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json19InternalInitializerD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `caP-y4fH-zQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json4FreeEPv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `bnk+edDbqMk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Array10push_frontERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MQxDxBzWz40` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Array5eraseERKNS1_8iteratorE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2mbVAtshWRI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ixJAz92H1uE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Array8iteratorC2ERKS2_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Hdc3DSIsG2I` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Array8iteratorC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `JSg1MSo7NKE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Array8iteratorD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `9niNrXsF3Bc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Array8pop_backEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `woec1toi4Xw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ArrayC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `6NlbpyIfUuY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ArrayC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `t6AHhkWu4mY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ArrayD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MIyMPw7YTGM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Value11referStringEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `m5J28iw-QPs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Value12referBooleanEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `QxjfcfBhEdc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Value12referIntegerEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R0ac5QOdlpo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Value13referUIntegerEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Nx5tez4siL4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Value9referRealEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `t7UEJrhojhk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2ENS0_9ValueTypeE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PWm9MyJJVqU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2EPKc](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `+hHa4QYtLY8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2ERKNS0_5ArrayE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `VH9dsKpQTmk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2ERKNS0_6ObjectE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `JXlrCLLCLmc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2ERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `veNviXt6wC0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `TZyuFeGuw9Y` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2Eb](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `OK4Ot0ue7J0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2Ed](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `23nm4oXHlfI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2El](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `EjxUpzR2Yx8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2Em](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `cqucBPSeVDA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `-wa17B7TGnw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json5ValueD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `0eUrW9JAxM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6MallocEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Xzy0onDzSAc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `lt3J4dPvEtM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6Object4PairC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Wu-urs6MMpQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6Object4PairD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `TM0Sj+Ebins` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6Object8iteratorC2ERKS2_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `r5CShmcua+M` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6Object8iteratorC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `OxO+V0ouoS4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6Object8iteratorD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `patTzc7zBjw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6ObjectC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `tSRqQ-+Y4PE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6ObjectC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `SkKczWHdklc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6ObjectD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `1siRBursJjw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6StringC2EPKc](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `N78gkQzXRL4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6StringC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `gALFgNfF+9I` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6StringC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `eG9E9M6XvTM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json6StringD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Ui7YFnSTCBw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json14InitParameter211getUserDataEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `GWtzpTKbZ+0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json14InitParameter212getAllocatorEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BykPm1ajcM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json14InitParameter217getFileBufferSizeEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Pcg6xQUhq9E` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PDkib0zyABg` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json5ValueixERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `clF7J7N9xXE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json6String2atEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `KrKM-5SdQrA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json6String4findEPKcm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `yjsf6SvXA08` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json6String4findEPKcmm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `wwDnDDsQVc0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json6String4findERKS1_m](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `awkeNR6Ukp8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json6String5rfindEPKcm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `VpQBXbaHXOo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json6String5rfindEPKcmm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `A29RHrlfTM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZNK3sce4Json6String5rfindERKS1_m](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `-R9mdXBspeQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. From c3cfc125236258ef640cd2bfb7d9c4b9a63ef4e2 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:25:51 +0200 Subject: [PATCH 032/318] fix(libSceJson2): implement constructor and destructor variants --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 41 +++- .../libs/prx/libSceJson2/InventoryExports.cpp | 170 --------------- core/libs/tests/GuestJsonAliases.cpp | 204 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 34 --- 5 files changed, 249 insertions(+), 207 deletions(-) create mode 100644 core/libs/tests/GuestJsonAliases.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 722cd23d32..c6547a41ed 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_alias_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonAliases.cpp) + target_include_directories(guest_json_alias_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_alias_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_alias_tests) + add_test(NAME guest_json_aliases COMMAND guest_json_alias_tests) add_test_executable(guest_json_tests ${LIBS_INCLUDE_DIR}/tests/GuestJson.cpp) target_include_directories(guest_json_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_tests PRIVATE libSceJson2 libc) @@ -91,7 +96,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 8be4b4b071..3f308334b1 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2383,13 +2383,15 @@ void APS5_VABI _ZN3sce4Json5ValueC1El(Value* self, std::int64_t value) { Constru void APS5_VABI _ZN3sce4Json5ValueC1Em(Value* self, std::uint64_t value) { Construct(*self); self->type = TypeUInteger; self->uinteger = value; } void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value* self, double value) { Construct(*self); AssignReal(*self, value); } void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value* self, const char* text) { - if (text == nullptr) throw std::invalid_argument("sce::Json::Value: null text"); Construct(*self); + ScopedString temporary{std::string_view{text ? text : ""}}; + if (!temporary.string.text) return; SetType(*self, TypeString); - if (self->type == TypeString) AssignText(*self->string, text); + if (self->type == TypeString) AssignText(*self->string, View(temporary.string), true); } void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6StringE(Value* self, const String* text) { Construct(*self); + if (!text->text) return; SetType(*self, TypeString); if (self->type == TypeString) AssignText(*self->string, View(*text), true); } @@ -2849,4 +2851,39 @@ void APS5_VABI _ZN3sce4Json6Object4PairC1Ev(Pair* self) { *self = pair; BindChildren(self->value); } + +void APS5_VABI _ZN3sce4Json11InitializerC2Ev(void* self) { _ZN3sce4Json11InitializerC1Ev(self); } +void APS5_VABI _ZN3sce4Json11InitializerD2Ev(void* self) { _ZN3sce4Json11InitializerD1Ev(self); } +void APS5_VABI _ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm(InitParameterRtti* self, void* allocParam, void* userData, std::size_t fileBufferSize) { _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(self, allocParam, userData, fileBufferSize); } +void APS5_VABI _ZN3sce4Json5Array8iteratorC2ERKS2_(ArrayIterator* self, const ArrayIterator* other) { _ZN3sce4Json5Array8iteratorC1ERKS2_(self, other); } +void APS5_VABI _ZN3sce4Json5Array8iteratorC2Ev(ArrayIterator* self) { _ZN3sce4Json5Array8iteratorC1Ev(self); } +void APS5_VABI _ZN3sce4Json5Array8iteratorD2Ev(ArrayIterator* self) { _ZN3sce4Json5Array8iteratorD1Ev(self); } +void APS5_VABI _ZN3sce4Json5ArrayC2ERKS1_(Array* self, const Array* other) { _ZN3sce4Json5ArrayC1ERKS1_(self, other); } +void APS5_VABI _ZN3sce4Json5ArrayC2Ev(Array* self) { _ZN3sce4Json5ArrayC1Ev(self); } +void APS5_VABI _ZN3sce4Json5ArrayD2Ev(Array* self) { _ZN3sce4Json5ArrayD1Ev(self); } +void APS5_VABI _ZN3sce4Json5ValueC2ENS0_9ValueTypeE(Value* self, ValueType type) { _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(self, type); } +void APS5_VABI _ZN3sce4Json5ValueC2EPKc(Value* self, const char* text) { _ZN3sce4Json5ValueC1EPKc(self, text); } +void APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_5ArrayE(Value* self, const Array* array) { _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(self, array); } +void APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6ObjectE(Value* self, const Object* object) { _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(self, object); } +void APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6StringE(Value* self, const String* text) { _ZN3sce4Json5ValueC1ERKNS0_6StringE(self, text); } +void APS5_VABI _ZN3sce4Json5ValueC2ERKS1_(Value* self, const Value* other) { _ZN3sce4Json5ValueC1ERKS1_(self, other); } +void APS5_VABI _ZN3sce4Json5ValueC2Eb(Value* self, bool value) { _ZN3sce4Json5ValueC1Eb(self, value); } +void APS5_VABI _ZN3sce4Json5ValueC2Ed(Value* self, double value) { _ZN3sce4Json5ValueC1Ed(self, value); } +void APS5_VABI _ZN3sce4Json5ValueC2El(Value* self, std::int64_t value) { _ZN3sce4Json5ValueC1El(self, value); } +void APS5_VABI _ZN3sce4Json5ValueC2Em(Value* self, std::uint64_t value) { _ZN3sce4Json5ValueC1Em(self, value); } +void APS5_VABI _ZN3sce4Json5ValueC2Ev(Value* self) { _ZN3sce4Json5ValueC1Ev(self); } +void APS5_VABI _ZN3sce4Json5ValueD2Ev(Value* self) { _ZN3sce4Json5ValueD1Ev(self); } +void APS5_VABI _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(Pair* self, const String* key, const Value* value) { _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(self, key, value); } +void APS5_VABI _ZN3sce4Json6Object4PairC2Ev(Pair* self) { _ZN3sce4Json6Object4PairC1Ev(self); } +void APS5_VABI _ZN3sce4Json6Object4PairD2Ev(Pair* self) { _ZN3sce4Json6Object4PairD1Ev(self); } +void APS5_VABI _ZN3sce4Json6Object8iteratorC2ERKS2_(ObjectIterator* self, const ObjectIterator* other) { _ZN3sce4Json6Object8iteratorC1ERKS2_(self, other); } +void APS5_VABI _ZN3sce4Json6Object8iteratorC2Ev(ObjectIterator* self) { _ZN3sce4Json6Object8iteratorC1Ev(self); } +void APS5_VABI _ZN3sce4Json6Object8iteratorD2Ev(ObjectIterator* self) { _ZN3sce4Json6Object8iteratorD1Ev(self); } +void APS5_VABI _ZN3sce4Json6ObjectC2ERKS1_(Object* self, const Object* other) { _ZN3sce4Json6ObjectC1ERKS1_(self, other); } +void APS5_VABI _ZN3sce4Json6ObjectC2Ev(Object* self) { _ZN3sce4Json6ObjectC1Ev(self); } +void APS5_VABI _ZN3sce4Json6ObjectD2Ev(Object* self) { _ZN3sce4Json6ObjectD1Ev(self); } +void APS5_VABI _ZN3sce4Json6StringC2EPKc(String* self, const char* text) { _ZN3sce4Json6StringC1EPKc(self, text); } +void APS5_VABI _ZN3sce4Json6StringC2ERKS1_(String* self, const String* other) { _ZN3sce4Json6StringC1ERKS1_(self, other); } +void APS5_VABI _ZN3sce4Json6StringC2Ev(String* self) { _ZN3sce4Json6StringC1Ev(self); } +void APS5_VABI _ZN3sce4Json6StringD2Ev(String* self) { _ZN3sce4Json6StringD1Ev(self); } } diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index 20cbf74dd5..3a2e99ff14 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -10,16 +10,6 @@ int APS5_VABI _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9Value return 0; } -int APS5_VABI _ZN3sce4Json11InitializerC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json11InitializerD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json12MemAllocatorD0Ev() { NotImplemented_nid_no_patch(__func__); return 0; @@ -30,11 +20,6 @@ int APS5_VABI _ZN3sce4Json12MemAllocatorD1Ev() { return 0; } -int APS5_VABI _ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json19InternalInitializer19terminate_staticlibEv() { NotImplemented_nid_no_patch(__func__); return 0; @@ -85,41 +70,11 @@ int APS5_VABI _ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE() { return 0; } -int APS5_VABI _ZN3sce4Json5Array8iteratorC2ERKS2_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5Array8iteratorC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5Array8iteratorD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json5Array8pop_backEv() { NotImplemented_nid_no_patch(__func__); return 0; } -int APS5_VABI _ZN3sce4Json5ArrayC2ERKS1_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ArrayC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ArrayD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json5Value11referStringEv() { NotImplemented_nid_no_patch(__func__); return 0; @@ -155,136 +110,11 @@ int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_() { return 0; } -int APS5_VABI _ZN3sce4Json5ValueC2ENS0_9ValueTypeE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2EPKc() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_5ArrayE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6ObjectE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6StringE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2ERKS1_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2Eb() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2Ed() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2El() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2Em() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5ValueD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json6MallocEm() { NotImplemented_nid_no_patch(__func__); return 0; } -int APS5_VABI _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6Object4PairC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6Object4PairD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6Object8iteratorC2ERKS2_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6Object8iteratorC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6Object8iteratorD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6ObjectC2ERKS1_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6ObjectC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6ObjectD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6StringC2EPKc() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6StringC2ERKS1_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6StringC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json6StringD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZNK3sce4Json14InitParameter211getUserDataEv() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/tests/GuestJsonAliases.cpp b/core/libs/tests/GuestJsonAliases.cpp new file mode 100644 index 0000000000..1ffa256968 --- /dev/null +++ b/core/libs/tests/GuestJsonAliases.cpp @@ -0,0 +1,204 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Array { void* storage; }; +struct Object { void* storage; }; +struct Iterator { void* state; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; }; +using AliasAllocateCallback = void* (APS5_VABI*)(std::size_t, void*); +using AliasFreeCallback = void (APS5_VABI*)(void*, void*); +using AliasNotifyCallback = void (APS5_VABI*)(int, std::size_t, void*); +struct Callbacks { AliasAllocateCallback allocate; AliasFreeCallback free; AliasNotifyCallback notify; }; +static_assert(sizeof(Value) == 32 && sizeof(Parameter) == 24 && sizeof(Pair) == 48); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC2Ev(void*); +void APS5_VABI _ZN3sce4Json11InitializerD2Ev(void*); +void APS5_VABI _ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm(Parameter*, void*, void*, std::size_t); +void APS5_VABI _ZN3sce4Json5Array8iteratorC2ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC2Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD2Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5ArrayC2ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayC2Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayD2Ev(Array*); +void APS5_VABI _ZN3sce4Json5ValueC2ENS0_9ValueTypeE(Value*, std::int32_t); +void APS5_VABI _ZN3sce4Json5ValueC2EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6ObjectE(Value*, const Object*); +void APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6StringE(Value*, const String*); +void APS5_VABI _ZN3sce4Json5ValueC2ERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueC2Eb(Value*, bool); +void APS5_VABI _ZN3sce4Json5ValueC2Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueC2El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueC2Em(Value*, std::uint64_t); +void APS5_VABI _ZN3sce4Json5ValueC2Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD2Ev(Value*); +void APS5_VABI _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(Pair*, const String*, const Value*); +void APS5_VABI _ZN3sce4Json6Object4PairC2Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object4PairD2Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object8iteratorC2ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorC2Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD2Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC2ERKS1_(Object*, const Object*); +void APS5_VABI _ZN3sce4Json6ObjectC2Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD2Ev(Object*); +void APS5_VABI _ZN3sce4Json6StringC2EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringC2ERKS1_(String*, const String*); +void APS5_VABI _ZN3sce4Json6StringC2Ev(String*); +void APS5_VABI _ZN3sce4Json6StringD2Ev(String*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +void APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct AliasTrace { + std::map live; + int budget = -1; + bool once = false; + std::size_t requests = 0; + unsigned notifications = 0; +}; +static void* APS5_VABI AllocateAlias(std::size_t size, void* context) { + auto& trace = *static_cast(context); + ++trace.requests; + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(size); + Require(memory != nullptr, "Alias tracking allocator requires host memory"); + std::memset(memory, 0xa5, size); trace.live.emplace(memory, size); return memory; +} +static void APS5_VABI FreeAlias(void* memory, void* context) { + if (!memory) return; + Require(static_cast(context)->live.erase(memory) == 1, "Alias destruction must free each allocation through its owning context exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyAlias(int error, std::size_t, void* context) { + Require(static_cast(error) == 0x80848102, "Alias allocation failures must retain the Json2 error"); + ++static_cast(context)->notifications; +} +static const String* Text(const Value& value) { return reinterpret_cast(value.payload); } +static void CheckText(const Value& value, const char* expected) { + Require(value.type == 5 && value.payload && std::strcmp(_ZNK3sce4Json6String5c_strEv(Text(value)), expected) == 0, "Value constructor variants must preserve string contents"); +} +int main() { + AliasTrace trace; + Callbacks callbacks{AllocateAlias, FreeAlias, NotifyAlias}; + alignas(8) std::array settings; settings.fill(std::byte{0xa5}); + auto* parameter = reinterpret_cast(settings.data()); + _ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm(parameter, &callbacks, &trace, 8192); + Require(parameter->allocator && parameter->context == &trace && parameter->bufferSize == 8192, "RTTI constructor variant must retain its callbacks, context and buffer setting"); + for (std::size_t i = sizeof(Parameter); i < settings.size(); ++i) Require(settings[i] == std::byte{0xa5}, "RTTI construction must preserve bytes beyond its parameter block"); + std::array initializer; initializer.fill(0xa5); + _ZN3sce4Json11InitializerC2Ev(initializer.data()); + Require(initializer[0] == 0, "Initializer constructor variant must clear its active byte"); + for (std::size_t i = 1; i < initializer.size(); ++i) Require(initializer[i] == 0xa5, "Initializer construction must preserve every adjacent byte"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(initializer.data(), parameter) == 0, "Initialize through the constructor variant and RTTI allocator"); + const auto baseline = trace.live.size(); + for (std::int32_t type = 0; type < 8; ++type) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + _ZN3sce4Json5ValueC2ENS0_9ValueTypeE(&value, type); + Require(value.type == type && !value.parent && !value.auxiliary && value.reserved == 0xa5a5a5a5, "Typed value constructor variant must retain its type and inactive metadata"); + _ZN3sce4Json5ValueD2Ev(&value); + Require(trace.live.size() == baseline, "Typed constructor/destructor variants must reclaim their handles and storage"); + } + Value value{}; + _ZN3sce4Json5ValueC2Ev(&value); Require(value.type == 0, "Default value constructor variant must construct null"); _ZN3sce4Json5ValueD2Ev(&value); + _ZN3sce4Json5ValueC2Eb(&value, true); Require(value.type == 1 && (value.payload & 255) == 1, "Boolean constructor variant must preserve its payload"); _ZN3sce4Json5ValueD2Ev(&value); + _ZN3sce4Json5ValueC2El(&value, -77); Require(value.type == 2 && value.payload == std::bit_cast(std::int64_t{-77}), "Signed constructor variant must preserve all payload bits"); _ZN3sce4Json5ValueD2Ev(&value); + _ZN3sce4Json5ValueC2Em(&value, UINT64_MAX); Require(value.type == 3 && value.payload == UINT64_MAX, "Unsigned constructor variant must preserve all payload bits"); _ZN3sce4Json5ValueD2Ev(&value); + _ZN3sce4Json5ValueC2Ed(&value, 1.25); Require(value.type == 4 && value.payload == std::bit_cast(1.25), "Real constructor variant must preserve its payload"); _ZN3sce4Json5ValueD2Ev(&value); + String empty{}, text{}, copiedText{}; + _ZN3sce4Json6StringC2Ev(&empty); Require(empty.storage && !*_ZNK3sce4Json6String5c_strEv(&empty), "Default string constructor variant must own empty storage"); + _ZN3sce4Json6StringC2EPKc(&text, "abc"); _ZN3sce4Json6StringC2ERKS1_(&copiedText, &text); + Require(text.storage != copiedText.storage && std::strcmp(_ZNK3sce4Json6String5c_strEv(&copiedText), "abc") == 0, "String copy constructor variant must own independent storage"); + _ZN3sce4Json5ValueC2EPKc(&value, "abc"); CheckText(value, "abc"); + Value copiedValue{}; _ZN3sce4Json5ValueC2ERKS1_(&copiedValue, &value); CheckText(copiedValue, "abc"); + Require(copiedValue.payload != value.payload, "Value copy constructor variant must own an independent handle"); + _ZN3sce4Json5ValueD2Ev(&copiedValue); _ZN3sce4Json5ValueD2Ev(&value); + _ZN3sce4Json5ValueC2ERKNS0_6StringE(&value, &text); CheckText(value, "abc"); + Require(Text(value)->storage != text.storage, "String-to-value constructor variant must copy storage"); _ZN3sce4Json5ValueD2Ev(&value); + Array array{}, copiedArray{}; _ZN3sce4Json5ArrayC2Ev(&array); + _ZN3sce4Json5ValueC2El(&value, 42); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); + _ZN3sce4Json5ArrayC2ERKS1_(&copiedArray, &array); + Require(array.storage != copiedArray.storage && _ZNK3sce4Json5Array4sizeEv(&copiedArray) == 1 && _ZNK3sce4Json5Array5frontEv(&copiedArray)->payload == 42, "Array copy constructor variant must copy its entries into independent storage"); + Value wrapped{}; _ZN3sce4Json5ValueC2ERKNS0_5ArrayE(&wrapped, &array); + const auto* child = _ZNK3sce4Json5Array5frontEv(reinterpret_cast(wrapped.payload)); + Require(wrapped.type == 6 && child->parent == &wrapped && child->payload == 42, "Array-to-value constructor variant must bind copied children to their owner"); _ZN3sce4Json5ValueD2Ev(&wrapped); + Object object{}, copiedObject{}; _ZN3sce4Json6ObjectC2Ev(&object); + _ZN3sce4Json5Value3setEl(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &text), 42); + _ZN3sce4Json6ObjectC2ERKS1_(&copiedObject, &object); + Require(object.storage != copiedObject.storage && _ZNK3sce4Json6Object4sizeEv(&copiedObject) == 1, "Object copy constructor variant must own independent tree storage"); + _ZN3sce4Json5ValueC2ERKNS0_6ObjectE(&wrapped, &object); + child = _ZNK3sce4Json5ValueixEPKc(&wrapped, "abc"); + Require(wrapped.type == 7 && child->parent == &wrapped && child->payload == 42, "Object-to-value constructor variant must bind copied members to their owner"); _ZN3sce4Json5ValueD2Ev(&wrapped); + Pair pair{}; _ZN3sce4Json6Object4PairC2Ev(&pair); + Require(pair.key.storage && pair.value.type == 0, "Default pair constructor variant must construct its key and null value"); _ZN3sce4Json6Object4PairD2Ev(&pair); + _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(&pair, &text, &value); + Require(pair.key.storage != text.storage && pair.value.type == 2 && pair.value.payload == 42, "Pair constructor variant must copy its key and value"); _ZN3sce4Json6Object4PairD2Ev(&pair); + Iterator iterator{}, copy{}; + _ZN3sce4Json5Array8iteratorC2Ev(&iterator); _ZN3sce4Json5Array8iteratorC2ERKS2_(©, &iterator); + Require(iterator.state && copy.state && iterator.state != copy.state && !*static_cast(copy.state), "Default array iterator variants must own independent unbound states"); + _ZN3sce4Json5Array8iteratorD2Ev(©); _ZN3sce4Json5Array8iteratorD2Ev(&iterator); + _ZNK3sce4Json5Array5beginEv(&iterator, &array); _ZN3sce4Json5Array8iteratorC2ERKS2_(©, &iterator); + Require(copy.state != iterator.state && _ZNK3sce4Json5Array8iteratordeEv(©) == _ZNK3sce4Json5Array5frontEv(&array), "Array iterator copy variant must preserve the actual element identity"); + _ZN3sce4Json5Array8iteratorD2Ev(©); _ZN3sce4Json5Array8iteratorD2Ev(&iterator); + _ZN3sce4Json6Object8iteratorC2Ev(&iterator); _ZN3sce4Json6Object8iteratorC2ERKS2_(©, &iterator); + Require(iterator.state && copy.state && iterator.state != copy.state && !*static_cast(copy.state), "Default object iterator variants must own independent unbound states"); + _ZN3sce4Json6Object8iteratorD2Ev(©); _ZN3sce4Json6Object8iteratorD2Ev(&iterator); + _ZNK3sce4Json6Object5beginEv(&iterator, &object); _ZN3sce4Json6Object8iteratorC2ERKS2_(©, &iterator); + Require(copy.state != iterator.state && _ZNK3sce4Json6Object8iteratordeEv(©) == _ZNK3sce4Json6Object8iteratordeEv(&iterator), "Object iterator copy variant must preserve the actual member identity"); + _ZN3sce4Json6Object8iteratorD2Ev(©); _ZN3sce4Json6Object8iteratorD2Ev(&iterator); + _ZN3sce4Json6ObjectD2Ev(&copiedObject); _ZN3sce4Json6ObjectD2Ev(&object); + _ZN3sce4Json5ArrayD2Ev(&copiedArray); _ZN3sce4Json5ArrayD2Ev(&array); _ZN3sce4Json5ValueD2Ev(&value); + _ZN3sce4Json6StringD2Ev(&copiedText); _ZN3sce4Json6StringD2Ev(&text); _ZN3sce4Json6StringD2Ev(&empty); + Require(trace.live.size() == baseline, "All constructor/destructor variants must reclaim their owned resources"); + String absent{}; + const auto requests = trace.requests; + _ZN3sce4Json5ValueC2ERKNS0_6StringE(&value, &absent); + Require(value.type == 0 && trace.requests == requests, "Missing string storage must leave a null value without allocating"); _ZN3sce4Json5ValueD2Ev(&value); + _ZN3sce4Json5ValueC2EPKc(&value, nullptr); CheckText(value, ""); _ZN3sce4Json5ValueD2Ev(&value); + struct Failure { bool once; int budget; int type; unsigned notifications; std::size_t requests; const char* text; }; + const Failure failures[]{ + {false, 0, 0, 1, 1, ""}, {false, 1, 0, 2, 3, ""}, + {false, 2, 0, 1, 3, ""}, {false, 3, 0, 1, 4, ""}, + {false, 4, 5, 1, 5, ""}, {false, 5, 5, 0, 5, "abc"}, + {true, 0, 0, 1, 1, ""}, {true, 1, 5, 1, 4, ""}, + {true, 2, 0, 1, 3, ""}, {true, 3, 0, 1, 4, ""}, + {true, 4, 5, 1, 5, ""}, {true, 5, 5, 0, 5, "abc"}, + }; + for (const auto& failure : failures) { + const auto notifications = trace.notifications; + const auto before = trace.requests; + trace.budget = failure.budget; trace.once = failure.once; + _ZN3sce4Json5ValueC2EPKc(&value, "abc"); trace.budget = -1; trace.once = false; + Require(value.type == failure.type && trace.notifications - notifications == failure.notifications && trace.requests - before == failure.requests, "Text constructor variant must preserve temporary string allocation and failure admission"); + if (value.type == 5) CheckText(value, failure.text); + _ZN3sce4Json5ValueD2Ev(&value); + Require(trace.live.size() == baseline, "Failed constructor variants must reclaim their admitted prefix"); + } + _ZN3sce4Json11InitializerD2Ev(initializer.data()); + Require(initializer[0] == 0 && trace.live.empty(), "Initializer destructor variant must terminate its resources"); + for (std::size_t i = 1; i < initializer.size(); ++i) Require(initializer[i] == 0xa5, "Initializer destruction must preserve every adjacent byte"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index fd51f78b21..8e5de886ce 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -53,11 +53,8 @@ ### Unknown 3.20 import declarations - [_ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R996H0YIhiQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json11InitializerC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `6qFqND4iwPA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json11InitializerD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `qQZGdYkEytk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json12MemAllocatorD1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `O1uwoqazS2A` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json17InitParameterRttiC2EPNS0_14AllocParamRttiEPvm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Mp8DPb3M4I0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json19InternalInitializer19terminate_staticlibEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `dvp2zGSQzMs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BPSPc3pUMJo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json19InternalInitializerC1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `WntRKhYTmZw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. @@ -68,13 +65,7 @@ - [_ZN3sce4Json5Array10push_frontERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MQxDxBzWz40` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array5eraseERKNS1_8iteratorE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2mbVAtshWRI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ixJAz92H1uE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array8iteratorC2ERKS2_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Hdc3DSIsG2I` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array8iteratorC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `JSg1MSo7NKE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array8iteratorD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `9niNrXsF3Bc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array8pop_backEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `woec1toi4Xw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ArrayC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `6NlbpyIfUuY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ArrayC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `t6AHhkWu4mY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ArrayD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MIyMPw7YTGM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value11referStringEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `m5J28iw-QPs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value12referBooleanEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `QxjfcfBhEdc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value12referIntegerEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R0ac5QOdlpo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. @@ -82,32 +73,7 @@ - [_ZN3sce4Json5Value9referRealEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `t7UEJrhojhk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ENS0_9ValueTypeE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PWm9MyJJVqU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2EPKc](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `+hHa4QYtLY8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ERKNS0_5ArrayE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `VH9dsKpQTmk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ERKNS0_6ObjectE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `JXlrCLLCLmc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `veNviXt6wC0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `TZyuFeGuw9Y` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2Eb](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `OK4Ot0ue7J0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2Ed](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `23nm4oXHlfI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2El](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `EjxUpzR2Yx8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2Em](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `cqucBPSeVDA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `-wa17B7TGnw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5ValueD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `0eUrW9JAxM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json6MallocEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Xzy0onDzSAc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `lt3J4dPvEtM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object4PairC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Wu-urs6MMpQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object4PairD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `TM0Sj+Ebins` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object8iteratorC2ERKS2_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `r5CShmcua+M` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object8iteratorC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `OxO+V0ouoS4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6Object8iteratorD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `patTzc7zBjw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6ObjectC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `tSRqQ-+Y4PE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6ObjectC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `SkKczWHdklc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6ObjectD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `1siRBursJjw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6StringC2EPKc](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `N78gkQzXRL4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6StringC2ERKS1_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `gALFgNfF+9I` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6StringC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `eG9E9M6XvTM` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6StringD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Ui7YFnSTCBw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json14InitParameter211getUserDataEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `GWtzpTKbZ+0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json14InitParameter212getAllocatorEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BykPm1ajcM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json14InitParameter217getFileBufferSizeEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Pcg6xQUhq9E` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. From e841746687ca9f84f607f6faff2f27afba07d942 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:32:47 +0200 Subject: [PATCH 033/318] feat(libSceJson2): implement parameter getters and mutable references --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 10 ++ .../libs/prx/libSceJson2/InventoryExports.cpp | 45 ------ core/libs/tests/GuestJsonReferences.cpp | 138 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 9 -- 5 files changed, 154 insertions(+), 55 deletions(-) create mode 100644 core/libs/tests/GuestJsonReferences.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index c6547a41ed..101ec1f668 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_reference_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonReferences.cpp) + target_include_directories(guest_json_reference_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_reference_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_reference_tests) + add_test(NAME guest_json_references COMMAND guest_json_reference_tests) add_test_executable(guest_json_alias_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonAliases.cpp) target_include_directories(guest_json_alias_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_alias_tests PRIVATE libSceJson2 libc) @@ -96,7 +101,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 3f308334b1..4ef3ac566f 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2886,4 +2886,14 @@ void APS5_VABI _ZN3sce4Json6StringC2EPKc(String* self, const char* text) { _ZN3s void APS5_VABI _ZN3sce4Json6StringC2ERKS1_(String* self, const String* other) { _ZN3sce4Json6StringC1ERKS1_(self, other); } void APS5_VABI _ZN3sce4Json6StringC2Ev(String* self) { _ZN3sce4Json6StringC1Ev(self); } void APS5_VABI _ZN3sce4Json6StringD2Ev(String* self) { _ZN3sce4Json6StringD1Ev(self); } + +String* APS5_VABI _ZN3sce4Json5Value11referStringEv(Value* self) { return self->type == TypeString ? self->string : nullptr; } +bool* APS5_VABI _ZN3sce4Json5Value12referBooleanEv(Value* self) { return self->type == TypeBoolean ? &self->boolean : nullptr; } +std::int64_t* APS5_VABI _ZN3sce4Json5Value12referIntegerEv(Value* self) { return self->type == TypeInteger || self->type == TypeUInteger ? &self->integer : nullptr; } +std::uint64_t* APS5_VABI _ZN3sce4Json5Value13referUIntegerEv(Value* self) { return self->type == TypeInteger || self->type == TypeUInteger ? &self->uinteger : nullptr; } +double* APS5_VABI _ZN3sce4Json5Value9referRealEv(Value* self) { return self->type == TypeReal ? &self->real : nullptr; } +void* APS5_VABI _ZNK3sce4Json14InitParameter211getUserDataEv(const InitParameter2* self) { return self->userData; } +void* APS5_VABI _ZNK3sce4Json14InitParameter212getAllocatorEv(const InitParameter2* self) { return self->allocator; } +std::size_t APS5_VABI _ZNK3sce4Json14InitParameter217getFileBufferSizeEv(const InitParameter2* self) { return self->fileBufferSize; } +std::int32_t APS5_VABI _ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv(const InitParameter2* self) { return self->specialFloatFormatType; } } diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index 3a2e99ff14..95fd6ae54b 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -75,31 +75,6 @@ int APS5_VABI _ZN3sce4Json5Array8pop_backEv() { return 0; } -int APS5_VABI _ZN3sce4Json5Value11referStringEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5Value12referBooleanEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5Value12referIntegerEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5Value13referUIntegerEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5Value9referRealEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_() { NotImplemented_nid_no_patch(__func__); return 0; @@ -115,26 +90,6 @@ int APS5_VABI _ZN3sce4Json6MallocEm() { return 0; } -int APS5_VABI _ZNK3sce4Json14InitParameter211getUserDataEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json14InitParameter212getAllocatorEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json14InitParameter217getFileBufferSizeEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/tests/GuestJsonReferences.cpp b/core/libs/tests/GuestJsonReferences.cpp new file mode 100644 index 0000000000..a70e6effd0 --- /dev/null +++ b/core/libs/tests/GuestJsonReferences.cpp @@ -0,0 +1,138 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using Mismatch = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +static_assert(sizeof(Value) == 32 && sizeof(Parameter) == 40 && offsetof(Parameter, mode) == 24); +extern "C" { +void* APS5_VABI _ZNK3sce4Json14InitParameter211getUserDataEv(const Parameter*); +void* APS5_VABI _ZNK3sce4Json14InitParameter212getAllocatorEv(const Parameter*); +std::size_t APS5_VABI _ZNK3sce4Json14InitParameter217getFileBufferSizeEv(const Parameter*); +std::int32_t APS5_VABI _ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv(const Parameter*); +String* APS5_VABI _ZN3sce4Json5Value11referStringEv(Value*); +bool* APS5_VABI _ZN3sce4Json5Value12referBooleanEv(Value*); +std::int64_t* APS5_VABI _ZN3sce4Json5Value12referIntegerEv(Value*); +std::uint64_t* APS5_VABI _ZN3sce4Json5Value13referUIntegerEv(Value*); +double* APS5_VABI _ZN3sce4Json5Value9referRealEv(Value*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value*, bool); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, Mismatch, void*); +const bool* APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +static void CheckParameters() { + int first = 1, second = 2; + const Parameter parameters[]{ + {nullptr, nullptr, 0, 0, {1, 2, 3}}, + {&first, &second, 8192, 2, {4, 5, 6}}, + {&second, &first, std::numeric_limits::max(), -17, {7, 8, 9}}, + }; + for (const auto& parameter : parameters) { + const auto before = parameter; + Require(_ZNK3sce4Json14InitParameter212getAllocatorEv(¶meter) == parameter.allocator && _ZNK3sce4Json14InitParameter211getUserDataEv(¶meter) == parameter.context, "Parameter getters must return exact pointer fields"); + Require(_ZNK3sce4Json14InitParameter217getFileBufferSizeEv(¶meter) == parameter.bufferSize && _ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv(¶meter) == parameter.mode, "Parameter getters must return field bits without normalizing or validating settings"); + Require(std::memcmp(&before, ¶meter, sizeof(parameter)) == 0, "Parameter getters must not mutate the parameter block"); + } +} +static void CheckReferences() { + int storage = 1, replacement = 2; + String string{&storage}; + for (std::int32_t type : {0, 1, 2, 3, 4, 5, 6, 7, 8, -1, INT32_MIN, INT32_MAX}) { + Value value{}; value.type = type; + value.payload = type == 5 ? reinterpret_cast(&string) : UINT64_C(0x1122334455667788); + const auto before = value; + auto* boolean = _ZN3sce4Json5Value12referBooleanEv(&value); + auto* integer = _ZN3sce4Json5Value12referIntegerEv(&value); + auto* uinteger = _ZN3sce4Json5Value13referUIntegerEv(&value); + auto* real = _ZN3sce4Json5Value9referRealEv(&value); + auto* text = _ZN3sce4Json5Value11referStringEv(&value); + Require(static_cast(boolean) == (type == 1 ? &value.payload : nullptr), "Boolean references must identify this value's payload or return null"); + Require(static_cast(integer) == ((type == 2 || type == 3) ? &value.payload : nullptr) && static_cast(uinteger) == static_cast(integer), "Both integer reference functions must accept signed and unsigned values and share their actual storage"); + Require(static_cast(real) == (type == 4 ? &value.payload : nullptr) && text == (type == 5 ? &string : nullptr), "Real and string references must return compatible owned storage without fallback"); + Require(std::memcmp(&before, &value, sizeof(value)) == 0, "Obtaining references must preserve the value and its metadata"); + if (boolean) { + *boolean = true; + Require(value.payload == ((before.payload & ~UINT64_C(255)) | 1), "Boolean references must permit in-place writes without changing inactive payload bytes"); + } + if (integer) { + *integer = -123; + Require(value.payload == std::bit_cast(std::int64_t{-123}), "Signed reference writes must preserve their exact payload bits"); + *uinteger = UINT64_MAX; + Require(value.payload == UINT64_MAX, "Unsigned reference writes must update the same integer payload"); + } + if (real) { + *real = -1.25; + Require(value.payload == std::bit_cast(-1.25), "Real references must permit writes to the actual payload"); + } + if (text) { + text->storage = &replacement; + Require(string.storage == &replacement, "String references must expose the actual mutable string handle"); + } + Require(value.type == before.type && value.parent == before.parent && value.auxiliary == before.auxiliary, "Reference writes must preserve type and ownership metadata"); + } + Value missing{}; missing.type = 5; + Require(_ZN3sce4Json5Value11referStringEv(&missing) == nullptr, "A string value with missing storage must return null without allocating a replacement"); +} +struct ReferenceTrace { + std::map live; + bool rejectAllocations = false; +}; +static void* APS5_VABI AllocateReference(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(!trace.rejectAllocations, "Reference and parameter access must not allocate"); + void* memory = std::malloc(bytes); Require(memory != nullptr, "Reference allocator requires host memory"); + trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeReference(void*, void* memory, void* context) { + if (!memory) return; + Require(static_cast(context)->live.erase(memory) == 1, "Reference metadata must return through the owning allocator exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyReference(void*, int, std::size_t, void*) { Require(false, "Reference tests must not encounter allocation failure"); } +static void* const ReferenceVtable[]{nullptr, nullptr, reinterpret_cast(AllocateReference), reinterpret_cast(FreeReference), reinterpret_cast(NotifyReference)}; +struct ReferenceAllocator { void* const* vtable = ReferenceVtable; }; +struct Recovery { Value replacement{}; unsigned calls = 0; }; +static const Value& APS5_VABI RecoverReference(std::int32_t type, const Value*, const Value*, void* context) { + auto& recovery = *static_cast(context); ++recovery.calls; + Require(type == 1, "The preservation control must request a boolean replacement"); + return recovery.replacement; +} +int main() { + CheckParameters(); CheckReferences(); + ReferenceTrace trace; ReferenceAllocator allocator; + Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize reference callback tests"); + Value value{}; Recovery recovery; + _ZN3sce4Json5ValueC1Ev(&value); _ZN3sce4Json5ValueC1Eb(&recovery.replacement, true); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&value, RecoverReference, &recovery) == 0, "Register a mismatch handler for the callback-bypass test"); + trace.rejectAllocations = true; + CheckParameters(); CheckReferences(); + for (unsigned attempt = 0; attempt < 2; ++attempt) { + Require(!_ZN3sce4Json5Value12referBooleanEv(&value) && !_ZN3sce4Json5Value12referIntegerEv(&value) && !_ZN3sce4Json5Value13referUIntegerEv(&value) && !_ZN3sce4Json5Value9referRealEv(&value) && !_ZN3sce4Json5Value11referStringEv(&value), "Reference accessors must return null for mismatched values even when recovery is registered"); + Require(recovery.calls == attempt, "Reference accessors must never invoke recovery callbacks"); + Require(*_ZNK3sce4Json5Value10getBooleanEv(&value) && recovery.calls == attempt + 1, "Ordinary getters must preserve their working callback recovery behavior"); + } + trace.rejectAllocations = false; + _ZN3sce4Json5ValueD1Ev(&recovery.replacement); _ZN3sce4Json5ValueD1Ev(&value); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Reference teardown must reclaim all initialized resources and callback metadata"); + CheckParameters(); CheckReferences(); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 8e5de886ce..481fb50bd3 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -66,18 +66,9 @@ - [_ZN3sce4Json5Array5eraseERKNS1_8iteratorE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2mbVAtshWRI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ixJAz92H1uE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array8pop_backEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `woec1toi4Xw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value11referStringEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `m5J28iw-QPs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value12referBooleanEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `QxjfcfBhEdc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value12referIntegerEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R0ac5QOdlpo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value13referUIntegerEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Nx5tez4siL4` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value9referRealEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `t7UEJrhojhk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json6MallocEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Xzy0onDzSAc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json14InitParameter211getUserDataEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `GWtzpTKbZ+0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json14InitParameter212getAllocatorEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BykPm1ajcM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json14InitParameter217getFileBufferSizeEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Pcg6xQUhq9E` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json14InitParameter225getSpecialFloatFormatTypeEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PDkib0zyABg` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json5ValueixERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `clF7J7N9xXE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json6String2atEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `KrKM-5SdQrA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json6String4findEPKcm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `yjsf6SvXA08` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. From 764ab1720974e4c033cb30f3e98aff52b7431d0a Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:39:54 +0200 Subject: [PATCH 034/318] feat(libSceJson2): implement string access and substring search --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 ++- core/libs/prx/libSceJson2/Export.cpp | 52 +++++++++++++++- .../libs/prx/libSceJson2/InventoryExports.cpp | 35 ----------- core/libs/tests/GuestJsonStringSearch.cpp | 60 +++++++++++++++++++ docs/dev/debt/libSceJson2.md | 9 +-- 5 files changed, 118 insertions(+), 45 deletions(-) create mode 100644 core/libs/tests/GuestJsonStringSearch.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 101ec1f668..617ea4c000 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_string_search_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonStringSearch.cpp) + target_include_directories(guest_json_string_search_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_string_search_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_string_search_tests) + add_test(NAME guest_json_string_search COMMAND guest_json_string_search_tests) add_test_executable(guest_json_reference_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonReferences.cpp) target_include_directories(guest_json_reference_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_reference_tests PRIVATE libSceJson2 libc) @@ -101,7 +106,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 4ef3ac566f..2d2d553ad8 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -324,6 +324,45 @@ const char* Characters(const String& string) { return string.text ? string.text->characters : emptyCharacters; } +bool SearchEqual(const char* left, const char* right, std::size_t count) { + const auto bounded = std::bit_cast(static_cast(count)); + return bounded <= 0 || std::strncmp(left, right, static_cast(bounded)) == 0; +} + +std::size_t FindText(const String& string, const char* needle, std::size_t position, std::size_t count) { + constexpr auto missing = std::numeric_limits::max(); + if (!string.text) return missing; + const std::size_t length = string.text->length; + if (!count) return position <= length ? position : missing; + if (position >= length || count > length - position) return missing; + if (!needle) { NotImplemented_nid_no_patch(__func__); return missing; } + const char* begin = string.text->characters; + const char* cursor = begin + position; + while (cursor < begin + length) { + const char* candidate = std::strchr(cursor, static_cast(*needle)); + if (!candidate) return missing; + if (SearchEqual(candidate, needle, count)) return static_cast(candidate - begin); + cursor = candidate + 1; + } + return missing; +} + +std::size_t ReverseFindText(const String& string, const char* needle, std::size_t position, std::size_t count) { + constexpr auto missing = std::numeric_limits::max(); + if (!string.text) return missing; + const std::size_t length = string.text->length; + if (!count) return std::min(position, length); + if (count > length) return missing; + if (!needle) { NotImplemented_nid_no_patch(__func__); return missing; } + std::size_t index = std::min(position, length - count); + for (;;) { + const char* candidate = string.text->characters + index; + if (*candidate == *needle && SearchEqual(candidate, needle, count)) return index; + if (!index) return missing; + --index; + } +} + std::string_view View(const String& string) { return {Characters(string), string.text ? string.text->length : 0}; } @@ -2220,8 +2259,17 @@ String* APS5_VABI _ZNK3sce4Json6String6substrEmm(String* result, const String* s Construct(*result, View(*self).substr(position, length)); return result; } -std::size_t APS5_VABI _ZNK3sce4Json6String4findEcm(const String* self, char character, std::size_t position) { return View(*self).find(character, position); } -std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEcm(const String* self, char character, std::size_t position) { return View(*self).rfind(character, position); } +char APS5_VABI _ZNK3sce4Json6String2atEm(const String* self, std::size_t index) { + return !self->text ? 0 : index < self->text->length ? self->text->characters[index] : emptyCharacters[0]; +} +std::size_t APS5_VABI _ZNK3sce4Json6String4findEPKcm(const String* self, const char* needle, std::size_t position) { return FindText(*self, needle, position, needle ? std::strlen(needle) : 0); } +std::size_t APS5_VABI _ZNK3sce4Json6String4findEPKcmm(const String* self, const char* needle, std::size_t position, std::size_t count) { return FindText(*self, needle, position, count); } +std::size_t APS5_VABI _ZNK3sce4Json6String4findERKS1_m(const String* self, const String* needle, std::size_t position) { return FindText(*self, Characters(*needle), position, std::strlen(Characters(*needle))); } +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEPKcm(const String* self, const char* needle, std::size_t position) { return ReverseFindText(*self, needle, position, needle ? std::strlen(needle) : 0); } +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEPKcmm(const String* self, const char* needle, std::size_t position, std::size_t count) { return ReverseFindText(*self, needle, position, count); } +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindERKS1_m(const String* self, const String* needle, std::size_t position) { return ReverseFindText(*self, Characters(*needle), position, std::strlen(Characters(*needle))); } +std::size_t APS5_VABI _ZNK3sce4Json6String4findEcm(const String* self, char character, std::size_t position) { return FindText(*self, &character, position, 1); } +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEcm(const String* self, char character, std::size_t position) { return ReverseFindText(*self, &character, position, 1); } int APS5_VABI _ZNK3sce4Json6String7compareEPKc(const String* self, const char* text) { if (!text) throw std::invalid_argument("sce::Json::String::compare: null text"); return CompareText(Characters(*self), text); diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index 95fd6ae54b..bf13693713 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -95,39 +95,4 @@ int APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE() { return 0; } -int APS5_VABI _ZNK3sce4Json6String2atEm() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json6String4findEPKcm() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json6String4findEPKcmm() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json6String4findERKS1_m() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json6String5rfindEPKcm() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json6String5rfindEPKcmm() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZNK3sce4Json6String5rfindERKS1_m() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - } diff --git a/core/libs/tests/GuestJsonStringSearch.cpp b/core/libs/tests/GuestJsonStringSearch.cpp new file mode 100644 index 0000000000..763fe6e3da --- /dev/null +++ b/core/libs/tests/GuestJsonStringSearch.cpp @@ -0,0 +1,60 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct SearchText { char* characters; std::uint32_t length; std::uint32_t capacity; std::uint64_t reserved; }; +struct String { SearchText* text; }; +extern "C" { +char APS5_VABI _ZNK3sce4Json6String2atEm(const String*, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String4findEPKcm(const String*, const char*, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String4findEPKcmm(const String*, const char*, std::size_t, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String4findERKS1_m(const String*, const String*, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEPKcm(const String*, const char*, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEPKcmm(const String*, const char*, std::size_t, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindERKS1_m(const String*, const String*, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String4findEcm(const String*, char, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String5rfindEcm(const String*, char, std::size_t); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +int main() { + constexpr auto none = std::numeric_limits::max(); + char bytes[]{'a', 'b', 'X', '\0', 'X', static_cast(0xff), '\0'}; + SearchText header{bytes, 6, 7, UINT64_C(0x123456789abcdef0)}; + String text{&header}, missing{}; + char needleBytes[]{'X', '\0', 'q', '\0'}; + SearchText needleHeader{needleBytes, 3, 4, 0}; + String needle{&needleHeader}; + const auto before = header; + char snapshot[sizeof(bytes)]; std::memcpy(snapshot, bytes, sizeof(bytes)); + for (std::size_t i = 0; i < sizeof(bytes); ++i) { + Require(_ZNK3sce4Json6String2atEm(&text, i) == bytes[i], "String at must return the indexed byte or zero at the boundary"); + } + Require(_ZNK3sce4Json6String2atEm(&text, none) == 0 && _ZNK3sce4Json6String2atEm(&missing, 0) == 0, "Out-of-range and missing-storage access must return zero"); + Require(_ZNK3sce4Json6String4findEPKcm(&text, "X", 0) == 2 && _ZNK3sce4Json6String4findEPKcm(&text, "X", 3) == none && _ZNK3sce4Json6String4findEPKcm(&text, "X", 4) == 4, "Forward searches must stop at an embedded zero and honor the starting position"); + Require(_ZNK3sce4Json6String5rfindEPKcm(&text, "X", none) == 4 && _ZNK3sce4Json6String5rfindEPKcm(&text, "X", 3) == 2, "Reverse searches must scan bytes across embedded zeros"); + Require(_ZNK3sce4Json6String4findERKS1_m(&text, &needle, 0) == 2 && _ZNK3sce4Json6String5rfindERKS1_m(&text, &needle, none) == 4, "String needles must use their C-string prefix rather than their stored length"); + Require(_ZNK3sce4Json6String4findEPKcmm(&text, needleBytes, 0, 3) == 2 && _ZNK3sce4Json6String5rfindEPKcmm(&text, needleBytes, none, 3) == 2, "Counted matching must stop comparing at a shared zero"); + for (std::size_t position : {std::size_t{0}, std::size_t{6}, std::size_t{7}, none}) { + const auto forward = position <= 6 ? position : none; + const auto reverse = position < 6 ? position : 6; + Require(_ZNK3sce4Json6String4findEPKcm(&text, nullptr, position) == forward && _ZNK3sce4Json6String4findEPKcmm(&text, nullptr, position, 0) == forward && _ZNK3sce4Json6String4findERKS1_m(&text, &missing, position) == forward, "Empty forward needles must validate the starting position"); + Require(_ZNK3sce4Json6String5rfindEPKcm(&text, nullptr, position) == reverse && _ZNK3sce4Json6String5rfindEPKcmm(&text, nullptr, position, 0) == reverse && _ZNK3sce4Json6String5rfindERKS1_m(&text, &missing, position) == reverse, "Empty reverse needles must clamp the starting position"); + } + Require(_ZNK3sce4Json6String4findEPKcm(&missing, "", 0) == none && _ZNK3sce4Json6String5rfindEPKcm(&missing, "", 0) == none, "Missing haystack storage must reject even empty needles"); + Require(_ZNK3sce4Json6String4findEPKcmm(&text, nullptr, 0, none) == none && _ZNK3sce4Json6String5rfindEPKcmm(&text, nullptr, none, none) == none, "Oversized counts must be rejected before touching the needle"); + Require(_ZNK3sce4Json6String4findEcm(&text, '\0', 0) == 3 && _ZNK3sce4Json6String4findEcm(&text, 'X', 3) == none && _ZNK3sce4Json6String5rfindEcm(&text, '\0', none) == 3, "Character search must preserve forward C-string and reverse byte semantics"); + Require(_ZNK3sce4Json6String4findEcm(&text, static_cast(0xff), 4) == 5 && _ZNK3sce4Json6String5rfindEcm(&text, static_cast(0xff), none) == 5, "Character search must preserve high-bit byte values"); + char tailBytes[]{'a', 'b', 'X', '\0'}; + SearchText tailHeader{tailBytes, 3, 4, 0}; String tail{&tailHeader}; + Require(_ZNK3sce4Json6String4findEPKcmm(&tail, "X", 0, 2) == 2, "Forward candidates must retain native zero-terminated comparison beyond the initial length check"); + Require(_ZNK3sce4Json6String4findEcm(&tail, '\0', 0) == 3 && _ZNK3sce4Json6String4findEcm(&tail, '\0', 3) == none && _ZNK3sce4Json6String5rfindEcm(&tail, '\0', none) == none, "Only forward character search may include the trailing terminator after an admitted start"); + Require(std::memcmp(&header, &before, sizeof(header)) == 0 && std::memcmp(bytes, snapshot, sizeof(bytes)) == 0 && text.text == &header, "Read-only string operations must preserve all storage and metadata"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 481fb50bd3..b3f2b3b9fe 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -70,10 +70,5 @@ - [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json6MallocEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Xzy0onDzSAc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json5ValueixERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `clF7J7N9xXE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String2atEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `KrKM-5SdQrA` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String4findEPKcm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `yjsf6SvXA08` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String4findEPKcmm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `wwDnDDsQVc0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String4findERKS1_m](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `awkeNR6Ukp8` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String5rfindEPKcm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `VpQBXbaHXOo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String5rfindEPKcmm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `A29RHrlfTM0` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json6String5rfindERKS1_m](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `-R9mdXBspeQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. + +- Counted string searches with an admitted positive count and a null needle remain explicitly unsupported; the reference dereferences the invalid pointer. Oversized and zero counts retain their verified results. From 1d98d6f07e013c6903abf5daf9030a804a58c12a Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:42:59 +0200 Subject: [PATCH 035/318] feat(libSceJson2): implement allocator entry points and preserve destructor state --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 14 ++- .../libs/prx/libSceJson2/InventoryExports.cpp | 20 ---- core/libs/tests/GuestJsonAllocationEntry.cpp | 103 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 8 +- 5 files changed, 123 insertions(+), 29 deletions(-) create mode 100644 core/libs/tests/GuestJsonAllocationEntry.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 617ea4c000..10e24f9f72 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_allocation_entry_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonAllocationEntry.cpp) + target_include_directories(guest_json_allocation_entry_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_allocation_entry_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_allocation_entry_tests) + add_test(NAME guest_json_allocation_entry COMMAND guest_json_allocation_entry_tests) add_test_executable(guest_json_string_search_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonStringSearch.cpp) target_include_directories(guest_json_string_search_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_string_search_tests PRIVATE libSceJson2 libc) @@ -106,7 +111,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 2d2d553ad8..01340be636 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2200,8 +2200,18 @@ void APS5_VABI _ZN3sce4Json12MemAllocatorC2Ev(void* self) { *static_cast(self) = &JsonMemAllocatorVtable[2]; } -void APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void* self) { - *static_cast(self) = &JsonMemAllocatorVtable[2]; +void APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void*) {} + +void APS5_VABI _ZN3sce4Json12MemAllocatorD1Ev(void*) {} + +void* APS5_VABI _ZN3sce4Json6MallocEm(std::size_t size) { return Allocate(size); } + +void APS5_VABI _ZN3sce4Json4FreeEPv(void* memory) { Release(memory); } + +int APS5_VABI _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_(void*, void (APS5_VABI*)(int, ValueType, void*), void*) { + auto& state = State(); + std::lock_guard lock(state.mutex); + return state.allocation ? 0 : JsonErrorNotInitialized; } std::size_t _ZN3sce4Json6String4nposE = static_cast(-1); diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index bf13693713..a80e666949 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -5,21 +5,11 @@ extern "C" { -int APS5_VABI _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json12MemAllocatorD0Ev() { NotImplemented_nid_no_patch(__func__); return 0; } -int APS5_VABI _ZN3sce4Json12MemAllocatorD1Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json19InternalInitializer19terminate_staticlibEv() { NotImplemented_nid_no_patch(__func__); return 0; @@ -50,11 +40,6 @@ int APS5_VABI _ZN3sce4Json19InternalInitializerD2Ev() { return 0; } -int APS5_VABI _ZN3sce4Json4FreeEPv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE() { NotImplemented_nid_no_patch(__func__); return 0; @@ -85,11 +70,6 @@ int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_() { return 0; } -int APS5_VABI _ZN3sce4Json6MallocEm() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/tests/GuestJsonAllocationEntry.cpp b/core/libs/tests/GuestJsonAllocationEntry.cpp new file mode 100644 index 0000000000..d705a26cfc --- /dev/null +++ b/core/libs/tests/GuestJsonAllocationEntry.cpp @@ -0,0 +1,103 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Parameter { void* allocator; void* context; std::size_t fileBuffer; }; +using InfoCallback = void (APS5_VABI*)(int, std::int32_t, void*); +extern "C" { +void* APS5_VABI _ZN3sce4Json6MallocEm(std::size_t); +void APS5_VABI _ZN3sce4Json4FreeEPv(void*); +int APS5_VABI _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_(void*, InfoCallback, void*); +void APS5_VABI _ZN3sce4Json12MemAllocatorD1Ev(void*); +void APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct EntryTrace { + std::map live; + std::vector requests; + std::vector frees; + void* expectedAllocator = nullptr; + bool reject = false; + unsigned notices = 0; + std::size_t failedSize = 0; + int notifiedState = 0; + int firstRequestState = 0; + int laterRequestState = 0; +}; +static void APS5_VABI Info(int, std::int32_t, void*) { Require(false, "The reference allocator-info setter must not install or invoke the supplied callback"); } +static int InfoState() { return _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_(nullptr, Info, nullptr); } +static void* APS5_VABI EntryAllocate(void* self, std::size_t size, void* context) { + auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator, "Allocation dispatch must preserve the allocator receiver and user context"); + if (trace.requests.empty()) trace.firstRequestState = InfoState(); + else trace.laterRequestState = InfoState(); + trace.requests.push_back(size); + if (trace.reject) return nullptr; + void* pointer = std::malloc(size ? size : 1); + Require(pointer != nullptr && trace.live.emplace(pointer, size).second, "Tracking allocator must provide independently owned host storage"); + return pointer; +} +static void APS5_VABI EntryFree(void* self, void* memory, void* context) { + auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator, "Free dispatch must preserve the allocator receiver and user context"); + trace.frees.push_back(memory); + if (memory) { Require(trace.live.erase(memory) == 1, "Every allocation must be released exactly once through its allocator"); std::free(memory); } +} +static void APS5_VABI EntryNotify(void* self, int error, std::size_t size, void* context) { + auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator && static_cast(error) == 0x80848102, "Allocation failure must notify the owning allocator with the Json2 error"); + ++trace.notices; trace.failedSize = size; trace.notifiedState = InfoState(); +} +int main() { + std::uint64_t allocatorBytes[]{UINT64_C(0x1122334455667788), UINT64_C(0xaabbccddeeff0099)}; + const auto first = allocatorBytes[0], second = allocatorBytes[1]; + _ZN3sce4Json12MemAllocatorD2Ev(allocatorBytes); + Require(allocatorBytes[0] == first && allocatorBytes[1] == second, "Base allocator destruction must not rewrite the vtable or adjacent bytes"); + _ZN3sce4Json12MemAllocatorD1Ev(allocatorBytes); + Require(allocatorBytes[0] == first && allocatorBytes[1] == second, "Both non-deleting allocator destructors must leave the complete object unchanged"); + constexpr int notInitialized = static_cast(0x80848110); + Require(InfoState() == notInitialized && _ZN3sce4Json6MallocEm(17) == nullptr, "Allocation entry points must expose missing context before initialization"); + _ZN3sce4Json4FreeEPv(nullptr); + void* table[]{nullptr, nullptr, reinterpret_cast(&EntryAllocate), reinterpret_cast(&EntryFree), reinterpret_cast(&EntryNotify)}; + void** allocator = table; + EntryTrace trace; trace.expectedAllocator = &allocator; + Parameter parameter{&allocator, &trace, 8192}; + std::uint8_t initializer = 0xa5; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&initializer, ¶meter) == 0, "Entry-point tests require successful production initialization"); + Require(trace.firstRequestState == notInitialized && trace.laterRequestState == 0 && InfoState() == 0, "Allocator-info admission must follow context publication, including during resource initialization"); + std::uint8_t unrelated = 0; + Require(_ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_(&unrelated, nullptr, &trace) == 0, "Allocator-info admission must ignore the receiver, callback and context arguments"); + for (std::size_t size : {std::size_t{0}, std::size_t{1}, std::size_t{257}}) { + const auto requests = trace.requests.size(), frees = trace.frees.size(), live = trace.live.size(); + void* memory = _ZN3sce4Json6MallocEm(size); + Require(memory && trace.requests.size() == requests + 1 && trace.requests.back() == size && trace.live.at(memory) == size, "Malloc must forward exact size and return the allocator's unmodified pointer"); + _ZN3sce4Json4FreeEPv(memory); + Require(trace.frees.size() == frees + 1 && trace.frees.back() == memory && trace.live.size() == live, "Free must forward the exact allocation without retaining ownership"); + } + const auto frees = trace.frees.size(); + _ZN3sce4Json4FreeEPv(nullptr); + Require(trace.frees.size() == frees + 1 && trace.frees.back() == nullptr, "An active allocator must receive even a null Free argument"); + trace.reject = true; + Require(_ZN3sce4Json6MallocEm(4097) == nullptr && trace.notices == 1 && trace.failedSize == 4097 && trace.notifiedState == 0, "Failed Malloc must notify once with the original size while keeping the allocation context active"); + trace.reject = false; + void* retry = _ZN3sce4Json6MallocEm(9); + Require(retry != nullptr && trace.notices == 1, "A failed direct allocation must permit a later successful retry"); + _ZN3sce4Json4FreeEPv(retry); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Termination must release all required resources through the same allocator"); + const auto finalRequests = trace.requests.size(), finalFrees = trace.frees.size(); + Require(InfoState() == notInitialized && _ZN3sce4Json6MallocEm(13) == nullptr, "Termination must withdraw the allocation context"); + _ZN3sce4Json4FreeEPv(nullptr); + Require(trace.requests.size() == finalRequests && trace.frees.size() == finalFrees && trace.notices == 1, "Missing-context calls must not dispatch stale allocator callbacks"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index b3f2b3b9fe..37cfa29ecd 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -52,23 +52,19 @@ ### Unknown 3.20 import declarations -- [_ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `R996H0YIhiQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json12MemAllocatorD1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `O1uwoqazS2A` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. +- [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE`; the deleting base allocator destructor traps in the reference. This invalid destruction path remains explicitly unsupported. - [_ZN3sce4Json19InternalInitializer19terminate_staticlibEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `dvp2zGSQzMs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BPSPc3pUMJo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json19InternalInitializerC1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `WntRKhYTmZw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json19InternalInitializerC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2n+fxNdSZfs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json19InternalInitializerD1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `yudT4-bglko` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json19InternalInitializerD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `caP-y4fH-zQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json4FreeEPv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `bnk+edDbqMk` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array10push_frontERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MQxDxBzWz40` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array5eraseERKNS1_8iteratorE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2mbVAtshWRI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ixJAz92H1uE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array8pop_backEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `woec1toi4Xw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json6MallocEm](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `Xzy0onDzSAc` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json5ValueixERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `clF7J7N9xXE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- Counted string searches with an admitted positive count and a null needle remain explicitly unsupported; the reference dereferences the invalid pointer. Oversized and zero counts retain their verified results. +- [Counted string searches](../../../core/libs/prx/libSceJson2/Export.cpp) with an admitted positive count and a null needle remain explicitly unsupported; the reference dereferences the invalid pointer. Oversized and zero counts retain their verified results. From 869eab36e06da63297bdb0218a51db68e842ad3e Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:46:59 +0200 Subject: [PATCH 036/318] feat(libSceJson2): implement internal initializer lifecycle --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 33 ++++- .../libs/prx/libSceJson2/InventoryExports.cpp | 30 ----- .../tests/GuestJsonInternalInitializer.cpp | 126 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 6 - 5 files changed, 163 insertions(+), 39 deletions(-) create mode 100644 core/libs/tests/GuestJsonInternalInitializer.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 10e24f9f72..1ab6f9ded6 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_internal_initializer_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInternalInitializer.cpp) + target_include_directories(guest_json_internal_initializer_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_internal_initializer_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_internal_initializer_tests) + add_test(NAME guest_json_internal_initializer COMMAND guest_json_internal_initializer_tests) add_test_executable(guest_json_allocation_entry_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonAllocationEntry.cpp) target_include_directories(guest_json_allocation_entry_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_allocation_entry_tests PRIVATE libSceJson2 libc) @@ -111,7 +116,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 01340be636..b224567735 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -147,6 +147,7 @@ struct Globals { void* context = nullptr; std::int32_t floatFormat = 2; bool floatFormatEnabled = false; + bool paddedStringGrowth = false; bool initialized = false; AllocParamRtti rtti{}; Handlers handlers{}; @@ -387,8 +388,13 @@ bool Grow(Text& text, std::size_t requested) { throw std::length_error("sce::Json::String: capacity exceeds 32 bits"); if (requested <= text.capacity) return true; std::uint32_t capacity = static_cast(requested); - const auto sdk = CompiledSdk(); - if (sdk > 0x044fffff) { + bool padded; + { + auto& state = State(); + std::lock_guard lock(state.mutex); + padded = state.paddedStringGrowth; + } + if (padded) { capacity |= 0x10; if (capacity / 3 < text.capacity / 2) { const auto grown = std::uint64_t{text.capacity} + text.capacity / 2; @@ -1394,6 +1400,7 @@ void ReleaseInitialization() { } state.floatFormat = 2; state.floatFormatEnabled = false; + state.paddedStringGrowth = false; state.initialized = false; } @@ -1439,6 +1446,7 @@ int InitializeResources(void* self, const void* parameter, std::int32_t format, if (admitted) state.recoveryMutex = ModernValues() ? CreateKernelMutex() : state.internalMutex; if (!admitted || !state.recoveryMutex) { ReleaseInitialization(); return JsonErrorOutOfMemory; } } catch (...) { ReleaseInitialization(); throw; } + state.paddedStringGrowth = CompiledSdk() > 0x044fffff; state.initialized = true; *static_cast(self) = 1; return 0; @@ -2155,6 +2163,27 @@ int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* return InitializeResources(self, parameter, 2, false); } +void APS5_VABI _ZN3sce4Json19InternalInitializerC1Ev(void* self) { _ZN3sce4Json11InitializerC1Ev(self); } + +void APS5_VABI _ZN3sce4Json19InternalInitializerC2Ev(void* self) { _ZN3sce4Json11InitializerC1Ev(self); } + +void APS5_VABI _ZN3sce4Json19InternalInitializerD1Ev(void* self) { + _ZN3sce4Json11Initializer9terminateEv(self); + _ZN3sce4Json11Initializer9terminateEv(self); +} + +void APS5_VABI _ZN3sce4Json19InternalInitializerD2Ev(void* self) { _ZN3sce4Json19InternalInitializerD1Ev(self); } + +int APS5_VABI _ZN3sce4Json19InternalInitializer19terminate_staticlibEv(void* self) { return _ZN3sce4Json11Initializer9terminateEv(self); } + +int APS5_VABI _ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(void* self, const void* parameter, std::int32_t option) { + auto& state = State(); + std::lock_guard callbackLock(state.callbackMutex); + std::lock_guard lock(state.mutex); + state.paddedStringGrowth = option == 1; + return _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(self, parameter); +} + int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { auto& state = State(); std::lock_guard callbackLock(state.callbackMutex); diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index a80e666949..4d1ec0b2a4 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -10,36 +10,6 @@ int APS5_VABI _ZN3sce4Json12MemAllocatorD0Ev() { return 0; } -int APS5_VABI _ZN3sce4Json19InternalInitializer19terminate_staticlibEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json19InternalInitializerC1Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json19InternalInitializerC2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json19InternalInitializerD1Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json19InternalInitializerD2Ev() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/tests/GuestJsonInternalInitializer.cpp b/core/libs/tests/GuestJsonInternalInitializer.cpp new file mode 100644 index 0000000000..6eb178844b --- /dev/null +++ b/core/libs/tests/GuestJsonInternalInitializer.cpp @@ -0,0 +1,126 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Parameter { void* allocator; void* context; std::size_t fileBuffer; }; +using InfoCallback = void (APS5_VABI*)(int, std::int32_t, void*); +extern "C" { +void APS5_VABI _ZN3sce4Json19InternalInitializerC1Ev(void*); +void APS5_VABI _ZN3sce4Json19InternalInitializerC2Ev(void*); +void APS5_VABI _ZN3sce4Json19InternalInitializerD1Ev(void*); +void APS5_VABI _ZN3sce4Json19InternalInitializerD2Ev(void*); +int APS5_VABI _ZN3sce4Json19InternalInitializer19terminate_staticlibEv(void*); +int APS5_VABI _ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(void*, const Parameter*, std::int32_t); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(void*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(void*); +void APS5_VABI _ZN3sce4Json6String7reserveEm(void*, std::size_t); +std::size_t APS5_VABI _ZNK3sce4Json6String8capacityEv(const void*); +void* APS5_VABI _ZN3sce4Json6MallocEm(std::size_t); +void APS5_VABI _ZN3sce4Json4FreeEPv(void*); +int APS5_VABI _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_(void*, InfoCallback, void*); +void APS5_VABI _ZN3sce4Json12MemAllocatorD1Ev(void*); +void APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct EntryTrace { + std::map live; + std::vector requests; + std::vector frees; + void* expectedAllocator = nullptr; + bool reject = false; + int budget = -1; + unsigned notices = 0; + std::size_t failedSize = 0; + int notifiedState = 0; + int firstRequestState = 0; + int laterRequestState = 0; +}; +static void APS5_VABI Info(int, std::int32_t, void*) { Require(false, "The reference allocator-info setter must not install or invoke the supplied callback"); } +static int InfoState() { return _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_(nullptr, Info, nullptr); } +static void* APS5_VABI EntryAllocate(void* self, std::size_t size, void* context) { + auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator, "Allocation dispatch must preserve the allocator receiver and user context"); + if (trace.requests.empty()) trace.firstRequestState = InfoState(); + else trace.laterRequestState = InfoState(); + trace.requests.push_back(size); + if (trace.reject || trace.budget == 0) return nullptr; + if (trace.budget > 0) --trace.budget; + void* pointer = std::malloc(size ? size : 1); + Require(pointer != nullptr && trace.live.emplace(pointer, size).second, "Tracking allocator must provide independently owned host storage"); + return pointer; +} +static void APS5_VABI EntryFree(void* self, void* memory, void* context) { + auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator, "Free dispatch must preserve the allocator receiver and user context"); + trace.frees.push_back(memory); + if (memory) { Require(trace.live.erase(memory) == 1, "Every allocation must be released exactly once through its allocator"); std::free(memory); } +} +static void APS5_VABI EntryNotify(void* self, int error, std::size_t size, void* context) { + auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator && static_cast(error) == 0x80848102, "Allocation failure must notify the owning allocator with the Json2 error"); + ++trace.notices; trace.failedSize = size; trace.notifiedState = InfoState(); +} +int main() { + constexpr auto already = static_cast(0x80848111); + constexpr auto missing = static_cast(0x80848110); + constexpr auto invalid = static_cast(0x80848120); + constexpr auto noMemory = static_cast(0x80848102); + void* table[]{nullptr, nullptr, reinterpret_cast(&EntryAllocate), reinterpret_cast(&EntryFree), reinterpret_cast(&EntryNotify)}; + void** allocator = table; + for (std::int32_t option : {0, 1, 2, -1}) { + for (int budget = 0; budget <= 12; ++budget) { + EntryTrace trace; trace.expectedAllocator = &allocator; trace.budget = budget; + Parameter parameter{&allocator, &trace, 8192}; + std::uint8_t object[]{0xa5, 0x19, 0x28, 0x37}; + if (budget % 2) _ZN3sce4Json19InternalInitializerC1Ev(object); + else _ZN3sce4Json19InternalInitializerC2Ev(object); + Require(object[0] == 0 && object[1] == 0x19 && object[2] == 0x28 && object[3] == 0x37, "Internal constructors must initialize only the active byte"); + Require(_ZN3sce4Json19InternalInitializer19terminate_staticlibEv(object) == missing, "Inactive internal termination must return not-initialized"); + Require(_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(object, nullptr, option) == invalid, "Internal initialization must reject a missing parameter"); + Parameter empty{}; + Require(_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(object, &empty, option) == noMemory, "Legacy internal initialization must report a missing allocator as out-of-memory"); + const int result = _ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(object, ¶meter, option); + if (budget < 12) { + Require(result == noMemory && object[0] == 0 && trace.live.empty(), "Every required-resource allocation failure must roll back ownership and clear the active byte"); + Require((budget == 0) == (trace.notices == 0), "The initial context failure must omit notification while resource failures notify"); + trace.budget = -1; + Require(_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(object, ¶meter, option) == 0, "Retry must succeed without reconstructing the failed internal initializer"); + } else Require(result == 0, "Internal initialization must admit every option value after allocating all resources"); + trace.budget = -1; + Require(object[0] == 1 && trace.live.size() == 12, "Internal initialization must own the same twelve required resources as legacy public initialization"); + void* text = nullptr; + _ZN3sce4Json6StringC1EPKc(&text, "abc"); + _ZN3sce4Json6String7reserveEm(&text, 5); + Require(_ZNK3sce4Json6String8capacityEv(&text) == 21, "Successful initialization must replace the requested growth option with the compiled SDK policy"); + _ZN3sce4Json6StringD1Ev(&text); + for (std::int32_t rejectedOption : {0, 1, 2, -1}) { + std::uint8_t other = 0; + Require(_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(&other, nullptr, rejectedOption) == already && other == 0 && object[0] == 1, "Global-context admission must precede parameter validation without taking ownership in another initializer"); + _ZN3sce4Json6StringC1EPKc(&text, "abc"); + _ZN3sce4Json6String7reserveEm(&text, 5); + Require(_ZNK3sce4Json6String8capacityEv(&text) == (rejectedOption == 1 ? 21 : 5), "Even a rejected internal initialization must update the string-growth option"); + _ZN3sce4Json6StringD1Ev(&text); + Require(_ZN3sce4Json19InternalInitializer19terminate_staticlibEv(&other) == missing && trace.live.size() == 12, "An inactive initializer must not terminate another initializer's resources"); + } + if (budget % 3 == 0) _ZN3sce4Json19InternalInitializerD1Ev(object); + else if (budget % 3 == 1) _ZN3sce4Json19InternalInitializerD2Ev(object); + else Require(_ZN3sce4Json19InternalInitializer19terminate_staticlibEv(object) == 0, "Explicit internal termination must succeed for its owning initializer"); + Require(object[0] == 0 && object[1] == 0x19 && object[2] == 0x28 && object[3] == 0x37 && trace.live.empty(), "Every internal teardown entry point must release all resources exactly once and preserve adjacent bytes"); + const auto frees = trace.frees.size(); + _ZN3sce4Json19InternalInitializerD1Ev(object); + _ZN3sce4Json19InternalInitializerD2Ev(object); + Require(trace.frees.size() == frees && InfoState() == missing, "Repeated internal destruction must not dispatch stale frees"); + } + } + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 37cfa29ecd..c3acd02228 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -53,12 +53,6 @@ ### Unknown 3.20 import declarations - [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE`; the deleting base allocator destructor traps in the reference. This invalid destruction path remains explicitly unsupported. -- [_ZN3sce4Json19InternalInitializer19terminate_staticlibEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `dvp2zGSQzMs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `BPSPc3pUMJo` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializerC1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `WntRKhYTmZw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializerC2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2n+fxNdSZfs` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializerD1Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `yudT4-bglko` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json19InternalInitializerD2Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `caP-y4fH-zQ` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array10push_frontERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MQxDxBzWz40` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array5eraseERKNS1_8iteratorE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2mbVAtshWRI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ixJAz92H1uE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. From 57b302c9a985720e1826ad764c1abccdc7cda800 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:49:38 +0200 Subject: [PATCH 037/318] feat(libSceJson2): implement array erasure and back removal --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 27 +++ .../libs/prx/libSceJson2/InventoryExports.cpp | 10 - core/libs/tests/GuestJsonArrayMutation.cpp | 191 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 5 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 core/libs/tests/GuestJsonArrayMutation.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 1ab6f9ded6..de2cedb8fa 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_array_mutation_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonArrayMutation.cpp) + target_include_directories(guest_json_array_mutation_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_array_mutation_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_array_mutation_tests) + add_test(NAME guest_json_array_mutation COMMAND guest_json_array_mutation_tests) add_test_executable(guest_json_internal_initializer_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInternalInitializer.cpp) target_include_directories(guest_json_internal_initializer_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_internal_initializer_tests PRIVATE libSceJson2 libc) @@ -116,7 +121,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index b224567735..cecd1cd633 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2358,6 +2358,33 @@ const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array* self) { if (!self->items->head) return reinterpret_cast(16); return &self->items->back(); } +void APS5_VABI _ZN3sce4Json5Array8pop_backEv(Array* self) { if (self->items) self->items->PopBack(); } + +ArrayIterator* APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(ArrayIterator* result, Array* self, const ArrayIterator* position) { + result->state = static_cast(Allocate(sizeof(ArrayIteratorState))); + if (result->state) result->state->position = nullptr; + if (!self->items) return result; + if (!position || !position->state || !position->state->position || position->state->position == self->items->head) { + Release(result->state); + result->state = nullptr; + NotImplemented_nid_no_patch(__func__); + } + auto* link = position->state->position; + auto* next = link->next; + next->previous = link->previous; + link->previous->next = next; + Destroy(link->value); + Release(link); + --self->items->count; + if (next) { + if (result->state) result->state->position = next; + } else { + Release(result->state); + result->state = nullptr; + } + return result; +} + void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array* self) { if (!self->items || !self->items->head) NotImplemented_nid_no_patch(__func__); if (self->items->empty()) throw std::out_of_range("sce::Json::Array::pop_front: empty array"); diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index 4d1ec0b2a4..ce04fb9625 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -15,21 +15,11 @@ int APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE() { return 0; } -int APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE() { NotImplemented_nid_no_patch(__func__); return 0; } -int APS5_VABI _ZN3sce4Json5Array8pop_backEv() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/tests/GuestJsonArrayMutation.cpp b/core/libs/tests/GuestJsonArrayMutation.cpp new file mode 100644 index 0000000000..3d5e514815 --- /dev/null +++ b/core/libs/tests/GuestJsonArrayMutation.cpp @@ -0,0 +1,191 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Iterator { void* state; }; +struct Object { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5Array8pop_backEv(Array*); +Iterator* APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(Iterator*, Array*, const Iterator*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ArrayTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::set headers; +}; +static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeArray(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); + } + Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Array notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; +struct ArrayAllocator { void* const* vtable = ArrayVtable; }; +static void CheckLinks(const Array& array, std::size_t count, std::initializer_list values = {}) { + Require(values.size() == 0 || values.size() == count, "Expected array values must cover every live node"); + const auto* header = static_cast(array.storage); + Require(header && header[1] == count, "Array header must expose its native element count"); + const auto* head = reinterpret_cast(header[0]); + const auto* previous = head; + std::size_t seen = 0; + for (auto* node = reinterpret_cast(head[1]); node != head; node = reinterpret_cast(node[1])) { + Require(node[0] == reinterpret_cast(previous), "Every array link must retain its predecessor"); + const auto* value = reinterpret_cast(node + 2); + Require(++seen <= count, "Array links must close at the sentinel"); + const auto expected = values.size() ? *(values.begin() + seen - 1) : 10 * seen; + Require(!value->parent && !value->auxiliary && value->type == 2 && value->payload == expected, "Array nodes must expose their inline values and ownership"); + previous = node; + } + Require(seen == count && head[0] == reinterpret_cast(previous), "Array sentinel must close both list directions"); +} +int main() { + ArrayTrace trace; ArrayAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize array mutation contract"); + const auto baseline = trace.live.size(); + Array absent{}; + const auto beforeMissing = trace.requests.size(); + _ZN3sce4Json5Array8pop_backEv(&absent); + Require(trace.requests.size() == beforeMissing, "Popping missing storage must do nothing without allocating"); + Iterator missingResult{}; + Require(_ZN3sce4Json5Array5eraseERKNS1_8iteratorE(&missingResult, &absent, nullptr) == &missingResult && missingResult.state && *static_cast(missingResult.state) == nullptr, "Erase with missing array storage must return an allocated unbound iterator without reading the input iterator"); + _ZN3sce4Json5Array8iteratorD1Ev(&missingResult); + for (bool rejectResult : {false, true}) { + for (unsigned index : {0u, 1u, 2u}) { + Array array{}; _ZN3sce4Json5ArrayC1Ev(&array); + Value value{}; + for (int number : {10, 20, 30}) { + _ZN3sce4Json5ValueC1El(&value, number); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); + _ZN3sce4Json5ValueD1Ev(&value); + } + Iterator selected{}, survivor{}, end{}; + _ZNK3sce4Json5Array5beginEv(&selected, &array); + _ZN3sce4Json5Array8iterator7advanceEm(&selected, index); + _ZNK3sce4Json5Array5beginEv(&survivor, &array); + _ZN3sce4Json5Array8iterator7advanceEm(&survivor, index == 0 ? 1 : 0); + _ZNK3sce4Json5Array3endEv(&end, &array); + const auto* survivorAddress = _ZNK3sce4Json5Array8iteratordeEv(&survivor); + const auto* removed = _ZNK3sce4Json5Array8iteratordeEv(&selected); + void* selectedNode = *static_cast(selected.state); + void* nextNode = static_cast(selectedNode)[1]; + const auto requests = trace.requests.size(); + const auto notices = trace.notifications; + trace.budget = rejectResult ? 0 : -1; + Iterator result{}; + Require(_ZN3sce4Json5Array5eraseERKNS1_8iteratorE(&result, &array, &selected) == &result, "Erase must return the caller's hidden result object"); + trace.budget = -1; + Require(trace.requests.size() == requests + 1 && trace.requests.back() == 8 && trace.notifications == notices + (rejectResult ? 1 : 0), "Erase must request exactly one iterator state and notify only when that allocation fails"); + Require(!trace.live.count(selectedNode) && _ZNK3sce4Json5Array4sizeEv(&array) == 2, "Erase must remove the requested node even when its result-state allocation fails"); + Require(result.state ? *static_cast(result.state) == nextNode : rejectResult, "The returned iterator must identify the successor or remain absent after allocation failure"); + Require(_ZNK3sce4Json5Array8iteratordeEv(&survivor) == survivorAddress && survivorAddress != removed, "Erasure must preserve every surviving inline value and iterator identity"); + if (index == 0) CheckLinks(array, 2, {20, 30}); + else if (index == 1) CheckLinks(array, 2, {10, 30}); + else CheckLinks(array, 2, {10, 20}); + _ZN3sce4Json5Array8iteratorD1Ev(&result); + _ZN3sce4Json5Array8iteratorD1Ev(&selected); + _ZN3sce4Json5Array8iteratorD1Ev(&end); + const auto popRequests = trace.requests.size(); + _ZN3sce4Json5Array8pop_backEv(&array); + Require(trace.requests.size() == popRequests && _ZNK3sce4Json5Array4sizeEv(&array) == 1, "Pop back must destroy the final node without allocating"); + _ZN3sce4Json5Array8pop_backEv(&array); + _ZN3sce4Json5Array8pop_backEv(&array); + CheckLinks(array, 0); + _ZN3sce4Json5Array8iteratorD1Ev(&survivor); + _ZN3sce4Json5ArrayD1Ev(&array); + Require(trace.live.size() == baseline, "Mutation and iterator teardown must restore allocator ownership without leaks"); + } + } + for (bool erase : {false, true}) { + Array array{}; _ZN3sce4Json5ArrayC1Ev(&array); + Value source{}; _ZN3sce4Json5ValueC1EPKc(&source, "owned payload"); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &source); + const auto sourceAllocations = trace.live.size(); + if (erase) { + Iterator selected{}, result{}; _ZNK3sce4Json5Array5beginEv(&selected, &array); + trace.budget = 0; + _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(&result, &array, &selected); + trace.budget = -1; + Require(!result.state, "Rejected erase result allocation must stay absent while destroying its owned payload"); + _ZN3sce4Json5Array8iteratorD1Ev(&selected); + } else _ZN3sce4Json5Array8pop_backEv(&array); + Require(trace.live.size() + 4 == sourceAllocations && source.type == 5 && source.payload, "Removal must release the string node, handle, header and characters while preserving the independent source"); + _ZN3sce4Json5ValueD1Ev(&source); + _ZN3sce4Json5ArrayD1Ev(&array); + Require(trace.live.size() == baseline, "Owned-payload removal must leave no allocator resources behind"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate the mutation allocation context without outstanding resources"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index c3acd02228..84498a25cf 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -54,11 +54,11 @@ - [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE`; the deleting base allocator destructor traps in the reference. This invalid destruction path remains explicitly unsupported. - [_ZN3sce4Json5Array10push_frontERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MQxDxBzWz40` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array5eraseERKNS1_8iteratorE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `2mbVAtshWRI` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ixJAz92H1uE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array8pop_backEv](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `woec1toi4Xw` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json5ValueixERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `clF7J7N9xXE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [Counted string searches](../../../core/libs/prx/libSceJson2/Export.cpp) with an admitted positive count and a null needle remain explicitly unsupported; the reference dereferences the invalid pointer. Oversized and zero counts retain their verified results. + +- [sce::Json::Array::erase](../../../core/libs/prx/libSceJson2/Export.cpp) - invalid, missing or sentinel positions in an allocated array remain explicitly unsupported. Reference invalid-iterator paths can corrupt storage or fault; valid erasure retains its mutation when returned-iterator allocation fails. From 43c313fbecf774c4e88068a8fe11ea973e9e5a97 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:55:21 +0200 Subject: [PATCH 038/318] feat(libSceJson2): implement array insertion and payload transfer protocol --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 104 +++++++++- .../libs/prx/libSceJson2/InventoryExports.cpp | 10 - core/libs/tests/GuestJsonArrayInsertion.cpp | 191 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 5 files changed, 299 insertions(+), 17 deletions(-) create mode 100644 core/libs/tests/GuestJsonArrayInsertion.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index de2cedb8fa..ce3fb3bd02 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_array_insertion_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonArrayInsertion.cpp) + target_include_directories(guest_json_array_insertion_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_array_insertion_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_array_insertion_tests) + add_test(NAME guest_json_array_insertion COMMAND guest_json_array_insertion_tests) add_test_executable(guest_json_array_mutation_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonArrayMutation.cpp) target_include_directories(guest_json_array_mutation_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_array_mutation_tests PRIVATE libSceJson2 libc) @@ -121,7 +126,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index cecd1cd633..75c9f3e85b 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1180,12 +1180,46 @@ void Construct(Value& v) { v.type = TypeNull; } +template +void TransferChildren(Value& destination, TLink* first, TLink* head, TNext next, TValue value) { + auto* begin = static_cast(Allocate(sizeof(TLink*))); + if (begin) *begin = first; + for (;;) { + auto* end = static_cast(Allocate(sizeof(TLink*))); + if (end) *end = head; + const bool more = begin && end && *begin != *end; + Release(end); + if (!more) break; + if (!*begin) { Release(begin); NotImplemented_nid_no_patch(__func__); } + value(*begin).parent = &destination; + *begin = next(*begin); + } + Release(begin); +} + void TransferPayload(Node& destination, Node& source) { - if (source.type == TypeBoolean) destination.boolean = source.boolean; - else if (source.type != TypeNull) std::memcpy(&destination.payload, &source.payload, sizeof(destination.payload)); + if (destination.auxiliary || source.auxiliary) return; + Clear(destination); + if (source.type <= TypeReal) AssignNode(destination, source); + else if (source.type <= TypeObject) { + destination.payload = source.payload; + source.payload = 0; + if (source.type == TypeArray && destination.array) { + auto* items = destination.array->items; + auto* head = items ? items->head : nullptr; + TransferChildren(destination, head ? head->next : nullptr, head, + [](ArrayLink* link) { return link->next; }, [](ArrayLink* link) -> Value& { return link->value; }); + } else if (source.type == TypeObject && destination.object) { + auto* items = destination.object->items; + auto* head = items ? items->head : nullptr; + TransferChildren(destination, head ? head->left : nullptr, head, + [](ObjectLink* link) { return NextObjectLink(link); }, [](ObjectLink* link) -> Value& { return link->pair.value; }); + } + } else return; + destination.parent = source.parent; + source.parent = nullptr; destination.type = source.type; - source.type = TypeNull; - BindChildren(destination); + Clear(source); } bool HasStorage(const String& value) { return value.text != nullptr; } @@ -1868,6 +1902,42 @@ void AppendArray(Array* self, const Value* value) { if (operation.Finish()) self->items->PopBack(); } +ArrayLink* InsertArrayLink(ArrayStorage& items, ArrayLink* position, const Value& value) { + auto* link = static_cast(Allocate(sizeof(ArrayLink))); + if (!link) return items.head; + try { CopyInto(link->value, value); } + catch (...) { Release(link); throw; } + link->previous = position->previous; + link->next = position; + position->previous->next = link; + position->previous = link; + ++items.count; + return link; +} + +void PrependArray(Array* self, const Value* value) { + if (!self->items) return; + AllocationOperation operation; + if (!ModernValues()) { + if (self->items->head) InsertArrayLink(*self->items, self->items->head->next, *value); + } else { + ScopedValue copy; + copy.value.parent = self->items->Owner(); + AssignNode(copy.value, *value); + Value empty{}; + if (self->items->head) InsertArrayLink(*self->items, self->items->head->next, empty); + if (self->items->count) TransferPayload(self->items->head->next->value, copy.value); + } + if (operation.Finish() && self->items->head && self->items->count) { + auto* link = self->items->head->next; + link->next->previous = self->items->head; + self->items->head->next = link->next; + Destroy(link->value); + Release(link); + --self->items->count; + } +} + using ParserReadCallback = int (APS5_VABI*)(char&, void*); class Parser { @@ -2322,6 +2392,32 @@ void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array* self) { Construct(*self); } void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array* self, const Array* other) { CopyArray(*self, *other); } void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array* self) { Destroy(*self); } Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array* self, const Array* other) { AssignArray(*self, *other); return self; } +void APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(Array* self, const Value* value) { PrependArray(self, value); } + +ArrayIterator* APS5_VABI _ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE(ArrayIterator* result, Array* self, const ArrayIterator* position, const Value* value) { + result->state = static_cast(Allocate(sizeof(ArrayIteratorState))); + if (result->state) result->state->position = nullptr; + if (!self->items) return result; + if (!position || !position->state || !position->state->position || !self->items->head) { + Release(result->state); result->state = nullptr; + NotImplemented_nid_no_patch(__func__); + } + ArrayLink* inserted; + if (!ModernValues()) inserted = InsertArrayLink(*self->items, position->state->position, *value); + else { + ScopedValue copy; + copy.value.parent = self->items->Owner(); + AssignNode(copy.value, *value); + AllocationOperation operation; + Value empty{}; + inserted = InsertArrayLink(*self->items, position->state->position, empty); + if (!operation.Finish()) TransferPayload(inserted->value, copy.value); + } + if (inserted) { if (result->state) result->state->position = inserted; } + else { Release(result->state); result->state = nullptr; } + return result; +} + void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array* self, const Value* value) { AppendArray(self, value); } ArrayIterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(ArrayIterator* result, const Array* self) { diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index ce04fb9625..e8594fcbef 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -10,16 +10,6 @@ int APS5_VABI _ZN3sce4Json12MemAllocatorD0Ev() { return 0; } -int APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/tests/GuestJsonArrayInsertion.cpp b/core/libs/tests/GuestJsonArrayInsertion.cpp new file mode 100644 index 0000000000..6dc06cb998 --- /dev/null +++ b/core/libs/tests/GuestJsonArrayInsertion.cpp @@ -0,0 +1,191 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Iterator { void* state; }; +struct Object { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(Array*, const Value*); +Iterator* APS5_VABI _ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE(Iterator*, Array*, const Iterator*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5Array8pop_backEv(Array*); +Iterator* APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(Iterator*, Array*, const Iterator*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ArrayTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::set headers; +}; +static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeArray(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); + } + Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Array notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; +struct ArrayAllocator { void* const* vtable = ArrayVtable; }; +static void Fill(Array& array) { + _ZN3sce4Json5ArrayC1Ev(&array); + for (int number : {10, 20, 30}) { + Value value{}; _ZN3sce4Json5ValueC1El(&value, number); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &value); + _ZN3sce4Json5ValueD1Ev(&value); + } +} +static void CheckSequence(const Array& array, bool front, bool once, int budget) { + const bool inserted = front ? budget >= 4 : (once ? budget != 4 : budget >= 5); + const bool lostFirst = front && !inserted && (!once || budget == 3); + const auto* head = *static_cast(array.storage); + auto* node = static_cast(head[1]); + unsigned seen = 0; + for (int slot = 0; slot < 4; ++slot) { + const bool newSlot = slot == (front ? 0 : 1); + if (newSlot && !inserted) continue; + const int original = front ? slot : (slot == 0 ? 1 : slot); + if (!newSlot && lostFirst && original == 1) continue; + Require(node != head, "Insertion must preserve the expected list length"); + const auto* value = reinterpret_cast(node + 2); + if (newSlot) { + const bool emptyNull = !front && once && (budget == 1 || budget == 2); + Require(value->type == (emptyNull ? 0 : 5), "Insertion must retain the native partial-copy type after one-shot failure"); + if (!emptyNull) { + const char* expected = !front && once && budget == 3 ? "" : "abc"; + Require(value->payload && std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(value->payload)), expected) == 0, "Insertion must retain the native partial string payload"); + } + } else Require(value->type == 2 && value->payload == static_cast(10 * original), "Insertion failure must preserve the reference's surviving original values"); + node = static_cast(node[1]); ++seen; + } + Require(node == head && _ZNK3sce4Json5Array4sizeEv(&array) == seen, "Insertion links and element count must agree"); +} +int main() { + ArrayTrace trace; ArrayAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize insertion allocation contracts"); + const auto baseline = trace.live.size(); + for (bool front : {false, true}) for (bool once : {false, true}) for (int budget = 0; budget <= 5; ++budget) { + Array array{}; Fill(array); + Value source{}; _ZN3sce4Json5ValueC1EPKc(&source, "abc"); + Iterator position{}, result{}; _ZNK3sce4Json5Array5beginEv(&position, &array); _ZN3sce4Json5Array8iterator7advanceEm(&position, 1); + const auto* survivor = _ZNK3sce4Json5Array8iteratordeEv(&position); + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = budget; trace.once = once; + if (front) _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(&array, &source); + else Require(_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE(&result, &array, &position, &source) == &result, "Insert must return the caller's hidden iterator object"); + trace.budget = -1; trace.once = false; + CheckSequence(array, front, once, budget); + Require(_ZNK3sce4Json5Array8iteratordeEv(&position) == survivor && survivor->payload == 20, "Insertion must preserve surviving node and iterator identity"); + std::vector expected; + if (!front) expected.push_back(8); + expected.push_back(8); + const int copied = budget - (front ? 0 : 1); + if (copied >= 1 || (!front && once && budget == 0)) expected.push_back(24); + if (copied >= 2 || (!front && once && budget == 0)) expected.push_back(4); + expected.push_back(48); + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, "Insertion must preserve allocation order across result, copied storage and list-node admission"); + const unsigned failed = once ? (budget < (front ? 4 : 5) ? 1 : 0) : (front ? (budget < 3 ? 2 : budget == 3 ? 1 : 0) : (budget == 0 ? 3 : budget < 4 ? 2 : budget == 4 ? 1 : 0)); + Require(trace.notifications == notices + failed, "Every rejected insertion allocation must notify exactly once"); + if (!front) { + Require((result.state != nullptr) == (budget != 0), "Result allocation failure must not suppress later insertion work"); + if (result.state) { + const bool nodeFailed = once ? budget == 4 : budget < 5; + auto* head = *static_cast(array.storage); + auto* returned = *static_cast(result.state); + Require(nodeFailed ? returned == head : returned != head && static_cast(returned)[1] == *static_cast(position.state), "Insert must return the admitted node or the sentinel when node allocation fails"); + } + } + Require(source.type == 5 && std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(source.payload)), "abc") == 0, "Insertion must preserve its independently owned source"); + _ZN3sce4Json5Array8iteratorD1Ev(&result); _ZN3sce4Json5Array8iteratorD1Ev(&position); + _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ArrayD1Ev(&array); + Require(trace.live.size() == baseline, "Every insertion failure must remain destructible without leaked ownership"); + } + for (bool front : {false, true}) { + Array inner{}; Fill(inner); + Value source{}; _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&source, &inner); + Array destination{}; _ZN3sce4Json5ArrayC1Ev(&destination); + Iterator position{}, result{}; _ZNK3sce4Json5Array3endEv(&position, &destination); + const auto requests = trace.requests.size(); + if (front) _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(&destination, &source); + else _ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE(&result, &destination, &position, &source); + const auto* inserted = _ZNK3sce4Json5Array5frontEv(&destination); + Require(inserted->type == 6 && inserted->payload != source.payload && inserted->parent == nullptr, "Nested insertion must own an independent array payload with the destination's parent"); + auto* nested = reinterpret_cast(inserted->payload); + auto* head = *static_cast(nested->storage); + for (auto* node = static_cast(head[1]); node != head; node = static_cast(node[1])) Require(reinterpret_cast(node + 2)->parent == inserted, "Payload transfer must rebind every child to the actual inserted value"); + Require(trace.requests.size() >= requests + 5 && std::vector(trace.requests.end() - 5, trace.requests.end()) == std::vector({8, 8, 8, 8, 8}), "Nested payload transfer must allocate its begin and repeated end iterator states through the supplied allocator"); + _ZN3sce4Json5Array8iteratorD1Ev(&result); _ZN3sce4Json5Array8iteratorD1Ev(&position); + _ZN3sce4Json5ArrayD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ArrayD1Ev(&inner); + Require(trace.live.size() == baseline, "Nested insertion and iterator traversal must release every admitted allocation"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate insertion allocation contracts"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 84498a25cf..43758c0a53 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -53,8 +53,6 @@ ### Unknown 3.20 import declarations - [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE`; the deleting base allocator destructor traps in the reference. This invalid destruction path remains explicitly unsupported. -- [_ZN3sce4Json5Array10push_frontERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `MQxDxBzWz40` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ixJAz92H1uE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZNK3sce4Json5ValueixERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `clF7J7N9xXE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. @@ -62,3 +60,5 @@ - [Counted string searches](../../../core/libs/prx/libSceJson2/Export.cpp) with an admitted positive count and a null needle remain explicitly unsupported; the reference dereferences the invalid pointer. Oversized and zero counts retain their verified results. - [sce::Json::Array::erase](../../../core/libs/prx/libSceJson2/Export.cpp) - invalid, missing or sentinel positions in an allocated array remain explicitly unsupported. Reference invalid-iterator paths can corrupt storage or fault; valid erasure retains its mutation when returned-iterator allocation fails. + +- [Array insertion and payload transfer](../../../core/libs/prx/libSceJson2/Export.cpp) - string-copy allocation failures and successful nested-array transfer follow the reference protocol. Nested collection transfer failures, missing-sentinel paths, invalid insertion iterators and older-SDK callback combinations still need direct regression coverage. Invalid insertion iterators are explicitly unsupported. From 4da5016a4ae9bc472fa6610035b2fccbadc0f4d4 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 06:59:54 +0200 Subject: [PATCH 039/318] feat(libSceJson2): implement const keyed lookup and iterator failure recovery --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 66 +++++--- .../libs/prx/libSceJson2/InventoryExports.cpp | 5 - core/libs/tests/GuestJsonKeyLookup.cpp | 149 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 3 +- 5 files changed, 200 insertions(+), 30 deletions(-) create mode 100644 core/libs/tests/GuestJsonKeyLookup.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index ce3fb3bd02..d9ce41c8d1 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_key_lookup_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonKeyLookup.cpp) + target_include_directories(guest_json_key_lookup_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_key_lookup_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_key_lookup_tests) + add_test(NAME guest_json_key_lookup COMMAND guest_json_key_lookup_tests) add_test_executable(guest_json_array_insertion_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonArrayInsertion.cpp) target_include_directories(guest_json_array_insertion_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_array_insertion_tests PRIVATE libSceJson2 libc) @@ -126,7 +131,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 75c9f3e85b..9b0142df65 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1548,6 +1548,45 @@ const Value& MissingElement(const Value* self, int error, const String* key, con return Defaults(TypeNull); } +const Value* LookupKey(const Value* self, const String* key) { + if (!key) { NotImplemented_nid_no_patch(__func__); return nullptr; } + auto moduleFallback = [] { + auto& state = State(); + std::lock_guard lock(state.mutex); + return state.fallback; + }; + if (!key->text) return moduleFallback(); + struct IteratorState { + ObjectIteratorState* state = static_cast(Allocate(sizeof(ObjectIteratorState))); + IteratorState() { if (state) state->position = nullptr; } + ~IteratorState() { Release(state); } + }; + if (self->type == TypeObject) { + IteratorState found; + if (!self->object) { NotImplemented_nid_no_patch(__func__); return nullptr; } + auto* items = self->object->items; + if (found.state) found.state->position = items ? items->Find(Characters(*key)) : nullptr; + if (items && !items->head) { Release(found.state); found.state = nullptr; } + bool present; + { + IteratorState end; + if (end.state) end.state->position = items ? items->head : nullptr; + if (items && !items->head) { Release(end.state); end.state = nullptr; } + present = found.state && end.state && found.state->position != end.state->position; + } + if (present) return &found.state->position->pair.value; + if (ModernValues()) { + const auto* result = &MissingElement(self, static_cast(0x80848115), key, nullptr); + if (moduleFallback() || RootOf(*self).auxiliary) return result; + } + } else if (ModernValues()) { + const auto* result = &MissingElement(self, static_cast(0x80848117), key, nullptr); + if (moduleFallback() || RootOf(*self).auxiliary) return result; + } + const auto* auxiliary = RootOf(*self).auxiliary; + return auxiliary && auxiliary->fallback ? auxiliary->fallback : moduleFallback(); +} + template int SetHandler(Node& n, HandlerRecord* Auxiliary::* member, TCallback callback, void* context, bool modernOnly = false) { if (n.parent) return static_cast(0x80848104); @@ -2697,22 +2736,10 @@ std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value* self) { if (n.type == TypeObject) return n.object->items->size(); return 0; } +const Value* APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE(const Value* self, const String* key) { return LookupKey(self, key); } const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value* self, const char* key) { - String text{}; - Construct(text, key ? key : ""); - const Value* result; - try { - if (self->type == TypeObject) { - const auto* node = self->object->items ? self->object->items->Find(Characters(text)) : nullptr; - if (node && !node->nil) { Destroy(text); return &node->pair.value; } - } - result = &MissingElement(self, static_cast(self->type == TypeObject ? 0x80848115 : 0x80848117), &text, nullptr); - } catch (...) { - Destroy(text); - throw; - } - Destroy(text); - return result; + ScopedString text(key ? key : ""); + return LookupKey(self, &text.string); } const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value* self, std::size_t index) { if (self->type == TypeArray && index < self->array->items->size()) { @@ -2733,14 +2760,7 @@ void APS5_VABI _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(Value* self, ValueType type) SetType(*self, type); } const Value* APS5_VABI _ZNK3sce4Json5Value8getValueEm(const Value* self, std::size_t index) { return _ZNK3sce4Json5ValueixEm(self, index); } -const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value* self, const String* key) { - const Node& n = NodeOf(*self); - if (key && n.type == TypeObject && n.object->items) { - const auto* node = n.object->items->Find(Characters(*key)); - if (node && !node->nil) return &node->pair.value; - } - return &MissingElement(self, static_cast(n.type == TypeObject ? 0x80848115 : 0x80848117), key, nullptr); -} +const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value* self, const String* key) { return LookupKey(self, key); } Value* APS5_VABI _ZN3sce4Json5Value10referValueEm(Value* self, std::size_t index) { if (self->type == TypeArray && index < self->array->items->size()) { auto it = self->array->items->begin(); diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index e8594fcbef..39bc8548ce 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -20,9 +20,4 @@ int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_() { return 0; } -int APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - } diff --git a/core/libs/tests/GuestJsonKeyLookup.cpp b/core/libs/tests/GuestJsonKeyLookup.cpp new file mode 100644 index 0000000000..c6225fc264 --- /dev/null +++ b/core/libs/tests/GuestJsonKeyLookup.cpp @@ -0,0 +1,149 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Iterator { void* state; }; +struct Object { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using LookupCallback = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); +extern "C" { +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE(const Value*, const String*); +const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value*, const String*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, LookupCallback, void*); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5Array8pop_backEv(Array*); +Iterator* APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(Iterator*, Array*, const Iterator*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ArrayTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::set headers; +}; +static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeArray(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); + } + Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Array notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; +struct ArrayAllocator { void* const* vtable = ArrayVtable; }; +struct LookupTrace { unsigned calls = 0; const String* key = nullptr; const Value* source = nullptr; }; +static void APS5_VABI LookupFailure(int error, const String* key, const std::size_t* index, const Value* parent, const Value* value, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848115 && !index && !parent, "Rejected object lookup must preserve its error, key form and parent"); + ++trace.calls; trace.key = key; trace.source = value; +} +int main() { + ArrayTrace trace; ArrayAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + Value root{}; _ZN3sce4Json5ValueC1Ev(&root); + String missingHeader{}; + Require(_ZNK3sce4Json5ValueixERKNS0_6StringE(&root, &missingHeader) == nullptr, "Before initialization a headerless key must return the absent module fallback without allocation"); + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize keyed lookup contract"); + const char json[] = "{\"a\":7,\"\":9}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, json, sizeof(json) - 1) == 0, "Parse lookup values including an empty key"); + String key{}; _ZN3sce4Json6StringC1EPKc(&key, "a"); + String absent{}; _ZN3sce4Json6StringC1EPKc(&absent, "absent"); + const auto* moduleFallback = _ZNK3sce4Json5ValueixERKNS0_6StringE(&root, &missingHeader); + Require(moduleFallback && moduleFallback->type == 0, "Headerless keys must select the module fallback"); + LookupTrace lookup; + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, LookupFailure, &lookup) == 0, "Register object lookup recovery"); + const auto* rootFallback = _ZNK3sce4Json5ValueixERKNS0_6StringE(&root, &absent); + Require(rootFallback && rootFallback != moduleFallback && rootFallback->parent == &root && lookup.calls == 1, "Missing keys must retain root-local fallback identity"); + const auto* actual = _ZNK3sce4Json5ValueixERKNS0_6StringE(&root, &key); + Require(actual && actual->type == 3 && actual->payload == 7 && actual->parent == &root, "Successful const lookup must return the actual owned element"); + const auto baseline = trace.live.size(); + for (bool getter : {false, true}) for (bool once : {false, true}) for (int budget : {0, 1, 2}) { + const auto requests = trace.requests.size(); const auto notices = trace.notifications; const auto calls = lookup.calls; + trace.budget = budget; trace.once = once; + const auto* result = getter ? _ZNK3sce4Json5Value8getValueERKNS0_6StringE(&root, &key) : _ZNK3sce4Json5ValueixERKNS0_6StringE(&root, &key); + trace.budget = -1; trace.once = false; + Require(result == (budget < 2 ? rootFallback : actual) && trace.live.size() == baseline, "Both keyed exports must preserve fallback identity and release temporary iterators"); + Require(trace.requests.size() == requests + 2 && trace.requests[requests] == 8 && trace.requests[requests + 1] == 8, "Every object lookup must allocate find and end iterator states in order"); + const auto failed = budget == 2 ? 0u : once ? 1u : budget == 0 ? 2u : 1u; + Require(trace.notifications == notices + failed && lookup.calls == calls + (budget < 2 ? 1 : 0), "Iterator allocation failure must notify and dispatch missing-element recovery even for an existing key"); + if (budget < 2) Require(lookup.key == &key && lookup.source == &root, "Failure recovery must borrow the exact supplied key and source objects"); + } + const auto requests = trace.requests.size(); const auto calls = lookup.calls; + Require(_ZNK3sce4Json5ValueixERKNS0_6StringE(&root, &missingHeader) == moduleFallback && _ZNK3sce4Json5Value8getValueERKNS0_6StringE(&root, &missingHeader) == moduleFallback && lookup.calls == calls && trace.requests.size() == requests, "A missing string header must bypass iterator allocation, recovery and the root fallback"); + trace.budget = 0; + Require(_ZNK3sce4Json5ValueixEPKc(&root, "a") == moduleFallback && lookup.calls == calls && trace.requests.size() == requests + 1 && trace.requests.back() == 24, "C-string key-header failure must bypass lookup and recovery"); + trace.budget = 1; trace.once = true; + const auto* emptyKey = _ZNK3sce4Json5ValueixEPKc(&root, "a"); + trace.budget = -1; trace.once = false; + Require(emptyKey && emptyKey->type == 3 && emptyKey->payload == 9 && lookup.calls == calls && trace.live.size() == baseline, "One-shot character-allocation failure must admit the constructed empty key and look up that key faithfully"); + _ZN3sce4Json6StringD1Ev(&key); _ZN3sce4Json6StringD1Ev(&absent); _ZN3sce4Json5ValueD1Ev(&root); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Destroy all lookup resources and callbacks"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 43758c0a53..da104b17b9 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -55,10 +55,11 @@ - [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE`; the deleting base allocator destructor traps in the reference. This invalid destruction path remains explicitly unsupported. - [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZNK3sce4Json5ValueixERKNS0_6StringE](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `clF7J7N9xXE` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [Counted string searches](../../../core/libs/prx/libSceJson2/Export.cpp) with an admitted positive count and a null needle remain explicitly unsupported; the reference dereferences the invalid pointer. Oversized and zero counts retain their verified results. - [sce::Json::Array::erase](../../../core/libs/prx/libSceJson2/Export.cpp) - invalid, missing or sentinel positions in an allocated array remain explicitly unsupported. Reference invalid-iterator paths can corrupt storage or fault; valid erasure retains its mutation when returned-iterator allocation fails. - [Array insertion and payload transfer](../../../core/libs/prx/libSceJson2/Export.cpp) - string-copy allocation failures and successful nested-array transfer follow the reference protocol. Nested collection transfer failures, missing-sentinel paths, invalid insertion iterators and older-SDK callback combinations still need direct regression coverage. Invalid insertion iterators are explicitly unsupported. + +- [Value keyed lookup](../../../core/libs/prx/libSceJson2/Export.cpp) - valid string-key overloads and C-string key-construction failures follow the observed iterator allocation and fallback protocol. Missing object handles, corrupted tree storage, lifecycle reentry during lookup and remaining older-SDK callback combinations remain unverified; invalid reference arguments are explicitly unsupported. From 295abeebb4b51bbe98564477b1fd495f71dc4c62 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 07:05:07 +0200 Subject: [PATCH 040/318] feat(libSceJson2): implement callback serialization protocol --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 141 +++++++++++++++ .../libs/prx/libSceJson2/InventoryExports.cpp | 10 - .../tests/GuestJsonSerializationCallback.cpp | 171 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 5 files changed, 320 insertions(+), 13 deletions(-) create mode 100644 core/libs/tests/GuestJsonSerializationCallback.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index d9ce41c8d1..a950e58591 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_serialization_callback_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonSerializationCallback.cpp) + target_include_directories(guest_json_serialization_callback_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_serialization_callback_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_serialization_callback_tests) + add_test(NAME guest_json_serialization_callback COMMAND guest_json_serialization_callback_tests) add_test_executable(guest_json_key_lookup_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonKeyLookup.cpp) target_include_directories(guest_json_key_lookup_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_key_lookup_tests PRIVATE libSceJson2 libc) @@ -131,7 +136,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 9b0142df65..515e60a398 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1977,6 +1977,141 @@ void PrependArray(Array* self, const Value* value) { } } +using SerializationCallback = int (APS5_VABI*)(String&, void*); + +template +struct SerializationIterator { + TLink** state; + SerializationIterator(TLink* position, bool invalid) : state(static_cast(Allocate(sizeof(TLink*)))) { + if (state) *state = position; + if (invalid) { Release(state); state = nullptr; } + } + ~SerializationIterator() { Release(state); } + bool Different(TLink* position, bool invalid) const { + SerializationIterator other(position, invalid); + return state && other.state && *state != *other.state; + } +}; + +class CallbackSerializer { +public: + CallbackSerializer(String& buffer, String* destination, SerializationCallback callback, void* context, bool collect) + : buffer(buffer), destination(destination), callback(callback), context(context), collect(collect) {} + int SerializeValue(const Value& value) { + if (value.type == TypeReal && !std::isfinite(value.real) && FloatConfiguration().enabled) { + CallbackScope scope; + const auto handlers = HandlersOf(value); + if (scope.entered && handlers.specialFloat) { + ScopedValue replacement; + bool error = false; + RecoverSpecial(value, handlers, 0, value.real, replacement.value, &error); + return error ? JsonErrorSpecialFloat : SerializeValue(replacement.value); + } + } + if (value.type >= TypeNull && value.type <= TypeReal) { + std::string scalar; + const int status = Serialize(scalar, value); + if (status < 0) return status; + AppendText(buffer, scalar); + return Emit(); + } + if (value.type == TypeString) { + if (!value.string) { NotImplemented_nid_no_patch(__func__); return 0; } + Escape(*value.string); + return Emit(); + } + if (value.type == TypeArray) { + if (!value.array) { NotImplemented_nid_no_patch(__func__); return 0; } + auto* items = value.array->items; + auto* head = items ? items->head : nullptr; + return Collection(head ? head->next : nullptr, head, items && !head, '[', ']', + [](ArrayLink* link) { return link->next; }, + [this](ArrayLink* link) { return SerializeValue(link->value); }); + } + if (value.type == TypeObject) { + if (!value.object) { NotImplemented_nid_no_patch(__func__); return 0; } + auto* items = value.object->items; + auto* head = items ? items->head : nullptr; + return Collection(head ? head->left : nullptr, head, items && !head, '{', '}', + [](ObjectLink* link) { return NextObjectLink(link); }, + [this](ObjectLink* link) { Escape(link->pair.key); Character(':'); return SerializeValue(link->pair.value); }); + } + return 0; + } +private: + String& buffer; + String* destination; + SerializationCallback callback; + void* context; + bool collect; + void Character(char character) { if (character) AppendText(buffer, std::string_view(&character, 1)); } + int Emit() { + if (collect) { + if (destination) { AppendText(*destination, Characters(buffer)); ClearText(buffer); } + return callback(*destination, context); + } + const int result = callback(buffer, context); + ClearText(buffer); + return result; + } + void Escape(const String& string) { + Character('"'); + for (const unsigned char character : View(string)) { + switch (character) { + case '\b': AppendText(buffer, "\\b"); break; + case '\t': AppendText(buffer, "\\t"); break; + case '\n': AppendText(buffer, "\\n"); break; + case '\f': AppendText(buffer, "\\f"); break; + case '\r': AppendText(buffer, "\\r"); break; + case '"': AppendText(buffer, "\\\""); break; + case '/': AppendText(buffer, "\\/"); break; + case '\\': AppendText(buffer, "\\\\"); break; + default: + if (character < 0x20 || character == 0x7f) { + char escaped[7]; + std::snprintf(escaped, sizeof(escaped), "\\u%04x", character); + AppendText(buffer, escaped); + } else Character(static_cast(character)); + } + } + Character('"'); + } + template + int Collection(TLink* first, TLink* head, bool invalid, char opening, char closing, TNext next, TSerialize serialize) { + Character(opening); + int status = Emit(); + if (status < 0) return status; + { + SerializationIterator iterator(first, invalid); + if (iterator.Different(head, invalid)) { + do { + if (iterator.Different(first, invalid)) Character(','); + if (!*iterator.state) { NotImplemented_nid_no_patch(__func__); return 0; } + status = serialize(*iterator.state); + if (status < 0) break; + *iterator.state = next(*iterator.state); + } while (iterator.Different(head, invalid)); + } + } + if (status < 0) return status; + Character(closing); + return Emit(); + } +}; + +int SerializeCallbackValue(Value* value, String* destination, SerializationCallback callback, void* context, bool collect) { + { + auto& state = State(); + std::lock_guard lock(state.mutex); + if (!state.allocation) return JsonErrorNotInitialized; + } + if (!callback || (collect && !destination)) { NotImplemented_nid_no_patch(__func__); return 0; } + if (!ModernValues()) { NotImplemented_nid_no_patch(__func__); return 0; } + ScopedString buffer(std::string_view{}); + CallbackSerializer serializer(buffer.string, destination, callback, context, collect); + return serializer.SerializeValue(*value); +} + using ParserReadCallback = int (APS5_VABI*)(char&, void*); class Parser { @@ -2793,6 +2928,12 @@ Object* APS5_VABI _ZN3sce4Json5Value11referObjectEv(Value* self) { if (n.type != TypeObject) throw std::runtime_error("sce::Json::Value::referObject: value is not an object"); return n.object; } +int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(Value* self, SerializationCallback callback, void* context) { + return SerializeCallbackValue(self, nullptr, callback, context, false); +} +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_(Value* self, String* output, SerializationCallback callback, void* context) { + return SerializeCallbackValue(self, output, callback, context, true); +} int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value* self, String* out) { std::string text; const int result = Serialize(text, *self); diff --git a/core/libs/prx/libSceJson2/InventoryExports.cpp b/core/libs/prx/libSceJson2/InventoryExports.cpp index 39bc8548ce..80741ba21c 100644 --- a/core/libs/prx/libSceJson2/InventoryExports.cpp +++ b/core/libs/prx/libSceJson2/InventoryExports.cpp @@ -10,14 +10,4 @@ int APS5_VABI _ZN3sce4Json12MemAllocatorD0Ev() { return 0; } -int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - } diff --git a/core/libs/tests/GuestJsonSerializationCallback.cpp b/core/libs/tests/GuestJsonSerializationCallback.cpp new file mode 100644 index 0000000000..933204be64 --- /dev/null +++ b/core/libs/tests/GuestJsonSerializationCallback.cpp @@ -0,0 +1,171 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Iterator { void* state; }; +struct Object { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using SerializationCallback = int (APS5_VABI*)(String&, void*); +extern "C" { +int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(Value*, SerializationCallback, void*); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_(Value*, String*, SerializationCallback, void*); +String* APS5_VABI _ZN3sce4Json6String6appendEPKc(String*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE(const Value*, const String*); +const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value*, const String*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5Array8pop_backEv(Array*); +Iterator* APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(Iterator*, Array*, const Iterator*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ArrayTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::set headers; +}; +static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeArray(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); + } + Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Array notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; +struct ArrayAllocator { void* const* vtable = ArrayVtable; }; +struct SerializationTrace { + std::vector chunks; + String* identity = nullptr; + int stopAt = -1; + int result = 23; + bool mutate = false; +}; +static int APS5_VABI Observe(String& chunk, void* context) { + auto& trace = *static_cast(context); + if (!trace.identity) trace.identity = &chunk; + Require(trace.identity == &chunk, "Serialization must reuse the same chunk handle or supplied output object throughout the call"); + trace.chunks.emplace_back(_ZNK3sce4Json6String5c_strEv(&chunk)); + if (trace.mutate) _ZN3sce4Json6String6appendEPKc(&chunk, "!"); + return static_cast(trace.chunks.size()) == trace.stopAt ? -77 : trace.result; +} +int main() { + ArrayTrace allocation; ArrayAllocator allocator; + Parameter parameter{&allocator, &allocation, 8192, 2, {}}; + Value root{}; _ZN3sce4Json5ValueC1Ev(&root); + SerializationTrace before; + Require(static_cast(_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(&root, Observe, &before)) == 0x80848110 && before.chunks.empty(), "Callback serialization before initialization must refuse without invoking the callback"); + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize serialization callbacks"); + const char json[] = R"({"a":[1,true,"x\n/\u007f"],"b":{}})"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, json, sizeof(json) - 1) == 0, "Parse nested callback serialization input"); + const std::vector chunks{"{", "\"a\":[", "1", ",true", ",\"x\\n\\/\\u007f\"", "]", ",\"b\":{", "}", "}"}; + const auto baseline = allocation.live.size(); + for (bool collect : {false, true}) for (bool mutate : {false, true}) for (int stop : {-1, 1, 3, 9}) { + String output{}; _ZN3sce4Json6StringC1EPKc(&output, "prefix"); + SerializationTrace trace; trace.stopAt = stop; trace.mutate = mutate; + if (collect) trace.identity = &output; + const int status = collect ? _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_(&root, &output, Observe, &trace) : _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(&root, Observe, &trace); + Require(status == (stop < 0 ? trace.result : -77), "Serialization must propagate the last callback result and stop on negative results"); + const auto count = stop < 0 ? chunks.size() : static_cast(stop); + Require(trace.chunks.size() == count, "Callback errors must stop traversal without emitting later or closing chunks"); + std::string cumulative = "prefix"; + for (std::size_t i = 0; i < count; ++i) { + cumulative += chunks[i]; + Require(trace.chunks[i] == (collect ? cumulative : chunks[i]), "Callback chunks must preserve structural boundaries, slash/DEL escaping and cumulative output semantics"); + if (collect && mutate) cumulative += '!'; + } + Require(std::string(_ZNK3sce4Json6String5c_strEv(&output)) == (collect ? cumulative : "prefix"), "Callback mutations must persist only in the supplied cumulative output, while streaming chunks are cleared after delivery"); + _ZN3sce4Json6StringD1Ev(&output); + Require(allocation.live.size() == baseline, "Successful and interrupted traversals must release all temporary buffers and iterator states"); + } + Value scalar{}; _ZN3sce4Json5ValueC1El(&scalar, 1234); + for (bool collect : {false, true}) for (bool once : {false, true}) for (int budget : {0, 1, 2, 3}) { + String output{}; _ZN3sce4Json6StringC1EPKc(&output, "p"); + const auto requests = allocation.requests.size(); const auto notices = allocation.notifications; + allocation.budget = budget; allocation.once = once; + SerializationTrace trace; + const int status = collect ? _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_(&scalar, &output, Observe, &trace) : _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(&scalar, Observe, &trace); + allocation.budget = -1; allocation.once = false; + Require(status == 23 && trace.chunks.size() == 1, "Allocation rejection must retain the reference callback-driven return contract"); + const bool complete = budget >= (collect ? 3 : 2); + Require(trace.chunks[0] == (collect ? complete ? "p1234" : "p" : complete ? "1234" : ""), "Allocation failures must expose the native admitted partial chunk or output"); + std::vector expected{24}; + if (budget >= 1) expected.push_back(21); + if (collect && budget >= 2) expected.push_back(22); + Require(std::vector(allocation.requests.begin() + requests, allocation.requests.end()) == expected, "Scalar callback serialization must allocate its header, chunk and optional cumulative-output growth in order"); + Require(allocation.notifications == notices + (complete ? 0 : 1), "Each scalar serialization rejection must notify exactly once"); + _ZN3sce4Json6StringD1Ev(&output); + Require(allocation.live.size() == baseline, "Rejected serialization allocations must not leak admitted storage"); + } + _ZN3sce4Json5ValueD1Ev(&scalar); _ZN3sce4Json5ValueD1Ev(&root); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && allocation.live.empty(), "Terminate callback serialization allocation context"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index da104b17b9..72b8d2fd4d 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -53,8 +53,6 @@ ### Unknown 3.20 import declarations - [_ZN3sce4Json12MemAllocatorD0Ev](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `ypazoTrKYhE`; the deleting base allocator destructor traps in the reference. This invalid destruction path remains explicitly unsupported. -- [_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `PyMbL9F2oBY` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. -- [_ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_](../../../core/libs/prx/libSceJson2/InventoryExports.cpp) (libSceJson2) - NID `l1URiQJlSoU` from the PS5 3.20 export inventory; prototype and behavior are unverified. Declaration is for import resolution only. - [Counted string searches](../../../core/libs/prx/libSceJson2/Export.cpp) with an admitted positive count and a null needle remain explicitly unsupported; the reference dereferences the invalid pointer. Oversized and zero counts retain their verified results. @@ -63,3 +61,5 @@ - [Array insertion and payload transfer](../../../core/libs/prx/libSceJson2/Export.cpp) - string-copy allocation failures and successful nested-array transfer follow the reference protocol. Nested collection transfer failures, missing-sentinel paths, invalid insertion iterators and older-SDK callback combinations still need direct regression coverage. Invalid insertion iterators are explicitly unsupported. - [Value keyed lookup](../../../core/libs/prx/libSceJson2/Export.cpp) - valid string-key overloads and C-string key-construction failures follow the observed iterator allocation and fallback protocol. Missing object handles, corrupted tree storage, lifecycle reentry during lookup and remaining older-SDK callback combinations remain unverified; invalid reference arguments are explicitly unsupported. + +- [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - modern chunk boundaries, callback return propagation, output mutation and scalar allocation rejection are reference-tested. Older-SDK serialization is explicitly unsupported until its parent-rebinding protocol is verified. Structural mutation of the traversed collection, nested iterator failures, non-finite recovery allocation failures and lifecycle reentry require further audit. Null callback or collecting-output references are explicitly unsupported. The existing non-callback serializer still follows a separate path and needs the same streaming/allocation audit. From 3b93f433ed06ac2b7bece670e2ece4e49fe7cd91 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 07:09:04 +0200 Subject: [PATCH 041/318] fix(libSceJson2): share serialization output and refresh live collection positions --- core/libs/prx/libSceJson2/Export.cpp | 47 ++++++++++++---- .../tests/GuestJsonSerializationCallback.cpp | 55 +++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 +- 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 515e60a398..78db96d5b8 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1993,6 +1993,9 @@ struct SerializationIterator { } }; +template +struct SerializationPosition { TLink* first; TLink* head; bool invalid; }; + class CallbackSerializer { public: CallbackSerializer(String& buffer, String* destination, SerializationCallback callback, void* context, bool collect) @@ -2022,17 +2025,25 @@ public: } if (value.type == TypeArray) { if (!value.array) { NotImplemented_nid_no_patch(__func__); return 0; } - auto* items = value.array->items; - auto* head = items ? items->head : nullptr; - return Collection(head ? head->next : nullptr, head, items && !head, '[', ']', + auto position = [&value] { + if (value.type != TypeArray || !value.array) NotImplemented_nid_no_patch(__func__); + auto* items = value.array->items; + auto* head = items ? items->head : nullptr; + return SerializationPosition{head ? head->next : nullptr, head, items && !head}; + }; + return Collection(position, '[', ']', [](ArrayLink* link) { return link->next; }, [this](ArrayLink* link) { return SerializeValue(link->value); }); } if (value.type == TypeObject) { if (!value.object) { NotImplemented_nid_no_patch(__func__); return 0; } - auto* items = value.object->items; - auto* head = items ? items->head : nullptr; - return Collection(head ? head->left : nullptr, head, items && !head, '{', '}', + auto position = [&value] { + if (value.type != TypeObject || !value.object) NotImplemented_nid_no_patch(__func__); + auto* items = value.object->items; + auto* head = items ? items->head : nullptr; + return SerializationPosition{head ? head->left : nullptr, head, items && !head}; + }; + return Collection(position, '{', '}', [](ObjectLink* link) { return NextObjectLink(link); }, [this](ObjectLink* link) { Escape(link->pair.key); Character(':'); return SerializeValue(link->pair.value); }); } @@ -2076,21 +2087,25 @@ private: } Character('"'); } - template - int Collection(TLink* first, TLink* head, bool invalid, char opening, char closing, TNext next, TSerialize serialize) { + template + int Collection(TPosition position, char opening, char closing, TNext next, TSerialize serialize) { Character(opening); int status = Emit(); if (status < 0) return status; { - SerializationIterator iterator(first, invalid); - if (iterator.Different(head, invalid)) { + auto current = position(); + SerializationIterator iterator(current.first, current.invalid); + current = position(); + if (iterator.Different(current.head, current.invalid)) { do { - if (iterator.Different(first, invalid)) Character(','); + current = position(); + if (iterator.Different(current.first, current.invalid)) Character(','); if (!*iterator.state) { NotImplemented_nid_no_patch(__func__); return 0; } status = serialize(*iterator.state); if (status < 0) break; *iterator.state = next(*iterator.state); - } while (iterator.Different(head, invalid)); + current = position(); + } while (iterator.Different(current.head, current.invalid)); } } if (status < 0) return status; @@ -2099,6 +2114,8 @@ private: } }; +int APS5_VABI SerializationSucceeded(String&, void*) { return 0; } + int SerializeCallbackValue(Value* value, String* destination, SerializationCallback callback, void* context, bool collect) { { auto& state = State(); @@ -2935,6 +2952,12 @@ int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_(Value* self return SerializeCallbackValue(self, output, callback, context, true); } int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value* self, String* out) { + { + auto& state = State(); + std::lock_guard lock(state.mutex); + if (!state.allocation) return JsonErrorNotInitialized; + } + if (ModernValues()) return SerializeCallbackValue(self, out, SerializationSucceeded, nullptr, true); std::string text; const int result = Serialize(text, *self); AppendText(*out, text); diff --git a/core/libs/tests/GuestJsonSerializationCallback.cpp b/core/libs/tests/GuestJsonSerializationCallback.cpp index 933204be64..bffb7adf8c 100644 --- a/core/libs/tests/GuestJsonSerializationCallback.cpp +++ b/core/libs/tests/GuestJsonSerializationCallback.cpp @@ -19,6 +19,9 @@ struct Pair { String key; std::uint64_t reserved; Value value; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; using SerializationCallback = int (APS5_VABI*)(String&, void*); extern "C" { +void APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(Array*, const Value*); +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(Value*, SerializationCallback, void*); int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_(Value*, String*, SerializationCallback, void*); String* APS5_VABI _ZN3sce4Json6String6appendEPKc(String*, const char*); @@ -116,18 +119,46 @@ static int APS5_VABI Observe(String& chunk, void* context) { if (trace.mutate) _ZN3sce4Json6String6appendEPKc(&chunk, "!"); return static_cast(trace.chunks.size()) == trace.stopAt ? -77 : trace.result; } +struct CollectionChange { + const Array* array; + int mode; + std::vector chunks; +}; +static int APS5_VABI PrependAtOpening(String& chunk, void* context) { + auto& change = *static_cast(context); + change.chunks.emplace_back(_ZNK3sce4Json6String5c_strEv(&chunk)); + if (change.chunks.size() == 1) { + Value value{}; _ZN3sce4Json5ValueC1El(&value, 0); + if (change.mode == 0) _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(const_cast(change.array), &value); + else if (change.mode == 1) _ZN3sce4Json5Array5clearEv(const_cast(change.array)); + else _ZN3sce4Json5Array9push_backERKNS0_5ValueE(const_cast(change.array), &value); + _ZN3sce4Json5ValueD1Ev(&value); + } + return 0; +} int main() { ArrayTrace allocation; ArrayAllocator allocator; Parameter parameter{&allocator, &allocation, 8192, 2, {}}; Value root{}; _ZN3sce4Json5ValueC1Ev(&root); SerializationTrace before; Require(static_cast(_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(&root, Observe, &before)) == 0x80848110 && before.chunks.empty(), "Callback serialization before initialization must refuse without invoking the callback"); + String beforeOutput{}; + Require(static_cast(_ZN3sce4Json5Value9serializeERNS0_6StringE(&root, &beforeOutput)) == 0x80848110 && !beforeOutput.storage, "Ordinary serialization before initialization must refuse without constructing output storage"); std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize serialization callbacks"); const char json[] = R"({"a":[1,true,"x\n/\u007f"],"b":{}})"; Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, json, sizeof(json) - 1) == 0, "Parse nested callback serialization input"); const std::vector chunks{"{", "\"a\":[", "1", ",true", ",\"x\\n\\/\\u007f\"", "]", ",\"b\":{", "}", "}"}; const auto baseline = allocation.live.size(); + String ordinary{}; + _ZN3sce4Json6StringC1EPKc(&ordinary, "prefix"); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&root, &ordinary) == 0, "Ordinary serialization must use the collecting protocol with a successful internal sink"); + std::string expectedOrdinary = "prefix"; + for (const auto& chunk : chunks) expectedOrdinary += chunk; + Require(std::string(_ZNK3sce4Json6String5c_strEv(&ordinary)) == expectedOrdinary, "Ordinary serialization must preserve the same slash and DEL escaping as callback serialization"); + _ZN3sce4Json6StringD1Ev(&ordinary); + Require(allocation.live.size() == baseline, "Ordinary serialization must release its temporary buffer and collection iterators"); + for (bool collect : {false, true}) for (bool mutate : {false, true}) for (int stop : {-1, 1, 3, 9}) { String output{}; _ZN3sce4Json6StringC1EPKc(&output, "prefix"); SerializationTrace trace; trace.stopAt = stop; trace.mutate = mutate; @@ -146,6 +177,16 @@ int main() { _ZN3sce4Json6StringD1Ev(&output); Require(allocation.live.size() == baseline, "Successful and interrupted traversals must release all temporary buffers and iterator states"); } + for (int mode : {0, 1, 2}) { + Value changing{}; _ZN3sce4Json5ValueC1Ev(&changing); + const char arrayJson[] = "[1,2]"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&changing, arrayJson, sizeof(arrayJson) - 1) == 0, "Parse collection mutation input"); + CollectionChange change{_ZNK3sce4Json5Value8getArrayEv(&changing), mode, {}}; + const auto expected = mode == 0 ? std::vector({"[", "0", ",1", ",2", "]"}) : mode == 1 ? std::vector({"[", "]"}) : std::vector({"[", "1", ",2", ",0", "]"}); + Require(_ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(&changing, PrependAtOpening, &change) == 0 && change.chunks == expected, "Serialization must obtain the live collection after the opening callback prepends, clears or appends entries"); + _ZN3sce4Json5ValueD1Ev(&changing); + Require(allocation.live.size() == baseline, "Collection mutation during serialization must leave no temporary or transferred allocations"); + } Value scalar{}; _ZN3sce4Json5ValueC1El(&scalar, 1234); for (bool collect : {false, true}) for (bool once : {false, true}) for (int budget : {0, 1, 2, 3}) { String output{}; _ZN3sce4Json6StringC1EPKc(&output, "p"); @@ -165,6 +206,20 @@ int main() { _ZN3sce4Json6StringD1Ev(&output); Require(allocation.live.size() == baseline, "Rejected serialization allocations must not leak admitted storage"); } + for (bool once : {false, true}) for (int budget : {0, 1, 2, 3}) { + String output{}; _ZN3sce4Json6StringC1EPKc(&output, "p"); + const auto requests = allocation.requests.size(); const auto notices = allocation.notifications; + allocation.budget = budget; allocation.once = once; + const int status = _ZN3sce4Json5Value9serializeERNS0_6StringE(&scalar, &output); + allocation.budget = -1; allocation.once = false; + Require(status == 0 && std::string(_ZNK3sce4Json6String5c_strEv(&output)) == (budget >= 3 ? "p1234" : "p"), "Ordinary serialization must preserve the native partial-output and return contracts under allocation failure"); + std::vector expected{24}; + if (budget >= 1) expected.push_back(21); + if (budget >= 2) expected.push_back(22); + Require(std::vector(allocation.requests.begin() + requests, allocation.requests.end()) == expected && allocation.notifications == notices + (budget < 3 ? 1 : 0), "Ordinary serialization must use the same allocator requests and failure notifications as the collecting callback path"); + _ZN3sce4Json6StringD1Ev(&output); + Require(allocation.live.size() == baseline, "Ordinary serialization failures must not leak storage"); + } _ZN3sce4Json5ValueD1Ev(&scalar); _ZN3sce4Json5ValueD1Ev(&root); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && allocation.live.empty(), "Terminate callback serialization allocation context"); return 0; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 72b8d2fd4d..70fa0d94fa 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -62,4 +62,4 @@ - [Value keyed lookup](../../../core/libs/prx/libSceJson2/Export.cpp) - valid string-key overloads and C-string key-construction failures follow the observed iterator allocation and fallback protocol. Missing object handles, corrupted tree storage, lifecycle reentry during lookup and remaining older-SDK callback combinations remain unverified; invalid reference arguments are explicitly unsupported. -- [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - modern chunk boundaries, callback return propagation, output mutation and scalar allocation rejection are reference-tested. Older-SDK serialization is explicitly unsupported until its parent-rebinding protocol is verified. Structural mutation of the traversed collection, nested iterator failures, non-finite recovery allocation failures and lifecycle reentry require further audit. Null callback or collecting-output references are explicitly unsupported. The existing non-callback serializer still follows a separate path and needs the same streaming/allocation audit. +- [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - modern chunk boundaries, callback return propagation, output mutation and scalar allocation rejection are reference-tested. Older-SDK serialization is explicitly unsupported until its parent-rebinding protocol is verified. Array prepend, clear and append during the opening callback are reference-tested; other structural mutations, nested iterator failures, non-finite recovery allocation failures and lifecycle reentry require further audit. Null callback or collecting-output references are explicitly unsupported. The modern non-callback serializer shares the verified collecting path, including escaping and scalar allocation failures. Its older-SDK path retains the previous implementation and still needs the streaming/allocation audit. From 4cc432e29aae946c433d7704656c53b23a58d633 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 07:19:33 +0200 Subject: [PATCH 042/318] fix(libSceJson2): reproduce legacy serialization metadata and real formatting --- core/libs/prx/libSceJson2/CMakeLists.txt | 12 + core/libs/prx/libSceJson2/Export.cpp | 64 +++++- .../tests/GuestJsonLegacySerialization.cpp | 215 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 282 insertions(+), 13 deletions(-) create mode 100644 core/libs/tests/GuestJsonLegacySerialization.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index a950e58591..9d63f01c95 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -195,3 +195,15 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + add_test_executable(guest_json_legacy_serialization_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonLegacySerialization.cpp Export.cpp) + target_include_directories(guest_json_legacy_serialization_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_legacy_serialization_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_legacy_serialization_tests) + add_test(NAME guest_json_legacy_serialization COMMAND guest_json_legacy_serialization_tests) + if(WIN32) + set_tests_properties(guest_json_legacy_serialization PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 78db96d5b8..b53966f71b 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1996,11 +1996,54 @@ struct SerializationIterator { template struct SerializationPosition { TLink* first; TLink* head; bool invalid; }; +void PrepareLegacySerialization(Value& value, const Value* parent) { + value.parent = const_cast(parent); + if (parent) { + if (value.auxiliary) { + if (value.auxiliary->fallback) { Destroy(*value.auxiliary->fallback); Release(value.auxiliary->fallback); } + Release(value.auxiliary); + value.auxiliary = nullptr; + } + } else { + EnsureAuxiliary(value); + if (value.auxiliary) { value.auxiliary->nullAccess = nullptr; value.auxiliary->nullAccessContext = nullptr; } + } +} + +void AppendLegacyReal(String& output, double value) { + float magnitude = static_cast(value); + const bool negative = magnitude < 0; + if (negative) magnitude = -magnitude; + const bool unordered = std::isnan(magnitude); + const std::uint64_t integer = unordered ? UINT64_C(0x8000000000000000) : + magnitude <= 2147483648.0f ? static_cast(magnitude) : UINT64_C(2147483647); + float fraction = unordered || magnitude > 2147483648.0f ? 0.0f : magnitude - static_cast(integer); + char digits[6]; + bool nonzero = false; + for (char& digit : digits) { + fraction *= 10.0f; + const auto number = static_cast(fraction) % 10; + digit = static_cast('0' + number); + nonzero = nonzero || number != 0; + } + char text[32]; + char* cursor = text; + if (negative) *cursor++ = '-'; + char whole[20]; + unsigned length = 0; + auto remaining = integer; + do { whole[length++] = static_cast('0' + remaining % 10); remaining /= 10; } while (remaining); + while (length) *cursor++ = whole[--length]; + if (nonzero) { *cursor++ = '.'; std::memcpy(cursor, digits, sizeof(digits)); cursor += sizeof(digits); } + AppendText(output, std::string_view(text, cursor - text)); +} + class CallbackSerializer { public: CallbackSerializer(String& buffer, String* destination, SerializationCallback callback, void* context, bool collect) : buffer(buffer), destination(destination), callback(callback), context(context), collect(collect) {} - int SerializeValue(const Value& value) { + int SerializeValue(const Value& value, const Value* parent = nullptr) { + if (!ModernValues()) PrepareLegacySerialization(const_cast(value), parent); if (value.type == TypeReal && !std::isfinite(value.real) && FloatConfiguration().enabled) { CallbackScope scope; const auto handlers = HandlersOf(value); @@ -2008,9 +2051,10 @@ public: ScopedValue replacement; bool error = false; RecoverSpecial(value, handlers, 0, value.real, replacement.value, &error); - return error ? JsonErrorSpecialFloat : SerializeValue(replacement.value); + return error ? JsonErrorSpecialFloat : SerializeValue(replacement.value, parent); } } + if (value.type == TypeReal && !FloatConfiguration().enabled) { AppendLegacyReal(buffer, value.real); return Emit(); } if (value.type >= TypeNull && value.type <= TypeReal) { std::string scalar; const int status = Serialize(scalar, value); @@ -2033,7 +2077,7 @@ public: }; return Collection(position, '[', ']', [](ArrayLink* link) { return link->next; }, - [this](ArrayLink* link) { return SerializeValue(link->value); }); + [this, &value](ArrayLink* link) { return SerializeValue(link->value, &value); }); } if (value.type == TypeObject) { if (!value.object) { NotImplemented_nid_no_patch(__func__); return 0; } @@ -2045,7 +2089,7 @@ public: }; return Collection(position, '{', '}', [](ObjectLink* link) { return NextObjectLink(link); }, - [this](ObjectLink* link) { Escape(link->pair.key); Character(':'); return SerializeValue(link->pair.value); }); + [this, &value](ObjectLink* link) { Escape(link->pair.key); Character(':'); return SerializeValue(link->pair.value, &value); }); } return 0; } @@ -2123,7 +2167,6 @@ int SerializeCallbackValue(Value* value, String* destination, SerializationCallb if (!state.allocation) return JsonErrorNotInitialized; } if (!callback || (collect && !destination)) { NotImplemented_nid_no_patch(__func__); return 0; } - if (!ModernValues()) { NotImplemented_nid_no_patch(__func__); return 0; } ScopedString buffer(std::string_view{}); CallbackSerializer serializer(buffer.string, destination, callback, context, collect); return serializer.SerializeValue(*value); @@ -2957,11 +3000,7 @@ int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value* self, String* ou std::lock_guard lock(state.mutex); if (!state.allocation) return JsonErrorNotInitialized; } - if (ModernValues()) return SerializeCallbackValue(self, out, SerializationSucceeded, nullptr, true); - std::string text; - const int result = Serialize(text, *self); - AppendText(*out, text); - return result; + return SerializeCallbackValue(self, out, SerializationSucceeded, nullptr, true); } void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value* self, String* out) { std::string text; @@ -2969,7 +3008,10 @@ void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value* self, Str else { if (self->type == TypeArray) text = "array"; else if (self->type == TypeObject) text = "object"; - else if (self->type == TypeReal) AppendReal(text, self->real); + else if (self->type == TypeReal) { + if (!FloatConfiguration().enabled) { AppendLegacyReal(*out, self->real); return; } + AppendReal(text, self->real); + } else Serialize(text, *self); AppendText(*out, text); } diff --git a/core/libs/tests/GuestJsonLegacySerialization.cpp b/core/libs/tests/GuestJsonLegacySerialization.cpp new file mode 100644 index 0000000000..13882c0ef8 --- /dev/null +++ b/core/libs/tests/GuestJsonLegacySerialization.cpp @@ -0,0 +1,215 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Iterator { void* state; }; +struct Object { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using LegacyNullCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using SerializationCallback = int (APS5_VABI*)(String&, void*); +extern "C" { +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, LegacyNullCallback, void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value*, std::size_t); +void APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value*, String*); +void APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(Array*, const Value*); +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +int APS5_VABI _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(Value*, SerializationCallback, void*); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_(Value*, String*, SerializationCallback, void*); +String* APS5_VABI _ZN3sce4Json6String6appendEPKc(String*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE(const Value*, const String*); +const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value*, const String*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5Array8pop_backEv(Array*); +Iterator* APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(Iterator*, Array*, const Iterator*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ArrayTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::set headers; +}; +static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeArray(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); + } + Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Array notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; +struct ArrayAllocator { void* const* vtable = ArrayVtable; }; +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + alignas(8) static const std::uint32_t parameters[]{0, 0, 0, 0, 0x05000000, 0}; + return parameters; +} +struct LegacySerializationTrace { std::vector chunks; }; +static int APS5_VABI ObserveLegacy(String& output, void* context) { + static_cast(context)->chunks.emplace_back(_ZNK3sce4Json6String5c_strEv(&output)); + return 23; +} +static const Value& APS5_VABI LegacyNull(std::int32_t, const Value*, void* context) { + Require(false, "Legacy serialization must clear the root null-access callback before later recovery"); + return *static_cast(context); +} +int main() { + ArrayTrace trace; ArrayAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&initializer, ¶meter) == 0, "Initialize legacy serialization contracts"); + const auto baseline = trace.live.size(); + struct Case { double value; const char* text; }; + const Case cases[]{ + {0.0, "0"}, {-0.0, "0"}, {1.0, "1"}, {1.25, "1.250000"}, + {0.0000001, "0"}, {-0.0000001, "-0"}, {-0.0000005, "-0"}, + {1.0000005, "1"}, {1.0000015, "1.000001"}, {1234567.875, "1234567.875000"}, + {1e8, "100000000"}, {1e10, "2147483647"}, {1e20, "2147483647"}, {-1e20, "-2147483647"}, + {std::numeric_limits::infinity(), "2147483647"}, {-std::numeric_limits::infinity(), "-2147483647"}, + {std::numeric_limits::quiet_NaN(), "9223372036854775808"}, + {std::bit_cast(UINT64_C(0xfff8000000000001)), "9223372036854775808"}, + }; + for (const auto& test : cases) for (int mode : {0, 1, 2, 3}) { + Value value{}; _ZN3sce4Json5ValueC1Ed(&value, test.value); + String output{}; _ZN3sce4Json6StringC1EPKc(&output, "p"); + const auto requests = trace.requests.size(); + LegacySerializationTrace callback; + int status = 0; + if (mode == 0) status = _ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output); + else if (mode == 1) status = _ZN3sce4Json5Value9serializeEPFiRNS0_6StringEPvES4_(&value, ObserveLegacy, &callback); + else if (mode == 2) status = _ZN3sce4Json5Value9serializeERNS0_6StringEPFiS3_PvES4_(&value, &output, ObserveLegacy, &callback); + else _ZNK3sce4Json5Value8toStringERNS0_6StringE(&value, &output); + Require(value.type == 4 && value.payload == std::bit_cast(test.value), "Formatting must preserve the original double payload bits, including signed zero and NaN payloads"); + const std::string expected = std::string("p") + test.text; + Require(status == (mode == 1 || mode == 2 ? 23 : 0), "Legacy serialization must preserve callback return values"); + Require(std::string(_ZNK3sce4Json6String5c_strEv(&output)) == (mode == 1 ? "p" : expected), "Legacy real formatting must preserve float32 conversion, truncation, sign and range behavior"); + if (mode == 1 || mode == 2) Require(callback.chunks == std::vector{mode == 1 ? std::string(test.text) : expected}, "Legacy callback serialization must emit the same formatted scalar through its selected output mode"); + std::vector allocations; + const auto length = std::strlen(test.text); + if (mode != 3) { + allocations = {24, 24, 32, (length + 1) | 16}; + Require(!value.parent && value.auxiliary && trace.live.at(value.auxiliary) == 24, "Legacy serialization must admit the root's allocator-backed auxiliary record"); + auto* fallback = static_cast(value.auxiliary)[2]; + Require(fallback && trace.live.at(fallback) == 32 && static_cast(fallback)->parent == &value, "The legacy root fallback must be allocator-backed and bound to its owning root"); + } else Require(!value.auxiliary, "Direct toString formatting must not perform serialization's metadata admission"); + if (mode != 1) allocations.push_back((length + 2) | 16); + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == allocations, "Legacy formatting and metadata admission must preserve the observed allocation order and request sizes"); + _ZN3sce4Json6StringD1Ev(&output); _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Legacy scalar serialization must release all buffers and root metadata on destruction"); + } + for (bool once : {false, true}) for (int budget = 0; budget <= 5; ++budget) { + Value value{}; _ZN3sce4Json5ValueC1Ed(&value, 1.25); + String output{}; _ZN3sce4Json6StringC1EPKc(&output, "p"); + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = budget; trace.once = once; + const int status = _ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output); + trace.budget = -1; trace.once = false; + const bool complete = budget >= 5 || (once && (budget == 1 || budget == 2)); + Require(status == 0 && std::string(_ZNK3sce4Json6String5c_strEv(&output)) == (complete ? "p1.250000" : "p"), "Legacy serialization must retain its observed partial output rather than inventing an out-of-memory return"); + const bool hasAuxiliary = budget >= 2 || (once && budget == 0); + Require((value.auxiliary != nullptr) == hasAuxiliary, "Rejected metadata admission must preserve the native partial auxiliary ownership"); + if (value.auxiliary) Require((static_cast(value.auxiliary)[2] != nullptr) == (budget >= 3 || (once && budget == 0)), "Fallback rejection must retain the admitted auxiliary record without fabricating a fallback"); + std::vector expected{24, 24}; + if (hasAuxiliary) expected.push_back(32); + if (budget != 0) expected.push_back(25); + if (budget >= 4 || (once && (budget == 1 || budget == 2))) expected.push_back(26); + const unsigned failed = once ? budget < 5 ? 1 : 0 : budget < 3 ? 2 : budget < 5 ? 1 : 0; + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected && trace.notifications == notices + failed, "Legacy metadata and output failures must preserve exact allocator request order and notification counts"); + _ZN3sce4Json6StringD1Ev(&output); _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Every partial legacy serialization state must remain destructible without leaked ownership"); + } + Value registered{}; _ZN3sce4Json5ValueC1El(®istered, 7); + String registeredOutput{}; _ZN3sce4Json6StringC1EPKc(®isteredOutput, ""); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(®istered, ®isteredOutput) == 0, "Admit metadata before registering a legacy root callback"); + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(®istered, LegacyNull, ®istered) == 0, "Register an existing legacy root null callback"); + void* registeredAuxiliary = registered.auxiliary; + void* registeredFallback = static_cast(registeredAuxiliary)[2]; + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(®istered, ®isteredOutput) == 0 && registered.auxiliary == registeredAuxiliary && static_cast(registeredAuxiliary)[2] == registeredFallback && !static_cast(registeredAuxiliary)[0] && !static_cast(registeredAuxiliary)[1], "Legacy serialization must reset existing root callback fields while preserving auxiliary and fallback identity"); + _ZN3sce4Json6StringD1Ev(®isteredOutput); _ZN3sce4Json5ValueD1Ev(®istered); + Require(trace.live.size() == baseline, "Reset legacy root handlers must retain normal cleanup ownership"); + Value root{}; _ZN3sce4Json5ValueC1Ev(&root); + const char json[] = "[1,2]"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, json, sizeof(json) - 1) == 0, "Parse legacy collection metadata input"); + auto* child = const_cast(_ZNK3sce4Json5ValueixEm(&root, 0)); + String first{}; _ZN3sce4Json6StringC1EPKc(&first, ""); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(child, &first) == 0 && !child->parent && child->auxiliary, "Serializing a legacy child as a root must detach its parent and admit root metadata"); + void* childAuxiliary = child->auxiliary; + void* childFallback = static_cast(childAuxiliary)[2]; + String whole{}; _ZN3sce4Json6StringC1EPKc(&whole, ""); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&root, &whole) == 0 && std::strcmp(_ZNK3sce4Json6String5c_strEv(&whole), "[1,2]") == 0, "Legacy collection serialization must preserve its actual contents"); + Require(child->parent == &root && !child->auxiliary && !trace.live.count(childAuxiliary) && !trace.live.count(childFallback), "Serializing the parent must rebind the child and release its former root auxiliary record and fallback"); + _ZN3sce4Json6StringD1Ev(&first); _ZN3sce4Json6StringD1Ev(&whole); _ZN3sce4Json5ValueD1Ev(&root); + Require(trace.live.size() == baseline && _ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Legacy metadata transitions must leave no outstanding ownership"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 70fa0d94fa..5b1d24b986 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -7,7 +7,7 @@ - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size and double overflow throw rather than inventing a result. Numeric-token validation, more complex parser allocation failures and cross-thread operation behavior remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. -- [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter differs from the host formatter and remains unsupported for non-finite serialization. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. +- [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter is now reproduced through the inspected float32 conversion, six truncating fractional digits, range saturation and NaN integer result. The regression covers signed zero, precision boundaries, large values, infinities and NaNs; floating-point rounding modes and exception flags remain unverified. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - `Pair(const String&, const Value&)` and default `Pair()` zero the 8 bytes between the key and value; what the console keeps there is unknown. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - libc-heap failures and untested legacy callback combinations remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Additional legacy compatibility remains unverified. @@ -62,4 +62,4 @@ - [Value keyed lookup](../../../core/libs/prx/libSceJson2/Export.cpp) - valid string-key overloads and C-string key-construction failures follow the observed iterator allocation and fallback protocol. Missing object handles, corrupted tree storage, lifecycle reentry during lookup and remaining older-SDK callback combinations remain unverified; invalid reference arguments are explicitly unsupported. -- [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - modern chunk boundaries, callback return propagation, output mutation and scalar allocation rejection are reference-tested. Older-SDK serialization is explicitly unsupported until its parent-rebinding protocol is verified. Array prepend, clear and append during the opening callback are reference-tested; other structural mutations, nested iterator failures, non-finite recovery allocation failures and lifecycle reentry require further audit. Null callback or collecting-output references are explicitly unsupported. The modern non-callback serializer shares the verified collecting path, including escaping and scalar allocation failures. Its older-SDK path retains the previous implementation and still needs the streaming/allocation audit. +- [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - nested iterator/metadata failures, non-finite recovery failures, lifecycle reentry, other legacy callback combinations and floating-point environment behavior still require audit. Null callback/output references are explicitly unsupported. All serialization shares the collecting/traversal protocol. Reference tests cover modern chunk boundaries, callback results, output mutation, scalar allocation failures and opening-callback array mutations. SDK 5 tests cover root admission/reset, child metadata release, numeric edge cases and persistent/one-shot root admission failures. From dc82a592ea8855f65438e45b642ed9e4c849edd1 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 07:34:38 +0200 Subject: [PATCH 043/318] fix(libSceJson2): reproduce decimal range conversion and native rounding --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 57 ++++++-- core/libs/tests/GuestJson.cpp | 46 +++---- core/libs/tests/GuestJsonDecimalRange.cpp | 152 ++++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 5 files changed, 224 insertions(+), 42 deletions(-) create mode 100644 core/libs/tests/GuestJsonDecimalRange.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 9d63f01c95..e26be55400 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_decimal_range_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonDecimalRange.cpp) + target_include_directories(guest_json_decimal_range_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_decimal_range_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_decimal_range_tests) + add_test(NAME guest_json_decimal_range COMMAND guest_json_decimal_range_tests) add_test_executable(guest_json_serialization_callback_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonSerializationCallback.cpp) target_include_directories(guest_json_serialization_callback_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_serialization_callback_tests PRIVATE libSceJson2 libc) @@ -136,7 +141,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index b53966f71b..e8be894349 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -6,13 +6,12 @@ #include #include #include +#include #include #include -#include #include #include #include -#include #include #include #include @@ -2174,6 +2173,46 @@ int SerializeCallbackValue(Value* value, String* destination, SerializationCallb using ParserReadCallback = int (APS5_VABI*)(char&, void*); +double ParseDecimal(std::string_view text) { + std::size_t cursor = 0; + while (cursor < text.size() && (text[cursor] == ' ' || (text[cursor] >= '\t' && text[cursor] <= '\r'))) ++cursor; + double sign = 1.0; + while (cursor < text.size() && (text[cursor] == '+' || text[cursor] == '-')) { + if (text[cursor++] == '-') sign = -sign; + } + auto digit = [&](std::size_t index) { return index < text.size() && text[index] >= '0' && text[index] <= '9'; }; + double magnitude = 0; + bool admitted = false; + while (digit(cursor)) { + volatile double scaled = magnitude * 10.0; + magnitude = scaled + (text[cursor++] - '0'); + admitted = true; + } + if (cursor < text.size() && text[cursor] == '.') { + ++cursor; + double divisor = 10.0; + while (digit(cursor)) { + magnitude += (text[cursor++] - '0') / divisor; + divisor *= 10.0; + admitted = true; + } + } + if (!admitted) return 0; + if (cursor < text.size() && (text[cursor] == 'e' || text[cursor] == 'E' || text[cursor] == 'd' || text[cursor] == 'D')) { + ++cursor; + bool divide = false; + if (cursor < text.size() && (text[cursor] == '+' || text[cursor] == '-')) divide = text[cursor++] == '-'; + if (!digit(cursor)) return 0; + std::uint32_t exponent = 0; + while (digit(cursor)) exponent = exponent * 10 + static_cast(text[cursor++] - '0'); + while (exponent-- && magnitude != 0 && !std::isinf(magnitude)) { + if (divide) magnitude /= 10.0; + else magnitude *= 10.0; + } + } + return magnitude * sign; +} + class Parser { public: Parser(const char* text, std::size_t size) : text(text), end(text + size) {} @@ -2330,19 +2369,9 @@ private: } } } - double real = 0; - const auto parsed = std::from_chars(text.data(), text.data() + text.size(), real); - bool converted = parsed.ec == std::errc{} && parsed.ptr == text.data() + text.size(); - if (parsed.ec == std::errc::result_out_of_range) { - std::istringstream stream(std::string{text}); - stream.imbue(std::locale::classic()); - stream >> real; - converted = !stream.fail() && stream.eof(); - } - if (!converted || !std::isfinite(real)) throw std::runtime_error("sce::Json::Parser::parse: number " + std::string{text} + " overflows a double; the console's behaviour is unverified"); - SetType(out, TypeReal); - out.real = real; + const double real = ParseDecimal(text); Admit(out); + AssignReal(out, real); return true; } bool value(Node& out) { diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index 9f8f8ee842..05b6b86c4d 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "JsonTestAllocator.hpp" +#include #include #include #include @@ -264,16 +265,12 @@ static void ParseAndRoundTrip() { Require(nullTextRejected); Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, "", 0) == JsonErrorParse); Require(_ZNK3sce4Json5Value7getTypeEv(&root) == TypeObject && _ZNK3sce4Json5Value5countEv(&root) == 8); - for (const char* overflow : {"1e309", "[-1e309]", "{\"a\":[1,1e400]}"}) { + for (const auto& [overflow, expected] : std::vector>{ + {"1e309", "inf"}, {"[-1e309]", "[-inf]"}, {"{\"a\":[1,1e400]}", "{\"a\":[1,inf]}"}}) { Value partial{}; _ZN3sce4Json5ValueC1Ev(&partial); - bool unverified = false; - try { - _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&partial, overflow, std::strlen(overflow)); - } catch (const std::runtime_error& error) { - unverified = std::string(error.what()).find("unverified") != std::string::npos; - } - Require(unverified); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&partial, overflow, std::strlen(overflow)) == 0); + Require(Serialize(partial) == expected); _ZN3sce4Json5ValueD1Ev(&partial); } _ZN3sce4Json5ValueD1Ev(&root); @@ -1416,8 +1413,12 @@ static int NumericLocale() { ++failures; } }; - for (const auto number : {1.5, -0.125, 1.25e30, std::numeric_limits::max(), - std::numeric_limits::min(), std::numeric_limits::denorm_min()}) { + for (const auto [number, expected] : std::vector>{ + {1.5, UINT64_C(0x3ff8000000000000)}, {-0.125, UINT64_C(0xbfc0000000000000)}, + {1.25e30, UINT64_C(0x462f8def8808b023)}, + {std::numeric_limits::max(), UINT64_C(0x7feffffffffffff9)}, + {std::numeric_limits::min(), UINT64_C(0x0010000000000003)}, + {std::numeric_limits::denorm_min(), UINT64_C(1)}}) { Value value{}; _ZN3sce4Json5ValueC1Ed(&value, number); const auto text = Serialize(value); @@ -1425,21 +1426,21 @@ static int NumericLocale() { Value parsed{}; _ZN3sce4Json5ValueC1Ev(&parsed); const int result = _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&parsed, text.c_str(), text.size()); - check(result == 0 && *_ZNK3sce4Json5Value7getRealEv(&parsed) == number, "Numeric round trip", text.c_str()); + check(result == 0 && std::bit_cast(*_ZNK3sce4Json5Value7getRealEv(&parsed)) == expected, "Reference decimal conversion", text.c_str()); _ZN3sce4Json5ValueD1Ev(&parsed); _ZN3sce4Json5ValueD1Ev(&value); } - for (const auto [text, number] : {std::pair{"1.5", 1.5}, {"-1.25e2", -125.0}, - {"4.9406564584124654e-324", std::numeric_limits::denorm_min()}, - {"2.4703282292062327e-324", 0.0}, - {"2.4703282292062328e-324", std::numeric_limits::denorm_min()}, - {"18446744073709551616", 18446744073709551616.0}, - {"1e-999", 0.0}, {"-1e-999", -0.0}}) { + for (const auto [text, expected] : std::vector>{ + {"1.5", UINT64_C(0x3ff8000000000000)}, {"-1.25e2", UINT64_C(0xc05f400000000000)}, + {"4.9406564584124654e-324", UINT64_C(1)}, {"2.4703282292062327e-324", UINT64_C(0)}, + {"2.4703282292062328e-324", UINT64_C(0)}, + {"18446744073709551616", UINT64_C(0x43f0000000000000)}, + {"1e-999", UINT64_C(0)}, {"-1e-999", UINT64_C(0x8000000000000000)}}) { Value value{}; _ZN3sce4Json5ValueC1Ev(&value); const int result = _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, text, std::strlen(text)); const auto parsed = *_ZNK3sce4Json5Value7getRealEv(&value); - check(result == 0 && parsed == number && std::signbit(parsed) == std::signbit(number), "Decimal-point parsing", text); + check(result == 0 && std::bit_cast(parsed) == expected, "Decimal-point parsing", text); _ZN3sce4Json5ValueD1Ev(&value); } Value root{}; @@ -1451,13 +1452,8 @@ static int NumericLocale() { check(serialized == document, "Nested numeric serialization", serialized.c_str()); check(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, "1,5", 3) == 0 && _ZNK3sce4Json5Value7getTypeEv(&root) == TypeUInteger && *_ZNK3sce4Json5Value11getUIntegerEv(&root) == 1, "Stop at a decimal comma", "1,5"); - bool overflow = false; - try { - _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, "1e309", 5); - } catch (const std::runtime_error&) { - overflow = true; - } - check(overflow, "Retain overflow rejection", "1e309"); + check(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, "1e309", 5) == 0 && + std::isinf(*_ZNK3sce4Json5Value7getRealEv(&root)), "Reference overflow admission", "1e309"); _ZN3sce4Json5ValueD1Ev(&root); check(std::strcmp(std::localeconv()->decimal_point, ",") == 0, "Preserve host locale", selected); std::setlocale(LC_NUMERIC, previous.c_str()); diff --git a/core/libs/tests/GuestJsonDecimalRange.cpp b/core/libs/tests/GuestJsonDecimalRange.cpp new file mode 100644 index 0000000000..91cefb7401 --- /dev/null +++ b/core/libs/tests/GuestJsonDecimalRange.cpp @@ -0,0 +1,152 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Iterator { void* state; }; +struct Object { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using LookupCallback = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); +extern "C" { +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE(const Value*, const String*); +const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value*, const String*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, LookupCallback, void*); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5Array8pop_backEv(Array*); +Iterator* APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(Iterator*, Array*, const Iterator*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ArrayTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::set headers; +}; +static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; +} +static void APS5_VABI FreeArray(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); + } + Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Array notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; +struct ArrayAllocator { void* const* vtable = ArrayVtable; }; +int main() { + ArrayTrace trace; ArrayAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize decimal underflow contracts"); + const auto baseline = trace.live.size(); + struct Case { const char* text; std::uint64_t bits; }; + const Case cases[]{ + {"1e309", UINT64_C(0x7ff0000000000000)}, {"-1e309", UINT64_C(0xfff0000000000000)}, + {"1.25e30", UINT64_C(0x462f8def8808b023)}, {"18446744073709551616", UINT64_C(0x43f0000000000000)}, + {"1e-999", 0}, {"-1e-999", UINT64_C(0x8000000000000000)}, + {"2.4703282292062327e-324", 0}, {"-2.4703282292062327e-324", UINT64_C(0x8000000000000000)}, + {"2.4703282292062328e-324", 0}, {"-2.4703282292062328e-324", UINT64_C(0x8000000000000000)}, + {"4.9406564584124654e-324", 1}, {"-4.9406564584124654e-324", UINT64_C(0x8000000000000001)}, + {"2.2250738585072014e-308", UINT64_C(0x0010000000000003)}, + {"1.7976931348623157e308", UINT64_C(0x7feffffffffffff9)}, + }; + for (const auto& test : cases) { + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, test.text, std::strlen(test.text)) == 0, "Decimal range conversion must preserve native admission"); + Require(value.type == 4 && value.payload == test.bits, "Decimal conversion must preserve the reference arithmetic, subnormal bits and the sign of underflowed zero"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Underflow parsing must release token, value and auxiliary ownership"); + } + for (const auto& test : {std::pair{"[-1e309]", "[-inf]"}, std::pair{"{\"a\":[1,1e400]}", "{\"a\":[1,inf]}"}}) { + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + String output{}; _ZN3sce4Json6StringC1EPKc(&output, ""); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, test.first, std::strlen(test.first)) == 0 && _ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output) == 0 && std::strcmp(_ZNK3sce4Json6String5c_strEv(&output), test.second) == 0, "Nested decimal overflow must preserve the reference collection and real-conversion behavior"); + _ZN3sce4Json6StringD1Ev(&output); _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == baseline, "Nested overflow must release all token, collection and auxiliary storage"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate decimal range allocation context"); + for (int mode : {0, 1}) { + parameter.mode = mode; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize overflow conversion settings"); + const auto resources = trace.live.size(); + for (const char* text : {"1e309", "-1e309"}) { + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, text, std::strlen(text)) == 0, "Overflow conversion must honor the configured special-float mode"); + if (mode == 0) Require(value.type == 5 && value.payload && std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(value.payload)), text[0] == '-' ? "-inf" : "inf") == 0, "String mode must convert overflowing decimal values through the normal real setter"); + else Require(value.type == 0, "Null mode must convert overflowing decimal values to null"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live.size() == resources, "Overflow conversion must release admitted value and auxiliary storage"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate overflow conversion settings"); + } + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 5b1d24b986..c4d55c2f49 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -2,9 +2,9 @@ ### Unknown function info -- [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) uses locale-independent `to_chars` with 17 significant digits and `from_chars`, following JSON's decimal-point grammar in RFC 8259 section 6. Out-of-range input falls back to a classic-locale stream so underflow retains the prior signed-zero behavior and overflow throws. Console rounding and formatting are unverified. +- [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) - floating-point environment behavior and malformed numeric-token grammar remain unverified. Valid decimal conversion follows the instructions at `0x7750`: separate multiply/add accumulation, per-digit fractional division and repeated exponent scaling. Reference-backed exact-bit tests cover signed underflow, subnormals, normal boundaries, large finite values and overflow across special-float modes. These results intentionally retain the reference rounding, which differs from standard correctly rounded conversion. Formatting still uses locale-independent `to_chars` with 17 significant digits for modern settings; broader formatting equivalence remains unverified. - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) - the console result for an empty array remains unverified; the existing host behavior throws `std::out_of_range`. Missing storage throws unsupported. Nonempty removal releases the allocator-owned node without allocating and preserves surviving element/iterator identity. -- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size and double overflow throw rather than inventing a result. Numeric-token validation, more complex parser allocation failures and cross-thread operation behavior remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. +- [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size throws; decimal overflow follows the configured real-conversion mode. Numeric-token validation, more complex parser allocation failures and cross-thread operation behavior remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter is now reproduced through the inspected float32 conversion, six truncating fractional digits, range saturation and NaN integer result. The regression covers signed zero, precision boundaries, large values, infinities and NaNs; floating-point rounding modes and exception flags remain unverified. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. From cfa085a6f823170ee12cae0ddfae18f230197494 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 07:43:25 +0200 Subject: [PATCH 044/318] fix(libSceJson2): preserve parser parent and auxiliary ownership --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 57 +++--- core/libs/tests/GuestJsonParserMetadata.cpp | 196 ++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 232 insertions(+), 32 deletions(-) create mode 100644 core/libs/tests/GuestJsonParserMetadata.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index e26be55400..34466f8bd5 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_parser_metadata_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonParserMetadata.cpp) + target_include_directories(guest_json_parser_metadata_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_parser_metadata_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_parser_metadata_tests) + add_test(NAME guest_json_parser_metadata COMMAND guest_json_parser_metadata_tests) add_test_executable(guest_json_decimal_range_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonDecimalRange.cpp) target_include_directories(guest_json_decimal_range_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_decimal_range_tests PRIVATE libSceJson2 libc) @@ -141,7 +146,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index e8be894349..f4874473c3 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1133,8 +1133,7 @@ void Clear(Node& n) { n.type = TypeNull; } -void Destroy(Value& v) { - Clear(v); +void ReleaseValueAuxiliary(Value& v) { if (v.auxiliary) { if (ModernValues()) { if (v.auxiliary->typeMismatch) Release(v.auxiliary->typeMismatch); @@ -1152,6 +1151,11 @@ void Destroy(Value& v) { Release(v.auxiliary); v.auxiliary = nullptr; } +} + +void Destroy(Value& v) { + Clear(v); + ReleaseValueAuxiliary(v); if (!ModernValues()) EraseLegacyHandler(v); } @@ -1995,17 +1999,12 @@ struct SerializationIterator { template struct SerializationPosition { TLink* first; TLink* head; bool invalid; }; -void PrepareLegacySerialization(Value& value, const Value* parent) { +void PrepareValueAdmission(Value& value, const Value* parent) { value.parent = const_cast(parent); - if (parent) { - if (value.auxiliary) { - if (value.auxiliary->fallback) { Destroy(*value.auxiliary->fallback); Release(value.auxiliary->fallback); } - Release(value.auxiliary); - value.auxiliary = nullptr; - } - } else { + if (parent) ReleaseValueAuxiliary(value); + else { EnsureAuxiliary(value); - if (value.auxiliary) { value.auxiliary->nullAccess = nullptr; value.auxiliary->nullAccessContext = nullptr; } + if (!ModernValues() && value.auxiliary) { value.auxiliary->nullAccess = nullptr; value.auxiliary->nullAccessContext = nullptr; } } } @@ -2042,7 +2041,7 @@ public: CallbackSerializer(String& buffer, String* destination, SerializationCallback callback, void* context, bool collect) : buffer(buffer), destination(destination), callback(callback), context(context), collect(collect) {} int SerializeValue(const Value& value, const Value* parent = nullptr) { - if (!ModernValues()) PrepareLegacySerialization(const_cast(value), parent); + if (!ModernValues()) PrepareValueAdmission(const_cast(value), parent); if (value.type == TypeReal && !std::isfinite(value.real) && FloatConfiguration().enabled) { CallbackScope scope; const auto handlers = HandlersOf(value); @@ -2324,10 +2323,8 @@ private: take(); return true; } - void Admit(Node& out) { - if (!out.parent) EnsureAuxiliary(out); - } - bool number(Node& out) { + void Admit(Node& out, const Value* parent) { PrepareValueAdmission(out, parent); } + bool number(Node& out, const Value* parent) { ScopedString token{std::string_view{}}; auto append = [&](char character) { AppendText(token.string, {&character, 1}); }; if (peek() >= 0 && peek() == '-') append(static_cast(take())); @@ -2351,38 +2348,38 @@ private: if (text.empty()) { SetType(out, TypeUInteger); out.uinteger = 0; - Admit(out); + Admit(out, parent); return true; } if (integral) { errno = 0; if (text[0] == '-') { const long long value = std::strtoll(Characters(token.string), nullptr, 10); - if (errno == 0) { SetType(out, TypeInteger); out.integer = value; Admit(out); return true; } + if (errno == 0) { SetType(out, TypeInteger); out.integer = value; Admit(out, parent); return true; } } else { const unsigned long long value = std::strtoull(Characters(token.string), nullptr, 10); if (errno == 0) { SetType(out, TypeUInteger); out.uinteger = value; - Admit(out); + Admit(out, parent); return true; } } } const double real = ParseDecimal(text); - Admit(out); + Admit(out, parent); AssignReal(out, real); return true; } - bool value(Node& out) { + bool value(Node& out, const Value* parent = nullptr) { if (AllocationOperation::Failed() || peek() < 0) return false; switch (peek()) { - case 'n': if (!literal("null")) return false; SetType(out, TypeNull); Admit(out); return true; - case 't': if (!literal("true")) return false; SetType(out, TypeBoolean); out.boolean = true; Admit(out); return true; - case 'f': if (!literal("false")) return false; SetType(out, TypeBoolean); out.boolean = false; Admit(out); return true; + case 'n': if (!literal("null")) return false; SetType(out, TypeNull); Admit(out, parent); return true; + case 't': if (!literal("true")) return false; SetType(out, TypeBoolean); out.boolean = true; Admit(out, parent); return true; + case 'f': if (!literal("false")) return false; SetType(out, TypeBoolean); out.boolean = false; Admit(out, parent); return true; case '"': { SetType(out, TypeString); - Admit(out); + Admit(out, parent); if (out.type != TypeString) throw GuestOutOfMemory{}; if (AllocationOperation::Failed()) return false; ParsedText text{*out.string}; @@ -2393,7 +2390,7 @@ private: take(); ++depth; SetType(out, TypeArray); - Admit(out); + Admit(out, parent); if (out.type != TypeArray) throw GuestOutOfMemory{}; skip(); if (peek() >= 0 && peek() == ']') { take(); --depth; return true; } @@ -2406,7 +2403,7 @@ private: auto& item = out.array->items->back(); item.parent = &out; skip(); - if (!value(item)) return false; + if (!value(item, &out)) return false; skip(); if (peek() >= 0 && peek() == ',') { take(); continue; } if (peek() >= 0 && peek() == ']') { take(); --depth; return true; } @@ -2418,7 +2415,7 @@ private: take(); ++depth; SetType(out, TypeObject); - Admit(out); + Admit(out, parent); if (out.type != TypeObject) throw GuestOutOfMemory{}; skip(); if (peek() >= 0 && peek() == '}') { take(); --depth; return true; } @@ -2432,7 +2429,7 @@ private: take(); skip(); Value& entry = *ObjectAccess(*out.object, key.string); - if (!value(entry)) return false; + if (!value(entry, &out)) return false; ClearText(key.string); skip(); if (peek() >= 0 && peek() == ',') { take(); continue; } @@ -2442,7 +2439,7 @@ private: } default: if (peek() != '-' && (peek() < '0' || peek() > '9')) return false; - return number(out); + return number(out, parent); } } }; diff --git a/core/libs/tests/GuestJsonParserMetadata.cpp b/core/libs/tests/GuestJsonParserMetadata.cpp new file mode 100644 index 0000000000..1c6f3982d2 --- /dev/null +++ b/core/libs/tests/GuestJsonParserMetadata.cpp @@ -0,0 +1,196 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Array { void* storage; }; +struct Iterator { void* state; }; +struct Object { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using LookupCallback = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); +using NullCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using MismatchCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +using FloatCallback = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +extern "C" { +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullCallback, void*); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, MismatchCallback, void*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatCallback, void*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixERKNS0_6StringE(const Value*, const String*); +const Value* APS5_VABI _ZNK3sce4Json5Value8getValueERKNS0_6StringE(const Value*, const String*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, LookupCallback, void*); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5Array8pop_backEv(Array*); +Iterator* APS5_VABI _ZN3sce4Json5Array5eraseERKNS1_8iteratorE(Iterator*, Array*, const Iterator*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array5clearEv(Array*); +void APS5_VABI _ZN3sce4Json5Array9pop_frontEv(Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorC1ERKS2_(Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); +void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); +const Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct ArrayTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::set headers; + std::vector frees; + std::map ids; + std::size_t nextId = 0; +}; +static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); trace.ids[memory] = ++trace.nextId; return memory; +} +static void APS5_VABI FreeArray(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + Require(trace.ids.count(memory) == 1, "Freed metadata must identify a currently live allocation generation"); + trace.frees.push_back(trace.ids.at(memory)); trace.ids.erase(memory); + if (trace.headers.erase(memory)) { + const auto* words = static_cast(memory); + Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); + } + Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Array notification must identify the failed allocation"); ++trace.notifications; +} +static void* const ArrayVtable[]{nullptr, nullptr, reinterpret_cast(AllocateArray), reinterpret_cast(FreeArray), reinterpret_cast(NotifyArray)}; +struct ArrayAllocator { void* const* vtable = ArrayVtable; }; +static const Value& APS5_VABI RejectNull(std::int32_t, const Value*, void* context) { + Require(false, "Parser metadata admission must not invoke null recovery"); + return *static_cast(context); +} +static const Value& APS5_VABI RejectMismatch(std::int32_t, const Value* value, const Value*, void*) { + Require(false, "Parser metadata admission must not invoke mismatch recovery"); + return *value; +} +static Value* APS5_VABI RejectFloat(Value* result, std::int32_t, double, const Value*, void*, bool*) { + Require(false, "Finite parser metadata admission must not invoke special-float recovery"); + return result; +} +static void APS5_VABI RejectElement(int, const String*, const std::size_t*, const Value*, const Value*, void*) { + Require(false, "Parser metadata admission must not invoke element recovery"); +} +static Value* MakeRoot(Value& root) { + _ZN3sce4Json5ValueC1Ev(&root); + constexpr char json[] = "{\"child\":7}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, json, sizeof(json) - 1) == 0, "Parse an owned child for metadata admission"); + auto* child = const_cast(_ZNK3sce4Json5ValueixEPKc(&root, "child")); + Require(child && child->parent == &root && !child->auxiliary && child->type == 3 && child->payload == 7, "Initial child ownership must match the containing object"); + return child; +} +int main() { + ArrayTrace trace; ArrayAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize parser metadata transitions"); + const auto baseline = trace.live.size(); + for (bool once : {false, true}) for (int budget : {0, 1, 2}) { + Value root{}; auto* child = MakeRoot(root); + const Value before = *child; + const auto beforeInvalid = trace.requests.size(); + Require(static_cast(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(child, "tru", 3)) == 0x80848101 && std::memcmp(child, &before, sizeof(before)) == 0 && trace.requests.size() == beforeInvalid, "An incomplete literal must preserve the existing child and metadata without admission"); + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = budget; trace.once = once; + const int status = _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(child, "true", 4); + trace.budget = -1; trace.once = false; + Require(status == (budget < 2 ? static_cast(0x80848102) : 0) && !child->parent && child->type == 1 && (child->payload & 255) == 1, "A recognized public parse must detach the child and retain its admitted scalar even when root metadata allocation fails"); + Require((child->auxiliary != nullptr) == (budget != 0), "Auxiliary allocation failure must leave the detached child without invented metadata"); + if (child->auxiliary) Require((static_cast(child->auxiliary)[2] != nullptr) == (budget == 2), "Fallback failure must retain its admitted auxiliary record without synthesizing a fallback"); + const auto expected = budget == 0 ? std::vector{72} : std::vector{72, 32}; + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected && trace.notifications == notices + (budget < 2 ? 1 : 0), "Child-to-root admission must preserve exact allocation order and notification counts"); + void* auxiliary = child->auxiliary; + const auto retryRequests = trace.requests.size(); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(child, "false", 5) == 0 && !child->parent && child->type == 1 && (child->payload & 255) == 0, "Retry must parse normally after independent admission failure"); + Require(trace.requests.size() == retryRequests + (budget == 0 ? 2 : 0) && (budget == 0 || child->auxiliary == auxiliary), "Retry must reuse admitted metadata and must not recreate a missing fallback"); + _ZN3sce4Json5ValueD1Ev(&root); + Require(trace.live.size() == baseline, "The containing object's destruction must release detached child metadata through its original allocator"); + } + struct Recognized { const char* text; std::int32_t type; }; + for (const auto& input : {Recognized{"null", 0}, Recognized{"true", 1}, Recognized{"false", 1}, Recognized{"123", 3}, Recognized{"3.5", 4}, Recognized{"\"x\"", 5}, Recognized{"[1]", 6}, Recognized{"{\"x\":1}", 7}}) { + Value root{}; auto* child = MakeRoot(root); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(child, input.text, std::strlen(input.text)) == 0 && child->type == input.type && !child->parent && child->auxiliary && static_cast(static_cast(child->auxiliary)[2])->parent == child, "Every recognized scalar or collection parsed directly into a child must admit it as a root"); + if (input.type == 6) Require(_ZNK3sce4Json5Array5frontEv(reinterpret_cast(child->payload))->parent == child, "A detached parsed array must bind its newly parsed elements to its actual value"); + if (input.type == 7) Require(_ZNK3sce4Json5ValueixEPKc(child, "x")->parent == child, "A detached parsed object must bind its newly parsed entries to its actual value"); + _ZN3sce4Json5ValueD1Ev(&root); + Require(trace.live.size() == baseline, "Every recognized child-to-root type transition must retain complete cleanup ownership"); + } + Value root{}; auto* child = MakeRoot(root); + void* rootAuxiliary = root.auxiliary; + void* rootFallback = static_cast(rootAuxiliary)[2]; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(child, "true", 4) == 0, "Admit a child as an independent root before registering handlers"); + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(child, RejectNull, child) == 0 && + _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(child, RejectMismatch, child) == 0 && + _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(child, RejectFloat, child) == 0 && + _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(child, RejectElement, child) == 0, "Register all four callback forms on the detached root"); + auto** auxiliary = static_cast(child->auxiliary); + const std::vector owned{auxiliary[3], auxiliary[4], auxiliary[5], auxiliary[2], child->auxiliary}; + for (void* allocation : owned) Require(allocation && trace.live.count(allocation), "All callback records, fallback and auxiliary storage must belong to the supplied allocator"); + const auto beforeRootParse = trace.requests.size(); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(child, "false", 5) == 0 && child->auxiliary == auxiliary && !child->parent && trace.requests.size() == beforeRootParse, "Modern root reparsing must reuse its existing auxiliary record without allocating"); + Require(auxiliary[0] == reinterpret_cast(&RejectNull) && auxiliary[1] == child && auxiliary[2] == owned[3] && auxiliary[3] == owned[0] && auxiliary[4] == owned[1] && auxiliary[5] == owned[2], "Modern root admission must preserve every existing callback, context, handler record and fallback identity"); + std::vector ownedIds; + for (void* allocation : owned) ownedIds.push_back(trace.ids.at(allocation)); + const auto frees = trace.frees.size(); + constexpr char replacement[] = "{\"child\":false}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, replacement, sizeof(replacement) - 1) == 0, "Reparse the containing object using its existing entry"); + Require(child == _ZNK3sce4Json5ValueixEPKc(&root, "child") && child->parent == &root && !child->auxiliary && child->type == 1 && (child->payload & 255) == 0, "Nested admission must preserve node identity, rebind its parent and remove former root metadata"); + std::vector released; + for (auto i = frees; i < trace.frees.size(); ++i) for (auto id : ownedIds) if (id == trace.frees[i]) released.push_back(id); + Require(released == ownedIds && root.auxiliary == rootAuxiliary && static_cast(rootAuxiliary)[2] == rootFallback, "Nested admission must release every handler record, fallback and auxiliary in native order while preserving root metadata"); + _ZN3sce4Json5ValueD1Ev(&root); + Require(trace.live.size() == baseline && _ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty() && trace.ids.empty(), "Parser metadata transitions must leave no outstanding ownership"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index c4d55c2f49..37d87e833e 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -42,7 +42,7 @@ - [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copies, alternate SDK behavior and concurrent failure accounting remain unverified. -- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. More complex parser low-memory paths, callback-driven allocation failures and child-metadata rebinding remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. +- [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. More complex parser low-memory paths and callback-driven allocation failures remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. - [sce::Json::Parser array allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - numeric token storage uses the supplied allocator; array parsing uses the public append implementation. Parser and append operations share the depth/error flag observed at `0x6840`, `0x310`, `0x340` and `0xb880`, retaining inherited errors until the outer operation finishes. Twenty-four reference-backed cases cover persistent and one-shot failures through successful parsing, partial counts, numeric-zero fallback after token-allocation failure, string suffixes, parent links, notifications and cleanup. Missing-sentinel parsing throws unsupported because reference execution crashes; callback-bearing collections and concurrent behavior remain incomplete. @@ -63,3 +63,5 @@ - [Value keyed lookup](../../../core/libs/prx/libSceJson2/Export.cpp) - valid string-key overloads and C-string key-construction failures follow the observed iterator allocation and fallback protocol. Missing object handles, corrupted tree storage, lifecycle reentry during lookup and remaining older-SDK callback combinations remain unverified; invalid reference arguments are explicitly unsupported. - [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - nested iterator/metadata failures, non-finite recovery failures, lifecycle reentry, other legacy callback combinations and floating-point environment behavior still require audit. Null callback/output references are explicitly unsupported. All serialization shares the collecting/traversal protocol. Reference tests cover modern chunk boundaries, callback results, output mutation, scalar allocation failures and opening-callback array mutations. SDK 5 tests cover root admission/reset, child metadata release, numeric edge cases and persistent/one-shot root admission failures. + +- [Parser parent and auxiliary transitions](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-driven reentry, more complex nested metadata failures and alternate SDK combinations still require audit. Reference tests cover direct parsing into an owned child across eight value forms, six root-admission failure/retry cases, incomplete-literal preservation, modern root callback retention and nested release of three handler records, fallback and auxiliary in native order. Allocation generations distinguish release from later address reuse. From 471a5b97c4d912566d79b5c541ac698ede57536f Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:10:09 +0200 Subject: [PATCH 045/318] fix(libSceJson2): preserve internal keys through callback-return cloning --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 5 +- .../libs/tests/GuestJsonRecoveryAllocator.cpp | 475 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 + 4 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 core/libs/tests/GuestJsonRecoveryAllocator.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 34466f8bd5..0aa09ecfca 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_recovery_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonRecoveryAllocator.cpp) + target_include_directories(guest_json_recovery_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_recovery_allocator_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_recovery_allocator_tests) + add_test(NAME guest_json_recovery_allocator COMMAND guest_json_recovery_allocator_tests) add_test_executable(guest_json_parser_metadata_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonParserMetadata.cpp) target_include_directories(guest_json_parser_metadata_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_parser_metadata_tests PRIVATE libSceJson2 libc) @@ -146,7 +151,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index f4874473c3..78283d5657 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1870,7 +1870,8 @@ void AssignObject(Object& destination, const Object& source) { if (!pending.items) return; pending.items->SetOwner(destination.items->Owner()); try { - for (const auto& pair : *source.items) { + for (auto position = source.items->begin(), end = source.items->end(); position != end; ++position) { + const auto& pair = *position; ScopedValue placeholder; SetType(placeholder.value, pair.value.type); ScopedPair copied; @@ -1883,7 +1884,7 @@ void AssignObject(Object& destination, const Object& source) { ObjectLink* node; bool inserted; { - InlineText key; key.Assign(Characters(pair.key)); + InlineText key; key.Assign(position.node->key.characters); TemporaryTreeEntry entry(key.text, copied.pair); node = pending.items->Insert(entry.key.text, entry.pair, inserted); } diff --git a/core/libs/tests/GuestJsonRecoveryAllocator.cpp b/core/libs/tests/GuestJsonRecoveryAllocator.cpp new file mode 100644 index 0000000000..6759a73727 --- /dev/null +++ b/core/libs/tests/GuestJsonRecoveryAllocator.cpp @@ -0,0 +1,475 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +static_assert(sizeof(Value) == 32 && sizeof(Parameter) == 40); +using FloatCallback = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +extern "C" { +Value* APS5_VABI _ZN3sce4Json5Value10referValueERKNS0_6StringE(Value*, const String*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatCallback, void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct RecoveryAllocationTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::map ids; + std::size_t nextId = 0; +}; +static void* APS5_VABI AllocateRecovery(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); trace.ids[memory] = ++trace.nextId; return memory; +} +static void APS5_VABI FreeRecovery(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + Require(trace.ids.count(memory) == 1, "Freed recovery storage must identify a currently live allocation generation"); + trace.ids.erase(memory); + Require(trace.live.erase(memory) == 1, "Recovery storage must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyRecovery(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Recovery notification must identify the failed allocation"); ++trace.notifications; +} +static void* const RecoveryVtable[]{nullptr, nullptr, reinterpret_cast(AllocateRecovery), reinterpret_cast(FreeRecovery), reinterpret_cast(NotifyRecovery)}; +struct RecoveryAllocator { void* const* vtable = RecoveryVtable; }; +struct RecoveryContext { const Value* replacement; bool object; unsigned calls = 0; int function = -1; bool errorPointer = false; }; +static Value* APS5_VABI RecoverAllocated(Value* result, std::int32_t function, double number, const Value* parent, void* context, bool* error) { + auto& trace = *static_cast(context); + ++trace.calls; trace.function = function; trace.errorPointer = error != nullptr; + Require(number == std::numeric_limits::infinity() && !parent, "Recovery must receive the original non-finite value and immediate parent"); + if (trace.object) _ZN3sce4Json5ValueC1ERKS1_(result, trace.replacement); + else _ZN3sce4Json5ValueC1EPKc(result, "recovery"); + if (error) *error = false; + return result; +} +static std::string Contents(Value& value) { + if (value.type == 0) return "null"; + if (value.type == 4) { Require(value.payload == std::bit_cast(std::numeric_limits::infinity()), "Serialization recovery must preserve the source real payload bits"); return "inf"; } + if (value.type == 5) return _ZNK3sce4Json6String5c_strEv(reinterpret_cast(value.payload)); + String output{}; _ZN3sce4Json6StringC1EPKc(&output, ""); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output) == 0, "Partial callback-return objects must remain serializable"); + std::string result = _ZNK3sce4Json6String5c_strEv(&output); + _ZN3sce4Json6StringD1Ev(&output); + return result; +} +struct Expected { + bool object; bool format; bool once; int budget; int type; + const char* contents; const char* output; std::size_t requests; unsigned notifications; std::uint64_t order; int lookupType; +}; +static const Expected Cases[]{ + {0, 0, 0, 0, 0, "null", "p", 1, 1, UINT64_C(0xaf63d54c8601e0a7), -2}, + {0, 0, 0, 1, 0, "null", "p", 3, 2, UINT64_C(0xa9485c18e1e33846), -2}, + {0, 0, 0, 2, 0, "null", "p", 3, 1, UINT64_C(0xa9485c18e1e33846), -2}, + {0, 0, 0, 3, 0, "null", "p", 4, 1, UINT64_C(0x892cdc47d518c7ba), -2}, + {0, 0, 0, 4, 0, "null", "p", 6, 2, UINT64_C(0xaebceda7a971a713), -2}, + {0, 0, 0, 5, 0, "null", "p", 6, 1, UINT64_C(0xaebceda7a971a713), -2}, + {0, 0, 0, 6, 0, "null", "p", 7, 1, UINT64_C(0x5caedee4ec1ed7b1), -2}, + {0, 0, 0, 7, 0, "null", "p", 9, 2, UINT64_C(0x768bd046d9a8a6e0), -2}, + {0, 0, 0, 8, 0, "null", "p", 9, 1, UINT64_C(0x768bd046d9a8a6e0), -2}, + {0, 0, 0, 9, 0, "null", "p", 10, 1, UINT64_C(0x1839e063d993b768), -2}, + {0, 0, 0, 10, 5, "", "p", 11, 1, UINT64_C(0xbe0faaaab60099d3), -2}, + {0, 0, 0, 11, 5, "recovery", "p", 11, 0, UINT64_C(0xbe0faaaab60099d3), -2}, + {0, 0, 1, 0, 0, "null", "p", 1, 1, UINT64_C(0xaf63d54c8601e0a7), -2}, + {0, 0, 1, 1, 5, "", "p", 8, 1, UINT64_C(0xfc01dbf4e232edfa), -2}, + {0, 0, 1, 2, 0, "null", "p", 3, 1, UINT64_C(0xa9485c18e1e33846), -2}, + {0, 0, 1, 3, 0, "null", "p", 4, 1, UINT64_C(0x892cdc47d518c7ba), -2}, + {0, 0, 1, 4, 5, "", "p", 9, 1, UINT64_C(0x768f2646d9ab6ed9), -2}, + {0, 0, 1, 5, 0, "null", "p", 6, 1, UINT64_C(0xaebceda7a971a713), -2}, + {0, 0, 1, 6, 0, "null", "p", 7, 1, UINT64_C(0x5caedee4ec1ed7b1), -2}, + {0, 0, 1, 7, 5, "", "p", 10, 1, UINT64_C(0x1839e063d993b768), -2}, + {0, 0, 1, 8, 0, "null", "p", 9, 1, UINT64_C(0x768bd046d9a8a6e0), -2}, + {0, 0, 1, 9, 0, "null", "p", 10, 1, UINT64_C(0x1839e063d993b768), -2}, + {0, 0, 1, 10, 5, "", "p", 11, 1, UINT64_C(0xbe0faaaab60099d3), -2}, + {0, 0, 1, 11, 5, "recovery", "p", 11, 0, UINT64_C(0xbe0faaaab60099d3), -2}, + {0, 1, 0, 0, 4, "inf", "p", 2, 2, UINT64_C(0x08843007b530e48d), -2}, + {0, 1, 0, 1, 4, "inf", "p", 3, 2, UINT64_C(0xa9823518e2146e48), -2}, + {0, 1, 0, 2, 4, "inf", "p", 5, 3, UINT64_C(0xc691469ccd6898fb), -2}, + {0, 1, 0, 3, 4, "inf", "p", 5, 2, UINT64_C(0xc691469ccd6898fb), -2}, + {0, 1, 0, 4, 4, "inf", "p", 6, 2, UINT64_C(0xd162e67108ad626b), -2}, + {0, 1, 0, 5, 4, "inf", "p", 8, 3, UINT64_C(0xac5d752721a88d90), -2}, + {0, 1, 0, 6, 4, "inf", "p", 8, 2, UINT64_C(0xac5d752721a88d90), -2}, + {0, 1, 0, 7, 4, "inf", "p", 9, 2, UINT64_C(0x8b87da7e318e2776), -2}, + {0, 1, 0, 8, 4, "inf", "p", 11, 3, UINT64_C(0x7c2105433836840e), -2}, + {0, 1, 0, 9, 4, "inf", "p", 19, 10, UINT64_C(0xaef2a03652afb7be), -2}, + {0, 1, 0, 10, 4, "inf", "p", 11, 1, UINT64_C(0x7c20ff43383679dc), -2}, + {0, 1, 0, 11, 4, "inf", "p\"recovery\"", 11, 0, UINT64_C(0x7c20ff43383679dc), -2}, + {0, 1, 1, 0, 4, "inf", "p", 9, 1, UINT64_C(0x8b87d67e318e20aa), -2}, + {0, 1, 1, 1, 4, "inf", "pnull", 4, 1, UINT64_C(0x1cae974828b789ba), -2}, + {0, 1, 1, 2, 4, "inf", "p\"\"", 9, 1, UINT64_C(0x60e103758cc7062a), -2}, + {0, 1, 1, 3, 4, "inf", "pnull", 6, 1, UINT64_C(0xd173e97108bbdab7), -2}, + {0, 1, 1, 4, 4, "inf", "pnull", 7, 1, UINT64_C(0x78700f11be9e5a67), -2}, + {0, 1, 1, 5, 4, "inf", "p\"\"", 10, 1, UINT64_C(0xa609f86e3495cb61), -2}, + {0, 1, 1, 6, 4, "inf", "pnull", 9, 1, UINT64_C(0x8b5b977e31687ab2), -2}, + {0, 1, 1, 7, 4, "inf", "pnull", 10, 1, UINT64_C(0xa5ffa46e348ce820), -2}, + {0, 1, 1, 8, 4, "inf", "p\"\"", 11, 1, UINT64_C(0x7c21074338368774), -2}, + {0, 1, 1, 9, 4, "inf", "precovery\"", 12, 1, UINT64_C(0x22a0063884a26faf), -2}, + {0, 1, 1, 10, 4, "inf", "p", 11, 1, UINT64_C(0x7c20ff43383679dc), -2}, + {0, 1, 1, 11, 4, "inf", "p\"recovery\"", 11, 0, UINT64_C(0x7c20ff43383679dc), -2}, + {1, 0, 0, 0, 0, "null", "p", 1, 1, UINT64_C(0xaf63c54c8601c577), -2}, + {1, 0, 0, 1, 0, "null", "p", 2, 1, UINT64_C(0x084d9807b50252d5), -2}, + {1, 0, 0, 2, 0, "null", "p", 3, 1, UINT64_C(0x1e2c121892f29727), -2}, + {1, 0, 0, 3, 0, "null", "p", 5, 2, UINT64_C(0x7a8da821d2d372b7), -2}, + {1, 0, 0, 4, 0, "null", "p", 6, 2, UINT64_C(0x10e130793c36cb2d), -2}, + {1, 0, 0, 5, 0, "null", "p", 10, 5, UINT64_C(0xb187c4272f9942bf), -2}, + {1, 0, 0, 6, 0, "null", "p", 11, 5, UINT64_C(0x10b56be74c59bf9d), -2}, + {1, 0, 0, 7, 0, "null", "p", 12, 5, UINT64_C(0xe7ef7936b9e1d087), -2}, + {1, 0, 0, 8, 0, "null", "p", 14, 6, UINT64_C(0x2f22e7a49d59a9e5), -2}, + {1, 0, 0, 9, 0, "null", "p", 14, 5, UINT64_C(0x2f22e7a49d59a9e5), -2}, + {1, 0, 0, 10, 0, "null", "p", 16, 6, UINT64_C(0x3eca414f9a84948f), -2}, + {1, 0, 0, 11, 0, "null", "p", 16, 5, UINT64_C(0x3eca414f9a84948f), -2}, + {1, 0, 0, 12, 0, "null", "p", 17, 5, UINT64_C(0xe4f17e1059ea318d), -2}, + {1, 0, 0, 13, 0, "null", "p", 18, 5, UINT64_C(0xb242dac6f307d43f), -2}, + {1, 0, 0, 14, 0, "null", "p", 19, 5, UINT64_C(0x3c673d38f9599c87), -2}, + {1, 0, 0, 15, 0, "null", "p", 19, 4, UINT64_C(0x3c673d38f9599c87), -2}, + {1, 0, 0, 16, 0, "null", "p", 20, 4, UINT64_C(0x28b4cf2275458c17), -2}, + {1, 0, 0, 17, 0, "null", "p", 20, 3, UINT64_C(0x28b4cf2275458c17), -2}, + {1, 0, 0, 18, 0, "null", "p", 21, 3, UINT64_C(0xbd17e68bbdc2b29d), -2}, + {1, 0, 0, 19, 0, "null", "p", 21, 2, UINT64_C(0xbd17e68bbdc2b29d), -2}, + {1, 0, 0, 20, 0, "null", "p", 24, 4, UINT64_C(0x8aa851c4ba2dfe0d), -2}, + {1, 0, 0, 21, 0, "null", "p", 24, 3, UINT64_C(0x8aa851c4ba2dfe0d), -2}, + {1, 0, 0, 22, 0, "null", "p", 25, 3, UINT64_C(0x960824483eb6c5ad), -2}, + {1, 0, 0, 23, 0, "null", "p", 25, 2, UINT64_C(0x960824483eb6c5ad), -2}, + {1, 0, 0, 24, 0, "null", "p", 26, 2, UINT64_C(0xa664ebc29063a4cf), -2}, + {1, 0, 0, 25, 0, "null", "p", 27, 2, UINT64_C(0x211dfc9b594e1afc), -2}, + {1, 0, 0, 26, 0, "null", "p", 27, 1, UINT64_C(0x211dfc9b594e1afc), -2}, + {1, 0, 0, 27, 0, "null", "p", 28, 1, UINT64_C(0x940f17f8bfb7a3d4), -2}, + {1, 0, 0, 28, 0, "null", "p", 29, 1, UINT64_C(0x4d4977adc50b3874), -2}, + {1, 0, 0, 29, 0, "null", "p", 31, 2, UINT64_C(0x96ac3aa3f26673dc), -2}, + {1, 0, 0, 30, 0, "null", "p", 32, 2, UINT64_C(0x6d883894e473365c), -2}, + {1, 0, 0, 31, 0, "null", "p", 36, 5, UINT64_C(0xa689d44f1e8c390a), -2}, + {1, 0, 0, 32, 0, "null", "p", 37, 5, UINT64_C(0xb52aeaa3bc7e29da), -2}, + {1, 0, 0, 33, 0, "null", "p", 38, 5, UINT64_C(0x2bdfebe44514ddfa), -2}, + {1, 0, 0, 34, 0, "null", "p", 40, 6, UINT64_C(0x52ab3c4c65391168), -2}, + {1, 0, 0, 35, 0, "null", "p", 40, 5, UINT64_C(0x52ab3c4c65391168), -2}, + {1, 0, 0, 36, 0, "null", "p", 42, 6, UINT64_C(0xd4a882bc4fea0176), -2}, + {1, 0, 0, 37, 0, "null", "p", 42, 5, UINT64_C(0xd4a882bc4fea0176), -2}, + {1, 0, 0, 38, 0, "null", "p", 43, 5, UINT64_C(0xa7aebec77c7e3806), -2}, + {1, 0, 0, 39, 0, "null", "p", 44, 5, UINT64_C(0x2c2b5af6b31d5a8a), -2}, + {1, 0, 0, 40, 0, "null", "p", 45, 5, UINT64_C(0xe2e498da0c1ac778), -2}, + {1, 0, 0, 41, 0, "null", "p", 45, 4, UINT64_C(0xe2e498da0c1ac778), -2}, + {1, 0, 0, 42, 0, "null", "p", 46, 4, UINT64_C(0x3097224f03ed303a), -2}, + {1, 0, 0, 43, 0, "null", "p", 46, 3, UINT64_C(0x3097224f03ed303a), -2}, + {1, 0, 0, 44, 0, "null", "p", 47, 3, UINT64_C(0xddcbae42c02cbfc6), -2}, + {1, 0, 0, 45, 0, "null", "p", 47, 2, UINT64_C(0xddcbae42c02cbfc6), -2}, + {1, 0, 0, 46, 0, "null", "p", 50, 4, UINT64_C(0xc12186d4776e2450), -2}, + {1, 0, 0, 47, 0, "null", "p", 50, 3, UINT64_C(0xc12186d4776e2450), -2}, + {1, 0, 0, 48, 0, "null", "p", 51, 3, UINT64_C(0x6659e106d2e5e0e6), -2}, + {1, 0, 0, 49, 0, "null", "p", 51, 2, UINT64_C(0x6659e106d2e5e0e6), -2}, + {1, 0, 0, 50, 0, "null", "p", 52, 2, UINT64_C(0xd0d0ac985ccb66fa), -2}, + {1, 0, 0, 51, 0, "null", "p", 53, 2, UINT64_C(0x9dff9ee5ada2d05f), -2}, + {1, 0, 0, 52, 0, "null", "p", 53, 1, UINT64_C(0x9dff9ee5ada2d05f), -2}, + {1, 0, 0, 53, 0, "null", "p", 54, 1, UINT64_C(0x1c2b7f460ba847cd), -2}, + {1, 0, 0, 54, 0, "null", "p", 55, 1, UINT64_C(0x8630e905cef1bd5f), -2}, + {1, 0, 0, 55, 7, "{}", "p", 56, 1, UINT64_C(0xf6d973dea4c4fecd), -1}, + {1, 0, 0, 56, 7, "{}", "p", 57, 1, UINT64_C(0x387e8851fabcb25f), -1}, + {1, 0, 0, 57, 7, "{}", "p", 61, 4, UINT64_C(0x3956b24396957ae5), -1}, + {1, 0, 0, 58, 7, "{}", "p", 62, 4, UINT64_C(0xd13b7fd8c3c27f9f), -1}, + {1, 0, 0, 59, 7, "{}", "p", 63, 4, UINT64_C(0xf2c99a54716bbccd), -1}, + {1, 0, 0, 60, 7, "{}", "p", 65, 5, UINT64_C(0xf2ece4ea49ff28d7), -1}, + {1, 0, 0, 61, 7, "{}", "p", 65, 4, UINT64_C(0xf2ece4ea49ff28d7), -1}, + {1, 0, 0, 62, 7, "{}", "p", 67, 5, UINT64_C(0xa80b0851a72bbbf5), -1}, + {1, 0, 0, 63, 7, "{}", "p", 67, 4, UINT64_C(0xa80b0851a72bbbf5), -1}, + {1, 0, 0, 64, 7, "{}", "p", 68, 4, UINT64_C(0x355849f180ded08f), -1}, + {1, 0, 0, 65, 7, "{}", "p", 69, 4, UINT64_C(0x84428c5dfaf81765), -1}, + {1, 0, 0, 66, 7, "{}", "p", 70, 4, UINT64_C(0x012028e3c8954fdd), -1}, + {1, 0, 0, 67, 7, "{}", "p", 70, 3, UINT64_C(0x012028e3c8954fdd), -1}, + {1, 0, 0, 68, 7, "{}", "p", 71, 3, UINT64_C(0x4b691e0db8a2f8dd), -1}, + {1, 0, 0, 69, 7, "{}", "p", 71, 2, UINT64_C(0x4b691e0db8a2f8dd), -1}, + {1, 0, 0, 70, 7, "{}", "p", 72, 2, UINT64_C(0xc739a650bd76c67f), -1}, + {1, 0, 0, 71, 7, "{}", "p", 72, 1, UINT64_C(0xc739a650bd76c67f), -1}, + {1, 0, 0, 72, 7, "{\"\":null}", "p", 75, 3, UINT64_C(0x9e8b9c317656d9bf), -1}, + {1, 0, 0, 73, 7, "{\"\":null}", "p", 75, 2, UINT64_C(0x9e8b9c317656d9bf), 0}, + {1, 0, 0, 74, 7, "{\"\":null}", "p", 76, 2, UINT64_C(0xbe361b0c15b0d18f), 0}, + {1, 0, 0, 75, 7, "{\"k\":null}", "p", 76, 1, UINT64_C(0xbe361b0c15b0d18f), 0}, + {1, 0, 0, 76, 7, "{\"k\":null}", "p", 77, 1, UINT64_C(0xe6c18c88db742395), 0}, + {1, 0, 0, 77, 7, "{\"k\":\"\"}", "p", 78, 1, UINT64_C(0x8f05688ce6588214), 5}, + {1, 0, 0, 78, 7, "{\"k\":\"recovery\"}", "p", 78, 0, UINT64_C(0x8f05688ce6588214), 5}, + {1, 0, 1, 0, 0, "null", "p", 1, 1, UINT64_C(0xaf63c54c8601c577), -2}, + {1, 0, 1, 1, 0, "null", "p", 2, 1, UINT64_C(0x084d9807b50252d5), -2}, + {1, 0, 1, 2, 0, "null", "p", 3, 1, UINT64_C(0x1e2c121892f29727), -2}, + {1, 0, 1, 3, 7, "{}", "p", 14, 1, UINT64_C(0xd76d5b07238706ad), -1}, + {1, 0, 1, 4, 7, "{}", "p", 15, 1, UINT64_C(0xccfbd0932ed1c61f), -1}, + {1, 0, 1, 5, 7, "{\"k\":\"recovery\"}", "p", 73, 1, UINT64_C(0x15fb0ebd0a33b7ca), 5}, + {1, 0, 1, 6, 7, "{\"k\":\"recovery\"}", "p", 74, 1, UINT64_C(0x68d020c916f67c54), 5}, + {1, 0, 1, 7, 7, "{\"\":\"recovery\"}", "p", 63, 1, UINT64_C(0x2d23dde27a017aa6), 5}, + {1, 0, 1, 8, 7, "{\"\":\"recovery\"}", "p", 67, 1, UINT64_C(0xe8726979e3018f08), 5}, + {1, 0, 1, 9, 7, "{\"\":\"recovery\"}", "p", 65, 1, UINT64_C(0xc4d03eb0173d3988), 5}, + {1, 0, 1, 10, 7, "{\"\":\"recovery\"}", "p", 68, 1, UINT64_C(0x59a5ec0540fd0250), 5}, + {1, 0, 1, 11, 7, "{\"k\":\"recovery\"}", "p", 75, 1, UINT64_C(0xc4135e36c1583f72), 5}, + {1, 0, 1, 12, 7, "{\"k\":\"recovery\"}", "p", 76, 1, UINT64_C(0x8ede4064f5a34ddc), 5}, + {1, 0, 1, 13, 7, "{\"k\":\"\"}", "p", 67, 1, UINT64_C(0x8c6b9268a2c5cfcd), -1}, + {1, 0, 1, 14, 7, "{\"k\":\"\"}", "p", 68, 1, UINT64_C(0x9f6328349f5efe35), -1}, + {1, 0, 1, 15, 7, "{\"\":\"\"}", "p", 64, 1, UINT64_C(0xddef821334a86667), 5}, + {1, 0, 1, 16, 7, "{\"\":\"\"}", "p", 66, 1, UINT64_C(0x9eaa88f7ed6fe65d), 5}, + {1, 0, 1, 17, 7, "{\"k\":null}", "p", 56, 1, UINT64_C(0x8250699960f4e85f), 0}, + {1, 0, 1, 18, 7, "{\"k\":null}", "p", 57, 1, UINT64_C(0x9b6c3d4dacd7a045), 0}, + {1, 0, 1, 19, 7, "{}", "p", 30, 1, UINT64_C(0x81db53d0e75644ef), -1}, + {1, 0, 1, 20, 7, "{\"k\":\"\"}", "p", 69, 1, UINT64_C(0xd73452dd7647589d), -1}, + {1, 0, 1, 21, 7, "{\"\":\"\"}", "p", 66, 1, UINT64_C(0x9eaa88f7ed6fe65d), 5}, + {1, 0, 1, 22, 7, "{\"\":\"\"}", "p", 67, 1, UINT64_C(0x96214c6d9d4d7ffd), 5}, + {1, 0, 1, 23, 7, "{\"k\":null}", "p", 58, 1, UINT64_C(0x5334f2ccc09e7717), 0}, + {1, 0, 1, 24, 7, "{\"k\":null}", "p", 59, 1, UINT64_C(0x82d1353a8c895ebd), 0}, + {1, 0, 1, 25, 7, "{\"k\":\"\"}", "p", 76, 1, UINT64_C(0x1be6a6f37e090450), 5}, + {1, 0, 1, 26, 0, "null", "p", 27, 1, UINT64_C(0x211dfc9b594e1afc), -2}, + {1, 0, 1, 27, 0, "null", "p", 28, 1, UINT64_C(0x940f17f8bfb7a3d4), -2}, + {1, 0, 1, 28, 0, "null", "p", 29, 1, UINT64_C(0x4d4977adc50b3874), -2}, + {1, 0, 1, 29, 7, "{}", "p", 35, 1, UINT64_C(0x961e76ecf8fd871c), -1}, + {1, 0, 1, 30, 7, "{}", "p", 36, 1, UINT64_C(0xef7cb822b58db19c), -1}, + {1, 0, 1, 31, 7, "{\"k\":\"recovery\"}", "p", 73, 1, UINT64_C(0xdbd99fe5ed4883e4), 5}, + {1, 0, 1, 32, 7, "{\"k\":\"recovery\"}", "p", 74, 1, UINT64_C(0x6a386f444a5e4254), 5}, + {1, 0, 1, 33, 7, "{\"\":\"recovery\"}", "p", 67, 1, UINT64_C(0x7e21bf21deb3808c), 5}, + {1, 0, 1, 34, 7, "{\"\":\"recovery\"}", "p", 71, 1, UINT64_C(0x982621b7c5f73376), 5}, + {1, 0, 1, 35, 7, "{\"\":\"recovery\"}", "p", 69, 1, UINT64_C(0x77d0ac84582d5f7a), 5}, + {1, 0, 1, 36, 7, "{\"\":\"recovery\"}", "p", 72, 1, UINT64_C(0xee3d5bf84f6d2a40), 5}, + {1, 0, 1, 37, 7, "{\"k\":\"recovery\"}", "p", 75, 1, UINT64_C(0xc5d9c4afc4709dd4), 5}, + {1, 0, 1, 38, 7, "{\"k\":\"recovery\"}", "p", 76, 1, UINT64_C(0x8cfe23e5d4680904), 5}, + {1, 0, 1, 39, 7, "{\"k\":\"\"}", "p", 71, 1, UINT64_C(0xe7ff3f815d66ff2e), -1}, + {1, 0, 1, 40, 7, "{\"k\":\"\"}", "p", 72, 1, UINT64_C(0x3296cb16b10379e8), -1}, + {1, 0, 1, 41, 7, "{\"\":\"\"}", "p", 69, 1, UINT64_C(0xeb080365bc3f30bc), 5}, + {1, 0, 1, 42, 7, "{\"\":\"\"}", "p", 71, 1, UINT64_C(0xdf67a4e26b3d2a1e), 5}, + {1, 0, 1, 43, 7, "{\"k\":null}", "p", 65, 1, UINT64_C(0xa0086042559c8e1c), 0}, + {1, 0, 1, 44, 7, "{\"k\":null}", "p", 66, 1, UINT64_C(0x51cdd5df396e0ba0), 0}, + {1, 0, 1, 45, 7, "{}", "p", 51, 1, UINT64_C(0x383161da3f804726), -1}, + {1, 0, 1, 46, 7, "{\"k\":\"\"}", "p", 73, 1, UINT64_C(0x7eb80b99470a2eba), -1}, + {1, 0, 1, 47, 7, "{\"\":\"\"}", "p", 71, 1, UINT64_C(0xdf67a4e26b3d2a1e), 5}, + {1, 0, 1, 48, 7, "{\"\":\"\"}", "p", 72, 1, UINT64_C(0x8f4766e11afb39d4), 5}, + {1, 0, 1, 49, 7, "{\"k\":null}", "p", 67, 1, UINT64_C(0x279a838f9bdfaefc), 0}, + {1, 0, 1, 50, 7, "{\"k\":null}", "p", 68, 1, UINT64_C(0x5a342f85a466b390), 0}, + {1, 0, 1, 51, 7, "{\"k\":\"\"}", "p", 77, 1, UINT64_C(0xe6c18c88db742395), 5}, + {1, 0, 1, 52, 0, "null", "p", 53, 1, UINT64_C(0x9dff9ee5ada2d05f), -2}, + {1, 0, 1, 53, 0, "null", "p", 54, 1, UINT64_C(0x1c2b7f460ba847cd), -2}, + {1, 0, 1, 54, 0, "null", "p", 55, 1, UINT64_C(0x8630e905cef1bd5f), -2}, + {1, 0, 1, 55, 7, "{}", "p", 56, 1, UINT64_C(0xf6d973dea4c4fecd), -1}, + {1, 0, 1, 56, 7, "{}", "p", 57, 1, UINT64_C(0x387e8851fabcb25f), -1}, + {1, 0, 1, 57, 7, "{\"k\":\"recovery\"}", "p", 73, 1, UINT64_C(0x95a65de0d1459ffa), 5}, + {1, 0, 1, 58, 7, "{\"k\":\"recovery\"}", "p", 74, 1, UINT64_C(0xa620195a9833c2b4), 5}, + {1, 0, 1, 59, 7, "{\"\":\"recovery\"}", "p", 71, 1, UINT64_C(0x454d188779b08d02), 5}, + {1, 0, 1, 60, 7, "{\"\":\"recovery\"}", "p", 75, 1, UINT64_C(0x22580a8396a3a0ac), 5}, + {1, 0, 1, 61, 7, "{\"\":\"recovery\"}", "p", 73, 1, UINT64_C(0x3acc0530cfa4dcec), 5}, + {1, 0, 1, 62, 7, "{\"\":\"recovery\"}", "p", 76, 1, UINT64_C(0xb9a8b3368d755ba4), 5}, + {1, 0, 1, 63, 7, "{\"k\":\"recovery\"}", "p", 75, 1, UINT64_C(0x91af3c44d0365042), 5}, + {1, 0, 1, 64, 7, "{\"k\":\"recovery\"}", "p", 76, 1, UINT64_C(0x41b64226cd9d271c), 5}, + {1, 0, 1, 65, 7, "{\"k\":\"\"}", "p", 75, 1, UINT64_C(0xbbddd48e5d45fa45), -1}, + {1, 0, 1, 66, 7, "{\"k\":\"\"}", "p", 76, 1, UINT64_C(0x402f245441340d5d), -1}, + {1, 0, 1, 67, 7, "{\"\":\"\"}", "p", 74, 1, UINT64_C(0x66aaf18c794fd2ff), 5}, + {1, 0, 1, 68, 7, "{\"\":\"\"}", "p", 76, 1, UINT64_C(0xbe140f0c1593d6c5), 5}, + {1, 0, 1, 69, 7, "{\"k\":null}", "p", 74, 1, UINT64_C(0x91322a4e151994d7), 0}, + {1, 0, 1, 70, 7, "{\"k\":null}", "p", 75, 1, UINT64_C(0x9e8ba6317656eabd), 0}, + {1, 0, 1, 71, 7, "{}", "p", 72, 1, UINT64_C(0xc739a650bd76c67f), -1}, + {1, 0, 1, 72, 7, "{\"k\":\"\"}", "p", 77, 1, UINT64_C(0xe6c18c88db742395), -1}, + {1, 0, 1, 73, 7, "{\"\":\"\"}", "p", 76, 1, UINT64_C(0xbe140f0c1593d6c5), 5}, + {1, 0, 1, 74, 7, "{\"\":\"\"}", "p", 77, 1, UINT64_C(0xe6c18c88db742395), 5}, + {1, 0, 1, 75, 7, "{\"k\":null}", "p", 76, 1, UINT64_C(0xbe361b0c15b0d18f), 0}, + {1, 0, 1, 76, 7, "{\"k\":null}", "p", 77, 1, UINT64_C(0xe6c18c88db742395), 0}, + {1, 0, 1, 77, 7, "{\"k\":\"\"}", "p", 78, 1, UINT64_C(0x8f05688ce6588214), 5}, + {1, 0, 1, 78, 7, "{\"k\":\"recovery\"}", "p", 78, 0, UINT64_C(0x8f05688ce6588214), 5}, + {1, 1, 0, 0, 4, "inf", "p", 2, 2, UINT64_C(0x08842007b530c95d), -2}, + {1, 1, 0, 1, 4, "inf", "p", 3, 2, UINT64_C(0xa94bb518e1e60558), -2}, + {1, 1, 0, 2, 4, "inf", "p", 4, 2, UINT64_C(0x925e3a47da742eb6), -2}, + {1, 1, 0, 3, 4, "inf", "p", 5, 2, UINT64_C(0x28fb3718324d62b8), -2}, + {1, 1, 0, 4, 4, "inf", "p", 7, 3, UINT64_C(0x5e96ce153c7cfbb8), -2}, + {1, 1, 0, 5, 4, "inf", "p", 8, 3, UINT64_C(0x76f63f179f84d58e), -2}, + {1, 1, 0, 6, 4, "inf", "p", 12, 6, UINT64_C(0xa616d1c6ca4b37b8), -2}, + {1, 1, 0, 7, 4, "inf", "p", 13, 6, UINT64_C(0x7b61fcc43cb7c366), -2}, + {1, 1, 0, 8, 4, "inf", "p", 14, 6, UINT64_C(0x55e9654cf5f40670), -2}, + {1, 1, 0, 9, 4, "inf", "p", 16, 7, UINT64_C(0x03166de64beed216), -2}, + {1, 1, 0, 10, 4, "inf", "p", 16, 6, UINT64_C(0x03166de64beed216), -2}, + {1, 1, 0, 11, 4, "inf", "p", 18, 7, UINT64_C(0x61960677c5c6ef08), -2}, + {1, 1, 0, 12, 4, "inf", "p", 18, 6, UINT64_C(0x61960677c5c6ef08), -2}, + {1, 1, 0, 13, 4, "inf", "p", 19, 6, UINT64_C(0x7fb54dffd01f18d6), -2}, + {1, 1, 0, 14, 4, "inf", "p", 20, 6, UINT64_C(0x3351fbcf95f4f238), -2}, + {1, 1, 0, 15, 4, "inf", "p", 21, 6, UINT64_C(0x1d49b0fdcdfd5028), -2}, + {1, 1, 0, 16, 4, "inf", "p", 21, 5, UINT64_C(0x1d49b0fdcdfd5028), -2}, + {1, 1, 0, 17, 4, "inf", "p", 22, 5, UINT64_C(0x95f8c66497bb53a0), -2}, + {1, 1, 0, 18, 4, "inf", "p", 22, 4, UINT64_C(0x95f8c66497bb53a0), -2}, + {1, 1, 0, 19, 4, "inf", "p", 23, 4, UINT64_C(0xd152956a474667e6), -2}, + {1, 1, 0, 20, 4, "inf", "p", 23, 3, UINT64_C(0xd152956a474667e6), -2}, + {1, 1, 0, 21, 4, "inf", "p", 26, 5, UINT64_C(0xf155ca552e66308e), -2}, + {1, 1, 0, 22, 4, "inf", "p", 26, 4, UINT64_C(0xf155ca552e66308e), -2}, + {1, 1, 0, 23, 4, "inf", "p", 27, 4, UINT64_C(0x3b227d6a79828cd6), -2}, + {1, 1, 0, 24, 4, "inf", "p", 27, 3, UINT64_C(0x3b227d6a79828cd6), -2}, + {1, 1, 0, 25, 4, "inf", "p", 28, 3, UINT64_C(0x73861aec2a4cce28), -2}, + {1, 1, 0, 26, 4, "inf", "p", 29, 3, UINT64_C(0x90c3c74bdb6352f9), -2}, + {1, 1, 0, 27, 4, "inf", "p", 29, 2, UINT64_C(0x90c3c74bdb6352f9), -2}, + {1, 1, 0, 28, 4, "inf", "p", 30, 2, UINT64_C(0x5fd285e5c9a097b5), -2}, + {1, 1, 0, 29, 4, "inf", "p", 31, 2, UINT64_C(0x71deb9759aa6efd1), -2}, + {1, 1, 0, 30, 4, "inf", "p", 35, 5, UINT64_C(0xcc66351878f7eaae), -2}, + {1, 1, 0, 31, 4, "inf", "p", 36, 5, UINT64_C(0x938d793a6f66fdfa), -2}, + {1, 1, 0, 32, 4, "inf", "p", 40, 8, UINT64_C(0x9ac679090babd038), -2}, + {1, 1, 0, 33, 4, "inf", "p", 41, 8, UINT64_C(0xb441ea5b2ee4ef6c), -2}, + {1, 1, 0, 34, 4, "inf", "p", 42, 8, UINT64_C(0xe25b516d74e1ce28), -2}, + {1, 1, 0, 35, 4, "inf", "p", 44, 9, UINT64_C(0x2adccef255d56a56), -2}, + {1, 1, 0, 36, 4, "inf", "p", 44, 8, UINT64_C(0x2adccef255d56a56), -2}, + {1, 1, 0, 37, 4, "inf", "p", 46, 9, UINT64_C(0x661c8ee64bff6094), -2}, + {1, 1, 0, 38, 4, "inf", "p", 46, 8, UINT64_C(0x661c8ee64bff6094), -2}, + {1, 1, 0, 39, 4, "inf", "p", 47, 8, UINT64_C(0xb77b424f5a5c3448), -2}, + {1, 1, 0, 40, 4, "inf", "p", 48, 8, UINT64_C(0xfb8a4df5be8fafe8), -2}, + {1, 1, 0, 41, 4, "inf", "p", 49, 8, UINT64_C(0xb303414a8c5a6cba), -2}, + {1, 1, 0, 42, 4, "inf", "p", 49, 7, UINT64_C(0xb303414a8c5a6cba), -2}, + {1, 1, 0, 43, 4, "inf", "p", 50, 7, UINT64_C(0x0c8984c4638db2a8), -2}, + {1, 1, 0, 44, 4, "inf", "p", 50, 6, UINT64_C(0x0c8984c4638db2a8), -2}, + {1, 1, 0, 45, 4, "inf", "p", 51, 6, UINT64_C(0xc499abf4c8d2d6f8), -2}, + {1, 1, 0, 46, 4, "inf", "p", 51, 5, UINT64_C(0xc499abf4c8d2d6f8), -2}, + {1, 1, 0, 47, 4, "inf", "p", 54, 7, UINT64_C(0xee5659e8ca9e200e), -2}, + {1, 1, 0, 48, 4, "inf", "p", 54, 6, UINT64_C(0xee5659e8ca9e200e), -2}, + {1, 1, 0, 49, 4, "inf", "p", 55, 6, UINT64_C(0x313624099079c388), -2}, + {1, 1, 0, 50, 4, "inf", "p", 55, 5, UINT64_C(0x313624099079c388), -2}, + {1, 1, 0, 51, 4, "inf", "p", 56, 5, UINT64_C(0x4eb737700cb0e2b8), -2}, + {1, 1, 0, 52, 4, "inf", "p", 57, 5, UINT64_C(0xf578edebf5d33e57), -2}, + {1, 1, 0, 53, 4, "inf", "p", 57, 4, UINT64_C(0xf578edebf5d33e57), -2}, + {1, 1, 0, 54, 4, "inf", "p", 58, 4, UINT64_C(0x106c88119e6508ba), -2}, + {1, 1, 0, 55, 4, "inf", "p{", 58, 3, UINT64_C(0x106c88119e6508ba), -2}, + {1, 1, 0, 56, 4, "inf", "p{", 58, 2, UINT64_C(0x106c88119e6508ba), -2}, + {1, 1, 0, 57, 4, "inf", "p{", 74, 17, UINT64_C(0xb580ca8d7702ee9e), -2}, + {1, 1, 0, 58, 4, "inf", "p{", 74, 16, UINT64_C(0xb580ca8d7702ee9e), -2}, + {1, 1, 0, 59, 4, "inf", "p{\"k\":\"recovery\"", 61, 2, UINT64_C(0x303529626abee9ac), -2}, + {1, 1, 0, 60, 4, "inf", "p{\"k\":\"recovery\"", 61, 1, UINT64_C(0x303529626abee9ac), -2}, + {1, 1, 0, 61, 4, "inf", "p{\"k\":\"recovery\"}", 61, 0, UINT64_C(0x303529626abee9ac), -2}, + {1, 1, 1, 0, 4, "inf", "p", 57, 1, UINT64_C(0x357a103f541adf8f), -2}, + {1, 1, 1, 1, 4, "inf", "pnull", 4, 1, UINT64_C(0x91aa0747d9db038a), -2}, + {1, 1, 1, 2, 4, "inf", "pnull", 5, 1, UINT64_C(0x2a4ba818336b39e0), -2}, + {1, 1, 1, 3, 4, "inf", "pnull", 6, 1, UINT64_C(0xf0414c1d797eadaa), -2}, + {1, 1, 1, 4, 4, "inf", "p{}", 15, 1, UINT64_C(0xb8a11f4b85cfdf68), -2}, + {1, 1, 1, 5, 4, "inf", "p{}", 16, 1, UINT64_C(0x515bb1e0a9377c8a), -2}, + {1, 1, 1, 6, 4, "inf", "p{\"k\":\"recovery\"}", 56, 1, UINT64_C(0x315f2eac37d67b5a), -2}, + {1, 1, 1, 7, 4, "inf", "p{\"k\":\"recovery\"}", 57, 1, UINT64_C(0xcbfa0862d979100c), -2}, + {1, 1, 1, 8, 4, "inf", "p{\"\":\"recovery\"}", 50, 1, UINT64_C(0xd905de7dd0c987e6), -2}, + {1, 1, 1, 9, 4, "inf", "p{\"\":\"recovery\"}", 54, 1, UINT64_C(0x88b0fc6aef4df998), -2}, + {1, 1, 1, 10, 4, "inf", "p{\"\":\"recovery\"}", 52, 1, UINT64_C(0x0ed068e5aec81d18), -2}, + {1, 1, 1, 11, 4, "inf", "p{\"\":\"recovery\"}", 55, 1, UINT64_C(0x9ae649625b45ca98), -2}, + {1, 1, 1, 12, 4, "inf", "p{\"k\":\"recovery\"}", 58, 1, UINT64_C(0x0625a9a88129bed2), -2}, + {1, 1, 1, 13, 4, "inf", "p{\"k\":\"recovery\"}", 59, 1, UINT64_C(0x303de3c047695c84), -2}, + {1, 1, 1, 14, 4, "inf", "p{\"k\":\"\"}", 54, 1, UINT64_C(0xb8015f1c36338c04), -2}, + {1, 1, 1, 15, 4, "inf", "p{\"k\":\"\"}", 55, 1, UINT64_C(0xdb7517e493e6a9b4), -2}, + {1, 1, 1, 16, 4, "inf", "p{\"\":\"\"}", 52, 1, UINT64_C(0x7da62e92d17ad572), -2}, + {1, 1, 1, 17, 4, "inf", "p{\"\":\"\"}", 54, 1, UINT64_C(0x8d200b5a3b7e1874), -2}, + {1, 1, 1, 18, 4, "inf", "p{\"k\":null}", 48, 1, UINT64_C(0x573ea7945c700032), -2}, + {1, 1, 1, 19, 4, "inf", "p{\"k\":null}", 49, 1, UINT64_C(0x27b1db6071fdac8c), -2}, + {1, 1, 1, 20, 4, "inf", "p{}", 31, 1, UINT64_C(0xce687ad15e20ed42), -2}, + {1, 1, 1, 21, 4, "inf", "p{\"k\":\"\"}", 56, 1, UINT64_C(0x3dd0ee7536859154), -2}, + {1, 1, 1, 22, 4, "inf", "p{\"\":\"\"}", 54, 1, UINT64_C(0x8d200b5a3b7e1874), -2}, + {1, 1, 1, 23, 4, "inf", "p{\"\":\"\"}", 55, 1, UINT64_C(0x39985a010413873c), -2}, + {1, 1, 1, 24, 4, "inf", "p{\"k\":null}", 50, 1, UINT64_C(0xe2169cee4d02037a), -2}, + {1, 1, 1, 25, 4, "inf", "p{\"k\":null}", 51, 1, UINT64_C(0x6668922033679bd4), -2}, + {1, 1, 1, 26, 4, "inf", "p{\"k\":\"\"}", 60, 1, UINT64_C(0x7a7b3b316939da83), -2}, + {1, 1, 1, 27, 4, "inf", "pnull", 30, 1, UINT64_C(0x5ffe94e5c9c5ec1d), -2}, + {1, 1, 1, 28, 4, "inf", "pnull", 31, 1, UINT64_C(0x735128759be1a9f9), -2}, + {1, 1, 1, 29, 4, "inf", "pnull", 32, 1, UINT64_C(0x2464e9d5c9a96f25), -2}, + {1, 1, 1, 30, 4, "inf", "p{}", 36, 1, UINT64_C(0x2959406bddec2341), -2}, + {1, 1, 1, 31, 4, "inf", "p{}", 37, 1, UINT64_C(0xff7ab32c63614925), -2}, + {1, 1, 1, 32, 4, "inf", "p{\"k\":\"recovery\"}", 56, 1, UINT64_C(0x1d97c31770f583e8), -2}, + {1, 1, 1, 33, 4, "inf", "p{\"k\":\"recovery\"}", 57, 1, UINT64_C(0xf80d82cd3044afcc), -2}, + {1, 1, 1, 34, 4, "inf", "p{\"\":\"recovery\"}", 54, 1, UINT64_C(0xda80aa339eb5c7f8), -2}, + {1, 1, 1, 35, 4, "inf", "p{\"\":\"recovery\"}", 58, 1, UINT64_C(0x625b8d667a7a4bda), -2}, + {1, 1, 1, 36, 4, "inf", "p{\"\":\"recovery\"}", 56, 1, UINT64_C(0xd1b802c73f7ec5a6), -2}, + {1, 1, 1, 37, 4, "inf", "p{\"\":\"recovery\"}", 59, 1, UINT64_C(0x4d6683f6c036dbd8), -2}, + {1, 1, 1, 38, 4, "inf", "p{\"k\":\"recovery\"}", 58, 1, UINT64_C(0xb64549e8f90110c8), -2}, + {1, 1, 1, 39, 4, "inf", "p{\"k\":\"recovery\"}", 59, 1, UINT64_C(0x5e11fc32d5a26abc), -2}, + {1, 1, 1, 40, 4, "inf", "p{\"k\":\"\"}", 58, 1, UINT64_C(0x9c11d723b8ba768f), -2}, + {1, 1, 1, 41, 4, "inf", "p{\"k\":\"\"}", 59, 1, UINT64_C(0xa9a483305377d8b9), -2}, + {1, 1, 1, 42, 4, "inf", "p{\"\":\"\"}", 57, 1, UINT64_C(0x58e686a85e22d0bf), -2}, + {1, 1, 1, 43, 4, "inf", "p{\"\":\"\"}", 59, 1, UINT64_C(0xd8a3bfa99189d665), -2}, + {1, 1, 1, 44, 4, "inf", "p{\"k\":null}", 57, 1, UINT64_C(0xb7cd827aae5b9de3), -2}, + {1, 1, 1, 45, 4, "inf", "p{\"k\":null}", 58, 1, UINT64_C(0xc901aa4562715073), -2}, + {1, 1, 1, 46, 4, "inf", "p{}", 52, 1, UINT64_C(0xb9b076a5e7fcb303), -2}, + {1, 1, 1, 47, 4, "inf", "p{\"k\":\"\"}", 60, 1, UINT64_C(0x7a7b3b316939da83), -2}, + {1, 1, 1, 48, 4, "inf", "p{\"\":\"\"}", 59, 1, UINT64_C(0xd8a3bfa99189d665), -2}, + {1, 1, 1, 49, 4, "inf", "p{\"\":\"\"}", 60, 1, UINT64_C(0x7a7b3b316939da83), -2}, + {1, 1, 1, 50, 4, "inf", "p{\"k\":null}", 59, 1, UINT64_C(0x8242d1842bb69e53), -2}, + {1, 1, 1, 51, 4, "inf", "p{\"k\":null}", 60, 1, UINT64_C(0x7a7b3b316939da83), -2}, + {1, 1, 1, 52, 4, "inf", "p{\"k\":\"\"}", 61, 1, UINT64_C(0x303529626abee9ac), -2}, + {1, 1, 1, 53, 4, "inf", "p\"k\":\"recovery\"}", 62, 1, UINT64_C(0x621da11fedc3cd77), -2}, + {1, 1, 1, 54, 4, "inf", "p\"k\":\"recovery\"}", 63, 1, UINT64_C(0x54123ca5b4a95d58), -2}, + {1, 1, 1, 55, 4, "inf", "p{}", 58, 1, UINT64_C(0x106c88119e6508ba), -2}, + {1, 1, 1, 56, 4, "inf", "p{}", 58, 1, UINT64_C(0x106c88119e6508ba), -2}, + {1, 1, 1, 57, 4, "inf", "p{\"k\":\"recovery\"}", 61, 1, UINT64_C(0x303529626abee9ac), -2}, + {1, 1, 1, 58, 4, "inf", "p{k\":\"recovery\"}", 62, 1, UINT64_C(0x52c21e3b3173e466), -2}, + {1, 1, 1, 59, 4, "inf", "p{\"k\":\"recovery\"}", 61, 1, UINT64_C(0x303529626abee9ac), -2}, + {1, 1, 1, 60, 4, "inf", "p{\"k\":\"recovery\"", 61, 1, UINT64_C(0x303529626abee9ac), -2}, + {1, 1, 1, 61, 4, "inf", "p{\"k\":\"recovery\"}", 61, 0, UINT64_C(0x303529626abee9ac), -2}, +}; +int main() { + RecoveryAllocationTrace trace; RecoveryAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize callback-return allocation contracts"); + Value replacement{}; _ZN3sce4Json5ValueC1Ev(&replacement); + const char document[] = "{\"k\":\"recovery\"}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&replacement, document, sizeof(document) - 1) == 0 && Contents(replacement) == document, "Construct the independently owned callback-return template"); + String lookupKey{}; _ZN3sce4Json6StringC1EPKc(&lookupKey, "k"); + const auto baseline = trace.live; + const auto baselineIds = trace.ids; + for (const auto& expected : Cases) { + Value value{}; _ZN3sce4Json5ValueC1Ed(&value, expected.format ? std::numeric_limits::infinity() : 0.125); + RecoveryContext callback{&replacement, expected.object}; + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, RecoverAllocated, &callback) == 0, "Register allocation-backed recovery"); + void* auxiliary = value.auxiliary; + void* fallback = static_cast(auxiliary)[2]; + void* record = static_cast(auxiliary)[4]; + const auto auxiliaryId = trace.ids.at(auxiliary), fallbackId = trace.ids.at(fallback), recordId = trace.ids.at(record); + String output{}; _ZN3sce4Json6StringC1EPKc(&output, "p"); + const auto requests = trace.requests.size(); const auto notifications = trace.notifications; + trace.budget = expected.budget; trace.once = expected.once; + int status = 0; + if (expected.format) status = _ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output); + else _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + trace.budget = -1; trace.once = false; + const auto end = trace.requests.size(); + const auto errors = trace.notifications - notifications; + Require(value.type == expected.type && Contents(value) == expected.contents && std::strcmp(_ZNK3sce4Json6String5c_strEv(&output), expected.output) == 0, "Callback-return allocation failure must preserve the reference's exact partial value and output"); + if (expected.object) { + const auto* found = value.type == 7 ? _ZN3sce4Json5Value10referValueERKNS0_6StringE(&value, &lookupKey) : nullptr; + const int lookupType = value.type == 7 ? (found ? found->type : -1) : -2; + Require(lookupType == expected.lookupType, "Callback-return cloning must preserve internal tree-key lookup independently of the public key text"); + } + std::uint64_t order = UINT64_C(14695981039346656037); + for (auto i = requests; i < end; ++i) { order ^= trace.requests[i]; order *= UINT64_C(1099511628211); } + if (end - requests != expected.requests || order != expected.order || errors != expected.notifications) std::fprintf(stderr, "Recovery trace differs: object=%d format=%d once=%d budget=%d requests=%zu/%zu notices=%u/%u\n", expected.object, expected.format, expected.once, expected.budget, end - requests, expected.requests, errors, expected.notifications); + Require(end - requests == expected.requests && order == expected.order && errors == expected.notifications, "Callback construction, cloning and final assignment must preserve exact request order and failure notifications"); + if (expected.format) Require(status == 0, "Serialization must preserve the reference callback-recovery status"); + Require(callback.calls == 1 && callback.function == (expected.format ? 0 : 1) && callback.errorPointer == expected.format, "Recovery dispatch must retain callback function, invocation count, error-pointer shape and observed serialization status"); + Require(!value.parent && value.auxiliary == auxiliary && static_cast(auxiliary)[2] == fallback && static_cast(auxiliary)[4] == record, "Partial callback cloning must preserve destination callback metadata and fallback identity"); + Require(trace.ids.count(auxiliary) && trace.ids.count(fallback) && trace.ids.count(record), "Destination recovery metadata must retain live allocator ownership"); + Require(trace.ids.at(auxiliary) == auxiliaryId && trace.ids.at(fallback) == fallbackId && trace.ids.at(record) == recordId && static_cast(record)[0] == reinterpret_cast(&RecoverAllocated) && static_cast(record)[1] == &callback && static_cast(fallback)->parent == &value, "Recovery metadata must retain its allocation generation, handler, context and fallback owner"); + Require(Contents(replacement) == document, "Failure recovery must preserve the independently owned source template"); + _ZN3sce4Json6StringD1Ev(&output); _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live == baseline && trace.ids == baselineIds, "Every callback-return failure state must reclaim temporary ownership and remain destructible"); + } + _ZN3sce4Json6StringD1Ev(&lookupKey); + _ZN3sce4Json5ValueD1Ev(&replacement); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty() && trace.ids.empty(), "Callback-return allocation checks must release all allocator-owned resources"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 37d87e833e..9d0011997a 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -65,3 +65,5 @@ - [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - nested iterator/metadata failures, non-finite recovery failures, lifecycle reentry, other legacy callback combinations and floating-point environment behavior still require audit. Null callback/output references are explicitly unsupported. All serialization shares the collecting/traversal protocol. Reference tests cover modern chunk boundaries, callback results, output mutation, scalar allocation failures and opening-callback array mutations. SDK 5 tests cover root admission/reset, child metadata release, numeric edge cases and persistent/one-shot root admission failures. - [Parser parent and auxiliary transitions](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-driven reentry, more complex nested metadata failures and alternate SDK combinations still require audit. Reference tests cover direct parsing into an owned child across eight value forms, six root-admission failure/retry cases, incomplete-literal preservation, modern root callback retention and nested release of three handler records, fallback and auxiliary in native order. Allocation generations distinguish release from later address reuse. + +- [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - nested array returns, more complex object shapes, inherited failure scopes and alternate SDK combinations remain unverified. Reference tests cover 330 persistent/one-shot cases through successful string and one-entry object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. From 38b15758fdd1f374f73574a77e90f9f7ff15fdc0 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:14:25 +0200 Subject: [PATCH 046/318] test(libSceJson2): verify nested callback recovery allocation failures --- .../libs/tests/GuestJsonRecoveryAllocator.cpp | 514 +++++++++++++++++- docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 512 insertions(+), 4 deletions(-) diff --git a/core/libs/tests/GuestJsonRecoveryAllocator.cpp b/core/libs/tests/GuestJsonRecoveryAllocator.cpp index 6759a73727..695e38e82c 100644 --- a/core/libs/tests/GuestJsonRecoveryAllocator.cpp +++ b/core/libs/tests/GuestJsonRecoveryAllocator.cpp @@ -87,7 +87,7 @@ static std::string Contents(Value& value) { return result; } struct Expected { - bool object; bool format; bool once; int budget; int type; + std::uint8_t object; bool format; bool once; int budget; int type; const char* contents; const char* output; std::size_t requests; unsigned notifications; std::uint64_t order; int lookupType; }; static const Expected Cases[]{ @@ -421,6 +421,510 @@ static const Expected Cases[]{ {1, 1, 1, 59, 4, "inf", "p{\"k\":\"recovery\"}", 61, 1, UINT64_C(0x303529626abee9ac), -2}, {1, 1, 1, 60, 4, "inf", "p{\"k\":\"recovery\"", 61, 1, UINT64_C(0x303529626abee9ac), -2}, {1, 1, 1, 61, 4, "inf", "p{\"k\":\"recovery\"}", 61, 0, UINT64_C(0x303529626abee9ac), -2}, + {2, 0, 0, 0, 0, "null", "p", 1, 1, UINT64_C(0xaf63c54c8601c577), -2}, + {2, 0, 0, 1, 0, "null", "p", 2, 1, UINT64_C(0x084d9807b50252d5), -2}, + {2, 0, 0, 2, 0, "null", "p", 5, 3, UINT64_C(0xd5e2502297cbec7f), -2}, + {2, 0, 0, 3, 0, "null", "p", 5, 2, UINT64_C(0xd5e2502297cbec7f), -2}, + {2, 0, 0, 4, 0, "null", "p", 6, 2, UINT64_C(0x3c3901c7f0249a5d), -2}, + {2, 0, 0, 5, 0, "null", "p", 10, 5, UINT64_C(0x62f0e976a47f1e5d), -2}, + {2, 0, 0, 6, 0, "null", "p", 11, 5, UINT64_C(0x0ff5e378802151e7), -2}, + {2, 0, 0, 7, 0, "null", "p", 12, 5, UINT64_C(0x5b010d0357a36655), -2}, + {2, 0, 0, 8, 0, "null", "p", 12, 4, UINT64_C(0x5b010d0357a36655), -2}, + {2, 0, 0, 9, 0, "null", "p", 13, 4, UINT64_C(0x01a1f1adc408aba7), -2}, + {2, 0, 0, 10, 0, "null", "p", 14, 4, UINT64_C(0xe8f4f20b1cc04b85), -2}, + {2, 0, 0, 11, 0, "null", "p", 16, 5, UINT64_C(0x3713a0a2974a381d), -2}, + {2, 0, 0, 12, 0, "null", "p", 16, 4, UINT64_C(0x3713a0a2974a381d), -2}, + {2, 0, 0, 13, 0, "null", "p", 17, 4, UINT64_C(0x8682ef73727aab77), -2}, + {2, 0, 0, 14, 0, "null", "p", 18, 4, UINT64_C(0x65ba56ded1a9f0b5), -2}, + {2, 0, 0, 15, 0, "null", "p", 19, 4, UINT64_C(0xe1a0f051903ffaff), -2}, + {2, 0, 0, 16, 0, "null", "p", 21, 5, UINT64_C(0x26dd6278cb42aa8f), -2}, + {2, 0, 0, 17, 0, "null", "p", 22, 5, UINT64_C(0xabd24466a76de815), -2}, + {2, 0, 0, 18, 0, "null", "p", 22, 4, UINT64_C(0xabd24466a76de815), -2}, + {2, 0, 0, 19, 0, "null", "p", 23, 4, UINT64_C(0x1fa4cf6e592324e7), -2}, + {2, 0, 0, 20, 0, "null", "p", 24, 4, UINT64_C(0x8db9defb4ed6e27d), -2}, + {2, 0, 0, 21, 0, "null", "p", 25, 4, UINT64_C(0x4e4b45fec78b6fce), -2}, + {2, 0, 0, 22, 0, "null", "p", 25, 3, UINT64_C(0x4e4b45fec78b6fce), -2}, + {2, 0, 0, 23, 0, "null", "p", 26, 3, UINT64_C(0x1f9de5ed6029a77a), -2}, + {2, 0, 0, 24, 0, "null", "p", 27, 3, UINT64_C(0xc749bc5b78e58b26), -2}, + {2, 0, 0, 25, 0, "null", "p", 27, 2, UINT64_C(0xc749bc5b78e58b26), -2}, + {2, 0, 0, 26, 0, "null", "p", 28, 2, UINT64_C(0x87d63d6e6e0b7f2a), -2}, + {2, 0, 0, 27, 0, "null", "p", 29, 2, UINT64_C(0xdc01a4a4fd15a38e), -2}, + {2, 0, 0, 28, 0, "null", "p", 30, 2, UINT64_C(0xecdb0a5a0c2141d2), -2}, + {2, 0, 0, 29, 0, "null", "p", 31, 2, UINT64_C(0x9922d1029c3d7e66), -2}, + {2, 0, 0, 30, 0, "null", "p", 32, 2, UINT64_C(0x74edd56f7d93044a), -2}, + {2, 0, 0, 31, 0, "null", "p", 33, 2, UINT64_C(0x43778e726115a32e), -2}, + {2, 0, 0, 32, 0, "null", "p", 34, 2, UINT64_C(0xbb10725af8d968f2), -2}, + {2, 0, 0, 33, 0, "null", "p", 37, 4, UINT64_C(0x1bc79002c244ddb0), -2}, + {2, 0, 0, 34, 0, "null", "p", 39, 5, UINT64_C(0xe68750e3d28b26fe), -2}, + {2, 0, 0, 35, 0, "null", "p", 39, 4, UINT64_C(0xe68750e3d28b26fe), -2}, + {2, 0, 0, 36, 0, "null", "p", 41, 5, UINT64_C(0xa6cd9e55cb763a84), -2}, + {2, 0, 0, 37, 0, "null", "p", 41, 4, UINT64_C(0xa6cd9e55cb763a84), -2}, + {2, 0, 0, 38, 0, "null", "p", 42, 4, UINT64_C(0xeeec001e3cf3acae), -2}, + {2, 0, 0, 39, 0, "null", "p", 42, 3, UINT64_C(0xeeec001e3cf3acae), -2}, + {2, 0, 0, 40, 0, "null", "p", 43, 3, UINT64_C(0x3845eb624cdaabc4), -2}, + {2, 0, 0, 41, 0, "null", "p", 43, 2, UINT64_C(0x3845eb624cdaabc4), -2}, + {2, 0, 0, 42, 0, "null", "p", 45, 3, UINT64_C(0xc95774996f552f82), -2}, + {2, 0, 0, 43, 0, "null", "p", 45, 2, UINT64_C(0xc95774996f552f82), -2}, + {2, 0, 0, 44, 0, "null", "p", 46, 2, UINT64_C(0x74df0eb82dd11a74), -2}, + {2, 0, 0, 45, 0, "null", "p", 46, 1, UINT64_C(0x74df0eb82dd11a74), -2}, + {2, 0, 0, 46, 0, "null", "p", 47, 1, UINT64_C(0x682056f5da4fbcbc), -2}, + {2, 0, 0, 47, 0, "null", "p", 50, 3, UINT64_C(0x1ab9bdb9424d66cc), -2}, + {2, 0, 0, 48, 0, "null", "p", 50, 2, UINT64_C(0x1ab9bdb9424d66cc), -2}, + {2, 0, 0, 49, 0, "null", "p", 51, 2, UINT64_C(0xb7c285cbaa276b34), -2}, + {2, 0, 0, 50, 0, "null", "p", 55, 5, UINT64_C(0x9424298cac093d34), -2}, + {2, 0, 0, 51, 0, "null", "p", 56, 5, UINT64_C(0x347fcf61ec77d4fc), -2}, + {2, 0, 0, 52, 0, "null", "p", 57, 5, UINT64_C(0xc8c0ee9ba81adea4), -2}, + {2, 0, 0, 53, 0, "null", "p", 57, 4, UINT64_C(0xc8c0ee9ba81adea4), -2}, + {2, 0, 0, 54, 0, "null", "p", 58, 4, UINT64_C(0x6a12467e2f46cd44), -2}, + {2, 0, 0, 55, 0, "null", "p", 59, 4, UINT64_C(0x675a7db8dfd2380c), -2}, + {2, 0, 0, 56, 0, "null", "p", 61, 5, UINT64_C(0xf358f2078632f01c), -2}, + {2, 0, 0, 57, 0, "null", "p", 61, 4, UINT64_C(0xf358f2078632f01c), -2}, + {2, 0, 0, 58, 0, "null", "p", 62, 4, UINT64_C(0xf7023016a7bffac4), -2}, + {2, 0, 0, 59, 0, "null", "p", 63, 4, UINT64_C(0x5c31ebcd95ba888c), -2}, + {2, 0, 0, 60, 0, "null", "p", 64, 4, UINT64_C(0xbfc0288355164f74), -2}, + {2, 0, 0, 61, 0, "null", "p", 66, 5, UINT64_C(0x3c1aaba80763f04c), -2}, + {2, 0, 0, 62, 0, "null", "p", 67, 5, UINT64_C(0x70ea0a0f3208cabc), -2}, + {2, 0, 0, 63, 0, "null", "p", 67, 4, UINT64_C(0x70ea0a0f3208cabc), -2}, + {2, 0, 0, 64, 0, "null", "p", 68, 4, UINT64_C(0x8be363d140998bdc), -2}, + {2, 0, 0, 65, 0, "null", "p", 69, 4, UINT64_C(0x1cce49169e498e2c), -2}, + {2, 0, 0, 66, 0, "null", "p", 70, 4, UINT64_C(0x963b4977257eeb2d), -2}, + {2, 0, 0, 67, 0, "null", "p", 70, 3, UINT64_C(0x963b4977257eeb2d), -2}, + {2, 0, 0, 68, 0, "null", "p", 71, 3, UINT64_C(0x50c8aa75059d1427), -2}, + {2, 0, 0, 69, 0, "null", "p", 72, 3, UINT64_C(0x727e6bd728c0c50d), -2}, + {2, 0, 0, 70, 0, "null", "p", 72, 2, UINT64_C(0x727e6bd728c0c50d), -2}, + {2, 0, 0, 71, 0, "null", "p", 73, 2, UINT64_C(0x4d96439a3f8ec77f), -2}, + {2, 0, 0, 72, 0, "null", "p", 74, 2, UINT64_C(0x65a4561a001088cd), -2}, + {2, 0, 0, 73, 0, "null", "p", 75, 2, UINT64_C(0xc580d32e1b03375f), -2}, + {2, 0, 0, 74, 0, "null", "p", 76, 2, UINT64_C(0x9cccae57e631d82d), -2}, + {2, 0, 0, 75, 0, "null", "p", 77, 2, UINT64_C(0xa2eaa45c27c7763f), -2}, + {2, 0, 0, 76, 0, "null", "p", 78, 2, UINT64_C(0x9cb37f979861790d), -2}, + {2, 0, 0, 77, 0, "null", "p", 79, 2, UINT64_C(0xa6e79397edfd089f), -2}, + {2, 0, 0, 78, 0, "null", "p", 82, 4, UINT64_C(0xf9119f97a351f67f), -2}, + {2, 0, 0, 79, 0, "null", "p", 84, 5, UINT64_C(0xa6ad7c44581adb95), -2}, + {2, 0, 0, 80, 0, "null", "p", 84, 4, UINT64_C(0xa6ad7c44581adb95), -2}, + {2, 0, 0, 81, 0, "null", "p", 86, 5, UINT64_C(0x7e1c2c7a609d2a27), -2}, + {2, 0, 0, 82, 0, "null", "p", 86, 4, UINT64_C(0x7e1c2c7a609d2a27), -2}, + {2, 0, 0, 83, 0, "null", "p", 87, 4, UINT64_C(0x700c2bc962c0dd37), -2}, + {2, 0, 0, 84, 0, "null", "p", 87, 3, UINT64_C(0x700c2bc962c0dd37), -2}, + {2, 0, 0, 85, 0, "null", "p", 88, 3, UINT64_C(0x997f5834c283b41f), -2}, + {2, 0, 0, 86, 0, "null", "p", 88, 2, UINT64_C(0x997f5834c283b41f), -2}, + {2, 0, 0, 87, 0, "null", "p", 90, 3, UINT64_C(0x1d73fdf588d7ef35), -2}, + {2, 0, 0, 88, 0, "null", "p", 90, 2, UINT64_C(0x1d73fdf588d7ef35), -2}, + {2, 0, 0, 89, 0, "null", "p", 91, 2, UINT64_C(0xe41c293786fcd79d), -2}, + {2, 0, 0, 90, 0, "null", "p", 91, 1, UINT64_C(0xe41c293786fcd79d), -2}, + {2, 0, 0, 91, 0, "null", "p", 92, 1, UINT64_C(0x98b1c65a5fa29627), -2}, + {2, 0, 0, 92, 6, "[]", "p", 94, 2, UINT64_C(0x2e21f08ba84ea50f), -2}, + {2, 0, 0, 93, 6, "[]", "p", 94, 1, UINT64_C(0x2e21f08ba84ea50f), -2}, + {2, 0, 0, 94, 6, "[]", "p", 95, 1, UINT64_C(0xb250fc4efda2ca0d), -2}, + {2, 0, 0, 95, 6, "[]", "p", 99, 4, UINT64_C(0x4f89b9b9cb02910d), -2}, + {2, 0, 0, 96, 6, "[]", "p", 100, 4, UINT64_C(0xe36e6c3931047cd7), -2}, + {2, 0, 0, 97, 6, "[]", "p", 101, 4, UINT64_C(0xd458342f0f8181d5), -2}, + {2, 0, 0, 98, 6, "[]", "p", 101, 3, UINT64_C(0xd458342f0f8181d5), -2}, + {2, 0, 0, 99, 6, "[null]", "p", 102, 3, UINT64_C(0x53b420f7595503f7), -2}, + {2, 0, 0, 100, 6, "[null]", "p", 103, 3, UINT64_C(0x3581634c071e8725), -2}, + {2, 0, 0, 101, 6, "[[]]", "p", 105, 4, UINT64_C(0x5da1acf27d62e94d), -2}, + {2, 0, 0, 102, 6, "[[]]", "p", 105, 3, UINT64_C(0x5da1acf27d62e94d), -2}, + {2, 0, 0, 103, 6, "[[]]", "p", 106, 3, UINT64_C(0x219ef50a4a5f0987), -2}, + {2, 0, 0, 104, 6, "[[]]", "p", 107, 3, UINT64_C(0x5058ef7cd5824115), -2}, + {2, 0, 0, 105, 6, "[[]]", "p", 108, 3, UINT64_C(0x24681e1db9aba8ff), -2}, + {2, 0, 0, 106, 6, "[[]]", "p", 110, 4, UINT64_C(0xc3d3dbbd453ec2ff), -2}, + {2, 0, 0, 107, 6, "[[]]", "p", 111, 4, UINT64_C(0xa52f2ae95e6c62a5), -2}, + {2, 0, 0, 108, 6, "[[]]", "p", 111, 3, UINT64_C(0xa52f2ae95e6c62a5), -2}, + {2, 0, 0, 109, 6, "[[null]]", "p", 112, 3, UINT64_C(0x1c470f8b72cda0e7), -2}, + {2, 0, 0, 110, 6, "[[null]]", "p", 113, 3, UINT64_C(0x4f0a40f3c446b1ad), -2}, + {2, 0, 0, 111, 6, "[[\"\"]]", "p", 114, 3, UINT64_C(0x8c8fd0367f4f1ec4), -2}, + {2, 0, 0, 112, 6, "[[\"recovery\"]]", "p", 114, 2, UINT64_C(0x8c8fd0367f4f1ec4), -2}, + {2, 0, 0, 113, 6, "[[\"recovery\"]]", "p", 115, 2, UINT64_C(0x27b3c09a539f280c), -2}, + {2, 0, 0, 114, 6, "[[\"recovery\"]]", "p", 116, 2, UINT64_C(0x1510723c16fda92c), -2}, + {2, 0, 0, 115, 6, "[[\"recovery\"]]", "p", 116, 1, UINT64_C(0x1510723c16fda92c), -2}, + {2, 0, 0, 116, 6, "[[\"recovery\"],null]", "p", 117, 1, UINT64_C(0xc89b401b1106682c), -2}, + {2, 0, 0, 117, 6, "[[\"recovery\"],null]", "p", 118, 1, UINT64_C(0xe635f9fdede2cc64), -2}, + {2, 0, 0, 118, 6, "[[\"recovery\"],null]", "p", 119, 1, UINT64_C(0x1083d67b3860b864), -2}, + {2, 0, 0, 119, 6, "[[\"recovery\"],{}]", "p", 120, 1, UINT64_C(0x70bdb760cc591b8c), -2}, + {2, 0, 0, 120, 6, "[[\"recovery\"],{}]", "p", 121, 1, UINT64_C(0xeb7a7d7b3b6a646c), -2}, + {2, 0, 0, 121, 6, "[[\"recovery\"],{}]", "p", 122, 1, UINT64_C(0x8b878465f5c86d24), -2}, + {2, 0, 0, 122, 6, "[[\"recovery\"],{}]", "p", 123, 1, UINT64_C(0xdfb34540a391b824), -2}, + {2, 0, 0, 123, 6, "[[\"recovery\"],{}]", "p", 126, 3, UINT64_C(0x4e7f4bd6054c0f1e), -2}, + {2, 0, 0, 124, 6, "[[\"recovery\"],{}]", "p", 128, 4, UINT64_C(0xd0b0968fe04c8144), -2}, + {2, 0, 0, 125, 6, "[[\"recovery\"],{}]", "p", 128, 3, UINT64_C(0xd0b0968fe04c8144), -2}, + {2, 0, 0, 126, 6, "[[\"recovery\"],{}]", "p", 130, 4, UINT64_C(0x0f522f8844e15982), -2}, + {2, 0, 0, 127, 6, "[[\"recovery\"],{}]", "p", 130, 3, UINT64_C(0x0f522f8844e15982), -2}, + {2, 0, 0, 128, 6, "[[\"recovery\"],{}]", "p", 131, 3, UINT64_C(0x1cb2c08d27459c90), -2}, + {2, 0, 0, 129, 6, "[[\"recovery\"],{}]", "p", 131, 2, UINT64_C(0x1cb2c08d27459c90), -2}, + {2, 0, 0, 130, 6, "[[\"recovery\"],{}]", "p", 132, 2, UINT64_C(0x09e813d9bbc19cb2), -2}, + {2, 0, 0, 131, 6, "[[\"recovery\"],{}]", "p", 132, 1, UINT64_C(0x09e813d9bbc19cb2), -2}, + {2, 0, 0, 132, 6, "[[\"recovery\"],{\"\":0}]", "p", 134, 2, UINT64_C(0x81f6cedef9521a98), -2}, + {2, 0, 0, 133, 6, "[[\"recovery\"],{\"\":0}]", "p", 134, 1, UINT64_C(0x81f6cedef9521a98), -2}, + {2, 0, 0, 134, 6, "[[\"recovery\"],{\"\":0}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 0, 135, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 0, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 0, 0, "null", "p", 1, 1, UINT64_C(0xaf63c54c8601c577), -2}, + {2, 0, 1, 1, 0, "null", "p", 2, 1, UINT64_C(0x084d9807b50252d5), -2}, + {2, 0, 1, 2, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 3, 6, "[]", "p", 14, 1, UINT64_C(0xe61a5f2be2380035), -2}, + {2, 0, 1, 4, 6, "[]", "p", 15, 1, UINT64_C(0x3444f23f5be2600f), -2}, + {2, 0, 1, 5, 6, "[[\"recovery\"],{\"k\":1}]", "p", 131, 1, UINT64_C(0x7707bc21130f5bae), -2}, + {2, 0, 1, 6, 6, "[[\"recovery\"],{\"k\":1}]", "p", 132, 1, UINT64_C(0x2488217525922a20), -2}, + {2, 0, 1, 7, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 8, 6, "[{\"k\":1}]", "p", 88, 1, UINT64_C(0x962fddd74d36c07f), -2}, + {2, 0, 1, 9, 6, "[null,{\"k\":1}]", "p", 91, 1, UINT64_C(0x8c9cf7e8fd16f66d), -2}, + {2, 0, 1, 10, 6, "[null,{\"k\":1}]", "p", 92, 1, UINT64_C(0xb9bb8920405251af), -2}, + {2, 0, 1, 11, 6, "[[],{\"k\":1}]", "p", 115, 1, UINT64_C(0xe6edb009277e435d), -2}, + {2, 0, 1, 12, 6, "[[],{\"k\":1}]", "p", 114, 1, UINT64_C(0xe0f78966f158d3d7), -2}, + {2, 0, 1, 13, 6, "[[],{\"k\":1}]", "p", 115, 1, UINT64_C(0xe6edb009277e435d), -2}, + {2, 0, 1, 14, 6, "[[],{\"k\":1}]", "p", 116, 1, UINT64_C(0x0e4bd4713cfa929f), -2}, + {2, 0, 1, 15, 6, "[[],{\"k\":1}]", "p", 117, 1, UINT64_C(0x7499037f3f29dab5), -2}, + {2, 0, 1, 16, 6, "[[\"recovery\"],{\"k\":1}]", "p", 134, 1, UINT64_C(0x047f47a3f3e985c0), -2}, + {2, 0, 1, 17, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 18, 6, "[[],{\"k\":1}]", "p", 120, 1, UINT64_C(0xd36e3a8f4ece6abf), -2}, + {2, 0, 1, 19, 6, "[[null],{\"k\":1}]", "p", 123, 1, UINT64_C(0x1162f4e2b7c0964d), -2}, + {2, 0, 1, 20, 6, "[[null],{\"k\":1}]", "p", 124, 1, UINT64_C(0x8200a888509879f7), -2}, + {2, 0, 1, 21, 6, "[[\"\"],{\"k\":1}]", "p", 133, 1, UINT64_C(0xab82a2db1293d4a4), -2}, + {2, 0, 1, 22, 6, "[[\"recovery\"],{\"k\":1}]", "p", 131, 1, UINT64_C(0x29cf3c457348784e), -2}, + {2, 0, 1, 23, 6, "[[\"recovery\"],{\"k\":1}]", "p", 132, 1, UINT64_C(0x9879fdbf86ab412a), -2}, + {2, 0, 1, 24, 6, "[[\"recovery\"],{\"k\":1}]", "p", 133, 1, UINT64_C(0x811dd14cbc76d616), -2}, + {2, 0, 1, 25, 6, "[[\"recovery\"]]", "p", 70, 1, UINT64_C(0xee001ae21f987432), -2}, + {2, 0, 1, 26, 6, "[[\"recovery\"],null]", "p", 73, 1, UINT64_C(0x2a367fa959c067e0), -2}, + {2, 0, 1, 27, 6, "[[\"recovery\"],null]", "p", 74, 1, UINT64_C(0xc6b149f98ccf3efc), -2}, + {2, 0, 1, 28, 6, "[[\"recovery\"],null]", "p", 75, 1, UINT64_C(0x318f98678c3190e8), -2}, + {2, 0, 1, 29, 6, "[[\"recovery\"],{}]", "p", 96, 1, UINT64_C(0x5a0e500c3d11d1cc), -2}, + {2, 0, 1, 30, 6, "[[\"recovery\"],{}]", "p", 97, 1, UINT64_C(0xdfd95d45102e4158), -2}, + {2, 0, 1, 31, 6, "[[\"recovery\"],{}]", "p", 98, 1, UINT64_C(0x1e923d9e3135ee54), -2}, + {2, 0, 1, 32, 6, "[[\"recovery\"],{}]", "p", 99, 1, UINT64_C(0x25637f963a6e0c80), -2}, + {2, 0, 1, 33, 6, "[[\"recovery\"],{\"\":1}]", "p", 120, 1, UINT64_C(0xaf2bbc597666e8b6), -2}, + {2, 0, 1, 34, 6, "[[\"recovery\"],{\"\":1}]", "p", 124, 1, UINT64_C(0x286966ae72068854), -2}, + {2, 0, 1, 35, 6, "[[\"recovery\"],{\"\":1}]", "p", 122, 1, UINT64_C(0x70bdfbd9cc778848), -2}, + {2, 0, 1, 36, 6, "[[\"recovery\"],{\"\":1}]", "p", 125, 1, UINT64_C(0x7f9ec3b496f20e76), -2}, + {2, 0, 1, 37, 6, "[[\"recovery\"],{\"k\":0}]", "p", 127, 1, UINT64_C(0xe435cc31d865c04c), -2}, + {2, 0, 1, 38, 6, "[[\"recovery\"],{\"k\":0}]", "p", 128, 1, UINT64_C(0x0bc93ca23c973146), -2}, + {2, 0, 1, 39, 6, "[[\"recovery\"],{\"\":0}]", "p", 124, 1, UINT64_C(0x5b0438a962c028ee), -2}, + {2, 0, 1, 40, 6, "[[\"recovery\"],{\"\":0}]", "p", 126, 1, UINT64_C(0x749e33fe09846f14), -2}, + {2, 0, 1, 41, 6, "[[\"recovery\"],{}]", "p", 108, 1, UINT64_C(0x1d1930c0b39ff2d6), -2}, + {2, 0, 1, 42, 6, "[[\"recovery\"],{\"k\":0}]", "p", 129, 1, UINT64_C(0x39b811df8065a440), -2}, + {2, 0, 1, 43, 6, "[[\"recovery\"],{\"\":0}]", "p", 126, 1, UINT64_C(0x749e33fe09846f14), -2}, + {2, 0, 1, 44, 6, "[[\"recovery\"],{\"\":0}]", "p", 127, 1, UINT64_C(0xa9ae8c0106305f02), -2}, + {2, 0, 1, 45, 0, "null", "p", 46, 1, UINT64_C(0x74df0eb82dd11a74), -2}, + {2, 0, 1, 46, 0, "null", "p", 47, 1, UINT64_C(0x682056f5da4fbcbc), -2}, + {2, 0, 1, 47, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 48, 6, "[]", "p", 54, 1, UINT64_C(0xd866a499e8246f4c), -2}, + {2, 0, 1, 49, 6, "[]", "p", 55, 1, UINT64_C(0x3843fa114a3e82b4), -2}, + {2, 0, 1, 50, 6, "[[\"recovery\"],{\"k\":1}]", "p", 131, 1, UINT64_C(0xf8f5f9bbf251a3ae), -2}, + {2, 0, 1, 51, 6, "[[\"recovery\"],{\"k\":1}]", "p", 132, 1, UINT64_C(0xfbbd747ae65c5fd6), -2}, + {2, 0, 1, 52, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 53, 6, "[{\"k\":1}]", "p", 105, 1, UINT64_C(0xdf45e48aebd07bac), -2}, + {2, 0, 1, 54, 6, "[null,{\"k\":1}]", "p", 107, 1, UINT64_C(0xf5386814407a5c18), -2}, + {2, 0, 1, 55, 6, "[null,{\"k\":1}]", "p", 108, 1, UINT64_C(0x75ecae360ec86ae0), -2}, + {2, 0, 1, 56, 6, "[[],{\"k\":1}]", "p", 121, 1, UINT64_C(0x9ecbb9e5b73e2b08), -2}, + {2, 0, 1, 57, 6, "[[],{\"k\":1}]", "p", 120, 1, UINT64_C(0x6352de65e7a42820), -2}, + {2, 0, 1, 58, 6, "[[],{\"k\":1}]", "p", 121, 1, UINT64_C(0x9ecbb9e5b73e2b08), -2}, + {2, 0, 1, 59, 6, "[[],{\"k\":1}]", "p", 122, 1, UINT64_C(0xf997bcd7489c6570), -2}, + {2, 0, 1, 60, 6, "[[],{\"k\":1}]", "p", 123, 1, UINT64_C(0xa5cb6694f3def5d8), -2}, + {2, 0, 1, 61, 6, "[[\"recovery\"],{\"k\":1}]", "p", 134, 1, UINT64_C(0x1cff01046f9d81fe), -2}, + {2, 0, 1, 62, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 63, 6, "[[],{\"k\":1}]", "p", 126, 1, UINT64_C(0x6e46235ae155ad00), -2}, + {2, 0, 1, 64, 6, "[[null],{\"k\":1}]", "p", 128, 1, UINT64_C(0x4d9e4c0942fb1314), -2}, + {2, 0, 1, 65, 6, "[[null],{\"k\":1}]", "p", 129, 1, UINT64_C(0x7db796f6516c6044), -2}, + {2, 0, 1, 66, 6, "[[\"\"],{\"k\":1}]", "p", 134, 1, UINT64_C(0x44b6e6e072fd4815), -2}, + {2, 0, 1, 67, 6, "[[\"recovery\"],{\"k\":1}]", "p", 131, 1, UINT64_C(0x5bba7eb92882c42e), -2}, + {2, 0, 1, 68, 6, "[[\"recovery\"],{\"k\":1}]", "p", 132, 1, UINT64_C(0xb32dfae5f269e0bc), -2}, + {2, 0, 1, 69, 6, "[[\"recovery\"],{\"k\":1}]", "p", 133, 1, UINT64_C(0x288b68f5f56c0c8e), -2}, + {2, 0, 1, 70, 6, "[[\"recovery\"]]", "p", 93, 1, UINT64_C(0x2c089b9238e3038c), -2}, + {2, 0, 1, 71, 6, "[[\"recovery\"],null]", "p", 95, 1, UINT64_C(0x9c59076cabc41196), -2}, + {2, 0, 1, 72, 6, "[[\"recovery\"],null]", "p", 96, 1, UINT64_C(0x945067c4b0b7de34), -2}, + {2, 0, 1, 73, 6, "[[\"recovery\"],null]", "p", 97, 1, UINT64_C(0xf3cc42e2cf1b7eb6), -2}, + {2, 0, 1, 74, 6, "[[\"recovery\"],{}]", "p", 108, 1, UINT64_C(0x75541d23077a1f54), -2}, + {2, 0, 1, 75, 6, "[[\"recovery\"],{}]", "p", 109, 1, UINT64_C(0x1597eae911857f46), -2}, + {2, 0, 1, 76, 6, "[[\"recovery\"],{}]", "p", 110, 1, UINT64_C(0xc524da67a13a4e74), -2}, + {2, 0, 1, 77, 6, "[[\"recovery\"],{}]", "p", 111, 1, UINT64_C(0x6981ea88761ff5e6), -2}, + {2, 0, 1, 78, 6, "[[\"recovery\"],{\"\":1}]", "p", 124, 1, UINT64_C(0xc3d0929287f61588), -2}, + {2, 0, 1, 79, 6, "[[\"recovery\"],{\"\":1}]", "p", 128, 1, UINT64_C(0x8669172f2a49755a), -2}, + {2, 0, 1, 80, 6, "[[\"recovery\"],{\"\":1}]", "p", 126, 1, UINT64_C(0x860088c0a1b6b29a), -2}, + {2, 0, 1, 81, 6, "[[\"recovery\"],{\"\":1}]", "p", 129, 1, UINT64_C(0x4a2488df4a66032a), -2}, + {2, 0, 1, 82, 6, "[[\"recovery\"],{\"k\":0}]", "p", 130, 1, UINT64_C(0xaa382e1de6f59b70), -2}, + {2, 0, 1, 83, 6, "[[\"recovery\"],{\"k\":0}]", "p", 131, 1, UINT64_C(0x423f69eae99f2980), -2}, + {2, 0, 1, 84, 6, "[[\"recovery\"],{\"\":0}]", "p", 128, 1, UINT64_C(0xb99b0f902499c508), -2}, + {2, 0, 1, 85, 6, "[[\"recovery\"],{\"\":0}]", "p", 130, 1, UINT64_C(0xebd8dfdbead7bca2), -2}, + {2, 0, 1, 86, 6, "[[\"recovery\"],{}]", "p", 120, 1, UINT64_C(0xce91b9a105dc6166), -2}, + {2, 0, 1, 87, 6, "[[\"recovery\"],{\"k\":0}]", "p", 132, 1, UINT64_C(0x404963a2c4a10eb0), -2}, + {2, 0, 1, 88, 6, "[[\"recovery\"],{\"\":0}]", "p", 130, 1, UINT64_C(0xebd8dfdbead7bca2), -2}, + {2, 0, 1, 89, 6, "[[\"recovery\"],{\"\":0}]", "p", 131, 1, UINT64_C(0x3e79f273ddcb978a), -2}, + {2, 0, 1, 90, 0, "null", "p", 91, 1, UINT64_C(0xe41c293786fcd79d), -2}, + {2, 0, 1, 91, 0, "null", "p", 92, 1, UINT64_C(0x98b1c65a5fa29627), -2}, + {2, 0, 1, 92, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 93, 6, "[]", "p", 94, 1, UINT64_C(0x2e21f08ba84ea50f), -2}, + {2, 0, 1, 94, 6, "[]", "p", 95, 1, UINT64_C(0xb250fc4efda2ca0d), -2}, + {2, 0, 1, 95, 6, "[[\"recovery\"],{\"k\":1}]", "p", 131, 1, UINT64_C(0x6f9d196c9bd4b80e), -2}, + {2, 0, 1, 96, 6, "[[\"recovery\"],{\"k\":1}]", "p", 132, 1, UINT64_C(0xee016a80c22b8f04), -2}, + {2, 0, 1, 97, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 98, 6, "[{\"k\":1}]", "p", 122, 1, UINT64_C(0x7892afa12ce1ab6d), -2}, + {2, 0, 1, 99, 6, "[null,{\"k\":1}]", "p", 123, 1, UINT64_C(0x3767948effb80947), -2}, + {2, 0, 1, 100, 6, "[null,{\"k\":1}]", "p", 124, 1, UINT64_C(0x506c0994a5d90c1d), -2}, + {2, 0, 1, 101, 6, "[[],{\"k\":1}]", "p", 127, 1, UINT64_C(0x9176b3ac4cb728b7), -2}, + {2, 0, 1, 102, 6, "[[],{\"k\":1}]", "p", 126, 1, UINT64_C(0xd58a76f1a9068035), -2}, + {2, 0, 1, 103, 6, "[[],{\"k\":1}]", "p", 127, 1, UINT64_C(0x9176b3ac4cb728b7), -2}, + {2, 0, 1, 104, 6, "[[],{\"k\":1}]", "p", 128, 1, UINT64_C(0xffbfa858653271ad), -2}, + {2, 0, 1, 105, 6, "[[],{\"k\":1}]", "p", 129, 1, UINT64_C(0xf4cd6d09dfbfe95f), -2}, + {2, 0, 1, 106, 6, "[[\"recovery\"],{\"k\":1}]", "p", 134, 1, UINT64_C(0x258db63c2da5cad4), -2}, + {2, 0, 1, 107, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 108, 6, "[[],{\"k\":1}]", "p", 132, 1, UINT64_C(0x587e897afbcc8e9d), -2}, + {2, 0, 1, 109, 6, "[[null],{\"k\":1}]", "p", 133, 1, UINT64_C(0x574165db06887297), -2}, + {2, 0, 1, 110, 6, "[[null],{\"k\":1}]", "p", 134, 1, UINT64_C(0x44b6e6e072fd4815), -2}, + {2, 0, 1, 111, 6, "[[\"\"],{\"k\":1}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 112, 6, "[[\"recovery\"],{\"k\":1}]", "p", 131, 1, UINT64_C(0xbeffc9b3b016a96e), -2}, + {2, 0, 1, 113, 6, "[[\"recovery\"],{\"k\":1}]", "p", 132, 1, UINT64_C(0xcc99fca554c5d906), -2}, + {2, 0, 1, 114, 6, "[[\"recovery\"],{\"k\":1}]", "p", 133, 1, UINT64_C(0xc35b653033194ba6), -2}, + {2, 0, 1, 115, 6, "[[\"recovery\"]]", "p", 116, 1, UINT64_C(0x1510723c16fda92c), -2}, + {2, 0, 1, 116, 6, "[[\"recovery\"],null]", "p", 117, 1, UINT64_C(0xc89b401b1106682c), -2}, + {2, 0, 1, 117, 6, "[[\"recovery\"],null]", "p", 118, 1, UINT64_C(0xe635f9fdede2cc64), -2}, + {2, 0, 1, 118, 6, "[[\"recovery\"],null]", "p", 119, 1, UINT64_C(0x1083d67b3860b864), -2}, + {2, 0, 1, 119, 6, "[[\"recovery\"],{}]", "p", 120, 1, UINT64_C(0x70bdb760cc591b8c), -2}, + {2, 0, 1, 120, 6, "[[\"recovery\"],{}]", "p", 121, 1, UINT64_C(0xeb7a7d7b3b6a646c), -2}, + {2, 0, 1, 121, 6, "[[\"recovery\"],{}]", "p", 122, 1, UINT64_C(0x8b878465f5c86d24), -2}, + {2, 0, 1, 122, 6, "[[\"recovery\"],{}]", "p", 123, 1, UINT64_C(0xdfb34540a391b824), -2}, + {2, 0, 1, 123, 6, "[[\"recovery\"],{\"\":1}]", "p", 128, 1, UINT64_C(0xf0e26e8f61eedf0e), -2}, + {2, 0, 1, 124, 6, "[[\"recovery\"],{\"\":1}]", "p", 132, 1, UINT64_C(0x898913a98d73fe48), -2}, + {2, 0, 1, 125, 6, "[[\"recovery\"],{\"\":1}]", "p", 130, 1, UINT64_C(0x0e41f58843f9c3c4), -2}, + {2, 0, 1, 126, 6, "[[\"recovery\"],{\"\":1}]", "p", 133, 1, UINT64_C(0x2c9072f93d1f7bca), -2}, + {2, 0, 1, 127, 6, "[[\"recovery\"],{\"k\":0}]", "p", 133, 1, UINT64_C(0x7f8a662f14e7da72), -2}, + {2, 0, 1, 128, 6, "[[\"recovery\"],{\"k\":0}]", "p", 134, 1, UINT64_C(0x81e270def940d1d4), -2}, + {2, 0, 1, 129, 6, "[[\"recovery\"],{\"\":0}]", "p", 132, 1, UINT64_C(0x0959c1d9bb490c16), -2}, + {2, 0, 1, 130, 6, "[[\"recovery\"],{\"\":0}]", "p", 134, 1, UINT64_C(0x81f6cedef9521a98), -2}, + {2, 0, 1, 131, 6, "[[\"recovery\"],{}]", "p", 132, 1, UINT64_C(0x09e813d9bbc19cb2), -2}, + {2, 0, 1, 132, 6, "[[\"recovery\"],{\"k\":0}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 133, 6, "[[\"recovery\"],{\"\":0}]", "p", 134, 1, UINT64_C(0x81f6cedef9521a98), -2}, + {2, 0, 1, 134, 6, "[[\"recovery\"],{\"\":0}]", "p", 135, 1, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 0, 1, 135, 6, "[[\"recovery\"],{\"k\":1}]", "p", 135, 0, UINT64_C(0x287c1ee1a68333ae), -2}, + {2, 1, 0, 0, 4, "inf", "p", 2, 2, UINT64_C(0x08842007b530c95d), -2}, + {2, 1, 0, 1, 4, "inf", "p", 3, 2, UINT64_C(0xa94bb518e1e60558), -2}, + {2, 1, 0, 2, 4, "inf", "p", 4, 2, UINT64_C(0x925e3a47da742eb6), -2}, + {2, 1, 0, 3, 4, "inf", "p", 7, 4, UINT64_C(0x873fb06eb56bb310), -2}, + {2, 1, 0, 4, 4, "inf", "p", 7, 3, UINT64_C(0x873fb06eb56bb310), -2}, + {2, 1, 0, 5, 4, "inf", "p", 8, 3, UINT64_C(0x21be1d1f5889f9de), -2}, + {2, 1, 0, 6, 4, "inf", "p", 12, 6, UINT64_C(0xefae9e7acaf5f4de), -2}, + {2, 1, 0, 7, 4, "inf", "p", 13, 6, UINT64_C(0x4657d233983eb2a8), -2}, + {2, 1, 0, 8, 4, "inf", "p", 14, 6, UINT64_C(0xf0284721aceafbd6), -2}, + {2, 1, 0, 9, 4, "inf", "p", 14, 5, UINT64_C(0xf0284721aceafbd6), -2}, + {2, 1, 0, 10, 4, "inf", "p", 15, 5, UINT64_C(0x442e86010f8af8c8), -2}, + {2, 1, 0, 11, 4, "inf", "p", 16, 5, UINT64_C(0x85c06de61045b546), -2}, + {2, 1, 0, 12, 4, "inf", "p", 18, 6, UINT64_C(0x41d98e2f4ad3c91e), -2}, + {2, 1, 0, 13, 4, "inf", "p", 18, 5, UINT64_C(0x41d98e2f4ad3c91e), -2}, + {2, 1, 0, 14, 4, "inf", "p", 19, 5, UINT64_C(0x790df96af2244a78), -2}, + {2, 1, 0, 15, 4, "inf", "p", 20, 5, UINT64_C(0x376dc67cd2ba1f16), -2}, + {2, 1, 0, 16, 4, "inf", "p", 21, 5, UINT64_C(0x3de77c10479abc20), -2}, + {2, 1, 0, 17, 4, "inf", "p", 23, 6, UINT64_C(0x168f5f4ea0496820), -2}, + {2, 1, 0, 18, 4, "inf", "p", 24, 6, UINT64_C(0xbff01ebe0def54c6), -2}, + {2, 1, 0, 19, 4, "inf", "p", 24, 5, UINT64_C(0xbff01ebe0def54c6), -2}, + {2, 1, 0, 20, 4, "inf", "p", 25, 5, UINT64_C(0x6e0c55c54e01aed8), -2}, + {2, 1, 0, 21, 4, "inf", "p", 26, 5, UINT64_C(0x672952c86689b87e), -2}, + {2, 1, 0, 22, 4, "inf", "p", 27, 5, UINT64_C(0x985cc89ee676aa03), -2}, + {2, 1, 0, 23, 4, "inf", "p", 27, 4, UINT64_C(0x985cc89ee676aa03), -2}, + {2, 1, 0, 24, 4, "inf", "p", 28, 4, UINT64_C(0x76a37085ec026a7b), -2}, + {2, 1, 0, 25, 4, "inf", "p", 29, 4, UINT64_C(0x67ea086e2017149b), -2}, + {2, 1, 0, 26, 4, "inf", "p", 29, 3, UINT64_C(0x67ea086e2017149b), -2}, + {2, 1, 0, 27, 4, "inf", "p", 30, 3, UINT64_C(0xa9799d2086fb771b), -2}, + {2, 1, 0, 28, 4, "inf", "p", 31, 3, UINT64_C(0x2528a044e77d2923), -2}, + {2, 1, 0, 29, 4, "inf", "p", 32, 3, UINT64_C(0xb64fe715f69e6223), -2}, + {2, 1, 0, 30, 4, "inf", "p", 33, 3, UINT64_C(0x0d6544514aa46a7b), -2}, + {2, 1, 0, 31, 4, "inf", "p", 34, 3, UINT64_C(0x5206e02138248d5b), -2}, + {2, 1, 0, 32, 4, "inf", "p", 35, 3, UINT64_C(0xb647ba71f04df7e3), -2}, + {2, 1, 0, 33, 4, "inf", "p", 36, 3, UINT64_C(0xca3aa1997d6e5ce3), -2}, + {2, 1, 0, 34, 4, "inf", "p", 39, 5, UINT64_C(0x049c604f5086fbf5), -2}, + {2, 1, 0, 35, 4, "inf", "p", 41, 6, UINT64_C(0xd30bbe54aa67b803), -2}, + {2, 1, 0, 36, 4, "inf", "p", 41, 5, UINT64_C(0xd30bbe54aa67b803), -2}, + {2, 1, 0, 37, 4, "inf", "p", 43, 6, UINT64_C(0x565578045ddc3491), -2}, + {2, 1, 0, 38, 4, "inf", "p", 43, 5, UINT64_C(0x565578045ddc3491), -2}, + {2, 1, 0, 39, 4, "inf", "p", 44, 5, UINT64_C(0x90309b76bc8f1147), -2}, + {2, 1, 0, 40, 4, "inf", "p", 44, 4, UINT64_C(0x90309b76bc8f1147), -2}, + {2, 1, 0, 41, 4, "inf", "p", 45, 4, UINT64_C(0x5ddf35f5bf677aa1), -2}, + {2, 1, 0, 42, 4, "inf", "p", 45, 3, UINT64_C(0x5ddf35f5bf677aa1), -2}, + {2, 1, 0, 43, 4, "inf", "p", 47, 4, UINT64_C(0x1676278f9958f91f), -2}, + {2, 1, 0, 44, 4, "inf", "p", 47, 3, UINT64_C(0x1676278f9958f91f), -2}, + {2, 1, 0, 45, 4, "inf", "p", 48, 3, UINT64_C(0xda674d01c347d925), -2}, + {2, 1, 0, 46, 4, "inf", "p", 48, 2, UINT64_C(0xda674d01c347d925), -2}, + {2, 1, 0, 47, 4, "inf", "p", 49, 2, UINT64_C(0x65f30dfed3920179), -2}, + {2, 1, 0, 48, 4, "inf", "p", 54, 6, UINT64_C(0x914f3d1bf9fab8ba), -2}, + {2, 1, 0, 49, 4, "inf", "p", 54, 5, UINT64_C(0x914f3d1bf9fab8ba), -2}, + {2, 1, 0, 50, 4, "inf", "p", 55, 5, UINT64_C(0xde0c49db59af6846), -2}, + {2, 1, 0, 51, 4, "inf", "p", 59, 8, UINT64_C(0xb7b6ab5b9ab29b66), -2}, + {2, 1, 0, 52, 4, "inf", "p", 60, 8, UINT64_C(0xd3548f8598abb40a), -2}, + {2, 1, 0, 53, 4, "inf", "p", 61, 8, UINT64_C(0xca2dac3b6af6a2c6), -2}, + {2, 1, 0, 54, 4, "inf", "p", 61, 7, UINT64_C(0xca2dac3b6af6a2c6), -2}, + {2, 1, 0, 55, 4, "inf", "p", 62, 7, UINT64_C(0x6c79bb8c2f845522), -2}, + {2, 1, 0, 56, 4, "inf", "p", 63, 7, UINT64_C(0x507800877df2db7e), -2}, + {2, 1, 0, 57, 4, "inf", "p", 65, 8, UINT64_C(0xc7d73b525cb35cde), -2}, + {2, 1, 0, 58, 4, "inf", "p", 65, 7, UINT64_C(0xc7d73b525cb35cde), -2}, + {2, 1, 0, 59, 4, "inf", "p", 66, 7, UINT64_C(0x8fcd0dc6b2475962), -2}, + {2, 1, 0, 60, 4, "inf", "p", 67, 7, UINT64_C(0x1a10f9f3af53143e), -2}, + {2, 1, 0, 61, 4, "inf", "p", 68, 7, UINT64_C(0x8d9413d55ceccf42), -2}, + {2, 1, 0, 62, 4, "inf", "p", 70, 8, UINT64_C(0x3705ed93c71c7dba), -2}, + {2, 1, 0, 63, 4, "inf", "p", 71, 8, UINT64_C(0xb689c5816d1f6a9e), -2}, + {2, 1, 0, 64, 4, "inf", "p", 71, 7, UINT64_C(0xb689c5816d1f6a9e), -2}, + {2, 1, 0, 65, 4, "inf", "p", 72, 7, UINT64_C(0x569f42d7c1e146ba), -2}, + {2, 1, 0, 66, 4, "inf", "p", 73, 7, UINT64_C(0x7000e960937be48e), -2}, + {2, 1, 0, 67, 4, "inf", "p", 74, 7, UINT64_C(0x1516607c80e41fc9), -2}, + {2, 1, 0, 68, 4, "inf", "p", 74, 6, UINT64_C(0x1516607c80e41fc9), -2}, + {2, 1, 0, 69, 4, "inf", "p", 75, 6, UINT64_C(0x96750cb7cbb0a4bb), -2}, + {2, 1, 0, 70, 4, "inf", "p", 76, 6, UINT64_C(0x470cf046dcbd8819), -2}, + {2, 1, 0, 71, 4, "inf", "p", 76, 5, UINT64_C(0x470cf046dcbd8819), -2}, + {2, 1, 0, 72, 4, "inf", "p", 77, 5, UINT64_C(0xc53df0bc2e598963), -2}, + {2, 1, 0, 73, 4, "inf", "p", 78, 5, UINT64_C(0x0f9a2d48a9309429), -2}, + {2, 1, 0, 74, 4, "inf", "p", 79, 5, UINT64_C(0xa7bfb696b47ccdb3), -2}, + {2, 1, 0, 75, 4, "inf", "p", 80, 5, UINT64_C(0xac5533df7aaf9119), -2}, + {2, 1, 0, 76, 4, "inf", "p", 81, 5, UINT64_C(0x3b3567c0e6b18803), -2}, + {2, 1, 0, 77, 4, "inf", "p", 82, 5, UINT64_C(0x06d0d4ed3a90d6c9), -2}, + {2, 1, 0, 78, 4, "inf", "p", 83, 5, UINT64_C(0x9d0b9317b69dcdd3), -2}, + {2, 1, 0, 79, 4, "inf", "p", 86, 7, UINT64_C(0xb3b840baf7320603), -2}, + {2, 1, 0, 80, 4, "inf", "p", 88, 8, UINT64_C(0x4089d5d92e008641), -2}, + {2, 1, 0, 81, 4, "inf", "p", 88, 7, UINT64_C(0x4089d5d92e008641), -2}, + {2, 1, 0, 82, 4, "inf", "p", 90, 8, UINT64_C(0x4ab828316be566fb), -2}, + {2, 1, 0, 83, 4, "inf", "p", 90, 7, UINT64_C(0x4ab828316be566fb), -2}, + {2, 1, 0, 84, 4, "inf", "p", 91, 7, UINT64_C(0x842d7ddee33895bb), -2}, + {2, 1, 0, 85, 4, "inf", "p", 91, 6, UINT64_C(0x842d7ddee33895bb), -2}, + {2, 1, 0, 86, 4, "inf", "p", 92, 6, UINT64_C(0xcca48c677db770e3), -2}, + {2, 1, 0, 87, 4, "inf", "p", 92, 5, UINT64_C(0xcca48c677db770e3), -2}, + {2, 1, 0, 88, 4, "inf", "p", 94, 6, UINT64_C(0x3bd0b6d0949cbca1), -2}, + {2, 1, 0, 89, 4, "inf", "p", 94, 5, UINT64_C(0x3bd0b6d0949cbca1), -2}, + {2, 1, 0, 90, 4, "inf", "p", 95, 5, UINT64_C(0x4afa4667b947c869), -2}, + {2, 1, 0, 91, 4, "inf", "p", 95, 4, UINT64_C(0x4afa4667b947c869), -2}, + {2, 1, 0, 92, 4, "inf", "p", 96, 4, UINT64_C(0xc3cf3a0ff19c4214), -2}, + {2, 1, 0, 93, 4, "inf", "p[", 96, 3, UINT64_C(0xc3cf3a0ff19c4214), -2}, + {2, 1, 0, 94, 4, "inf", "p[", 96, 2, UINT64_C(0xc3cf3a0ff19c4214), -2}, + {2, 1, 0, 95, 4, "inf", "p[", 102, 7, UINT64_C(0xe297ab3390bad1e8), -2}, + {2, 1, 0, 96, 4, "inf", "p[", 102, 6, UINT64_C(0xe297ab3390bad1e8), -2}, + {2, 1, 0, 97, 4, "inf", "p[[", 102, 5, UINT64_C(0xe297ab3390bad1e8), -2}, + {2, 1, 0, 98, 4, "inf", "p[[", 102, 4, UINT64_C(0xe297ab3390bad1e8), -2}, + {2, 1, 0, 99, 4, "inf", "p[[", 114, 15, UINT64_C(0x374af64aaff30d64), -2}, + {2, 1, 0, 100, 4, "inf", "p[[", 114, 14, UINT64_C(0x374af64aaff30d64), -2}, + {2, 1, 0, 101, 4, "inf", "p[[\"recovery\"", 105, 4, UINT64_C(0xce60b037a8707bbe), -2}, + {2, 1, 0, 102, 4, "inf", "p[[\"recovery\"", 105, 3, UINT64_C(0xce60b037a8707bbe), -2}, + {2, 1, 0, 103, 4, "inf", "p[[\"recovery\"]", 105, 2, UINT64_C(0xce60b037a8707bbe), -2}, + {2, 1, 0, 104, 4, "inf", "p[[\"recovery\"]", 111, 7, UINT64_C(0x354affc149c2c62a), -2}, + {2, 1, 0, 105, 4, "inf", "p[[\"recovery\"]", 112, 7, UINT64_C(0x894ddfea63a49cfc), -2}, + {2, 1, 0, 106, 4, "inf", "p[[\"recovery\"],{", 111, 5, UINT64_C(0x354affc149c2c62a), -2}, + {2, 1, 0, 107, 4, "inf", "p[[\"recovery\"],{", 111, 4, UINT64_C(0x354affc149c2c62a), -2}, + {2, 1, 0, 108, 4, "inf", "p[[\"recovery\"],{", 118, 10, UINT64_C(0x885508e0a1d3c190), -2}, + {2, 1, 0, 109, 4, "inf", "p[[\"recovery\"],{", 118, 9, UINT64_C(0x885508e0a1d3c190), -2}, + {2, 1, 0, 110, 4, "inf", "p[[\"recovery\"],{", 115, 5, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 0, 111, 4, "inf", "p[[\"recovery\"],{\"k\":1", 115, 4, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 0, 112, 4, "inf", "p[[\"recovery\"],{\"k\":1", 115, 3, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 0, 113, 4, "inf", "p[[\"recovery\"],{\"k\":1}", 115, 2, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 0, 114, 4, "inf", "p[[\"recovery\"],{\"k\":1}", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 0, 115, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 0, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 0, 4, "inf", "p", 105, 1, UINT64_C(0xfac7f1cbf21951a9), -2}, + {2, 1, 1, 1, 4, "inf", "pnull", 4, 1, UINT64_C(0x91aa0747d9db038a), -2}, + {2, 1, 1, 2, 4, "inf", "pnull", 5, 1, UINT64_C(0x2a4ba818336b39e0), -2}, + {2, 1, 1, 3, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 4, 4, "inf", "p[]", 15, 1, UINT64_C(0x0c8974e4b6ecf110), -2}, + {2, 1, 1, 5, 4, "inf", "p[]", 16, 1, UINT64_C(0x69af6a031832f12a), -2}, + {2, 1, 1, 6, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 111, 1, UINT64_C(0x3f9605949f0b61dc), -2}, + {2, 1, 1, 7, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 112, 1, UINT64_C(0x55e1e9f354626c96), -2}, + {2, 1, 1, 8, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 9, 4, "inf", "p[{\"k\":1}]", 75, 1, UINT64_C(0x591aac29dc23081c), -2}, + {2, 1, 1, 10, 4, "inf", "p[null,{\"k\":1}]", 80, 1, UINT64_C(0xaba4dcbfd93fbe89), -2}, + {2, 1, 1, 11, 4, "inf", "p[null,{\"k\":1}]", 81, 1, UINT64_C(0x732879a78cb66343), -2}, + {2, 1, 1, 12, 4, "inf", "p[[],{\"k\":1}]", 97, 1, UINT64_C(0x73efebbb25af1480), -2}, + {2, 1, 1, 13, 4, "inf", "p[[],{\"k\":1}]", 96, 1, UINT64_C(0xf8c138a5ea57881e), -2}, + {2, 1, 1, 14, 4, "inf", "p[[],{\"k\":1}]", 97, 1, UINT64_C(0x73efebbb25af1480), -2}, + {2, 1, 1, 15, 4, "inf", "p[[],{\"k\":1}]", 98, 1, UINT64_C(0xcddc111172cf8e06), -2}, + {2, 1, 1, 16, 4, "inf", "p[[],{\"k\":1}]", 99, 1, UINT64_C(0x6f4ae13a6baf83f8), -2}, + {2, 1, 1, 17, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 114, 1, UINT64_C(0x3a7766a8697c2a66), -2}, + {2, 1, 1, 18, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 19, 4, "inf", "p[[],{\"k\":1}]", 102, 1, UINT64_C(0x11e37283dc8dd7d6), -2}, + {2, 1, 1, 20, 4, "inf", "p[[null],{\"k\":1}]", 107, 1, UINT64_C(0x848dda53d220f8fb), -2}, + {2, 1, 1, 21, 4, "inf", "p[[null],{\"k\":1}]", 108, 1, UINT64_C(0xa9473dc35a917371), -2}, + {2, 1, 1, 22, 4, "inf", "p[[\"\"],{\"k\":1}]", 113, 1, UINT64_C(0xc5fb8ea3c2b0b1c3), -2}, + {2, 1, 1, 23, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 111, 1, UINT64_C(0xf51092ee3a9ad1fc), -2}, + {2, 1, 1, 24, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 112, 1, UINT64_C(0xedfb1fb02fae5c84), -2}, + {2, 1, 1, 25, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 113, 1, UINT64_C(0x36f8caa611be6164), -2}, + {2, 1, 1, 26, 4, "inf", "p[[\"recovery\"]]", 63, 1, UINT64_C(0xc8600ba3b9a0000c), -2}, + {2, 1, 1, 27, 4, "inf", "p[[\"recovery\"],null]", 69, 1, UINT64_C(0x7846bb94c3fa45a4), -2}, + {2, 1, 1, 28, 4, "inf", "p[[\"recovery\"],null]", 70, 1, UINT64_C(0x34916d381ad6105c), -2}, + {2, 1, 1, 29, 4, "inf", "p[[\"recovery\"],null]", 71, 1, UINT64_C(0xb57428a5b1a5535c), -2}, + {2, 1, 1, 30, 4, "inf", "p[[\"recovery\"],{}]", 84, 1, UINT64_C(0x305be21460b5f84e), -2}, + {2, 1, 1, 31, 4, "inf", "p[[\"recovery\"],{}]", 85, 1, UINT64_C(0x13c0590bc3f0feae), -2}, + {2, 1, 1, 32, 4, "inf", "p[[\"recovery\"],{}]", 86, 1, UINT64_C(0x0cfdde51f4c27156), -2}, + {2, 1, 1, 33, 4, "inf", "p[[\"recovery\"],{}]", 87, 1, UINT64_C(0x8f32c5b6747d2856), -2}, + {2, 1, 1, 34, 4, "inf", "p[[\"recovery\"],{\"\":1}]", 104, 1, UINT64_C(0x488f2fce5d41cff4), -2}, + {2, 1, 1, 35, 4, "inf", "p[[\"recovery\"],{\"\":1}]", 108, 1, UINT64_C(0xd595ff26e9136a52), -2}, + {2, 1, 1, 36, 4, "inf", "p[[\"recovery\"],{\"\":1}]", 106, 1, UINT64_C(0xd2bb220845d5b4be), -2}, + {2, 1, 1, 37, 4, "inf", "p[[\"recovery\"],{\"\":1}]", 109, 1, UINT64_C(0x6ca21d89f8353260), -2}, + {2, 1, 1, 38, 4, "inf", "p[[\"recovery\"],{\"k\":0}]", 110, 1, UINT64_C(0x449e6807405be748), -2}, + {2, 1, 1, 39, 4, "inf", "p[[\"recovery\"],{\"k\":0}]", 111, 1, UINT64_C(0xa260c5b4fb642196), -2}, + {2, 1, 1, 40, 4, "inf", "p[[\"recovery\"],{\"\":0}]", 108, 1, UINT64_C(0xfc1524aac194c92c), -2}, + {2, 1, 1, 41, 4, "inf", "p[[\"recovery\"],{\"\":0}]", 110, 1, UINT64_C(0xf81d5cfa5e6b6022), -2}, + {2, 1, 1, 42, 4, "inf", "p[[\"recovery\"],{}]", 96, 1, UINT64_C(0xa69ee9fea9eb4938), -2}, + {2, 1, 1, 43, 4, "inf", "p[[\"recovery\"],{\"k\":0}]", 112, 1, UINT64_C(0x132c5d9212db309c), -2}, + {2, 1, 1, 44, 4, "inf", "p[[\"recovery\"],{\"\":0}]", 110, 1, UINT64_C(0xf81d5cfa5e6b6022), -2}, + {2, 1, 1, 45, 4, "inf", "p[[\"recovery\"],{\"\":0}]", 111, 1, UINT64_C(0x043e77994de74fd4), -2}, + {2, 1, 1, 46, 4, "inf", "pnull", 49, 1, UINT64_C(0x65610cfed31611a9), -2}, + {2, 1, 1, 47, 4, "inf", "pnull", 50, 1, UINT64_C(0xce02370181186f9d), -2}, + {2, 1, 1, 48, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 49, 4, "inf", "p[]", 55, 1, UINT64_C(0xc3754b6adc679d65), -2}, + {2, 1, 1, 50, 4, "inf", "p[]", 56, 1, UINT64_C(0xde5a0015199ced49), -2}, + {2, 1, 1, 51, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 111, 1, UINT64_C(0x74016e17e4a9193c), -2}, + {2, 1, 1, 52, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 112, 1, UINT64_C(0xe38a0224c4ca8bf8), -2}, + {2, 1, 1, 53, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 54, 4, "inf", "p[{\"k\":1}]", 92, 1, UINT64_C(0x0143704d13918f93), -2}, + {2, 1, 1, 55, 4, "inf", "p[null,{\"k\":1}]", 96, 1, UINT64_C(0x67f7c86fc86745e8), -2}, + {2, 1, 1, 56, 4, "inf", "p[null,{\"k\":1}]", 97, 1, UINT64_C(0xd472dda62fc8b51c), -2}, + {2, 1, 1, 57, 4, "inf", "p[[],{\"k\":1}]", 103, 1, UINT64_C(0x4422ace3e1108167), -2}, + {2, 1, 1, 58, 4, "inf", "p[[],{\"k\":1}]", 102, 1, UINT64_C(0x208e048921aeb27b), -2}, + {2, 1, 1, 59, 4, "inf", "p[[],{\"k\":1}]", 103, 1, UINT64_C(0x4422ace3e1108167), -2}, + {2, 1, 1, 60, 4, "inf", "p[[],{\"k\":1}]", 104, 1, UINT64_C(0xc940fad8d887805b), -2}, + {2, 1, 1, 61, 4, "inf", "p[[],{\"k\":1}]", 105, 1, UINT64_C(0xdd2801bf456fb6c7), -2}, + {2, 1, 1, 62, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 114, 1, UINT64_C(0x38b6d551ca08b970), -2}, + {2, 1, 1, 63, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 64, 4, "inf", "p[[],{\"k\":1}]", 108, 1, UINT64_C(0x88b0a306097cbe3b), -2}, + {2, 1, 1, 65, 4, "inf", "p[[null],{\"k\":1}]", 112, 1, UINT64_C(0x92fbfabfe2bd4d90), -2}, + {2, 1, 1, 66, 4, "inf", "p[[null],{\"k\":1}]", 113, 1, UINT64_C(0x8d853497518ee95c), -2}, + {2, 1, 1, 67, 4, "inf", "p[[\"\"],{\"k\":1}]", 114, 1, UINT64_C(0xcdc25dcc8c6fd758), -2}, + {2, 1, 1, 68, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 111, 1, UINT64_C(0x905ea4dbf4f5ae3c), -2}, + {2, 1, 1, 69, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 112, 1, UINT64_C(0xf52f68a5508e71e2), -2}, + {2, 1, 1, 70, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 113, 1, UINT64_C(0xef2a9233221136ec), -2}, + {2, 1, 1, 71, 4, "inf", "p[[\"recovery\"]]", 86, 1, UINT64_C(0x1948be9f7de8afae), -2}, + {2, 1, 1, 72, 4, "inf", "p[[\"recovery\"],null]", 91, 1, UINT64_C(0xf7140d424fe8b6ee), -2}, + {2, 1, 1, 73, 4, "inf", "p[[\"recovery\"],null]", 92, 1, UINT64_C(0x8315c9b651e19040), -2}, + {2, 1, 1, 74, 4, "inf", "p[[\"recovery\"],null]", 93, 1, UINT64_C(0xe0c1f4211c5b60be), -2}, + {2, 1, 1, 75, 4, "inf", "p[[\"recovery\"],{}]", 96, 1, UINT64_C(0xc0ede8404dad0c5a), -2}, + {2, 1, 1, 76, 4, "inf", "p[[\"recovery\"],{}]", 97, 1, UINT64_C(0x9da215612280ff2c), -2}, + {2, 1, 1, 77, 4, "inf", "p[[\"recovery\"],{}]", 98, 1, UINT64_C(0x202ed5d544886c8a), -2}, + {2, 1, 1, 78, 4, "inf", "p[[\"recovery\"],{}]", 99, 1, UINT64_C(0xb8acaca1beac85bc), -2}, + {2, 1, 1, 79, 4, "inf", "p[[\"recovery\"],{\"\":1}]", 108, 1, UINT64_C(0x52121bbdbfb0762a), -2}, + {2, 1, 1, 80, 4, "inf", "p[[\"recovery\"],{\"\":1}]", 112, 1, UINT64_C(0x48642ece843ee99c), -2}, + {2, 1, 1, 81, 4, "inf", "p[[\"recovery\"],{\"\":1}]", 110, 1, UINT64_C(0x397fa09381d5f75c), -2}, + {2, 1, 1, 82, 4, "inf", "p[[\"recovery\"],{\"\":1}]", 113, 1, UINT64_C(0x10cf6f3e6fb9025c), -2}, + {2, 1, 1, 83, 4, "inf", "p[[\"recovery\"],{\"k\":0}]", 113, 1, UINT64_C(0x71198b642b887e5c), -2}, + {2, 1, 1, 84, 4, "inf", "p[[\"recovery\"],{\"k\":0}]", 114, 1, UINT64_C(0xce2d81c99c68799c), -2}, + {2, 1, 1, 85, 4, "inf", "p[[\"recovery\"],{\"\":0}]", 112, 1, UINT64_C(0x79982ee8a36ed02a), -2}, + {2, 1, 1, 86, 4, "inf", "p[[\"recovery\"],{\"\":0}]", 114, 1, UINT64_C(0x476bb0dcbf90c4a4), -2}, + {2, 1, 1, 87, 4, "inf", "p[[\"recovery\"],{}]", 108, 1, UINT64_C(0x1c22249ac336200c), -2}, + {2, 1, 1, 88, 4, "inf", "p[[\"recovery\"],{\"k\":0}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 89, 4, "inf", "p[[\"recovery\"],{\"\":0}]", 114, 1, UINT64_C(0x476bb0dcbf90c4a4), -2}, + {2, 1, 1, 90, 4, "inf", "p[[\"recovery\"],{\"\":0}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 91, 4, "inf", "p[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x5b13a2832940b6f4), -2}, + {2, 1, 1, 92, 4, "inf", "p[\"recovery\"],{\"k\":1}]", 116, 1, UINT64_C(0xbb5fcbcb673868bd), -2}, + {2, 1, 1, 93, 4, "inf", "p[]", 96, 1, UINT64_C(0xc3cf3a0ff19c4214), -2}, + {2, 1, 1, 94, 4, "inf", "p[]", 96, 1, UINT64_C(0xc3cf3a0ff19c4214), -2}, + {2, 1, 1, 95, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 96, 4, "inf", "p[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 97, 4, "inf", "p[[],{\"k\":1}]", 111, 1, UINT64_C(0xcc5ceaf4e3a4f3aa), -2}, + {2, 1, 1, 98, 4, "inf", "p[[],{\"k\":1}]", 111, 1, UINT64_C(0xcc5ceaf4e3a4f3aa), -2}, + {2, 1, 1, 99, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 100, 4, "inf", "p[[recovery\"],{\"k\":1}]", 116, 1, UINT64_C(0x23a78c04006531a6), -2}, + {2, 1, 1, 101, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 102, 4, "inf", "p[[\"recovery\",{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 103, 4, "inf", "p[[\"recovery\"]]", 105, 1, UINT64_C(0xce60b037a8707bbe), -2}, + {2, 1, 1, 104, 4, "inf", "p[[\"recovery\"]{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 105, 4, "inf", "p[[\"recovery\"]{\"k\":1}]", 116, 1, UINT64_C(0x5652dc9a31d4a622), -2}, + {2, 1, 1, 106, 4, "inf", "p[[\"recovery\"],{}]", 111, 1, UINT64_C(0x354affc149c2c62a), -2}, + {2, 1, 1, 107, 4, "inf", "p[[\"recovery\"],{}]", 111, 1, UINT64_C(0x354affc149c2c62a), -2}, + {2, 1, 1, 108, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 109, 4, "inf", "p[[\"recovery\"],{k\":1}]", 116, 1, UINT64_C(0xd4561202a8aba466), -2}, + {2, 1, 1, 110, 4, "inf", "p[[\"recovery\"],{}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 111, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 112, 4, "inf", "p[[\"recovery\"],{\"k\":1]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 113, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 114, 4, "inf", "p[[\"recovery\"],{\"k\":1}", 115, 1, UINT64_C(0x10e2790b3ef7283c), -2}, + {2, 1, 1, 115, 4, "inf", "p[[\"recovery\"],{\"k\":1}]", 115, 0, UINT64_C(0x10e2790b3ef7283c), -2}, }; int main() { RecoveryAllocationTrace trace; RecoveryAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; @@ -429,12 +933,15 @@ int main() { Value replacement{}; _ZN3sce4Json5ValueC1Ev(&replacement); const char document[] = "{\"k\":\"recovery\"}"; Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&replacement, document, sizeof(document) - 1) == 0 && Contents(replacement) == document, "Construct the independently owned callback-return template"); + Value nestedReplacement{}; _ZN3sce4Json5ValueC1Ev(&nestedReplacement); + const char nestedDocument[] = "[[\"recovery\"],{\"k\":1}]"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&nestedReplacement, nestedDocument, sizeof(nestedDocument) - 1) == 0 && Contents(nestedReplacement) == nestedDocument, "Construct the independently owned nested recovery template"); String lookupKey{}; _ZN3sce4Json6StringC1EPKc(&lookupKey, "k"); const auto baseline = trace.live; const auto baselineIds = trace.ids; for (const auto& expected : Cases) { Value value{}; _ZN3sce4Json5ValueC1Ed(&value, expected.format ? std::numeric_limits::infinity() : 0.125); - RecoveryContext callback{&replacement, expected.object}; + RecoveryContext callback{expected.object == 2 ? &nestedReplacement : &replacement, expected.object != 0}; Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, RecoverAllocated, &callback) == 0, "Register allocation-backed recovery"); void* auxiliary = value.auxiliary; void* fallback = static_cast(auxiliary)[2]; @@ -464,11 +971,12 @@ int main() { Require(!value.parent && value.auxiliary == auxiliary && static_cast(auxiliary)[2] == fallback && static_cast(auxiliary)[4] == record, "Partial callback cloning must preserve destination callback metadata and fallback identity"); Require(trace.ids.count(auxiliary) && trace.ids.count(fallback) && trace.ids.count(record), "Destination recovery metadata must retain live allocator ownership"); Require(trace.ids.at(auxiliary) == auxiliaryId && trace.ids.at(fallback) == fallbackId && trace.ids.at(record) == recordId && static_cast(record)[0] == reinterpret_cast(&RecoverAllocated) && static_cast(record)[1] == &callback && static_cast(fallback)->parent == &value, "Recovery metadata must retain its allocation generation, handler, context and fallback owner"); - Require(Contents(replacement) == document, "Failure recovery must preserve the independently owned source template"); + Require(Contents(replacement) == document && Contents(nestedReplacement) == nestedDocument, "Failure recovery must preserve the independently owned source template"); _ZN3sce4Json6StringD1Ev(&output); _ZN3sce4Json5ValueD1Ev(&value); Require(trace.live == baseline && trace.ids == baselineIds, "Every callback-return failure state must reclaim temporary ownership and remain destructible"); } _ZN3sce4Json6StringD1Ev(&lookupKey); + _ZN3sce4Json5ValueD1Ev(&nestedReplacement); _ZN3sce4Json5ValueD1Ev(&replacement); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty() && trace.ids.empty(), "Callback-return allocation checks must release all allocator-owned resources"); return 0; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 9d0011997a..c0989f381a 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -66,4 +66,4 @@ - [Parser parent and auxiliary transitions](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-driven reentry, more complex nested metadata failures and alternate SDK combinations still require audit. Reference tests cover direct parsing into an owned child across eight value forms, six root-admission failure/retry cases, incomplete-literal preservation, modern root callback retention and nested release of three handler records, fallback and auxiliary in native order. Allocation generations distinguish release from later address reuse. -- [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - nested array returns, more complex object shapes, inherited failure scopes and alternate SDK combinations remain unverified. Reference tests cover 330 persistent/one-shot cases through successful string and one-entry object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. +- [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - more complex object shapes, inherited failure scopes and alternate SDK combinations remain unverified. Reference tests cover 834 persistent/one-shot cases through successful string, one-entry object and nested array/object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. From 505d9aa7f3af5d195452091b16d407db02a771dc Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:19:49 +0200 Subject: [PATCH 047/318] fix(libSceJson2): preserve initialization cleanup allocator effects --- core/libs/prx/libSceJson2/CMakeLists.txt | 15 ++ core/libs/prx/libSceJson2/Export.cpp | 23 +++- .../tests/GuestJsonInitializationRelease.cpp | 130 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 +- 4 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 core/libs/tests/GuestJsonInitializationRelease.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 0aa09ecfca..8a8fdbb10d 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,21 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_initialization_release_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationRelease.cpp) + target_include_directories(guest_json_initialization_release_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_initialization_release_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_initialization_release_tests) + add_test(NAME guest_json_initialization_release COMMAND guest_json_initialization_release_tests) + add_test_executable(guest_json_legacy_initialization_release_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationRelease.cpp Export.cpp) + target_compile_definitions(guest_json_legacy_initialization_release_tests PRIVATE JSON2_LEGACY_TEST) + target_include_directories(guest_json_legacy_initialization_release_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_legacy_initialization_release_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_legacy_initialization_release_tests) + add_test(NAME guest_json_legacy_initialization_release COMMAND guest_json_legacy_initialization_release_tests) + if(WIN32) + set_tests_properties(guest_json_initialization_release guest_json_legacy_initialization_release PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() add_test_executable(guest_json_recovery_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonRecoveryAllocator.cpp) target_include_directories(guest_json_recovery_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_recovery_allocator_tests PRIVATE libSceJson2 libc) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 78283d5657..a0b553974d 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1396,10 +1396,14 @@ const Value& Defaults(ValueType type) { return defaults.values[type]; } -void ResetDefaults() { +void ResetDefaults(bool releaseMissing = false) { auto& defaults = SharedDefaults(); for (unsigned type = 0; type < 8; ++type) { - if (defaults.ready[type]) Destroy(defaults.values[type]); + if (defaults.ready[type]) { + const bool missing = type >= TypeString && !defaults.values[type].payload; + Destroy(defaults.values[type]); + if (releaseMissing && missing) Release(nullptr); + } defaults.values[type] = {}; defaults.ready[type] = false; } @@ -1420,15 +1424,20 @@ PthreadMutex* CreateKernelMutex() { void ReleaseInitialization() { auto& state = State(); + if (!ModernValues() && !state.legacyHandlers) Release(nullptr); ReleaseLegacyHandlers(); - if (state.fallback) { Destroy(*state.fallback); Release(state.fallback); state.fallback = nullptr; } - if (state.recoveryMutex && state.recoveryMutex != state.internalMutex) { - scePthreadMutexDestroy(state.recoveryMutex); + if (state.fallback) Destroy(*state.fallback); + Release(state.fallback); + state.fallback = nullptr; + if (ModernValues()) { + if (state.recoveryMutex) scePthreadMutexDestroy(state.recoveryMutex); Release(state.recoveryMutex); } state.recoveryMutex = nullptr; - if (state.internalMutex) { scePthreadMutexDestroy(state.internalMutex); Release(state.internalMutex); state.internalMutex = nullptr; } - ResetDefaults(); + if (state.internalMutex) scePthreadMutexDestroy(state.internalMutex); + Release(state.internalMutex); + state.internalMutex = nullptr; + ResetDefaults(true); if (state.allocation) { auto* allocation = state.allocation; const auto context = *allocation; diff --git a/core/libs/tests/GuestJsonInitializationRelease.cpp b/core/libs/tests/GuestJsonInitializationRelease.cpp new file mode 100644 index 0000000000..203eeec600 --- /dev/null +++ b/core/libs/tests/GuestJsonInitializationRelease.cpp @@ -0,0 +1,130 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +} +#if defined(JSON2_LEGACY_TEST) +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + static const std::uint32_t words[]{0, 0, 0, 0, 0x05000000}; + return words; +} +#endif +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct InitializationTrace { + std::map ids; + std::vector releases; + std::size_t requests = 0; + int budget = -1; + bool once = false; +}; +static void* APS5_VABI AllocateTracked(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + ++trace.requests; + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); + Require(trace.ids.emplace(memory, trace.requests).second, "Each allocation must have an independent live identity"); + return memory; +} +static void APS5_VABI FreeTracked(void*, void* memory, void* context) { + auto& trace = *static_cast(context); + trace.releases.push_back(memory ? trace.ids.at(memory) : 0); + if (!memory) return; + Require(trace.ids.erase(memory) == 1, "Cleanup must release each admitted allocation through its owner exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyTracked(void*, int error, std::size_t, void*) { + Require(static_cast(error) == 0x80848102, "Initialization allocation failures must report the observed error"); +} +static void* const Vtable[]{nullptr, nullptr, reinterpret_cast(AllocateTracked), reinterpret_cast(FreeTracked), reinterpret_cast(NotifyTracked)}; +struct InitializationAllocator { void* const* vtable = Vtable; }; +struct Expected { bool once; int budget; bool admitted; std::vector releases; }; +static const Expected Cases[]{ +#if defined(JSON2_LEGACY_TEST) + {0, 0, false, {}}, + {0, 1, false, {0,0,0,0,0,0,1}}, + {0, 2, false, {0,0,0,2,0,0,1}}, + {0, 3, false, {0,0,0,3,2,0,0,1}}, + {0, 4, false, {0,0,0,3,2,4,0,1}}, + {0, 5, false, {0,0,0,3,2,5,4,0,1}}, + {0, 6, false, {0,0,0,3,2,6,5,4,0,1}}, + {0, 7, false, {0,0,0,3,2,6,5,4,7,1}}, + {0, 8, false, {8,0,0,0,3,2,6,5,4,7,1}}, + {0, 9, false, {0,0,0,3,2,6,5,4,9,8,7,1}}, + {0, 10, false, {0,10,0,3,2,6,5,4,9,8,7,1}}, + {0, 11, true, {0,10,11,3,2,6,5,4,9,8,7,1}}, + {0, 12, true, {0,10,11,3,2,6,5,4,9,8,7,1}}, + {1, 0, false, {}}, + {1, 1, false, {0,9,10,0,5,4,3,8,7,6,1}}, + {1, 2, true, {0,10,11,2,6,5,4,9,8,7,1}}, + {1, 3, false, {0,8,9,3,2,0,7,6,5,1}}, + {1, 4, true, {0,9,10,3,2,4,8,7,6,1}}, + {1, 5, true, {0,10,11,3,2,5,4,9,8,7,1}}, + {1, 6, false, {0,8,9,3,2,6,5,4,0,1}}, + {1, 7, true, {0,9,10,3,2,6,5,4,7,1}}, + {1, 8, true, {8,0,10,11,3,2,6,5,4,7,1}}, + {1, 9, false, {0,0,11,3,2,6,5,4,9,8,7,1}}, + {1, 10, false, {0,10,0,3,2,6,5,4,9,8,7,1}}, + {1, 11, true, {0,10,11,3,2,6,5,4,9,8,7,1}}, + {1, 12, true, {0,10,11,3,2,6,5,4,9,8,7,1}}, +#else + {0, 0, false, {}}, + {0, 1, false, {0,0,0,0,0,0,1}}, + {0, 2, false, {0,0,0,2,0,0,1}}, + {0, 3, false, {0,0,0,3,2,0,0,1}}, + {0, 4, false, {0,0,0,3,2,4,0,1}}, + {0, 5, false, {0,0,0,3,2,5,4,0,1}}, + {0, 6, false, {0,0,0,3,2,6,5,4,0,1}}, + {0, 7, false, {0,0,0,3,2,6,5,4,7,1}}, + {0, 8, false, {8,0,0,0,3,2,6,5,4,7,1}}, + {0, 9, false, {0,0,0,3,2,6,5,4,9,8,7,1}}, + {0, 10, false, {10,0,0,3,2,6,5,4,9,8,7,1}}, + {0, 11, false, {10,0,11,3,2,6,5,4,9,8,7,1}}, + {0, 12, true, {10,12,11,3,2,6,5,4,9,8,7,1}}, + {1, 0, false, {}}, + {1, 1, false, {9,0,10,0,5,4,3,8,7,6,1}}, + {1, 2, true, {10,12,11,2,6,5,4,9,8,7,1}}, + {1, 3, false, {8,0,9,3,2,0,7,6,5,1}}, + {1, 4, true, {9,11,10,3,2,4,8,7,6,1}}, + {1, 5, true, {10,12,11,3,2,5,4,9,8,7,1}}, + {1, 6, false, {8,0,9,3,2,6,5,4,0,1}}, + {1, 7, true, {9,11,10,3,2,6,5,4,7,1}}, + {1, 8, true, {8,10,12,11,3,2,6,5,4,7,1}}, + {1, 9, false, {0,0,11,3,2,6,5,4,9,8,7,1}}, + {1, 10, false, {10,0,0,3,2,6,5,4,9,8,7,1}}, + {1, 11, false, {10,0,11,3,2,6,5,4,9,8,7,1}}, + {1, 12, true, {10,12,11,3,2,6,5,4,9,8,7,1}}, +#endif +}; +int main() { + for (const auto& expected : Cases) { + InitializationTrace trace; trace.budget = expected.budget; trace.once = expected.once; + InitializationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + const int status = _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter); + Require(status == (expected.admitted ? 0 : static_cast(0x80848102)) && initializer == expected.admitted, "Failure admission must preserve the reference status and initializer active byte"); + if (expected.admitted) Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate admitted partial resources"); + Require(trace.ids.empty() && initializer == 0, "Rollback and termination must reclaim every admitted resource"); + Require(trace.releases == expected.releases, "Cleanup must preserve exact allocation-identity order including observable null frees"); + Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(&initializer)) == 0x80848110 && trace.releases == expected.releases, "Repeated termination must report inactive state without allocator effects"); + trace.budget = -1; trace.once = false; trace.requests = 0; trace.releases.clear(); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1, "Retry must work on the same initializer after every failure schedule"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.ids.empty(), "Retry termination must reclaim complete resource ownership"); + Require(trace.releases == Cases[12].releases, "Retry must restore the full healthy teardown protocol"); + } + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index c0989f381a..6cfdf12a37 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -36,7 +36,7 @@ - [sce::Json::Value nested array allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. -- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - initialization follows `0x1340`, `0xda0` and `0x940`: a 24-byte allocation context owns the original allocator, user context and file-buffer setting; default handles/storage, a 32-byte fallback and kernel mutex handles are allocated before admission. Inner default failures can retain usable outer handles; required-resource failure reclaims ownership and leaves retry legal. SDK 6 uses two recursive kernel mutex handles, while the legacy path shares one. Local tests substitute the kernel boundary and cover all twelve SDK 6 allocation positions, partial admission and retry. Failed kernel mutex setup or locking throws rather than fabricating success; terminating from an active callback remains unsupported. These checks do not prove Linux/Windows integration or every reentrant/concurrent lifecycle. +- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - kernel setup/locking failures and termination during callbacks remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Initialization at `0x1340`, `0xda0` and `0x940` owns a 24-byte allocator context, default handles/storage, fallback and recursive mutex handles (two for SDK 6, one for SDK 5). Reference tests cover persistent/one-shot allocation failures, partial admission, exact release identities including null frees, repeated termination and retry on the same initializer. Kernel setup succeeds at the substituted boundary. The reference skips mutex creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. - [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. From cdf706ca3842eaddefa8e89e7b628293c1843ef4 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:26:13 +0200 Subject: [PATCH 048/318] fix(libSceJson2): preserve parser callback failure ordering --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 9 +- .../libs/tests/GuestJsonInheritedRecovery.cpp | 187 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 199 insertions(+), 8 deletions(-) create mode 100644 core/libs/tests/GuestJsonInheritedRecovery.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 8a8fdbb10d..6e70ab2b7f 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_inherited_recovery_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInheritedRecovery.cpp) + target_include_directories(guest_json_inherited_recovery_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_inherited_recovery_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_inherited_recovery_tests) + add_test(NAME guest_json_inherited_recovery COMMAND guest_json_inherited_recovery_tests) add_test_executable(guest_json_initialization_release_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationRelease.cpp) target_include_directories(guest_json_initialization_release_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_initialization_release_tests PRIVATE libSceJson2 libc) @@ -166,7 +171,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index a0b553974d..474ef5f37d 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2227,10 +2227,7 @@ public: Parser(const char* text, std::size_t size) : text(text), end(text + size) {} Parser(ParserReadCallback read, void* context) : read(read), context(context) {} - bool Parse(Node& out) { - skip(); - return value(out); - } + bool Parse(Node& out) { return value(out); } int Error() const { return readError < 0 ? readError : JsonErrorParse; } @@ -2382,7 +2379,9 @@ private: return true; } bool value(Node& out, const Value* parent = nullptr) { - if (AllocationOperation::Failed() || peek() < 0) return false; + if (AllocationOperation::Failed()) return false; + skip(); + if (peek() < 0) return false; switch (peek()) { case 'n': if (!literal("null")) return false; SetType(out, TypeNull); Admit(out, parent); return true; case 't': if (!literal("true")) return false; SetType(out, TypeBoolean); out.boolean = true; Admit(out, parent); return true; diff --git a/core/libs/tests/GuestJsonInheritedRecovery.cpp b/core/libs/tests/GuestJsonInheritedRecovery.cpp new file mode 100644 index 0000000000..4266ac1d60 --- /dev/null +++ b/core/libs/tests/GuestJsonInheritedRecovery.cpp @@ -0,0 +1,187 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +static_assert(sizeof(Value) == 32 && sizeof(Parameter) == 40); +using ReadCallback = int (APS5_VABI*)(char&, void*); +using FloatCallback = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +extern "C" { +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(Value*, ReadCallback, void*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatCallback, void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +} +static void Require(bool valid, const char* message) { + if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct RecoveryAllocationTrace { + std::map live; + std::vector requests; + int budget = -1; + bool once = false; + unsigned notifications = 0; + std::map ids; + std::size_t nextId = 0; +}; +static void* APS5_VABI AllocateRecovery(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); trace.ids[memory] = ++trace.nextId; return memory; +} +static void APS5_VABI FreeRecovery(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + Require(trace.ids.count(memory) == 1, "Freed recovery storage must identify a currently live allocation generation"); + trace.ids.erase(memory); + Require(trace.live.erase(memory) == 1, "Recovery storage must release through its owning allocator"); std::free(memory); +} +static void APS5_VABI NotifyRecovery(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Recovery notification must identify the failed allocation"); ++trace.notifications; +} +static void* const RecoveryVtable[]{nullptr, nullptr, reinterpret_cast(AllocateRecovery), reinterpret_cast(FreeRecovery), reinterpret_cast(NotifyRecovery)}; +struct RecoveryAllocator { void* const* vtable = RecoveryVtable; }; +struct RecoveryContext { const Value* replacement; bool object; unsigned calls = 0; int function = -1; bool errorPointer = false; }; +static Value* APS5_VABI RecoverAllocated(Value* result, std::int32_t function, double number, const Value* parent, void* context, bool* error) { + auto& trace = *static_cast(context); + ++trace.calls; trace.function = function; trace.errorPointer = error != nullptr; + Require(number == std::numeric_limits::infinity() && !parent, "Recovery must receive the original non-finite value and immediate parent"); + if (trace.object) _ZN3sce4Json5ValueC1ERKS1_(result, trace.replacement); + else _ZN3sce4Json5ValueC1EPKc(result, "recovery"); + if (error) *error = false; + return result; +} +static std::string Contents(Value& value) { + if (value.type == 0) return "null"; + if (value.type == 4) { Require(value.payload == std::bit_cast(std::numeric_limits::infinity()), "Serialization recovery must preserve the source real payload bits"); return "inf"; } + if (value.type == 5) return _ZNK3sce4Json6String5c_strEv(reinterpret_cast(value.payload)); + String output{}; _ZN3sce4Json6StringC1EPKc(&output, ""); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output) == 0, "Partial callback-return objects must remain serializable"); + std::string result = _ZNK3sce4Json6String5c_strEv(&output); + _ZN3sce4Json6StringD1Ev(&output); + return result; +} +struct ReadContext { + RecoveryAllocationTrace* allocations; + bool fail; + Value* assignment; + Value* format; + Value* nested; + int nestedStatus = 0; + int formatStatus = 0; + String* output; + unsigned reads = 0; + unsigned nestedReads = 0; +}; +static int APS5_VABI ReadTrue(char& character, void* data) { + auto& position = *static_cast(data); + if (position >= 4) return static_cast(0x80848101); + character = "true"[position++]; + return 0; +} +static int APS5_VABI ReadNested(char& character, void* data) { + auto& context = *static_cast(data); + if (!context.reads++) { + if (context.fail) { + String failed{}; + context.allocations->budget = 0; context.allocations->once = true; + _ZN3sce4Json6StringC1EPKc(&failed, "x"); + context.allocations->budget = -1; context.allocations->once = false; + _ZN3sce4Json6StringD1Ev(&failed); + } + context.nestedStatus = _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(context.nested, ReadTrue, &context.nestedReads); + _ZN3sce4Json5Value3setEd(context.assignment, std::numeric_limits::infinity()); + context.formatStatus = _ZN3sce4Json5Value9serializeERNS0_6StringE(context.format, context.output); + } + if (context.reads > 4) return static_cast(0x80848101); + character = "true"[context.reads - 1]; + return 0; +} +struct Expected { + const char* document; + bool fail; + int type; + const char* assignment; + const char* output; + std::size_t requests; + std::uint64_t order; +}; +static const Expected Cases[]{ + {"\"recovery\"", 0, 5, "recovery", "p\"recovery\"", 22, UINT64_C(0x512fbd7009730792)}, + {"\"recovery\"", 1, 5, "recovery", "p\"recovery\"", 21, UINT64_C(0xb7c45554dbd6d058)}, + {"{\"k\":\"recovery\"}", 0, 7, "{\"k\":\"recovery\"}", "p{\"k\":\"recovery\"}", 143, UINT64_C(0x105ecd3d3b0befa3)}, + {"{\"k\":\"recovery\"}", 1, 7, "{\"k\":\"\"}", "p{\"k\":\"\"}", 137, UINT64_C(0x1c0f8b9f8e134cd2)}, + {"[[\"recovery\"],{\"k\":1}]", 0, 6, "[[\"recovery\"],{\"k\":1}]", "p[[\"recovery\"],{\"k\":1}]", 254, UINT64_C(0xd1da135cb823d2ad)}, + {"[[\"recovery\"],{\"k\":1}]", 1, 6, "[[],{}]", "p[[],{}]", 136, UINT64_C(0xe4d4124493072e0a)}, +}; +int main() { + RecoveryAllocationTrace allocations; RecoveryAllocator allocator; + Parameter parameter{&allocator, &allocations, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize inherited callback recovery contracts"); + const auto baseline = allocations.live; + const auto baselineIds = allocations.ids; + for (const auto& expected : Cases) { + Value replacement{}, assignment{}, format{}, nested{}, outer{}; + _ZN3sce4Json5ValueC1Ev(&replacement); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&replacement, expected.document, std::strlen(expected.document)) == 0, "Construct independent replacement template"); + _ZN3sce4Json5ValueC1Ed(&assignment, 0.125); + _ZN3sce4Json5ValueC1Ed(&format, std::numeric_limits::infinity()); + _ZN3sce4Json5ValueC1Ed(&nested, 0.125); _ZN3sce4Json5ValueC1Ed(&outer, 0.125); + RecoveryContext assignCallback{&replacement, true}, formatCallback{&replacement, true}; + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&assignment, RecoverAllocated, &assignCallback) == 0 && + _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&format, RecoverAllocated, &formatCallback) == 0, "Register assignment and formatting recovery"); + void* assignmentAuxiliary = assignment.auxiliary; + void* formatAuxiliary = format.auxiliary; + const auto assignmentId = allocations.ids.at(assignmentAuxiliary), formatId = allocations.ids.at(formatAuxiliary); + String output{}; _ZN3sce4Json6StringC1EPKc(&output, "p"); + ReadContext context{&allocations, expected.fail, &assignment, &format, &nested, 0, 0, &output}; + const auto first = allocations.requests.size(); const auto notifications = allocations.notifications; + const int status = _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(&outer, ReadNested, &context); + const auto end = allocations.requests.size(); + Require(status == (expected.fail ? static_cast(0x80848102) : 0) && outer.type == 1 && (outer.payload & 0xff) == 1 && context.reads == 4, + "A failure raised by the reader must preserve literal completion while reporting the outer operation failure"); + Require(context.nestedStatus == status && nested.type == (expected.fail ? 4 : 1) && context.nestedReads == (expected.fail ? 0u : 4u), "A nested parse must observe the inherited failure without consuming the outer failure state"); + Require(expected.fail ? nested.payload == std::bit_cast(0.125) : (nested.payload & 0xff) == 1, "Refused nested parsing must preserve its original scalar bits"); + Require(assignment.type == expected.type && Contents(assignment) == expected.assignment && context.formatStatus == 0 && + std::strcmp(_ZNK3sce4Json6String5c_strEv(&output), expected.output) == 0, "Inherited failures must preserve reference callback-return partial collections and serialization"); + std::uint64_t order = UINT64_C(14695981039346656037); + for (auto i = first; i < end; ++i) { order ^= allocations.requests[i]; order *= UINT64_C(1099511628211); } + Require(end - first == expected.requests && order == expected.order && allocations.notifications - notifications == expected.fail, "Nested callback recovery must preserve exact allocation request order and notifications"); + Require(assignCallback.calls == 1 && assignCallback.function == 1 && !assignCallback.errorPointer && formatCallback.calls == 1 && formatCallback.function == 0 && formatCallback.errorPointer, "Nested assignment and serialization must preserve their distinct callback ABIs"); + Require(assignment.auxiliary == assignmentAuxiliary && format.auxiliary == formatAuxiliary && allocations.ids.at(assignmentAuxiliary) == assignmentId && allocations.ids.at(formatAuxiliary) == formatId, "Inherited failures must preserve destination metadata identities and generations"); + Require(Contents(replacement) == (replacement.type == 5 ? "recovery" : expected.document), "Nested recovery must preserve its independent source template"); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&nested, "true", 4) == 0 && nested.type == 1 && (nested.payload & 0xff) == 1, "Outermost completion must clear the inherited failure and permit the next operation"); + _ZN3sce4Json6StringD1Ev(&output); + for (auto* value : {&replacement, &assignment, &format, &nested, &outer}) _ZN3sce4Json5ValueD1Ev(value); + Require(allocations.live == baseline && allocations.ids == baselineIds, "Nested operations must release all temporary ownership without replacing module resources"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && allocations.live.empty() && allocations.ids.empty(), "Release inherited recovery resources"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 6cfdf12a37..e89e48c1dc 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -64,6 +64,6 @@ - [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - nested iterator/metadata failures, non-finite recovery failures, lifecycle reentry, other legacy callback combinations and floating-point environment behavior still require audit. Null callback/output references are explicitly unsupported. All serialization shares the collecting/traversal protocol. Reference tests cover modern chunk boundaries, callback results, output mutation, scalar allocation failures and opening-callback array mutations. SDK 5 tests cover root admission/reset, child metadata release, numeric edge cases and persistent/one-shot root admission failures. -- [Parser parent and auxiliary transitions](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-driven reentry, more complex nested metadata failures and alternate SDK combinations still require audit. Reference tests cover direct parsing into an owned child across eight value forms, six root-admission failure/retry cases, incomplete-literal preservation, modern root callback retention and nested release of three handler records, fallback and auxiliary in native order. Allocation generations distinguish release from later address reuse. +- [Parser parent and auxiliary transitions](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-notification reentry, more complex nested metadata failures and alternate SDK combinations still require audit. Reader-driven allocation failures are checked before starting a value, following `0x5d30`; an already-started literal still completes, while outermost completion reports and clears the shared failure. Reference tests cover direct parsing into an owned child across eight value forms, six root-admission failure/retry cases, incomplete-literal preservation, modern root callback retention and nested release of three handler records, fallback and auxiliary in native order. Allocation generations distinguish release from later address reuse. -- [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - more complex object shapes, inherited failure scopes and alternate SDK combinations remain unverified. Reference tests cover 834 persistent/one-shot cases through successful string, one-entry object and nested array/object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. +- [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - more complex object shapes, allocator-notification reentry and alternate SDK combinations remain unverified. Six inherited-failure cases verify reader-driven failure, nested parse refusal, string/object/nested-array callback returns, outer literal completion, allocation-order fingerprints and subsequent retry. Reference tests cover 834 persistent/one-shot cases through successful string, one-entry object and nested array/object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. From d491273fd3638b7bfc113c16763aad653b832871 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:30:26 +0200 Subject: [PATCH 049/318] fix(libSceJson2): reclaim kernel setup resources on exceptions --- core/libs/prx/libSceJson2/CMakeLists.txt | 9 ++ core/libs/prx/libSceJson2/Export.cpp | 19 ++- .../tests/GuestJsonKernelInitialization.cpp | 148 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 +- 4 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 core/libs/tests/GuestJsonKernelInitialization.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 6e70ab2b7f..738d771fb7 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,15 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_kernel_initialization_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonKernelInitialization.cpp Export.cpp) + target_include_directories(guest_json_kernel_initialization_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_kernel_initialization_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_kernel_initialization_tests) + add_test(NAME guest_json_kernel_initialization COMMAND guest_json_kernel_initialization_tests) + if(WIN32) + set_tests_properties(guest_json_kernel_initialization PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() add_test_executable(guest_json_inherited_recovery_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInheritedRecovery.cpp) target_include_directories(guest_json_inherited_recovery_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_inherited_recovery_tests PRIVATE libSceJson2 libc) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 474ef5f37d..0421959ec4 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1414,12 +1414,19 @@ PthreadMutex* CreateKernelMutex() { if (!mutex) return nullptr; *mutex = nullptr; PthreadMutexattr attribute = nullptr; - if (scePthreadMutexattrInit(&attribute)) { Release(mutex); NotImplemented_nid_no_patch(__func__); } - const auto typeStatus = scePthreadMutexattrSettype(&attribute, 2); - const auto status = typeStatus ? typeStatus : scePthreadMutexInit(mutex, &attribute, "SceJson_internal"); - scePthreadMutexattrDestroy(&attribute); - if (status) { Release(mutex); NotImplemented_nid_no_patch(__func__); } - return mutex; + try { + if (scePthreadMutexattrInit(&attribute)) NotImplemented_nid_no_patch(__func__); + const auto typeStatus = scePthreadMutexattrSettype(&attribute, 2); + const auto status = typeStatus ? typeStatus : scePthreadMutexInit(mutex, &attribute, "SceJson_internal"); + scePthreadMutexattrDestroy(&attribute); + if (status) NotImplemented_nid_no_patch(__func__); + return mutex; + } catch (...) { + if (attribute) scePthreadMutexattrDestroy(&attribute); + if (*mutex) scePthreadMutexDestroy(mutex); + Release(mutex); + throw; + } } void ReleaseInitialization() { diff --git a/core/libs/tests/GuestJsonKernelInitialization.cpp b/core/libs/tests/GuestJsonKernelInitialization.cpp new file mode 100644 index 0000000000..ce7b6d7b1b --- /dev/null +++ b/core/libs/tests/GuestJsonKernelInitialization.cpp @@ -0,0 +1,148 @@ +#include "prx/libkernel/Pthread/include/Mutex.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +static std::uint32_t Sdk = 0x06000000; +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + static std::uint32_t words[]{0, 0, 0, 0, 0}; + words[4] = Sdk; + return words; +} +struct GuestTrace { std::map live; std::size_t generation = 0; }; +static void* APS5_VABI AllocateTracked(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); + Require(trace.live.emplace(memory, ++trace.generation).second, "Guest allocation identities must be independently live"); + return memory; +} +static void APS5_VABI FreeTracked(void*, void* memory, void* context) { + if (!memory) return; + Require(static_cast(context)->live.erase(memory) == 1, "Each guest resource must return to its allocator exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyTracked(void*, int, std::size_t, void*) { + Require(false, "Host kernel failures must not fabricate guest allocator failures"); +} +static void* const Vtable[]{nullptr, nullptr, reinterpret_cast(AllocateTracked), reinterpret_cast(FreeTracked), reinterpret_cast(NotifyTracked)}; +struct Allocator { void* const* vtable = Vtable; }; +enum class Failure { None, AttributeError, AttributeException, TypeError, MutexError, MutexException }; +struct KernelTrace { + Failure failure = Failure::None; + unsigned target = 0; + unsigned attempts = 0; + unsigned attributesCreated = 0; + unsigned attributesDestroyed = 0; + unsigned mutexesCreated = 0; + unsigned mutexesDestroyed = 0; + std::map attributes; + std::map mutexes; +}; +static KernelTrace Kernel; +static bool At(Failure failure) { return Kernel.failure == failure && Kernel.attempts == Kernel.target; } +extern "C" int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attribute) { + ++Kernel.attempts; + if (At(Failure::AttributeError)) return static_cast(0x8002000c); + if (At(Failure::AttributeException)) throw std::bad_alloc(); + *attribute = reinterpret_cast(new int(0)); + Require(Kernel.attributes.emplace(*attribute, 0).second, "Kernel attributes must have independent live identities"); + ++Kernel.attributesCreated; + return 0; +} +extern "C" int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attribute, int type) { + Require(type == 2 && Kernel.attributes.count(*attribute), "Json2 mutexes must request recursive attributes"); + if (At(Failure::TypeError)) return static_cast(0x80020016); + Kernel.attributes.at(*attribute) = type; + return 0; +} +extern "C" int APS5_VABI scePthreadMutexattrDestroy(PthreadMutexattr* attribute) { + Require(Kernel.attributes.erase(*attribute) == 1, "Kernel attributes must be destroyed exactly once"); + delete reinterpret_cast(*attribute); + *attribute = nullptr; + ++Kernel.attributesDestroyed; + return 0; +} +extern "C" int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attribute, const char* name) { + Require(Kernel.attributes.at(*attribute) == 2 && std::strcmp(name, "SceJson_internal") == 0 && !*mutex, "Mutex creation must receive recursive attributes and initialized handle storage"); + if (At(Failure::MutexError)) return static_cast(0x8002000c); + if (At(Failure::MutexException)) throw std::bad_alloc(); + *mutex = reinterpret_cast(new std::recursive_mutex); + Require(Kernel.mutexes.emplace(*mutex, 0).second, "Kernel mutexes must have independent live identities"); + ++Kernel.mutexesCreated; + return 0; +} +extern "C" int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex) { + Require(Kernel.mutexes.count(*mutex) && Kernel.mutexes.at(*mutex) == 0, "Only a live unlocked kernel mutex may be destroyed"); + Kernel.mutexes.erase(*mutex); + delete reinterpret_cast(*mutex); + *mutex = reinterpret_cast(std::uintptr_t{2}); + ++Kernel.mutexesDestroyed; + return 0; +} +extern "C" int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex) { + Require(Kernel.mutexes.count(*mutex), "Only a live kernel mutex may be locked"); + reinterpret_cast(*mutex)->lock(); + ++Kernel.mutexes.at(*mutex); + return 0; +} +extern "C" int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex) { + Require(Kernel.mutexes.count(*mutex) && Kernel.mutexes.at(*mutex), "Kernel unlocking must balance an owned lock"); + --Kernel.mutexes.at(*mutex); + reinterpret_cast(*mutex)->unlock(); + return 0; +} +static void Check(Failure failure, unsigned target) { + Kernel = {}; Kernel.failure = failure; Kernel.target = target; + GuestTrace guest; Allocator allocator; Parameter parameter{&allocator, &guest, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + bool failed = false; + try { + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && failure == Failure::None, "A failed kernel setup must not report successful initialization"); + } catch (const std::bad_alloc&) { + Require(failure == Failure::AttributeException || failure == Failure::MutexException, "Host allocation exceptions must retain their actual type"); + failed = true; + } catch (const std::runtime_error& error) { + Require((failure == Failure::AttributeError || failure == Failure::TypeError || failure == Failure::MutexError) && std::strstr(error.what(), "CreateKernelMutex"), "Unverified kernel status failures must remain explicitly unsupported"); + failed = true; + } + const unsigned mutexCount = Sdk < 0x06000000 ? 1u : 2u; + if (failure == Failure::None) { + Require(!failed && Kernel.mutexes.size() == mutexCount && Kernel.attributes.empty(), "Healthy initialization must own the SDK-specific recursive mutex resources"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate healthy resources"); + } else { + Require(failed && initializer == 0 && Kernel.attempts == target, "Kernel setup failure must leave the initializer inactive at the requested attempt"); + } + Require(guest.live.empty() && Kernel.attributes.empty() && Kernel.mutexes.empty(), "Every kernel setup failure must roll back guest handles, attributes and previously created mutexes"); + Require(Kernel.attributesCreated == Kernel.attributesDestroyed && Kernel.mutexesCreated == Kernel.mutexesDestroyed, "Rollback must balance each kernel resource exactly once"); + const auto generation = guest.generation; + Kernel.failure = Failure::None; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && guest.generation > generation && Kernel.mutexes.size() == mutexCount, "Retry must admit a fresh resource generation using the same initializer"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && guest.live.empty() && Kernel.attributes.empty() && Kernel.mutexes.empty(), "Retry teardown must reclaim complete guest and kernel ownership"); + Require(Kernel.attributesCreated == Kernel.attributesDestroyed && Kernel.mutexesCreated == Kernel.mutexesDestroyed, "Retry must preserve balanced kernel ownership"); +} +int main() { + for (auto sdk : {0x05000000u, 0x06000000u}) { + Sdk = sdk; + Check(Failure::None, 0); + for (auto failure : {Failure::AttributeError, Failure::AttributeException, Failure::TypeError, Failure::MutexError, Failure::MutexException}) + for (unsigned target = 1; target <= (sdk < 0x06000000 ? 1u : 2u); ++target) Check(failure, target); + } + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index e89e48c1dc..6acb1ef025 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -36,7 +36,7 @@ - [sce::Json::Value nested array allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. -- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - kernel setup/locking failures and termination during callbacks remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Initialization at `0x1340`, `0xda0` and `0x940` owns a 24-byte allocator context, default handles/storage, fallback and recursive mutex handles (two for SDK 6, one for SDK 5). Reference tests cover persistent/one-shot allocation failures, partial admission, exact release identities including null frees, repeated termination and retry on the same initializer. Kernel setup succeeds at the substituted boundary. The reference skips mutex creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. +- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - returned kernel setup/locking errors and callback-time termination remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Reference tests cover SDK 5/6 allocation failures, partial admission, exact release identities including null frees, repeated termination and retry. Host-boundary tests exercise 17 healthy/error/exception schedules, including allocation exceptions from either mutex constructor, balanced guest/attribute/mutex ownership and retry on the same initializer. Setup exceptions reclaim local resources before global rollback. The reference skips creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. - [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. From 0b50db7be03a457080a07419d74785432263a701 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:38:07 +0200 Subject: [PATCH 050/318] test(libSceJson2): verify allocator notification reentry ordering --- .../libs/tests/GuestJsonInheritedRecovery.cpp | 42 ++++++++++++++++--- docs/dev/debt/libSceJson2.md | 4 +- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/core/libs/tests/GuestJsonInheritedRecovery.cpp b/core/libs/tests/GuestJsonInheritedRecovery.cpp index 4266ac1d60..72b1827c8f 100644 --- a/core/libs/tests/GuestJsonInheritedRecovery.cpp +++ b/core/libs/tests/GuestJsonInheritedRecovery.cpp @@ -45,6 +45,9 @@ struct RecoveryAllocationTrace { unsigned notifications = 0; std::map ids; std::size_t nextId = 0; + void (*onNotify)(RecoveryAllocationTrace&) = nullptr; + void* notificationContext = nullptr; + bool notifying = false; }; static void* APS5_VABI AllocateRecovery(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); trace.requests.push_back(bytes); @@ -64,6 +67,11 @@ static void APS5_VABI FreeRecovery(void*, void* memory, void* context) { static void APS5_VABI NotifyRecovery(void*, int error, std::size_t bytes, void* context) { auto& trace = *static_cast(context); Require(static_cast(error) == 0x80848102 && trace.requests.back() == bytes, "Recovery notification must identify the failed allocation"); ++trace.notifications; + if (trace.onNotify && !trace.notifying) { + trace.notifying = true; + trace.onNotify(trace); + trace.notifying = false; + } } static void* const RecoveryVtable[]{nullptr, nullptr, reinterpret_cast(AllocateRecovery), reinterpret_cast(FreeRecovery), reinterpret_cast(NotifyRecovery)}; struct RecoveryAllocator { void* const* vtable = RecoveryVtable; }; @@ -98,6 +106,7 @@ struct ReadContext { String* output; unsigned reads = 0; unsigned nestedReads = 0; + unsigned hooks = 0; }; static int APS5_VABI ReadTrue(char& character, void* data) { auto& position = *static_cast(data); @@ -123,6 +132,15 @@ static int APS5_VABI ReadNested(char& character, void* data) { character = "true"[context.reads - 1]; return 0; } +static void RunNotification(RecoveryAllocationTrace& allocations) { + auto& context = *static_cast(allocations.notificationContext); + ++context.hooks; + if (context.fail) { allocations.budget = 0; allocations.once = true; } + context.nestedStatus = _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(context.nested, ReadTrue, &context.nestedReads); + allocations.budget = -1; allocations.once = false; + _ZN3sce4Json5Value3setEd(context.assignment, std::numeric_limits::infinity()); + context.formatStatus = _ZN3sce4Json5Value9serializeERNS0_6StringE(context.format, context.output); +} struct Expected { const char* document; bool fail; @@ -131,6 +149,7 @@ struct Expected { const char* output; std::size_t requests; std::uint64_t order; + bool notification = false; }; static const Expected Cases[]{ {"\"recovery\"", 0, 5, "recovery", "p\"recovery\"", 22, UINT64_C(0x512fbd7009730792)}, @@ -139,6 +158,12 @@ static const Expected Cases[]{ {"{\"k\":\"recovery\"}", 1, 7, "{\"k\":\"\"}", "p{\"k\":\"\"}", 137, UINT64_C(0x1c0f8b9f8e134cd2)}, {"[[\"recovery\"],{\"k\":1}]", 0, 6, "[[\"recovery\"],{\"k\":1}]", "p[[\"recovery\"],{\"k\":1}]", 254, UINT64_C(0xd1da135cb823d2ad)}, {"[[\"recovery\"],{\"k\":1}]", 1, 6, "[[],{}]", "p[[],{}]", 136, UINT64_C(0xe4d4124493072e0a)}, + {"\"recovery\"", 0, 5, "recovery", "p\"recovery\"", 21, UINT64_C(0x1e6ba634008a1e40), true}, + {"\"recovery\"", 1, 5, "recovery", "p\"recovery\"", 20, UINT64_C(0x4f5af7c5c1ffdcb2), true}, + {"{\"k\":\"recovery\"}", 0, 7, "{\"k\":\"recovery\"}", "p{\"k\":\"recovery\"}", 142, UINT64_C(0x1b1dd3ba293b3ae5), true}, + {"{\"k\":\"recovery\"}", 1, 7, "{\"k\":\"\"}", "p{\"k\":\"\"}", 136, UINT64_C(0x2c958fa5f20dbd84), true}, + {"[[\"recovery\"],{\"k\":1}]", 0, 6, "[[\"recovery\"],{\"k\":1}]", "p[[\"recovery\"],{\"k\":1}]", 253, UINT64_C(0x1e7b03d5c679de77), true}, + {"[[\"recovery\"],{\"k\":1}]", 1, 6, "[[],{}]", "p[[],{}]", 135, UINT64_C(0x2d2754b3359f2448), true}, }; int main() { RecoveryAllocationTrace allocations; RecoveryAllocator allocator; @@ -163,17 +188,24 @@ int main() { String output{}; _ZN3sce4Json6StringC1EPKc(&output, "p"); ReadContext context{&allocations, expected.fail, &assignment, &format, &nested, 0, 0, &output}; const auto first = allocations.requests.size(); const auto notifications = allocations.notifications; - const int status = _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(&outer, ReadNested, &context); + int status; + if (expected.notification) { + allocations.onNotify = RunNotification; allocations.notificationContext = &context; + allocations.budget = 0; allocations.once = true; + status = _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&outer, "true", 4); + allocations.onNotify = nullptr; allocations.notificationContext = nullptr; + allocations.budget = -1; allocations.once = false; + } else status = _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(&outer, ReadNested, &context); const auto end = allocations.requests.size(); - Require(status == (expected.fail ? static_cast(0x80848102) : 0) && outer.type == 1 && (outer.payload & 0xff) == 1 && context.reads == 4, + Require(status == (expected.fail || expected.notification ? static_cast(0x80848102) : 0) && outer.type == 1 && (outer.payload & 0xff) == 1 && context.reads == (expected.notification ? 0u : 4u) && context.hooks == (expected.notification ? 1u : 0u), "A failure raised by the reader must preserve literal completion while reporting the outer operation failure"); - Require(context.nestedStatus == status && nested.type == (expected.fail ? 4 : 1) && context.nestedReads == (expected.fail ? 0u : 4u), "A nested parse must observe the inherited failure without consuming the outer failure state"); - Require(expected.fail ? nested.payload == std::bit_cast(0.125) : (nested.payload & 0xff) == 1, "Refused nested parsing must preserve its original scalar bits"); + Require(context.nestedStatus == (expected.fail ? static_cast(0x80848102) : 0) && nested.type == (expected.fail && !expected.notification ? 4 : 1) && context.nestedReads == (expected.fail && !expected.notification ? 0u : 4u), "A nested parse must observe the inherited failure without consuming the outer failure state"); + Require(expected.fail && !expected.notification ? nested.payload == std::bit_cast(0.125) : (nested.payload & 0xff) == 1, "Refused nested parsing must preserve its original scalar bits"); Require(assignment.type == expected.type && Contents(assignment) == expected.assignment && context.formatStatus == 0 && std::strcmp(_ZNK3sce4Json6String5c_strEv(&output), expected.output) == 0, "Inherited failures must preserve reference callback-return partial collections and serialization"); std::uint64_t order = UINT64_C(14695981039346656037); for (auto i = first; i < end; ++i) { order ^= allocations.requests[i]; order *= UINT64_C(1099511628211); } - Require(end - first == expected.requests && order == expected.order && allocations.notifications - notifications == expected.fail, "Nested callback recovery must preserve exact allocation request order and notifications"); + Require(end - first == expected.requests && order == expected.order && allocations.notifications - notifications == unsigned(expected.fail) + unsigned(expected.notification), "Nested callback recovery must preserve exact allocation request order and notifications"); Require(assignCallback.calls == 1 && assignCallback.function == 1 && !assignCallback.errorPointer && formatCallback.calls == 1 && formatCallback.function == 0 && formatCallback.errorPointer, "Nested assignment and serialization must preserve their distinct callback ABIs"); Require(assignment.auxiliary == assignmentAuxiliary && format.auxiliary == formatAuxiliary && allocations.ids.at(assignmentAuxiliary) == assignmentId && allocations.ids.at(formatAuxiliary) == formatId, "Inherited failures must preserve destination metadata identities and generations"); Require(Contents(replacement) == (replacement.type == 5 ? "recovery" : expected.document), "Nested recovery must preserve its independent source template"); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 6acb1ef025..97b87215ce 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -64,6 +64,6 @@ - [Value callback serialization](../../../core/libs/prx/libSceJson2/Export.cpp) - nested iterator/metadata failures, non-finite recovery failures, lifecycle reentry, other legacy callback combinations and floating-point environment behavior still require audit. Null callback/output references are explicitly unsupported. All serialization shares the collecting/traversal protocol. Reference tests cover modern chunk boundaries, callback results, output mutation, scalar allocation failures and opening-callback array mutations. SDK 5 tests cover root admission/reset, child metadata release, numeric edge cases and persistent/one-shot root admission failures. -- [Parser parent and auxiliary transitions](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-notification reentry, more complex nested metadata failures and alternate SDK combinations still require audit. Reader-driven allocation failures are checked before starting a value, following `0x5d30`; an already-started literal still completes, while outermost completion reports and clears the shared failure. Reference tests cover direct parsing into an owned child across eight value forms, six root-admission failure/retry cases, incomplete-literal preservation, modern root callback retention and nested release of three handler records, fallback and auxiliary in native order. Allocation generations distinguish release from later address reuse. +- [Parser parent and auxiliary transitions](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-driven lifecycle changes, more complex nested metadata failures and alternate SDK combinations still require audit. Six notifier-reentry cases cover failure publication after notification, nested failures, callback-return copying and retry. Reader-driven allocation failures are checked before starting a value, following `0x5d30`; an already-started literal still completes, while outermost completion reports and clears the shared failure. Reference tests cover direct parsing into an owned child across eight value forms, six root-admission failure/retry cases, incomplete-literal preservation, modern root callback retention and nested release of three handler records, fallback and auxiliary in native order. Allocation generations distinguish release from later address reuse. -- [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - more complex object shapes, allocator-notification reentry and alternate SDK combinations remain unverified. Six inherited-failure cases verify reader-driven failure, nested parse refusal, string/object/nested-array callback returns, outer literal completion, allocation-order fingerprints and subsequent retry. Reference tests cover 834 persistent/one-shot cases through successful string, one-entry object and nested array/object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. +- [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - more complex object shapes, allocator-driven lifecycle changes and alternate SDK combinations remain unverified. Twelve reader/notifier-reentry cases verify delayed failure publication, inherited nested failures, string/object/nested-array callback returns, literal completion, request-order fingerprints and retry. Reference tests cover 834 persistent/one-shot cases through successful string, one-entry object and nested array/object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. From 12d398ca39a2287257b301ef4a81c0b27600e480 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:46:16 +0200 Subject: [PATCH 051/318] test(libSceJson2): verify initializer ownership during allocator callbacks --- .../tests/GuestJsonInternalInitializer.cpp | 34 +++++++++++++++++-- docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/core/libs/tests/GuestJsonInternalInitializer.cpp b/core/libs/tests/GuestJsonInternalInitializer.cpp index 6eb178844b..d7c6998b66 100644 --- a/core/libs/tests/GuestJsonInternalInitializer.cpp +++ b/core/libs/tests/GuestJsonInternalInitializer.cpp @@ -27,6 +27,7 @@ void APS5_VABI _ZN3sce4Json12MemAllocatorD1Ev(void*); void APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void*); void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const void*); int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); } static void Require(bool valid, const char* message) { @@ -44,15 +45,36 @@ struct EntryTrace { int notifiedState = 0; int firstRequestState = 0; int laterRequestState = 0; + void* owner = nullptr; + unsigned allocationReentries = 0; + unsigned notificationReentries = 0; + unsigned releaseReentries = 0; }; +static void CheckReentry(EntryTrace& trace) { + std::uint8_t other[]{0, 0x19, 0x28, 0x37}; + const auto active = *static_cast(trace.owner); + constexpr auto already = static_cast(0x80848111); + constexpr auto missing = static_cast(0x80848110); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(trace.owner, nullptr) == already && + _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(trace.owner, nullptr) == already, + "Allocator callback reentry must reject both duplicate initialization forms before parameter validation"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(other, nullptr) == already && + _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(other, nullptr) == already, + "An allocator callback must not publish a competing initializer while the module context is owned"); + Require(_ZN3sce4Json11Initializer9terminateEv(other) == missing && other[0] == 0 && other[1] == 0x19 && other[2] == 0x28 && other[3] == 0x37, + "An inactive callback-local initializer must not terminate another initializer's resources"); + Require(*static_cast(trace.owner) == active, "Rejected callback lifecycle operations must preserve the owning initializer active byte"); +} static void APS5_VABI Info(int, std::int32_t, void*) { Require(false, "The reference allocator-info setter must not install or invoke the supplied callback"); } static int InfoState() { return _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9ValueTypeEPvES3_(nullptr, Info, nullptr); } static void* APS5_VABI EntryAllocate(void* self, std::size_t size, void* context) { auto& trace = *static_cast(context); Require(self == trace.expectedAllocator, "Allocation dispatch must preserve the allocator receiver and user context"); - if (trace.requests.empty()) trace.firstRequestState = InfoState(); - else trace.laterRequestState = InfoState(); + const int admission = InfoState(); + if (trace.requests.empty()) trace.firstRequestState = admission; + else trace.laterRequestState = admission; trace.requests.push_back(size); + if (trace.owner && admission == 0) { CheckReentry(trace); ++trace.allocationReentries; } if (trace.reject || trace.budget == 0) return nullptr; if (trace.budget > 0) --trace.budget; void* pointer = std::malloc(size ? size : 1); @@ -63,12 +85,14 @@ static void APS5_VABI EntryFree(void* self, void* memory, void* context) { auto& trace = *static_cast(context); Require(self == trace.expectedAllocator, "Free dispatch must preserve the allocator receiver and user context"); trace.frees.push_back(memory); + if (trace.owner) { CheckReentry(trace); ++trace.releaseReentries; } if (memory) { Require(trace.live.erase(memory) == 1, "Every allocation must be released exactly once through its allocator"); std::free(memory); } } static void APS5_VABI EntryNotify(void* self, int error, std::size_t size, void* context) { auto& trace = *static_cast(context); Require(self == trace.expectedAllocator && static_cast(error) == 0x80848102, "Allocation failure must notify the owning allocator with the Json2 error"); ++trace.notices; trace.failedSize = size; trace.notifiedState = InfoState(); + if (trace.owner) { CheckReentry(trace); ++trace.notificationReentries; } } int main() { constexpr auto already = static_cast(0x80848111); @@ -77,6 +101,7 @@ int main() { constexpr auto noMemory = static_cast(0x80848102); void* table[]{nullptr, nullptr, reinterpret_cast(&EntryAllocate), reinterpret_cast(&EntryFree), reinterpret_cast(&EntryNotify)}; void** allocator = table; + unsigned allocationReentries = 0, notificationReentries = 0, releaseReentries = 0; for (std::int32_t option : {0, 1, 2, -1}) { for (int budget = 0; budget <= 12; ++budget) { EntryTrace trace; trace.expectedAllocator = &allocator; trace.budget = budget; @@ -87,6 +112,7 @@ int main() { Require(object[0] == 0 && object[1] == 0x19 && object[2] == 0x28 && object[3] == 0x37, "Internal constructors must initialize only the active byte"); Require(_ZN3sce4Json19InternalInitializer19terminate_staticlibEv(object) == missing, "Inactive internal termination must return not-initialized"); Require(_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(object, nullptr, option) == invalid, "Internal initialization must reject a missing parameter"); + trace.owner = object; Parameter empty{}; Require(_ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(object, &empty, option) == noMemory, "Legacy internal initialization must report a missing allocator as out-of-memory"); const int result = _ZN3sce4Json19InternalInitializer20initialize_staticlibEPKNS0_13InitParameterENS1_10InitOptionE(object, ¶meter, option); @@ -120,7 +146,11 @@ int main() { _ZN3sce4Json19InternalInitializerD1Ev(object); _ZN3sce4Json19InternalInitializerD2Ev(object); Require(trace.frees.size() == frees && InfoState() == missing, "Repeated internal destruction must not dispatch stale frees"); + allocationReentries += trace.allocationReentries; + notificationReentries += trace.notificationReentries; + releaseReentries += trace.releaseReentries; } } + Require(allocationReentries && notificationReentries && releaseReentries, "Lifecycle rejection must be exercised through actual allocate, notify and free callbacks"); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 97b87215ce..2b27334b39 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -36,7 +36,7 @@ - [sce::Json::Value nested array allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. -- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - returned kernel setup/locking errors and callback-time termination remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Reference tests cover SDK 5/6 allocation failures, partial admission, exact release identities including null frees, repeated termination and retry. Host-boundary tests exercise 17 healthy/error/exception schedules, including allocation exceptions from either mutex constructor, balanced guest/attribute/mutex ownership and retry on the same initializer. Setup exceptions reclaim local resources before global rollback. The reference skips creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. +- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - returned kernel setup/locking errors and callback-time termination remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Reference tests cover SDK 5/6 allocation failures, partial admission, null frees and retry; SDK 6 tests also reject duplicate initialization and inactive termination from allocate/notify/free callbacks while the context remains published. Host-boundary tests exercise 17 healthy/error/exception schedules, including allocation exceptions from either mutex constructor, balanced guest/attribute/mutex ownership and retry on the same initializer. Setup exceptions reclaim local resources before global rollback. The reference skips creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. - [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. From 878268c30091f84a7673a03d33397e2f8960b424 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:50:25 +0200 Subject: [PATCH 052/318] fix(libSceJson2): preserve inactive object pair bytes --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 20 ++--- core/libs/tests/GuestJson.cpp | 2 +- core/libs/tests/GuestJsonPairLayout.cpp | 100 +++++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 +- 5 files changed, 114 insertions(+), 17 deletions(-) create mode 100644 core/libs/tests/GuestJsonPairLayout.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 738d771fb7..d9c11cc24c 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_pair_layout_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonPairLayout.cpp) + target_include_directories(guest_json_pair_layout_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_pair_layout_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_pair_layout_tests) + add_test(NAME guest_json_pair_layout COMMAND guest_json_pair_layout_tests) add_test_executable(guest_json_kernel_initialization_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonKernelInitialization.cpp Export.cpp) target_include_directories(guest_json_kernel_initialization_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_kernel_initialization_tests PRIVATE libkernel libc) @@ -180,7 +185,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 0421959ec4..f47a54c1d1 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -3224,17 +3224,13 @@ bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value* self) { } void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair* self, const String* key, const Value* value) { - Pair pair{}; - pair.reserved = 0; - Construct(pair.key, *key); + Construct(self->key, *key); try { - CopyInto(pair.value, *value); + CopyInto(self->value, *value); } catch (...) { - Destroy(pair.key); + Destroy(self->key); throw; } - *self = pair; - BindChildren(self->value); } void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair* self) { @@ -3317,17 +3313,13 @@ void APS5_VABI _ZN3sce4Json6Object8iteratorC1Ev(ObjectIterator* self) { } void APS5_VABI _ZN3sce4Json6Object4PairC1Ev(Pair* self) { - Pair pair{}; - pair.reserved = 0; - Construct(pair.key, std::string()); + Construct(self->key, std::string_view{}); try { - Construct(pair.value); + Construct(self->value); } catch (...) { - Destroy(pair.key); + Destroy(self->key); throw; } - *self = pair; - BindChildren(self->value); } void APS5_VABI _ZN3sce4Json11InitializerC2Ev(void* self) { _ZN3sce4Json11InitializerC1Ev(self); } diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index 05b6b86c4d..db3130ac87 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -389,7 +389,7 @@ static void ObjectPair() { Pair pair{}; std::memset(&pair, 0x5a, sizeof(pair)); _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &value); - Require(pair.reserved == 0); + Require(pair.reserved == UINT64_C(0x5a5a5a5a5a5a5a5a)); Require(pair.key.text != key.text && _ZNK3sce4Json5Value9getStringEv(&pair.value) != _ZNK3sce4Json5Value9getStringEv(&value)); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), "answer") == 0); Require(_ZNK3sce4Json5Value7getTypeEv(&pair.value) == TypeString); diff --git a/core/libs/tests/GuestJsonPairLayout.cpp b/core/libs/tests/GuestJsonPairLayout.cpp new file mode 100644 index 0000000000..8cc17e3aa8 --- /dev/null +++ b/core/libs/tests/GuestJsonPairLayout.cpp @@ -0,0 +1,100 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +void APS5_VABI _ZN3sce4Json6Object4PairC1Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object4PairC2Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair*, const String*, const Value*); +void APS5_VABI _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(Pair*, const String*, const Value*); +void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object4PairD2Ev(Pair*); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct PairTrace { std::map live; std::size_t generation = 0; }; +static void* APS5_VABI AllocatePair(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); + Require(trace.live.emplace(memory, ++trace.generation).second, "Pair storage must have an independently owned allocation generation"); + return memory; +} +static void APS5_VABI FreePair(void*, void* memory, void* context) { + if (!memory) return; + Require(static_cast(context)->live.erase(memory) == 1, "Pair teardown must release each allocation through its owner exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyPair(void*, int, std::size_t, void*) { Require(false, "Healthy pair construction must not fail allocation"); } +static void* const PairVtable[]{nullptr, nullptr, reinterpret_cast(AllocatePair), reinterpret_cast(FreePair), reinterpret_cast(NotifyPair)}; +struct PairAllocator { void* const* vtable = PairVtable; }; +static std::string Serialize(Value& value) { + String output{}; _ZN3sce4Json6StringC1EPKc(&output, ""); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output) == 0, "Constructed pair values must remain serializable"); + std::string text = _ZNK3sce4Json6String5c_strEv(&output); + _ZN3sce4Json6StringD1Ev(&output); + return text; +} +static void CheckMetadata(const Pair& pair, std::uint64_t pattern) { + Require(pair.reserved == pattern, "Pair construction must preserve the eight inactive bytes between key and value"); + Require(!pair.value.parent && !pair.value.auxiliary && pair.value.reserved == static_cast(pattern), "Pair value construction must initialize ownership without overwriting inactive reserved bytes"); +} +int main() { + PairTrace trace; PairAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize pair layout contracts"); + const auto baseline = trace.live; + for (unsigned char poison : {0x5a, 0xa5}) for (bool base : {false, true}) { + const auto pattern = UINT64_C(0x0101010101010101) * poison; + Pair empty; std::memset(&empty, poison, sizeof(empty)); + if (base) _ZN3sce4Json6Object4PairC2Ev(&empty); + else _ZN3sce4Json6Object4PairC1Ev(&empty); + CheckMetadata(empty, pattern); + Require(empty.key.storage && !_ZNK3sce4Json6String5c_strEv(&empty.key)[0] && empty.value.type == 0 && empty.value.payload == pattern, "Default pairs must construct an empty key and null value while preserving inactive payload bytes"); + if (base) _ZN3sce4Json6Object4PairD2Ev(&empty); + else _ZN3sce4Json6Object4PairD1Ev(&empty); + Require(trace.live == baseline, "Default pair destruction must preserve module allocation identities"); + for (const char* document : {"null", "false", "true", "42", "-7", "1.25", "\"x\"", "[1]", "{\"a\":1}"}) { + String key{}; _ZN3sce4Json6StringC1EPKc(&key, "key"); + Value source{}; _ZN3sce4Json5ValueC1Ev(&source); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&source, document, std::strlen(document)) == 0, "Construct the independently owned source value"); + Pair copy; std::memset(©, poison, sizeof(copy)); + if (base) _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(©, &key, &source); + else _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(©, &key, &source); + CheckMetadata(copy, pattern); + Require(copy.key.storage != key.storage && std::strcmp(_ZNK3sce4Json6String5c_strEv(©.key), "key") == 0 && copy.value.type == source.type, "Pair construction must independently own its key and copy the exact source type"); + if (source.type == 0) Require(copy.value.payload == pattern, "Null pair values must preserve inactive payload bytes"); + else if (source.type == 1) Require(copy.value.payload == ((pattern & ~UINT64_C(0xff)) | (source.payload & 0xff)), "Boolean pair values must write only their active payload byte"); + else if (source.type <= 4) Require(copy.value.payload == source.payload, "Numeric pair values must retain exact source payload bits"); + else Require(copy.value.payload != source.payload, "Complex pair values must own independent outer handles"); + Require(Serialize(copy.value) == document && Serialize(source) == document, "Pair copies must preserve source and destination contents independently"); + if (base) _ZN3sce4Json6Object4PairD2Ev(©); + else _ZN3sce4Json6Object4PairD1Ev(©); + _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json6StringD1Ev(&key); + Require(trace.live == baseline, "Pair copies must release their complete independent ownership"); + } + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Release pair layout resources"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 2b27334b39..56b980ba5b 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -8,7 +8,7 @@ - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter is now reproduced through the inspected float32 conversion, six truncating fractional digits, range saturation and NaN integer result. The regression covers signed zero, precision boundaries, large values, infinities and NaNs; floating-point rounding modes and exception flags remain unverified. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. -- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - `Pair(const String&, const Value&)` and default `Pair()` zero the 8 bytes between the key and value; what the console keeps there is unknown. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. +- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocation failures and nested child binding still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - libc-heap failures and untested legacy callback combinations remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Additional legacy compatibility remains unverified. - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback-value failure behavior and compatibility below SDK 6 remain incomplete. From 4ef2ff175a03f9f7316304c5a986f2cd2132d891 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 08:59:15 +0200 Subject: [PATCH 053/318] fix(libSceJson2): preserve pair and failed copy release order --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 10 +- core/libs/tests/GuestJsonPairAllocator.cpp | 519 +++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 533 insertions(+), 7 deletions(-) create mode 100644 core/libs/tests/GuestJsonPairAllocator.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index d9c11cc24c..dbb990d34b 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_pair_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonPairAllocator.cpp) + target_include_directories(guest_json_pair_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_pair_allocator_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_pair_allocator_tests) + add_test(NAME guest_json_pair_allocator COMMAND guest_json_pair_allocator_tests) add_test_executable(guest_json_pair_layout_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonPairLayout.cpp) target_include_directories(guest_json_pair_layout_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_pair_layout_tests PRIVATE libSceJson2 libc) @@ -185,7 +190,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_pair_allocator guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index f47a54c1d1..21a2c1fb6a 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -992,9 +992,9 @@ void ReleaseLegacyHandlers() { state.legacyHandlers = nullptr; } -void Construct(Object& object) { +void Construct(Object& object, bool releaseMissingHeader = false) { object.items = static_cast(Allocate(ModernValues() ? 32 : 24)); - if (!object.items) return; + if (!object.items) { if (releaseMissingHeader) Release(nullptr); return; } object.items->count = 0; object.items->head = NewObjectHead(); object.items->SetOwner(nullptr); @@ -1848,7 +1848,7 @@ void AssignArray(Array& destination, const Array& source) { } Array pending{}; Construct(pending); - if (!pending.items) return; + if (!pending.items) { Release(nullptr); return; } if (!pending.items->head) { Destroy(pending); return; } pending.items->SetOwner(destination.items->Owner()); try { @@ -1882,7 +1882,7 @@ void AssignObject(Object& destination, const Object& source) { return; } Object pending{}; - Construct(pending); + Construct(pending, true); if (!pending.items) return; pending.items->SetOwner(destination.items->Owner()); try { @@ -3234,8 +3234,8 @@ void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair* self } void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair* self) { - Destroy(self->key); Destroy(self->value); + Destroy(self->key); } ObjectIterator* APS5_VABI _ZN3sce4Json6Object6insertERKNS1_4PairE(ObjectIterator* result, Object* self, const Pair* pair) { diff --git a/core/libs/tests/GuestJsonPairAllocator.cpp b/core/libs/tests/GuestJsonPairAllocator.cpp new file mode 100644 index 0000000000..3a6234947f --- /dev/null +++ b/core/libs/tests/GuestJsonPairAllocator.cpp @@ -0,0 +1,519 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +void APS5_VABI _ZN3sce4Json6Object4PairC1Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object4PairC2Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair*, const String*, const Value*); +void APS5_VABI _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(Pair*, const String*, const Value*); +void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object4PairD2Ev(Pair*); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct PairTrace { + std::map live; + std::size_t generation = 0; + std::vector requests; + std::vector releases; + int budget = -1; + bool once = false; + unsigned notices = 0; +}; +static void* APS5_VABI AllocatePair(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); + Require(trace.live.emplace(memory, ++trace.generation).second, "Pair storage must have an independently owned allocation generation"); + return memory; +} +static void APS5_VABI FreePair(void*, void* memory, void* context) { + auto& trace = *static_cast(context); + trace.releases.push_back(memory ? trace.live.at(memory) : 0); + if (!memory) return; + Require(trace.live.erase(memory) == 1, "Pair teardown must release each allocation through its owner exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyPair(void*, int error, std::size_t size, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == size, "Pair notification must identify the failed allocation request"); + ++trace.notices; +} +static void* const PairVtable[]{nullptr, nullptr, reinterpret_cast(AllocatePair), reinterpret_cast(FreePair), reinterpret_cast(NotifyPair)}; +struct PairAllocator { void* const* vtable = PairVtable; }; +static std::string Serialize(Value& value) { + String output{}; _ZN3sce4Json6StringC1EPKc(&output, ""); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output) == 0, "Constructed pair values must remain serializable"); + std::string text = _ZNK3sce4Json6String5c_strEv(&output); + _ZN3sce4Json6StringD1Ev(&output); + return text; +} +static void CheckMetadata(const Pair& pair, std::uint64_t pattern) { + Require(pair.reserved == pattern, "Pair construction must preserve the eight inactive bytes between key and value"); + Require(!pair.value.parent && !pair.value.auxiliary && pair.value.reserved == static_cast(pattern), "Pair value construction must initialize ownership without overwriting inactive reserved bytes"); +} +struct InspectArrayLink { InspectArrayLink* previous; InspectArrayLink* next; Value value; }; +struct InspectArray { InspectArrayLink* head; std::size_t count; std::uint64_t reserved; Value* owner; }; +struct InspectObjectLink { InspectObjectLink* parent; InspectObjectLink* left; InspectObjectLink* right; unsigned char key[24]; Pair pair; std::uint8_t color; std::uint8_t nil; unsigned char reserved[6]; }; +struct InspectObject { InspectObjectLink* head; std::size_t count; std::uint64_t reserved; Value* owner; }; +static_assert(sizeof(InspectArrayLink) == 48 && offsetof(InspectArrayLink, value) == 16); +static_assert(sizeof(InspectObjectLink) == 104 && offsetof(InspectObjectLink, nil) == 97 && offsetof(InspectObjectLink, pair) == 48); +static void Shape(Value& value, const Value* parent, std::string& output); +static int Identity(const Value* actual, const Value* expected) { return actual ? (actual == expected ? 1 : 2) : 0; } +static void TreeShape(InspectObjectLink* node, const Value* parent, std::string& output) { + if (node->nil) return; + TreeShape(node->left, parent, output); + Shape(node->pair.value, parent, output); + TreeShape(node->right, parent, output); +} +static void Shape(Value& value, const Value* parent, std::string& output) { + output += std::to_string(value.type) + ":" + std::to_string(Identity(value.parent, parent)); + if (value.type == 6) { + auto* header = *reinterpret_cast(value.payload); + if (!header) { output += "x"; return; } + output += "a" + std::to_string(header->count) + ":" + std::to_string(Identity(header->owner, &value)) + "["; + if (!header->head) { output += "missing]"; return; } + for (auto* node = header->head->next; node != header->head; node = node->next) Shape(node->value, &value, output); + output += "]"; + } + if (value.type == 7) { + auto* header = *reinterpret_cast(value.payload); + if (!header) { output += "x"; return; } + output += "o" + std::to_string(header->count) + ":" + std::to_string(Identity(header->owner, &value)) + "{"; + if (!header->head) { output += "missing}"; return; } + TreeShape(header->head->parent, &value, output); + output += "}"; + } + output += ";"; +} +static std::uint64_t Fingerprint(const std::vector& values, std::size_t first, std::size_t end, std::size_t generation = 0) { + std::uint64_t hash = UINT64_C(14695981039346656037); + for (auto i = first; i < end; ++i) { + hash ^= values[i] ? values[i] - generation : 0; + hash *= UINT64_C(1099511628211); + } + return hash; +} +struct Expected { + unsigned source; + bool once; + int budget; + int type; + const char* key; + const char* text; + const char* shape; + unsigned notifications; + std::size_t requests; + std::uint64_t requestOrder; + std::size_t constructionReleases; + std::uint64_t constructionOrder; + std::size_t destructionReleases; + std::uint64_t destructionOrder; +}; +static const Expected Cases[]{ + {0, 0, 0, 0, "", "null", "0:0;", 1, 1, UINT64_C(0xaf63d54c8601e0a7), 0, UINT64_C(0xcbf29ce484222325), 0, UINT64_C(0xcbf29ce484222325)}, + {0, 0, 1, 0, "", "null", "0:0;", 0, 1, UINT64_C(0xaf63d54c8601e0a7), 0, UINT64_C(0xcbf29ce484222325), 1, UINT64_C(0xaf63bc4c8601b62c)}, + {0, 1, 0, 0, "", "null", "0:0;", 1, 1, UINT64_C(0xaf63d54c8601e0a7), 0, UINT64_C(0xcbf29ce484222325), 0, UINT64_C(0xcbf29ce484222325)}, + {0, 1, 1, 0, "", "null", "0:0;", 0, 1, UINT64_C(0xaf63d54c8601e0a7), 0, UINT64_C(0xcbf29ce484222325), 1, UINT64_C(0xaf63bc4c8601b62c)}, + {1, 0, 0, 0, "", "null", "0:0;", 1, 1, UINT64_C(0xaf63d54c8601e0a7), 0, UINT64_C(0xcbf29ce484222325), 0, UINT64_C(0xcbf29ce484222325)}, + {1, 0, 1, 0, "", "null", "0:0;", 1, 2, UINT64_C(0x08841407b530b4f9), 0, UINT64_C(0xcbf29ce484222325), 1, UINT64_C(0xaf63bc4c8601b62c)}, + {1, 0, 2, 0, "key", "null", "0:0;", 0, 2, UINT64_C(0x08841407b530b4f9), 0, UINT64_C(0xcbf29ce484222325), 2, UINT64_C(0x08395307b4f1348c)}, + {1, 1, 0, 0, "", "null", "0:0;", 1, 1, UINT64_C(0xaf63d54c8601e0a7), 0, UINT64_C(0xcbf29ce484222325), 0, UINT64_C(0xcbf29ce484222325)}, + {1, 1, 1, 0, "", "null", "0:0;", 1, 2, UINT64_C(0x08841407b530b4f9), 0, UINT64_C(0xcbf29ce484222325), 1, UINT64_C(0xaf63bc4c8601b62c)}, + {1, 1, 2, 0, "key", "null", "0:0;", 0, 2, UINT64_C(0x08841407b530b4f9), 0, UINT64_C(0xcbf29ce484222325), 2, UINT64_C(0x08395307b4f1348c)}, + {2, 0, 0, 0, "", "null", "0:0;", 2, 2, UINT64_C(0x08842007b530c95d), 0, UINT64_C(0xcbf29ce484222325), 0, UINT64_C(0xcbf29ce484222325)}, + {2, 0, 1, 0, "", "null", "0:0;", 2, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 1, UINT64_C(0xaf63bc4c8601b62c)}, + {2, 0, 2, 0, "key", "null", "0:0;", 1, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 2, UINT64_C(0x08395307b4f1348c)}, + {2, 0, 3, 0, "key", "null", "0:0;", 1, 4, UINT64_C(0x29e493479f20d661), 1, UINT64_C(0xaf63be4c8601b992), 2, UINT64_C(0x08395307b4f1348c)}, + {2, 0, 4, 5, "key", "\"\"", "5:0;", 1, 5, UINT64_C(0x503caab364cc52b8), 0, UINT64_C(0xcbf29ce484222325), 4, UINT64_C(0xb345225e3644edb5)}, + {2, 0, 5, 5, "key", "\"recovery\"", "5:0;", 0, 5, UINT64_C(0x503caab364cc52b8), 0, UINT64_C(0xcbf29ce484222325), 5, UINT64_C(0x3f3fcc261a338844)}, + {2, 1, 0, 5, "", "\"recovery\"", "5:0;", 1, 4, UINT64_C(0x919fd647d9d25bc2), 0, UINT64_C(0xcbf29ce484222325), 3, UINT64_C(0xe1efc21870f169c3)}, + {2, 1, 1, 5, "", "\"recovery\"", "5:0;", 1, 5, UINT64_C(0x503caab364cc52b8), 0, UINT64_C(0xcbf29ce484222325), 4, UINT64_C(0xb345225e3644edb5)}, + {2, 1, 2, 0, "key", "null", "0:0;", 1, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 2, UINT64_C(0x08395307b4f1348c)}, + {2, 1, 3, 0, "key", "null", "0:0;", 1, 4, UINT64_C(0x29e493479f20d661), 1, UINT64_C(0xaf63be4c8601b992), 2, UINT64_C(0x08395307b4f1348c)}, + {2, 1, 4, 5, "key", "\"\"", "5:0;", 1, 5, UINT64_C(0x503caab364cc52b8), 0, UINT64_C(0xcbf29ce484222325), 4, UINT64_C(0xb345225e3644edb5)}, + {2, 1, 5, 5, "key", "\"recovery\"", "5:0;", 0, 5, UINT64_C(0x503caab364cc52b8), 0, UINT64_C(0xcbf29ce484222325), 5, UINT64_C(0x3f3fcc261a338844)}, + {3, 0, 0, 0, "", "null", "0:0;", 2, 2, UINT64_C(0x08842007b530c95d), 0, UINT64_C(0xcbf29ce484222325), 0, UINT64_C(0xcbf29ce484222325)}, + {3, 0, 1, 0, "", "null", "0:0;", 2, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 1, UINT64_C(0xaf63bc4c8601b62c)}, + {3, 0, 2, 0, "key", "null", "0:0;", 1, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 2, UINT64_C(0x08395307b4f1348c)}, + {3, 0, 3, 0, "key", "null", "0:0;", 1, 4, UINT64_C(0x29e49b479f20e3f9), 1, UINT64_C(0xaf63be4c8601b992), 2, UINT64_C(0x08395307b4f1348c)}, + {3, 0, 4, 0, "key", "null", "0:0;", 1, 5, UINT64_C(0x50576bb364e2af63), 2, UINT64_C(0x0824ed07b4dfde30), 2, UINT64_C(0x08395307b4f1348c)}, + {3, 0, 5, 7, "key", "{}", "7:0o0:1{};", 1, 6, UINT64_C(0x673b44d46d2fced9), 1, UINT64_C(0xaf63bd4c8601b7df), 5, UINT64_C(0x3f3fcc261a338844)}, + {3, 0, 6, 7, "key", "{}", "7:0o0:1{};", 1, 7, UINT64_C(0x9984a5f5883c36c3), 1, UINT64_C(0xaf63bb4c8601b479), 5, UINT64_C(0x3f3fcc261a338844)}, + {3, 0, 7, 7, "key", "{}", "7:0o0:1{};", 12, 19, UINT64_C(0x57b5704e320be159), 2, UINT64_C(0x08218e07b4dd06ec), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 8, 7, "key", "{}", "7:0o0:1{};", 12, 20, UINT64_C(0xf367fb55d54c9203), 3, UINT64_C(0x1e87e8189340ae76), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 9, 7, "key", "{}", "7:0o0:1{};", 13, 22, UINT64_C(0xf5aab40101cc37d3), 4, UINT64_C(0xf241d4ba207d8e95), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 10, 7, "key", "{}", "7:0o0:1{};", 12, 22, UINT64_C(0xf5aab40101cc37d3), 5, UINT64_C(0x7e6b1d36db19d80d), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 11, 7, "key", "{}", "7:0o0:1{};", 13, 24, UINT64_C(0xfe76783938049f49), 6, UINT64_C(0x74e8056682af8fd0), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 12, 7, "key", "{}", "7:0o0:1{};", 12, 24, UINT64_C(0xfe76783938049f49), 7, UINT64_C(0x7693bb7c7546b046), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 13, 7, "key", "{}", "7:0o0:1{};", 13, 26, UINT64_C(0xde319778ed12258b), 8, UINT64_C(0x4468dd3409107207), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 14, 7, "key", "{}", "7:0o0:1{};", 12, 26, UINT64_C(0xde319778ed12258b), 9, UINT64_C(0xad55aca26461f6ad), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 15, 7, "key", "{}", "7:0o0:1{};", 12, 27, UINT64_C(0xc7ec986f14c73639), 10, UINT64_C(0x4235c494c06acdf8), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 16, 7, "key", "{}", "7:0o0:1{};", 14, 30, UINT64_C(0x8a81bd5371368fab), 12, UINT64_C(0x57e8bc377377a1a8), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 17, 7, "key", "{}", "7:0o0:1{};", 13, 30, UINT64_C(0x8a81bd5371368fab), 13, UINT64_C(0x1268d2826954245d), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 18, 7, "key", "{}", "7:0o0:1{};", 13, 31, UINT64_C(0x96301b105a1ce429), 13, UINT64_C(0xf35671decbb938c3), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 19, 7, "key", "{}", "7:0o0:1{};", 12, 31, UINT64_C(0x96301b105a1ce429), 14, UINT64_C(0x0cab5608e565adf6), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 20, 7, "key", "{}", "7:0o0:1{};", 12, 32, UINT64_C(0x3bc91ee93ed610f9), 15, UINT64_C(0x794fa5db173e30f0), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 21, 7, "key", "{}", "7:0o0:1{};", 11, 32, UINT64_C(0x3bc91ee93ed610f9), 16, UINT64_C(0x41fe94923964f0cb), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 22, 7, "key", "{}", "7:0o0:1{};", 11, 33, UINT64_C(0x8b101e49bc1d3b51), 17, UINT64_C(0xbbeef34e524f8b27), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 23, 7, "key", "{}", "7:0o0:1{};", 10, 33, UINT64_C(0x8b101e49bc1d3b51), 18, UINT64_C(0x655747e04dd5b78a), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 24, 7, "key", "{}", "7:0o0:1{};", 10, 34, UINT64_C(0x4670bb37ce1cd743), 19, UINT64_C(0x575dab813e787668), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 25, 7, "key", "{}", "7:0o0:1{};", 11, 36, UINT64_C(0xbdb571159b86105b), 21, UINT64_C(0xad7895c02a7368e9), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 26, 7, "key", "{}", "7:0o0:1{};", 10, 36, UINT64_C(0xbdb571159b86105b), 22, UINT64_C(0xca7465d28bfd5461), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 27, 7, "key", "{}", "7:0o0:1{};", 10, 37, UINT64_C(0x44d2b5c68fb25c01), 22, UINT64_C(0xad57f867f29b0856), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 28, 7, "key", "{}", "7:0o0:1{};", 9, 37, UINT64_C(0x44d2b5c68fb25c01), 23, UINT64_C(0xa4f6f9a06ded132c), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {3, 0, 29, 7, "key", "{\"\":null}", "7:0o1:1{0:0;};", 9, 38, UINT64_C(0x98134cae02268541), 23, UINT64_C(0xa4f6f9a06ded132c), 6, UINT64_C(0x5883fada9831d117)}, + {3, 0, 30, 7, "key", "{\"\":null}", "7:0o1:1{0:0;};", 10, 40, UINT64_C(0xef7eec8809648b01), 23, UINT64_C(0xa4f6f9a06ded132c), 7, UINT64_C(0x6c521e5e6410ac8b)}, + {3, 0, 31, 7, "key", "{\"\":null}", "7:0o1:1{0:0;};", 10, 41, UINT64_C(0xe9e59080116a59f1), 23, UINT64_C(0xa4f6f9a06ded132c), 8, UINT64_C(0x86a3a0747ab65d86)}, + {3, 0, 32, 7, "key", "{\"a\":null}", "7:0o1:1{0:0;};", 9, 41, UINT64_C(0xe9e59080116a59f1), 23, UINT64_C(0xa4f6f9a06ded132c), 9, UINT64_C(0xacf3512bbda582e0)}, + {3, 0, 33, 7, "key", "{\"a\":null}", "7:0o1:1{0:0;};", 9, 42, UINT64_C(0xeb3e6815095e9583), 24, UINT64_C(0x7e7974b5e6a575d5), 9, UINT64_C(0xacf3512bbda582e0)}, + {3, 0, 34, 7, "key", "{\"a\":[]}", "7:0o1:1{6:0a0:1[missing]};", 10, 44, UINT64_C(0xfbc65d9e64e2dd2b), 24, UINT64_C(0x0d47adecfbfc430c), 11, UINT64_C(0xf238d2bde87a012f)}, + {3, 0, 35, 7, "key", "{\"a\":[]}", "7:0o1:1{6:0a0:1[];};", 9, 44, UINT64_C(0xfbc65d9e64e2dd2b), 24, UINT64_C(0x0d47adecfbfc430c), 12, UINT64_C(0xaf25e2b8d279d422)}, + {3, 0, 36, 7, "key", "{\"a\":[]}", "7:0o1:1{6:0a0:1[];};", 9, 45, UINT64_C(0x62622be92d64ec21), 24, UINT64_C(0x88b696849119ed00), 12, UINT64_C(0xaf25e2b8d279d422)}, + {3, 0, 37, 7, "key", "{\"a\":[]}", "7:0o1:1{6:1a0:1[];};", 9, 46, UINT64_C(0xd5087cc731249313), 26, UINT64_C(0x945498db768529bf), 12, UINT64_C(0x1402fa0bcc4f6376)}, + {3, 0, 38, 7, "key", "{\"a\":[]}", "7:0o1:1{6:1a0:1[];};", 9, 47, UINT64_C(0xdb649185d9c21c79), 26, UINT64_C(0xf414310c6d6a89e5), 12, UINT64_C(0x1402fa0bcc4f6376)}, + {3, 0, 39, 7, "key", "{\"a\":[]}", "7:0o1:1{6:1a0:1[];};", 9, 48, UINT64_C(0x82d36a62e335fbeb), 27, UINT64_C(0x02643bbfdf0d87ea), 12, UINT64_C(0x11ebb0aab788d892)}, + {3, 0, 40, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 8, 48, UINT64_C(0x82d36a62e335fbeb), 27, UINT64_C(0x02643bbfdf0d87ea), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 41, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 8, 49, UINT64_C(0xb283cbd1b5b5f209), 28, UINT64_C(0xa0a087072c09fde7), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 42, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 8, 50, UINT64_C(0x8a9162aba4d6acf3), 29, UINT64_C(0x9c20542712c0a0ef), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 43, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 9, 52, UINT64_C(0x50f37a8bb3884709), 30, UINT64_C(0xcfe6f26979584abc), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 44, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 8, 52, UINT64_C(0x50f37a8bb3884709), 31, UINT64_C(0xf18fa8ba86856bca), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 45, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 9, 54, UINT64_C(0x62ceafb6eec98ffb), 32, UINT64_C(0xa11b393a4792a7cd), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 46, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 8, 54, UINT64_C(0x62ceafb6eec98ffb), 33, UINT64_C(0x9fda7bb9fe326757), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 47, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 8, 55, UINT64_C(0x6b6a32eab66db2f9), 34, UINT64_C(0x7cc8e9986e8bf23c), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 48, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 8, 56, UINT64_C(0xaecb93612cbf336b), 35, UINT64_C(0x684ccd1b16fbce2e), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 49, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 8, 57, UINT64_C(0x4858d152b25ec04b), 36, UINT64_C(0xf3d7ebb38a437d51), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 50, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 7, 57, UINT64_C(0x4858d152b25ec04b), 37, UINT64_C(0x3c9b54e75e043a07), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 51, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 7, 58, UINT64_C(0xfa7380bc30541f43), 38, UINT64_C(0x557d299a96c25a7a), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 52, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 6, 58, UINT64_C(0xfa7380bc30541f43), 39, UINT64_C(0x11fc770274e9a8ea), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 53, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 6, 59, UINT64_C(0x72407261f2c46ee9), 40, UINT64_C(0x8be5dea685297ad5), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 54, 7, "key", "{\"a\":[true]}", "7:0o1:1{6:1a1:1[1:1;];};", 5, 59, UINT64_C(0x72407261f2c46ee9), 41, UINT64_C(0xc570cb729193a42f), 13, UINT64_C(0xe364508a410c0844)}, + {3, 0, 55, 7, "key", "{\"a\":[true],\"\":null}", "7:0o2:1{6:1a1:1[1:1;];0:0;};", 7, 62, UINT64_C(0xdd828768b0673cb1), 41, UINT64_C(0xc570cb729193a42f), 14, UINT64_C(0xd51537a7c0e9393d)}, + {3, 0, 56, 7, "key", "{\"a\":[true],\"\":null}", "7:0o2:1{6:1a1:1[1:1;];0:0;};", 6, 62, UINT64_C(0xdd828768b0673cb1), 41, UINT64_C(0xc570cb729193a42f), 15, UINT64_C(0x42cc3f104f6e4243)}, + {3, 0, 57, 7, "key", "{\"a\":[true],\"\":null}", "7:0o2:1{6:1a1:1[1:1;];0:0;};", 6, 63, UINT64_C(0x6d02e9bcd600b499), 41, UINT64_C(0xc570cb729193a42f), 16, UINT64_C(0x2a0391b8de74bd18)}, + {3, 0, 58, 7, "key", "{\"a\":[true],\"k\":null}", "7:0o2:1{6:1a1:1[1:1;];0:0;};", 5, 63, UINT64_C(0x6d02e9bcd600b499), 41, UINT64_C(0xc570cb729193a42f), 17, UINT64_C(0xb4aa802afc47ece0)}, + {3, 0, 59, 7, "key", "{\"a\":[true],\"k\":null}", "7:0o2:1{6:1a1:1[1:1;];0:0;};", 5, 64, UINT64_C(0x5f482a0d0dc8019b), 42, UINT64_C(0x65054fd4e246057e), 17, UINT64_C(0xb4aa802afc47ece0)}, + {3, 0, 60, 7, "key", "{\"a\":[true],\"k\":\"\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 5, 65, UINT64_C(0x4b4d917e57f86132), 41, UINT64_C(0xc570cb729193a42f), 19, UINT64_C(0x62909e1cec7ae1d3)}, + {3, 0, 61, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 65, UINT64_C(0x4b4d917e57f86132), 41, UINT64_C(0xc570cb729193a42f), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 62, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 66, UINT64_C(0x972352fcc3940a32), 42, UINT64_C(0x28714bacd96afc31), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 63, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 67, UINT64_C(0x9ac6fafb0fd4419a), 43, UINT64_C(0x642ec0bd91056886), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 64, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 68, UINT64_C(0x06fc689c01f51722), 44, UINT64_C(0xa43d450f6a509362), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 65, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 5, 70, UINT64_C(0x0bea874fe70c8b8c), 45, UINT64_C(0x1fceee1263cab1b3), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 66, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 70, UINT64_C(0x0bea874fe70c8b8c), 46, UINT64_C(0xc764841a6cf73959), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 67, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 5, 72, UINT64_C(0x5cf6bc91b66abd26), 47, UINT64_C(0xca2d698260e6ceec), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 68, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 72, UINT64_C(0x5cf6bc91b66abd26), 48, UINT64_C(0x176b5b2a4ffd504a), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 69, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 73, UINT64_C(0x78e6de7db4f012a6), 49, UINT64_C(0xfce7936c92f12f1b), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 70, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 74, UINT64_C(0x783a50ac559170ce), 50, UINT64_C(0x91a5fbcf95a76d5d), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 71, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 5, 76, UINT64_C(0x5a15fa6e26c98802), 52, UINT64_C(0x23a18c828327ce44), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 72, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 5, 77, UINT64_C(0xa82a00a0586d91aa), 52, UINT64_C(0xcae276259cf4c86c), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 73, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 77, UINT64_C(0xa82a00a0586d91aa), 53, UINT64_C(0xeaa902eb2ee886e5), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 74, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 4, 78, UINT64_C(0x626f4aa7f91ef32c), 54, UINT64_C(0x486d2771cfef87e7), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 75, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 3, 78, UINT64_C(0x626f4aa7f91ef32c), 55, UINT64_C(0xb7e4a214cb3bfcc8), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 76, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 3, 79, UINT64_C(0xb91abd6c8103cf8a), 56, UINT64_C(0x7862cb1cd7eccd70), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 77, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 2, 79, UINT64_C(0xb91abd6c8103cf8a), 57, UINT64_C(0x77989b1f55d4e559), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 78, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 2, 80, UINT64_C(0x8b4ae15f38aa07ee), 58, UINT64_C(0x7d721ed202be022f), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 79, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 2, 81, UINT64_C(0x5a447acd48eac8b2), 59, UINT64_C(0x4bb22e9ffd64c5f0), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 80, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 2, 82, UINT64_C(0x4e1a64d2e7c37f86), 61, UINT64_C(0x80ae7b36aecc0210), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 81, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 2, 83, UINT64_C(0x7a59405fd132616a), 61, UINT64_C(0x7f28d8f6eb248efd), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 82, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 1, 83, UINT64_C(0x7a59405fd132616a), 62, UINT64_C(0x048d19db3da2b56b), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 0, 83, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 3, 86, UINT64_C(0x9f5b0f8c3ddf0238), 62, UINT64_C(0x048d19db3da2b56b), 21, UINT64_C(0x690a605d38c59743)}, + {3, 0, 84, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 2, 86, UINT64_C(0x9f5b0f8c3ddf0238), 62, UINT64_C(0x048d19db3da2b56b), 22, UINT64_C(0x6a3f1a2a443276c5)}, + {3, 0, 85, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 2, 87, UINT64_C(0xa6a9474d21df7fca), 62, UINT64_C(0x048d19db3da2b56b), 23, UINT64_C(0x3f954ced238070f8)}, + {3, 0, 86, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 87, UINT64_C(0xa6a9474d21df7fca), 62, UINT64_C(0x048d19db3da2b56b), 24, UINT64_C(0x601715e2f78af79c)}, + {3, 0, 87, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 88, UINT64_C(0x112412108ec65a9e), 63, UINT64_C(0x333ba418287f24ee), 24, UINT64_C(0x601715e2f78af79c)}, + {3, 0, 88, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 89, UINT64_C(0xe6a5a8229b0c9002), 64, UINT64_C(0x72c6e24c51035e34), 24, UINT64_C(0x601715e2f78af79c)}, + {3, 0, 89, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 90, UINT64_C(0xf80cd4cd7658e9c6), 63, UINT64_C(0xe3718003aacfdf89), 27, UINT64_C(0x23c931a0f86ee070)}, + {3, 0, 90, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 91, UINT64_C(0xd6b74720191512aa), 63, UINT64_C(0x859ce136ed5a19e3), 27, UINT64_C(0x23c931a0f86ee070)}, + {3, 0, 91, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 1, 92, UINT64_C(0xee80658a9ece807e), 65, UINT64_C(0x6974aedd6bbd4bfe), 27, UINT64_C(0xb55d4a66e9e7e0ec)}, + {3, 0, 92, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 1, 93, UINT64_C(0x12aca08bd8e3a562), 65, UINT64_C(0x7cb2eddc0608127a), 27, UINT64_C(0xb55d4a66e9e7e0ec)}, + {3, 0, 93, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 3, 96, UINT64_C(0x3c333c9bf74f1224), 66, UINT64_C(0x154d3274d8faa6c5), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 94, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 4, 98, UINT64_C(0xb940cbe38e8e601e), 67, UINT64_C(0xae9fccd09bb2a1a9), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 95, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 3, 98, UINT64_C(0xb940cbe38e8e601e), 68, UINT64_C(0x61f4292581fe1020), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 96, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 4, 100, UINT64_C(0xfcbcf9f4cc7993e0), 69, UINT64_C(0xe79fa9bc314eb694), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 97, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 3, 100, UINT64_C(0xfcbcf9f4cc7993e0), 70, UINT64_C(0x588604dbc9a84b89), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 98, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 3, 101, UINT64_C(0x30f9e0f7075fc2b6), 71, UINT64_C(0x920d6877a5ad485b), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 99, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 2, 101, UINT64_C(0x30f9e0f7075fc2b6), 72, UINT64_C(0xf5adbbea7893fef0), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 100, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 2, 102, UINT64_C(0x9729f5c186b3bae0), 73, UINT64_C(0x29f0f6fd5ed356c0), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 101, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 1, 102, UINT64_C(0x9729f5c186b3bae0), 74, UINT64_C(0xa9a8d50e804943ed), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 0, 102, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 2, 104, UINT64_C(0x234227d767437cfa), 74, UINT64_C(0xa9a8d50e804943ed), 28, UINT64_C(0x5765ff51e1564dec)}, + {3, 0, 103, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 104, UINT64_C(0x234227d767437cfa), 74, UINT64_C(0xa9a8d50e804943ed), 29, UINT64_C(0x4d11975435094ccb)}, + {3, 0, 104, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 74, UINT64_C(0xa9a8d50e804943ed), 30, UINT64_C(0xb50667d7d86f8d99)}, + {3, 0, 105, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 0, 105, UINT64_C(0x2ce6ab0477ad5968), 74, UINT64_C(0xa9a8d50e804943ed), 31, UINT64_C(0x98a23224cfb0ffe0)}, + {3, 1, 0, 7, "", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 104, UINT64_C(0xa89112e8bdfbac92), 74, UINT64_C(0x6ea88119e6b74a5d), 29, UINT64_C(0x15bba93be47766f3)}, + {3, 1, 1, 7, "", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 74, UINT64_C(0x3a169d78c5b81187), 30, UINT64_C(0xcddf81110b348447)}, + {3, 1, 2, 0, "key", "null", "0:0;", 1, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 2, UINT64_C(0x08395307b4f1348c)}, + {3, 1, 3, 0, "key", "null", "0:0;", 1, 4, UINT64_C(0x29e49b479f20e3f9), 1, UINT64_C(0xaf63be4c8601b992), 2, UINT64_C(0x08395307b4f1348c)}, + {3, 1, 4, 0, "key", "null", "0:0;", 1, 5, UINT64_C(0x50576bb364e2af63), 2, UINT64_C(0x0824ed07b4dfde30), 2, UINT64_C(0x08395307b4f1348c)}, + {3, 1, 5, 7, "key", "{}", "7:0o0:1{};", 1, 6, UINT64_C(0x673b44d46d2fced9), 1, UINT64_C(0xaf63bd4c8601b7df), 5, UINT64_C(0x3f3fcc261a338844)}, + {3, 1, 6, 7, "key", "{}", "7:0o0:1{};", 1, 7, UINT64_C(0x9984a5f5883c36c3), 1, UINT64_C(0xaf63bb4c8601b479), 5, UINT64_C(0x3f3fcc261a338844)}, + {3, 1, 7, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 91, UINT64_C(0x60517b283d4e7f28), 59, UINT64_C(0x70305a3ea5d29b0b), 31, UINT64_C(0x1987bbdf4f551fe8)}, + {3, 1, 8, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 92, UINT64_C(0x2cdb62dd8baca92e), 60, UINT64_C(0x19657326d928cdd8), 31, UINT64_C(0x2c299448de62a072)}, + {3, 1, 9, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 73, UINT64_C(0x1fc7c6ebd1b98ec6), 31, UINT64_C(0xe32cffe4946de372)}, + {3, 1, 10, 7, "key", "{\"\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 98, UINT64_C(0x27dfd47a22924cde), 68, UINT64_C(0xad21aae5eb3e9d5c), 29, UINT64_C(0xe76f73f6e413690f)}, + {3, 1, 11, 7, "key", "{\"\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x54436514d18ad928), 71, UINT64_C(0xfeaee9cf685870dc), 30, UINT64_C(0xfad1497156057d51)}, + {3, 1, 12, 7, "key", "{\"\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 100, UINT64_C(0x0ec693ac9e264c48), 70, UINT64_C(0x46f3d395149d83b7), 29, UINT64_C(0x9292c45666d5f087)}, + {3, 1, 13, 7, "key", "{\"\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0x998db6ea3f64f750), 72, UINT64_C(0x6a8506724de1c15f), 30, UINT64_C(0x51fafc33547ec99a)}, + {3, 1, 14, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 94, UINT64_C(0xdd430a82fe7fe43e), 62, UINT64_C(0x6c1ea5a1d32dce85), 31, UINT64_C(0xaa062e38a46dc362)}, + {3, 1, 15, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 95, UINT64_C(0x020439d52e8973d0), 63, UINT64_C(0x6916ebb1bcc7d741), 31, UINT64_C(0xe99cd0eb7f04b0c0)}, + {3, 1, 16, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 73, UINT64_C(0x010e7d553fd5eda0), 31, UINT64_C(0xe32cffe4946de372)}, + {3, 1, 17, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 104, UINT64_C(0x7d3ce9fb1cdc6896), 73, UINT64_C(0x1798f8572a342022), 31, UINT64_C(0xe70b60a34f05f80c)}, + {3, 1, 18, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 73, UINT64_C(0x8df3952bc81fb19a), 31, UINT64_C(0xe32cffe4946de372)}, + {3, 1, 19, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 100, UINT64_C(0x2dfc24178ad786ae), 70, UINT64_C(0xab432aea1b343eb1), 29, UINT64_C(0x3bd25fc07522b6c7)}, + {3, 1, 20, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 101, UINT64_C(0xb1577d1dfe0dc2be), 71, UINT64_C(0xdd3a2a6eb7fab273), 29, UINT64_C(0xa4dacec9da60a18d)}, + {3, 1, 21, 7, "key", "{\"\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 99, UINT64_C(0x4f117cd47c5b07a8), 70, UINT64_C(0x68097b7d7ba01328), 28, UINT64_C(0x9d231f94a3d8505f)}, + {3, 1, 22, 7, "key", "{\"\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 101, UINT64_C(0xaf7fa9cbc785a706), 71, UINT64_C(0xdd3a2a6eb7fab273), 29, UINT64_C(0xa4dacec9da60a18d)}, + {3, 1, 23, 7, "key", "{\"a\":null,\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{0:0;5:1;7:1o1:1{3:1;};};", 1, 93, UINT64_C(0xfe8c535b61975fc0), 65, UINT64_C(0x52dcce1da8492cfc), 27, UINT64_C(0x6b2d1037cf396c3a)}, + {3, 1, 24, 7, "key", "{\"a\":null,\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{0:0;5:1;7:1o1:1{3:1;};};", 1, 94, UINT64_C(0x49656a6b416c8aae), 66, UINT64_C(0xb6023d8fb31fa833), 27, UINT64_C(0x351d6894476b65d4)}, + {3, 1, 25, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x5b8d98fa549824d6), 71, UINT64_C(0x3236647f1b32c90a), 30, UINT64_C(0x4c922b10d249c073)}, + {3, 1, 26, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 101, UINT64_C(0xfd645a891ab048d8), 71, UINT64_C(0x441133cd64aeff34), 30, UINT64_C(0x4bdf8fc323423212)}, + {3, 1, 27, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x5b8d98fa549824d6), 71, UINT64_C(0x6997d907cb99bf48), 30, UINT64_C(0x4c922b10d249c073)}, + {3, 1, 28, 7, "key", "{\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o2:1{5:1;7:1o1:1{3:1;};};", 1, 94, UINT64_C(0xc58789bf9039d336), 70, UINT64_C(0xa480724f4c9ef05f), 23, UINT64_C(0xaefbe7fa88ebe124)}, + {3, 1, 29, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x5b8d98fa549824d6), 72, UINT64_C(0xe344b830f3a11d8a), 29, UINT64_C(0xab5acbf8e6ca0bdf)}, + {3, 1, 30, 7, "key", "{\"\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 101, UINT64_C(0xaf7fa9cbc785a706), 72, UINT64_C(0x902c140387af47d3), 28, UINT64_C(0x66b09ba8d6eda091)}, + {3, 1, 31, 7, "key", "{\"\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x5b8d98fa549824d6), 72, UINT64_C(0xe344b830f3a11d8a), 29, UINT64_C(0xab5acbf8e6ca0bdf)}, + {3, 1, 32, 7, "key", "{\"a\":null,\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{0:0;5:1;7:1o1:1{3:1;};};", 1, 98, UINT64_C(0x63e791635b56fa46), 70, UINT64_C(0x8a8854f75fca9d93), 27, UINT64_C(0x2dfa80357ea0bdb4)}, + {3, 1, 33, 7, "key", "{\"a\":null,\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{0:0;5:1;7:1o1:1{3:1;};};", 1, 99, UINT64_C(0x4a646907627eac30), 71, UINT64_C(0x18b59e7d5b2fe673), 27, UINT64_C(0x2c0f7f6633f03dbe)}, + {3, 1, 34, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x5b8d98fa549824d6), 71, UINT64_C(0x8291d9ce0eae8d1d), 30, UINT64_C(0xeda7450c41cedbee)}, + {3, 1, 35, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 101, UINT64_C(0xfd5972cc2d7ea8a8), 71, UINT64_C(0x931494827e967b6a), 30, UINT64_C(0x52b5b7d7887eb988)}, + {3, 1, 36, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:0a0:1[];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x5b8d98fa549824d6), 71, UINT64_C(0xa4496da3e2ecc4d3), 30, UINT64_C(0x5c17b3d1c0a1b56e)}, + {3, 1, 37, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a0:1[];5:1;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0x7bf409aa512991e0), 73, UINT64_C(0x1091b5eee814c0e9), 30, UINT64_C(0x21d27a28fc8ef91c)}, + {3, 1, 38, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a0:1[];5:1;7:1o1:1{3:1;};};", 1, 104, UINT64_C(0xe05d289941f12cfe), 73, UINT64_C(0xf799f9b52163552a), 30, UINT64_C(0xe93f3e2fb84426fe)}, + {3, 1, 39, 7, "key", "{\"a\":[],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a0:1[];5:1;7:1o1:1{3:1;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 74, UINT64_C(0xf82ec970d3cfc2e0), 30, UINT64_C(0x11bb3330ad0999c0)}, + {3, 1, 40, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 100, UINT64_C(0x835a6ff205dd6146), 68, UINT64_C(0x3b1c5be6adc3631c), 31, UINT64_C(0x9c1079f29a41bd24)}, + {3, 1, 41, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 101, UINT64_C(0x8d63785d9cbce268), 69, UINT64_C(0xa8cdc57a4d527c5c), 31, UINT64_C(0x520bc292ee5b0f92)}, + {3, 1, 42, 7, "key", "{\"a\":[true],\"\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 98, UINT64_C(0x520ba6b37d1f237e), 68, UINT64_C(0x2d7baac348149c62), 29, UINT64_C(0x8098f4d3abda36e7)}, + {3, 1, 43, 7, "key", "{\"a\":[true],\"\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0xec8ebd121f055228), 71, UINT64_C(0x2b515550cedda7c8), 30, UINT64_C(0x86928d977d93f2bb)}, + {3, 1, 44, 7, "key", "{\"a\":[true],\"\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 100, UINT64_C(0x66382e2716717628), 70, UINT64_C(0x0bdc83cc3c8b5cef), 29, UINT64_C(0x4ecf792831a2b853)}, + {3, 1, 45, 7, "key", "{\"a\":[true],\"\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0xe8acd57ecd9d0bd8), 72, UINT64_C(0xc0509a5f17c137f9), 30, UINT64_C(0xd78e7d4db72bdd00)}, + {3, 1, 46, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x2a58dcccc2fcfd9e), 70, UINT64_C(0x8d2f336da17fdc39), 31, UINT64_C(0x413ebc4e90581e70)}, + {3, 1, 47, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0x03a8c5ca4270b260), 71, UINT64_C(0xa46bf71b697ba90f), 31, UINT64_C(0x117b52679e50447a)}, + {3, 1, 48, 7, "key", "{\"k\":\"\",\"a\":[true],\"o\":{\"q\":1}}", "7:0o3:1{5:0;6:1a1:1[1:1;];7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x684c99dda1421b91), 72, UINT64_C(0xe793b57af9534fae), 29, UINT64_C(0x67733b1248cb0261)}, + {3, 1, 49, 7, "key", "{\"k\":\"\",\"a\":[true],\"o\":{\"q\":1}}", "7:0o3:1{5:0;6:1a1:1[1:1;];7:1o1:1{3:1;};};", 1, 103, UINT64_C(0xc103fd077d12a831), 73, UINT64_C(0x2fb728bc07b7ae3c), 29, UINT64_C(0xe2affcd4663837ef)}, + {3, 1, 50, 7, "key", "{\"a\":[true],\"\":\"\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:0;7:1o1:1{3:1;};};", 1, 101, UINT64_C(0xf9b8778ce5892e03), 72, UINT64_C(0xa6bcc4a926c9b17f), 28, UINT64_C(0xfe73d9bce6994e15)}, + {3, 1, 51, 7, "key", "{\"a\":[true],\"\":\"\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:0;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0xba3b00c1c283e5e9), 73, UINT64_C(0x2fb728bc07b7ae3c), 29, UINT64_C(0xd1fe98172432957d)}, + {3, 1, 52, 7, "key", "{\"a\":[true],\"k\":null,\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];0:0;7:1o1:1{3:1;};};", 1, 101, UINT64_C(0xa5355b8a98f6cb8b), 72, UINT64_C(0xa6bcc4a926c9b17f), 28, UINT64_C(0xfe73d9bce6994e15)}, + {3, 1, 53, 7, "key", "{\"a\":[true],\"k\":null,\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];0:0;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0xe52e3935b858e4b9), 73, UINT64_C(0x48dbc55354a5d5f7), 28, UINT64_C(0xffb0dab42fe56888)}, + {3, 1, 54, 7, "key", "{\"a\":[true],\"o\":{\"q\":1}}", "7:0o2:1{6:1a1:1[1:1;];7:1o1:1{3:1;};};", 1, 99, UINT64_C(0x9796ad14651215a3), 74, UINT64_C(0x95d5583f8bb4071a), 24, UINT64_C(0xd729a5f6b715fe63)}, + {3, 1, 55, 7, "key", "{\"a\":[true],\"k\":\"\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:0;7:1o1:1{3:1;};};", 1, 104, UINT64_C(0x555db4043c851841), 74, UINT64_C(0x54851835c64759d3), 29, UINT64_C(0xe734f45587bfefcb)}, + {3, 1, 56, 7, "key", "{\"a\":[true],\"\":\"\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:0;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0xba3b00c1c283e5e9), 74, UINT64_C(0xb84e1822ab185956), 28, UINT64_C(0xb37adbd00a14818f)}, + {3, 1, 57, 7, "key", "{\"a\":[true],\"\":\"\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:0;7:1o1:1{3:1;};};", 1, 104, UINT64_C(0x555db4043c851841), 74, UINT64_C(0x54851835c64759d3), 29, UINT64_C(0xe734f45587bfefcb)}, + {3, 1, 58, 7, "key", "{\"a\":[true],\"k\":null,\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];0:0;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0x70181eb04816bf33), 74, UINT64_C(0xb84e1822ab185956), 28, UINT64_C(0xb37adbd00a14818f)}, + {3, 1, 59, 7, "key", "{\"a\":[true],\"k\":null,\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];0:0;7:1o1:1{3:1;};};", 1, 104, UINT64_C(0x555db4043c851841), 75, UINT64_C(0x93238bb2f717715c), 28, UINT64_C(0x6b0001fee18a4342)}, + {3, 1, 60, 7, "key", "{\"a\":[true],\"k\":\"\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 74, UINT64_C(0x152a73fddccffd18), 30, UINT64_C(0x2919611d528894b6)}, + {3, 1, 61, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 91, UINT64_C(0xc4be6f24764b02dc), 59, UINT64_C(0x70b92cf6f5e020be), 31, UINT64_C(0xb62ceac18acff7bb)}, + {3, 1, 62, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 92, UINT64_C(0x61f826a2990473dc), 60, UINT64_C(0xf32be6d03cbedc94), 31, UINT64_C(0xde4498e1728d6c04)}, + {3, 1, 63, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 93, UINT64_C(0x969ed331086ff354), 61, UINT64_C(0xd551a80e1affde03), 31, UINT64_C(0xaf4ee52420b1c109)}, + {3, 1, 64, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 98, UINT64_C(0x79853297e30e1d4c), 68, UINT64_C(0xb65d101d0531c0a2), 29, UINT64_C(0x46c716fcc1328a3d)}, + {3, 1, 65, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x12ef72ce739f4d0e), 71, UINT64_C(0xb4e6bb7515f3cf72), 30, UINT64_C(0x99227e2156785871)}, + {3, 1, 66, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 100, UINT64_C(0x6f83d649fe8c8c72), 70, UINT64_C(0x44eac57908774015), 29, UINT64_C(0x5fcef1cfacefbfb7)}, + {3, 1, 67, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0x2804f6f1e71ce3f8), 72, UINT64_C(0xb02c45a9887be1a2), 30, UINT64_C(0xf770764b4ef7ecf9)}, + {3, 1, 68, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 94, UINT64_C(0x7c26769eabcb532c), 62, UINT64_C(0x73f92b18a24d6d95), 31, UINT64_C(0x824ac31fee78d5ba)}, + {3, 1, 69, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 95, UINT64_C(0xafbd5cdea9c2bcac), 63, UINT64_C(0xb0539f5485162014), 31, UINT64_C(0xdbe2448368048507)}, + {3, 1, 70, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 96, UINT64_C(0xb33822e607e26de4), 64, UINT64_C(0x33f9c22d193bedf6), 31, UINT64_C(0x51382e7a7c032910)}, + {3, 1, 71, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 104, UINT64_C(0xc06e6a557e87d880), 73, UINT64_C(0x7a0b8e235a97a118), 31, UINT64_C(0xd26bf9954e0a7e12)}, + {3, 1, 72, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 73, UINT64_C(0x61bb4f31921812e7), 31, UINT64_C(0x2e08c6c7d56c3047)}, + {3, 1, 73, 7, "key", "{\"o\":{},\"a\":[true],\"k\":\"recovery\"}", "7:0o3:1{7:0o0:1{};6:1a1:1[1:1;];5:1;};", 1, 89, UINT64_C(0x109ea5036371589a), 62, UINT64_C(0x340e380554d0657b), 26, UINT64_C(0xd9113597a2d80f61)}, + {3, 1, 74, 7, "key", "{\"o\":{},\"a\":[true],\"k\":\"recovery\"}", "7:0o3:1{7:0o0:1{};6:1a1:1[1:1;];5:1;};", 1, 90, UINT64_C(0x9244c3a01dcace7c), 63, UINT64_C(0xd24b3c75b921bd74), 26, UINT64_C(0x1c37ca450bfc8a3d)}, + {3, 1, 75, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 88, UINT64_C(0x9f631208c5a8d666), 62, UINT64_C(0xe93b1055211f4f47), 25, UINT64_C(0xd4a2aa2952606b15)}, + {3, 1, 76, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 90, UINT64_C(0xd3b8a67d8b3576f8), 63, UINT64_C(0xd24b3c75b921bd74), 26, UINT64_C(0xf4e67f641f73ecad)}, + {3, 1, 77, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 82, UINT64_C(0xe0e7732a58e74766), 57, UINT64_C(0x77989b1f55d4e559), 24, UINT64_C(0x607bb464128767f0)}, + {3, 1, 78, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 83, UINT64_C(0x7d5bc46063a8c1e2), 58, UINT64_C(0x7d721ed202be022f), 24, UINT64_C(0x6c0528f9d1ca95c4)}, + {3, 1, 79, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 84, UINT64_C(0x0c778e767eab97ae), 59, UINT64_C(0x4bb22e9ffd64c5f0), 24, UINT64_C(0xedf2404c46b20bf4)}, + {3, 1, 80, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 90, UINT64_C(0x304cacc2dca590f6), 63, UINT64_C(0x29777f1b2af8f733), 27, UINT64_C(0x756f2f41adc7104a)}, + {3, 1, 81, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 91, UINT64_C(0xd6b74720191512aa), 63, UINT64_C(0xb522ebbedf5074e6), 27, UINT64_C(0x122bfe467a81181b)}, + {3, 1, 82, 7, "key", "{\"a\":[true],\"k\":\"recovery\"}", "7:0o2:1{6:1a1:1[1:1;];5:1;};", 1, 83, UINT64_C(0x7a59405fd132616a), 62, UINT64_C(0x048d19db3da2b56b), 20, UINT64_C(0x1a0725f3cd62f080)}, + {3, 1, 83, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 91, UINT64_C(0xd6b74720191512aa), 64, UINT64_C(0x72c6e24c51035e34), 26, UINT64_C(0x4fe2311fc08c62d1)}, + {3, 1, 84, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 90, UINT64_C(0xd3b8a67d8b3576f8), 64, UINT64_C(0x0c961ea2c6d9bdd0), 25, UINT64_C(0xb192d70ba7d71d53)}, + {3, 1, 85, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 91, UINT64_C(0xd6b74720191512aa), 64, UINT64_C(0x72c6e24c51035e34), 26, UINT64_C(0x4fe2311fc08c62d1)}, + {3, 1, 86, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 87, UINT64_C(0xa6a9474d21df7fca), 62, UINT64_C(0x048d19db3da2b56b), 24, UINT64_C(0x601715e2f78af79c)}, + {3, 1, 87, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 88, UINT64_C(0x112412108ec65a9e), 63, UINT64_C(0x333ba418287f24ee), 24, UINT64_C(0x601715e2f78af79c)}, + {3, 1, 88, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":null}", "7:0o3:1{6:1a1:1[1:1;];5:1;0:0;};", 1, 89, UINT64_C(0xe6a5a8229b0c9002), 64, UINT64_C(0x72c6e24c51035e34), 24, UINT64_C(0x601715e2f78af79c)}, + {3, 1, 89, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 90, UINT64_C(0xf80cd4cd7658e9c6), 63, UINT64_C(0xe3718003aacfdf89), 27, UINT64_C(0x23c931a0f86ee070)}, + {3, 1, 90, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:0o0:1{};};", 1, 91, UINT64_C(0xd6b74720191512aa), 63, UINT64_C(0x859ce136ed5a19e3), 27, UINT64_C(0x23c931a0f86ee070)}, + {3, 1, 91, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 1, 92, UINT64_C(0xee80658a9ece807e), 65, UINT64_C(0x6974aedd6bbd4bfe), 27, UINT64_C(0xb55d4a66e9e7e0ec)}, + {3, 1, 92, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 1, 93, UINT64_C(0x12aca08bd8e3a562), 65, UINT64_C(0x7cb2eddc0608127a), 27, UINT64_C(0xb55d4a66e9e7e0ec)}, + {3, 1, 93, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 98, UINT64_C(0xfc0a73e323c44ba4), 68, UINT64_C(0x61f4292581fe1020), 29, UINT64_C(0x32b15a146cdd8b6f)}, + {3, 1, 94, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 102, UINT64_C(0x2c63d677b55e3dfa), 71, UINT64_C(0xd9342f88c76e636d), 30, UINT64_C(0xf08facd70b71fb88)}, + {3, 1, 95, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 100, UINT64_C(0xfdcd03f4cd60d80e), 70, UINT64_C(0x588604dbc9a84b89), 29, UINT64_C(0x176bd9a267ef7e3b)}, + {3, 1, 96, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 1, 103, UINT64_C(0x9c1dc1d99d63a20c), 72, UINT64_C(0xf5adbbea7893fef0), 30, UINT64_C(0x584f1dfc2923487b)}, + {3, 1, 97, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 103, UINT64_C(0xc99430c7bfe41ee4), 72, UINT64_C(0xf5adbbea7893fef0), 30, UINT64_C(0x584f1dfc2923487b)}, + {3, 1, 98, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 104, UINT64_C(0x238cd5d76782dd1e), 73, UINT64_C(0x29f0f6fd5ed356c0), 30, UINT64_C(0x404b79f3610714de)}, + {3, 1, 99, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 102, UINT64_C(0x985bf7c187b7d7dc), 72, UINT64_C(0xf5adbbea7893fef0), 29, UINT64_C(0x79af5c5dc615f4f7)}, + {3, 1, 100, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 104, UINT64_C(0x234227d767437cfa), 73, UINT64_C(0x29f0f6fd5ed356c0), 30, UINT64_C(0x404b79f3610714de)}, + {3, 1, 101, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o0:1{};};", 1, 102, UINT64_C(0x9729f5c186b3bae0), 74, UINT64_C(0xa9a8d50e804943ed), 27, UINT64_C(0x998885b959d4fc68)}, + {3, 1, 102, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 74, UINT64_C(0xa9a8d50e804943ed), 30, UINT64_C(0xb50667d7d86f8d99)}, + {3, 1, 103, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 104, UINT64_C(0x234227d767437cfa), 74, UINT64_C(0xa9a8d50e804943ed), 29, UINT64_C(0x4d11975435094ccb)}, + {3, 1, 104, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"\":0}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:0;};};", 1, 105, UINT64_C(0x2ce6ab0477ad5968), 74, UINT64_C(0xa9a8d50e804943ed), 30, UINT64_C(0xb50667d7d86f8d99)}, + {3, 1, 105, 7, "key", "{\"a\":[true],\"k\":\"recovery\",\"o\":{\"q\":1}}", "7:0o3:1{6:1a1:1[1:1;];5:1;7:1o1:1{3:1;};};", 0, 105, UINT64_C(0x2ce6ab0477ad5968), 74, UINT64_C(0xa9a8d50e804943ed), 31, UINT64_C(0x98a23224cfb0ffe0)}, + {4, 0, 0, 0, "", "null", "0:0;", 2, 2, UINT64_C(0x08842007b530c95d), 0, UINT64_C(0xcbf29ce484222325), 0, UINT64_C(0xcbf29ce484222325)}, + {4, 0, 1, 0, "", "null", "0:0;", 2, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 1, UINT64_C(0xaf63bc4c8601b62c)}, + {4, 0, 2, 0, "key", "null", "0:0;", 1, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 2, UINT64_C(0x08395307b4f1348c)}, + {4, 0, 3, 0, "key", "null", "0:0;", 1, 4, UINT64_C(0x29e49b479f20e3f9), 1, UINT64_C(0xaf63be4c8601b992), 2, UINT64_C(0x08395307b4f1348c)}, + {4, 0, 4, 6, "key", "[]", "6:0a0:1[missing]", 2, 6, UINT64_C(0x67f9d4d46dd1ec91), 1, UINT64_C(0xaf63bd4c8601b7df), 4, UINT64_C(0xb345225e3644edb5)}, + {4, 0, 5, 6, "key", "[]", "6:0a0:1[];", 1, 6, UINT64_C(0x67f9d4d46dd1ec91), 1, UINT64_C(0xaf63bd4c8601b7df), 5, UINT64_C(0x3f3fcc261a338844)}, + {4, 0, 6, 6, "key", "[]", "6:0a0:1[];", 1, 7, UINT64_C(0x7f7145f69bb51593), 1, UINT64_C(0xaf63bb4c8601b479), 5, UINT64_C(0x3f3fcc261a338844)}, + {4, 0, 7, 6, "key", "[]", "6:0a0:1[];", 4, 11, UINT64_C(0x0ef52d9e6a1d5a93), 2, UINT64_C(0x08218e07b4dd06ec), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {4, 0, 8, 6, "key", "[]", "6:0a0:1[];", 4, 12, UINT64_C(0xa2e7cea7d9800249), 3, UINT64_C(0x1e87e8189340ae76), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {4, 0, 9, 6, "key", "[]", "6:0a0:1[];", 4, 13, UINT64_C(0x1fe4f8370a5a89cb), 4, UINT64_C(0xf241d4ba207d8e95), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {4, 0, 10, 6, "key", "[]", "6:0a0:1[];", 3, 13, UINT64_C(0x1fe4f8370a5a89cb), 5, UINT64_C(0x7e6b1d36db19d80d), 5, UINT64_C(0x18d2da27b6b8ff60)}, + {4, 0, 11, 6, "key", "[null]", "6:0a1:1[0:0;];", 3, 14, UINT64_C(0x8d23b086984bf3e9), 5, UINT64_C(0x7e6b1d36db19d80d), 6, UINT64_C(0xe5658011eb170905)}, + {4, 0, 12, 6, "key", "[null]", "6:0a1:1[0:0;];", 3, 15, UINT64_C(0x4ef6f5b452adc8bb), 6, UINT64_C(0xe60984573090fde3), 6, UINT64_C(0xe5658011eb170905)}, + {4, 0, 13, 6, "key", "[[]]", "6:0a1:1[6:0a0:1[missing]];", 4, 17, UINT64_C(0x9b59153f217aca13), 6, UINT64_C(0xfe81bb0663d6faef), 8, UINT64_C(0xd594f473ed6d03b0)}, + {4, 0, 14, 6, "key", "[[]]", "6:0a1:1[6:0a0:1[];];", 3, 17, UINT64_C(0x9b59153f217aca13), 6, UINT64_C(0xfe81bb0663d6faef), 9, UINT64_C(0x0f060cdee7c6d79c)}, + {4, 0, 15, 6, "key", "[[]]", "6:0a1:1[6:0a0:1[];];", 3, 18, UINT64_C(0xa2f3e5456d9fe0d9), 6, UINT64_C(0x02c88c156775a22c), 9, UINT64_C(0x0f060cdee7c6d79c)}, + {4, 0, 16, 6, "key", "[[]]", "6:0a1:1[6:1a0:1[];];", 3, 19, UINT64_C(0xdf5265fa0b5e084b), 8, UINT64_C(0x7640b8a62d1fce5a), 9, UINT64_C(0x3a62d1866b18afc8)}, + {4, 0, 17, 6, "key", "[[]]", "6:0a1:1[6:1a0:1[];];", 3, 20, UINT64_C(0x927dfae12a29b7e1), 8, UINT64_C(0x1700d06a36def24f), 9, UINT64_C(0x3a62d1866b18afc8)}, + {4, 0, 18, 6, "key", "[[]]", "6:0a1:1[6:1a0:1[];];", 4, 22, UINT64_C(0xe866f0c62e93f421), 9, UINT64_C(0xf24b4415cd4b36d9), 9, UINT64_C(0x08d2d2ce6ebd7ac4)}, + {4, 0, 19, 6, "key", "[[]]", "6:0a1:1[6:1a0:1[];];", 4, 23, UINT64_C(0xaa96fa88c4c89fbb), 10, UINT64_C(0x8d143888e63f9b66), 9, UINT64_C(0x08d2d2ce6ebd7ac4)}, + {4, 0, 20, 6, "key", "[[]]", "6:0a1:1[6:1a0:1[];];", 3, 23, UINT64_C(0xaa96fa88c4c89fbb), 11, UINT64_C(0xaa5c60bc7d9fe37a), 9, UINT64_C(0x08d2d2ce6ebd7ac4)}, + {4, 0, 21, 6, "key", "[[null]]", "6:0a1:1[6:1a1:1[0:0;];];", 3, 24, UINT64_C(0xa746b16660fe9f39), 11, UINT64_C(0xaa5c60bc7d9fe37a), 10, UINT64_C(0x340b2d803261506f)}, + {4, 0, 22, 6, "key", "[[null]]", "6:0a1:1[6:1a1:1[0:0;];];", 3, 25, UINT64_C(0xc63af9f71f110db3), 12, UINT64_C(0xac32a94474c4bc64), 10, UINT64_C(0x340b2d803261506f)}, + {4, 0, 23, 6, "key", "[[\"\"]]", "6:0a1:1[6:1a1:1[5:1;];];", 3, 26, UINT64_C(0xdf2316e9c5861876), 11, UINT64_C(0xaa5c60bc7d9fe37a), 12, UINT64_C(0xc1928666ae5d4a6a)}, + {4, 0, 24, 6, "key", "[[\"recovery\"]]", "6:0a1:1[6:1a1:1[5:1;];];", 2, 26, UINT64_C(0xdf2316e9c5861876), 11, UINT64_C(0xaa5c60bc7d9fe37a), 13, UINT64_C(0xc70e457618281d24)}, + {4, 0, 25, 6, "key", "[[\"recovery\"]]", "6:0a1:1[6:1a1:1[5:1;];];", 2, 27, UINT64_C(0xae82353aa2adaf82), 12, UINT64_C(0x283aa048ed3b7e47), 13, UINT64_C(0xc70e457618281d24)}, + {4, 0, 26, 6, "key", "[[\"recovery\"]]", "6:0a1:1[6:1a1:1[5:1;];];", 2, 28, UINT64_C(0x33fb34a26c513b2e), 13, UINT64_C(0xc3114ee2a85daebf), 13, UINT64_C(0xc70e457618281d24)}, + {4, 0, 27, 6, "key", "[[\"recovery\"]]", "6:0a1:1[6:1a1:1[5:1;];];", 1, 28, UINT64_C(0x33fb34a26c513b2e), 14, UINT64_C(0x0e819e1d9dff465c), 13, UINT64_C(0xc70e457618281d24)}, + {4, 0, 28, 6, "key", "[[\"recovery\"],null]", "6:0a2:1[6:1a1:1[5:1;];0:0;];", 1, 29, UINT64_C(0xa51595fe0e078192), 14, UINT64_C(0x0e819e1d9dff465c), 14, UINT64_C(0xf7ddd57b3ac52386)}, + {4, 0, 29, 6, "key", "[[\"recovery\"],null]", "6:0a2:1[6:1a1:1[5:1;];0:0;];", 1, 30, UINT64_C(0x8b2f90b1d6c16176), 15, UINT64_C(0x27a4fc9d713e1dcf), 14, UINT64_C(0xf7ddd57b3ac52386)}, + {4, 0, 30, 6, "key", "[[\"recovery\"],null]", "6:0a2:1[6:1a1:1[5:1;];0:0;];", 1, 31, UINT64_C(0x4333fc2fea9805fa), 16, UINT64_C(0xb34108ccff78652b), 14, UINT64_C(0xf7ddd57b3ac52386)}, + {4, 0, 31, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:0o0:1{};];", 1, 32, UINT64_C(0xc95b5f6ba051f16e), 15, UINT64_C(0x60ba726eb8e9eafe), 17, UINT64_C(0x1fbdcc46db94b748)}, + {4, 0, 32, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:0o0:1{};];", 1, 33, UINT64_C(0x783429e16b3c8d32), 15, UINT64_C(0xcdedec10fab123de), 17, UINT64_C(0x1fbdcc46db94b748)}, + {4, 0, 33, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 1, 34, UINT64_C(0x7d303c0937e3b596), 17, UINT64_C(0x0e22e134963af1ff), 17, UINT64_C(0xc47810ca7ea08204)}, + {4, 0, 34, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 1, 35, UINT64_C(0x9cac01a9f7ee3e9a), 17, UINT64_C(0x9d9d28d543e28d8d), 17, UINT64_C(0xc47810ca7ea08204)}, + {4, 0, 35, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 3, 38, UINT64_C(0x449d737b6d7045cc), 18, UINT64_C(0x634cab1e73430dc8), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 36, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 4, 40, UINT64_C(0xd8f447b1c5fb9216), 19, UINT64_C(0xe73f82bc8b648d68), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 37, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 3, 40, UINT64_C(0xd8f447b1c5fb9216), 20, UINT64_C(0xa643db5d08c58deb), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 38, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 4, 42, UINT64_C(0x502e87ffe0865028), 21, UINT64_C(0xf96ee5bab261b4ef), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 39, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 3, 42, UINT64_C(0x502e87ffe0865028), 22, UINT64_C(0x3d36dc7057ad8e3c), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 40, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 3, 43, UINT64_C(0x916885ca66ca7fee), 23, UINT64_C(0x8993257e27b863ee), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 41, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 2, 43, UINT64_C(0x916885ca66ca7fee), 24, UINT64_C(0xb355554d556a2233), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 42, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 2, 44, UINT64_C(0xde8cf0eca99ecd68), 25, UINT64_C(0xc52002f46dcf2203), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 43, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 1, 44, UINT64_C(0xde8cf0eca99ecd68), 26, UINT64_C(0x3628a04973bf9fa0), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 0, 44, 6, "key", "[[\"recovery\"],{\"\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 2, 46, UINT64_C(0x2e23e78c9567bb32), 26, UINT64_C(0x3628a04973bf9fa0), 18, UINT64_C(0xed49e4670d08ef12)}, + {4, 0, 45, 6, "key", "[[\"recovery\"],{\"\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 46, UINT64_C(0x2e23e78c9567bb32), 26, UINT64_C(0x3628a04973bf9fa0), 19, UINT64_C(0xc217b532e08d04bf)}, + {4, 0, 46, 6, "key", "[[\"recovery\"],{\"\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0x3628a04973bf9fa0), 20, UINT64_C(0xdda94b1d3104c283)}, + {4, 0, 47, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 0, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0x3628a04973bf9fa0), 21, UINT64_C(0xf0d02532e8d6f88c)}, + {4, 1, 0, 6, "", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 46, UINT64_C(0x92dd27940b88cdee), 26, UINT64_C(0x709eb1bd3f80136c), 19, UINT64_C(0xcc473bf941fe55b3)}, + {4, 1, 1, 6, "", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0xdc138678b8467c04), 20, UINT64_C(0x39553f39ddffd6fd)}, + {4, 1, 2, 0, "key", "null", "0:0;", 1, 3, UINT64_C(0xa922fa18e1c37583), 0, UINT64_C(0xcbf29ce484222325), 2, UINT64_C(0x08395307b4f1348c)}, + {4, 1, 3, 0, "key", "null", "0:0;", 1, 4, UINT64_C(0x29e49b479f20e3f9), 1, UINT64_C(0xaf63be4c8601b992), 2, UINT64_C(0x08395307b4f1348c)}, + {4, 1, 4, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 25, UINT64_C(0x61ac871f77e3e9ef), 21, UINT64_C(0x745e4c543ea53008)}, + {4, 1, 5, 6, "key", "[]", "6:0a0:1[];", 1, 6, UINT64_C(0x67f9d4d46dd1ec91), 1, UINT64_C(0xaf63bd4c8601b7df), 5, UINT64_C(0x3f3fcc261a338844)}, + {4, 1, 6, 6, "key", "[]", "6:0a0:1[];", 1, 7, UINT64_C(0x7f7145f69bb51593), 1, UINT64_C(0xaf63bb4c8601b479), 5, UINT64_C(0x3f3fcc261a338844)}, + {4, 1, 7, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 43, UINT64_C(0x62d08462f86429b0), 21, UINT64_C(0xbc4aea29c30224e9), 21, UINT64_C(0x2195d3c69dbd95a4)}, + {4, 1, 8, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 44, UINT64_C(0x7887af0e0f014fba), 22, UINT64_C(0x0e0620c2831b2fa8), 21, UINT64_C(0xd4097f69bb24ded4)}, + {4, 1, 9, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 25, UINT64_C(0xe9abf74807bfdead), 21, UINT64_C(0xc78a1d641ccded9c)}, + {4, 1, 10, 6, "key", "[{\"k\":1}]", "6:0a1:1[7:1o1:1{3:1;};];", 1, 34, UINT64_C(0xaca4dcad768758b3), 20, UINT64_C(0x2b1254f33a770e0f), 13, UINT64_C(0x45a62b3550081448)}, + {4, 1, 11, 6, "key", "[null,{\"k\":1}]", "6:0a2:1[0:0;7:1o1:1{3:1;};];", 1, 35, UINT64_C(0x10f4d932a9856119), 20, UINT64_C(0xf63d69101e63889e), 14, UINT64_C(0x8afec3ac50f976fd)}, + {4, 1, 12, 6, "key", "[null,{\"k\":1}]", "6:0a2:1[0:0;7:1o1:1{3:1;};];", 1, 36, UINT64_C(0x463f5c38b808c643), 21, UINT64_C(0xb1e86ad690faa15b), 14, UINT64_C(0x4363778309ad8d0d)}, + {4, 1, 13, 6, "key", "[[],{\"k\":1}]", "6:0a2:1[6:0a0:1[];7:1o1:1{3:1;};];", 1, 39, UINT64_C(0xa97deac3ff265a29), 21, UINT64_C(0x1282acbd55a15693), 17, UINT64_C(0xe511760c18ca0850)}, + {4, 1, 14, 6, "key", "[[],{\"k\":1}]", "6:0a2:1[6:0a0:1[];7:1o1:1{3:1;};];", 1, 38, UINT64_C(0xd16edb648a6ce1ab), 21, UINT64_C(0xd366c3eec13151bd), 17, UINT64_C(0xa892f032cc56afb4)}, + {4, 1, 15, 6, "key", "[[],{\"k\":1}]", "6:0a2:1[6:0a0:1[];7:1o1:1{3:1;};];", 1, 39, UINT64_C(0xa97deac3ff265a29), 21, UINT64_C(0x430b89b128d98bc9), 17, UINT64_C(0x0ffc238e12d8a034)}, + {4, 1, 16, 6, "key", "[[],{\"k\":1}]", "6:0a2:1[6:1a0:1[];7:1o1:1{3:1;};];", 1, 40, UINT64_C(0x064800083d112433), 23, UINT64_C(0x7e3dde20e2c4af28), 17, UINT64_C(0xa9c98a3ab9b643c0)}, + {4, 1, 17, 6, "key", "[[],{\"k\":1}]", "6:0a2:1[6:1a0:1[];7:1o1:1{3:1;};];", 1, 41, UINT64_C(0x960821877b217001), 23, UINT64_C(0xc7fcea40891a9322), 17, UINT64_C(0xae16f39d760e7cc8)}, + {4, 1, 18, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 46, UINT64_C(0x4adbffdc69541eea), 24, UINT64_C(0x76857573ce5936df), 21, UINT64_C(0xae785b5857c280a0)}, + {4, 1, 19, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 25, UINT64_C(0x4b847b2b3084d143), 21, UINT64_C(0xd558d4feac336e68)}, + {4, 1, 20, 6, "key", "[[],{\"k\":1}]", "6:0a2:1[6:1a0:1[];7:1o1:1{3:1;};];", 1, 44, UINT64_C(0x4e84d7d79cb12b43), 26, UINT64_C(0x2827871680da954c), 17, UINT64_C(0x282bda8ef28461dc)}, + {4, 1, 21, 6, "key", "[[null],{\"k\":1}]", "6:0a2:1[6:1a1:1[0:0;];7:1o1:1{3:1;};];", 1, 45, UINT64_C(0x50892a76fb509509), 26, UINT64_C(0x2fe37edc7395b8a5), 18, UINT64_C(0x9b143889fe3f908f)}, + {4, 1, 22, 6, "key", "[[null],{\"k\":1}]", "6:0a2:1[6:1a1:1[0:0;];7:1o1:1{3:1;};];", 1, 46, UINT64_C(0x3e337cb4623144cb), 27, UINT64_C(0xf0778577f32c1518), 18, UINT64_C(0x76545b682a66d80f)}, + {4, 1, 23, 6, "key", "[[\"\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0xba5d9fa327aa54c7), 20, UINT64_C(0x152a5133a69aacd2)}, + {4, 1, 24, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 43, UINT64_C(0x545e9aeceaf07070), 21, UINT64_C(0xb358f17a3ddbd0f3), 21, UINT64_C(0x7f7246030eab6d5c)}, + {4, 1, 25, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 44, UINT64_C(0x5de102f5bf79df84), 22, UINT64_C(0x6d16a137cb9e81de), 21, UINT64_C(0xc94584b061292514)}, + {4, 1, 26, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 45, UINT64_C(0xfdf3d577ce49b8f8), 23, UINT64_C(0x7116aa28dc5729ce), 21, UINT64_C(0x7fb420974764fe44)}, + {4, 1, 27, 6, "key", "[[\"recovery\"]]", "6:0a1:1[6:1a1:1[5:1;];];", 1, 28, UINT64_C(0x33fb34a26c513b2e), 14, UINT64_C(0x0e819e1d9dff465c), 13, UINT64_C(0xc70e457618281d24)}, + {4, 1, 28, 6, "key", "[[\"recovery\"],null]", "6:0a2:1[6:1a1:1[5:1;];0:0;];", 1, 29, UINT64_C(0xa51595fe0e078192), 14, UINT64_C(0x0e819e1d9dff465c), 14, UINT64_C(0xf7ddd57b3ac52386)}, + {4, 1, 29, 6, "key", "[[\"recovery\"],null]", "6:0a2:1[6:1a1:1[5:1;];0:0;];", 1, 30, UINT64_C(0x8b2f90b1d6c16176), 15, UINT64_C(0x27a4fc9d713e1dcf), 14, UINT64_C(0xf7ddd57b3ac52386)}, + {4, 1, 30, 6, "key", "[[\"recovery\"],null]", "6:0a2:1[6:1a1:1[5:1;];0:0;];", 1, 31, UINT64_C(0x4333fc2fea9805fa), 16, UINT64_C(0xb34108ccff78652b), 14, UINT64_C(0xf7ddd57b3ac52386)}, + {4, 1, 31, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:0o0:1{};];", 1, 32, UINT64_C(0xc95b5f6ba051f16e), 15, UINT64_C(0x60ba726eb8e9eafe), 17, UINT64_C(0x1fbdcc46db94b748)}, + {4, 1, 32, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:0o0:1{};];", 1, 33, UINT64_C(0x783429e16b3c8d32), 15, UINT64_C(0xcdedec10fab123de), 17, UINT64_C(0x1fbdcc46db94b748)}, + {4, 1, 33, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 1, 34, UINT64_C(0x7d303c0937e3b596), 17, UINT64_C(0x0e22e134963af1ff), 17, UINT64_C(0xc47810ca7ea08204)}, + {4, 1, 34, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 1, 35, UINT64_C(0x9cac01a9f7ee3e9a), 17, UINT64_C(0x9d9d28d543e28d8d), 17, UINT64_C(0xc47810ca7ea08204)}, + {4, 1, 35, 6, "key", "[[\"recovery\"],{\"\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 40, UINT64_C(0xa3938fb1a75a386c), 20, UINT64_C(0xa643db5d08c58deb), 19, UINT64_C(0xfa57cf84a8b53e9b)}, + {4, 1, 36, 6, "key", "[[\"recovery\"],{\"\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 44, UINT64_C(0x9e7a111eaeebebf2), 23, UINT64_C(0xb411294cf9d6b5fc), 20, UINT64_C(0x76cdc90ed9ee8f5a)}, + {4, 1, 37, 6, "key", "[[\"recovery\"],{\"\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 42, UINT64_C(0x4eb201ffdf431a66), 22, UINT64_C(0x3d36dc7057ad8e3c), 19, UINT64_C(0xb4b45f3eec4e7aa7)}, + {4, 1, 38, 6, "key", "[[\"recovery\"],{\"\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 1, 45, UINT64_C(0x32b0d42505b4e994), 24, UINT64_C(0xb355554d556a2233), 20, UINT64_C(0xe8452972970cdff5)}, + {4, 1, 39, 6, "key", "[[\"recovery\"],{\"k\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 45, UINT64_C(0x939601fb7cc7322c), 24, UINT64_C(0xb355554d556a2233), 20, UINT64_C(0xe8452972970cdff5)}, + {4, 1, 40, 6, "key", "[[\"recovery\"],{\"k\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 46, UINT64_C(0x2e38658c95793a56), 25, UINT64_C(0xc52002f46dcf2203), 20, UINT64_C(0x69a2acc0a7d1c9c0)}, + {4, 1, 41, 6, "key", "[[\"recovery\"],{\"\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 44, UINT64_C(0xdf1b42ecaa175e04), 24, UINT64_C(0xb355554d556a2233), 19, UINT64_C(0x957fcb3b059cab5b)}, + {4, 1, 42, 6, "key", "[[\"recovery\"],{\"\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 46, UINT64_C(0x2e23e78c9567bb32), 25, UINT64_C(0xc52002f46dcf2203), 20, UINT64_C(0x69a2acc0a7d1c9c0)}, + {4, 1, 43, 6, "key", "[[\"recovery\"],{}]", "6:0a2:1[6:1a1:1[5:1;];7:1o0:1{};];", 1, 44, UINT64_C(0xde8cf0eca99ecd68), 26, UINT64_C(0x3628a04973bf9fa0), 17, UINT64_C(0x6bb402bf755397c8)}, + {4, 1, 44, 6, "key", "[[\"recovery\"],{\"k\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0x3628a04973bf9fa0), 20, UINT64_C(0xdda94b1d3104c283)}, + {4, 1, 45, 6, "key", "[[\"recovery\"],{\"\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 46, UINT64_C(0x2e23e78c9567bb32), 26, UINT64_C(0x3628a04973bf9fa0), 19, UINT64_C(0xc217b532e08d04bf)}, + {4, 1, 46, 6, "key", "[[\"recovery\"],{\"\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0x3628a04973bf9fa0), 20, UINT64_C(0xdda94b1d3104c283)}, + {4, 1, 47, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 0, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0x3628a04973bf9fa0), 21, UINT64_C(0xf0d02532e8d6f88c)}, +}; +int main() { + PairTrace trace; PairAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize pair allocation contracts"); + const char* documents[]{"", "null", "\"recovery\"", "{\"k\":\"recovery\",\"o\":{\"q\":1},\"a\":[true]}", "[[\"recovery\"],{\"k\":1}]"}; + const auto moduleBaseline = trace.live; + for (const auto& expected : Cases) for (bool base : {false, true}) { + String key{}; _ZN3sce4Json6StringC1EPKc(&key, "key"); + Value source{}; _ZN3sce4Json5ValueC1Ev(&source); + const char* document = documents[expected.source]; + if (expected.source) Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&source, document, std::strlen(document)) == 0, "Construct independent pair source"); + const auto sourceText = Serialize(source); + const auto baseline = trace.live; + Pair pair; std::memset(&pair, 0xa5, sizeof(pair)); + const auto first = trace.requests.size(), freeFirst = trace.releases.size(), generation = trace.generation; + const auto notifications = trace.notices; + trace.budget = expected.budget; trace.once = expected.once; + if (!expected.source) { + if (base) _ZN3sce4Json6Object4PairC2Ev(&pair); + else _ZN3sce4Json6Object4PairC1Ev(&pair); + } else { + if (base) _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); + else _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); + } + const auto end = trace.requests.size(), freeEnd = trace.releases.size(); + const auto errors = trace.notices - notifications; + trace.budget = -1; trace.once = false; + CheckMetadata(pair, UINT64_C(0xa5a5a5a5a5a5a5a5)); + std::string shape; Shape(pair.value, nullptr, shape); + const auto text = Serialize(pair.value); + Require(pair.value.type == expected.type && shape == expected.shape && text == expected.text && + std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), expected.key) == 0, + "Partial pair construction must preserve reference contents, collection counts, child parents and storage owners"); + Require(end - first == expected.requests && Fingerprint(trace.requests, first, end) == expected.requestOrder && errors == expected.notifications, + "Pair construction must preserve exact allocation request order and notifications at every failure point"); + Require(freeEnd - freeFirst == expected.constructionReleases && Fingerprint(trace.releases, freeFirst, freeEnd, generation) == expected.constructionOrder, + "Failed pair copies must preserve exact temporary release identities and observable null frees"); + const auto teardown = trace.releases.size(); + if (base) _ZN3sce4Json6Object4PairD2Ev(&pair); + else _ZN3sce4Json6Object4PairD1Ev(&pair); + Require(trace.releases.size() - teardown == expected.destructionReleases && Fingerprint(trace.releases, teardown, trace.releases.size(), generation) == expected.destructionOrder, + "Pair destruction must release the value before its key in the reference allocation-identity order"); + Require(trace.live == baseline && Serialize(source) == sourceText, "Partial pair cleanup must preserve every source allocation generation and value"); + _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json6StringD1Ev(&key); + Require(trace.live == moduleBaseline, "Pair failure schedules must leave only the original module resources"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Release pair allocation resources"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 56b980ba5b..ca6037d519 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -8,7 +8,7 @@ - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter is now reproduced through the inspected float32 conversion, six truncating fractional digits, range saturation and NaN integer result. The regression covers signed zero, precision boundaries, large values, infinities and NaNs; floating-point rounding modes and exception flags remain unverified. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. -- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocation failures and nested child binding still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. +- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and older-SDK copies still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - libc-heap failures and untested legacy callback combinations remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Additional legacy compatibility remains unverified. - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback-value failure behavior and compatibility below SDK 6 remain incomplete. @@ -67,3 +67,5 @@ - [Parser parent and auxiliary transitions](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-driven lifecycle changes, more complex nested metadata failures and alternate SDK combinations still require audit. Six notifier-reentry cases cover failure publication after notification, nested failures, callback-return copying and retry. Reader-driven allocation failures are checked before starting a value, following `0x5d30`; an already-started literal still completes, while outermost completion reports and clears the shared failure. Reference tests cover direct parsing into an owned child across eight value forms, six root-admission failure/retry cases, incomplete-literal preservation, modern root callback retention and nested release of three handler records, fallback and auxiliary in native order. Allocation generations distinguish release from later address reuse. - [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - more complex object shapes, allocator-driven lifecycle changes and alternate SDK combinations remain unverified. Twelve reader/notifier-reentry cases verify delayed failure publication, inherited nested failures, string/object/nested-array callback returns, literal completion, request-order fingerprints and retry. Reference tests cover 834 persistent/one-shot cases through successful string, one-entry object and nested array/object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. + +- [Object Pair allocation and teardown](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-bearing sources, allocator-driven reentry and alternate SDK behavior remain unverified. Reference tests cover 660 modern persistent/one-shot cases through every allocation position for default, null, string, nested-object and nested-array copies across both constructor/destructor aliases. Checks preserve inactive bytes, partial contents, collection counts, child parents, storage owners, allocation generations, request/notification order and construction/destruction release identities. Pair teardown releases the value before the key, following `0xc2b0`. Failed temporary header allocation in modern collection assignment still calls the allocator free function with null, following `0xbc80` and `0xd400`; failed sentinel allocation reclaims its admitted header. From a9dd8a8f42c1a80ecad9f2776ada1473f7c1ca90 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 09:07:45 +0200 Subject: [PATCH 054/318] test(libSceJson2): verify callback ownership across value copies --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/tests/GuestJsonCallbackCopies.cpp | 183 ++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 6 +- 3 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 core/libs/tests/GuestJsonCallbackCopies.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index dbb990d34b..100f9906ad 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_callback_copies_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCallbackCopies.cpp) + target_include_directories(guest_json_callback_copies_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_callback_copies_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_callback_copies_tests) + add_test(NAME guest_json_callback_copies COMMAND guest_json_callback_copies_tests) add_test_executable(guest_json_pair_allocator_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonPairAllocator.cpp) target_include_directories(guest_json_pair_allocator_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_pair_allocator_tests PRIVATE libSceJson2 libc) @@ -190,7 +195,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_pair_allocator guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_callback_copies guest_json_pair_allocator guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/tests/GuestJsonCallbackCopies.cpp b/core/libs/tests/GuestJsonCallbackCopies.cpp new file mode 100644 index 0000000000..5e9e8b6f3e --- /dev/null +++ b/core/libs/tests/GuestJsonCallbackCopies.cpp @@ -0,0 +1,183 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Array { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using NullHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using MismatchHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +using FloatHandler = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +using ElementHandler = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair*, const String*, const Value*); +void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair*); +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullHandler, void*); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, MismatchHandler, void*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatHandler, void*); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, ElementHandler, void*); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct CopyAllocations { std::map live; std::size_t generation = 0; }; +static void* APS5_VABI AllocateCopy(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); + Require(trace.live.emplace(memory, ++trace.generation).second, "Callback copy allocations must have independent live generations"); + return memory; +} +static void APS5_VABI FreeCopy(void*, void* memory, void* context) { + if (!memory) return; + Require(static_cast(context)->live.erase(memory) == 1, "Callback copy teardown must release each allocation through its owner exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyCopy(void*, int, std::size_t, void*) { Require(false, "Healthy callback copies must not fail allocation"); } +static void* const CopyVtable[]{nullptr, nullptr, reinterpret_cast(AllocateCopy), reinterpret_cast(FreeCopy), reinterpret_cast(NotifyCopy)}; +struct CopyAllocator { void* const* vtable = CopyVtable; }; +struct CopyContext { + Value replacement{}; + const Value* receiver = nullptr; + unsigned nulls = 0, mismatches = 0, floats = 0, elements = 0; +}; +static const Value& APS5_VABI NullCopy(std::int32_t type, const Value* parent, void* context) { + Require(type == 2 && !parent, "Copied null recovery must preserve requested type and parent ABI"); + auto& state = *static_cast(context); ++state.nulls; + return state.replacement; +} +static const Value& APS5_VABI MismatchCopy(std::int32_t type, const Value* self, const Value* parent, void* context) { + auto& state = *static_cast(context); + Require(type == 2 && self == state.receiver && !parent, "Copied mismatch recovery must preserve receiver, type and parent ABI"); + ++state.mismatches; + return state.replacement; +} +static Value* APS5_VABI FloatCopy(Value* result, std::int32_t function, double number, const Value* parent, void* context, bool* error) { + Require(function == 1 && number == std::numeric_limits::infinity() && !parent && !error, "Copied float recovery must preserve its assignment ABI"); + auto& state = *static_cast(context); ++state.floats; + _ZN3sce4Json5ValueC1El(result, static_cast(state.replacement.payload)); + return result; +} +static void APS5_VABI ElementCopy(int error, const String* key, const std::size_t* index, const Value* parent, const Value* self, void* context) { + auto& state = *static_cast(context); + Require(static_cast(error) == 0x80848115 && key && !index && !parent && self == state.receiver && std::strcmp(_ZNK3sce4Json6String5c_strEv(key), "missing") == 0, "Copied element recovery must retain its exact key, receiver and parent ABI"); + ++state.elements; +} +static void Register(Value& value, CopyContext& context) { + context.receiver = &value; + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&value, NullCopy, &context) == 0, "Register source null recovery"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&value, MismatchCopy, &context) == 0, "Register source mismatch recovery"); + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, FloatCopy, &context) == 0, "Register source float recovery"); + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&value, ElementCopy, &context) == 0, "Register source element recovery"); +} +struct MetadataSnapshot { + std::array bytes; + std::array allocations; + std::array generations; +}; +static MetadataSnapshot Capture(const Value& value, const CopyAllocations& trace) { + MetadataSnapshot snapshot{}; + const auto* words = static_cast(value.auxiliary); + snapshot.allocations = {value.auxiliary, words[2], words[3], words[4], words[5]}; + std::memcpy(snapshot.bytes.data(), value.auxiliary, snapshot.bytes.size()); + for (unsigned i = 0; i < snapshot.allocations.size(); ++i) snapshot.generations[i] = trace.live.at(snapshot.allocations[i]); + return snapshot; +} +static void Verify(const Value& value, const MetadataSnapshot& snapshot, const CopyAllocations& trace) { + Require(value.auxiliary == snapshot.allocations[0] && std::memcmp(value.auxiliary, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "Copying must preserve complete source/destination auxiliary bytes and handler/context identities"); + for (unsigned i = 0; i < snapshot.allocations.size(); ++i) Require(trace.live.at(snapshot.allocations[i]) == snapshot.generations[i], "Copying must preserve auxiliary, fallback and all three callback-record allocation generations"); + Require(static_cast(snapshot.allocations[1])->parent == &value, "Copied callback metadata must retain its original fallback owner"); +} +static std::string Serialize(Value& value) { + String output{}; _ZN3sce4Json6StringC1EPKc(&output, ""); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &output) == 0, "Callback copy source must remain serializable"); + std::string text = _ZNK3sce4Json6String5c_strEv(&output); + _ZN3sce4Json6StringD1Ev(&output); + return text; +} +int main() { + CopyAllocations allocations; CopyAllocator allocator; Parameter parameter{&allocator, &allocations, 8192, 2, {}}; + std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize callback copy contracts"); + const auto baseline = allocations.live; + for (const char* document : {"null", "1.25", "{\"n\":null,\"s\":\"text\"}"}) for (unsigned mode = 0; mode < 5; ++mode) { + Value source{}, copy{}; Pair pair{}; String key{}; Array array{}; + _ZN3sce4Json5ValueC1Ev(&source); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&source, document, std::strlen(document)) == 0, "Construct callback-bearing source"); + CopyContext sourceContext, destinationContext; + _ZN3sce4Json5ValueC1El(&sourceContext.replacement, 77); _ZN3sce4Json5ValueC1El(&destinationContext.replacement, 99); + Register(source, sourceContext); + const auto sourceMetadata = Capture(source, allocations); + Value* target = © + MetadataSnapshot destinationMetadata{}; + if (mode == 0) _ZN3sce4Json5ValueC1ERKS1_(©, &source); + else if (mode == 1 || mode == 2) { + _ZN3sce4Json5ValueC1Ev(©); + if (mode == 2) { Register(copy, destinationContext); destinationMetadata = Capture(copy, allocations); } + _ZN3sce4Json5Value3setERKS1_(©, &source); + } else if (mode == 3) { + _ZN3sce4Json6StringC1EPKc(&key, "key"); + _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); target = &pair.value; + } else { + _ZN3sce4Json5ArrayC1Ev(&array); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &source); target = const_cast(_ZNK3sce4Json5Array5frontEv(&array)); + } + Require(!target->parent && target->type == source.type && Serialize(*target) == document, "Copies must retain source contents with independent root ownership"); + if (mode == 2) Verify(*target, destinationMetadata, allocations); + else Require(!target->auxiliary, "Modern constructors and unregistered destinations must not inherit source-local callback metadata"); + const auto* sourceInteger = _ZNK3sce4Json5Value10getIntegerEv(&source); + const auto* targetInteger = _ZNK3sce4Json5Value10getIntegerEv(target); + Require(sourceInteger == reinterpret_cast(&sourceContext.replacement.payload) && *sourceInteger == 77, "Source recovery must retain its exact borrowed replacement identity"); + Require(mode == 2 ? targetInteger == reinterpret_cast(&destinationContext.replacement.payload) && *targetInteger == 99 : *targetInteger == 0, "Copies must use destination handlers or typed defaults rather than source callbacks"); + if (source.type == 7) { _ZNK3sce4Json5ValueixEPKc(&source, "missing"); _ZNK3sce4Json5ValueixEPKc(target, "missing"); } + _ZN3sce4Json5Value3setEd(target, std::numeric_limits::infinity()); + Require(target->type == (mode == 2 ? 2 : 4) && target->payload == (mode == 2 ? UINT64_C(99) : std::bit_cast(std::numeric_limits::infinity())), "Copies must retain destination float recovery without dispatching source handlers"); + Require(sourceContext.nulls == unsigned(source.type == 0) && sourceContext.mismatches == unsigned(source.type != 0) && !sourceContext.floats && sourceContext.elements == unsigned(source.type == 7), "Copy consumers must not invoke source callbacks except explicit source operations"); + Require(destinationContext.nulls == unsigned(mode == 2 && source.type == 0) && destinationContext.mismatches == unsigned(mode == 2 && source.type != 0) && destinationContext.floats == unsigned(mode == 2) && destinationContext.elements == unsigned(mode == 2 && source.type == 7), "Assignment must preserve each destination callback and context independently"); + Verify(source, sourceMetadata, allocations); + Require(Serialize(source) == document, "Copied value mutation must preserve its independent source contents"); + if (mode == 2) Verify(*target, destinationMetadata, allocations); + if (mode == 3) { _ZN3sce4Json6Object4PairD1Ev(&pair); _ZN3sce4Json6StringD1Ev(&key); } + else if (mode == 4) _ZN3sce4Json5ArrayD1Ev(&array); + else _ZN3sce4Json5ValueD1Ev(©); + Verify(source, sourceMetadata, allocations); + _ZN3sce4Json5Value3setEd(&source, std::numeric_limits::infinity()); + Require(sourceContext.floats == 1 && source.type == 2 && source.payload == 77, "Source callbacks must remain usable after copied storage and destination callbacks are destroyed"); + Verify(source, sourceMetadata, allocations); + _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ValueD1Ev(&sourceContext.replacement); _ZN3sce4Json5ValueD1Ev(&destinationContext.replacement); + Require(allocations.live == baseline, "Callback copy cleanup must reclaim every temporary generation without replacing module resources"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && allocations.live.empty(), "Release callback copy resources"); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index ca6037d519..5ddad44e4f 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -40,7 +40,7 @@ - [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. -- [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copies, alternate SDK behavior and concurrent failure accounting remain unverified. +- [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copy failures, alternate SDK behavior and concurrent failure accounting remain unverified. - [sce::Json::Parser value admission](../../../core/libs/prx/libSceJson2/Export.cpp) - recognized values change type before root metadata admission, following `0x5d30` and `0x3280`; incomplete scalar tokens allocate no auxiliary record. Twelve reference-backed string cases cover persistent and one-shot failure at every allocation position, retained type, error status, request/notification counts, character retries and cleanup. A failed character allocation can discard the first character while later characters are admitted. More complex parser low-memory paths and callback-driven allocation failures remain incomplete; reference execution terminated during one missing-sentinel array case, so that case does not establish a verified recovery contract. @@ -68,4 +68,6 @@ - [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - more complex object shapes, allocator-driven lifecycle changes and alternate SDK combinations remain unverified. Twelve reader/notifier-reentry cases verify delayed failure publication, inherited nested failures, string/object/nested-array callback returns, literal completion, request-order fingerprints and retry. Reference tests cover 834 persistent/one-shot cases through successful string, one-entry object and nested array/object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. -- [Object Pair allocation and teardown](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-bearing sources, allocator-driven reentry and alternate SDK behavior remain unverified. Reference tests cover 660 modern persistent/one-shot cases through every allocation position for default, null, string, nested-object and nested-array copies across both constructor/destructor aliases. Checks preserve inactive bytes, partial contents, collection counts, child parents, storage owners, allocation generations, request/notification order and construction/destruction release identities. Pair teardown releases the value before the key, following `0xc2b0`. Failed temporary header allocation in modern collection assignment still calls the allocator free function with null, following `0xbc80` and `0xd400`; failed sentinel allocation reclaims its admitted header. +- [Object Pair allocation and teardown](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-bearing source failures, allocator-driven reentry and alternate SDK behavior remain unverified. Reference tests cover 660 modern persistent/one-shot cases through every allocation position for default, null, string, nested-object and nested-array copies across both constructor/destructor aliases. Checks preserve inactive bytes, partial contents, collection counts, child parents, storage owners, allocation generations, request/notification order and construction/destruction release identities. Pair teardown releases the value before the key, following `0xc2b0`. Failed temporary header allocation in modern collection assignment still calls the allocator free function with null, following `0xbc80` and `0xd400`; failed sentinel allocation reclaims its admitted header. + +- [Modern callback-bearing copies](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation-failure combinations, nested owners, lifecycle changes from callbacks and alternate SDK behavior remain unverified. Fifteen reference-backed cases exercise null, real and object sources with all four local handlers through Value construction, assignment with/without destination handlers, Pair construction and standalone Array insertion. Modern copies discard source-local handlers; assignment retains destination handlers. Tests invoke null/mismatch/element/float recovery and verify complete auxiliary bytes, fallback ownership, all callback-record generations, borrowed replacement identity and source-handler usability after copy destruction. From 859dd1d4cd1a928e706c08ac94e7ee58646fc1a6 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 09:11:55 +0200 Subject: [PATCH 055/318] test(libSceJson2): preserve source callbacks through pair copy failures --- core/libs/tests/GuestJsonPairAllocator.cpp | 86 +++++++++++++++++++++- docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/core/libs/tests/GuestJsonPairAllocator.cpp b/core/libs/tests/GuestJsonPairAllocator.cpp index 3a6234947f..6b9f6b49b5 100644 --- a/core/libs/tests/GuestJsonPairAllocator.cpp +++ b/core/libs/tests/GuestJsonPairAllocator.cpp @@ -1,4 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include +#include #include #include #include @@ -12,7 +14,19 @@ struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint3 struct String { void* storage; }; struct Pair { String key; std::uint64_t reserved; Value value; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using NullHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using MismatchHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +using FloatHandler = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +using ElementHandler = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); extern "C" { +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullHandler, void*); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, MismatchHandler, void*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatHandler, void*); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, ElementHandler, void*); void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); @@ -468,17 +482,73 @@ static const Expected Cases[]{ {4, 1, 46, 6, "key", "[[\"recovery\"],{\"\":0}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:0;};];", 1, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0x3628a04973bf9fa0), 20, UINT64_C(0xdda94b1d3104c283)}, {4, 1, 47, 6, "key", "[[\"recovery\"],{\"k\":1}]", "6:0a2:1[6:1a1:1[5:1;];7:1o1:1{3:1;};];", 0, 47, UINT64_C(0xcebda3e1df431290), 26, UINT64_C(0x3628a04973bf9fa0), 21, UINT64_C(0xf0d02532e8d6f88c)}, }; +struct PairCallbackContext { + Value replacement{}; + const Value* receiver = nullptr; + unsigned nulls = 0, mismatches = 0, floats = 0, elements = 0; +}; +static const Value& APS5_VABI NullCopy(std::int32_t type, const Value* parent, void* context) { + Require(type == 2 && !parent, "Copied null recovery must preserve requested type and parent ABI"); + auto& state = *static_cast(context); ++state.nulls; + return state.replacement; +} +static const Value& APS5_VABI MismatchCopy(std::int32_t type, const Value* self, const Value* parent, void* context) { + auto& state = *static_cast(context); + Require(type == 2 && self == state.receiver && !parent, "Copied mismatch recovery must preserve receiver, type and parent ABI"); + ++state.mismatches; + return state.replacement; +} +static Value* APS5_VABI FloatCopy(Value* result, std::int32_t function, double number, const Value* parent, void* context, bool* error) { + Require(function == 1 && number == std::numeric_limits::infinity() && !parent && !error, "Copied float recovery must preserve its assignment ABI"); + auto& state = *static_cast(context); ++state.floats; + _ZN3sce4Json5ValueC1El(result, static_cast(state.replacement.payload)); + return result; +} +static void APS5_VABI ElementCopy(int error, const String* key, const std::size_t* index, const Value* parent, const Value* self, void* context) { + auto& state = *static_cast(context); + Require(static_cast(error) == 0x80848115 && key && !index && !parent && self == state.receiver && std::strcmp(_ZNK3sce4Json6String5c_strEv(key), "missing") == 0, "Copied element recovery must retain its exact key, receiver and parent ABI"); + ++state.elements; +} +static void Register(Value& value, PairCallbackContext& context) { + context.receiver = &value; + Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&value, NullCopy, &context) == 0, "Register source null recovery"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&value, MismatchCopy, &context) == 0, "Register source mismatch recovery"); + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, FloatCopy, &context) == 0, "Register source float recovery"); + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&value, ElementCopy, &context) == 0, "Register source element recovery"); +} +struct MetadataSnapshot { + std::array bytes; + std::array allocations; + std::array generations; +}; +static MetadataSnapshot Capture(const Value& value, const PairTrace& trace) { + MetadataSnapshot snapshot{}; + const auto* words = static_cast(value.auxiliary); + snapshot.allocations = {value.auxiliary, words[2], words[3], words[4], words[5]}; + std::memcpy(snapshot.bytes.data(), value.auxiliary, snapshot.bytes.size()); + for (unsigned i = 0; i < snapshot.allocations.size(); ++i) snapshot.generations[i] = trace.live.at(snapshot.allocations[i]); + return snapshot; +} +static void Verify(const Value& value, const MetadataSnapshot& snapshot, const PairTrace& trace) { + Require(value.auxiliary == snapshot.allocations[0] && std::memcmp(value.auxiliary, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "Copying must preserve complete source/destination auxiliary bytes and handler/context identities"); + for (unsigned i = 0; i < snapshot.allocations.size(); ++i) Require(trace.live.at(snapshot.allocations[i]) == snapshot.generations[i], "Copying must preserve auxiliary, fallback and all three callback-record allocation generations"); + Require(static_cast(snapshot.allocations[1])->parent == &value, "Copied callback metadata must retain its original fallback owner"); +} int main() { PairTrace trace; PairAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize pair allocation contracts"); const char* documents[]{"", "null", "\"recovery\"", "{\"k\":\"recovery\",\"o\":{\"q\":1},\"a\":[true]}", "[[\"recovery\"],{\"k\":1}]"}; const auto moduleBaseline = trace.live; - for (const auto& expected : Cases) for (bool base : {false, true}) { + for (const auto& expected : Cases) for (bool base : {false, true}) for (bool callbackSource : {false, true}) { String key{}; _ZN3sce4Json6StringC1EPKc(&key, "key"); Value source{}; _ZN3sce4Json5ValueC1Ev(&source); const char* document = documents[expected.source]; if (expected.source) Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&source, document, std::strlen(document)) == 0, "Construct independent pair source"); + PairCallbackContext context; + _ZN3sce4Json5ValueC1El(&context.replacement, 77); + MetadataSnapshot sourceMetadata{}; + if (callbackSource) { Register(source, context); sourceMetadata = Capture(source, trace); } const auto sourceText = Serialize(source); const auto baseline = trace.live; Pair pair; std::memset(&pair, 0xa5, sizeof(pair)); @@ -511,7 +581,19 @@ int main() { Require(trace.releases.size() - teardown == expected.destructionReleases && Fingerprint(trace.releases, teardown, trace.releases.size(), generation) == expected.destructionOrder, "Pair destruction must release the value before its key in the reference allocation-identity order"); Require(trace.live == baseline && Serialize(source) == sourceText, "Partial pair cleanup must preserve every source allocation generation and value"); - _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json6StringD1Ev(&key); + if (callbackSource) { + Verify(source, sourceMetadata, trace); + const auto* borrowed = _ZNK3sce4Json5Value10getIntegerEv(&source); + Require(borrowed == reinterpret_cast(&context.replacement.payload) && *borrowed == 77, + "Source null/mismatch callbacks must retain their borrowed result identity after failed copies are destroyed"); + const bool nullSource = source.type == 0, objectSource = source.type == 7; + if (objectSource) _ZNK3sce4Json5ValueixEPKc(&source, "missing"); + _ZN3sce4Json5Value3setEd(&source, std::numeric_limits::infinity()); + Require(source.type == 2 && source.payload == 77 && context.nulls == unsigned(nullSource) && context.mismatches == unsigned(!nullSource) && context.elements == unsigned(objectSource) && context.floats == 1, + "Every partial-copy failure must leave the source's four independent handlers usable and correctly bound"); + Verify(source, sourceMetadata, trace); + } + _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ValueD1Ev(&context.replacement); _ZN3sce4Json6StringD1Ev(&key); Require(trace.live == moduleBaseline, "Pair failure schedules must leave only the original module resources"); } Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Release pair allocation resources"); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 5ddad44e4f..5dc4942ad8 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -68,6 +68,6 @@ - [Special-float callback-return allocation](../../../core/libs/prx/libSceJson2/Export.cpp) - more complex object shapes, allocator-driven lifecycle changes and alternate SDK combinations remain unverified. Twelve reader/notifier-reentry cases verify delayed failure publication, inherited nested failures, string/object/nested-array callback returns, literal completion, request-order fingerprints and retry. Reference tests cover 834 persistent/one-shot cases through successful string, one-entry object and nested array/object recovery for assignment and serialization. Checks include partial contents, internal-key lookup, allocation-order fingerprints, notifications, callback ABI, destination metadata generations and complete cleanup. Object assignment copies the internal tree key separately from the public pair key, following `0xd400`; these can diverge after allocation failure. -- [Object Pair allocation and teardown](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-bearing source failures, allocator-driven reentry and alternate SDK behavior remain unverified. Reference tests cover 660 modern persistent/one-shot cases through every allocation position for default, null, string, nested-object and nested-array copies across both constructor/destructor aliases. Checks preserve inactive bytes, partial contents, collection counts, child parents, storage owners, allocation generations, request/notification order and construction/destruction release identities. Pair teardown releases the value before the key, following `0xc2b0`. Failed temporary header allocation in modern collection assignment still calls the allocator free function with null, following `0xbc80` and `0xd400`; failed sentinel allocation reclaims its admitted header. +- [Object Pair allocation and teardown](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and alternate SDK behavior remain unverified. Reference tests cover 1,320 modern persistent/one-shot cases with and without four registered source handlers through every allocation position for default, null, string, nested-object and nested-array copies across both constructor/destructor aliases. Checks preserve inactive bytes, partial contents, collection counts, child parents, storage owners, allocation generations, request/notification order and construction/destruction release identities. Source auxiliary bytes, generations and handlers survive cleanup. Pair teardown releases the value before the key, following `0xc2b0`. Failed temporary header allocation in modern collection assignment still calls the allocator free function with null, following `0xbc80` and `0xd400`; failed sentinel allocation reclaims its admitted header. - [Modern callback-bearing copies](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation-failure combinations, nested owners, lifecycle changes from callbacks and alternate SDK behavior remain unverified. Fifteen reference-backed cases exercise null, real and object sources with all four local handlers through Value construction, assignment with/without destination handlers, Pair construction and standalone Array insertion. Modern copies discard source-local handlers; assignment retains destination handlers. Tests invoke null/mismatch/element/float recovery and verify complete auxiliary bytes, fallback ownership, all callback-record generations, borrowed replacement identity and source-handler usability after copy destruction. From c2776bddd18996df89a837bf1ba3e72147d3f009 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 09:24:48 +0200 Subject: [PATCH 056/318] fix(libSceJson2): reject notifier teardown during active allocation operations --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- core/libs/prx/libSceJson2/Export.cpp | 16 +- .../libs/tests/GuestJsonCallbackLifecycle.cpp | 142 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 6 +- 4 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 core/libs/tests/GuestJsonCallbackLifecycle.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 100f9906ad..2839ec0f8e 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_callback_lifecycle_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCallbackLifecycle.cpp) + target_include_directories(guest_json_callback_lifecycle_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_callback_lifecycle_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_callback_lifecycle_tests) + add_test(NAME guest_json_callback_lifecycle COMMAND guest_json_callback_lifecycle_tests) add_test_executable(guest_json_callback_copies_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCallbackCopies.cpp) target_include_directories(guest_json_callback_copies_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_callback_copies_tests PRIVATE libSceJson2 libc) @@ -195,7 +200,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_callback_copies guest_json_pair_allocator guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_callback_lifecycle guest_json_callback_copies guest_json_pair_allocator guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 21a2c1fb6a..1e2da97dbd 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -166,6 +166,15 @@ Globals& State() { return globals; } +thread_local std::uint32_t notificationDepth = 0; + +struct NotificationScope { + NotificationScope() { ++notificationDepth; } + ~NotificationScope() { --notificationDepth; } + NotificationScope(const NotificationScope&) = delete; + NotificationScope& operator=(const NotificationScope&) = delete; +}; + struct KernelMutexLock { PthreadMutex* mutex; explicit KernelMutexLock(PthreadMutex* value) : mutex(value) { @@ -227,7 +236,10 @@ void* Allocate(const AllocationContext& context, std::size_t size) { void* memory = allocate(context.allocator, size, context.userData); if (!memory) { const auto notify = reinterpret_cast(table[4]); - notify(context.allocator, JsonErrorOutOfMemory, size, context.userData); + { + NotificationScope scope; + notify(context.allocator, JsonErrorOutOfMemory, size, context.userData); + } auto& state = State(); std::lock_guard lock(state.mutex); KernelMutexLock kernelLock(state.internalMutex); @@ -2535,7 +2547,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; - if (state.callbackActive) NotImplemented_nid_no_patch(__func__); + if (state.callbackActive || (notificationDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); state.callback = nullptr; state.context = nullptr; state.handlers = {}; diff --git a/core/libs/tests/GuestJsonCallbackLifecycle.cpp b/core/libs/tests/GuestJsonCallbackLifecycle.cpp new file mode 100644 index 0000000000..cd855893b8 --- /dev/null +++ b/core/libs/tests/GuestJsonCallbackLifecycle.cpp @@ -0,0 +1,142 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using FloatHandler = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatHandler, void*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +void* APS5_VABI _ZN3sce4Json6MallocEm(std::size_t); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +enum class NoticeAction { None, Terminate, RejectTermination }; +struct LifecycleTrace { + std::map live; + std::size_t generation = 0; + bool failNext = false; + std::uint8_t* owner = nullptr; + NoticeAction noticeAction = NoticeAction::None; + unsigned notices = 0; + unsigned refused = 0; +}; +static void* APS5_VABI AllocateLifecycle(void*, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + if (trace.failNext) { trace.failNext = false; return nullptr; } + void* memory = std::malloc(bytes); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); + Require(trace.live.emplace(memory, ++trace.generation).second, "Lifecycle resources must have independently live allocation generations"); + return memory; +} +static void APS5_VABI FreeLifecycle(void*, void* memory, void* context) { + if (!memory) return; + Require(static_cast(context)->live.erase(memory) == 1, "Lifecycle cleanup must release each allocation through its owner exactly once"); + std::free(memory); +} +static void RejectOwnerTermination(LifecycleTrace& trace) { + const auto before = trace.live; + bool rejected = false; + try { _ZN3sce4Json11Initializer9terminateEv(trace.owner); } + catch (const std::runtime_error& error) { + Require(std::strstr(error.what(), "terminate"), "Unsafe callback termination must use the explicit unsupported boundary"); + rejected = true; + } + Require(rejected, "Termination inside an active callback operation must be explicitly rejected before teardown"); + Require(*trace.owner == 1 && trace.live == before, "Rejected callback termination must preserve the entire module allocation generation and active byte"); + ++trace.refused; +} +static void APS5_VABI NotifyLifecycle(void*, int error, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(static_cast(error) == 0x80848102 && bytes == (trace.noticeAction == NoticeAction::Terminate ? 8u : 72u), "Lifecycle notifier must receive the actual failed allocation size and module error"); + ++trace.notices; + if (trace.noticeAction == NoticeAction::Terminate) { + try { + Require(_ZN3sce4Json11Initializer9terminateEv(trace.owner) == 0 && *trace.owner == 0 && trace.live.empty(), "A standalone Malloc notifier must retain reference-supported module termination"); + } catch (const std::runtime_error&) { Require(false, "Reference-supported standalone notifier termination must not be rejected"); } + } else if (trace.noticeAction == NoticeAction::RejectTermination) RejectOwnerTermination(trace); + else Require(false, "Healthy lifecycle operations must not unexpectedly fail allocation"); +} +static void* const LifecycleVtable[]{nullptr, nullptr, reinterpret_cast(AllocateLifecycle), reinterpret_cast(FreeLifecycle), reinterpret_cast(NotifyLifecycle)}; +struct LifecycleAllocator { void* const* vtable = LifecycleVtable; }; +struct RecoveryContext { LifecycleTrace* trace; bool terminateOwner; unsigned calls = 0; }; +static Value* APS5_VABI RecoverLifecycle(Value* result, std::int32_t function, double number, const Value* parent, void* context, bool* error) { + auto& state = *static_cast(context); + Require(function == 1 && number == std::numeric_limits::infinity() && !parent && !error, "Lifecycle recovery must preserve its assignment callback ABI"); + ++state.calls; + std::uint8_t inactive = 0; + Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&inactive, nullptr)) == 0x80848111 && inactive == 0, "Recovery reentry must reject a competing initializer before parameter validation"); + Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(&inactive)) == 0x80848110 && *state.trace->owner == 1, "Inactive termination must preserve active recovery and module ownership"); + if (state.terminateOwner) RejectOwnerTermination(*state.trace); + _ZN3sce4Json5ValueC1El(result, 7); + return result; +} +static void CheckRecovery(bool terminateOwner) { + LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; trace.owner = &initializer; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize recovery lifecycle contracts"); + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + RecoveryContext context{&trace, terminateOwner}; + Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, RecoverLifecycle, &context) == 0, "Register lifecycle recovery"); + const auto baseline = trace.live; + void* auxiliary = value.auxiliary; + for (unsigned attempt = 1; attempt <= 2; ++attempt) { + _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); + Require(value.type == 2 && value.payload == 7 && value.auxiliary == auxiliary && context.calls == attempt && trace.refused == (terminateOwner ? attempt : 0) && initializer == 1 && trace.live == baseline, + "Handled callback lifecycle operations must preserve recovery metadata, repeatability and complete module ownership"); + } + _ZN3sce4Json5ValueD1Ev(&value); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Ordinary termination must work after recovery callback reentry"); +} +static void CheckNotice(bool activeOperation) { + LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; trace.owner = &initializer; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize notifier lifecycle contracts"); + const auto baseline = trace.live; + trace.noticeAction = activeOperation ? NoticeAction::RejectTermination : NoticeAction::Terminate; + trace.failNext = true; + if (activeOperation) { + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + Require(static_cast(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, "true", 4)) == 0x80848102 && value.type == 1 && (value.payload & 0xff) == 1 && !value.auxiliary && initializer == 1 && trace.live == baseline, + "Rejected notifier termination must retain normal parser out-of-memory behavior and module ownership"); + Require(trace.refused == 1 && trace.notices == 1, "Active notifier termination must be rejected exactly once at the actual failure boundary"); + trace.noticeAction = NoticeAction::None; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, "true", 4) == 0 && value.auxiliary && initializer == 1, "Parser retry must work after a caught notifier lifecycle error"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live == baseline && _ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Ordinary teardown must reclaim resources after notifier refusal and retry"); + } else { + Require(!_ZN3sce4Json6MallocEm(8) && initializer == 0 && trace.live.empty() && trace.notices == 1 && !trace.refused, + "Standalone allocation failure must preserve successful notifier termination and its null allocation result"); + } + Require(initializer == 0 && trace.live.empty(), "Notifier lifecycle cases must fully release the original module generation"); + const auto generation = trace.generation; + trace.noticeAction = NoticeAction::None; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == 12, "The same initializer must admit a fresh module generation after notifier-driven termination or refusal"); + for (const auto& [memory, identity] : trace.live) Require(memory && identity > generation, "Lifecycle retry must never retain stale allocation identities"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release retried notifier lifecycle generation"); +} +int main() { + CheckRecovery(false); + CheckRecovery(true); + CheckNotice(false); + CheckNotice(true); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 5dc4942ad8..415f6d2f3f 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -5,7 +5,7 @@ - [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) - floating-point environment behavior and malformed numeric-token grammar remain unverified. Valid decimal conversion follows the instructions at `0x7750`: separate multiply/add accumulation, per-digit fractional division and repeated exponent scaling. Reference-backed exact-bit tests cover signed underflow, subnormals, normal boundaries, large finite values and overflow across special-float modes. These results intentionally retain the reference rounding, which differs from standard correctly rounded conversion. Formatting still uses locale-independent `to_chars` with 17 significant digits for modern settings; broader formatting equivalence remains unverified. - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) - the console result for an empty array remains unverified; the existing host behavior throws `std::out_of_range`. Missing storage throws unsupported. Nonempty removal releases the allocator-owned node without allocating and preserves surviving element/iterator identity. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size throws; decimal overflow follows the configured real-conversion mode. Numeric-token validation, more complex parser allocation failures and cross-thread operation behavior remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. -- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Kernel API failure paths, reentrant initialization/termination and full guest execution remain incomplete. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. +- [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Remaining kernel-status and callback/concurrent lifecycle paths, plus full guest execution, remain unverified. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter is now reproduced through the inspected float32 conversion, six truncating fractional digits, range saturation and NaN integer result. The regression covers signed zero, precision boundaries, large values, infinities and NaNs; floating-point rounding modes and exception flags remain unverified. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and older-SDK copies still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. @@ -36,7 +36,7 @@ - [sce::Json::Value nested array allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. -- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - returned kernel setup/locking errors and callback-time termination remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Reference tests cover SDK 5/6 allocation failures, partial admission, null frees and retry; SDK 6 tests also reject duplicate initialization and inactive termination from allocate/notify/free callbacks while the context remains published. Host-boundary tests exercise 17 healthy/error/exception schedules, including allocation exceptions from either mutex constructor, balanced guest/attribute/mutex ownership and retry on the same initializer. Setup exceptions reclaim local resources before global rollback. The reference skips creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. +- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - returned kernel setup/locking errors, recovery-callback termination and active-operation notifier termination remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Reference tests cover SDK 5/6 allocation failures, partial admission, null frees and retry; SDK 6 tests also reject duplicate initialization and inactive termination from allocate/notify/free callbacks while the context remains published. Seventeen host-boundary schedules verify mutex-construction errors/exceptions, balanced guest/attribute/mutex ownership and retry. Setup exceptions reclaim local resources before global rollback. The reference skips creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. - [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. @@ -71,3 +71,5 @@ - [Object Pair allocation and teardown](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and alternate SDK behavior remain unverified. Reference tests cover 1,320 modern persistent/one-shot cases with and without four registered source handlers through every allocation position for default, null, string, nested-object and nested-array copies across both constructor/destructor aliases. Checks preserve inactive bytes, partial contents, collection counts, child parents, storage owners, allocation generations, request/notification order and construction/destruction release identities. Source auxiliary bytes, generations and handlers survive cleanup. Pair teardown releases the value before the key, following `0xc2b0`. Failed temporary header allocation in modern collection assignment still calls the allocator free function with null, following `0xbc80` and `0xd400`; failed sentinel allocation reclaims its admitted header. - [Modern callback-bearing copies](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation-failure combinations, nested owners, lifecycle changes from callbacks and alternate SDK behavior remain unverified. Fifteen reference-backed cases exercise null, real and object sources with all four local handlers through Value construction, assignment with/without destination handlers, Pair construction and standalone Array insertion. Modern copies discard source-local handlers; assignment retains destination handlers. Tests invoke null/mismatch/element/float recovery and verify complete auxiliary bytes, fallback ownership, all callback-record generations, borrowed replacement identity and source-handler usability after copy destruction. + +- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - direct allocate/free callback termination, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and notifier termination within an active allocation operation are explicitly unsupported before teardown. The reference returned success, then reused a freed mutex slot after recovery or a null slot at parser exit; both faulted at the substituted kernel boundary. These are not console measurements. Reference-backed controls preserve inactive-initializer termination and standalone Malloc-notifier termination/reinitialization. Production tests verify refusal before ownership changes, repeatable recovery, parser OOM/retry and fresh module generations. From 3ed71062c4b1284580cf2b756080960504d7da5c Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 09:32:28 +0200 Subject: [PATCH 057/318] fix(libSceJson2): guard active allocation and free callback teardown --- core/libs/prx/libSceJson2/Export.cpp | 23 ++++--- .../libs/tests/GuestJsonCallbackLifecycle.cpp | 61 ++++++++++++++++++- docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 1e2da97dbd..24e100d8fc 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -166,13 +166,13 @@ Globals& State() { return globals; } -thread_local std::uint32_t notificationDepth = 0; +thread_local std::uint32_t allocatorCallbackDepth = 0; -struct NotificationScope { - NotificationScope() { ++notificationDepth; } - ~NotificationScope() { --notificationDepth; } - NotificationScope(const NotificationScope&) = delete; - NotificationScope& operator=(const NotificationScope&) = delete; +struct AllocatorCallbackScope { + AllocatorCallbackScope() { ++allocatorCallbackDepth; } + ~AllocatorCallbackScope() { --allocatorCallbackDepth; } + AllocatorCallbackScope(const AllocatorCallbackScope&) = delete; + AllocatorCallbackScope& operator=(const AllocatorCallbackScope&) = delete; }; struct KernelMutexLock { @@ -233,11 +233,15 @@ AllocParamRtti RttiCallbacks() { void* Allocate(const AllocationContext& context, std::size_t size) { auto* const* table = *static_cast(context.allocator); const auto allocate = reinterpret_cast(table[2]); - void* memory = allocate(context.allocator, size, context.userData); + void* memory; + { + AllocatorCallbackScope scope; + memory = allocate(context.allocator, size, context.userData); + } if (!memory) { const auto notify = reinterpret_cast(table[4]); { - NotificationScope scope; + AllocatorCallbackScope scope; notify(context.allocator, JsonErrorOutOfMemory, size, context.userData); } auto& state = State(); @@ -251,6 +255,7 @@ void* Allocate(const AllocationContext& context, std::size_t size) { void Release(const AllocationContext& context, void* memory) { auto* const* table = *static_cast(context.allocator); const auto free = reinterpret_cast(table[3]); + AllocatorCallbackScope scope; free(context.allocator, memory, context.userData); } @@ -2547,7 +2552,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; - if (state.callbackActive || (notificationDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); + if (state.callbackActive || (allocatorCallbackDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); state.callback = nullptr; state.context = nullptr; state.handlers = {}; diff --git a/core/libs/tests/GuestJsonCallbackLifecycle.cpp b/core/libs/tests/GuestJsonCallbackLifecycle.cpp index cd855893b8..6e238c14af 100644 --- a/core/libs/tests/GuestJsonCallbackLifecycle.cpp +++ b/core/libs/tests/GuestJsonCallbackLifecycle.cpp @@ -22,22 +22,30 @@ void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatHandler, void*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); void* APS5_VABI _ZN3sce4Json6MallocEm(std::size_t); +void APS5_VABI _ZN3sce4Json4FreeEPv(void*); } static void Require(bool condition, const char* message) { if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } } enum class NoticeAction { None, Terminate, RejectTermination }; +enum class AllocatorAction { None, RejectAllocate, RejectFree, TerminateFree }; struct LifecycleTrace { std::map live; std::size_t generation = 0; bool failNext = false; std::uint8_t* owner = nullptr; NoticeAction noticeAction = NoticeAction::None; + AllocatorAction allocatorAction = AllocatorAction::None; unsigned notices = 0; unsigned refused = 0; }; +static void RejectOwnerTermination(LifecycleTrace& trace); static void* APS5_VABI AllocateLifecycle(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); + if (trace.allocatorAction == AllocatorAction::RejectAllocate) { + trace.allocatorAction = AllocatorAction::None; + RejectOwnerTermination(trace); + } if (trace.failNext) { trace.failNext = false; return nullptr; } void* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); @@ -46,8 +54,20 @@ static void* APS5_VABI AllocateLifecycle(void*, std::size_t bytes, void* context return memory; } static void APS5_VABI FreeLifecycle(void*, void* memory, void* context) { + auto& trace = *static_cast(context); + if (trace.allocatorAction == AllocatorAction::RejectFree) { + trace.allocatorAction = AllocatorAction::None; + RejectOwnerTermination(trace); + } else if (trace.allocatorAction == AllocatorAction::TerminateFree) { + trace.allocatorAction = AllocatorAction::None; + const auto identity = trace.live.at(memory); + try { + Require(_ZN3sce4Json11Initializer9terminateEv(trace.owner) == 0 && *trace.owner == 0 && trace.live.size() == 1 && trace.live.at(memory) == identity, + "Standalone free-callback termination must release module resources while retaining the current allocation for its original free callback"); + } catch (const std::runtime_error&) { Require(false, "Reference-supported standalone free-callback termination must not be rejected"); } + } if (!memory) return; - Require(static_cast(context)->live.erase(memory) == 1, "Lifecycle cleanup must release each allocation through its owner exactly once"); + Require(trace.live.erase(memory) == 1, "Lifecycle cleanup must release each allocation through its owner exactly once"); std::free(memory); } static void RejectOwnerTermination(LifecycleTrace& trace) { @@ -133,10 +153,49 @@ static void CheckNotice(bool activeOperation) { for (const auto& [memory, identity] : trace.live) Require(memory && identity > generation, "Lifecycle retry must never retain stale allocation identities"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release retried notifier lifecycle generation"); } +static void CheckAllocator(bool allocate) { + LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; trace.owner = &initializer; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize direct allocator lifecycle contracts"); + const auto baseline = trace.live; + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + trace.allocatorAction = allocate ? AllocatorAction::RejectAllocate : AllocatorAction::RejectFree; + const char* document = allocate ? "true" : "42"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, document, std::strlen(document)) == 0 && initializer == 1 && trace.refused == 1 && trace.allocatorAction == AllocatorAction::None, + "Caught allocator callback termination must preserve actual parser completion and the active module"); + Require(allocate ? value.type == 1 && (value.payload & 0xff) == 1 : value.type == 3 && value.payload == 42, + "Allocator lifecycle refusal must preserve the parser's exact scalar result"); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, "false", 5) == 0 && value.type == 1 && !(value.payload & 0xff) && trace.refused == 1, + "Following operations must remain usable after allocation/free callback lifecycle refusal"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live == baseline && _ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), + "Direct allocator lifecycle refusal must preserve original module generations through ordinary teardown"); +} +static void CheckStandaloneFree() { + LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; trace.owner = &initializer; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0, "Initialize standalone free lifecycle contract"); + void* memory = _ZN3sce4Json6MallocEm(8); + Require(memory != nullptr, "Standalone lifecycle preservation requires a real allocated buffer"); + trace.allocatorAction = AllocatorAction::TerminateFree; + _ZN3sce4Json4FreeEPv(memory); + Require(initializer == 0 && trace.live.empty() && trace.allocatorAction == AllocatorAction::None && !trace.refused, + "Standalone free must complete callback-driven termination and release the original buffer"); + const auto generation = trace.generation; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == 12, + "Initialization must remain usable after standalone free-callback termination"); + for (const auto& [pointer, identity] : trace.live) Require(pointer && identity > generation, "Reinitialization must own fresh resource generations after free-callback termination"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release standalone free lifecycle retry"); +} int main() { CheckRecovery(false); CheckRecovery(true); CheckNotice(false); CheckNotice(true); + CheckAllocator(true); + CheckAllocator(false); + CheckStandaloneFree(); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 415f6d2f3f..0c4f6bc7f0 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -36,7 +36,7 @@ - [sce::Json::Value nested array allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local same-source tests cover every allocation point in copying and assigning an array containing a string, integer and inner array containing another string. Persistent and single-allocation failures preserve the observed null entries, empty strings, missing sentinels, parent links, error counts and actual owned cleanup. These checks do not complete nested object failure coverage, callback-return cloning failures, legacy SDK behavior or full guest execution. -- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - returned kernel setup/locking errors, recovery-callback termination and active-operation notifier termination remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Reference tests cover SDK 5/6 allocation failures, partial admission, null frees and retry; SDK 6 tests also reject duplicate initialization and inactive termination from allocate/notify/free callbacks while the context remains published. Seventeen host-boundary schedules verify mutex-construction errors/exceptions, balanced guest/attribute/mutex ownership and retry. Setup exceptions reclaim local resources before global rollback. The reference skips creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. +- [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - returned kernel setup/locking errors, recovery-callback termination and active-operation allocator-callback termination remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Reference tests cover SDK 5/6 allocation failures, partial admission, null frees and retry; SDK 6 tests also reject duplicate initialization and inactive termination from allocate/notify/free callbacks while the context remains published. Seventeen host-boundary schedules verify mutex-construction errors/exceptions, balanced guest/attribute/mutex ownership and retry. Setup exceptions reclaim local resources before global rollback. The reference skips creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. - [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. @@ -72,4 +72,4 @@ - [Modern callback-bearing copies](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation-failure combinations, nested owners, lifecycle changes from callbacks and alternate SDK behavior remain unverified. Fifteen reference-backed cases exercise null, real and object sources with all four local handlers through Value construction, assignment with/without destination handlers, Pair construction and standalone Array insertion. Modern copies discard source-local handlers; assignment retains destination handlers. Tests invoke null/mismatch/element/float recovery and verify complete auxiliary bytes, fallback ownership, all callback-record generations, borrowed replacement identity and source-handler usability after copy destruction. -- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - direct allocate/free callback termination, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and notifier termination within an active allocation operation are explicitly unsupported before teardown. The reference returned success, then reused a freed mutex slot after recovery or a null slot at parser exit; both faulted at the substituted kernel boundary. These are not console measurements. Reference-backed controls preserve inactive-initializer termination and standalone Malloc-notifier termination/reinitialization. Production tests verify refusal before ownership changes, repeatable recovery, parser OOM/retry and fresh module generations. +- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - teardown reentry, pre-publication allocation callbacks, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and allocate/notify/free callback termination inside an active allocation operation are explicitly unsupported before teardown. Reference recovery/notifier probes reused freed/null mutex slots after termination and faulted at the substituted kernel boundary; direct allocate/free probes also faulted after termination returned success. These are not console measurements. Reference-backed controls preserve inactive-initializer termination, standalone Malloc-notifier termination and standalone Free-callback termination/reinitialization. Production tests verify ownership-preserving refusal, repeatable recovery, successful parse continuation, OOM/retry and fresh module generations. From ceb12c906ca5ed1e0a1cca96451f966f4f808aef Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 09:39:27 +0200 Subject: [PATCH 058/318] fix(libSceJson2): reject recursive module teardown --- core/libs/prx/libSceJson2/Export.cpp | 12 +++++++- .../libs/tests/GuestJsonCallbackLifecycle.cpp | 28 ++++++++++++++++++- docs/dev/debt/libSceJson2.md | 4 ++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 24e100d8fc..4fb08eb1d9 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -148,6 +148,7 @@ struct Globals { bool floatFormatEnabled = false; bool paddedStringGrowth = false; bool initialized = false; + bool terminating = false; AllocParamRtti rtti{}; Handlers handlers{}; AllocationContext* allocation = nullptr; @@ -175,6 +176,14 @@ struct AllocatorCallbackScope { AllocatorCallbackScope& operator=(const AllocatorCallbackScope&) = delete; }; +struct TerminationScope { + bool& active; + explicit TerminationScope(bool& value) : active(value) { active = true; } + ~TerminationScope() { active = false; } + TerminationScope(const TerminationScope&) = delete; + TerminationScope& operator=(const TerminationScope&) = delete; +}; + struct KernelMutexLock { PthreadMutex* mutex; explicit KernelMutexLock(PthreadMutex* value) : mutex(value) { @@ -2552,7 +2561,8 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; - if (state.callbackActive || (allocatorCallbackDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); + if (state.terminating || state.callbackActive || (allocatorCallbackDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); + TerminationScope termination(state.terminating); state.callback = nullptr; state.context = nullptr; state.handlers = {}; diff --git a/core/libs/tests/GuestJsonCallbackLifecycle.cpp b/core/libs/tests/GuestJsonCallbackLifecycle.cpp index 6e238c14af..02eb7269f9 100644 --- a/core/libs/tests/GuestJsonCallbackLifecycle.cpp +++ b/core/libs/tests/GuestJsonCallbackLifecycle.cpp @@ -38,6 +38,8 @@ struct LifecycleTrace { AllocatorAction allocatorAction = AllocatorAction::None; unsigned notices = 0; unsigned refused = 0; + unsigned freeCalls = 0; + unsigned teardownTarget = 0; }; static void RejectOwnerTermination(LifecycleTrace& trace); static void* APS5_VABI AllocateLifecycle(void*, std::size_t bytes, void* context) { @@ -55,6 +57,11 @@ static void* APS5_VABI AllocateLifecycle(void*, std::size_t bytes, void* context } static void APS5_VABI FreeLifecycle(void*, void* memory, void* context) { auto& trace = *static_cast(context); + ++trace.freeCalls; + if (trace.teardownTarget && trace.freeCalls == trace.teardownTarget) { + trace.teardownTarget = 0; + RejectOwnerTermination(trace); + } if (trace.allocatorAction == AllocatorAction::RejectFree) { trace.allocatorAction = AllocatorAction::None; RejectOwnerTermination(trace); @@ -78,7 +85,7 @@ static void RejectOwnerTermination(LifecycleTrace& trace) { Require(std::strstr(error.what(), "terminate"), "Unsafe callback termination must use the explicit unsupported boundary"); rejected = true; } - Require(rejected, "Termination inside an active callback operation must be explicitly rejected before teardown"); + Require(rejected, "Unsafe callback termination must be explicitly rejected before teardown"); Require(*trace.owner == 1 && trace.live == before, "Rejected callback termination must preserve the entire module allocation generation and active byte"); ++trace.refused; } @@ -189,6 +196,24 @@ static void CheckStandaloneFree() { for (const auto& [pointer, identity] : trace.live) Require(pointer && identity > generation, "Reinitialization must own fresh resource generations after free-callback termination"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release standalone free lifecycle retry"); } +static void CheckTeardownReentry(unsigned target) { + LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; trace.owner = &initializer; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && trace.live.size() == 12, + "Teardown reentry requires the actual twelve independently allocated module resources"); + trace.freeCalls = 0; trace.teardownTarget = target; + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty() && trace.freeCalls == 12 && trace.refused == 1 && !trace.teardownTarget, + "Each teardown callback must reject nested owner termination while allowing the outer call to release every resource exactly once"); + const auto generation = trace.generation; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == 12, + "Initialization must admit a fresh module after rejected teardown reentry at every release position"); + for (const auto& [pointer, identity] : trace.live) Require(pointer && identity > generation, "Teardown reentry retry must retain no old allocation generations"); + try { + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty() && trace.refused == 1, + "The teardown guard must reset so ordinary termination remains usable for the next module generation"); + } catch (const std::runtime_error&) { Require(false, "Completed teardown must clear its guard before ordinary termination of a fresh module generation"); } +} int main() { CheckRecovery(false); CheckRecovery(true); @@ -197,5 +222,6 @@ int main() { CheckAllocator(true); CheckAllocator(false); CheckStandaloneFree(); + for (unsigned target = 1; target <= 12; ++target) CheckTeardownReentry(target); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 0c4f6bc7f0..dee2b0854e 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -72,4 +72,6 @@ - [Modern callback-bearing copies](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation-failure combinations, nested owners, lifecycle changes from callbacks and alternate SDK behavior remain unverified. Fifteen reference-backed cases exercise null, real and object sources with all four local handlers through Value construction, assignment with/without destination handlers, Pair construction and standalone Array insertion. Modern copies discard source-local handlers; assignment retains destination handlers. Tests invoke null/mismatch/element/float recovery and verify complete auxiliary bytes, fallback ownership, all callback-record generations, borrowed replacement identity and source-handler usability after copy destruction. -- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - teardown reentry, pre-publication allocation callbacks, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and allocate/notify/free callback termination inside an active allocation operation are explicitly unsupported before teardown. Reference recovery/notifier probes reused freed/null mutex slots after termination and faulted at the substituted kernel boundary; direct allocate/free probes also faulted after termination returned success. These are not console measurements. Reference-backed controls preserve inactive-initializer termination, standalone Malloc-notifier termination and standalone Free-callback termination/reinitialization. Production tests verify ownership-preserving refusal, repeatable recovery, successful parse continuation, OOM/retry and fresh module generations. +- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - composite-destruction callbacks and teardown exceptions, pre-publication allocation callbacks, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and allocate/notify/free callback termination inside an active allocation operation are explicitly unsupported before teardown. Reference recovery/notifier probes reused freed/null mutex slots after termination and faulted at the substituted kernel boundary; direct allocate/free probes also faulted after termination returned success. These are not console measurements. Reference-backed controls preserve inactive-initializer termination, standalone Malloc-notifier termination and standalone Free-callback termination/reinitialization. Production tests verify ownership-preserving refusal, repeatable recovery, successful parse continuation, OOM/retry and fresh module generations. + +- [Termination reentry](../../../core/libs/prx/libSceJson2/Export.cpp) - callback exceptions during partially completed teardown and older-SDK combinations remain unverified. Nested owner termination while cleanup is already active is explicitly unsupported before further ownership changes. The reference allocator observed a duplicate free request when a teardown free callback recursively terminated the module; a standalone Free callback retained its target until its original release completed. Twelve production cases exercise reentry at every normal module release position, require one refusal and exactly twelve releases, and verify that the outer call completes and the next module generation initializes and terminates normally. From 27e55c8cb47849ee0ba79d5b0c3bad6ec918cbf7 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 09:50:55 +0200 Subject: [PATCH 059/318] test(libSceJson2): verify initialization callback publication ownership --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +- .../GuestJsonInitializationPublication.cpp | 126 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 core/libs/tests/GuestJsonInitializationPublication.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 2839ec0f8e..8c24e562a8 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_initialization_publication_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationPublication.cpp) + target_include_directories(guest_json_initialization_publication_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_initialization_publication_tests PRIVATE libSceJson2 libkernel libc) + configure_windows_unwind(guest_json_initialization_publication_tests) + add_test(NAME guest_json_initialization_publication COMMAND guest_json_initialization_publication_tests) add_test_executable(guest_json_callback_lifecycle_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCallbackLifecycle.cpp) target_include_directories(guest_json_callback_lifecycle_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_callback_lifecycle_tests PRIVATE libSceJson2 libc) @@ -200,7 +205,7 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) - set_tests_properties(guest_json guest_json_callback_lifecycle guest_json_callback_copies guest_json_pair_allocator guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json guest_json_initialization_publication guest_json_callback_lifecycle guest_json_callback_copies guest_json_pair_allocator guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/tests/GuestJsonInitializationPublication.cpp b/core/libs/tests/GuestJsonInitializationPublication.cpp new file mode 100644 index 0000000000..eeee2cf2b7 --- /dev/null +++ b/core/libs/tests/GuestJsonInitializationPublication.cpp @@ -0,0 +1,126 @@ +#include "prx/libkernel/Pthread/include/Mutex.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +static int Initialize(bool legacy, void* self, const Parameter* parameter) { + return legacy ? _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(self, parameter) : + _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(self, parameter); +} +enum class Publication { Handoff, CompleteNested, Overwrite }; +struct AllocationIdentity { + std::size_t size; + std::size_t generation; + bool operator==(const AllocationIdentity&) const = default; +}; +struct PublicationTrace { + std::map live; + std::map nested; + std::vector releases; + std::size_t generation = 0; + std::size_t nestedGeneration = 0; + Publication action = Publication::Handoff; + bool armed = false; + bool nestedLegacy = false; + std::uint8_t* nestedOwner = nullptr; + const Parameter* nestedParameter = nullptr; +}; +static void* APS5_VABI AllocatePublication(void*, std::size_t size, void* context) { + auto& trace = *static_cast(context); + if (trace.armed) { + trace.armed = false; + Require(Initialize(trace.nestedLegacy, trace.nestedOwner, trace.nestedParameter) == 0 && *trace.nestedOwner == 1 && trace.live.size() == 12, + "Nested initialization in the unpublished-context callback must publish its actual complete resource generation"); + trace.nested = trace.live; trace.nestedGeneration = trace.generation; + if (trace.action == Publication::CompleteNested) + Require(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner) == 0 && trace.live.empty() && *trace.nestedOwner == 0, "The initial allocation callback may complete a nested module lifecycle before returning its allocation"); + if (trace.action == Publication::Handoff) return nullptr; + } + void* memory = std::malloc(size); + Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, size); + Require(trace.live.emplace(memory, AllocationIdentity{size, ++trace.generation}).second, "Publication resources must have independently owned live generations"); + return memory; +} +static void APS5_VABI FreePublication(void*, void* memory, void* context) { + if (!memory) return; + auto& trace = *static_cast(context); + trace.releases.push_back(trace.live.at(memory).generation); + Require(trace.live.erase(memory) == 1, "Publication cleanup must release each allocation through its owning allocator exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyPublication(void*, int, std::size_t, void*) { Require(false, "Failure of the unpublished context allocation must omit allocator notification"); } +static void* const PublicationVtable[]{nullptr, nullptr, reinterpret_cast(AllocatePublication), reinterpret_cast(FreePublication), reinterpret_cast(NotifyPublication)}; +struct PublicationAllocator { void* const* vtable = PublicationVtable; }; +static void ReclaimOrphanedFixtureEpoch(PublicationTrace& trace) { + Require(trace.live == trace.nested && trace.live.size() == 12, "The fixture may reclaim only the exact orphaned epoch already observed after library teardown"); + unsigned mutexes = 0; + for (const auto& [pointer, identity] : trace.nested) { + if (identity.generation + 1 >= trace.nestedGeneration) { + Require(identity.size == sizeof(PthreadMutex) && scePthreadMutexDestroy(static_cast(pointer)) == 0, + "Fixture cleanup must destroy the two real orphaned kernel mutex resources before reclaiming their handle storage"); + ++mutexes; + } + } + Require(mutexes == 2, "The SDK 6 orphaned epoch must retain its two actual mutex handles"); + for (const auto& [pointer, identity] : trace.nested) { + Require(trace.live.at(pointer) == identity, "Fixture reclamation must retain the observed orphan allocation generation"); + FreePublication(nullptr, pointer, &trace); + } + Require(trace.live.empty(), "Caller-owned allocator fixture cleanup must reclaim the measured orphaned epoch"); +} +static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool nestedLegacy) { + PublicationTrace trace; PublicationAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t outer = 0, nested = 0; + _ZN3sce4Json11InitializerC1Ev(&outer); _ZN3sce4Json11InitializerC1Ev(&nested); + trace.action = action; trace.armed = true; trace.nestedLegacy = nestedLegacy; + trace.nestedOwner = sharedOwner ? &outer : &nested; trace.nestedParameter = ¶meter; + const int status = Initialize(outerLegacy, &outer, ¶meter); + Require(!trace.armed && trace.nested.size() == 12 && trace.nestedGeneration == 12, "Every publication case must exercise actual first-allocation callback reentry"); + if (action == Publication::Handoff) { + Require(static_cast(status) == 0x80848102 && *trace.nestedOwner == 1 && outer == unsigned(sharedOwner) && trace.live == trace.nested && trace.releases.empty(), + "A failed outer context allocation must retain the nested initializer's ownership, active byte and untouched resource generation"); + Require(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner) == 0 && *trace.nestedOwner == 0 && trace.live.empty(), "The nested owner must remain able to terminate after outer initialization reports OOM"); + } else { + Require(status == 0 && outer == 1 && trace.live.size() == (action == Publication::Overwrite ? 24u : 12u), + "Successful outer initialization must preserve the reference resource count after nested publication"); + if (action == Publication::CompleteNested) { + Require(nested == 0 && trace.releases.size() == 12, "Completed nested teardown must reclaim exactly its original resource generation"); + for (const auto& [pointer, identity] : trace.live) Require(pointer && identity.generation > trace.nestedGeneration, "Outer initialization after a nested lifecycle must own an entirely fresh generation"); + } + Require(_ZN3sce4Json11Initializer9terminateEv(&outer) == 0 && outer == 0, "Terminate the final published outer module generation"); + if (action == Publication::Overwrite) { + Require(trace.live == trace.nested && nested == unsigned(!sharedOwner) && trace.releases.size() == 12, + "Double successful publication must preserve the reference's orphaned first epoch instead of claiming the library reclaimed it"); + const auto released = trace.releases; + Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner)) == 0x80848110 && trace.live == trace.nested && trace.releases == released, + "An overwritten nested owner must report missing module state without releasing its orphaned epoch"); + ReclaimOrphanedFixtureEpoch(trace); + } else Require(trace.live.empty(), "A completed nested lifecycle must not orphan module resources"); + } + std::uint8_t retry = 0; _ZN3sce4Json11InitializerC1Ev(&retry); + const auto generation = trace.generation; + Require(Initialize(outerLegacy, &retry, ¶meter) == 0 && retry == 1 && trace.live.size() == 12, "A fresh initializer must work after every measured publication outcome"); + for (const auto& [pointer, identity] : trace.live) Require(pointer && identity.generation > generation, "Publication retry must retain no old fixture allocation generations"); + Require(_ZN3sce4Json11Initializer9terminateEv(&retry) == 0 && retry == 0 && trace.live.empty(), "Release the fresh publication generation"); +} +int main() { + for (auto action : {Publication::Handoff, Publication::CompleteNested, Publication::Overwrite}) + for (bool sharedOwner : {false, true}) for (bool outerLegacy : {false, true}) for (bool nestedLegacy : {false, true}) + Check(action, sharedOwner, outerLegacy, nestedLegacy); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index dee2b0854e..6696a9cc1b 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -72,6 +72,8 @@ - [Modern callback-bearing copies](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation-failure combinations, nested owners, lifecycle changes from callbacks and alternate SDK behavior remain unverified. Fifteen reference-backed cases exercise null, real and object sources with all four local handlers through Value construction, assignment with/without destination handlers, Pair construction and standalone Array insertion. Modern copies discard source-local handlers; assignment retains destination handlers. Tests invoke null/mismatch/element/float recovery and verify complete auxiliary bytes, fallback ownership, all callback-record generations, borrowed replacement identity and source-handler usability after copy destruction. -- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - composite-destruction callbacks and teardown exceptions, pre-publication allocation callbacks, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and allocate/notify/free callback termination inside an active allocation operation are explicitly unsupported before teardown. Reference recovery/notifier probes reused freed/null mutex slots after termination and faulted at the substituted kernel boundary; direct allocate/free probes also faulted after termination returned success. These are not console measurements. Reference-backed controls preserve inactive-initializer termination, standalone Malloc-notifier termination and standalone Free-callback termination/reinitialization. Production tests verify ownership-preserving refusal, repeatable recovery, successful parse continuation, OOM/retry and fresh module generations. +- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - composite-destruction callbacks, teardown exceptions and unmeasured publication variants, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and allocate/notify/free callback termination inside an active allocation operation are explicitly unsupported before teardown. Reference recovery/notifier probes reused freed/null mutex slots after termination and faulted at the substituted kernel boundary; direct allocate/free probes also faulted after termination returned success. These are not console measurements. Reference-backed controls preserve inactive-initializer termination, standalone Malloc-notifier termination and standalone Free-callback termination/reinitialization. Production tests verify ownership-preserving refusal, repeatable recovery, successful parse continuation, OOM/retry and fresh module generations. - [Termination reentry](../../../core/libs/prx/libSceJson2/Export.cpp) - callback exceptions during partially completed teardown and older-SDK combinations remain unverified. Nested owner termination while cleanup is already active is explicitly unsupported before further ownership changes. The reference allocator observed a duplicate free request when a teardown free callback recursively terminated the module; a standalone Free callback retained its target until its original release completed. Twelve production cases exercise reentry at every normal module release position, require one refusal and exactly twelve releases, and verify that the outer call completes and the next module generation initializes and terminates normally. + +- [Initialization callback publication](../../../core/libs/prx/libSceJson2/Export.cpp) - distinct allocator domains, RTTI wrappers, SDK 5 and concurrent publication remain unverified. Twenty-four SDK 6 reference cases cover both initializer overloads, shared/separate owners and first-allocation callback reentry. Outer context-allocation failure preserves a published nested owner; a completed nested lifecycle permits a fresh outer generation. Double successful publication orphans the first twelve allocations: outer termination reclaims only its own generation and nested termination subsequently reports not-initialized. Tests assert this observed leak and owner-byte behavior before caller-owned fixture cleanup destroys the two orphaned kernel mutexes and frees the orphaned raw allocations. That final fixture reclamation is not library behavior. From 5c76b9a8d0ff4838744d1768b5efdc0571cd34c9 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 09:57:42 +0200 Subject: [PATCH 060/318] test(libSceJson2): verify allocator domains during initialization reentry --- .../GuestJsonInitializationPublication.cpp | 96 +++++++++++++------ docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 66 insertions(+), 32 deletions(-) diff --git a/core/libs/tests/GuestJsonInitializationPublication.cpp b/core/libs/tests/GuestJsonInitializationPublication.cpp index eeee2cf2b7..7911863c8e 100644 --- a/core/libs/tests/GuestJsonInitializationPublication.cpp +++ b/core/libs/tests/GuestJsonInitializationPublication.cpp @@ -38,16 +38,33 @@ struct PublicationTrace { bool nestedLegacy = false; std::uint8_t* nestedOwner = nullptr; const Parameter* nestedParameter = nullptr; + PublicationTrace* nestedTrace = nullptr; + void* expectedAllocator = nullptr; }; -static void* APS5_VABI AllocatePublication(void*, std::size_t size, void* context) { +struct ContextRecord { void* allocator; void* userData; std::size_t bufferSize; }; +static void CheckContext(const PublicationTrace& trace, std::size_t generation, const Parameter& parameter) { + unsigned found = 0; + for (const auto& [pointer, identity] : trace.live) { + if (identity.generation != generation) continue; + Require(identity.size == sizeof(ContextRecord), "Publication must allocate its actual context record in the owning domain"); + ContextRecord context{}; std::memcpy(&context, pointer, sizeof(context)); + Require(context.allocator == parameter.allocator && context.userData == parameter.context && context.bufferSize == parameter.bufferSize, + "Publication must snapshot the correct allocator receiver, user context and file-buffer setting for each domain"); + ++found; + } + Require(found == 1, "The published allocation context must retain its exact allocation generation"); +} +static void* APS5_VABI AllocatePublication(void* self, std::size_t size, void* context) { auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator, "Publication allocation must preserve its exact allocator receiver and user context"); if (trace.armed) { trace.armed = false; - Require(Initialize(trace.nestedLegacy, trace.nestedOwner, trace.nestedParameter) == 0 && *trace.nestedOwner == 1 && trace.live.size() == 12, + Require(Initialize(trace.nestedLegacy, trace.nestedOwner, trace.nestedParameter) == 0 && *trace.nestedOwner == 1 && trace.nestedTrace->live.size() == 12, "Nested initialization in the unpublished-context callback must publish its actual complete resource generation"); - trace.nested = trace.live; trace.nestedGeneration = trace.generation; + CheckContext(*trace.nestedTrace, trace.nestedTrace->generation - 11, *trace.nestedParameter); + trace.nested = trace.nestedTrace->live; trace.nestedGeneration = trace.nestedTrace->generation; if (trace.action == Publication::CompleteNested) - Require(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner) == 0 && trace.live.empty() && *trace.nestedOwner == 0, "The initial allocation callback may complete a nested module lifecycle before returning its allocation"); + Require(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner) == 0 && trace.nestedTrace->live.empty() && *trace.nestedOwner == 0, "The initial allocation callback may complete a nested module lifecycle before returning its allocation"); if (trace.action == Publication::Handoff) return nullptr; } void* memory = std::malloc(size); @@ -56,9 +73,10 @@ static void* APS5_VABI AllocatePublication(void*, std::size_t size, void* contex Require(trace.live.emplace(memory, AllocationIdentity{size, ++trace.generation}).second, "Publication resources must have independently owned live generations"); return memory; } -static void APS5_VABI FreePublication(void*, void* memory, void* context) { - if (!memory) return; +static void APS5_VABI FreePublication(void* self, void* memory, void* context) { auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator, "Publication release must preserve its exact allocator receiver and user context"); + if (!memory) return; trace.releases.push_back(trace.live.at(memory).generation); Require(trace.live.erase(memory) == 1, "Publication cleanup must release each allocation through its owning allocator exactly once"); std::free(memory); @@ -66,61 +84,77 @@ static void APS5_VABI FreePublication(void*, void* memory, void* context) { static void APS5_VABI NotifyPublication(void*, int, std::size_t, void*) { Require(false, "Failure of the unpublished context allocation must omit allocator notification"); } static void* const PublicationVtable[]{nullptr, nullptr, reinterpret_cast(AllocatePublication), reinterpret_cast(FreePublication), reinterpret_cast(NotifyPublication)}; struct PublicationAllocator { void* const* vtable = PublicationVtable; }; -static void ReclaimOrphanedFixtureEpoch(PublicationTrace& trace) { - Require(trace.live == trace.nested && trace.live.size() == 12, "The fixture may reclaim only the exact orphaned epoch already observed after library teardown"); +static void ReclaimOrphanedFixtureEpoch(PublicationTrace& trace, const std::map& nested, std::size_t generation) { + Require(trace.live == nested && trace.live.size() == 12, "The fixture may reclaim only the exact orphaned epoch already observed after library teardown"); unsigned mutexes = 0; - for (const auto& [pointer, identity] : trace.nested) { - if (identity.generation + 1 >= trace.nestedGeneration) { + for (const auto& [pointer, identity] : nested) { + if (identity.generation + 1 >= generation) { Require(identity.size == sizeof(PthreadMutex) && scePthreadMutexDestroy(static_cast(pointer)) == 0, "Fixture cleanup must destroy the two real orphaned kernel mutex resources before reclaiming their handle storage"); ++mutexes; } } Require(mutexes == 2, "The SDK 6 orphaned epoch must retain its two actual mutex handles"); - for (const auto& [pointer, identity] : trace.nested) { + for (const auto& [pointer, identity] : nested) { Require(trace.live.at(pointer) == identity, "Fixture reclamation must retain the observed orphan allocation generation"); - FreePublication(nullptr, pointer, &trace); + FreePublication(trace.expectedAllocator, pointer, &trace); } Require(trace.live.empty(), "Caller-owned allocator fixture cleanup must reclaim the measured orphaned epoch"); } -static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool nestedLegacy) { - PublicationTrace trace; PublicationAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; +static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool nestedLegacy, unsigned domain) { + PublicationTrace trace, separateTrace; + PublicationAllocator allocator, separateAllocator; + const bool separate = domain != 0; + trace.expectedAllocator = &allocator; + separateTrace.expectedAllocator = domain == 2 ? &separateAllocator : &allocator; + Parameter parameter{&allocator, &trace, 8192, 2, {}}; + Parameter separateParameter{separateTrace.expectedAllocator, &separateTrace, 16384, 2, {}}; std::uint8_t outer = 0, nested = 0; _ZN3sce4Json11InitializerC1Ev(&outer); _ZN3sce4Json11InitializerC1Ev(&nested); trace.action = action; trace.armed = true; trace.nestedLegacy = nestedLegacy; - trace.nestedOwner = sharedOwner ? &outer : &nested; trace.nestedParameter = ¶meter; + trace.nestedOwner = sharedOwner ? &outer : &nested; + trace.nestedParameter = separate ? &separateParameter : ¶meter; + trace.nestedTrace = separate ? &separateTrace : &trace; const int status = Initialize(outerLegacy, &outer, ¶meter); Require(!trace.armed && trace.nested.size() == 12 && trace.nestedGeneration == 12, "Every publication case must exercise actual first-allocation callback reentry"); if (action == Publication::Handoff) { - Require(static_cast(status) == 0x80848102 && *trace.nestedOwner == 1 && outer == unsigned(sharedOwner) && trace.live == trace.nested && trace.releases.empty(), + Require(static_cast(status) == 0x80848102 && *trace.nestedOwner == 1 && outer == unsigned(sharedOwner) && trace.nestedTrace->live == trace.nested && trace.releases.empty() && trace.nestedTrace->releases.empty(), "A failed outer context allocation must retain the nested initializer's ownership, active byte and untouched resource generation"); - Require(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner) == 0 && *trace.nestedOwner == 0 && trace.live.empty(), "The nested owner must remain able to terminate after outer initialization reports OOM"); + Require(trace.live.size() == (separate ? 0u : 12u), "Outer context failure must allocate no resources in a separate outer allocator domain"); + Require(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner) == 0 && *trace.nestedOwner == 0 && trace.live.empty() && separateTrace.live.empty(), + "The nested owner must terminate through its original allocator after outer initialization reports OOM"); } else { - Require(status == 0 && outer == 1 && trace.live.size() == (action == Publication::Overwrite ? 24u : 12u), - "Successful outer initialization must preserve the reference resource count after nested publication"); + Require(status == 0 && outer == 1 && trace.live.size() == (action == Publication::Overwrite && !separate ? 24u : 12u), + "Successful outer initialization must preserve the reference resource count in each allocator domain"); + CheckContext(trace, separate ? 1u : trace.nestedGeneration + 1, parameter); if (action == Publication::CompleteNested) { - Require(nested == 0 && trace.releases.size() == 12, "Completed nested teardown must reclaim exactly its original resource generation"); - for (const auto& [pointer, identity] : trace.live) Require(pointer && identity.generation > trace.nestedGeneration, "Outer initialization after a nested lifecycle must own an entirely fresh generation"); + Require(nested == 0 && trace.nestedTrace->releases.size() == 12, "Completed nested teardown must reclaim exactly its original resource generation"); + Require(!separate || (separateTrace.live.empty() && trace.releases.empty()), "A completed nested lifecycle must release only its owning allocator domain"); + for (const auto& [pointer, identity] : trace.live) Require(pointer && identity.generation > (separate ? 0u : trace.nestedGeneration), "Outer initialization after a nested lifecycle must own a fresh generation in its own domain"); + } else if (separate) { + Require(separateTrace.live == trace.nested && separateTrace.releases.empty(), "Outer publication must not reclaim or replace the nested allocator's orphaned generation"); } Require(_ZN3sce4Json11Initializer9terminateEv(&outer) == 0 && outer == 0, "Terminate the final published outer module generation"); if (action == Publication::Overwrite) { - Require(trace.live == trace.nested && nested == unsigned(!sharedOwner) && trace.releases.size() == 12, - "Double successful publication must preserve the reference's orphaned first epoch instead of claiming the library reclaimed it"); - const auto released = trace.releases; - Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner)) == 0x80848110 && trace.live == trace.nested && trace.releases == released, - "An overwritten nested owner must report missing module state without releasing its orphaned epoch"); - ReclaimOrphanedFixtureEpoch(trace); - } else Require(trace.live.empty(), "A completed nested lifecycle must not orphan module resources"); + Require(trace.nestedTrace->live == trace.nested && nested == unsigned(!sharedOwner) && trace.releases.size() == 12 && (!separate || (trace.live.empty() && separateTrace.releases.empty())), + "Double successful publication must retain the reference's orphaned first epoch in its original allocator domain"); + const auto released = trace.releases, nestedReleased = trace.nestedTrace->releases; + Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner)) == 0x80848110 && trace.nestedTrace->live == trace.nested && trace.releases == released && trace.nestedTrace->releases == nestedReleased, + "An overwritten nested owner must report missing module state without releasing resources through either allocator domain"); + ReclaimOrphanedFixtureEpoch(*trace.nestedTrace, trace.nested, trace.nestedGeneration); + } else Require(trace.live.empty() && separateTrace.live.empty(), "A completed nested lifecycle must not orphan module resources in either allocator domain"); } std::uint8_t retry = 0; _ZN3sce4Json11InitializerC1Ev(&retry); - const auto generation = trace.generation; + const auto generation = trace.generation, separateGeneration = separateTrace.generation; + const auto separateReleases = separateTrace.releases; Require(Initialize(outerLegacy, &retry, ¶meter) == 0 && retry == 1 && trace.live.size() == 12, "A fresh initializer must work after every measured publication outcome"); for (const auto& [pointer, identity] : trace.live) Require(pointer && identity.generation > generation, "Publication retry must retain no old fixture allocation generations"); - Require(_ZN3sce4Json11Initializer9terminateEv(&retry) == 0 && retry == 0 && trace.live.empty(), "Release the fresh publication generation"); + Require(_ZN3sce4Json11Initializer9terminateEv(&retry) == 0 && retry == 0 && trace.live.empty() && separateTrace.live.empty() && separateTrace.generation == separateGeneration && separateTrace.releases == separateReleases, + "A fresh outer generation must not call into a stale nested allocator domain"); } int main() { for (auto action : {Publication::Handoff, Publication::CompleteNested, Publication::Overwrite}) for (bool sharedOwner : {false, true}) for (bool outerLegacy : {false, true}) for (bool nestedLegacy : {false, true}) - Check(action, sharedOwner, outerLegacy, nestedLegacy); + for (unsigned domain = 0; domain < 3; ++domain) Check(action, sharedOwner, outerLegacy, nestedLegacy, domain); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 6696a9cc1b..34b564ca81 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -76,4 +76,4 @@ - [Termination reentry](../../../core/libs/prx/libSceJson2/Export.cpp) - callback exceptions during partially completed teardown and older-SDK combinations remain unverified. Nested owner termination while cleanup is already active is explicitly unsupported before further ownership changes. The reference allocator observed a duplicate free request when a teardown free callback recursively terminated the module; a standalone Free callback retained its target until its original release completed. Twelve production cases exercise reentry at every normal module release position, require one refusal and exactly twelve releases, and verify that the outer call completes and the next module generation initializes and terminates normally. -- [Initialization callback publication](../../../core/libs/prx/libSceJson2/Export.cpp) - distinct allocator domains, RTTI wrappers, SDK 5 and concurrent publication remain unverified. Twenty-four SDK 6 reference cases cover both initializer overloads, shared/separate owners and first-allocation callback reentry. Outer context-allocation failure preserves a published nested owner; a completed nested lifecycle permits a fresh outer generation. Double successful publication orphans the first twelve allocations: outer termination reclaims only its own generation and nested termination subsequently reports not-initialized. Tests assert this observed leak and owner-byte behavior before caller-owned fixture cleanup destroys the two orphaned kernel mutexes and frees the orphaned raw allocations. That final fixture reclamation is not library behavior. +- [Initialization callback publication](../../../core/libs/prx/libSceJson2/Export.cpp) - RTTI wrappers, SDK 5 and concurrent publication remain unverified. Seventy-two SDK 6 reference cases cover both initializer overloads, shared/separate owners, shared or distinct receiver/context domains and saved file-buffer settings. Outer context-allocation failure preserves the nested owner; a completed nested lifecycle permits a fresh outer generation. Double successful publication orphans the first twelve allocations in their original domain: outer termination reclaims only its own epoch and nested termination reports not-initialized. Tests assert receiver/context routing, exact resource generations and that retry never calls a stale domain. After verifying the observed leak, the caller-owned fixture destroys its two orphaned mutexes and frees the orphaned raw storage; this is fixture reclamation rather than library cleanup. From a193210018c17225f0d35b3f1f1e7200306cf77b Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 10:16:16 +0200 Subject: [PATCH 061/318] test(libSceJson2): verify shared RTTI callback publication --- .../GuestJsonInitializationPublication.cpp | 80 ++++++++++++++++--- docs/dev/debt/libSceJson2.md | 4 +- 2 files changed, 74 insertions(+), 10 deletions(-) diff --git a/core/libs/tests/GuestJsonInitializationPublication.cpp b/core/libs/tests/GuestJsonInitializationPublication.cpp index 7911863c8e..7d7f0b2888 100644 --- a/core/libs/tests/GuestJsonInitializationPublication.cpp +++ b/core/libs/tests/GuestJsonInitializationPublication.cpp @@ -8,7 +8,13 @@ #include struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using RttiAllocate = void* (APS5_VABI*)(std::size_t, void*); +using RttiFree = void (APS5_VABI*)(void*, void*); +using RttiNotify = void (APS5_VABI*)(int, std::size_t, void*); +struct RttiCallbacks { RttiAllocate allocate; RttiFree free; RttiNotify notify; }; extern "C" { +void APS5_VABI _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(Parameter*, const RttiCallbacks*, void*, std::size_t); +void APS5_VABI _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(Parameter*, const RttiCallbacks*, void*); void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); @@ -40,6 +46,13 @@ struct PublicationTrace { const Parameter* nestedParameter = nullptr; PublicationTrace* nestedTrace = nullptr; void* expectedAllocator = nullptr; + std::size_t rttiAllocations[2]{}; + std::size_t rttiReleases[2]{}; + bool deferredConfiguration = false; + Parameter* pendingParameter = nullptr; + const RttiCallbacks* pendingCallbacks = nullptr; + void* pendingContext = nullptr; + void* adapter = nullptr; }; struct ContextRecord { void* allocator; void* userData; std::size_t bufferSize; }; static void CheckContext(const PublicationTrace& trace, std::size_t generation, const Parameter& parameter) { @@ -54,11 +67,14 @@ static void CheckContext(const PublicationTrace& trace, std::size_t generation, } Require(found == 1, "The published allocation context must retain its exact allocation generation"); } -static void* APS5_VABI AllocatePublication(void* self, std::size_t size, void* context) { - auto& trace = *static_cast(context); - Require(self == trace.expectedAllocator, "Publication allocation must preserve its exact allocator receiver and user context"); +static void ConfigureRtti(bool legacy, Parameter& parameter, const RttiCallbacks& callbacks, void* context); +static void* AllocateStorage(PublicationTrace& trace, std::size_t size) { if (trace.armed) { trace.armed = false; + if (trace.deferredConfiguration) { + ConfigureRtti(trace.nestedLegacy, *trace.pendingParameter, *trace.pendingCallbacks, trace.pendingContext); + Require(trace.pendingParameter->allocator == trace.adapter, "Reentrant RTTI configuration must update the shared adapter before context publication"); + } Require(Initialize(trace.nestedLegacy, trace.nestedOwner, trace.nestedParameter) == 0 && *trace.nestedOwner == 1 && trace.nestedTrace->live.size() == 12, "Nested initialization in the unpublished-context callback must publish its actual complete resource generation"); CheckContext(*trace.nestedTrace, trace.nestedTrace->generation - 11, *trace.nestedParameter); @@ -73,14 +89,43 @@ static void* APS5_VABI AllocatePublication(void* self, std::size_t size, void* c Require(trace.live.emplace(memory, AllocationIdentity{size, ++trace.generation}).second, "Publication resources must have independently owned live generations"); return memory; } -static void APS5_VABI FreePublication(void* self, void* memory, void* context) { - auto& trace = *static_cast(context); - Require(self == trace.expectedAllocator, "Publication release must preserve its exact allocator receiver and user context"); +static void FreeStorage(PublicationTrace& trace, void* memory) { if (!memory) return; trace.releases.push_back(trace.live.at(memory).generation); Require(trace.live.erase(memory) == 1, "Publication cleanup must release each allocation through its owning allocator exactly once"); std::free(memory); } +static void* APS5_VABI AllocatePublication(void* self, std::size_t size, void* context) { + auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator, "Publication allocation must preserve its exact allocator receiver and user context"); + return AllocateStorage(trace, size); +} +static void APS5_VABI FreePublication(void* self, void* memory, void* context) { + auto& trace = *static_cast(context); + Require(self == trace.expectedAllocator, "Publication release must preserve its exact allocator receiver and user context"); + FreeStorage(trace, memory); +} +static void* APS5_VABI AllocateRttiFirst(std::size_t size, void* context) { + auto& trace = *static_cast(context); ++trace.rttiAllocations[0]; + return AllocateStorage(trace, size); +} +static void* APS5_VABI AllocateRttiSecond(std::size_t size, void* context) { + auto& trace = *static_cast(context); ++trace.rttiAllocations[1]; + return AllocateStorage(trace, size); +} +static void APS5_VABI FreeRttiFirst(void* memory, void* context) { + auto& trace = *static_cast(context); ++trace.rttiReleases[0]; + FreeStorage(trace, memory); +} +static void APS5_VABI FreeRttiSecond(void* memory, void* context) { + auto& trace = *static_cast(context); ++trace.rttiReleases[1]; + FreeStorage(trace, memory); +} +static void APS5_VABI NotifyRtti(int, std::size_t, void*) { Require(false, "Failure of the unpublished RTTI context allocation must omit notification"); } +static void ConfigureRtti(bool legacy, Parameter& parameter, const RttiCallbacks& callbacks, void* context) { + if (legacy) _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(¶meter, &callbacks, context, parameter.bufferSize); + else _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(¶meter, &callbacks, context); +} static void APS5_VABI NotifyPublication(void*, int, std::size_t, void*) { Require(false, "Failure of the unpublished context allocation must omit allocator notification"); } static void* const PublicationVtable[]{nullptr, nullptr, reinterpret_cast(AllocatePublication), reinterpret_cast(FreePublication), reinterpret_cast(NotifyPublication)}; struct PublicationAllocator { void* const* vtable = PublicationVtable; }; @@ -97,11 +142,11 @@ static void ReclaimOrphanedFixtureEpoch(PublicationTrace& trace, const std::map< Require(mutexes == 2, "The SDK 6 orphaned epoch must retain its two actual mutex handles"); for (const auto& [pointer, identity] : nested) { Require(trace.live.at(pointer) == identity, "Fixture reclamation must retain the observed orphan allocation generation"); - FreePublication(trace.expectedAllocator, pointer, &trace); + FreeStorage(trace, pointer); } Require(trace.live.empty(), "Caller-owned allocator fixture cleanup must reclaim the measured orphaned epoch"); } -static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool nestedLegacy, unsigned domain) { +static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool nestedLegacy, unsigned domain, bool rtti, bool deferredConfiguration) { PublicationTrace trace, separateTrace; PublicationAllocator allocator, separateAllocator; const bool separate = domain != 0; @@ -109,6 +154,16 @@ static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool n separateTrace.expectedAllocator = domain == 2 ? &separateAllocator : &allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; Parameter separateParameter{separateTrace.expectedAllocator, &separateTrace, 16384, 2, {}}; + const RttiCallbacks first{AllocateRttiFirst, FreeRttiFirst, NotifyRtti}, second{AllocateRttiSecond, FreeRttiSecond, NotifyRtti}; + if (rtti) { + ConfigureRtti(outerLegacy, parameter, first, &trace); + if (!deferredConfiguration) ConfigureRtti(nestedLegacy, separateParameter, second, &separateTrace); + Require(parameter.allocator && (deferredConfiguration || parameter.allocator == separateParameter.allocator) && parameter.context == &trace && separateParameter.context == &separateTrace, + "Both RTTI parameter families must share the adapter while retaining independent user contexts"); + trace.expectedAllocator = parameter.allocator; separateTrace.expectedAllocator = parameter.allocator; + trace.deferredConfiguration = deferredConfiguration; + trace.pendingParameter = &separateParameter; trace.pendingCallbacks = &second; trace.pendingContext = &separateTrace; trace.adapter = parameter.allocator; + } std::uint8_t outer = 0, nested = 0; _ZN3sce4Json11InitializerC1Ev(&outer); _ZN3sce4Json11InitializerC1Ev(&nested); trace.action = action; trace.armed = true; trace.nestedLegacy = nestedLegacy; @@ -151,10 +206,17 @@ static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool n for (const auto& [pointer, identity] : trace.live) Require(pointer && identity.generation > generation, "Publication retry must retain no old fixture allocation generations"); Require(_ZN3sce4Json11Initializer9terminateEv(&retry) == 0 && retry == 0 && trace.live.empty() && separateTrace.live.empty() && separateTrace.generation == separateGeneration && separateTrace.releases == separateReleases, "A fresh outer generation must not call into a stale nested allocator domain"); + if (rtti) { + Require(trace.rttiAllocations[0] == unsigned(deferredConfiguration) && !trace.rttiReleases[0] && !separateTrace.rttiAllocations[0] && !separateTrace.rttiReleases[0] && trace.rttiAllocations[1] && trace.rttiReleases[1], + "RTTI module allocation and release must use the most recently configured callback table even through an older parameter object"); + Require(!separate || (separateTrace.rttiAllocations[1] == 12 && separateTrace.rttiReleases[1] == (action == Publication::Overwrite ? 0u : 12u)), "RTTI callbacks must preserve the nested user context and leave an orphaned epoch unreleased by the library"); + } } int main() { for (auto action : {Publication::Handoff, Publication::CompleteNested, Publication::Overwrite}) for (bool sharedOwner : {false, true}) for (bool outerLegacy : {false, true}) for (bool nestedLegacy : {false, true}) - for (unsigned domain = 0; domain < 3; ++domain) Check(action, sharedOwner, outerLegacy, nestedLegacy, domain); + for (bool rtti : {false, true}) for (unsigned domain = 0; domain < (rtti ? 2u : 3u); ++domain) + for (unsigned deferred = 0; deferred < (rtti ? 2u : 1u); ++deferred) + Check(action, sharedOwner, outerLegacy, nestedLegacy, domain, rtti, deferred != 0); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 34b564ca81..ab6146f7b6 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -76,4 +76,6 @@ - [Termination reentry](../../../core/libs/prx/libSceJson2/Export.cpp) - callback exceptions during partially completed teardown and older-SDK combinations remain unverified. Nested owner termination while cleanup is already active is explicitly unsupported before further ownership changes. The reference allocator observed a duplicate free request when a teardown free callback recursively terminated the module; a standalone Free callback retained its target until its original release completed. Twelve production cases exercise reentry at every normal module release position, require one refusal and exactly twelve releases, and verify that the outer call completes and the next module generation initializes and terminates normally. -- [Initialization callback publication](../../../core/libs/prx/libSceJson2/Export.cpp) - RTTI wrappers, SDK 5 and concurrent publication remain unverified. Seventy-two SDK 6 reference cases cover both initializer overloads, shared/separate owners, shared or distinct receiver/context domains and saved file-buffer settings. Outer context-allocation failure preserves the nested owner; a completed nested lifecycle permits a fresh outer generation. Double successful publication orphans the first twelve allocations in their original domain: outer termination reclaims only its own epoch and nested termination reports not-initialized. Tests assert receiver/context routing, exact resource generations and that retry never calls a stale domain. After verifying the observed leak, the caller-owned fixture destroys its two orphaned mutexes and frees the orphaned raw storage; this is fixture reclamation rather than library cleanup. +- [Initialization callback publication](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK 5 and concurrent publication remain unverified. Seventy-two direct-allocator SDK 6 reference cases cover both initializer overloads, shared/separate owners, shared or distinct receiver/context domains and saved file-buffer settings. Outer context-allocation failure preserves the nested owner; a completed nested lifecycle permits a fresh outer generation. Double successful publication orphans the first twelve allocations in their original domain: outer termination reclaims only its own epoch and nested termination reports not-initialized. Tests assert receiver/context routing, exact resource generations and that retry never calls a stale domain. After verifying the observed leak, the caller-owned fixture destroys its two orphaned mutexes and frees the orphaned raw storage; this is fixture reclamation rather than library cleanup. + +- [RTTI initialization publication](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK 5, concurrent reconfiguration and thrown allocator callbacks remain unverified. Ninety-six additional SDK 6 reference cases cover legacy construction/newer RTTI setters, shared/independent contexts and configuration before initialization or inside its first allocation callback. All parameter objects share one adapter. An already-entered first allocation uses the original callback once; subsequent allocations and releases use the latest configured table while preserving each parameter context and file-buffer setting. Tests cover handoff, nested lifecycle completion, orphaned epochs and retry, and distinguish actual RTTI release callbacks from later allocator-fixture reclamation. From f98efccb39c16f0f9e32ea8264b0d8192aaf5b88 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 10:26:35 +0200 Subject: [PATCH 062/318] test(libSceJson2): verify SDK 5 initialization publication --- core/libs/prx/libSceJson2/CMakeLists.txt | 10 +++++ .../GuestJsonInitializationPublication.cpp | 39 ++++++++++++------- docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 8c24e562a8..b65ce66edc 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,16 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_legacy_initialization_publication_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationPublication.cpp Export.cpp) + target_compile_definitions(guest_json_legacy_initialization_publication_tests PRIVATE JSON2_PUBLICATION_LEGACY_SDK) + target_include_directories(guest_json_legacy_initialization_publication_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_legacy_initialization_publication_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_legacy_initialization_publication_tests) + add_test(NAME guest_json_legacy_initialization_publication COMMAND guest_json_legacy_initialization_publication_tests) + if(WIN32) + set_tests_properties(guest_json_legacy_initialization_publication PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() add_test_executable(guest_json_initialization_publication_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationPublication.cpp) target_include_directories(guest_json_initialization_publication_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_initialization_publication_tests PRIVATE libSceJson2 libkernel libc) diff --git a/core/libs/tests/GuestJsonInitializationPublication.cpp b/core/libs/tests/GuestJsonInitializationPublication.cpp index 7d7f0b2888..8d480a70f2 100644 --- a/core/libs/tests/GuestJsonInitializationPublication.cpp +++ b/core/libs/tests/GuestJsonInitializationPublication.cpp @@ -20,6 +20,19 @@ int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); } +#if defined(JSON2_PUBLICATION_LEGACY_SDK) +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + static const std::uint32_t words[]{0, 0, 0, 0, 0x05000000}; + return words; +} +static constexpr std::size_t ModuleResources = 11; +static constexpr unsigned KernelResources = 1; +static constexpr std::size_t ModuleFreeCalls = ModuleResources + 1; +#else +static constexpr std::size_t ModuleResources = 12; +static constexpr unsigned KernelResources = 2; +static constexpr std::size_t ModuleFreeCalls = ModuleResources; +#endif static void Require(bool condition, const char* message) { if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } } @@ -75,9 +88,9 @@ static void* AllocateStorage(PublicationTrace& trace, std::size_t size) { ConfigureRtti(trace.nestedLegacy, *trace.pendingParameter, *trace.pendingCallbacks, trace.pendingContext); Require(trace.pendingParameter->allocator == trace.adapter, "Reentrant RTTI configuration must update the shared adapter before context publication"); } - Require(Initialize(trace.nestedLegacy, trace.nestedOwner, trace.nestedParameter) == 0 && *trace.nestedOwner == 1 && trace.nestedTrace->live.size() == 12, + Require(Initialize(trace.nestedLegacy, trace.nestedOwner, trace.nestedParameter) == 0 && *trace.nestedOwner == 1 && trace.nestedTrace->live.size() == ModuleResources, "Nested initialization in the unpublished-context callback must publish its actual complete resource generation"); - CheckContext(*trace.nestedTrace, trace.nestedTrace->generation - 11, *trace.nestedParameter); + CheckContext(*trace.nestedTrace, trace.nestedTrace->generation - (ModuleResources - 1), *trace.nestedParameter); trace.nested = trace.nestedTrace->live; trace.nestedGeneration = trace.nestedTrace->generation; if (trace.action == Publication::CompleteNested) Require(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner) == 0 && trace.nestedTrace->live.empty() && *trace.nestedOwner == 0, "The initial allocation callback may complete a nested module lifecycle before returning its allocation"); @@ -130,16 +143,16 @@ static void APS5_VABI NotifyPublication(void*, int, std::size_t, void*) { Requir static void* const PublicationVtable[]{nullptr, nullptr, reinterpret_cast(AllocatePublication), reinterpret_cast(FreePublication), reinterpret_cast(NotifyPublication)}; struct PublicationAllocator { void* const* vtable = PublicationVtable; }; static void ReclaimOrphanedFixtureEpoch(PublicationTrace& trace, const std::map& nested, std::size_t generation) { - Require(trace.live == nested && trace.live.size() == 12, "The fixture may reclaim only the exact orphaned epoch already observed after library teardown"); + Require(trace.live == nested && trace.live.size() == ModuleResources, "The fixture may reclaim only the exact orphaned epoch already observed after library teardown"); unsigned mutexes = 0; for (const auto& [pointer, identity] : nested) { - if (identity.generation + 1 >= generation) { + if (identity.generation + KernelResources > generation) { Require(identity.size == sizeof(PthreadMutex) && scePthreadMutexDestroy(static_cast(pointer)) == 0, - "Fixture cleanup must destroy the two real orphaned kernel mutex resources before reclaiming their handle storage"); + "Fixture cleanup must destroy the actual orphaned kernel mutex resources before reclaiming their handle storage"); ++mutexes; } } - Require(mutexes == 2, "The SDK 6 orphaned epoch must retain its two actual mutex handles"); + Require(mutexes == KernelResources, "The orphaned epoch must retain its SDK-specific count of actual mutex handles"); for (const auto& [pointer, identity] : nested) { Require(trace.live.at(pointer) == identity, "Fixture reclamation must retain the observed orphan allocation generation"); FreeStorage(trace, pointer); @@ -171,19 +184,19 @@ static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool n trace.nestedParameter = separate ? &separateParameter : ¶meter; trace.nestedTrace = separate ? &separateTrace : &trace; const int status = Initialize(outerLegacy, &outer, ¶meter); - Require(!trace.armed && trace.nested.size() == 12 && trace.nestedGeneration == 12, "Every publication case must exercise actual first-allocation callback reentry"); + Require(!trace.armed && trace.nested.size() == ModuleResources && trace.nestedGeneration == ModuleResources, "Every publication case must exercise actual first-allocation callback reentry"); if (action == Publication::Handoff) { Require(static_cast(status) == 0x80848102 && *trace.nestedOwner == 1 && outer == unsigned(sharedOwner) && trace.nestedTrace->live == trace.nested && trace.releases.empty() && trace.nestedTrace->releases.empty(), "A failed outer context allocation must retain the nested initializer's ownership, active byte and untouched resource generation"); - Require(trace.live.size() == (separate ? 0u : 12u), "Outer context failure must allocate no resources in a separate outer allocator domain"); + Require(trace.live.size() == (separate ? 0u : ModuleResources), "Outer context failure must allocate no resources in a separate outer allocator domain"); Require(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner) == 0 && *trace.nestedOwner == 0 && trace.live.empty() && separateTrace.live.empty(), "The nested owner must terminate through its original allocator after outer initialization reports OOM"); } else { - Require(status == 0 && outer == 1 && trace.live.size() == (action == Publication::Overwrite && !separate ? 24u : 12u), + Require(status == 0 && outer == 1 && trace.live.size() == (action == Publication::Overwrite && !separate ? 2 * ModuleResources : ModuleResources), "Successful outer initialization must preserve the reference resource count in each allocator domain"); CheckContext(trace, separate ? 1u : trace.nestedGeneration + 1, parameter); if (action == Publication::CompleteNested) { - Require(nested == 0 && trace.nestedTrace->releases.size() == 12, "Completed nested teardown must reclaim exactly its original resource generation"); + Require(nested == 0 && trace.nestedTrace->releases.size() == ModuleResources, "Completed nested teardown must reclaim exactly its original resource generation"); Require(!separate || (separateTrace.live.empty() && trace.releases.empty()), "A completed nested lifecycle must release only its owning allocator domain"); for (const auto& [pointer, identity] : trace.live) Require(pointer && identity.generation > (separate ? 0u : trace.nestedGeneration), "Outer initialization after a nested lifecycle must own a fresh generation in its own domain"); } else if (separate) { @@ -191,7 +204,7 @@ static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool n } Require(_ZN3sce4Json11Initializer9terminateEv(&outer) == 0 && outer == 0, "Terminate the final published outer module generation"); if (action == Publication::Overwrite) { - Require(trace.nestedTrace->live == trace.nested && nested == unsigned(!sharedOwner) && trace.releases.size() == 12 && (!separate || (trace.live.empty() && separateTrace.releases.empty())), + Require(trace.nestedTrace->live == trace.nested && nested == unsigned(!sharedOwner) && trace.releases.size() == ModuleResources && (!separate || (trace.live.empty() && separateTrace.releases.empty())), "Double successful publication must retain the reference's orphaned first epoch in its original allocator domain"); const auto released = trace.releases, nestedReleased = trace.nestedTrace->releases; Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(trace.nestedOwner)) == 0x80848110 && trace.nestedTrace->live == trace.nested && trace.releases == released && trace.nestedTrace->releases == nestedReleased, @@ -202,14 +215,14 @@ static void Check(Publication action, bool sharedOwner, bool outerLegacy, bool n std::uint8_t retry = 0; _ZN3sce4Json11InitializerC1Ev(&retry); const auto generation = trace.generation, separateGeneration = separateTrace.generation; const auto separateReleases = separateTrace.releases; - Require(Initialize(outerLegacy, &retry, ¶meter) == 0 && retry == 1 && trace.live.size() == 12, "A fresh initializer must work after every measured publication outcome"); + Require(Initialize(outerLegacy, &retry, ¶meter) == 0 && retry == 1 && trace.live.size() == ModuleResources, "A fresh initializer must work after every measured publication outcome"); for (const auto& [pointer, identity] : trace.live) Require(pointer && identity.generation > generation, "Publication retry must retain no old fixture allocation generations"); Require(_ZN3sce4Json11Initializer9terminateEv(&retry) == 0 && retry == 0 && trace.live.empty() && separateTrace.live.empty() && separateTrace.generation == separateGeneration && separateTrace.releases == separateReleases, "A fresh outer generation must not call into a stale nested allocator domain"); if (rtti) { Require(trace.rttiAllocations[0] == unsigned(deferredConfiguration) && !trace.rttiReleases[0] && !separateTrace.rttiAllocations[0] && !separateTrace.rttiReleases[0] && trace.rttiAllocations[1] && trace.rttiReleases[1], "RTTI module allocation and release must use the most recently configured callback table even through an older parameter object"); - Require(!separate || (separateTrace.rttiAllocations[1] == 12 && separateTrace.rttiReleases[1] == (action == Publication::Overwrite ? 0u : 12u)), "RTTI callbacks must preserve the nested user context and leave an orphaned epoch unreleased by the library"); + Require(!separate || (separateTrace.rttiAllocations[1] == ModuleResources && separateTrace.rttiReleases[1] == (action == Publication::Overwrite ? 0u : ModuleFreeCalls)), "RTTI callbacks must preserve the nested user context and leave an orphaned epoch unreleased by the library"); } } int main() { diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index ab6146f7b6..1fe93b5f53 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -76,6 +76,6 @@ - [Termination reentry](../../../core/libs/prx/libSceJson2/Export.cpp) - callback exceptions during partially completed teardown and older-SDK combinations remain unverified. Nested owner termination while cleanup is already active is explicitly unsupported before further ownership changes. The reference allocator observed a duplicate free request when a teardown free callback recursively terminated the module; a standalone Free callback retained its target until its original release completed. Twelve production cases exercise reentry at every normal module release position, require one refusal and exactly twelve releases, and verify that the outer call completes and the next module generation initializes and terminates normally. -- [Initialization callback publication](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK 5 and concurrent publication remain unverified. Seventy-two direct-allocator SDK 6 reference cases cover both initializer overloads, shared/separate owners, shared or distinct receiver/context domains and saved file-buffer settings. Outer context-allocation failure preserves the nested owner; a completed nested lifecycle permits a fresh outer generation. Double successful publication orphans the first twelve allocations in their original domain: outer termination reclaims only its own epoch and nested termination reports not-initialized. Tests assert receiver/context routing, exact resource generations and that retry never calls a stale domain. After verifying the observed leak, the caller-owned fixture destroys its two orphaned mutexes and frees the orphaned raw storage; this is fixture reclamation rather than library cleanup. +- [Initialization callback publication](../../../core/libs/prx/libSceJson2/Export.cpp) - concurrent publication remains unverified. One hundred forty-four direct-allocator reference cases span SDK 5/6, both initializer overloads, shared/separate owners and receiver/context domains. They verify saved file-buffer settings, nested handoff, completed nested lifecycles and retry. Double success orphans the first epoch in its original domain: eleven resources/one mutex for SDK 5, twelve/two for SDK 6. Outer termination reclaims only its own epoch and nested termination reports not-initialized. Tests assert ownership and stale-domain avoidance, then separately reclaim the observed orphaned epoch through the caller-owned fixture. SDK 5 teardown also forwards its observed null free; fixture reclamation is not library cleanup. -- [RTTI initialization publication](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK 5, concurrent reconfiguration and thrown allocator callbacks remain unverified. Ninety-six additional SDK 6 reference cases cover legacy construction/newer RTTI setters, shared/independent contexts and configuration before initialization or inside its first allocation callback. All parameter objects share one adapter. An already-entered first allocation uses the original callback once; subsequent allocations and releases use the latest configured table while preserving each parameter context and file-buffer setting. Tests cover handoff, nested lifecycle completion, orphaned epochs and retry, and distinguish actual RTTI release callbacks from later allocator-fixture reclamation. +- [RTTI initialization publication](../../../core/libs/prx/libSceJson2/Export.cpp) - concurrent reconfiguration and thrown allocator callbacks remain unverified. One hundred ninety-two additional SDK 5/6 reference cases cover legacy construction/newer RTTI setters, shared/independent contexts and configuration before initialization or inside its first allocation callback. All parameter objects share one adapter. An already-entered first allocation uses the original callback once; subsequent allocations and releases use the latest table while preserving each context and file-buffer setting. Tests cover handoff, lifecycle completion, orphaned epochs, SDK-specific mutex ownership and retry, distinguishing actual RTTI releases, SDK 5 null-free callbacks and later fixture reclamation. From d1c84725721083f637287c5c75fd7c374409527e Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 10:31:24 +0200 Subject: [PATCH 063/318] test(libSceJson2): verify interrupted nested array transfer --- core/libs/tests/GuestJsonArrayInsertion.cpp | 86 ++++++++++++++++++++- docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/core/libs/tests/GuestJsonArrayInsertion.cpp b/core/libs/tests/GuestJsonArrayInsertion.cpp index 6dc06cb998..4b86827a27 100644 --- a/core/libs/tests/GuestJsonArrayInsertion.cpp +++ b/core/libs/tests/GuestJsonArrayInsertion.cpp @@ -1,4 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include +#include #include #include #include @@ -70,6 +72,8 @@ struct ArrayTrace { bool once = false; unsigned notifications = 0; std::set headers; + std::map generations; + std::size_t generation = 0; }; static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); trace.requests.push_back(bytes); @@ -77,7 +81,9 @@ static void* APS5_VABI AllocateArray(void*, std::size_t bytes, void* context) { if (trace.budget > 0) --trace.budget; void* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); - std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); return memory; + std::memset(memory, 0xa5, bytes); trace.live.emplace(memory, bytes); + Require(trace.generations.emplace(memory, ++trace.generation).second, "Each insertion allocation must own a fresh live generation"); + return memory; } static void APS5_VABI FreeArray(void*, void* memory, void* context) { if (!memory) return; @@ -86,7 +92,7 @@ static void APS5_VABI FreeArray(void*, void* memory, void* context) { const auto* words = static_cast(memory); Require(words[0] == 0xffffffff && words[1] == UINT64_MAX, "Array teardown must expose the native header poison before allocator release"); } - Require(trace.live.erase(memory) == 1, "Array must release through its owning allocator"); std::free(memory); + Require(trace.live.erase(memory) == 1 && trace.generations.erase(memory) == 1, "Array must release through its owning allocator exactly once"); std::free(memory); } static void APS5_VABI NotifyArray(void*, int error, std::size_t bytes, void* context) { auto& trace = *static_cast(context); @@ -127,6 +133,79 @@ static void CheckSequence(const Array& array, bool front, bool once, int budget) } Require(node == head && _ZNK3sce4Json5Array4sizeEv(&array) == seen, "Insertion links and element count must agree"); } +struct TransferSnapshot { + void* pointer; + std::size_t generation; + std::vector bytes; +}; +static void CheckNestedTransferFailures(ArrayTrace& trace) { + const auto baseline = trace.generations; + for (bool front : {false, true}) for (bool once : {false, true}) for (int failure = 0; failure <= 5; ++failure) { + Array inner{}; Fill(inner); + Value source{}; _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&source, &inner); + Array destination{}; Fill(destination); + Iterator position{}, result{}; _ZNK3sce4Json5Array5beginEv(&position, &destination); + _ZN3sce4Json5Array8iterator7advanceEm(&position, 1); + const auto* survivor = _ZNK3sce4Json5Array8iteratordeEv(&position); + const auto sourceValue = source; + const auto* sourceArray = reinterpret_cast(source.payload); + auto* sourceHead = *static_cast(sourceArray->storage); + std::vector sourceAllocations; + const auto capture = [&](void* pointer) { + const auto size = trace.live.at(pointer); + const auto* bytes = static_cast(pointer); + sourceAllocations.push_back({pointer, trace.generations.at(pointer), {bytes, bytes + size}}); + }; + capture(reinterpret_cast(source.payload)); capture(sourceArray->storage); capture(const_cast(static_cast(sourceHead))); + for (auto* node = static_cast(sourceHead[1]); node != sourceHead; node = static_cast(node[1])) capture(const_cast(static_cast(node))); + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = (front ? 9 : 10) + failure; trace.once = once; + if (front) _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(&destination, &source); + else Require(_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE(&result, &destination, &position, &source) == &result, "Nested insertion must return its actual caller-owned iterator"); + trace.budget = -1; trace.once = false; + std::vector expected; + if (!front) expected.push_back(8); + for (const auto size : {8, 32, 48, 32, 48, 48, 48, 48, 48}) expected.push_back(size); + expected.insert(expected.end(), std::max(2, std::min(failure + 1, 5)), 8); + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, "Nested transfer must retain the measured copy, node and interrupted iterator allocation sequence"); + const unsigned failures = failure == 5 ? 0 : failure == 0 && !once ? 2 : 1; + Require(trace.notifications == notices + failures, "Nested transfer must notify every rejected iterator allocation without retrying the interrupted traversal"); + const bool rolledBack = front && failure < 5; + Require(_ZNK3sce4Json5Array4sizeEv(&destination) == (rolledBack ? 3u : 4u), "Prepending must remove its new entry after transfer failure while insertion retains it"); + Require(_ZNK3sce4Json5Array8iteratordeEv(&position) == survivor && survivor->type == 2 && survivor->payload == 20, "Interrupted nested transfer must preserve surviving iterator and element identity"); + auto* head = *static_cast(destination.storage); + auto* node = static_cast(head[1]); + const Value* inserted = nullptr; + for (int original = 0; original < 3; ++original) { + if (!rolledBack && original == (front ? 0 : 1)) { + inserted = reinterpret_cast(node + 2); node = static_cast(node[1]); + } + Require(node != head && reinterpret_cast(node + 2)->type == 2 && reinterpret_cast(node + 2)->payload == static_cast((original + 1) * 10), "Transfer failure must preserve the full order of all existing destination entries"); + node = static_cast(node[1]); + } + Require(node == head, "Interrupted nested transfer must preserve list closure"); + if (!rolledBack) { + Require(inserted && !inserted->parent && !inserted->auxiliary && inserted->type == 6 && inserted->payload && inserted->payload != source.payload, "Transferred nested arrays must own an independent root payload"); + if (!front) Require(_ZNK3sce4Json5Array8iteratordeEv(&result) == inserted, "Returning iterator must retain the new node after an interrupted child transfer"); + const auto* nested = reinterpret_cast(inserted->payload); + auto* nestedHead = *static_cast(nested->storage); + auto* child = static_cast(nestedHead[1]); + const auto rebound = std::max(0, std::min(failure - 1, 3)); + for (int index = 0; index < 3; ++index) { + Require(child != nestedHead, "Interrupted child rebinding must preserve every copied node"); + const auto* value = reinterpret_cast(child + 2); + Require(value->type == 2 && value->payload == static_cast((index + 1) * 10) && (value->parent == inserted) == (index < rebound), "Transfer must rebind only the child prefix admitted before iterator failure"); + child = static_cast(child[1]); + } + Require(child == nestedHead && _ZNK3sce4Json5Array4sizeEv(nested) == 3, "Nested payload count and circular links must survive interrupted transfer"); + } + Require(std::memcmp(&source, &sourceValue, sizeof(Value)) == 0, "Interrupted transfer must preserve the source Value bytes"); + for (const auto& snapshot : sourceAllocations) Require(trace.generations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "Interrupted transfer must preserve all source payload allocation generations and bytes"); + _ZN3sce4Json5Array8iteratorD1Ev(&result); _ZN3sce4Json5Array8iteratorD1Ev(&position); + _ZN3sce4Json5ArrayD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ArrayD1Ev(&inner); + Require(trace.generations == baseline, "Both retained and rolled-back nested transfers must reclaim every newly admitted allocation generation"); + } +} int main() { ArrayTrace trace; ArrayAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); @@ -186,6 +265,7 @@ int main() { _ZN3sce4Json5ArrayD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ArrayD1Ev(&inner); Require(trace.live.size() == baseline, "Nested insertion and iterator traversal must release every admitted allocation"); } - Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate insertion allocation contracts"); + CheckNestedTransferFailures(trace); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty() && trace.generations.empty(), "Terminate insertion allocation contracts"); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 1fe93b5f53..594fe41d05 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -58,7 +58,7 @@ - [sce::Json::Array::erase](../../../core/libs/prx/libSceJson2/Export.cpp) - invalid, missing or sentinel positions in an allocated array remain explicitly unsupported. Reference invalid-iterator paths can corrupt storage or fault; valid erasure retains its mutation when returned-iterator allocation fails. -- [Array insertion and payload transfer](../../../core/libs/prx/libSceJson2/Export.cpp) - string-copy allocation failures and successful nested-array transfer follow the reference protocol. Nested collection transfer failures, missing-sentinel paths, invalid insertion iterators and older-SDK callback combinations still need direct regression coverage. Invalid insertion iterators are explicitly unsupported. +- [Array insertion and payload transfer](../../../core/libs/prx/libSceJson2/Export.cpp) - object transfers, nested copy-stage failures, missing-sentinel paths and older-SDK callback combinations still need direct regression coverage. String-copy failures and successful nested-array transfer follow the reference protocol. Twenty-four additional reference-backed cases reject every temporary iterator allocation during transfer of a three-element nested array, using persistent and one-shot failure schedules. Insertion retains the new node and rebinds only the admitted child prefix; prepending removes the new entry on failure. Checks cover exact requests/notifications, surviving iterator identity, existing destination order, unchanged source bytes and allocation generations, result identity, nested links/counts and complete cleanup. Invalid insertion iterators are explicitly unsupported. - [Value keyed lookup](../../../core/libs/prx/libSceJson2/Export.cpp) - valid string-key overloads and C-string key-construction failures follow the observed iterator allocation and fallback protocol. Missing object handles, corrupted tree storage, lifecycle reentry during lookup and remaining older-SDK callback combinations remain unverified; invalid reference arguments are explicitly unsupported. From 6b37f97b336a84a8890e8f242ac27eee9ca6653f Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 10:35:21 +0200 Subject: [PATCH 064/318] test(libSceJson2): verify interrupted object payload transfer --- core/libs/tests/GuestJsonArrayInsertion.cpp | 81 ++++++++++++++------- docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 57 insertions(+), 26 deletions(-) diff --git a/core/libs/tests/GuestJsonArrayInsertion.cpp b/core/libs/tests/GuestJsonArrayInsertion.cpp index 4b86827a27..b941e4fdf4 100644 --- a/core/libs/tests/GuestJsonArrayInsertion.cpp +++ b/core/libs/tests/GuestJsonArrayInsertion.cpp @@ -20,6 +20,7 @@ struct Pair { String key; std::uint64_t reserved; Value value; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; extern "C" { void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value*, const Object*); void APS5_VABI _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(Array*, const Value*); Iterator* APS5_VABI _ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE(Iterator*, Array*, const Iterator*, const Value*); void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); @@ -53,6 +54,10 @@ const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object*); +Iterator* APS5_VABI _ZNK3sce4Json6Object3endEv(Iterator*, const Object*); +bool APS5_VABI _ZNK3sce4Json6Object8iteratorneERKS2_(const Iterator*, const Iterator*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEv(Iterator*); Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEi(Iterator*, int); @@ -140,32 +145,43 @@ struct TransferSnapshot { }; static void CheckNestedTransferFailures(ArrayTrace& trace) { const auto baseline = trace.generations; - for (bool front : {false, true}) for (bool once : {false, true}) for (int failure = 0; failure <= 5; ++failure) { - Array inner{}; Fill(inner); - Value source{}; _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&source, &inner); + for (bool object : {false, true}) for (bool front : {false, true}) for (bool once : {false, true}) for (int failure = 0; failure <= 5; ++failure) { + Array inner{}; Object innerObject{}; + if (object) { + _ZN3sce4Json6ObjectC1Ev(&innerObject); + int number = 10; + for (const char* text : {"a", "b", "c"}) { + String key{}; _ZN3sce4Json6StringC1EPKc(&key, text); + _ZN3sce4Json5ValueC1El(_ZN3sce4Json6ObjectixERKNS0_6StringE(&innerObject, &key), number); + number += 10; _ZN3sce4Json6StringD1Ev(&key); + } + } else Fill(inner); + const auto beforeSource = trace.generations; + Value source{}; + if (object) _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(&source, &innerObject); + else _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&source, &inner); + std::vector sourceAllocations; + for (const auto& [pointer, generation] : trace.generations) { + if (beforeSource.contains(pointer)) continue; + const auto* bytes = static_cast(pointer); + sourceAllocations.push_back({pointer, generation, {bytes, bytes + trace.live.at(pointer)}}); + } Array destination{}; Fill(destination); Iterator position{}, result{}; _ZNK3sce4Json5Array5beginEv(&position, &destination); _ZN3sce4Json5Array8iterator7advanceEm(&position, 1); const auto* survivor = _ZNK3sce4Json5Array8iteratordeEv(&position); const auto sourceValue = source; - const auto* sourceArray = reinterpret_cast(source.payload); - auto* sourceHead = *static_cast(sourceArray->storage); - std::vector sourceAllocations; - const auto capture = [&](void* pointer) { - const auto size = trace.live.at(pointer); - const auto* bytes = static_cast(pointer); - sourceAllocations.push_back({pointer, trace.generations.at(pointer), {bytes, bytes + size}}); - }; - capture(reinterpret_cast(source.payload)); capture(sourceArray->storage); capture(const_cast(static_cast(sourceHead))); - for (auto* node = static_cast(sourceHead[1]); node != sourceHead; node = static_cast(node[1])) capture(const_cast(static_cast(node))); + std::vector copyRequests = object + ? std::vector{8, 32, 104, 32, 104, 24, 2, 24, 2, 2, 2, 24, 2, 104, 2, 24, 2, 24, 2, 24, 2, 2, 2, 24, 2, 104, 2, 24, 2, 24, 2, 24, 2, 2, 2, 24, 2, 104, 2, 24, 2, 48} + : std::vector{8, 32, 48, 32, 48, 48, 48, 48, 48}; const auto requests = trace.requests.size(); const auto notices = trace.notifications; - trace.budget = (front ? 9 : 10) + failure; trace.once = once; + trace.budget = static_cast(copyRequests.size()) + (front ? 0 : 1) + failure; trace.once = once; if (front) _ZN3sce4Json5Array10push_frontERKNS0_5ValueE(&destination, &source); else Require(_ZN3sce4Json5Array6insertERKNS1_8iteratorERKNS0_5ValueE(&result, &destination, &position, &source) == &result, "Nested insertion must return its actual caller-owned iterator"); trace.budget = -1; trace.once = false; std::vector expected; if (!front) expected.push_back(8); - for (const auto size : {8, 32, 48, 32, 48, 48, 48, 48, 48}) expected.push_back(size); + expected.insert(expected.end(), copyRequests.begin(), copyRequests.end()); expected.insert(expected.end(), std::max(2, std::min(failure + 1, 5)), 8); Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, "Nested transfer must retain the measured copy, node and interrupted iterator allocation sequence"); const unsigned failures = failure == 5 ? 0 : failure == 0 && !once ? 2 : 1; @@ -185,24 +201,39 @@ static void CheckNestedTransferFailures(ArrayTrace& trace) { } Require(node == head, "Interrupted nested transfer must preserve list closure"); if (!rolledBack) { - Require(inserted && !inserted->parent && !inserted->auxiliary && inserted->type == 6 && inserted->payload && inserted->payload != source.payload, "Transferred nested arrays must own an independent root payload"); + Require(inserted && !inserted->parent && !inserted->auxiliary && inserted->type == (object ? 7 : 6) && inserted->payload && inserted->payload != source.payload, "Transferred nested collections must own an independent root payload"); if (!front) Require(_ZNK3sce4Json5Array8iteratordeEv(&result) == inserted, "Returning iterator must retain the new node after an interrupted child transfer"); - const auto* nested = reinterpret_cast(inserted->payload); - auto* nestedHead = *static_cast(nested->storage); - auto* child = static_cast(nestedHead[1]); const auto rebound = std::max(0, std::min(failure - 1, 3)); - for (int index = 0; index < 3; ++index) { - Require(child != nestedHead, "Interrupted child rebinding must preserve every copied node"); - const auto* value = reinterpret_cast(child + 2); + const auto checkChild = [&](const Value* value, int index) { Require(value->type == 2 && value->payload == static_cast((index + 1) * 10) && (value->parent == inserted) == (index < rebound), "Transfer must rebind only the child prefix admitted before iterator failure"); - child = static_cast(child[1]); + }; + if (object) { + const auto* nested = reinterpret_cast(inserted->payload); + Iterator current{}, end{}; _ZNK3sce4Json6Object5beginEv(¤t, nested); _ZNK3sce4Json6Object3endEv(&end, nested); + for (int index = 0; index < 3; ++index) { + Require(_ZNK3sce4Json6Object8iteratorneERKS2_(¤t, &end), "Interrupted object transfer must retain every copied tree node"); + const auto* pair = _ZNK3sce4Json6Object8iteratordeEv(¤t); + const char key[]{static_cast('a' + index), 0}; + Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair->key), key) == 0, "Object transfer must preserve ordered public keys"); + checkChild(&pair->value, index); _ZN3sce4Json6Object8iteratorppEv(¤t); + } + Require(!_ZNK3sce4Json6Object8iteratorneERKS2_(¤t, &end) && _ZNK3sce4Json6Object4sizeEv(nested) == 3, "Interrupted object transfer must preserve tree endpoints and count"); + _ZN3sce4Json6Object8iteratorD1Ev(¤t); _ZN3sce4Json6Object8iteratorD1Ev(&end); + } else { + const auto* nested = reinterpret_cast(inserted->payload); + auto* nestedHead = *static_cast(nested->storage); + auto* child = static_cast(nestedHead[1]); + for (int index = 0; index < 3; ++index) { + Require(child != nestedHead, "Interrupted child rebinding must preserve every copied node"); + checkChild(reinterpret_cast(child + 2), index); child = static_cast(child[1]); + } + Require(child == nestedHead && _ZNK3sce4Json5Array4sizeEv(nested) == 3, "Nested payload count and circular links must survive interrupted transfer"); } - Require(child == nestedHead && _ZNK3sce4Json5Array4sizeEv(nested) == 3, "Nested payload count and circular links must survive interrupted transfer"); } Require(std::memcmp(&source, &sourceValue, sizeof(Value)) == 0, "Interrupted transfer must preserve the source Value bytes"); for (const auto& snapshot : sourceAllocations) Require(trace.generations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "Interrupted transfer must preserve all source payload allocation generations and bytes"); _ZN3sce4Json5Array8iteratorD1Ev(&result); _ZN3sce4Json5Array8iteratorD1Ev(&position); - _ZN3sce4Json5ArrayD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ArrayD1Ev(&inner); + _ZN3sce4Json5ArrayD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ArrayD1Ev(&inner); _ZN3sce4Json6ObjectD1Ev(&innerObject); Require(trace.generations == baseline, "Both retained and rolled-back nested transfers must reclaim every newly admitted allocation generation"); } } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 594fe41d05..02ae4094f9 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -58,7 +58,7 @@ - [sce::Json::Array::erase](../../../core/libs/prx/libSceJson2/Export.cpp) - invalid, missing or sentinel positions in an allocated array remain explicitly unsupported. Reference invalid-iterator paths can corrupt storage or fault; valid erasure retains its mutation when returned-iterator allocation fails. -- [Array insertion and payload transfer](../../../core/libs/prx/libSceJson2/Export.cpp) - object transfers, nested copy-stage failures, missing-sentinel paths and older-SDK callback combinations still need direct regression coverage. String-copy failures and successful nested-array transfer follow the reference protocol. Twenty-four additional reference-backed cases reject every temporary iterator allocation during transfer of a three-element nested array, using persistent and one-shot failure schedules. Insertion retains the new node and rebinds only the admitted child prefix; prepending removes the new entry on failure. Checks cover exact requests/notifications, surviving iterator identity, existing destination order, unchanged source bytes and allocation generations, result identity, nested links/counts and complete cleanup. Invalid insertion iterators are explicitly unsupported. +- [Array insertion and payload transfer](../../../core/libs/prx/libSceJson2/Export.cpp) - nested copy-stage failures, missing-sentinel paths and older-SDK callback combinations still need direct regression coverage. String-copy failures and successful nested-array transfer follow the reference protocol. Forty-eight additional reference-backed cases reject every temporary iterator allocation during transfer of three-element nested arrays and objects, using persistent and one-shot failure schedules. Insertion retains the new node and rebinds only the admitted child prefix; prepending removes the new entry on failure. Checks cover exact requests/notifications, surviving iterator identity, existing destination order, unchanged source bytes and allocation generations, result identity, ordered object keys, nested links/counts and complete cleanup. Invalid insertion iterators are explicitly unsupported. - [Value keyed lookup](../../../core/libs/prx/libSceJson2/Export.cpp) - valid string-key overloads and C-string key-construction failures follow the observed iterator allocation and fallback protocol. Missing object handles, corrupted tree storage, lifecycle reentry during lookup and remaining older-SDK callback combinations remain unverified; invalid reference arguments are explicitly unsupported. From f9025a46bb39741558441f373c8396f6429e2931 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 10:39:52 +0200 Subject: [PATCH 065/318] test(libSceJson2): verify legacy Pair callback copy failures --- core/libs/tests/GuestJsonLegacyHandlers.cpp | 92 +++++++++++++++++++-- docs/dev/debt/libSceJson2.md | 4 +- 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index 567063856d..e461e3864e 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -8,13 +8,23 @@ #include #include #include +#include struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; struct Array { void* storage; }; +struct String { void* storage; }; +struct Pair { String key; std::uint64_t reserved; Value value; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; }; using Handler = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); using NullHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); extern "C" { +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair*, const String*, const Value*); +void APS5_VABI _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(Pair*, const String*, const Value*); +void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair*); +void APS5_VABI _ZN3sce4Json6Object4PairD2Ev(Pair*); void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const Parameter*); int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); @@ -43,6 +53,11 @@ struct LegacyTrace { std::map guest; std::map heap; int budget = -1; + bool once = false; + std::vector requests; + std::map guestGenerations; + std::map heapGenerations; + std::size_t generation = 0; unsigned notifications = 0; }; static LegacyTrace* HeapTrace = nullptr; @@ -55,27 +70,30 @@ extern "C" void* APS5_VABI _Znwm_nid_postfix(std::size_t bytes) { if (!memory) throw std::bad_alloc(); std::memset(memory, 0xa5, bytes); Require(HeapTrace && bytes == 56, "Legacy map nodes must use the actual libc new allocation boundary"); - HeapTrace->heap.emplace(memory, bytes); + Require(HeapTrace->heap.emplace(memory, bytes).second && HeapTrace->heapGenerations.emplace(memory, ++HeapTrace->generation).second, "Legacy libc allocations must own distinct live generations"); return memory; } extern "C" void APS5_VABI _ZdlPv_nid_postfix(void* memory) { - Require(HeapTrace && HeapTrace->heap.erase(memory) == 1, "Legacy map nodes must return through libc delete"); + Require(HeapTrace && HeapTrace->heap.erase(memory) == 1 && HeapTrace->heapGenerations.erase(memory) == 1, "Legacy map nodes must return through libc delete"); std::free(memory); } static void* APS5_VABI AllocateLegacy(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); - if (trace.budget == 0) return nullptr; + trace.requests.push_back(bytes); + if (trace.budget == 0) { if (trace.once) trace.budget = -1; return nullptr; } if (trace.budget > 0) --trace.budget; void* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); - std::memset(memory, 0xa5, bytes); trace.guest.emplace(memory, bytes); return memory; + std::memset(memory, 0xa5, bytes); + Require(trace.guest.emplace(memory, bytes).second && trace.guestGenerations.emplace(memory, ++trace.generation).second, "Legacy guest allocations must own distinct live generations"); + return memory; } static void APS5_VABI FreeLegacy(void*, void* memory, void* context) { if (!memory) return; - Require(static_cast(context)->guest.erase(memory) == 1, "Legacy metadata must return through its owning allocator"); + Require(static_cast(context)->guest.erase(memory) == 1 && static_cast(context)->guestGenerations.erase(memory) == 1, "Legacy metadata must return through its owning allocator"); std::free(memory); } static void APS5_VABI NotifyLegacy(void*, int error, std::size_t bytes, void* context) { - Require(static_cast(error) == 0x80848102 && (bytes == 24 || bytes == 48), "Legacy allocation failure must identify its map header or array node"); + Require(static_cast(error) == 0x80848102 && bytes == static_cast(context)->requests.back(), "Legacy allocation failure must identify its actual rejected request"); ++static_cast(context)->notifications; } static void* const LegacyVtable[]{nullptr, nullptr, reinterpret_cast(AllocateLegacy), reinterpret_cast(FreeLegacy), reinterpret_cast(NotifyLegacy)}; @@ -124,6 +142,65 @@ static const Value& APS5_VABI RecoverLegacyNull(std::int32_t requested, const Va Require(requested == 1 && !parent && *_ZNK3sce4Json5Value10getBooleanEv(value.other), "Legacy null recovery must allow independent mismatch recovery before the reentry guard"); return value.replacement->replacement; } +struct LegacySnapshot { void* pointer; std::size_t generation; std::vector bytes; }; +static void CheckPairCopies(LegacyTrace& trace, Context& context) { + const auto guestBaseline = trace.guestGenerations, heapBaseline = trace.heapGenerations; + for (bool text : {false, true}) for (bool once : {false, true}) for (unsigned alias = 0; alias < 2; ++alias) for (int budget = 0; budget <= (text ? 5 : 2); ++budget) { + Value source{}; ParseLegacy(source, text ? "\"abc\"" : "9"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &context) == 0, "Register the actual legacy Pair source map record"); + std::vector sourceAllocations; + for (const auto& [pointer, generation] : trace.guestGenerations) { + if (guestBaseline.contains(pointer)) continue; + const auto* bytes = static_cast(pointer); + sourceAllocations.push_back({pointer, generation, {bytes, bytes + trace.guest.at(pointer)}}); + } + const LegacyNode* sourceNode = nullptr; + for (const auto& [pointer, size] : trace.heap) { + const auto* node = static_cast(pointer); + if (!node->nil && node->key == &source) sourceNode = node; + } + Require(sourceNode && sourceNode->callback == RecoverLegacy && sourceNode->context == &context, "Pair copying requires an independently owned source callback record"); + const auto sourceGeneration = trace.heapGenerations.at(const_cast(sourceNode)); + const Value sourceValue = source; + String key{}; _ZN3sce4Json6StringC1EPKc(&key, "key"); + Pair pair; std::memset(&pair, 0xa5, sizeof(pair)); + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = budget; trace.once = once; + if (alias) _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); + else _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); + trace.budget = -1; trace.once = false; + std::vector expected{24}; + if (budget > 0) expected.push_back(4); + if (text) { + expected.push_back(8); + if ((once && budget < 2) || budget >= 3) expected.push_back(24); + if ((once && budget < 2) || budget >= 4) expected.push_back(4); + } + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, "Legacy Pair failure must preserve the measured key, Value handle, text header and character request order"); + const unsigned failed = text ? (once ? unsigned(budget < 5) : budget < 2 ? 2u : unsigned(budget < 5)) : unsigned(budget < 2); + Require(trace.notifications == notices + failed, "Legacy Pair copies must notify every admitted allocation failure"); + const int expectedType = text ? ((once && budget < 2) || budget >= 4 ? 5 : 0) : 3; + Require(pair.reserved == UINT64_C(0xa5a5a5a5a5a5a5a5) && !pair.value.parent && !pair.value.auxiliary && pair.value.type == expectedType, "Legacy Pair copies must preserve inactive Pair bytes and retain independent Value ownership"); + Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), budget < 2 ? "" : "key") == 0, "Legacy Pair key failures must preserve the measured empty or completed key"); + if (expectedType == 5) Require(pair.value.payload && std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(pair.value.payload)), budget == 4 ? "" : "abc") == 0, "Legacy Pair string failures must preserve the admitted partial text"); + if (!text) Require(pair.value.payload == 9, "Failed Pair key allocation must not suppress scalar copying"); + CheckMap(trace, {{&source, &context}, {&pair.value, &context}}); + const auto calls = context.calls; + Require(_ZNK3sce4Json5Value10getBooleanEv(&pair.value) == reinterpret_cast(&context.replacement.payload) && context.calls == calls + 1 && context.source == &pair.value && !context.parent, + "Every partial legacy Pair must retain its independently copied map callback and exact borrowed replacement identity"); + if (alias) _ZN3sce4Json6Object4PairD2Ev(&pair); + else _ZN3sce4Json6Object4PairD1Ev(&pair); + CheckMap(trace, {{&source, &context}}); + Require(std::memcmp(&source, &sourceValue, sizeof(source)) == 0 && trace.heapGenerations.at(const_cast(sourceNode)) == sourceGeneration && sourceNode->callback == RecoverLegacy && sourceNode->context == &context, + "Pair cleanup must preserve source bytes and its original callback-record generation"); + for (const auto& snapshot : sourceAllocations) Require(trace.guestGenerations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, + "Failed Pair copies must preserve all source guest allocation generations and bytes"); + Require(_ZNK3sce4Json5Value10getBooleanEv(&source) == reinterpret_cast(&context.replacement.payload) && context.source == &source && !context.parent, + "Source callbacks must remain usable after failed Pair copy teardown"); + _ZN3sce4Json6StringD1Ev(&key); _ZN3sce4Json5ValueD1Ev(&source); CheckMap(trace, {}); + Require(trace.guestGenerations == guestBaseline && trace.heapGenerations == heapBaseline, "Legacy Pair copies must reclaim every new generation in both allocator domains"); + } +} int main() { LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; @@ -177,6 +254,7 @@ int main() { Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&null, RecoverLegacyNull, &nullContext) == 0 && *_ZNK3sce4Json5Value10getBooleanEv(&null), "Legacy local null recovery must preserve its pre-guard dispatch"); _ZN3sce4Json5ValueD1Ev(&null); _ZN3sce4Json5ValueD1Ev(&assigned); _ZN3sce4Json5ValueD1Ev(©); _ZN3sce4Json5ValueD1Ev(&source); CheckMap(trace, {}); + CheckPairCopies(trace, first); std::array roots{}; std::map entries; for (unsigned index = 0; index < roots.size(); ++index) { auto& root = roots[index * 13 % roots.size()]; ParseLegacy(root); @@ -189,7 +267,7 @@ int main() { } _ZN3sce4Json5ValueD1Ev(&second.replacement); _ZN3sce4Json5ValueD1Ev(&first.replacement); Require(trace.guest.size() == baseline + 1 && trace.heap.size() == 1, "Empty legacy map storage must remain owned until termination"); - Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.guest.empty() && trace.heap.empty(), "Termination must release both allocator domains and all legacy map resources"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.guest.empty() && trace.heap.empty() && trace.guestGenerations.empty() && trace.heapGenerations.empty(), "Termination must release both allocator domains and all legacy map resources"); HeapTrace = nullptr; return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 02ae4094f9..bf43ed7bb1 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -8,7 +8,7 @@ - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Remaining kernel-status and callback/concurrent lifecycle paths, plus full guest execution, remain unverified. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter is now reproduced through the inspected float32 conversion, six truncating fractional digits, range saturation and NaN integer result. The regression covers signed zero, precision boundaries, large values, infinities and NaNs; floating-point rounding modes and exception flags remain unverified. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. -- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and older-SDK copies still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. +- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and nested older-SDK copies still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - libc-heap failures and untested legacy callback combinations remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Additional legacy compatibility remains unverified. - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback-value failure behavior and compatibility below SDK 6 remain incomplete. @@ -79,3 +79,5 @@ - [Initialization callback publication](../../../core/libs/prx/libSceJson2/Export.cpp) - concurrent publication remains unverified. One hundred forty-four direct-allocator reference cases span SDK 5/6, both initializer overloads, shared/separate owners and receiver/context domains. They verify saved file-buffer settings, nested handoff, completed nested lifecycles and retry. Double success orphans the first epoch in its original domain: eleven resources/one mutex for SDK 5, twelve/two for SDK 6. Outer termination reclaims only its own epoch and nested termination reports not-initialized. Tests assert ownership and stale-domain avoidance, then separately reclaim the observed orphaned epoch through the caller-owned fixture. SDK 5 teardown also forwards its observed null free; fixture reclamation is not library cleanup. - [RTTI initialization publication](../../../core/libs/prx/libSceJson2/Export.cpp) - concurrent reconfiguration and thrown allocator callbacks remain unverified. One hundred ninety-two additional SDK 5/6 reference cases cover legacy construction/newer RTTI setters, shared/independent contexts and configuration before initialization or inside its first allocation callback. All parameter objects share one adapter. An already-entered first allocation uses the original callback once; subsequent allocations and releases use the latest table while preserving each context and file-buffer setting. Tests cover handoff, lifecycle completion, orphaned epochs, SDK-specific mutex ownership and retry, distinguishing actual RTTI releases, SDK 5 null-free callbacks and later fixture reclamation. + +- [Legacy Pair callback copies](../../../core/libs/prx/libSceJson2/Export.cpp) - nested collections, inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. Thirty-six SDK 5 reference-backed cases cover numeric and string sources with a registered mismatch handler, both constructor/destructor aliases and every guest allocation failure position through success using persistent and one-shot schedules. Even a null partial copy retains its independent callback map record. Checks cover exact request/notification order, partial key/text admission, inactive Pair bytes, borrowed callback result identity, independent auxiliary ownership, source allocation bytes/generations and source callback usability after copy cleanup. Every case restores the original live generations in both the guest allocator and libc heap. From 00705e12d94aaf0cd501ad260e1ce952c8a94284 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 10:49:00 +0200 Subject: [PATCH 066/318] test(libSceJson2): verify nested legacy callback copy failures --- core/libs/tests/GuestJsonLegacyHandlers.cpp | 185 ++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 6 +- 2 files changed, 189 insertions(+), 2 deletions(-) diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index e461e3864e..308cafbcf7 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -9,6 +9,7 @@ #include #include #include +#include struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; struct Array { void* storage; }; @@ -201,6 +202,189 @@ static void CheckPairCopies(LegacyTrace& trace, Context& context) { Require(trace.guestGenerations == guestBaseline && trace.heapGenerations == heapBaseline, "Legacy Pair copies must reclaim every new generation in both allocator domains"); } } +struct LegacyArrayLink { LegacyArrayLink* previous; LegacyArrayLink* next; Value value; }; +struct LegacyArrayHeader { LegacyArrayLink* head; std::size_t count; std::uint64_t reserved; }; +struct LegacyTreeLink { LegacyTreeLink* parent; LegacyTreeLink* left; LegacyTreeLink* right; unsigned char key[24]; Pair pair; unsigned char color; unsigned char nil; unsigned char reserved[6]; }; +struct LegacyTreeHeader { LegacyTreeLink* head; std::size_t count; std::uint64_t reserved; }; +static_assert(sizeof(LegacyArrayHeader) == 24 && sizeof(LegacyTreeHeader) == 24 && sizeof(LegacyTreeLink) == 104); +static void CollectionShape(const Value& value, const Value* parent, std::string& output, std::vector& values); +static void CollectionTreeShape(const LegacyTreeLink* node, const Value* parent, std::string& output, std::vector& values) { + if (node->nil) return; + CollectionTreeShape(node->left, parent, output, values); + output += "k=" + std::string(*reinterpret_cast(node->key)) + "/q=" + _ZNK3sce4Json6String5c_strEv(&node->pair.key) + "/"; + CollectionShape(node->pair.value, parent, output, values); + CollectionTreeShape(node->right, parent, output, values); +} +static void CollectionShape(const Value& value, const Value* parent, std::string& output, std::vector& values) { + values.push_back(&value); + output += std::to_string(value.type) + ":" + std::to_string(value.parent ? value.parent == parent ? 1 : 2 : 0); + if (value.type > 0 && value.type < 5) output += "=" + std::to_string(value.payload); + if (value.type == 5) output += "s=" + std::string(value.payload ? _ZNK3sce4Json6String5c_strEv(reinterpret_cast(value.payload)) : ""); + if (value.type == 6) { + auto* header = value.payload ? *reinterpret_cast(value.payload) : nullptr; + if (!header) { output += "x;"; return; } + output += "a" + std::to_string(header->count) + "["; + if (!header->head) output += "missing"; + else for (auto* node = header->head->next; node != header->head; node = node->next) CollectionShape(node->value, &value, output, values); + output += "]"; + } + if (value.type == 7) { + auto* header = value.payload ? *reinterpret_cast(value.payload) : nullptr; + if (!header) { output += "x;"; return; } + output += "o" + std::to_string(header->count) + "{"; + if (!header->head) output += "missing"; + else CollectionTreeShape(header->head->parent, &value, output, values); + output += "}"; + } + output += ";"; +} +struct LegacyCollectionCase { + unsigned source; + bool once; + int budget; + unsigned notifications; + std::size_t requests; + std::uint64_t requestOrder; + const char* key; + const char* shape; +}; +static const LegacyCollectionCase CollectionCases[]{ + {0, 0, 0, 2, 2, UINT64_C(0x08842007b530c95d), "", "0:0;"}, + {0, 0, 1, 2, 3, UINT64_C(0xa922fa18e1c37583), "", "0:0;"}, + {0, 0, 2, 1, 3, UINT64_C(0xa922fa18e1c37583), "key", "0:0;"}, + {0, 0, 3, 1, 4, UINT64_C(0x29e493479f20d661), "key", "0:0;"}, + {0, 0, 4, 2, 6, UINT64_C(0x231a8cd446ee0ac9), "key", "6:0a0[missing];"}, + {0, 0, 5, 1, 6, UINT64_C(0x231a8cd446ee0ac9), "key", "6:0a0[missing];"}, + {0, 0, 6, 1, 7, UINT64_C(0x942845b4867ca51b), "key", "6:0a0[];"}, + {0, 0, 7, 1, 8, UINT64_C(0x3d1384c085cc7f49), "key", "6:0a1[0:0;];"}, + {0, 0, 8, 1, 9, UINT64_C(0x94a9e4235a7c56a3), "key", "6:0a1[0:0;];"}, + {0, 0, 9, 1, 10, UINT64_C(0x19057312c1477af1), "key", "6:0a1[0:0;];"}, + {0, 0, 10, 1, 11, UINT64_C(0xcbbd21de6c7551fb), "key", "6:0a1[7:0o0{missing};];"}, + {0, 0, 11, 1, 12, UINT64_C(0xa7b21ff24b599cc9), "key", "6:0a1[7:0o0{};];"}, + {0, 0, 12, 3, 15, UINT64_C(0x44ba72eae582feb9), "key", "6:0a1[7:0o1{k=/q=/0:0;};];"}, + {0, 0, 13, 2, 15, UINT64_C(0x44ba72eae582feb9), "key", "6:0a1[7:0o1{k=s/q=/0:0;};];"}, + {0, 0, 14, 2, 16, UINT64_C(0x4be45a23fda819b9), "key", "6:0a1[7:0o1{k=s/q=/0:0;};];"}, + {0, 0, 15, 1, 16, UINT64_C(0x4be45a23fda819b9), "key", "6:0a1[7:0o1{k=s/q=s/0:0;};];"}, + {0, 0, 16, 1, 17, UINT64_C(0x9d1ecc2804a38c93), "key", "6:0a1[7:0o1{k=s/q=s/0:0;};];"}, + {0, 0, 17, 1, 18, UINT64_C(0x9ee178ffe1e7da63), "key", "6:0a1[7:0o1{k=s/q=s/5:0s=;};];"}, + {0, 0, 18, 0, 18, UINT64_C(0x9ee178ffe1e7da63), "key", "6:0a1[7:0o1{k=s/q=s/5:0s=t;};];"}, + {0, 1, 0, 1, 17, UINT64_C(0xd9fd6582200984bd), "", "6:0a1[7:0o1{k=s/q=s/5:0s=t;};];"}, + {0, 1, 1, 1, 18, UINT64_C(0x9ee178ffe1e7da63), "", "6:0a1[7:0o1{k=s/q=s/5:0s=t;};];"}, + {0, 1, 2, 1, 3, UINT64_C(0xa922fa18e1c37583), "key", "0:0;"}, + {0, 1, 3, 1, 4, UINT64_C(0x29e493479f20d661), "key", "0:0;"}, + {0, 1, 4, 1, 18, UINT64_C(0x9ee178ffe1e7da63), "key", "6:0a1[7:0o1{k=s/q=s/5:0s=t;};];"}, + {0, 1, 5, 1, 6, UINT64_C(0x231a8cd446ee0ac9), "key", "6:0a0[missing];"}, + {0, 1, 6, 1, 7, UINT64_C(0x942845b4867ca51b), "key", "6:0a0[];"}, + {0, 1, 7, 1, 8, UINT64_C(0x3d1384c085cc7f49), "key", "6:0a1[0:0;];"}, + {0, 1, 8, 1, 9, UINT64_C(0x94a9e4235a7c56a3), "key", "6:0a1[0:0;];"}, + {0, 1, 9, 1, 10, UINT64_C(0x19057312c1477af1), "key", "6:0a1[0:0;];"}, + {0, 1, 10, 1, 11, UINT64_C(0xcbbd21de6c7551fb), "key", "6:0a1[7:0o0{missing};];"}, + {0, 1, 11, 1, 12, UINT64_C(0xa7b21ff24b599cc9), "key", "6:0a1[7:0o0{};];"}, + {0, 1, 12, 1, 18, UINT64_C(0x9ee178ffe1e7da63), "key", "6:0a1[7:0o1{k=/q=s/5:0s=t;};];"}, + {0, 1, 13, 1, 17, UINT64_C(0x68efa027e7058763), "key", "6:0a1[7:0o1{k=s/q=/5:0s=t;};];"}, + {0, 1, 14, 1, 18, UINT64_C(0x9ee178ffe1e7da63), "key", "6:0a1[7:0o1{k=s/q=/5:0s=t;};];"}, + {0, 1, 15, 1, 16, UINT64_C(0x4be45a23fda819b9), "key", "6:0a1[7:0o1{k=s/q=s/0:0;};];"}, + {0, 1, 16, 1, 17, UINT64_C(0x9d1ecc2804a38c93), "key", "6:0a1[7:0o1{k=s/q=s/0:0;};];"}, + {0, 1, 17, 1, 18, UINT64_C(0x9ee178ffe1e7da63), "key", "6:0a1[7:0o1{k=s/q=s/5:0s=;};];"}, + {0, 1, 18, 0, 18, UINT64_C(0x9ee178ffe1e7da63), "key", "6:0a1[7:0o1{k=s/q=s/5:0s=t;};];"}, + {1, 0, 0, 2, 2, UINT64_C(0x08842007b530c95d), "", "0:0;"}, + {1, 0, 1, 2, 3, UINT64_C(0xa922fa18e1c37583), "", "0:0;"}, + {1, 0, 2, 1, 3, UINT64_C(0xa922fa18e1c37583), "key", "0:0;"}, + {1, 0, 3, 1, 4, UINT64_C(0x29e493479f20d661), "key", "0:0;"}, + {1, 0, 4, 1, 5, UINT64_C(0x503c4bb364cbb14b), "key", "0:0;"}, + {1, 0, 5, 1, 6, UINT64_C(0x2225c4d4461dfe79), "key", "7:0o0{missing};"}, + {1, 0, 6, 1, 7, UINT64_C(0x242b85b324f6b6e3), "key", "7:0o0{};"}, + {1, 0, 7, 3, 10, UINT64_C(0x154035d1032f5a9b), "key", "7:0o1{k=/q=/0:0;};"}, + {1, 0, 8, 2, 10, UINT64_C(0x154035d1032f5a9b), "key", "7:0o1{k=a/q=/0:0;};"}, + {1, 0, 9, 2, 11, UINT64_C(0x4b61a128696594d3), "key", "7:0o1{k=a/q=/0:0;};"}, + {1, 0, 10, 1, 11, UINT64_C(0x4b61a128696594d3), "key", "7:0o1{k=a/q=a/0:0;};"}, + {1, 0, 11, 1, 12, UINT64_C(0x7c79a2ab179bd4f1), "key", "7:0o1{k=a/q=a/0:0;};"}, + {1, 0, 12, 2, 14, UINT64_C(0xa516a88d9f1de459), "key", "7:0o1{k=a/q=a/6:0a0[missing];};"}, + {1, 0, 13, 1, 14, UINT64_C(0xa516a88d9f1de459), "key", "7:0o1{k=a/q=a/6:0a0[missing];};"}, + {1, 0, 14, 1, 15, UINT64_C(0xa364d1a55fcb1e6b), "key", "7:0o1{k=a/q=a/6:0a0[];};"}, + {1, 0, 15, 1, 16, UINT64_C(0x6f6e9f01c624a239), "key", "7:0o1{k=a/q=a/6:0a1[0:0;];};"}, + {1, 0, 16, 1, 17, UINT64_C(0x7d9a5103b03f7e13), "key", "7:0o1{k=a/q=a/6:0a1[0:0;];};"}, + {1, 0, 17, 1, 18, UINT64_C(0xacb5ba447be336e3), "key", "7:0o1{k=a/q=a/6:0a1[5:0s=;];};"}, + {1, 0, 18, 0, 18, UINT64_C(0xacb5ba447be336e3), "key", "7:0o1{k=a/q=a/6:0a1[5:0s=t;];};"}, + {1, 1, 0, 1, 17, UINT64_C(0xc5a67b07339990bd), "", "7:0o1{k=a/q=a/6:0a1[5:0s=t;];};"}, + {1, 1, 1, 1, 18, UINT64_C(0xacb5ba447be336e3), "", "7:0o1{k=a/q=a/6:0a1[5:0s=t;];};"}, + {1, 1, 2, 1, 3, UINT64_C(0xa922fa18e1c37583), "key", "0:0;"}, + {1, 1, 3, 1, 4, UINT64_C(0x29e493479f20d661), "key", "0:0;"}, + {1, 1, 4, 1, 5, UINT64_C(0x503c4bb364cbb14b), "key", "0:0;"}, + {1, 1, 5, 1, 6, UINT64_C(0x2225c4d4461dfe79), "key", "7:0o0{missing};"}, + {1, 1, 6, 1, 7, UINT64_C(0x242b85b324f6b6e3), "key", "7:0o0{};"}, + {1, 1, 7, 1, 18, UINT64_C(0xacb5ba447be336e3), "key", "7:0o1{k=/q=a/6:0a1[5:0s=t;];};"}, + {1, 1, 8, 1, 17, UINT64_C(0xe052cbb56cadb10b), "key", "7:0o1{k=a/q=/6:0a1[5:0s=t;];};"}, + {1, 1, 9, 1, 18, UINT64_C(0xacb5ba447be336e3), "key", "7:0o1{k=a/q=/6:0a1[5:0s=t;];};"}, + {1, 1, 10, 1, 11, UINT64_C(0x4b61a128696594d3), "key", "7:0o1{k=a/q=a/0:0;};"}, + {1, 1, 11, 1, 12, UINT64_C(0x7c79a2ab179bd4f1), "key", "7:0o1{k=a/q=a/0:0;};"}, + {1, 1, 12, 1, 18, UINT64_C(0xacb5ba447be336e3), "key", "7:0o1{k=a/q=a/6:0a1[5:0s=t;];};"}, + {1, 1, 13, 1, 14, UINT64_C(0xa516a88d9f1de459), "key", "7:0o1{k=a/q=a/6:0a0[missing];};"}, + {1, 1, 14, 1, 15, UINT64_C(0xa364d1a55fcb1e6b), "key", "7:0o1{k=a/q=a/6:0a0[];};"}, + {1, 1, 15, 1, 16, UINT64_C(0x6f6e9f01c624a239), "key", "7:0o1{k=a/q=a/6:0a1[0:0;];};"}, + {1, 1, 16, 1, 17, UINT64_C(0x7d9a5103b03f7e13), "key", "7:0o1{k=a/q=a/6:0a1[0:0;];};"}, + {1, 1, 17, 1, 18, UINT64_C(0xacb5ba447be336e3), "key", "7:0o1{k=a/q=a/6:0a1[5:0s=;];};"}, + {1, 1, 18, 0, 18, UINT64_C(0xacb5ba447be336e3), "key", "7:0o1{k=a/q=a/6:0a1[5:0s=t;];};"}, +}; +static void CheckCollectionCopies(LegacyTrace& trace, Context& context) { + const auto guestBaseline = trace.guestGenerations, heapBaseline = trace.heapGenerations; + for (const auto& test : CollectionCases) for (unsigned alias = 0; alias < 2; ++alias) { + Value source{}; ParseLegacy(source, test.source ? R"({"a":["t"]})" : R"([{"s":"t"}])"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &context) == 0, "Register nested legacy Pair source recovery"); + std::vector sourceAllocations; + for (const auto& [pointer, generation] : trace.guestGenerations) { + if (guestBaseline.contains(pointer)) continue; + const auto* bytes = static_cast(pointer); + sourceAllocations.push_back({pointer, generation, {bytes, bytes + trace.guest.at(pointer)}}); + } + const LegacyNode* sourceNode = nullptr; + for (const auto& [pointer, size] : trace.heap) { + const auto* node = static_cast(pointer); + if (!node->nil && node->key == &source) sourceNode = node; + } + Require(sourceNode != nullptr, "Nested Pair copies require the actual source callback record"); + const auto sourceGeneration = trace.heapGenerations.at(const_cast(sourceNode)); + const Value sourceValue = source; + String key{}; _ZN3sce4Json6StringC1EPKc(&key, "key"); + Pair pair; std::memset(&pair, 0xa5, sizeof(pair)); + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = test.budget; trace.once = test.once; + if (alias) _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); + else _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); + trace.budget = -1; trace.once = false; + std::uint64_t requestOrder = UINT64_C(14695981039346656037); + for (auto i = requests; i < trace.requests.size(); ++i) { + requestOrder ^= trace.requests[i]; requestOrder *= UINT64_C(1099511628211); + } + Require(trace.notifications == notices + test.notifications && trace.requests.size() == requests + test.requests && requestOrder == test.requestOrder, + "Nested legacy Pair failures must preserve every request, notification and allocation-order fingerprint"); + Require(pair.reserved == UINT64_C(0xa5a5a5a5a5a5a5a5) && pair.value.reserved == UINT32_C(0xa5a5a5a5) && !pair.value.auxiliary && std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), test.key) == 0, + "Nested legacy copies must preserve inactive bytes, independent metadata and partial key admission"); + std::string shape; std::vector values; + CollectionShape(pair.value, nullptr, shape, values); + Require(shape == test.shape, "Nested legacy copies must preserve partial counts, missing sentinels, both object keys, scalar/string contents and child parent identities"); + CheckMap(trace, {{&source, &context}, {&pair.value, &context}}); + const auto calls = context.calls; + Require(_ZNK3sce4Json5Value10getBooleanEv(&pair.value) == reinterpret_cast(&context.replacement.payload) && context.calls == calls + 1 && context.source == &pair.value && !context.parent, + "Even an incomplete nested Pair must retain its independently copied callback and borrowed result identity"); + for (const auto* value : values) { + if (value == &pair.value) continue; + Require(!value->parent && !value->auxiliary && !*_ZNK3sce4Json5Value10getBooleanEv(value) && context.calls == calls + 1, + "Unbound legacy descendants must use typed defaults without inheriting the copied root callback"); + } + if (alias) _ZN3sce4Json6Object4PairD2Ev(&pair); + else _ZN3sce4Json6Object4PairD1Ev(&pair); + CheckMap(trace, {{&source, &context}}); + Require(std::memcmp(&source, &sourceValue, sizeof(source)) == 0 && trace.heapGenerations.at(const_cast(sourceNode)) == sourceGeneration && sourceNode->callback == RecoverLegacy && sourceNode->context == &context, + "Nested Pair teardown must preserve the source Value and callback-record generation"); + for (const auto& snapshot : sourceAllocations) Require(trace.guestGenerations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, + "Nested Pair failure and cleanup must preserve all source allocation generations and bytes"); + Require(_ZNK3sce4Json5Value10getBooleanEv(&source) == reinterpret_cast(&context.replacement.payload) && context.source == &source && !context.parent, + "Nested source recovery must remain usable after copied storage is destroyed"); + _ZN3sce4Json6StringD1Ev(&key); _ZN3sce4Json5ValueD1Ev(&source); CheckMap(trace, {}); + Require(trace.guestGenerations == guestBaseline && trace.heapGenerations == heapBaseline, "Nested legacy copies must restore every original generation in both allocation domains"); + } +} int main() { LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; @@ -255,6 +439,7 @@ int main() { _ZN3sce4Json5ValueD1Ev(&null); _ZN3sce4Json5ValueD1Ev(&assigned); _ZN3sce4Json5ValueD1Ev(©); _ZN3sce4Json5ValueD1Ev(&source); CheckMap(trace, {}); CheckPairCopies(trace, first); + CheckCollectionCopies(trace, first); std::array roots{}; std::map entries; for (unsigned index = 0; index < roots.size(); ++index) { auto& root = roots[index * 13 % roots.size()]; ParseLegacy(root); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index bf43ed7bb1..402148cff9 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -8,7 +8,7 @@ - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Remaining kernel-status and callback/concurrent lifecycle paths, plus full guest execution, remain unverified. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. - [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter is now reproduced through the inspected float32 conversion, six truncating fractional digits, range saturation and NaN integer result. The regression covers signed zero, precision boundaries, large values, infinities and NaNs; floating-point rounding modes and exception flags remain unverified. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. -- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and nested older-SDK copies still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. +- [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and inherited older-SDK failure scopes still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - libc-heap failures and untested legacy callback combinations remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Additional legacy compatibility remains unverified. - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback-value failure behavior and compatibility below SDK 6 remain incomplete. @@ -80,4 +80,6 @@ - [RTTI initialization publication](../../../core/libs/prx/libSceJson2/Export.cpp) - concurrent reconfiguration and thrown allocator callbacks remain unverified. One hundred ninety-two additional SDK 5/6 reference cases cover legacy construction/newer RTTI setters, shared/independent contexts and configuration before initialization or inside its first allocation callback. All parameter objects share one adapter. An already-entered first allocation uses the original callback once; subsequent allocations and releases use the latest table while preserving each context and file-buffer setting. Tests cover handoff, lifecycle completion, orphaned epochs, SDK-specific mutex ownership and retry, distinguishing actual RTTI releases, SDK 5 null-free callbacks and later fixture reclamation. -- [Legacy Pair callback copies](../../../core/libs/prx/libSceJson2/Export.cpp) - nested collections, inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. Thirty-six SDK 5 reference-backed cases cover numeric and string sources with a registered mismatch handler, both constructor/destructor aliases and every guest allocation failure position through success using persistent and one-shot schedules. Even a null partial copy retains its independent callback map record. Checks cover exact request/notification order, partial key/text admission, inactive Pair bytes, borrowed callback result identity, independent auxiliary ownership, source allocation bytes/generations and source callback usability after copy cleanup. Every case restores the original live generations in both the guest allocator and libc heap. +- [Legacy Pair callback copies](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. Thirty-six SDK 5 reference-backed cases cover numeric and string sources with a registered mismatch handler, both constructor/destructor aliases and every guest allocation failure position through success using persistent and one-shot schedules. Even a null partial copy retains its independent callback map record. Checks cover exact request/notification order, partial key/text admission, inactive Pair bytes, borrowed callback result identity, independent auxiliary ownership, source allocation bytes/generations and source callback usability after copy cleanup. Every case restores the original live generations in both the guest allocator and libc heap. + +- [Legacy nested Pair copies](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. One hundred fifty-two SDK 5 reference-backed cases cover an array containing an object/string and an object containing an array/string, both Pair constructor/destructor aliases and persistent/one-shot failure at every allocation position through eighteen-allocation success. Checks cover partial counts, missing sentinels, independent internal/public object keys, exact request/notification fingerprints, inactive bytes, source bytes/generations and both allocator domains. Copies retain the root mismatch-map callback independently of auxiliary metadata; unbound descendants use typed defaults without inheriting that callback. Root and source recovery remain callable after copy cleanup. From 70fa6d9d1a76a842536f617a6ea21ccd3e3c0d57 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 10:52:20 +0200 Subject: [PATCH 067/318] test(libSceJson2): verify inherited legacy map failures --- core/libs/tests/GuestJsonLegacyHandlers.cpp | 93 ++++++++++++++++++++- docs/dev/debt/libSceJson2.md | 4 +- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index 308cafbcf7..4572b2dba9 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -17,8 +17,11 @@ struct String { void* storage; }; struct Pair { String key; std::uint64_t reserved; Value value; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; }; using Handler = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +using ReadCallback = int (APS5_VABI*)(char&, void*); using NullHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); extern "C" { +void* APS5_VABI _ZN3sce4Json6MallocEm(std::size_t); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(Value*, ReadCallback, void*); void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); @@ -60,6 +63,8 @@ struct LegacyTrace { std::map heapGenerations; std::size_t generation = 0; unsigned notifications = 0; + void (*onNotify)(void*) = nullptr; + void* noticeContext = nullptr; }; static LegacyTrace* HeapTrace = nullptr; extern "C" const void* ApplicationProcessParameters_nid_no_patch() { @@ -95,7 +100,9 @@ static void APS5_VABI FreeLegacy(void*, void* memory, void* context) { } static void APS5_VABI NotifyLegacy(void*, int error, std::size_t bytes, void* context) { Require(static_cast(error) == 0x80848102 && bytes == static_cast(context)->requests.back(), "Legacy allocation failure must identify its actual rejected request"); - ++static_cast(context)->notifications; + auto& trace = *static_cast(context); + ++trace.notifications; + if (const auto action = trace.onNotify) { trace.onNotify = nullptr; action(trace.noticeContext); } } static void* const LegacyVtable[]{nullptr, nullptr, reinterpret_cast(AllocateLegacy), reinterpret_cast(FreeLegacy), reinterpret_cast(NotifyLegacy)}; struct LegacyAllocator { void* const* vtable = LegacyVtable; }; @@ -128,7 +135,7 @@ static void CheckMap(const LegacyTrace& trace, const std::map(node)) && node->parent == parent && !node->nil && node->color <= 1, "Legacy nodes must retain their libc ownership and parent link"); Require(!node->color || (!node->left->color && !node->right->color), "Legacy red nodes must have black children"); const auto left = visit(node->left, node); - Require(expected != entries.end() && node->key == expected->first && node->callback == RecoverLegacy && node->context == expected->second, "Legacy map entries must preserve sorted keys and callback identity"); + Require(expected != entries.end() && node->key == expected->first && node->callback == (expected->second ? RecoverLegacy : nullptr) && node->context == expected->second, "Legacy map entries must preserve sorted keys and callback identity"); ++expected; const auto right = visit(node->right, node); Require(left == right, "Legacy tree paths must preserve black height"); @@ -385,6 +392,87 @@ static void CheckCollectionCopies(LegacyTrace& trace, Context& context) { Require(trace.guestGenerations == guestBaseline && trace.heapGenerations == heapBaseline, "Nested legacy copies must restore every original generation in both allocation domains"); } } +struct MapReadContext { + LegacyTrace* trace; + Value* target; + Value* copy; + Context* replacement; + unsigned mode; + unsigned reads = 0; + int registration = -1; +}; +static void UpdateInheritedMap(void* data) { + auto& context = *static_cast(data); + context.registration = _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(context.target, RecoverLegacy, context.replacement); + _ZN3sce4Json5ValueC1ERKS1_(context.copy, context.target); +} +static int APS5_VABI ReadInheritedMap(char& character, void* data) { + auto& context = *static_cast(data); + if (!context.reads) { + if (context.mode) { + if (context.mode == 2) { context.trace->onNotify = UpdateInheritedMap; context.trace->noticeContext = &context; } + context.trace->budget = 0; context.trace->once = true; + Require(!_ZN3sce4Json6MallocEm(8),"Inherited map testing must reject the actual guest allocation"); + context.trace->budget = -1; context.trace->once = false; + } + if (context.mode != 2) UpdateInheritedMap(&context); + } + if (context.reads >= 4) return static_cast(0x80848101); + character = "true"[context.reads++]; return 0; +} +static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& second) { + const auto guestBaseline = trace.guestGenerations, heapBaseline = trace.heapGenerations; + for (bool existing : {false, true}) for (unsigned mode = 0; mode < 3; ++mode) { + Value target{}, copy{}, outer{}; ParseLegacy(target); + if (existing) Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&target, RecoverLegacy, &first) == 0, "Prepare an existing legacy mismatch record before inherited failure"); + std::vector targetAllocations; + for (const auto& [pointer, generation] : trace.guestGenerations) { + if (guestBaseline.contains(pointer)) continue; + const auto* bytes = static_cast(pointer); + targetAllocations.push_back({pointer, generation, {bytes, bytes + trace.guest.at(pointer)}}); + } + const Value targetValue = target; + _ZN3sce4Json5ValueC1Ev(&outer); + MapReadContext context{&trace, &target, ©, &second, mode}; + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + const int status = _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(&outer, ReadInheritedMap, &context); + const auto error = static_cast(0x80848102); + Require(status == (mode ? error : 0) && context.registration == (mode == 1 ? error : 0) && context.reads == 4 && outer.type == 1 && (outer.payload & 0xff) == 1 && outer.auxiliary, + "Legacy map updates must observe failure only after notifier return without clearing the outer parser error or interrupting its admitted literal"); + const auto expected = mode ? std::vector{8, 24, 32} : std::vector{24, 32}; + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected && trace.notifications == notices + unsigned(mode != 0) && !trace.onNotify, + "Inherited map registration must preserve actual failure, root-admission and notifier ordering"); + Context* targetContext = mode == 1 ? existing ? &first : nullptr : &second; + Context* copyContext = mode == 1 ? nullptr : &second; + CheckMap(trace, {{&target, targetContext}, {©, copyContext}}); + Require(!copy.parent && !copy.auxiliary && copy.type == 3 && copy.payload == 9 && std::memcmp(&target, &targetValue, sizeof(target)) == 0, + "Inherited callback-map failure must not discard copied scalar contents or change the source ownership metadata"); + const auto verifyRecovery = [&](const Value& value, Context* expectedContext) { + const auto firstCalls = first.calls, secondCalls = second.calls; + const auto* result = _ZNK3sce4Json5Value10getBooleanEv(&value); + if (expectedContext) Require(result == reinterpret_cast(&expectedContext->replacement.payload) && expectedContext->source == &value && !expectedContext->parent && expectedContext->calls == (expectedContext == &first ? firstCalls : secondCalls) + 1, + "Admitted legacy records must retain their exact callback context, receiver and borrowed result"); + else Require(!*result && first.calls == firstCalls && second.calls == secondCalls, + "An empty map entry retained after inherited failure must use the typed default without callback dispatch"); + }; + verifyRecovery(target, targetContext); verifyRecovery(copy, copyContext); + const auto guest = trace.guestGenerations, heap = trace.heapGenerations; + const auto beforeRetry = trace.requests.size(); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&target, RecoverLegacy, &second) == 0 && _ZN3sce4Json5ValueaSERKS1_(©, &target) == ©, + "Independent registration and copy retry must succeed after the outer failure scope completes"); + Require(trace.guestGenerations == guest && trace.heapGenerations == heap && trace.requests.size() == beforeRetry, + "Retry must reuse both existing or empty legacy callback records without replacing allocation generations"); + CheckMap(trace, {{&target, &second}, {©, &second}}); + verifyRecovery(target, &second); verifyRecovery(copy, &second); + for (const auto& snapshot : targetAllocations) Require(trace.guestGenerations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, + "Inherited failure, registration retry and copying must preserve the original source auxiliary and fallback generations"); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&outer, "false", 5) == 0 && outer.type == 1 && !(outer.payload & 0xff), + "The next independent parse must not inherit an already completed callback-map failure"); + _ZN3sce4Json5ValueD1Ev(©); _ZN3sce4Json5ValueD1Ev(&target); _ZN3sce4Json5ValueD1Ev(&outer); CheckMap(trace, {}); + Require(trace.guestGenerations == guestBaseline && trace.heapGenerations == heapBaseline, + "Inherited map updates and retries must reclaim every temporary generation in both allocator domains"); + } +} int main() { LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; @@ -440,6 +528,7 @@ int main() { CheckMap(trace, {}); CheckPairCopies(trace, first); CheckCollectionCopies(trace, first); + CheckInheritedMap(trace, first, second); std::array roots{}; std::map entries; for (unsigned index = 0; index < roots.size(); ++index) { auto& root = roots[index * 13 % roots.size()]; ParseLegacy(root); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 402148cff9..4977a0eb09 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -48,7 +48,7 @@ - [sce::Json::Parser object allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - object keys use one allocator-backed string reused between members, following `0x6250`; lookup closes its shared operation scope before assigning the exposed key and owner, following `0xda40`. Sixty-six reference-backed cases cover persistent and one-shot failure at each returning allocation position through a 33-allocation successful parse of an object containing a string and array. Checks include partial keys/members, numeric and string fallback values, parent links, error status, request/notification counts and complete allocator cleanup. The nested array sentinel failure is excluded because reference execution crashes; more complex nested objects, callback-driven allocation failures and concurrency remain unverified. -- [sce::Json::Collection inherited allocation errors](../../../core/libs/prx/libSceJson2/Export.cpp) - array assignment, object assignment/insertion and legacy mismatch-map updates now use the shared depth/error protocol instead of counter snapshots, following `0xbc80`, `0xd400`, `0xcd30` and `0x3b00`. Six reference-backed public parser/read-callback cases invoke array assignment, object assignment or insertion after a one-shot character allocation failure, plus healthy controls. Checks cover placeholder retention, skipped payload binding, unchanged source, outer error persistence, recovery on the next independent parse and complete cleanup. Baseline and targeted assignment/insertion mutations fail these checks. Nested legacy map updates, callback-return cloning and cross-thread failure propagation remain unverified. +- [sce::Json::Collection inherited allocation errors](../../../core/libs/prx/libSceJson2/Export.cpp) - array assignment, object assignment/insertion and legacy mismatch-map updates now use the shared depth/error protocol instead of counter snapshots, following `0xbc80`, `0xd400`, `0xcd30` and `0x3b00`. Six reference-backed public parser/read-callback cases invoke array assignment, object assignment or insertion after a one-shot character allocation failure, plus healthy controls. Checks cover placeholder retention, skipped payload binding, unchanged source, outer error persistence, recovery on the next independent parse and complete cleanup. Baseline and targeted assignment/insertion mutations fail these checks. Callback-return cloning and cross-thread failure propagation remain unverified; legacy map updates under inherited failure are covered separately below. ### Unknown 3.20 import declarations @@ -83,3 +83,5 @@ - [Legacy Pair callback copies](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. Thirty-six SDK 5 reference-backed cases cover numeric and string sources with a registered mismatch handler, both constructor/destructor aliases and every guest allocation failure position through success using persistent and one-shot schedules. Even a null partial copy retains its independent callback map record. Checks cover exact request/notification order, partial key/text admission, inactive Pair bytes, borrowed callback result identity, independent auxiliary ownership, source allocation bytes/generations and source callback usability after copy cleanup. Every case restores the original live generations in both the guest allocator and libc heap. - [Legacy nested Pair copies](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. One hundred fifty-two SDK 5 reference-backed cases cover an array containing an object/string and an object containing an array/string, both Pair constructor/destructor aliases and persistent/one-shot failure at every allocation position through eighteen-allocation success. Checks cover partial counts, missing sentinels, independent internal/public object keys, exact request/notification fingerprints, inactive bytes, source bytes/generations and both allocator domains. Copies retain the root mismatch-map callback independently of auxiliary metadata; unbound descendants use typed defaults without inheriting that callback. Root and source recovery remain callable after copy cleanup. + +- [Legacy callback map inherited failures](../../../core/libs/prx/libSceJson2/Export.cpp) - Pair/collection copies inside an inherited failure scope, allocator-driven lifecycle changes and cross-thread propagation remain unverified. Six SDK 5 reference-backed schedules exercise new/existing records through a real parser reader, with healthy controls, a published allocation error and registration inside the notifier before error publication. Published failure preserves an existing handler or retains a new empty record; scalar copying still completes but its callback record remains empty. Notifier-time registration and copying succeed before the shared error is published, while the outer parser completes its admitted literal and reports OOM. Checks cover actual allocation/notification order, callback ABI and borrowed results, exact auxiliary/fallback generations, record-reusing registration/copy retry, independent parser retry and complete cleanup in both allocator domains. From acafffb8979ba8ca1780c99d1d70f07e46e72fd5 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 11:02:24 +0200 Subject: [PATCH 068/318] fix(libSceJson2): preserve auxiliary state after callback termination --- core/libs/prx/libSceJson2/CMakeLists.txt | 5 + core/libs/prx/libSceJson2/Export.cpp | 5 + .../tests/GuestJsonCompositeDestruction.cpp | 157 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 core/libs/tests/GuestJsonCompositeDestruction.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index b65ce66edc..27eb9b800a 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,11 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_composite_destruction_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCompositeDestruction.cpp) + target_include_directories(guest_json_composite_destruction_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_composite_destruction_tests PRIVATE libSceJson2 libc) + configure_windows_unwind(guest_json_composite_destruction_tests) + add_test(NAME guest_json_composite_destruction COMMAND guest_json_composite_destruction_tests) add_test_executable(guest_json_legacy_initialization_publication_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationPublication.cpp Export.cpp) target_compile_definitions(guest_json_legacy_initialization_publication_tests PRIVATE JSON2_PUBLICATION_LEGACY_SDK) target_include_directories(guest_json_legacy_initialization_publication_tests PRIVATE ${LIBS_INCLUDE_DIR}) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 4fb08eb1d9..2bff5fc8a3 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1181,6 +1181,11 @@ void ReleaseValueAuxiliary(Value& v) { void Destroy(Value& v) { Clear(v); + { + auto& state = State(); + std::lock_guard lock(state.mutex); + if (!state.recoveryMutex) return; + } ReleaseValueAuxiliary(v); if (!ModernValues()) EraseLegacyHandler(v); } diff --git a/core/libs/tests/GuestJsonCompositeDestruction.cpp b/core/libs/tests/GuestJsonCompositeDestruction.cpp new file mode 100644 index 0000000000..39f1ee7e20 --- /dev/null +++ b/core/libs/tests/GuestJsonCompositeDestruction.cpp @@ -0,0 +1,157 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; +struct String { void* storage; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +using NullHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using MismatchHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); +using FloatHandler = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); +using ElementHandler = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD2Ev(Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullHandler, void*); +int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, MismatchHandler, void*); +int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatHandler, void*); +int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, ElementHandler, void*); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct DestructionTrace { + std::map live; + std::map module; + std::map source; + std::size_t generation = 0; + std::size_t sourceBase = 0; + void* receiver = nullptr; + std::uint8_t* owner = nullptr; + unsigned target = 0; + unsigned calls = 0; + unsigned terminations = 0; + bool observing = false; + std::vector releases; + std::vector moduleReleases; +}; +static void* APS5_VABI AllocateDestruction(void* receiver, std::size_t bytes, void* context) { + auto& trace = *static_cast(context); + Require(receiver == trace.receiver, "Composite allocations must retain their actual allocator receiver and context"); + void* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); + std::memset(memory, 0xa5, bytes); + Require(trace.live.emplace(memory, ++trace.generation).second, "Composite resources must own distinct live allocation generations"); + return memory; +} +static void ReleaseStorage(DestructionTrace& trace, void* memory) { + Require(trace.live.erase(memory) == 1, "Every composite resource must be reclaimed exactly once by its owning domain"); + std::free(memory); +} +static void APS5_VABI FreeDestruction(void* receiver, void* memory, void* context) { + auto& trace = *static_cast(context); + Require(receiver == trace.receiver, "Composite cleanup must preserve allocator receiver and context even through module termination"); + if (trace.observing) { + ++trace.calls; + Require(memory != nullptr, "Successful composite and module teardown must release their measured nonnull resources"); + const auto identity = trace.live.at(memory); + if (trace.module.contains(memory)) { + Require(trace.module.at(memory) == identity, "Nested module termination must retain its original allocation generations"); + trace.moduleReleases.push_back(identity); + } else { + Require(trace.source.contains(memory) && trace.source.at(memory) == identity, "Composite release callbacks must identify their exact original payload generation"); + trace.releases.push_back(identity - trace.sourceBase); + if (trace.target && trace.releases.size() == trace.target) { + trace.target = 0; + auto expected = trace.live; + for (const auto& [pointer, generation] : trace.module) Require(expected.erase(pointer) == 1, "Callback termination must begin with every original module resource still live"); + Require(_ZN3sce4Json11Initializer9terminateEv(trace.owner) == 0 && *trace.owner == 0 && trace.live == expected, + "Composite free callbacks must retain reference-supported module termination while preserving all remaining source allocations including the current release"); + ++trace.terminations; + } + } + } + if (memory) ReleaseStorage(trace, memory); +} +static void APS5_VABI NotifyDestruction(void*, int, std::size_t, void*) { Require(false, "Healthy composite destruction must not fail allocation"); } +static void* const DestructionVtable[]{nullptr, nullptr, reinterpret_cast(AllocateDestruction), reinterpret_cast(FreeDestruction), reinterpret_cast(NotifyDestruction)}; +struct DestructionAllocator { void* const* vtable = DestructionVtable; }; +static const std::vector ReleaseOrders[]{ + {5, 4, 3}, + {31, 30, 29, 28, 27, 26, 25, 24, 18, 17, 16, 15, 14, 13, 12, 3, 2, 1, 5, 4}, + {9, 8, 7, 6, 26, 25, 24, 23, 22, 21, 20, 13, 12, 11, 10, 3, 2, 1, 5, 4}, + {5, 4, 3, 8, 9, 10, 7, 6}, +}; +static void DestroyAlias(Value& value, unsigned alias) { + if (alias) _ZN3sce4Json5ValueD2Ev(&value); + else _ZN3sce4Json5ValueD1Ev(&value); +} +static void Check(unsigned shape, unsigned target, unsigned alias) { + DestructionTrace trace; DestructionAllocator allocator; trace.receiver = &allocator; + Parameter parameter{&allocator, &trace, 8192, 2, {}}; + std::uint8_t initializer = 0; trace.owner = &initializer; _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == 12, "Initialize the actual composite-destruction resource epoch"); + trace.module = trace.live; trace.sourceBase = trace.generation; + Value value; std::memset(&value, 0xa5, sizeof(value)); + if (shape == 0 || shape == 3) { + _ZN3sce4Json5ValueC1EPKc(&value, "text"); + if (shape == 3) Require(_ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&value, nullptr, nullptr) == 0 && + _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&value, nullptr, nullptr) == 0 && + _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, nullptr, nullptr) == 0 && + _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&value, nullptr, nullptr) == 0, "Admit all modern auxiliary callback records through their real allocation paths"); + } else { + _ZN3sce4Json5ValueC1Ev(&value); + const char* document = shape == 1 ? R"({"s":"text","a":[1]})" : R"(["text",{"s":"more"}])"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, document, std::strlen(document)) == 0, "Construct actual nested object/array ownership before callback destruction"); + } + for (const auto& [pointer, generation] : trace.live) if (!trace.module.contains(pointer)) trace.source.emplace(pointer, generation); + const auto& order = ReleaseOrders[shape]; + Require(trace.source.size() == order.size(), "Each composite payload must own its complete measured source resource epoch"); + void* auxiliary = value.auxiliary; + std::array auxiliaryBytes{}; + if (auxiliary) std::memcpy(auxiliaryBytes.data(), auxiliary, auxiliaryBytes.size()); + trace.target = target; trace.observing = true; + DestroyAlias(value, alias); + const unsigned released = target ? target : static_cast(order.size()); + Require(trace.releases == std::vector(order.begin(), order.begin() + released), "Composite destruction must preserve the exact native release-generation prefix before callback termination"); + auto remaining = trace.source; + for (unsigned index = 0; index < released; ++index) { + bool found = false; + for (auto iterator = remaining.begin(); iterator != remaining.end(); ++iterator) if (iterator->second == trace.sourceBase + order[index]) { remaining.erase(iterator); found = true; break; } + Require(found, "Every observed release must remove exactly its expected original payload generation"); + } + const bool retainedAuxiliary = target && ((shape == 1 || shape == 2) ? target <= 18 : shape == 3 && target <= 3); + Require(value.type == 0 && !value.parent && !value.payload && value.reserved == UINT32_C(0xa5a5a5a5) && value.auxiliary == (retainedAuxiliary ? auxiliary : nullptr), + "Destruction must skip auxiliary cleanup after payload callbacks terminate the module, retaining its original pointer rather than falsely clearing ownership"); + if (retainedAuxiliary) Require(std::memcmp(auxiliary, auxiliaryBytes.data(), auxiliaryBytes.size()) == 0, "Skipped auxiliary cleanup must preserve the complete original auxiliary bytes and callback/fallback identities"); + Require(trace.terminations == unsigned(target != 0) && !trace.target && trace.calls == released + (target ? 12u : 0u) && trace.moduleReleases.size() == (target ? 12u : 0u) && initializer == unsigned(!target) && trace.live == (target ? remaining : trace.module), + "Callback termination must reclaim only the module epoch and released source prefix while preserving the reference's orphaned allocations"); + const auto beforeRepeat = trace.live; const auto calls = trace.calls; + DestroyAlias(value, 1 - alias); + Require(trace.live == beforeRepeat && trace.calls == calls && value.auxiliary == (retainedAuxiliary ? auxiliary : nullptr), "A repeated destructor must preserve inactive-module auxiliary state without attempting another release"); + trace.observing = false; + if (!target) Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Ordinary composite destruction must remain fully reclaimable"); + else { + Require(trace.live == remaining, "Fixture reclamation may begin only after the exact orphaned source epoch has been observed"); + for (const auto& [pointer, generation] : remaining) { Require(trace.live.at(pointer) == generation, "Caller-owned fixture cleanup must retain each orphan allocation generation"); ReleaseStorage(trace, pointer); } + } + const auto generation = trace.generation; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == 12, "Initialization must admit a fresh module after every measured composite-destruction outcome"); + for (const auto& [pointer, identity] : trace.live) Require(pointer && identity > generation, "Composite lifecycle retry must retain no stale module or source resource generations"); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release the fresh composite lifecycle generation"); +} +int main() { + for (unsigned shape = 0; shape < 4; ++shape) for (unsigned target = 0; target <= ReleaseOrders[shape].size(); ++target) for (unsigned alias = 0; alias < 2; ++alias) Check(shape, target, alias); + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 4977a0eb09..075611d964 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -72,7 +72,7 @@ - [Modern callback-bearing copies](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation-failure combinations, nested owners, lifecycle changes from callbacks and alternate SDK behavior remain unverified. Fifteen reference-backed cases exercise null, real and object sources with all four local handlers through Value construction, assignment with/without destination handlers, Pair construction and standalone Array insertion. Modern copies discard source-local handlers; assignment retains destination handlers. Tests invoke null/mismatch/element/float recovery and verify complete auxiliary bytes, fallback ownership, all callback-record generations, borrowed replacement identity and source-handler usability after copy destruction. -- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - composite-destruction callbacks, teardown exceptions and unmeasured publication variants, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and allocate/notify/free callback termination inside an active allocation operation are explicitly unsupported before teardown. Reference recovery/notifier probes reused freed/null mutex slots after termination and faulted at the substituted kernel boundary; direct allocate/free probes also faulted after termination returned success. These are not console measurements. Reference-backed controls preserve inactive-initializer termination, standalone Malloc-notifier termination and standalone Free-callback termination/reinitialization. Production tests verify ownership-preserving refusal, repeatable recovery, successful parse continuation, OOM/retry and fresh module generations. +- [Callback-time termination](../../../core/libs/prx/libSceJson2/Export.cpp) - composite-destruction reinitialization, teardown exceptions and unmeasured publication variants, concurrent lifecycle, older SDKs and guest-kernel error responses remain unverified. Recovery-callback owner termination and allocate/notify/free callback termination inside an active allocation operation are explicitly unsupported before teardown. Reference recovery/notifier probes reused freed/null mutex slots after termination and faulted at the substituted kernel boundary; direct allocate/free probes also faulted after termination returned success. These are not console measurements. Reference-backed controls preserve inactive-initializer termination, standalone Malloc-notifier termination and standalone Free-callback termination/reinitialization. Production tests verify ownership-preserving refusal, repeatable recovery, successful parse continuation, OOM/retry and fresh module generations. - [Termination reentry](../../../core/libs/prx/libSceJson2/Export.cpp) - callback exceptions during partially completed teardown and older-SDK combinations remain unverified. Nested owner termination while cleanup is already active is explicitly unsupported before further ownership changes. The reference allocator observed a duplicate free request when a teardown free callback recursively terminated the module; a standalone Free callback retained its target until its original release completed. Twelve production cases exercise reentry at every normal module release position, require one refusal and exactly twelve releases, and verify that the outer call completes and the next module generation initializes and terminates normally. @@ -85,3 +85,5 @@ - [Legacy nested Pair copies](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. One hundred fifty-two SDK 5 reference-backed cases cover an array containing an object/string and an object containing an array/string, both Pair constructor/destructor aliases and persistent/one-shot failure at every allocation position through eighteen-allocation success. Checks cover partial counts, missing sentinels, independent internal/public object keys, exact request/notification fingerprints, inactive bytes, source bytes/generations and both allocator domains. Copies retain the root mismatch-map callback independently of auxiliary metadata; unbound descendants use typed defaults without inheriting that callback. Root and source recovery remain callable after copy cleanup. - [Legacy callback map inherited failures](../../../core/libs/prx/libSceJson2/Export.cpp) - Pair/collection copies inside an inherited failure scope, allocator-driven lifecycle changes and cross-thread propagation remain unverified. Six SDK 5 reference-backed schedules exercise new/existing records through a real parser reader, with healthy controls, a published allocation error and registration inside the notifier before error publication. Published failure preserves an existing handler or retains a new empty record; scalar copying still completes but its callback record remains empty. Notifier-time registration and copying succeed before the shared error is published, while the outer parser completes its admitted literal and reports OOM. Checks cover actual allocation/notification order, callback ABI and borrowed results, exact auxiliary/fallback generations, record-reusing registration/copy retry, independent parser retry and complete cleanup in both allocator domains. + +- [Composite-destruction termination](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-time reinitialization, alternate SDKs and thrown callbacks remain unverified. One hundred ten SDK 6 reference cases cover both Value destructor aliases, four payload/metadata shapes and every release position. Following `0x17e0`, destruction requires the published recovery mutex after clearing the payload; callback termination during payload release therefore preserves the auxiliary pointer and bytes. Checks cover exact release-generation prefixes, allocator receiver/context, module/source ownership, repeated destruction and fresh initialization. Early termination orphans the remaining caller allocations; final-release termination completes cleanup. The caller-owned fixture separately reclaims the observed orphaned epoch. The pre-fix destructor fails the auxiliary-retention regression. From defefd7905c0264c65e6a9332fafb6e3191b3121 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 11:10:05 +0200 Subject: [PATCH 069/318] fix(libSceJson2): reject termination during legacy auxiliary cleanup --- core/libs/prx/libSceJson2/CMakeLists.txt | 10 +++ core/libs/prx/libSceJson2/Export.cpp | 15 ++++- .../tests/GuestJsonCompositeDestruction.cpp | 61 ++++++++++++++----- docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 72 insertions(+), 18 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 27eb9b800a..7301737abd 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,16 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_legacy_composite_destruction_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCompositeDestruction.cpp Export.cpp) + target_compile_definitions(guest_json_legacy_composite_destruction_tests PRIVATE JSON2_DESTRUCTION_LEGACY_SDK) + target_include_directories(guest_json_legacy_composite_destruction_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_legacy_composite_destruction_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_legacy_composite_destruction_tests) + add_test(NAME guest_json_legacy_composite_destruction COMMAND guest_json_legacy_composite_destruction_tests) + if(WIN32) + set_tests_properties(guest_json_legacy_composite_destruction PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() add_test_executable(guest_json_composite_destruction_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCompositeDestruction.cpp) target_include_directories(guest_json_composite_destruction_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_composite_destruction_tests PRIVATE libSceJson2 libc) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 2bff5fc8a3..fa01ed1e72 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -168,6 +168,7 @@ Globals& State() { } thread_local std::uint32_t allocatorCallbackDepth = 0; +thread_local std::uint32_t legacyCleanupDepth = 0; struct AllocatorCallbackScope { AllocatorCallbackScope() { ++allocatorCallbackDepth; } @@ -176,6 +177,14 @@ struct AllocatorCallbackScope { AllocatorCallbackScope& operator=(const AllocatorCallbackScope&) = delete; }; +struct LegacyCleanupScope { + bool active; + explicit LegacyCleanupScope(bool legacy) : active(legacy) { if (active) ++legacyCleanupDepth; } + ~LegacyCleanupScope() { if (active) --legacyCleanupDepth; } + LegacyCleanupScope(const LegacyCleanupScope&) = delete; + LegacyCleanupScope& operator=(const LegacyCleanupScope&) = delete; +}; + struct TerminationScope { bool& active; explicit TerminationScope(bool& value) : active(value) { active = true; } @@ -1186,8 +1195,10 @@ void Destroy(Value& v) { std::lock_guard lock(state.mutex); if (!state.recoveryMutex) return; } + const bool legacy = !ModernValues(); + LegacyCleanupScope cleanup(legacy); ReleaseValueAuxiliary(v); - if (!ModernValues()) EraseLegacyHandler(v); + if (legacy) EraseLegacyHandler(v); } int EnsureAuxiliary(Node& n) { @@ -2566,7 +2577,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; - if (state.terminating || state.callbackActive || (allocatorCallbackDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); + if (state.terminating || state.callbackActive || (allocatorCallbackDepth && (state.allocationDepth || legacyCleanupDepth))) NotImplemented_nid_no_patch(__func__); TerminationScope termination(state.terminating); state.callback = nullptr; state.context = nullptr; diff --git a/core/libs/tests/GuestJsonCompositeDestruction.cpp b/core/libs/tests/GuestJsonCompositeDestruction.cpp index 39f1ee7e20..ac0ccb4392 100644 --- a/core/libs/tests/GuestJsonCompositeDestruction.cpp +++ b/core/libs/tests/GuestJsonCompositeDestruction.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; @@ -29,6 +30,22 @@ int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEP int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, FloatHandler, void*); int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, ElementHandler, void*); } +#if defined(JSON2_DESTRUCTION_LEGACY_SDK) +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + static const std::uint32_t words[]{0, 0, 0, 0, 0x05000000}; + return words; +} +static constexpr bool LegacySdk = true; +static constexpr unsigned ModuleResources = 11; +static constexpr unsigned AuxiliaryBytes = 24; +static constexpr unsigned ShapeCount = 3; +#else +static constexpr bool LegacySdk = false; +static constexpr unsigned ModuleResources = 12; +static constexpr unsigned AuxiliaryBytes = 72; +static constexpr unsigned ShapeCount = 4; +#endif +static constexpr unsigned ModuleFreeCalls = 12; static void Require(bool condition, const char* message) { if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } } @@ -43,6 +60,9 @@ struct DestructionTrace { unsigned target = 0; unsigned calls = 0; unsigned terminations = 0; + unsigned refusals = 0; + unsigned nullCalls = 0; + bool rejectTermination = false; bool observing = false; std::vector releases; std::vector moduleReleases; @@ -64,7 +84,7 @@ static void APS5_VABI FreeDestruction(void* receiver, void* memory, void* contex Require(receiver == trace.receiver, "Composite cleanup must preserve allocator receiver and context even through module termination"); if (trace.observing) { ++trace.calls; - Require(memory != nullptr, "Successful composite and module teardown must release their measured nonnull resources"); + if (!memory) { Require(LegacySdk, "Only the measured legacy module cleanup may forward a null free"); ++trace.nullCalls; return; } const auto identity = trace.live.at(memory); if (trace.module.contains(memory)) { Require(trace.module.at(memory) == identity, "Nested module termination must retain its original allocation generations"); @@ -74,11 +94,20 @@ static void APS5_VABI FreeDestruction(void* receiver, void* memory, void* contex trace.releases.push_back(identity - trace.sourceBase); if (trace.target && trace.releases.size() == trace.target) { trace.target = 0; - auto expected = trace.live; - for (const auto& [pointer, generation] : trace.module) Require(expected.erase(pointer) == 1, "Callback termination must begin with every original module resource still live"); - Require(_ZN3sce4Json11Initializer9terminateEv(trace.owner) == 0 && *trace.owner == 0 && trace.live == expected, - "Composite free callbacks must retain reference-supported module termination while preserving all remaining source allocations including the current release"); - ++trace.terminations; + if (trace.rejectTermination) { + const auto before = trace.live; + bool refused = false; + try { _ZN3sce4Json11Initializer9terminateEv(trace.owner); } + catch (const std::runtime_error& error) { Require(std::strstr(error.what(), "terminate"), "Unsafe legacy cleanup termination must use the explicit unsupported boundary"); refused = true; } + Require(refused && *trace.owner == 1 && trace.live == before, "Legacy auxiliary cleanup must reject callback termination before changing module ownership"); + ++trace.refusals; + } else { + auto expected = trace.live; + for (const auto& [pointer, generation] : trace.module) Require(expected.erase(pointer) == 1, "Callback termination must begin with every original module resource still live"); + Require(_ZN3sce4Json11Initializer9terminateEv(trace.owner) == 0 && *trace.owner == 0 && trace.live == expected, + "Composite free callbacks must retain reference-supported module termination while preserving all remaining source allocations including the current release"); + ++trace.terminations; + } } } } @@ -101,7 +130,7 @@ static void Check(unsigned shape, unsigned target, unsigned alias) { DestructionTrace trace; DestructionAllocator allocator; trace.receiver = &allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; std::uint8_t initializer = 0; trace.owner = &initializer; _ZN3sce4Json11InitializerC1Ev(&initializer); - Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == 12, "Initialize the actual composite-destruction resource epoch"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == ModuleResources, "Initialize the actual composite-destruction resource epoch"); trace.module = trace.live; trace.sourceBase = trace.generation; Value value; std::memset(&value, 0xa5, sizeof(value)); if (shape == 0 || shape == 3) { @@ -120,10 +149,12 @@ static void Check(unsigned shape, unsigned target, unsigned alias) { Require(trace.source.size() == order.size(), "Each composite payload must own its complete measured source resource epoch"); void* auxiliary = value.auxiliary; std::array auxiliaryBytes{}; - if (auxiliary) std::memcpy(auxiliaryBytes.data(), auxiliary, auxiliaryBytes.size()); + if (auxiliary) std::memcpy(auxiliaryBytes.data(), auxiliary, AuxiliaryBytes); + trace.rejectTermination = LegacySdk && shape != 0 && target > 18; + const bool terminated = target && !trace.rejectTermination; trace.target = target; trace.observing = true; DestroyAlias(value, alias); - const unsigned released = target ? target : static_cast(order.size()); + const unsigned released = terminated ? target : static_cast(order.size()); Require(trace.releases == std::vector(order.begin(), order.begin() + released), "Composite destruction must preserve the exact native release-generation prefix before callback termination"); auto remaining = trace.source; for (unsigned index = 0; index < released; ++index) { @@ -131,27 +162,27 @@ static void Check(unsigned shape, unsigned target, unsigned alias) { for (auto iterator = remaining.begin(); iterator != remaining.end(); ++iterator) if (iterator->second == trace.sourceBase + order[index]) { remaining.erase(iterator); found = true; break; } Require(found, "Every observed release must remove exactly its expected original payload generation"); } - const bool retainedAuxiliary = target && ((shape == 1 || shape == 2) ? target <= 18 : shape == 3 && target <= 3); + const bool retainedAuxiliary = terminated && ((shape == 1 || shape == 2) ? target <= 18 : shape == 3 && target <= 3); Require(value.type == 0 && !value.parent && !value.payload && value.reserved == UINT32_C(0xa5a5a5a5) && value.auxiliary == (retainedAuxiliary ? auxiliary : nullptr), "Destruction must skip auxiliary cleanup after payload callbacks terminate the module, retaining its original pointer rather than falsely clearing ownership"); - if (retainedAuxiliary) Require(std::memcmp(auxiliary, auxiliaryBytes.data(), auxiliaryBytes.size()) == 0, "Skipped auxiliary cleanup must preserve the complete original auxiliary bytes and callback/fallback identities"); - Require(trace.terminations == unsigned(target != 0) && !trace.target && trace.calls == released + (target ? 12u : 0u) && trace.moduleReleases.size() == (target ? 12u : 0u) && initializer == unsigned(!target) && trace.live == (target ? remaining : trace.module), + if (retainedAuxiliary) Require(std::memcmp(auxiliary, auxiliaryBytes.data(), AuxiliaryBytes) == 0, "Skipped auxiliary cleanup must preserve the complete original auxiliary bytes and callback/fallback identities"); + Require(trace.terminations == unsigned(terminated) && trace.refusals == unsigned(trace.rejectTermination) && !trace.target && trace.calls == released + (terminated ? ModuleFreeCalls : 0u) && trace.moduleReleases.size() == (terminated ? ModuleResources : 0u) && trace.nullCalls == (terminated ? ModuleFreeCalls - ModuleResources : 0u) && initializer == unsigned(!terminated) && trace.live == (terminated ? remaining : trace.module), "Callback termination must reclaim only the module epoch and released source prefix while preserving the reference's orphaned allocations"); const auto beforeRepeat = trace.live; const auto calls = trace.calls; DestroyAlias(value, 1 - alias); Require(trace.live == beforeRepeat && trace.calls == calls && value.auxiliary == (retainedAuxiliary ? auxiliary : nullptr), "A repeated destructor must preserve inactive-module auxiliary state without attempting another release"); trace.observing = false; - if (!target) Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Ordinary composite destruction must remain fully reclaimable"); + if (!terminated) Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Ordinary composite destruction must remain fully reclaimable"); else { Require(trace.live == remaining, "Fixture reclamation may begin only after the exact orphaned source epoch has been observed"); for (const auto& [pointer, generation] : remaining) { Require(trace.live.at(pointer) == generation, "Caller-owned fixture cleanup must retain each orphan allocation generation"); ReleaseStorage(trace, pointer); } } const auto generation = trace.generation; - Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == 12, "Initialization must admit a fresh module after every measured composite-destruction outcome"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == ModuleResources, "Initialization must admit a fresh module after every measured composite-destruction outcome"); for (const auto& [pointer, identity] : trace.live) Require(pointer && identity > generation, "Composite lifecycle retry must retain no stale module or source resource generations"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release the fresh composite lifecycle generation"); } int main() { - for (unsigned shape = 0; shape < 4; ++shape) for (unsigned target = 0; target <= ReleaseOrders[shape].size(); ++target) for (unsigned alias = 0; alias < 2; ++alias) Check(shape, target, alias); + for (unsigned shape = 0; shape < ShapeCount; ++shape) for (unsigned target = 0; target <= ReleaseOrders[shape].size(); ++target) for (unsigned alias = 0; alias < 2; ++alias) Check(shape, target, alias); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 075611d964..5bb4b43574 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -86,4 +86,6 @@ - [Legacy callback map inherited failures](../../../core/libs/prx/libSceJson2/Export.cpp) - Pair/collection copies inside an inherited failure scope, allocator-driven lifecycle changes and cross-thread propagation remain unverified. Six SDK 5 reference-backed schedules exercise new/existing records through a real parser reader, with healthy controls, a published allocation error and registration inside the notifier before error publication. Published failure preserves an existing handler or retains a new empty record; scalar copying still completes but its callback record remains empty. Notifier-time registration and copying succeed before the shared error is published, while the outer parser completes its admitted literal and reports OOM. Checks cover actual allocation/notification order, callback ABI and borrowed results, exact auxiliary/fallback generations, record-reusing registration/copy retry, independent parser retry and complete cleanup in both allocator domains. -- [Composite-destruction termination](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-time reinitialization, alternate SDKs and thrown callbacks remain unverified. One hundred ten SDK 6 reference cases cover both Value destructor aliases, four payload/metadata shapes and every release position. Following `0x17e0`, destruction requires the published recovery mutex after clearing the payload; callback termination during payload release therefore preserves the auxiliary pointer and bytes. Checks cover exact release-generation prefixes, allocator receiver/context, module/source ownership, repeated destruction and fresh initialization. Early termination orphans the remaining caller allocations; final-release termination completes cleanup. The caller-owned fixture separately reclaims the observed orphaned epoch. The pre-fix destructor fails the auxiliary-retention regression. +- [Composite-destruction termination](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-time reinitialization, additional legacy callback combinations and thrown callbacks remain unverified. One hundred ten SDK 6 reference cases cover both Value destructor aliases, four payload/metadata shapes and every release position. Following `0x17e0`, destruction requires the published recovery mutex after clearing the payload; callback termination during payload release therefore preserves the auxiliary pointer and bytes. Checks cover exact release-generation prefixes, allocator receiver/context, module/source ownership, repeated destruction and fresh initialization. Early termination orphans the remaining caller allocations; final-release termination completes cleanup. The caller-owned fixture separately reclaims the observed orphaned epoch. The pre-fix destructor fails the auxiliary-retention regression. + +- [Legacy auxiliary-cleanup termination](../../../core/libs/prx/libSceJson2/Export.cpp) - owner termination from allocator callbacks inside legacy auxiliary/map cleanup is explicitly unsupported before teardown. Eight reference probes across both Value destructor aliases returned termination success, then faulted through a null mutex slot at the substituted kernel boundary; these are not console measurements. Ninety-two SDK 5 production cases verify the refusal and 84 returning reference cases preserve ordinary destruction and early payload-release termination for strings and nested arrays/objects. Checks cover eleven module resources plus the observed null free, exact release prefixes, retained auxiliary bytes, allocator identities, repeated destruction and fresh initialization. The cleanup guard ends before later operations; modern metadata-release termination remains supported. Callback-time reinitialization and thrown callbacks remain unverified. From 0162e34ec37e705188b9bb69eeaf0b73f57f447c Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 15:27:18 +0200 Subject: [PATCH 070/318] fix(libSceJson2): allow callback-time reinitialization during cleanup --- core/libs/prx/libSceJson2/Export.cpp | 16 +--- .../tests/GuestJsonCompositeDestruction.cpp | 93 ++++++++++++------- docs/dev/debt/libSceJson2.md | 8 +- 3 files changed, 67 insertions(+), 50 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index fa01ed1e72..b3daff44f1 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -168,7 +168,6 @@ Globals& State() { } thread_local std::uint32_t allocatorCallbackDepth = 0; -thread_local std::uint32_t legacyCleanupDepth = 0; struct AllocatorCallbackScope { AllocatorCallbackScope() { ++allocatorCallbackDepth; } @@ -177,14 +176,6 @@ struct AllocatorCallbackScope { AllocatorCallbackScope& operator=(const AllocatorCallbackScope&) = delete; }; -struct LegacyCleanupScope { - bool active; - explicit LegacyCleanupScope(bool legacy) : active(legacy) { if (active) ++legacyCleanupDepth; } - ~LegacyCleanupScope() { if (active) --legacyCleanupDepth; } - LegacyCleanupScope(const LegacyCleanupScope&) = delete; - LegacyCleanupScope& operator=(const LegacyCleanupScope&) = delete; -}; - struct TerminationScope { bool& active; explicit TerminationScope(bool& value) : active(value) { active = true; } @@ -1000,6 +991,7 @@ int RegisterLegacyHandler(Value& value, TypeMismatchHandler callback, void* cont void EraseLegacyHandler(const Value& value) { auto& state = State(); + if (!state.recoveryMutex) NotImplemented_nid_no_patch(__func__); KernelMutexLock lock(state.recoveryMutex); if (state.legacyHandlers) state.legacyHandlers->Erase(state.legacyHandlers->Find(&value)); } @@ -1195,10 +1187,8 @@ void Destroy(Value& v) { std::lock_guard lock(state.mutex); if (!state.recoveryMutex) return; } - const bool legacy = !ModernValues(); - LegacyCleanupScope cleanup(legacy); ReleaseValueAuxiliary(v); - if (legacy) EraseLegacyHandler(v); + if (!ModernValues()) EraseLegacyHandler(v); } int EnsureAuxiliary(Node& n) { @@ -2577,7 +2567,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; - if (state.terminating || state.callbackActive || (allocatorCallbackDepth && (state.allocationDepth || legacyCleanupDepth))) NotImplemented_nid_no_patch(__func__); + if (state.terminating || state.callbackActive || (allocatorCallbackDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); TerminationScope termination(state.terminating); state.callback = nullptr; state.context = nullptr; diff --git a/core/libs/tests/GuestJsonCompositeDestruction.cpp b/core/libs/tests/GuestJsonCompositeDestruction.cpp index ac0ccb4392..f0faf7cb93 100644 --- a/core/libs/tests/GuestJsonCompositeDestruction.cpp +++ b/core/libs/tests/GuestJsonCompositeDestruction.cpp @@ -49,64 +49,75 @@ static constexpr unsigned ModuleFreeCalls = 12; static void Require(bool condition, const char* message) { if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } } +struct DestructionTrace; +struct DestructionDomain { DestructionTrace* trace; void* receiver; unsigned index; }; struct DestructionTrace { std::map live; std::map module; std::map source; std::size_t generation = 0; std::size_t sourceBase = 0; - void* receiver = nullptr; std::uint8_t* owner = nullptr; unsigned target = 0; unsigned calls = 0; unsigned terminations = 0; - unsigned refusals = 0; unsigned nullCalls = 0; - bool rejectTermination = false; + bool reinitialize = false; + bool allocatingReplacement = false; + unsigned replacementDomain = 0; + const Parameter* parameter = nullptr; + std::map replacementModule; bool observing = false; std::vector releases; + std::vector releaseDomains; std::vector moduleReleases; }; static void* APS5_VABI AllocateDestruction(void* receiver, std::size_t bytes, void* context) { - auto& trace = *static_cast(context); - Require(receiver == trace.receiver, "Composite allocations must retain their actual allocator receiver and context"); + const auto& domain = *static_cast(context); + auto& trace = *domain.trace; + Require(receiver == domain.receiver && domain.index == (trace.allocatingReplacement ? trace.replacementDomain : 0u), "Composite allocations must retain their actual allocator receiver and context"); void* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); std::memset(memory, 0xa5, bytes); Require(trace.live.emplace(memory, ++trace.generation).second, "Composite resources must own distinct live allocation generations"); return memory; } static void ReleaseStorage(DestructionTrace& trace, void* memory) { - Require(trace.live.erase(memory) == 1, "Every composite resource must be reclaimed exactly once by its owning domain"); + Require(trace.live.erase(memory) == 1, "Every composite resource must be reclaimed exactly once by the shared tracking backend"); std::free(memory); } static void APS5_VABI FreeDestruction(void* receiver, void* memory, void* context) { - auto& trace = *static_cast(context); - Require(receiver == trace.receiver, "Composite cleanup must preserve allocator receiver and context even through module termination"); + const auto& domain = *static_cast(context); + auto& trace = *domain.trace; + Require(receiver == domain.receiver, "Composite cleanup must preserve the receiver paired with its supplied callback context"); if (trace.observing) { ++trace.calls; - if (!memory) { Require(LegacySdk, "Only the measured legacy module cleanup may forward a null free"); ++trace.nullCalls; return; } + if (!memory) { Require(LegacySdk && domain.index == 0, "Only the measured legacy module cleanup may forward a null free"); ++trace.nullCalls; return; } const auto identity = trace.live.at(memory); if (trace.module.contains(memory)) { - Require(trace.module.at(memory) == identity, "Nested module termination must retain its original allocation generations"); + Require(trace.module.at(memory) == identity && domain.index == 0, "Nested module termination must retain its original allocation generations"); trace.moduleReleases.push_back(identity); } else { Require(trace.source.contains(memory) && trace.source.at(memory) == identity, "Composite release callbacks must identify their exact original payload generation"); - trace.releases.push_back(identity - trace.sourceBase); + trace.releases.push_back(identity - trace.sourceBase); trace.releaseDomains.push_back(domain.index); if (trace.target && trace.releases.size() == trace.target) { trace.target = 0; - if (trace.rejectTermination) { - const auto before = trace.live; - bool refused = false; - try { _ZN3sce4Json11Initializer9terminateEv(trace.owner); } - catch (const std::runtime_error& error) { Require(std::strstr(error.what(), "terminate"), "Unsafe legacy cleanup termination must use the explicit unsupported boundary"); refused = true; } - Require(refused && *trace.owner == 1 && trace.live == before, "Legacy auxiliary cleanup must reject callback termination before changing module ownership"); - ++trace.refusals; - } else { - auto expected = trace.live; - for (const auto& [pointer, generation] : trace.module) Require(expected.erase(pointer) == 1, "Callback termination must begin with every original module resource still live"); - Require(_ZN3sce4Json11Initializer9terminateEv(trace.owner) == 0 && *trace.owner == 0 && trace.live == expected, - "Composite free callbacks must retain reference-supported module termination while preserving all remaining source allocations including the current release"); - ++trace.terminations; + auto expected = trace.live; + for (const auto& [pointer, generation] : trace.module) Require(expected.erase(pointer) == 1, "Callback termination must begin with every original module resource still live"); + Require(_ZN3sce4Json11Initializer9terminateEv(trace.owner) == 0 && *trace.owner == 0 && trace.live == expected, + "Composite free callbacks must retain reference-supported module termination while preserving all remaining source allocations including the current release"); + ++trace.terminations; + if (trace.reinitialize) { + const auto generation = trace.generation; + trace.observing = false; trace.allocatingReplacement = true; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(trace.owner, trace.parameter) == 0 && *trace.owner == 1, + "The allocator callback may initialize a fresh module before returning its original payload release"); + trace.observing = true; trace.allocatingReplacement = false; + for (const auto& [pointer, identity] : trace.live) if (!expected.contains(pointer) || expected.at(pointer) != identity) { + Require(identity > generation, "Callback-time reinitialization must own a new module resource generation"); + trace.replacementModule.emplace(pointer, identity); + } + Require(trace.replacementModule.size() == ModuleResources, "Callback-time reinitialization must publish the complete SDK-specific module epoch"); + for (const auto& [pointer, identity] : expected) Require(trace.live.at(pointer) == identity, "Reinitialization must preserve every old payload allocation still awaiting release"); } } } @@ -126,9 +137,10 @@ static void DestroyAlias(Value& value, unsigned alias) { if (alias) _ZN3sce4Json5ValueD2Ev(&value); else _ZN3sce4Json5ValueD1Ev(&value); } -static void Check(unsigned shape, unsigned target, unsigned alias) { - DestructionTrace trace; DestructionAllocator allocator; trace.receiver = &allocator; - Parameter parameter{&allocator, &trace, 8192, 2, {}}; +static void Check(unsigned shape, unsigned target, unsigned alias, bool reinitialize, unsigned domain = 0) { + DestructionTrace trace; DestructionAllocator allocator, replacementAllocator; + DestructionDomain first{&trace, &allocator, 0}, second{&trace, domain == 2 ? &replacementAllocator : &allocator, 1}; + Parameter parameter{&allocator, &first, 8192, 2, {}}, replacement{second.receiver, &second, 4096, 2, {}}; std::uint8_t initializer = 0; trace.owner = &initializer; _ZN3sce4Json11InitializerC1Ev(&initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, ¶meter) == 0 && initializer == 1 && trace.live.size() == ModuleResources, "Initialize the actual composite-destruction resource epoch"); trace.module = trace.live; trace.sourceBase = trace.generation; @@ -150,29 +162,39 @@ static void Check(unsigned shape, unsigned target, unsigned alias) { void* auxiliary = value.auxiliary; std::array auxiliaryBytes{}; if (auxiliary) std::memcpy(auxiliaryBytes.data(), auxiliary, AuxiliaryBytes); - trace.rejectTermination = LegacySdk && shape != 0 && target > 18; - const bool terminated = target && !trace.rejectTermination; + trace.reinitialize = reinitialize; trace.parameter = domain ? &replacement : ¶meter; trace.replacementDomain = unsigned(domain != 0); + const bool terminated = target != 0; + const bool unsupportedCleanup = LegacySdk && shape != 0 && target > 18 && !reinitialize; trace.target = target; trace.observing = true; - DestroyAlias(value, alias); - const unsigned released = terminated ? target : static_cast(order.size()); + bool cleanupError = false; + try { DestroyAlias(value, alias); } + catch (const std::runtime_error& error) { + Require(unsupportedCleanup && std::strstr(error.what(), "EraseLegacyHandler"), "Only an actual missing legacy cleanup mutex may produce the explicit unsupported error"); + cleanupError = true; + } + Require(cleanupError == unsupportedCleanup, "Legacy destruction must reject the null-mutex cleanup boundary while preserving callback-time reinitialization"); + const unsigned released = terminated && !reinitialize ? target : static_cast(order.size()); Require(trace.releases == std::vector(order.begin(), order.begin() + released), "Composite destruction must preserve the exact native release-generation prefix before callback termination"); + std::vector expectedDomains(released, 0); + if (reinitialize) for (unsigned index = target; index < released; ++index) expectedDomains[index] = trace.replacementDomain; + Require(trace.releaseDomains == expectedDomains, "The already-entered free retains its original domain; subsequent releases must use the replacement allocator/context even for old source allocations"); auto remaining = trace.source; for (unsigned index = 0; index < released; ++index) { bool found = false; for (auto iterator = remaining.begin(); iterator != remaining.end(); ++iterator) if (iterator->second == trace.sourceBase + order[index]) { remaining.erase(iterator); found = true; break; } Require(found, "Every observed release must remove exactly its expected original payload generation"); } - const bool retainedAuxiliary = terminated && ((shape == 1 || shape == 2) ? target <= 18 : shape == 3 && target <= 3); + const bool retainedAuxiliary = terminated && !reinitialize && ((shape == 1 || shape == 2) ? target <= 18 : shape == 3 && target <= 3); Require(value.type == 0 && !value.parent && !value.payload && value.reserved == UINT32_C(0xa5a5a5a5) && value.auxiliary == (retainedAuxiliary ? auxiliary : nullptr), "Destruction must skip auxiliary cleanup after payload callbacks terminate the module, retaining its original pointer rather than falsely clearing ownership"); if (retainedAuxiliary) Require(std::memcmp(auxiliary, auxiliaryBytes.data(), AuxiliaryBytes) == 0, "Skipped auxiliary cleanup must preserve the complete original auxiliary bytes and callback/fallback identities"); - Require(trace.terminations == unsigned(terminated) && trace.refusals == unsigned(trace.rejectTermination) && !trace.target && trace.calls == released + (terminated ? ModuleFreeCalls : 0u) && trace.moduleReleases.size() == (terminated ? ModuleResources : 0u) && trace.nullCalls == (terminated ? ModuleFreeCalls - ModuleResources : 0u) && initializer == unsigned(!terminated) && trace.live == (terminated ? remaining : trace.module), + Require(trace.terminations == unsigned(terminated) && !trace.target && trace.calls == released + (terminated ? ModuleFreeCalls : 0u) && trace.moduleReleases.size() == (terminated ? ModuleResources : 0u) && trace.nullCalls == (terminated ? ModuleFreeCalls - ModuleResources : 0u) && initializer == unsigned(!terminated || reinitialize) && trace.live == (reinitialize ? trace.replacementModule : terminated ? remaining : trace.module), "Callback termination must reclaim only the module epoch and released source prefix while preserving the reference's orphaned allocations"); const auto beforeRepeat = trace.live; const auto calls = trace.calls; DestroyAlias(value, 1 - alias); Require(trace.live == beforeRepeat && trace.calls == calls && value.auxiliary == (retainedAuxiliary ? auxiliary : nullptr), "A repeated destructor must preserve inactive-module auxiliary state without attempting another release"); trace.observing = false; - if (!terminated) Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Ordinary composite destruction must remain fully reclaimable"); + if (!terminated || reinitialize) Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Ordinary composite destruction must remain fully reclaimable"); else { Require(trace.live == remaining, "Fixture reclamation may begin only after the exact orphaned source epoch has been observed"); for (const auto& [pointer, generation] : remaining) { Require(trace.live.at(pointer) == generation, "Caller-owned fixture cleanup must retain each orphan allocation generation"); ReleaseStorage(trace, pointer); } @@ -183,6 +205,7 @@ static void Check(unsigned shape, unsigned target, unsigned alias) { Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release the fresh composite lifecycle generation"); } int main() { - for (unsigned shape = 0; shape < ShapeCount; ++shape) for (unsigned target = 0; target <= ReleaseOrders[shape].size(); ++target) for (unsigned alias = 0; alias < 2; ++alias) Check(shape, target, alias); + for (unsigned shape = 0; shape < ShapeCount; ++shape) for (unsigned target = 0; target <= ReleaseOrders[shape].size(); ++target) for (unsigned alias = 0; alias < 2; ++alias) Check(shape, target, alias, false); + for (unsigned shape = 0; shape < ShapeCount; ++shape) for (unsigned target = 1; target <= ReleaseOrders[shape].size(); ++target) for (unsigned alias = 0; alias < 2; ++alias) for (unsigned domain = 0; domain < 3; ++domain) Check(shape, target, alias, true, domain); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 5bb4b43574..8f1857dd99 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -1,5 +1,7 @@ # libSceJson2 technical debt +SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGetCompiledSdkVersion` (process-parameter offset `0x10`). The PS5 SDK field at `0x14` and the console firmware version are distinct. + ### Unknown function info - [JSON real-number conversion](../../../core/libs/prx/libSceJson2/Export.cpp) - floating-point environment behavior and malformed numeric-token grammar remain unverified. Valid decimal conversion follows the instructions at `0x7750`: separate multiply/add accumulation, per-digit fractional division and repeated exponent scaling. Reference-backed exact-bit tests cover signed underflow, subnormals, normal boundaries, large finite values and overflow across special-float modes. These results intentionally retain the reference rounding, which differs from standard correctly rounded conversion. Formatting still uses locale-independent `to_chars` with 17 significant digits for modern settings; broader formatting equivalence remains unverified. @@ -86,6 +88,8 @@ - [Legacy callback map inherited failures](../../../core/libs/prx/libSceJson2/Export.cpp) - Pair/collection copies inside an inherited failure scope, allocator-driven lifecycle changes and cross-thread propagation remain unverified. Six SDK 5 reference-backed schedules exercise new/existing records through a real parser reader, with healthy controls, a published allocation error and registration inside the notifier before error publication. Published failure preserves an existing handler or retains a new empty record; scalar copying still completes but its callback record remains empty. Notifier-time registration and copying succeed before the shared error is published, while the outer parser completes its admitted literal and reports OOM. Checks cover actual allocation/notification order, callback ABI and borrowed results, exact auxiliary/fallback generations, record-reusing registration/copy retry, independent parser retry and complete cleanup in both allocator domains. -- [Composite-destruction termination](../../../core/libs/prx/libSceJson2/Export.cpp) - callback-time reinitialization, additional legacy callback combinations and thrown callbacks remain unverified. One hundred ten SDK 6 reference cases cover both Value destructor aliases, four payload/metadata shapes and every release position. Following `0x17e0`, destruction requires the published recovery mutex after clearing the payload; callback termination during payload release therefore preserves the auxiliary pointer and bytes. Checks cover exact release-generation prefixes, allocator receiver/context, module/source ownership, repeated destruction and fresh initialization. Early termination orphans the remaining caller allocations; final-release termination completes cleanup. The caller-owned fixture separately reclaims the observed orphaned epoch. The pre-fix destructor fails the auxiliary-retention regression. +- [Composite-destruction termination](../../../core/libs/prx/libSceJson2/Export.cpp) - additional legacy callback combinations and thrown callbacks remain unverified. One hundred ten SDK 6 reference cases cover both Value destructor aliases, four payload/metadata shapes and every release position. Following `0x17e0`, destruction requires the published recovery mutex after clearing the payload; callback termination during payload release therefore preserves the auxiliary pointer and bytes. Checks cover exact release-generation prefixes, allocator receiver/context, module/source ownership, repeated destruction and fresh initialization. Early termination orphans the remaining caller allocations; final-release termination completes cleanup. The caller-owned fixture separately reclaims the observed orphaned epoch. The pre-fix destructor fails the auxiliary-retention regression. -- [Legacy auxiliary-cleanup termination](../../../core/libs/prx/libSceJson2/Export.cpp) - owner termination from allocator callbacks inside legacy auxiliary/map cleanup is explicitly unsupported before teardown. Eight reference probes across both Value destructor aliases returned termination success, then faulted through a null mutex slot at the substituted kernel boundary; these are not console measurements. Ninety-two SDK 5 production cases verify the refusal and 84 returning reference cases preserve ordinary destruction and early payload-release termination for strings and nested arrays/objects. Checks cover eleven module resources plus the observed null free, exact release prefixes, retained auxiliary bytes, allocator identities, repeated destruction and fresh initialization. The cleanup guard ends before later operations; modern metadata-release termination remains supported. Callback-time reinitialization and thrown callbacks remain unverified. +- [Legacy auxiliary-cleanup termination](../../../core/libs/prx/libSceJson2/Export.cpp) - a missing recovery mutex after auxiliary-release callbacks is explicitly unsupported at legacy map cleanup, before kernel use. Eight reference probes across both Value destructor aliases returned termination success, then faulted through a null mutex slot at the substituted kernel boundary; these are not console measurements. Ninety-two termination-only production cases cover 84 returning reference cases and eight explicit errors. Checks preserve eleven module resources plus the observed null free, release prefixes, auxiliary state, allocator identities and retry. Termination itself remains allowed so its callback can reinitialize before cleanup resumes. Thrown callbacks remain unverified. + +- [Composite callback reinitialization](../../../core/libs/prx/libSceJson2/Export.cpp) - repeated nested reinitialization and thrown callbacks remain unverified. Five hundred sixty-four SDK 5/6 reference cases cover both destructor aliases, every release position and shared, changed-context or changed-receiver/context domains. The already-entered free keeps its original receiver/context; subsequent releases use the newly published allocator even for old allocations. Reinitialization admits a complete fresh module epoch and lets auxiliary/map cleanup finish. Tests verify release identities/order, domain routing, complete payload cleanup, repeated destruction and later module retry. The pooled backend records cross-domain dispatch; it does not establish safe transfer between independent allocator pools. From 306a228b4c61ba9187365772387bbad2ecb98506 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:33:18 +0200 Subject: [PATCH 071/318] test(libSceJson2): verify inherited Pair callback failures --- core/libs/tests/GuestJsonLegacyHandlers.cpp | 29 +++++++++++++++++---- docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index 4572b2dba9..01835095e8 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -400,11 +400,17 @@ struct MapReadContext { unsigned mode; unsigned reads = 0; int registration = -1; + Pair* pair = nullptr; + const String* key = nullptr; + unsigned alias = 0; }; static void UpdateInheritedMap(void* data) { auto& context = *static_cast(data); context.registration = _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(context.target, RecoverLegacy, context.replacement); - _ZN3sce4Json5ValueC1ERKS1_(context.copy, context.target); + if (context.pair) { + if (context.alias == 1) _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(context.pair, context.key, context.target); + else _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(context.pair, context.key, context.target); + } else _ZN3sce4Json5ValueC1ERKS1_(context.copy, context.target); } static int APS5_VABI ReadInheritedMap(char& character, void* data) { auto& context = *static_cast(data); @@ -422,8 +428,11 @@ static int APS5_VABI ReadInheritedMap(char& character, void* data) { } static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& second) { const auto guestBaseline = trace.guestGenerations, heapBaseline = trace.heapGenerations; - for (bool existing : {false, true}) for (unsigned mode = 0; mode < 3; ++mode) { - Value target{}, copy{}, outer{}; ParseLegacy(target); + for (bool existing : {false, true}) for (unsigned mode = 0; mode < 3; ++mode) for (unsigned alias = 0; alias < 3; ++alias) { + Value target{}, scalarCopy{}, outer{}; Pair pair; std::memset(&pair, 0xa5, sizeof(pair)); + Value& copy = alias ? pair.value : scalarCopy; + String key{}; if (alias) _ZN3sce4Json6StringC1EPKc(&key, "key"); + ParseLegacy(target); if (existing) Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&target, RecoverLegacy, &first) == 0, "Prepare an existing legacy mismatch record before inherited failure"); std::vector targetAllocations; for (const auto& [pointer, generation] : trace.guestGenerations) { @@ -434,12 +443,17 @@ static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& secon const Value targetValue = target; _ZN3sce4Json5ValueC1Ev(&outer); MapReadContext context{&trace, &target, ©, &second, mode}; + if (alias) { context.pair = &pair; context.key = &key; context.alias = alias; } const auto requests = trace.requests.size(); const auto notices = trace.notifications; const int status = _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(&outer, ReadInheritedMap, &context); const auto error = static_cast(0x80848102); Require(status == (mode ? error : 0) && context.registration == (mode == 1 ? error : 0) && context.reads == 4 && outer.type == 1 && (outer.payload & 0xff) == 1 && outer.auxiliary, "Legacy map updates must observe failure only after notifier return without clearing the outer parser error or interrupting its admitted literal"); - const auto expected = mode ? std::vector{8, 24, 32} : std::vector{24, 32}; + auto expected = mode ? std::vector{8} : std::vector{}; + if (alias) { expected.push_back(24); expected.push_back(4); } + expected.push_back(24); expected.push_back(32); + if (alias) Require(pair.reserved == UINT64_C(0xa5a5a5a5a5a5a5a5) && pair.key.storage != key.storage && std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), "key") == 0, + "Inherited failure must preserve independent Pair key ownership, complete key contents and inactive padding"); Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected && trace.notifications == notices + unsigned(mode != 0) && !trace.onNotify, "Inherited map registration must preserve actual failure, root-admission and notifier ordering"); Context* targetContext = mode == 1 ? existing ? &first : nullptr : &second; @@ -468,7 +482,12 @@ static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& secon "Inherited failure, registration retry and copying must preserve the original source auxiliary and fallback generations"); Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&outer, "false", 5) == 0 && outer.type == 1 && !(outer.payload & 0xff), "The next independent parse must not inherit an already completed callback-map failure"); - _ZN3sce4Json5ValueD1Ev(©); _ZN3sce4Json5ValueD1Ev(&target); _ZN3sce4Json5ValueD1Ev(&outer); CheckMap(trace, {}); + if (alias) { + if (alias == 1) _ZN3sce4Json6Object4PairD1Ev(&pair); + else _ZN3sce4Json6Object4PairD2Ev(&pair); + _ZN3sce4Json6StringD1Ev(&key); + } else _ZN3sce4Json5ValueD1Ev(©); + _ZN3sce4Json5ValueD1Ev(&target); _ZN3sce4Json5ValueD1Ev(&outer); CheckMap(trace, {}); Require(trace.guestGenerations == guestBaseline && trace.heapGenerations == heapBaseline, "Inherited map updates and retries must reclaim every temporary generation in both allocator domains"); } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 8f1857dd99..8881ea6026 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -86,7 +86,7 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Legacy nested Pair copies](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. One hundred fifty-two SDK 5 reference-backed cases cover an array containing an object/string and an object containing an array/string, both Pair constructor/destructor aliases and persistent/one-shot failure at every allocation position through eighteen-allocation success. Checks cover partial counts, missing sentinels, independent internal/public object keys, exact request/notification fingerprints, inactive bytes, source bytes/generations and both allocator domains. Copies retain the root mismatch-map callback independently of auxiliary metadata; unbound descendants use typed defaults without inheriting that callback. Root and source recovery remain callable after copy cleanup. -- [Legacy callback map inherited failures](../../../core/libs/prx/libSceJson2/Export.cpp) - Pair/collection copies inside an inherited failure scope, allocator-driven lifecycle changes and cross-thread propagation remain unverified. Six SDK 5 reference-backed schedules exercise new/existing records through a real parser reader, with healthy controls, a published allocation error and registration inside the notifier before error publication. Published failure preserves an existing handler or retains a new empty record; scalar copying still completes but its callback record remains empty. Notifier-time registration and copying succeed before the shared error is published, while the outer parser completes its admitted literal and reports OOM. Checks cover actual allocation/notification order, callback ABI and borrowed results, exact auxiliary/fallback generations, record-reusing registration/copy retry, independent parser retry and complete cleanup in both allocator domains. +- [Legacy callback map inherited failures](../../../core/libs/prx/libSceJson2/Export.cpp) - Collection copies inside an inherited failure scope, allocator-driven lifecycle changes and cross-thread propagation remain unverified. Eighteen SDK 5 reference cases exercise new/existing records through a parser reader: healthy controls, published allocation errors and notifier-time registration. Published failure preserves an existing handler or retains a new empty record; scalar Value and both Pair constructor aliases still copy the scalar but their callback record remains empty. Pair keys retain independently allocated complete text and inactive padding. Notifier-time registration and copying succeed before the shared error is published, while the outer parser completes its admitted literal and reports OOM. Checks cover allocation/notification order, borrowed callback results, auxiliary/fallback generations, record-reusing retry, independent parser retry and both allocator domains. - [Composite-destruction termination](../../../core/libs/prx/libSceJson2/Export.cpp) - additional legacy callback combinations and thrown callbacks remain unverified. One hundred ten SDK 6 reference cases cover both Value destructor aliases, four payload/metadata shapes and every release position. Following `0x17e0`, destruction requires the published recovery mutex after clearing the payload; callback termination during payload release therefore preserves the auxiliary pointer and bytes. Checks cover exact release-generation prefixes, allocator receiver/context, module/source ownership, repeated destruction and fresh initialization. Early termination orphans the remaining caller allocations; final-release termination completes cleanup. The caller-owned fixture separately reclaims the observed orphaned epoch. The pre-fix destructor fails the auxiliary-retention regression. From 6a66ea5400067e8ad335acf693a4e912f8f99c3d Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:35:53 +0200 Subject: [PATCH 072/318] test(libSceJson2): verify inherited collection callback copies --- core/libs/tests/GuestJsonLegacyHandlers.cpp | 25 ++++++++++++++++----- docs/dev/debt/libSceJson2.md | 3 ++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index 01835095e8..ea5fb5ae69 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -428,11 +428,11 @@ static int APS5_VABI ReadInheritedMap(char& character, void* data) { } static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& second) { const auto guestBaseline = trace.guestGenerations, heapBaseline = trace.heapGenerations; - for (bool existing : {false, true}) for (unsigned mode = 0; mode < 3; ++mode) for (unsigned alias = 0; alias < 3; ++alias) { + for (unsigned shape = 0; shape < 3; ++shape) for (bool existing : {false, true}) for (unsigned mode = 0; mode < 3; ++mode) for (unsigned alias = 0; alias < 3; ++alias) { Value target{}, scalarCopy{}, outer{}; Pair pair; std::memset(&pair, 0xa5, sizeof(pair)); Value& copy = alias ? pair.value : scalarCopy; String key{}; if (alias) _ZN3sce4Json6StringC1EPKc(&key, "key"); - ParseLegacy(target); + ParseLegacy(target, shape == 0 ? "9" : shape == 1 ? "[{\"s\":\"text\"}]" : "{\"a\":[\"text\"]}"); if (existing) Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&target, RecoverLegacy, &first) == 0, "Prepare an existing legacy mismatch record before inherited failure"); std::vector targetAllocations; for (const auto& [pointer, generation] : trace.guestGenerations) { @@ -451,6 +451,11 @@ static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& secon "Legacy map updates must observe failure only after notifier return without clearing the outer parser error or interrupting its admitted literal"); auto expected = mode ? std::vector{8} : std::vector{}; if (alias) { expected.push_back(24); expected.push_back(4); } + if (shape) { + const auto payload = shape == 1 ? std::vector{8, 24, 48, 48, 48, 8, 24, 104, 104, 104, 2, 24, 2, 8, 24, 5} + : std::vector{8, 24, 104, 104, 104, 2, 24, 2, 8, 24, 48, 48, 48, 8, 24, 5}; + expected.insert(expected.end(), payload.begin(), payload.end()); + } expected.push_back(24); expected.push_back(32); if (alias) Require(pair.reserved == UINT64_C(0xa5a5a5a5a5a5a5a5) && pair.key.storage != key.storage && std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), "key") == 0, "Inherited failure must preserve independent Pair key ownership, complete key contents and inactive padding"); @@ -459,8 +464,15 @@ static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& secon Context* targetContext = mode == 1 ? existing ? &first : nullptr : &second; Context* copyContext = mode == 1 ? nullptr : &second; CheckMap(trace, {{&target, targetContext}, {©, copyContext}}); - Require(!copy.parent && !copy.auxiliary && copy.type == 3 && copy.payload == 9 && std::memcmp(&target, &targetValue, sizeof(target)) == 0, - "Inherited callback-map failure must not discard copied scalar contents or change the source ownership metadata"); + Require(!copy.parent && !copy.auxiliary && copy.type == (shape ? shape == 1 ? 6 : 7 : 3) && (shape || copy.payload == 9) && std::memcmp(&target, &targetValue, sizeof(target)) == 0, + "Inherited callback-map failure must preserve copied payload type and source ownership metadata"); + const auto verifyShape = [&] { + std::string actual; std::vector values; + CollectionShape(copy, nullptr, actual, values); + const char* expectedShape = shape == 0 ? "3:0=9;" : shape == 1 ? "6:0a1[7:0o1{k=s/q=s/5:0s=text;};];" : "7:0o1{k=a/q=a/6:0a1[5:0s=text;];};"; + Require(actual == expectedShape, "Inherited copies must preserve complete nested payloads, independent internal/public keys and unbound descendant parents"); + }; + verifyShape(); const auto verifyRecovery = [&](const Value& value, Context* expectedContext) { const auto firstCalls = first.calls, secondCalls = second.calls; const auto* result = _ZNK3sce4Json5Value10getBooleanEv(&value); @@ -474,8 +486,9 @@ static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& secon const auto beforeRetry = trace.requests.size(); Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&target, RecoverLegacy, &second) == 0 && _ZN3sce4Json5ValueaSERKS1_(©, &target) == ©, "Independent registration and copy retry must succeed after the outer failure scope completes"); - Require(trace.guestGenerations == guest && trace.heapGenerations == heap && trace.requests.size() == beforeRetry, - "Retry must reuse both existing or empty legacy callback records without replacing allocation generations"); + Require((shape ? trace.guestGenerations.size() == guest.size() && trace.requests.size() > beforeRetry : trace.guestGenerations == guest && trace.requests.size() == beforeRetry) && trace.heapGenerations == heap, + "Retry must reuse legacy callback records and retain balanced payload ownership across collection replacement"); + verifyShape(); CheckMap(trace, {{&target, &second}, {©, &second}}); verifyRecovery(target, &second); verifyRecovery(copy, &second); for (const auto& snapshot : targetAllocations) Require(trace.guestGenerations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 8881ea6026..7d79bac3d7 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -86,7 +86,8 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Legacy nested Pair copies](../../../core/libs/prx/libSceJson2/Export.cpp) - inherited failure scopes, allocator-driven reentry and libc allocation exceptions remain unverified. One hundred fifty-two SDK 5 reference-backed cases cover an array containing an object/string and an object containing an array/string, both Pair constructor/destructor aliases and persistent/one-shot failure at every allocation position through eighteen-allocation success. Checks cover partial counts, missing sentinels, independent internal/public object keys, exact request/notification fingerprints, inactive bytes, source bytes/generations and both allocator domains. Copies retain the root mismatch-map callback independently of auxiliary metadata; unbound descendants use typed defaults without inheriting that callback. Root and source recovery remain callable after copy cleanup. -- [Legacy callback map inherited failures](../../../core/libs/prx/libSceJson2/Export.cpp) - Collection copies inside an inherited failure scope, allocator-driven lifecycle changes and cross-thread propagation remain unverified. Eighteen SDK 5 reference cases exercise new/existing records through a parser reader: healthy controls, published allocation errors and notifier-time registration. Published failure preserves an existing handler or retains a new empty record; scalar Value and both Pair constructor aliases still copy the scalar but their callback record remains empty. Pair keys retain independently allocated complete text and inactive padding. Notifier-time registration and copying succeed before the shared error is published, while the outer parser completes its admitted literal and reports OOM. Checks cover allocation/notification order, borrowed callback results, auxiliary/fallback generations, record-reusing retry, independent parser retry and both allocator domains. +- [Legacy callback map inherited failures](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-driven lifecycle changes, concurrent propagation and thrown callbacks remain unverified. Fifty-four SDK 5 reference cases cover scalar and nested Array/Object copies through Value and both Pair constructor aliases, new/existing records, healthy readers, published errors and notifier-time registration. Published failure preserves existing handlers or retains empty records; payload copying still completes. Notifier-time copies retain callbacks before error publication. Checks cover exact requests/notices, Pair padding and independent keys, nested contents and parent links, borrowed callback results, source bytes/generations, record-reusing retry and cleanup in both allocator domains. A mutation that skips collection copying after inherited failure is detected. + - [Composite-destruction termination](../../../core/libs/prx/libSceJson2/Export.cpp) - additional legacy callback combinations and thrown callbacks remain unverified. One hundred ten SDK 6 reference cases cover both Value destructor aliases, four payload/metadata shapes and every release position. Following `0x17e0`, destruction requires the published recovery mutex after clearing the payload; callback termination during payload release therefore preserves the auxiliary pointer and bytes. Checks cover exact release-generation prefixes, allocator receiver/context, module/source ownership, repeated destruction and fresh initialization. Early termination orphans the remaining caller allocations; final-release termination completes cleanup. The caller-owned fixture separately reclaims the observed orphaned epoch. The pre-fix destructor fails the auxiliary-retention regression. From 39e4a0036eb8554b58e49150abeab8ec6c4df2e9 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:37:45 +0200 Subject: [PATCH 073/318] docs(libSceJson2): record verified allocator exception mismatch --- docs/dev/debt/libSceJson2.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 7d79bac3d7..aac85e26cd 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -94,3 +94,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Legacy auxiliary-cleanup termination](../../../core/libs/prx/libSceJson2/Export.cpp) - a missing recovery mutex after auxiliary-release callbacks is explicitly unsupported at legacy map cleanup, before kernel use. Eight reference probes across both Value destructor aliases returned termination success, then faulted through a null mutex slot at the substituted kernel boundary; these are not console measurements. Ninety-two termination-only production cases cover 84 returning reference cases and eight explicit errors. Checks preserve eleven module resources plus the observed null free, release prefixes, auxiliary state, allocator identities and retry. Termination itself remains allowed so its callback can reinitialize before cleanup resumes. Thrown callbacks remain unverified. - [Composite callback reinitialization](../../../core/libs/prx/libSceJson2/Export.cpp) - repeated nested reinitialization and thrown callbacks remain unverified. Five hundred sixty-four SDK 5/6 reference cases cover both destructor aliases, every release position and shared, changed-context or changed-receiver/context domains. The already-entered free keeps its original receiver/context; subsequent releases use the newly published allocator even for old allocations. Reinitialization admits a complete fresh module epoch and lets auxiliary/map cleanup finish. Tests verify release identities/order, domain routing, complete payload cleanup, repeated destruction and later module retry. The pooled backend records cross-domain dispatch; it does not establish safe transfer between independent allocator pools. + +- [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - production currently rolls back thrown allocator callbacks, unlike the measured SDK 6 reference. Thirteen isolated native probes register all 305 FDEs from the module's single `zR` CIE. Throws at allocation positions 0–11 propagate without frees. A position-0 throw leaves retry available; positions 1–11 retain the admitted allocations, leave the owner inactive, reject termination as not initialized and reject retry as already initialized. Successful position 12 admits and releases all twelve resources. Host unwinding through mapped reference frames is verified; this does not establish guest exception integration or thrown kernel/free/notifier behavior. Regression coverage and faithful production handling remain required. From 14347afcecf63264c7f2dde34077feb405d41ec3 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:40:47 +0200 Subject: [PATCH 074/318] fix(libSceJson2): preserve partial initialization on allocator exceptions --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 ++ core/libs/prx/libSceJson2/Export.cpp | 27 +++--- .../GuestJsonInitializationExceptions.cpp | 89 +++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 +- 4 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 core/libs/tests/GuestJsonInitializationExceptions.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 7301737abd..0f83a099c7 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,13 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_initialization_exceptions_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationExceptions.cpp) + target_include_directories(guest_json_initialization_exceptions_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_initialization_exceptions_tests PRIVATE libSceJson2 libkernel libc) + configure_windows_unwind(guest_json_initialization_exceptions_tests) + foreach(position RANGE 0 12) + add_test(NAME guest_json_initialization_exception_${position} COMMAND guest_json_initialization_exceptions_tests ${position}) + endforeach() add_test_executable(guest_json_legacy_composite_destruction_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCompositeDestruction.cpp Export.cpp) target_compile_definitions(guest_json_legacy_composite_destruction_tests PRIVATE JSON2_DESTRUCTION_LEGACY_SDK) target_include_directories(guest_json_legacy_composite_destruction_tests PRIVATE ${LIBS_INCLUDE_DIR}) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index b3daff44f1..ddeafcf9a3 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1441,6 +1441,8 @@ void ResetDefaults(bool releaseMissing = false) { } } +void ReleaseInitialization(); + PthreadMutex* CreateKernelMutex() { auto* mutex = static_cast(Allocate(sizeof(PthreadMutex))); if (!mutex) return nullptr; @@ -1457,6 +1459,7 @@ PthreadMutex* CreateKernelMutex() { if (attribute) scePthreadMutexattrDestroy(&attribute); if (*mutex) scePthreadMutexDestroy(mutex); Release(mutex); + ReleaseInitialization(); throw; } } @@ -1518,19 +1521,17 @@ int InitializeResources(void* self, const void* parameter, std::int32_t format, state.allocation = allocation; state.floatFormat = format; state.floatFormatEnabled = modern; - try { - CreateDefault(TypeString); - CreateDefault(TypeArray); - CreateDefault(TypeObject); - state.fallback = static_cast(Allocate(sizeof(Value))); - if (state.fallback) Construct(*state.fallback); - state.internalMutex = CreateKernelMutex(); - const auto& defaults = SharedDefaults(); - const bool admitted = defaults.values[TypeString].string && defaults.values[TypeArray].array && - defaults.values[TypeObject].object && state.fallback && state.internalMutex; - if (admitted) state.recoveryMutex = ModernValues() ? CreateKernelMutex() : state.internalMutex; - if (!admitted || !state.recoveryMutex) { ReleaseInitialization(); return JsonErrorOutOfMemory; } - } catch (...) { ReleaseInitialization(); throw; } + CreateDefault(TypeString); + CreateDefault(TypeArray); + CreateDefault(TypeObject); + state.fallback = static_cast(Allocate(sizeof(Value))); + if (state.fallback) Construct(*state.fallback); + state.internalMutex = CreateKernelMutex(); + const auto& defaults = SharedDefaults(); + const bool admitted = defaults.values[TypeString].string && defaults.values[TypeArray].array && + defaults.values[TypeObject].object && state.fallback && state.internalMutex; + if (admitted) state.recoveryMutex = ModernValues() ? CreateKernelMutex() : state.internalMutex; + if (!admitted || !state.recoveryMutex) { ReleaseInitialization(); return JsonErrorOutOfMemory; } state.paddedStringGrowth = CompiledSdk() > 0x044fffff; state.initialized = true; *static_cast(self) = 1; diff --git a/core/libs/tests/GuestJsonInitializationExceptions.cpp b/core/libs/tests/GuestJsonInitializationExceptions.cpp new file mode 100644 index 0000000000..2b4ef37c76 --- /dev/null +++ b/core/libs/tests/GuestJsonInitializationExceptions.cpp @@ -0,0 +1,89 @@ +#include "prx/libkernel/Pthread/include/Mutex.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct Marker { const void* identity; }; +struct InitializationExceptionTrace { + void* allocator = nullptr; + unsigned target = 0; + std::vector requests; + std::vector admitted; + std::map live; + unsigned releases = 0; +}; +static void* APS5_VABI AllocateExceptionTracked(void* self, std::size_t size, void* data) { + auto& trace = *static_cast(data); + Require(self == trace.allocator, "Initialization must preserve allocator receiver and context"); + trace.requests.push_back(size); + if (trace.requests.size() == trace.target + 1) throw Marker{&trace}; + void* memory = std::malloc(size); + Require(memory && trace.live.emplace(memory, trace.admitted.size()).second, "Every admitted allocation requires independent ownership"); + std::memset(memory, 0xa5, size); + trace.admitted.push_back(memory); + return memory; +} +static void APS5_VABI ReleaseExceptionTracked(void* self, void* memory, void* data) { + auto& trace = *static_cast(data); + Require(self == trace.allocator && (!memory || trace.live.erase(memory) == 1), "Normal termination must release each admitted generation exactly once"); + ++trace.releases; + std::free(memory); +} +static void APS5_VABI NotifyExceptionTracked(void*, int, std::size_t, void*) { Require(false, "Thrown callbacks must not synthesize null-allocation notifications"); } +static void* const Table[]{nullptr, nullptr, reinterpret_cast(AllocateExceptionTracked), reinterpret_cast(ReleaseExceptionTracked), reinterpret_cast(NotifyExceptionTracked)}; +struct ExceptionAllocator { void* const* table = Table; }; +int main(int argc, char** argv) { + Require(argc == 2, "Each exception schedule requires an independent module process"); + const auto target = static_cast(std::strtoul(argv[1], nullptr, 10)); + Require(target <= 12, "Allocation schedule must include every initialization position and success"); + ExceptionAllocator allocator; InitializationExceptionTrace trace; trace.allocator = &allocator; trace.target = target; + Parameter parameter{&allocator, &trace, 16384, 0, {}}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + bool caught = false; + try { + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0, "Complete admission must initialize normally"); + } catch (const Marker& marker) { + Require(marker.identity == &trace, "Allocator exceptions must propagate with their original identity"); + caught = true; + } + Require(caught == (target < 12) && owner == unsigned(target == 12) && trace.releases == 0 && trace.live.size() == target, + "Thrown initialization callbacks must retain every prior allocation without publishing an active owner or rolling back"); + const std::array sizes{24, 8, 24, 8, 32, 48, 8, 32, 104, 32, 8, 8}; + const auto count = target < 12 ? target + 1 : 12; + Require(trace.requests == std::vector(sizes.begin(), sizes.begin() + count), "Exception schedules must cross the exact initialization resource sequence"); + const auto live = trace.live; + const auto requests = trace.requests; + const int notInitialized = static_cast(0x80848110), alreadyInitialized = static_cast(0x80848111); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == (caught ? notInitialized : 0), "An inactive owner must not reclaim the published partial context"); + trace.target = 1000; + if (target && caught) { + std::uint8_t other = 0; _ZN3sce4Json11InitializerC1Ev(&other); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == alreadyInitialized && + _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, ¶meter) == alreadyInitialized && + _ZN3sce4Json11Initializer9terminateEv(&other) == notInitialized && !owner && !other && trace.live == live && trace.requests == requests && !trace.releases, + "A published partial context must reject retries from both owners without changing allocations"); + if (target > 10) Require(scePthreadMutexDestroy(static_cast(trace.admitted[10])) == 0, "The fixture must reclaim its orphaned host mutex separately"); + for (const auto& [pointer, generation] : trace.live) std::free(pointer); + trace.live.clear(); + } else { + Require(trace.live.empty() && trace.releases == (target == 12 ? 12u : 0u), "Normal termination must reclaim all twelve resources"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0 && owner && trace.live.size() == 12, + "Unpublished exceptions and completed lifecycles must allow a fresh initialization"); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.live.empty(), "Fresh initialization must retain complete teardown"); + } + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index aac85e26cd..66926967c8 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -95,4 +95,4 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Composite callback reinitialization](../../../core/libs/prx/libSceJson2/Export.cpp) - repeated nested reinitialization and thrown callbacks remain unverified. Five hundred sixty-four SDK 5/6 reference cases cover both destructor aliases, every release position and shared, changed-context or changed-receiver/context domains. The already-entered free keeps its original receiver/context; subsequent releases use the newly published allocator even for old allocations. Reinitialization admits a complete fresh module epoch and lets auxiliary/map cleanup finish. Tests verify release identities/order, domain routing, complete payload cleanup, repeated destruction and later module retry. The pooled backend records cross-domain dispatch; it does not establish safe transfer between independent allocator pools. -- [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - production currently rolls back thrown allocator callbacks, unlike the measured SDK 6 reference. Thirteen isolated native probes register all 305 FDEs from the module's single `zR` CIE. Throws at allocation positions 0–11 propagate without frees. A position-0 throw leaves retry available; positions 1–11 retain the admitted allocations, leave the owner inactive, reject termination as not initialized and reject retry as already initialized. Successful position 12 admits and releases all twelve resources. Host unwinding through mapped reference frames is verified; this does not establish guest exception integration or thrown kernel/free/notifier behavior. Regression coverage and faithful production handling remain required. +- [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - older SDKs, notifier/free exceptions and guest exception integration remain unverified. Thirteen isolated SDK 6 reference-backed cases cover each allocation position and success. Throws propagate with their original identity and no frees. A first-allocation throw leaves retry available; later throws retain admitted resources, leave the owner inactive, reject termination as not initialized and reject either owner's retry as already initialized. The fixture separately reclaims orphaned storage and its host mutex after checking library behavior. Null-return rollback and explicit kernel setup error cleanup remain separate. The pre-fix implementation fails the no-rollback assertion; normal admission, termination and retry remain covered. From 71f9d0d7f7466c589935e6655b96d44858fc2ec6 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:42:58 +0200 Subject: [PATCH 075/318] test(libSceJson2): cover legacy and notifier initialization exceptions --- core/libs/prx/libSceJson2/CMakeLists.txt | 18 ++++++- .../GuestJsonInitializationExceptions.cpp | 47 ++++++++++++++----- docs/dev/debt/libSceJson2.md | 2 +- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 0f83a099c7..00aed88fd8 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -9,7 +9,23 @@ if(BUILD_TESTING) target_link_libraries(guest_json_initialization_exceptions_tests PRIVATE libSceJson2 libkernel libc) configure_windows_unwind(guest_json_initialization_exceptions_tests) foreach(position RANGE 0 12) - add_test(NAME guest_json_initialization_exception_${position} COMMAND guest_json_initialization_exceptions_tests ${position}) + foreach(callback allocate notify) + add_test(NAME guest_json_initialization_exception_${callback}_${position} COMMAND guest_json_initialization_exceptions_tests ${position} ${callback}) + endforeach() + endforeach() + add_test_executable(guest_json_legacy_initialization_exceptions_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationExceptions.cpp Export.cpp) + target_compile_definitions(guest_json_legacy_initialization_exceptions_tests PRIVATE JSON2_EXCEPTION_LEGACY_SDK) + target_include_directories(guest_json_legacy_initialization_exceptions_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_legacy_initialization_exceptions_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_legacy_initialization_exceptions_tests) + foreach(position RANGE 0 11) + foreach(callback allocate notify) + add_test(NAME guest_json_legacy_initialization_exception_${callback}_${position} COMMAND guest_json_legacy_initialization_exceptions_tests ${position} ${callback}) + if(WIN32) + set_tests_properties(guest_json_legacy_initialization_exception_${callback}_${position} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() endforeach() add_test_executable(guest_json_legacy_composite_destruction_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCompositeDestruction.cpp Export.cpp) target_compile_definitions(guest_json_legacy_composite_destruction_tests PRIVATE JSON2_DESTRUCTION_LEGACY_SDK) diff --git a/core/libs/tests/GuestJsonInitializationExceptions.cpp b/core/libs/tests/GuestJsonInitializationExceptions.cpp index 2b4ef37c76..10e6faa6f7 100644 --- a/core/libs/tests/GuestJsonInitializationExceptions.cpp +++ b/core/libs/tests/GuestJsonInitializationExceptions.cpp @@ -17,10 +17,23 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); static void Require(bool condition, const char* message) { if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } } +#if defined(JSON2_EXCEPTION_LEGACY_SDK) +static constexpr unsigned ResourceCount = 11; +static constexpr std::size_t HeaderSize = 24; +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + static const std::uint32_t words[]{0, 0, 0, 0, 0x05000000}; + return words; +} +#else +static constexpr unsigned ResourceCount = 12; +static constexpr std::size_t HeaderSize = 32; +#endif struct Marker { const void* identity; }; struct InitializationExceptionTrace { void* allocator = nullptr; unsigned target = 0; + bool notifier = false; + unsigned notices = 0; std::vector requests; std::vector admitted; std::map live; @@ -30,7 +43,10 @@ static void* APS5_VABI AllocateExceptionTracked(void* self, std::size_t size, vo auto& trace = *static_cast(data); Require(self == trace.allocator, "Initialization must preserve allocator receiver and context"); trace.requests.push_back(size); - if (trace.requests.size() == trace.target + 1) throw Marker{&trace}; + if (trace.requests.size() == trace.target + 1) { + if (trace.notifier) return nullptr; + throw Marker{&trace}; + } void* memory = std::malloc(size); Require(memory && trace.live.emplace(memory, trace.admitted.size()).second, "Every admitted allocation requires independent ownership"); std::memset(memory, 0xa5, size); @@ -43,32 +59,39 @@ static void APS5_VABI ReleaseExceptionTracked(void* self, void* memory, void* da ++trace.releases; std::free(memory); } -static void APS5_VABI NotifyExceptionTracked(void*, int, std::size_t, void*) { Require(false, "Thrown callbacks must not synthesize null-allocation notifications"); } +static void APS5_VABI NotifyExceptionTracked(void* self, int error, std::size_t size, void* data) { + auto& trace = *static_cast(data); + Require(trace.notifier && self == trace.allocator && error == static_cast(0x80848102) && size == trace.requests.back(), + "Notifier exceptions must retain receiver, context and the rejected allocation request"); + ++trace.notices; + throw Marker{&trace}; +} static void* const Table[]{nullptr, nullptr, reinterpret_cast(AllocateExceptionTracked), reinterpret_cast(ReleaseExceptionTracked), reinterpret_cast(NotifyExceptionTracked)}; struct ExceptionAllocator { void* const* table = Table; }; int main(int argc, char** argv) { - Require(argc == 2, "Each exception schedule requires an independent module process"); + Require(argc == 3, "Each exception schedule requires an independent module process"); const auto target = static_cast(std::strtoul(argv[1], nullptr, 10)); - Require(target <= 12, "Allocation schedule must include every initialization position and success"); - ExceptionAllocator allocator; InitializationExceptionTrace trace; trace.allocator = &allocator; trace.target = target; + Require(target <= ResourceCount, "Allocation schedule must include every initialization position and success"); + ExceptionAllocator allocator; InitializationExceptionTrace trace; trace.allocator = &allocator; trace.target = target; trace.notifier = std::strcmp(argv[2], "notify") == 0; + Require(trace.notifier || std::strcmp(argv[2], "allocate") == 0, "Each schedule requires an allocation or notifier exception"); Parameter parameter{&allocator, &trace, 16384, 0, {}}; std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); bool caught = false; try { - Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0, "Complete admission must initialize normally"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == (trace.notifier && !target ? static_cast(0x80848102) : 0), "Only an unpublished null context may return OOM without notifying"); } catch (const Marker& marker) { Require(marker.identity == &trace, "Allocator exceptions must propagate with their original identity"); caught = true; } - Require(caught == (target < 12) && owner == unsigned(target == 12) && trace.releases == 0 && trace.live.size() == target, + Require(caught == (target < ResourceCount && (!trace.notifier || target)) && owner == unsigned(target == ResourceCount) && trace.releases == 0 && trace.live.size() == target && trace.notices == unsigned(trace.notifier && target && target < ResourceCount), "Thrown initialization callbacks must retain every prior allocation without publishing an active owner or rolling back"); - const std::array sizes{24, 8, 24, 8, 32, 48, 8, 32, 104, 32, 8, 8}; - const auto count = target < 12 ? target + 1 : 12; + const std::array sizes{24, 8, 24, 8, HeaderSize, 48, 8, HeaderSize, 104, 32, 8, 8}; + const auto count = target < ResourceCount ? target + 1 : ResourceCount; Require(trace.requests == std::vector(sizes.begin(), sizes.begin() + count), "Exception schedules must cross the exact initialization resource sequence"); const auto live = trace.live; const auto requests = trace.requests; const int notInitialized = static_cast(0x80848110), alreadyInitialized = static_cast(0x80848111); - Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == (caught ? notInitialized : 0), "An inactive owner must not reclaim the published partial context"); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == (target < ResourceCount ? notInitialized : 0), "An inactive owner must not reclaim the published partial context"); trace.target = 1000; if (target && caught) { std::uint8_t other = 0; _ZN3sce4Json11InitializerC1Ev(&other); @@ -80,8 +103,8 @@ int main(int argc, char** argv) { for (const auto& [pointer, generation] : trace.live) std::free(pointer); trace.live.clear(); } else { - Require(trace.live.empty() && trace.releases == (target == 12 ? 12u : 0u), "Normal termination must reclaim all twelve resources"); - Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0 && owner && trace.live.size() == 12, + Require(trace.live.empty() && trace.releases == (target == ResourceCount ? 12u : 0u), "Normal termination must reclaim all twelve resources"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0 && owner && trace.live.size() == ResourceCount, "Unpublished exceptions and completed lifecycles must allow a fresh initialization"); Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.live.empty(), "Fresh initialization must retain complete teardown"); } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 66926967c8..3d16a4f508 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -95,4 +95,4 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Composite callback reinitialization](../../../core/libs/prx/libSceJson2/Export.cpp) - repeated nested reinitialization and thrown callbacks remain unverified. Five hundred sixty-four SDK 5/6 reference cases cover both destructor aliases, every release position and shared, changed-context or changed-receiver/context domains. The already-entered free keeps its original receiver/context; subsequent releases use the newly published allocator even for old allocations. Reinitialization admits a complete fresh module epoch and lets auxiliary/map cleanup finish. Tests verify release identities/order, domain routing, complete payload cleanup, repeated destruction and later module retry. The pooled backend records cross-domain dispatch; it does not establish safe transfer between independent allocator pools. -- [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - older SDKs, notifier/free exceptions and guest exception integration remain unverified. Thirteen isolated SDK 6 reference-backed cases cover each allocation position and success. Throws propagate with their original identity and no frees. A first-allocation throw leaves retry available; later throws retain admitted resources, leave the owner inactive, reject termination as not initialized and reject either owner's retry as already initialized. The fixture separately reclaims orphaned storage and its host mutex after checking library behavior. Null-return rollback and explicit kernel setup error cleanup remain separate. The pre-fix implementation fails the no-rollback assertion; normal admission, termination and retry remain covered. +- [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - free exceptions and guest exception integration remain unverified. Fifty isolated SDK 5/6 reference-backed cases cover allocator/notifier throws at each resource position and success. Throws propagate with their original identity and no frees. A first context allocation returning null reports OOM without notifying; a first allocator throw propagates. Both permit retry. Later throws retain admitted resources, leave owners inactive and reject termination/retry. Checks cover resource sizes, receiver/context, exact notifier error/size and both owners. The fixture separately reclaims orphaned storage and any host mutex. SDK 5 normal teardown forwards its observed null free. Null-return rollback and kernel setup error cleanup remain covered. Pre-fix rollback and swallowed-notifier mutations fail the regressions. From ca31eed4976d418027041d24045f2ad65df15f12 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:46:18 +0200 Subject: [PATCH 076/318] test(libSceJson2): verify interrupted termination ownership --- core/libs/prx/libSceJson2/CMakeLists.txt | 30 ++++++ .../tests/GuestJsonTerminationExceptions.cpp | 96 +++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 core/libs/tests/GuestJsonTerminationExceptions.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 00aed88fd8..f28fd60007 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,36 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_termination_exceptions_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonTerminationExceptions.cpp) + target_include_directories(guest_json_termination_exceptions_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_termination_exceptions_tests PRIVATE libSceJson2 libkernel libc) + configure_windows_unwind(guest_json_termination_exceptions_tests) + add_test_executable(guest_json_legacy_termination_exceptions_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonTerminationExceptions.cpp Export.cpp) + target_compile_definitions(guest_json_legacy_termination_exceptions_tests PRIVATE JSON2_TERMINATION_LEGACY_SDK) + target_include_directories(guest_json_legacy_termination_exceptions_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_legacy_termination_exceptions_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_legacy_termination_exceptions_tests) + foreach(flavor modern legacy) + if(flavor STREQUAL "legacy") + set(exception_target guest_json_legacy_termination_exceptions_tests) + else() + set(exception_target guest_json_termination_exceptions_tests) + endif() + add_test(NAME guest_json_${flavor}_termination_healthy COMMAND ${exception_target} 0 before) + foreach(position RANGE 1 12) + foreach(boundary before after) + add_test(NAME guest_json_${flavor}_termination_exception_${boundary}_${position} COMMAND ${exception_target} ${position} ${boundary}) + if(WIN32 AND flavor STREQUAL "legacy") + set_tests_properties(guest_json_${flavor}_termination_exception_${boundary}_${position} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() + endforeach() + if(WIN32 AND flavor STREQUAL "legacy") + set_tests_properties(guest_json_${flavor}_termination_healthy PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() add_test_executable(guest_json_initialization_exceptions_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonInitializationExceptions.cpp) target_include_directories(guest_json_initialization_exceptions_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_initialization_exceptions_tests PRIVATE libSceJson2 libkernel libc) diff --git a/core/libs/tests/GuestJsonTerminationExceptions.cpp b/core/libs/tests/GuestJsonTerminationExceptions.cpp new file mode 100644 index 0000000000..b55fdb23c1 --- /dev/null +++ b/core/libs/tests/GuestJsonTerminationExceptions.cpp @@ -0,0 +1,96 @@ +#include "prx/libkernel/Pthread/include/Mutex.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +} +#if defined(JSON2_TERMINATION_LEGACY_SDK) +static constexpr bool Legacy = true; +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + static const std::uint32_t words[]{0, 0, 0, 0, 0x05000000}; + return words; +} +#else +static constexpr bool Legacy = false; +#endif +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct TerminationMarker { const void* identity; }; +struct TerminationTrace { + void* allocator = nullptr; + unsigned target = 0; + bool after = false; + std::vector allocations; + std::map live; + std::vector releases; +}; +static void* APS5_VABI AllocateTerminationTracked(void* self, std::size_t size, void* data) { + auto& trace = *static_cast(data); + Require(self == trace.allocator, "Initialization must preserve the allocator receiver and context"); + void* memory = std::malloc(size); + Require(memory && trace.live.emplace(memory, trace.allocations.size() + 1).second, "Each initialization resource must have an independent live identity"); + std::memset(memory, 0xa5, size); + trace.allocations.push_back(memory); + return memory; +} +static void APS5_VABI ReleaseTerminationTracked(void* self, void* memory, void* data) { + auto& trace = *static_cast(data); + Require(self == trace.allocator && (!memory || trace.live.contains(memory)), "Teardown must preserve allocator identity and release only live resources"); + trace.releases.push_back(memory ? trace.live.at(memory) : 0); + if (trace.releases.size() == trace.target && !trace.after) throw TerminationMarker{&trace}; + if (memory) { trace.live.erase(memory); std::free(memory); } + if (trace.releases.size() == trace.target && trace.after) throw TerminationMarker{&trace}; +} +static void APS5_VABI NotifyTerminationTracked(void*, int, std::size_t, void*) { Require(false, "Release exceptions must not fabricate allocation errors"); } +static void* const Table[]{nullptr, nullptr, reinterpret_cast(AllocateTerminationTracked), reinterpret_cast(ReleaseTerminationTracked), reinterpret_cast(NotifyTerminationTracked)}; +struct TerminationAllocator { void* const* table = Table; }; +int main(int argc, char** argv) { + Require(argc == 3, "Every interrupted teardown requires an independent module process"); + TerminationAllocator allocator; TerminationTrace trace; trace.allocator = &allocator; + trace.target = static_cast(std::strtoul(argv[1], nullptr, 10)); trace.after = std::strcmp(argv[2], "after") == 0; + Require(trace.target <= 12 && (trace.after || std::strcmp(argv[2], "before") == 0), "The schedule must identify a release boundary"); + Parameter parameter{&allocator, &trace, 16384, 0, {}}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0 && owner && trace.live.size() == (Legacy ? 11u : 12u), "Initialize complete module ownership before release failure"); + const auto baseline = trace.live; + bool caught = false; + try { Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0, "Uninterrupted teardown must complete"); } + catch (const TerminationMarker& marker) { Require(marker.identity == &trace, "Free callback exceptions must preserve their identity"); caught = true; } + const std::array order = Legacy ? std::array{0, 10, 11, 3, 2, 6, 5, 4, 9, 8, 7, 1} + : std::array{10, 12, 11, 3, 2, 6, 5, 4, 9, 8, 7, 1}; + const unsigned calls = trace.target ? trace.target : 12; + Require(caught == bool(trace.target) && owner == unsigned(caught) && trace.releases == std::vector(order.begin(), order.begin() + calls), + "Interrupted teardown must retain the active owner and exact release-generation prefix"); + auto expected = baseline; + const unsigned completed = caught ? calls - unsigned(!trace.after) : calls; + for (unsigned i = 0; i < completed; ++i) if (order[i]) expected.erase(trace.allocations[order[i] - 1]); + Require(trace.live == expected, "Only frees completed before the exception may change resource ownership"); + const auto releases = trace.releases; + std::uint8_t other = 0; _ZN3sce4Json11InitializerC1Ev(&other); + if (caught) { + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == static_cast(0x80848111) && + _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, ¶meter) == static_cast(0x80848111) && + !other && trace.live == expected && trace.releases == releases, + "Both owners must reject initialization after incomplete teardown without further allocator effects"); + if (!Legacy && trace.target < 2) Require(scePthreadMutexDestroy(static_cast(trace.allocations[11])) == 0, "The fixture must separately reclaim the untouched recovery mutex"); + if (trace.target < 3) Require(scePthreadMutexDestroy(static_cast(trace.allocations[10])) == 0, "The fixture must separately reclaim the untouched internal mutex"); + for (const auto& [pointer, generation] : trace.live) std::free(pointer); + trace.live.clear(); + } else { + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == static_cast(0x80848110) && trace.releases == releases, "Completed teardown must remain inactive without repeated frees"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, ¶meter) == 0 && other, "Completed teardown must permit a new module generation"); + Require(_ZN3sce4Json11Initializer9terminateEv(&other) == 0 && !other && trace.live.empty(), "A fresh generation must retain complete cleanup"); + } + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 3d16a4f508..ac900735d1 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -95,4 +95,6 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Composite callback reinitialization](../../../core/libs/prx/libSceJson2/Export.cpp) - repeated nested reinitialization and thrown callbacks remain unverified. Five hundred sixty-four SDK 5/6 reference cases cover both destructor aliases, every release position and shared, changed-context or changed-receiver/context domains. The already-entered free keeps its original receiver/context; subsequent releases use the newly published allocator even for old allocations. Reinitialization admits a complete fresh module epoch and lets auxiliary/map cleanup finish. Tests verify release identities/order, domain routing, complete payload cleanup, repeated destruction and later module retry. The pooled backend records cross-domain dispatch; it does not establish safe transfer between independent allocator pools. -- [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - free exceptions and guest exception integration remain unverified. Fifty isolated SDK 5/6 reference-backed cases cover allocator/notifier throws at each resource position and success. Throws propagate with their original identity and no frees. A first context allocation returning null reports OOM without notifying; a first allocator throw propagates. Both permit retry. Later throws retain admitted resources, leave owners inactive and reject termination/retry. Checks cover resource sizes, receiver/context, exact notifier error/size and both owners. The fixture separately reclaims orphaned storage and any host mutex. SDK 5 normal teardown forwards its observed null free. Null-return rollback and kernel setup error cleanup remain covered. Pre-fix rollback and swallowed-notifier mutations fail the regressions. +- [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - rollback free exceptions and guest exception integration remain unverified. Fifty isolated SDK 5/6 reference-backed cases cover allocator/notifier throws at each resource position and success. Throws propagate with their original identity and no frees. A first context allocation returning null reports OOM without notifying; a first allocator throw propagates. Both permit retry. Later throws retain admitted resources, leave owners inactive and reject termination/retry. Checks cover resource sizes, receiver/context, exact notifier error/size and both owners. The fixture separately reclaims orphaned storage and any host mutex. SDK 5 normal teardown forwards its observed null free. Null-return rollback and kernel setup error cleanup remain covered. Pre-fix rollback and swallowed-notifier mutations fail the regressions. + +- [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - retry after interrupted teardown, nested cleanup and guest exception integration remain incomplete. Fifty isolated SDK 5/6 reference-backed cases cover throws before/after each of twelve free callbacks and healthy controls. Exceptions retain their identity, the owner stays active, completed frees match the exact generation prefix and either owner rejects initialization without allocator effects. Checks include the legacy null free and independently reclaimed fixture storage/untouched host mutexes. Early owner clearing is detected. Separate retry probes include native faults at the substituted kernel boundary and requests for already released resources; those outcomes are not returning reference contracts or console measurements. Returning retry schedules and safe refusal before invalid accesses still require production coverage. From b6ec65e7d044b75801bda65498effab3dff78186 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:48:48 +0200 Subject: [PATCH 077/318] fix(libSceJson2): preserve null frees on interrupted teardown retry --- core/libs/prx/libSceJson2/CMakeLists.txt | 15 ++++++++++ core/libs/prx/libSceJson2/Export.cpp | 2 +- .../tests/GuestJsonTerminationExceptions.cpp | 28 ++++++++++++++++++- docs/dev/debt/libSceJson2.md | 2 +- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index f28fd60007..08ca91e5da 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -29,6 +29,21 @@ if(BUILD_TESTING) endif() endforeach() endforeach() + foreach(position 1 5 6 8 9 11 12) + add_test(NAME guest_json_${flavor}_termination_retry_${position} COMMAND ${exception_target} ${position} before retry) + if(WIN32 AND flavor STREQUAL "legacy") + set_tests_properties(guest_json_${flavor}_termination_retry_${position} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() + if(flavor STREQUAL "legacy") + add_test(NAME guest_json_legacy_termination_retry_2 COMMAND ${exception_target} 2 before retry) + add_test(NAME guest_json_legacy_termination_retry_null_after COMMAND ${exception_target} 1 after retry) + if(WIN32) + set_tests_properties(guest_json_legacy_termination_retry_2 guest_json_legacy_termination_retry_null_after PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endif() if(WIN32 AND flavor STREQUAL "legacy") set_tests_properties(guest_json_${flavor}_termination_healthy PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$") diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index ddeafcf9a3..803d6f6e51 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1435,7 +1435,7 @@ void ResetDefaults(bool releaseMissing = false) { const bool missing = type >= TypeString && !defaults.values[type].payload; Destroy(defaults.values[type]); if (releaseMissing && missing) Release(nullptr); - } + } else if (releaseMissing && type >= TypeString) Release(nullptr); defaults.values[type] = {}; defaults.ready[type] = false; } diff --git a/core/libs/tests/GuestJsonTerminationExceptions.cpp b/core/libs/tests/GuestJsonTerminationExceptions.cpp index b55fdb23c1..44d488de53 100644 --- a/core/libs/tests/GuestJsonTerminationExceptions.cpp +++ b/core/libs/tests/GuestJsonTerminationExceptions.cpp @@ -56,7 +56,7 @@ static void APS5_VABI NotifyTerminationTracked(void*, int, std::size_t, void*) { static void* const Table[]{nullptr, nullptr, reinterpret_cast(AllocateTerminationTracked), reinterpret_cast(ReleaseTerminationTracked), reinterpret_cast(NotifyTerminationTracked)}; struct TerminationAllocator { void* const* table = Table; }; int main(int argc, char** argv) { - Require(argc == 3, "Every interrupted teardown requires an independent module process"); + Require(argc == 3 || argc == 4, "Every interrupted teardown requires an independent module process"); TerminationAllocator allocator; TerminationTrace trace; trace.allocator = &allocator; trace.target = static_cast(std::strtoul(argv[1], nullptr, 10)); trace.after = std::strcmp(argv[2], "after") == 0; Require(trace.target <= 12 && (trace.after || std::strcmp(argv[2], "before") == 0), "The schedule must identify a release boundary"); @@ -83,6 +83,32 @@ int main(int argc, char** argv) { _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, ¶meter) == static_cast(0x80848111) && !other && trace.live == expected && trace.releases == releases, "Both owners must reject initialization after incomplete teardown without further allocator effects"); + if (argc == 4) { + Require(std::strcmp(argv[3], "retry") == 0, "The additional schedule must request a returning reference retry"); + std::vector tail; + switch (trace.target) { + case 1: Require(!trace.after || Legacy, "Only the legacy null-free after-throw retry returns"); tail.assign(order.begin(), order.end()); break; + case 2: Require(Legacy && !trace.after, "Only the legacy fallback before-throw retry returns"); tail.assign(order.begin(), order.end()); break; + case 5: tail = {0, 0, 0, 2, 6, 5, 4, 9, 8, 7, 1}; break; + case 6: tail = {0, 0, 0, 0, 6, 5, 4, 9, 8, 7, 1}; break; + case 8: tail = {0, 0, 0, 0, 4, 9, 8, 7, 1}; break; + case 9: tail = {0, 0, 0, 0, 0, 9, 8, 7, 1}; break; + case 11: tail = {0, 0, 0, 0, 0, 7, 1}; break; + case 12: tail = {0, 0, 0, 0, 0, 0, 1}; break; + default: Require(false, "This retry has no returning reference contract"); + } + Require(!trace.after || (Legacy && trace.target == 1), "Returning resource retries must precede the rejected release"); + auto expectedRetry = releases; expectedRetry.insert(expectedRetry.end(), tail.begin(), tail.end()); + trace.target = 0; + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.live.empty() && trace.releases == expectedRetry, + "Returning teardown retry must reclaim the retained resources in exact reference order including null frees"); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == static_cast(0x80848110) && trace.releases == expectedRetry, + "Completed retry must not repeat any release"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, ¶meter) == 0 && other, + "Completed retry must permit a fresh owner generation"); + Require(_ZN3sce4Json11Initializer9terminateEv(&other) == 0 && !other && trace.live.empty(), "Fresh ownership must retain complete cleanup after interrupted retry"); + return 0; + } if (!Legacy && trace.target < 2) Require(scePthreadMutexDestroy(static_cast(trace.allocations[11])) == 0, "The fixture must separately reclaim the untouched recovery mutex"); if (trace.target < 3) Require(scePthreadMutexDestroy(static_cast(trace.allocations[10])) == 0, "The fixture must separately reclaim the untouched internal mutex"); for (const auto& [pointer, generation] : trace.live) std::free(pointer); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index ac900735d1..f531ac6d5a 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -97,4 +97,4 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - rollback free exceptions and guest exception integration remain unverified. Fifty isolated SDK 5/6 reference-backed cases cover allocator/notifier throws at each resource position and success. Throws propagate with their original identity and no frees. A first context allocation returning null reports OOM without notifying; a first allocator throw propagates. Both permit retry. Later throws retain admitted resources, leave owners inactive and reject termination/retry. Checks cover resource sizes, receiver/context, exact notifier error/size and both owners. The fixture separately reclaims orphaned storage and any host mutex. SDK 5 normal teardown forwards its observed null free. Null-return rollback and kernel setup error cleanup remain covered. Pre-fix rollback and swallowed-notifier mutations fail the regressions. -- [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - retry after interrupted teardown, nested cleanup and guest exception integration remain incomplete. Fifty isolated SDK 5/6 reference-backed cases cover throws before/after each of twelve free callbacks and healthy controls. Exceptions retain their identity, the owner stays active, completed frees match the exact generation prefix and either owner rejects initialization without allocator effects. Checks include the legacy null free and independently reclaimed fixture storage/untouched host mutexes. Early owner clearing is detected. Separate retry probes include native faults at the substituted kernel boundary and requests for already released resources; those outcomes are not returning reference contracts or console measurements. Returning retry schedules and safe refusal before invalid accesses still require production coverage. +- [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - faulting teardown retries, nested cleanup and guest exception integration remain incomplete. Fifty SDK 5/6 reference cases cover throws before/after twelve frees and healthy controls. Exceptions preserve identity, owner activity, exact release prefixes and initialization refusal. Sixteen returning retry cases cover the null-free sequence, retained generations, inactive repeats and fresh owners. Completed defaults still forward null frees during retry; the pre-fix `ready` gate skipped them and fails the regression. The fixture separately reclaims orphaned resources only in nonretry cases. Separate probes faulted at substituted kernel boundaries or requested already released resources; these are not returning contracts or console measurements. Safe refusal before invalid retry accesses remains required. From 649bed0946a870e0c6e6b74aa24a48a2696411c7 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:53:59 +0200 Subject: [PATCH 078/318] fix(libSceJson2): refuse retries through retired mutex slots --- core/libs/prx/libSceJson2/CMakeLists.txt | 14 ++++++++++++++ core/libs/prx/libSceJson2/Export.cpp | 15 +++++++++++++-- .../tests/GuestJsonTerminationExceptions.cpp | 17 ++++++++++++++++- docs/dev/debt/libSceJson2.md | 2 +- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 08ca91e5da..d84fb77989 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -29,6 +29,20 @@ if(BUILD_TESTING) endif() endforeach() endforeach() + if(flavor STREQUAL "legacy") + set(retired_positions 3) + else() + set(retired_positions 2 3) + endif() + foreach(position IN LISTS retired_positions) + foreach(boundary before after) + add_test(NAME guest_json_${flavor}_termination_retired_mutex_${boundary}_${position} COMMAND ${exception_target} ${position} ${boundary} refuse-mutex) + if(WIN32 AND flavor STREQUAL "legacy") + set_tests_properties(guest_json_${flavor}_termination_retired_mutex_${boundary}_${position} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() + endforeach() foreach(position 1 5 6 8 9 11 12) add_test(NAME guest_json_${flavor}_termination_retry_${position} COMMAND ${exception_target} ${position} before retry) if(WIN32 AND flavor STREQUAL "legacy") diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 803d6f6e51..c3947dba6f 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -156,6 +156,8 @@ struct Globals { LegacyHandlerMap* legacyHandlers = nullptr; PthreadMutex* internalMutex = nullptr; PthreadMutex* recoveryMutex = nullptr; + bool internalMutexRetired = false; + bool recoveryMutexRetired = false; std::recursive_mutex callbackMutex; bool callbackActive = false; std::uint32_t allocationDepth = 0; @@ -1472,13 +1474,21 @@ void ReleaseInitialization() { Release(state.fallback); state.fallback = nullptr; if (ModernValues()) { - if (state.recoveryMutex) scePthreadMutexDestroy(state.recoveryMutex); + if (state.recoveryMutex) { + scePthreadMutexDestroy(state.recoveryMutex); + state.recoveryMutexRetired = true; + } Release(state.recoveryMutex); } state.recoveryMutex = nullptr; - if (state.internalMutex) scePthreadMutexDestroy(state.internalMutex); + state.recoveryMutexRetired = false; + if (state.internalMutex) { + scePthreadMutexDestroy(state.internalMutex); + state.internalMutexRetired = true; + } Release(state.internalMutex); state.internalMutex = nullptr; + state.internalMutexRetired = false; ResetDefaults(true); if (state.allocation) { auto* allocation = state.allocation; @@ -2568,6 +2578,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; + if (state.internalMutexRetired || state.recoveryMutexRetired) NotImplemented_nid_no_patch(__func__); if (state.terminating || state.callbackActive || (allocatorCallbackDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); TerminationScope termination(state.terminating); state.callback = nullptr; diff --git a/core/libs/tests/GuestJsonTerminationExceptions.cpp b/core/libs/tests/GuestJsonTerminationExceptions.cpp index 44d488de53..1747da7595 100644 --- a/core/libs/tests/GuestJsonTerminationExceptions.cpp +++ b/core/libs/tests/GuestJsonTerminationExceptions.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; @@ -83,7 +85,20 @@ int main(int argc, char** argv) { _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, ¶meter) == static_cast(0x80848111) && !other && trace.live == expected && trace.releases == releases, "Both owners must reject initialization after incomplete teardown without further allocator effects"); - if (argc == 4) { + if (argc == 4 && std::strcmp(argv[3], "refuse-mutex") == 0) { + Require((Legacy && trace.target == 3) || (!Legacy && (trace.target == 2 || trace.target == 3)), "Refusal probes require a destroyed mutex slot"); + trace.target = 0; + for (unsigned attempt = 0; attempt < 2; ++attempt) { + bool refused = false; + try { _ZN3sce4Json11Initializer9terminateEv(&owner); } + catch (const std::runtime_error& error) { + refused = std::string(error.what()).find("_ZN3sce4Json11Initializer9terminateEv") != std::string::npos; + } + Require(refused && owner && trace.live == expected && trace.releases == releases, + "Retry must explicitly refuse destroyed mutex ownership before kernel use or further release effects"); + } + trace.target = calls; + } else if (argc == 4) { Require(std::strcmp(argv[3], "retry") == 0, "The additional schedule must request a returning reference retry"); std::vector tail; switch (trace.target) { diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index f531ac6d5a..0212869907 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -97,4 +97,4 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - rollback free exceptions and guest exception integration remain unverified. Fifty isolated SDK 5/6 reference-backed cases cover allocator/notifier throws at each resource position and success. Throws propagate with their original identity and no frees. A first context allocation returning null reports OOM without notifying; a first allocator throw propagates. Both permit retry. Later throws retain admitted resources, leave owners inactive and reject termination/retry. Checks cover resource sizes, receiver/context, exact notifier error/size and both owners. The fixture separately reclaims orphaned storage and any host mutex. SDK 5 normal teardown forwards its observed null free. Null-return rollback and kernel setup error cleanup remain covered. Pre-fix rollback and swallowed-notifier mutations fail the regressions. -- [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - faulting teardown retries, nested cleanup and guest exception integration remain incomplete. Fifty SDK 5/6 reference cases cover throws before/after twelve frees and healthy controls. Exceptions preserve identity, owner activity, exact release prefixes and initialization refusal. Sixteen returning retry cases cover the null-free sequence, retained generations, inactive repeats and fresh owners. Completed defaults still forward null frees during retry; the pre-fix `ready` gate skipped them and fails the regression. The fixture separately reclaims orphaned resources only in nonretry cases. Separate probes faulted at substituted kernel boundaries or requested already released resources; these are not returning contracts or console measurements. Safe refusal before invalid retry accesses remains required. +- [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - nested cleanup, non-mutex invalid retries and guest exception integration remain incomplete. Fifty SDK 5/6 reference cases cover throws around twelve frees and healthy controls, preserving exception identity, owner activity, release prefixes and initialization refusal. Sixteen returning retries cover null frees, retained generations and fresh owners. Completed defaults forward null frees during retry; the pre-fix `ready` gate fails this regression. Six production cases reject destroyed mutex slots before retry effects. Retirement survives a free callback exception and resets only after slot release returns. Separate probes faulted at substituted kernel boundaries or requested released resources; these are not returning contracts or console measurements. Fixture reclamation is separate from library cleanup. From b41af1f2bd3cbbaac9fe24e6ddeeceaf248018b0 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 18:56:57 +0200 Subject: [PATCH 079/318] fix(libSceJson2): refuse retries through retired default headers --- core/libs/prx/libSceJson2/CMakeLists.txt | 9 ++++++++ core/libs/prx/libSceJson2/Export.cpp | 21 ++++++++++++++++++- .../tests/GuestJsonTerminationExceptions.cpp | 7 ++++--- docs/dev/debt/libSceJson2.md | 2 +- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index d84fb77989..bc7cea971c 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -43,6 +43,15 @@ if(BUILD_TESTING) endif() endforeach() endforeach() + foreach(position 4 7 10) + foreach(boundary before after) + add_test(NAME guest_json_${flavor}_termination_retired_header_${boundary}_${position} COMMAND ${exception_target} ${position} ${boundary} refuse-header) + if(WIN32 AND flavor STREQUAL "legacy") + set_tests_properties(guest_json_${flavor}_termination_retired_header_${boundary}_${position} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() + endforeach() foreach(position 1 5 6 8 9 11 12) add_test(NAME guest_json_${flavor}_termination_retry_${position} COMMAND ${exception_target} ${position} before retry) if(WIN32 AND flavor STREQUAL "legacy") diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index c3947dba6f..f5edc5a292 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -158,6 +158,7 @@ struct Globals { PthreadMutex* recoveryMutex = nullptr; bool internalMutexRetired = false; bool recoveryMutexRetired = false; + std::uint8_t retiredDefaultHeaders = 0; std::recursive_mutex callbackMutex; bool callbackActive = false; std::uint32_t allocationDepth = 0; @@ -1073,30 +1074,38 @@ void CopyObjectStorage(ObjectStorage& destination, const ObjectStorage& source) void Destroy(Value& v); void CopyInto(Value& destination, const Value& source); +void SetDefaultHeaderRetired(ValueType type, const void* owner, bool retired); + void Destroy(String& s) { if (!s.text) return; ClearText(s); + SetDefaultHeaderRetired(TypeString, &s, true); PoisonText(*s.text); Release(s.text); s.text = nullptr; + SetDefaultHeaderRetired(TypeString, &s, false); } void Destroy(Array& a) { if (!a.items) return; a.items->ReleaseContents(); if (a.items->head) Release(a.items->head); + SetDefaultHeaderRetired(TypeArray, &a, true); a.items->head = reinterpret_cast(std::uintptr_t{0xffffffff}); a.items->count = std::numeric_limits::max(); Release(a.items); a.items = nullptr; + SetDefaultHeaderRetired(TypeArray, &a, false); } void Destroy(Object& object) { if (!object.items) return; object.items->clear(); if (object.items->head) Release(object.items->head); + SetDefaultHeaderRetired(TypeObject, &object, true); object.items->head = reinterpret_cast(std::uintptr_t{0xffffffff}); object.items->count = std::numeric_limits::max(); Release(object.items); object.items = nullptr; + SetDefaultHeaderRetired(TypeObject, &object, false); } const Node& RootOf(const Node& n) { @@ -1417,6 +1426,16 @@ DefaultStorage& SharedDefaults() { return defaults; } +void SetDefaultHeaderRetired(ValueType type, const void* owner, bool retired) { + auto& state = State(); + std::lock_guard lock(state.mutex); + auto& defaults = SharedDefaults(); + if (!defaults.ready[type] || defaults.values[type].payload != reinterpret_cast(owner)) return; + const auto mask = static_cast(1u << type); + if (retired) state.retiredDefaultHeaders |= mask; + else state.retiredDefaultHeaders &= static_cast(~mask); +} + const Value& Defaults(ValueType type) { auto& state = State(); std::lock_guard lock(state.mutex); @@ -2578,7 +2597,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { std::lock_guard callbackLock(state.callbackMutex); std::lock_guard lock(state.mutex); if (!state.initialized || !*static_cast(self)) return JsonErrorNotInitialized; - if (state.internalMutexRetired || state.recoveryMutexRetired) NotImplemented_nid_no_patch(__func__); + if (state.internalMutexRetired || state.recoveryMutexRetired || state.retiredDefaultHeaders) NotImplemented_nid_no_patch(__func__); if (state.terminating || state.callbackActive || (allocatorCallbackDepth && state.allocationDepth)) NotImplemented_nid_no_patch(__func__); TerminationScope termination(state.terminating); state.callback = nullptr; diff --git a/core/libs/tests/GuestJsonTerminationExceptions.cpp b/core/libs/tests/GuestJsonTerminationExceptions.cpp index 1747da7595..0d3f29a7e8 100644 --- a/core/libs/tests/GuestJsonTerminationExceptions.cpp +++ b/core/libs/tests/GuestJsonTerminationExceptions.cpp @@ -85,8 +85,9 @@ int main(int argc, char** argv) { _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, ¶meter) == static_cast(0x80848111) && !other && trace.live == expected && trace.releases == releases, "Both owners must reject initialization after incomplete teardown without further allocator effects"); - if (argc == 4 && std::strcmp(argv[3], "refuse-mutex") == 0) { - Require((Legacy && trace.target == 3) || (!Legacy && (trace.target == 2 || trace.target == 3)), "Refusal probes require a destroyed mutex slot"); + if (argc == 4 && (std::strcmp(argv[3], "refuse-mutex") == 0 || std::strcmp(argv[3], "refuse-header") == 0)) { + if (std::strcmp(argv[3], "refuse-header") == 0) Require(trace.target == 4 || trace.target == 7 || trace.target == 10, "Header refusal requires completed child destruction"); + else Require((Legacy && trace.target == 3) || (!Legacy && (trace.target == 2 || trace.target == 3)), "Refusal probes require a destroyed mutex slot"); trace.target = 0; for (unsigned attempt = 0; attempt < 2; ++attempt) { bool refused = false; @@ -95,7 +96,7 @@ int main(int argc, char** argv) { refused = std::string(error.what()).find("_ZN3sce4Json11Initializer9terminateEv") != std::string::npos; } Require(refused && owner && trace.live == expected && trace.releases == releases, - "Retry must explicitly refuse destroyed mutex ownership before kernel use or further release effects"); + "Retry must explicitly refuse retired ownership before kernel use or further release effects"); } trace.target = calls; } else if (argc == 4) { diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 0212869907..a833b635be 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -97,4 +97,4 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - rollback free exceptions and guest exception integration remain unverified. Fifty isolated SDK 5/6 reference-backed cases cover allocator/notifier throws at each resource position and success. Throws propagate with their original identity and no frees. A first context allocation returning null reports OOM without notifying; a first allocator throw propagates. Both permit retry. Later throws retain admitted resources, leave owners inactive and reject termination/retry. Checks cover resource sizes, receiver/context, exact notifier error/size and both owners. The fixture separately reclaims orphaned storage and any host mutex. SDK 5 normal teardown forwards its observed null free. Null-return rollback and kernel setup error cleanup remain covered. Pre-fix rollback and swallowed-notifier mutations fail the regressions. -- [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - nested cleanup, non-mutex invalid retries and guest exception integration remain incomplete. Fifty SDK 5/6 reference cases cover throws around twelve frees and healthy controls, preserving exception identity, owner activity, release prefixes and initialization refusal. Sixteen returning retries cover null frees, retained generations and fresh owners. Completed defaults forward null frees during retry; the pre-fix `ready` gate fails this regression. Six production cases reject destroyed mutex slots before retry effects. Retirement survives a free callback exception and resets only after slot release returns. Separate probes faulted at substituted kernel boundaries or requested released resources; these are not returning contracts or console measurements. Fixture reclamation is separate from library cleanup. +- [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - nested cleanup, callbacks freeing then throwing at other stages and guest exception integration remain incomplete. Fifty SDK 5/6 reference cases cover throws around twelve frees and healthy controls, preserving exception identity, owner activity, release prefixes and initialization refusal. Sixteen returning retries cover null frees, retained generations and fresh owners. Completed defaults forward null frees during retry; the old `ready` gate fails the regression. Eighteen production cases reject destroyed mutex slots or poisoned default inner headers before retry effects. Retirement survives callback exceptions and clears only after the corresponding release returns. Outer-handle retries remain available. Separate probes faulted at substituted kernel boundaries or requested released resources; these are not returning contracts or console measurements. Fixture reclamation is separate. From 2414406f60f8465780dc8e706cacd087a83363b3 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:00:20 +0200 Subject: [PATCH 080/318] fix(libSceJson2): preserve legacy map heap exception ownership --- core/libs/prx/libSceJson2/CMakeLists.txt | 7 +++ core/libs/prx/libSceJson2/Export.cpp | 3 +- core/libs/tests/GuestJsonLegacyHandlers.cpp | 63 ++++++++++++++++++++- docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index bc7cea971c..0c36d30acd 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -375,6 +375,13 @@ if(BUILD_TESTING) target_link_libraries(guest_json_legacy_handlers_tests PRIVATE libkernel libc) configure_windows_unwind(guest_json_legacy_handlers_tests) add_test(NAME guest_json_legacy_handlers COMMAND guest_json_legacy_handlers_tests) + foreach(position RANGE 0 2) + add_test(NAME guest_json_legacy_heap_exception_${position} COMMAND guest_json_legacy_handlers_tests ${position}) + if(WIN32) + set_tests_properties(guest_json_legacy_heap_exception_${position} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() if(WIN32) set_tests_properties(guest_json_legacy_handlers PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$") diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index f5edc5a292..bfc30866bf 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -979,8 +979,7 @@ int RegisterLegacyHandler(Value& value, TypeMismatchHandler callback, void* cont auto* map = static_cast(Allocate(sizeof(LegacyHandlerMap))); if (!map) return JsonErrorOutOfMemory; map->count = 0; - try { map->head = static_cast(_Znwm_nid_postfix(sizeof(LegacyHandlerNode))); } - catch (const std::bad_alloc&) { Release(map); NotImplemented_nid_no_patch(__func__); } + map->head = static_cast(_Znwm_nid_postfix(sizeof(LegacyHandlerNode))); map->head->parent = map->head; map->head->left = map->head; map->head->right = map->head; map->head->color = 0; map->head->nil = 1; state.legacyHandlers = map; diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index ea5fb5ae69..4d2bfa793b 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -57,6 +57,7 @@ struct LegacyTrace { std::map guest; std::map heap; int budget = -1; + int heapBudget = -1; bool once = false; std::vector requests; std::map guestGenerations; @@ -72,6 +73,9 @@ extern "C" const void* ApplicationProcessParameters_nid_no_patch() { return parameters; } extern "C" void* APS5_VABI _Znwm_nid_postfix(std::size_t bytes) { + Require(HeapTrace != nullptr, "Heap allocation requires its owning trace"); + if (HeapTrace->heapBudget == 0) throw std::bad_alloc(); + if (HeapTrace->heapBudget > 0) --HeapTrace->heapBudget; void* memory = std::malloc(bytes); if (!memory) throw std::bad_alloc(); std::memset(memory, 0xa5, bytes); @@ -505,7 +509,64 @@ static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& secon "Inherited map updates and retries must reclaim every temporary generation in both allocator domains"); } } -int main() { +static int CheckHeapExceptions(unsigned budget) { + Require(budget <= 2, "Heap exceptions must cover sentinel, entry and complete admission"); + LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; + Parameter parameter{&allocator, &trace, 16384}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&owner, ¶meter) == 0, "Initialize legacy heap exception resources"); + Context context; _ZN3sce4Json5ValueC1Eb(&context.replacement, true); + Value source{}; ParseLegacy(source); + const Value sourceValue = source; + const auto baseline = trace.guestGenerations; + std::vector snapshots; + for (const auto& [pointer, generation] : baseline) { + const auto* bytes = static_cast(pointer); + snapshots.push_back({pointer, generation, {bytes, bytes + trace.guest.at(pointer)}}); + } + trace.heapBudget = static_cast(budget); + const auto requests = trace.requests.size(); + bool caught = false; + try { Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &context) == 0, "Complete registration must succeed"); } + catch (const std::bad_alloc&) { caught = true; } + Require(caught == (budget < 2) && owner && trace.heap.size() == budget && trace.guest.size() == baseline.size() + 1 && + trace.requests.size() == requests + 1 && trace.requests.back() == 24 && !trace.notifications, + "Libc exceptions must propagate without guest OOM notifications or reclaiming the admitted map header"); + void* header = nullptr; + for (const auto& [pointer, generation] : trace.guestGenerations) if (!baseline.contains(pointer)) { Require(!header, "Admission creates only one guest map header"); header = pointer; } + Require(header && trace.guest.at(header) == 24, "The unpublished header must remain owned in its guest allocator domain"); + const auto headerGeneration = trace.guestGenerations.at(header); + if (!budget) { + std::array expected; expected.fill(0xa5); std::memset(expected.data() + 8, 0, 8); + Require(std::memcmp(header, expected.data(), expected.size()) == 0 && trace.heap.empty(), "Sentinel failure must leave only the count initialized in the unpublished header"); + } else CheckMap(trace, budget == 1 ? std::map{} : std::map{{&source, &context}}); + trace.heapBudget = -1; + const auto retryRequests = trace.requests.size(); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &context) == 0 && + trace.requests.size() == retryRequests + unsigned(!budget) && trace.heap.size() == 2 && trace.guestGenerations.at(header) == headerGeneration, + "Retry must replace only an unpublished map and reuse an already published sentinel"); + CheckMap(trace, {{&source, &context}}); + const auto calls = context.calls; + Require(_ZNK3sce4Json5Value10getBooleanEv(&source) == reinterpret_cast(&context.replacement.payload) && context.calls == calls + 1 && context.source == &source && !context.parent, + "Retried callback metadata must dispatch the original context and borrowed result"); + Require(std::memcmp(&source, &sourceValue, sizeof(source)) == 0, "Heap failure and retry must preserve source metadata ownership"); + for (const auto& snapshot : snapshots) Require(trace.guestGenerations.at(snapshot.pointer) == snapshot.generation && + std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "Heap failure must preserve preexisting allocation bytes and generations"); + _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ValueD1Ev(&context.replacement); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.heap.empty() && trace.heapGenerations.empty() && trace.guest.size() == unsigned(!budget), + "Termination must reclaim published map storage while preserving the orphaned unpublished header"); + if (!budget) { + Require(trace.guestGenerations.at(header) == headerGeneration, "Library cleanup must not falsely claim orphan reclamation"); + FreeLegacy(nullptr, header, &trace); + } + Require(trace.guest.empty() && trace.guestGenerations.empty(), "The fixture separately reclaims any orphaned allocation"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&owner, ¶meter) == 0 && + _ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.guest.empty() && trace.heap.empty(), "Heap exception cleanup must permit a fresh module generation"); + HeapTrace = nullptr; + return 0; +} +int main(int argc, char** argv) { + if (argc == 2) return CheckHeapExceptions(static_cast(std::strtoul(argv[1], nullptr, 10))); LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index a833b635be..b481a281bb 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -40,7 +40,7 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [sce::Json::Initializer resource admission](../../../core/libs/prx/libSceJson2/Export.cpp) - returned kernel setup/locking errors, recovery-callback termination and active-operation allocator-callback termination remain explicitly unsupported; concurrent/reentrant lifecycle and Linux/Windows integration remain unverified. Reference tests cover SDK 5/6 allocation failures, partial admission, null frees and retry; SDK 6 tests also reject duplicate initialization and inactive termination from allocate/notify/free callbacks while the context remains published. Seventeen host-boundary schedules verify mutex-construction errors/exceptions, balanced guest/attribute/mutex ownership and retry. Setup exceptions reclaim local resources before global rollback. The reference skips creation after attribute errors and ignores mutex-init/destroy results, potentially retaining unwritten handles; safely reproducing those states remains unverified. -- [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Failure to allocate the libc sentinel throws unsupported after reclaiming the unpublished header; reference exception unwinding is unverified. +- [sce::Json::Value legacy mismatch map](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc allocation exceptions remain incomplete. Map registration, lookup, copying and destruction follow `0x3b00`, `0x7230`, `0x22b0`, `0x1e40` and `0x1810`: a parsed root must already own auxiliary metadata; a shared 24-byte map header uses the supplied allocator, while its sentinel and entries use libc new/delete. Copies propagate callback records independently of auxiliary ownership. Empty map storage survives until termination. Local SDK 5 tests cover replacement, null handlers, parent dispatch, null-recovery ordering, swap temporary cleanup, tree balance, erasure and both allocator domains. Sentinel/entry allocation exceptions and retry follow the reference contracts below; broader exception paths remain unverified. - [sce::Json::Object nested allocation regressions](../../../core/libs/prx/libSceJson2/Export.cpp) - local tests cover 322 copy-construction and assignment cases for an object containing a string, inner object/string and array. Persistent and single-allocation failure budgets exercise every allocation position through successful completion (50 allocations for construction, 109 for assignment). Reference comparisons cover retained partial contents, collection counts, parent/owner links, request counts, notifications, unchanged source state and allocator cleanup. Construction may retain a source count larger than its admitted nodes and leave copied root entries unbound; tests preserve those observed semantics. Callback-bearing copy failures, alternate SDK behavior and concurrent failure accounting remain unverified. @@ -98,3 +98,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Initialization allocator exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - rollback free exceptions and guest exception integration remain unverified. Fifty isolated SDK 5/6 reference-backed cases cover allocator/notifier throws at each resource position and success. Throws propagate with their original identity and no frees. A first context allocation returning null reports OOM without notifying; a first allocator throw propagates. Both permit retry. Later throws retain admitted resources, leave owners inactive and reject termination/retry. Checks cover resource sizes, receiver/context, exact notifier error/size and both owners. The fixture separately reclaims orphaned storage and any host mutex. SDK 5 normal teardown forwards its observed null free. Null-return rollback and kernel setup error cleanup remain covered. Pre-fix rollback and swallowed-notifier mutations fail the regressions. - [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - nested cleanup, callbacks freeing then throwing at other stages and guest exception integration remain incomplete. Fifty SDK 5/6 reference cases cover throws around twelve frees and healthy controls, preserving exception identity, owner activity, release prefixes and initialization refusal. Sixteen returning retries cover null frees, retained generations and fresh owners. Completed defaults forward null frees during retry; the old `ready` gate fails the regression. Eighteen production cases reject destroyed mutex slots or poisoned default inner headers before retry effects. Retirement survives callback exceptions and clears only after the corresponding release returns. Outer-handle retries remain available. Separate probes faulted at substituted kernel boundaries or requested released resources; these are not returning contracts or console measurements. Fixture reclamation is separate. + +- [Legacy map heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - nested copy/erase exceptions and concurrent lock ownership remain unverified. Three isolated SDK 5 reference cases cover sentinel allocation failure, entry allocation failure and success. Sentinel failure propagates `std::bad_alloc` and orphans an unpublished 24-byte header with only its count initialized; retry allocates a replacement header. Entry failure retains the published sentinel and retry reuses it. Checks cover guest/libc domains, absent guest notifications, source bytes/generations, callback identity, termination and fresh initialization. The fixture reclaims the unpublished orphan only after asserting that library cleanup left it intact. The pre-fix implementation replaces the exception with an unsupported error and fails the regression. From b72dc8b4a341f064b8e35955a7ca7eb801dee91a Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:03:30 +0200 Subject: [PATCH 081/318] fix(libSceJson2): retain Pair ownership after callback copy exceptions --- core/libs/prx/libSceJson2/CMakeLists.txt | 9 +++ core/libs/prx/libSceJson2/Export.cpp | 7 +-- core/libs/tests/GuestJsonLegacyHandlers.cpp | 70 +++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 + 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 0c36d30acd..1eec9dee83 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -375,6 +375,15 @@ if(BUILD_TESTING) target_link_libraries(guest_json_legacy_handlers_tests PRIVATE libkernel libc) configure_windows_unwind(guest_json_legacy_handlers_tests) add_test(NAME guest_json_legacy_handlers COMMAND guest_json_legacy_handlers_tests) + foreach(shape RANGE 0 3) + foreach(alias RANGE 0 2) + add_test(NAME guest_json_legacy_copy_heap_exception_${shape}_${alias} COMMAND guest_json_legacy_handlers_tests ${shape} ${alias}) + if(WIN32) + set_tests_properties(guest_json_legacy_copy_heap_exception_${shape}_${alias} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() + endforeach() foreach(position RANGE 0 2) add_test(NAME guest_json_legacy_heap_exception_${position} COMMAND guest_json_legacy_handlers_tests ${position}) if(WIN32) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index bfc30866bf..bab1637857 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -3288,12 +3288,7 @@ bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value* self) { void APS5_VABI _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(Pair* self, const String* key, const Value* value) { Construct(self->key, *key); - try { - CopyInto(self->value, *value); - } catch (...) { - Destroy(self->key); - throw; - } + CopyInto(self->value, *value); } void APS5_VABI _ZN3sce4Json6Object4PairD1Ev(Pair* self) { diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index 4d2bfa793b..e80d91dffd 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -565,7 +565,77 @@ static int CheckHeapExceptions(unsigned budget) { HeapTrace = nullptr; return 0; } +static int CheckCopyHeapExceptions(unsigned shape, unsigned alias) { + Require(shape < 4 && alias < 3, "Heap-copy schedules cover scalar, string, nested collections and constructor aliases"); + LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&owner, ¶meter) == 0, "Initialize copy exception resources"); + Context context; _ZN3sce4Json5ValueC1Eb(&context.replacement, true); + Value source{}; ParseLegacy(source, shape == 0 ? "9" : shape == 1 ? "\"text\"" : shape == 2 ? "[{\"s\":\"text\"}]" : "{\"a\":[\"text\"]}"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &context) == 0, "Prepare source callback ownership"); + String key{}; _ZN3sce4Json6StringC1EPKc(&key, "key"); + Pair pair; std::memset(&pair, 0xa5, sizeof(pair)); Value scalar{}; Value& copy = alias ? pair.value : scalar; + const Value sourceValue = source; + const auto baseline = trace.guestGenerations, heapBaseline = trace.heapGenerations; + std::vector snapshots; + for (const auto& [pointer, generation] : baseline) { + if (trace.guest.at(pointer) == 24 && trace.heap.contains(*static_cast(pointer))) continue; + const auto* bytes = static_cast(pointer); + snapshots.push_back({pointer, generation, {bytes, bytes + trace.guest.at(pointer)}}); + } + const auto requests = trace.requests.size(); trace.heapBudget = 0; + bool caught = false; + try { + if (!alias) _ZN3sce4Json5ValueC1ERKS1_(©, &source); + else if (alias == 1) _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); + else _ZN3sce4Json6Object4PairC2ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &source); + } catch (const std::bad_alloc&) { caught = true; } + std::vector expected; + if (alias) expected = {24, 4}; + const auto payload = shape == 0 ? std::vector{} : shape == 1 ? std::vector{8, 24, 5} + : shape == 2 ? std::vector{8, 24, 48, 48, 48, 8, 24, 104, 104, 104, 2, 24, 2, 8, 24, 5} + : std::vector{8, 24, 104, 104, 104, 2, 24, 2, 8, 24, 48, 48, 48, 8, 24, 5}; + expected.insert(expected.end(), payload.begin(), payload.end()); + const unsigned retained = shape == 0 ? 0 : shape == 1 ? 3 : 14; + Require(caught && owner && !trace.notifications && trace.heapGenerations == heapBaseline && trace.guest.size() == baseline.size() + retained + unsigned(alias != 0) * 2 && + std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, + "Map-entry exceptions must preserve completed copy allocations without constructor rollback or guest OOM notices"); + CheckMap(trace, {{&source, &context}}); + const auto verifyPayload = [&] { + std::string actual; std::vector values; CollectionShape(copy, nullptr, actual, values); + const char* expectedShape = shape == 0 ? "3:0=9;" : shape == 1 ? "5:0s=text;" : shape == 2 ? "6:0a1[7:0o1{k=s/q=s/5:0s=text;};];" : "7:0o1{k=a/q=a/6:0a1[5:0s=text;];};"; + Require(actual == expectedShape && !copy.auxiliary, "Heap exceptions must preserve complete payloads and independent callback metadata"); + if (alias) Require(pair.key.storage != key.storage && std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), "key") == 0 && pair.reserved == UINT64_C(0xa5a5a5a5a5a5a5a5), + "Failed Pair constructors must retain their independent key and inactive padding"); + }; + verifyPayload(); + const auto calls = context.calls; + Require(!*_ZNK3sce4Json5Value10getBooleanEv(©) && context.calls == calls, "An unpublished copy callback must use the typed default"); + Require(_ZNK3sce4Json5Value10getBooleanEv(&source) == reinterpret_cast(&context.replacement.payload) && context.calls == calls + 1 && context.source == &source, + "Copy failure must preserve the source callback and borrowed result"); + trace.heapBudget = -1; + Require(_ZN3sce4Json5ValueaSERKS1_(©, &source) == ©, "Assignment retry must publish the copied callback"); + CheckMap(trace, {{&source, &context}, {©, &context}}); verifyPayload(); + Require(_ZNK3sce4Json5Value10getBooleanEv(©) == reinterpret_cast(&context.replacement.payload) && context.source == © && !context.parent, + "Retried copies must recover through their own callback identity"); + Require(std::memcmp(&source, &sourceValue, sizeof(source)) == 0, "Copy failure and retry must preserve source Value bytes"); + for (const auto& [pointer, generation] : baseline) Require(trace.guestGenerations.at(pointer) == generation, "Retry must preserve every preexisting allocation generation including shared map storage"); + for (const auto& snapshot : snapshots) Require(trace.guestGenerations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, + "Copy failure must preserve all preexisting allocation bytes and generations"); + if (!alias) _ZN3sce4Json5ValueD1Ev(©); + else if (alias == 1) _ZN3sce4Json6Object4PairD1Ev(&pair); + else _ZN3sce4Json6Object4PairD2Ev(&pair); + _ZN3sce4Json6StringD1Ev(&key); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ValueD1Ev(&context.replacement); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.guest.empty() && trace.heap.empty() && trace.guestGenerations.empty() && trace.heapGenerations.empty(), + "Failed construction, retry and explicit destruction must reclaim both allocator domains"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&owner, ¶meter) == 0 && _ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && trace.guest.empty() && trace.heap.empty(), + "Copy exceptions must permit a fresh module generation"); + HeapTrace = nullptr; + return 0; +} + int main(int argc, char** argv) { + if (argc == 3) return CheckCopyHeapExceptions(static_cast(std::strtoul(argv[1], nullptr, 10)), static_cast(std::strtoul(argv[2], nullptr, 10))); if (argc == 2) return CheckHeapExceptions(static_cast(std::strtoul(argv[1], nullptr, 10))); LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index b481a281bb..d417bdaf18 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -100,3 +100,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Termination free exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - nested cleanup, callbacks freeing then throwing at other stages and guest exception integration remain incomplete. Fifty SDK 5/6 reference cases cover throws around twelve frees and healthy controls, preserving exception identity, owner activity, release prefixes and initialization refusal. Sixteen returning retries cover null frees, retained generations and fresh owners. Completed defaults forward null frees during retry; the old `ready` gate fails the regression. Eighteen production cases reject destroyed mutex slots or poisoned default inner headers before retry effects. Retirement survives callback exceptions and clears only after the corresponding release returns. Outer-handle retries remain available. Separate probes faulted at substituted kernel boundaries or requested released resources; these are not returning contracts or console measurements. Fixture reclamation is separate. - [Legacy map heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - nested copy/erase exceptions and concurrent lock ownership remain unverified. Three isolated SDK 5 reference cases cover sentinel allocation failure, entry allocation failure and success. Sentinel failure propagates `std::bad_alloc` and orphans an unpublished 24-byte header with only its count initialized; retry allocates a replacement header. Entry failure retains the published sentinel and retry reuses it. Checks cover guest/libc domains, absent guest notifications, source bytes/generations, callback identity, termination and fresh initialization. The fixture reclaims the unpublished orphan only after asserting that library cleanup left it intact. The pre-fix implementation replaces the exception with an unsupported error and fails the regression. + +- [Legacy copy heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - deeper entry-allocation exceptions during collection insertion and concurrent lock ownership remain unverified. Twelve SDK 5 reference cases cover Value and both Pair constructor aliases with scalar, string and nested Array/Object sources. Failure to allocate the destination callback-map entry propagates `std::bad_alloc` after payload copying. Pair keys remain independently owned; constructor unwinding does not release them. Checks cover exact guest requests, complete payloads, padding, absent copy callbacks, preserved source callbacks/bytes/generations, shared-map invariants, assignment retry and cleanup in both allocator domains. The pre-fix Pair catch frees the key and fails the retained-allocation regression. Fresh initialization succeeds after explicit destruction and teardown. From 9b33f564568403509d207ed1774db3b0c7e00477 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:07:05 +0200 Subject: [PATCH 082/318] fix(libSceJson2): preserve Array state on child callback exceptions --- core/libs/prx/libSceJson2/CMakeLists.txt | 9 +++ core/libs/prx/libSceJson2/Export.cpp | 23 +++---- core/libs/tests/GuestJsonLegacyHandlers.cpp | 70 +++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 + 4 files changed, 90 insertions(+), 14 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 1eec9dee83..124fe9e0c3 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -375,6 +375,15 @@ if(BUILD_TESTING) target_link_libraries(guest_json_legacy_handlers_tests PRIVATE libkernel libc) configure_windows_unwind(guest_json_legacy_handlers_tests) add_test(NAME guest_json_legacy_handlers COMMAND guest_json_legacy_handlers_tests) + foreach(position RANGE 0 2) + foreach(alias RANGE 0 1) + add_test(NAME guest_json_legacy_array_heap_exception_${position}_${alias} COMMAND guest_json_legacy_handlers_tests array-heap ${position} ${alias}) + if(WIN32) + set_tests_properties(guest_json_legacy_array_heap_exception_${position}_${alias} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() + endforeach() foreach(shape RANGE 0 3) foreach(alias RANGE 0 2) add_test(NAME guest_json_legacy_copy_heap_exception_${shape}_${alias} COMMAND guest_json_legacy_handlers_tests ${shape} ${alias}) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index bab1637857..9ebe215c03 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1288,14 +1288,12 @@ void Construct(T*& destination, const TArgs&... arguments) { void CopyArrayStorage(ArrayStorage& destination, const ArrayStorage& source) { ArrayStorage pending{NewArrayHead(), 0, 0}; - try { - if (pending.head) { - for (const auto& item : source) { - try { CopyInto(pending.emplace_back(), item); } - catch (const GuestOutOfMemory&) { break; } - } + if (pending.head) { + for (const auto& item : source) { + try { CopyInto(pending.emplace_back(), item); } + catch (const GuestOutOfMemory&) { break; } } - } catch (...) { pending.clear(); if (pending.head) Release(pending.head); throw; } + } destination.clear(); if (destination.head) Release(destination.head); destination.head = pending.head; @@ -1303,13 +1301,10 @@ void CopyArrayStorage(ArrayStorage& destination, const ArrayStorage& source) { } void CopyArray(Array& destination, const Array& source) { - Array pending{}; - if (!source.items) { destination = pending; return; } - Construct(pending); - if (!pending.items) { destination = pending; return; } - try { CopyArrayStorage(*pending.items, *source.items); } - catch (...) { Destroy(pending); throw; } - destination = pending; + if (!source.items) { destination = {}; return; } + Construct(destination); + if (!destination.items) return; + CopyArrayStorage(*destination.items, *source.items); } void CopyObject(Object& destination, const Object& source) { diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index e80d91dffd..e6daca16aa 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -45,6 +45,9 @@ int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEP int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullHandler, void*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); +void APS5_VABI _ZN3sce4Json5ArrayC2ERKS1_(Array*, const Array*); +Array* APS5_VABI _ZN3sce4Json5ArrayaSERKS1_(Array*, const Array*); void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); @@ -634,7 +637,74 @@ static int CheckCopyHeapExceptions(unsigned shape, unsigned alias) { return 0; } +static int CheckArrayHeapExceptions(unsigned budget, unsigned alias) { + Require(budget <= 2 && alias < 2, "Array-copy schedules cover both child entries and constructor aliases"); + LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&owner, ¶meter) == 0, "Initialize array-copy exception resources"); + Context context; _ZN3sce4Json5ValueC1Eb(&context.replacement, true); Value source{}; ParseLegacy(source); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &context) == 0, "Prepare copied child callbacks"); + Array array{}; _ZN3sce4Json5ArrayC1Ev(&array); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &source); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &source); + auto* sourceHeader = static_cast(array.storage); + std::map entries{{&source, &context}}; + for (auto* node = sourceHeader->head->next; node != sourceHeader->head; node = node->next) entries.emplace(&node->value, &context); + CheckMap(trace, entries); + const auto baseline = trace.guestGenerations, heapBaseline = trace.heapGenerations; + const Value sourceValue = source; + std::vector snapshots; + for (const auto& [pointer, generation] : baseline) { + if (trace.guest.at(pointer) == 24 && trace.heap.contains(*static_cast(pointer))) continue; + const auto* bytes = static_cast(pointer); + snapshots.push_back({pointer, generation, {bytes, bytes + trace.guest.at(pointer)}}); + } + Array copy; std::memset(©, 0xa5, sizeof(copy)); const auto requests = trace.requests.size(); trace.heapBudget = static_cast(budget); + bool caught = false; + try { + if (alias) _ZN3sce4Json5ArrayC2ERKS1_(©, &array); + else _ZN3sce4Json5ArrayC1ERKS1_(©, &array); + } catch (const std::bad_alloc&) { caught = true; } + std::vector expected{24, 48, 48, 48}; if (budget) expected.push_back(48); + Require(caught == (budget < 2) && trace.guest.contains(copy.storage) && trace.guest.size() == baseline.size() + (budget == 1 ? 5u : 4u) && + trace.heap.size() == heapBaseline.size() + budget && !trace.notifications && std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, + "Interrupted Array construction must publish empty destination storage and retain the temporary list without rollback"); + auto* copyHeader = static_cast(copy.storage); + Require(copyHeader->count == (caught ? 0u : 2u) && copyHeader->head && (caught ? copyHeader->head->next == copyHeader->head && copyHeader->head->previous == copyHeader->head : true), + "Only completed list copying may replace the initial empty destination"); + std::map orphans; + if (caught) for (const auto& [pointer, generation] : trace.guestGenerations) { + if (!baseline.contains(pointer) && pointer != copy.storage && pointer != copyHeader->head) { + Require(trace.guest.at(pointer) == 48, "Only temporary list links become orphaned"); orphans.emplace(pointer, generation); + } + } + Require(orphans.size() == (caught ? budget + 2 : 0), "Failed copies retain the temporary sentinel and admitted child nodes"); + for (const auto& [pointer, generation] : trace.heapGenerations) if (!heapBaseline.contains(pointer)) { + const auto* node = static_cast(pointer); + Require(!node->nil && node->callback == RecoverLegacy && node->context == &context && node->key->type == 3 && node->key->payload == 9, + "Completed child callback records must remain independently published even in the orphaned temporary list"); + entries.emplace(node->key, &context); + } + CheckMap(trace, entries); trace.heapBudget = -1; + Require(_ZN3sce4Json5ArrayaSERKS1_(©, &array) == © && _ZNK3sce4Json5Array4sizeEv(©) == 2, "Assignment retry must replace the retained empty destination"); + for (const auto& [pointer, generation] : orphans) Require(trace.guestGenerations.at(pointer) == generation, "Retry must not claim orphaned temporary-list ownership"); + for (const auto& snapshot : snapshots) Require(trace.guestGenerations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, + "Interrupted copy and retry must preserve source storage bytes and generations"); + Require(std::memcmp(&source, &sourceValue, sizeof(source)) == 0, "Original callback source metadata must remain unchanged"); + const auto* child = _ZNK3sce4Json5Array5frontEv(©); + Require(_ZNK3sce4Json5Value10getBooleanEv(child) == reinterpret_cast(&context.replacement.payload) && context.source == child && !context.parent, + "Retried children must dispatch through their own callback records"); + _ZN3sce4Json5ArrayD1Ev(©); _ZN3sce4Json5ArrayD1Ev(&array); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ValueD1Ev(&context.replacement); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.heap.empty() && trace.heapGenerations.empty() && trace.guestGenerations == orphans, + "Module teardown reclaims callback records but cannot reclaim orphaned caller list nodes"); + for (const auto& [pointer, generation] : orphans) FreeLegacy(nullptr, pointer, &trace); + Require(trace.guest.empty() && _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&owner, ¶meter) == 0 && + _ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && trace.guest.empty() && trace.heap.empty(), "Separate fixture reclamation must leave fresh module initialization usable"); + HeapTrace = nullptr; + return 0; +} + int main(int argc, char** argv) { + if (argc == 4 && std::strcmp(argv[1], "array-heap") == 0) return CheckArrayHeapExceptions(static_cast(std::strtoul(argv[2], nullptr, 10)), static_cast(std::strtoul(argv[3], nullptr, 10))); if (argc == 3) return CheckCopyHeapExceptions(static_cast(std::strtoul(argv[1], nullptr, 10)), static_cast(std::strtoul(argv[2], nullptr, 10))); if (argc == 2) return CheckHeapExceptions(static_cast(std::strtoul(argv[1], nullptr, 10))); LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index d417bdaf18..d683946a8f 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -102,3 +102,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Legacy map heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - nested copy/erase exceptions and concurrent lock ownership remain unverified. Three isolated SDK 5 reference cases cover sentinel allocation failure, entry allocation failure and success. Sentinel failure propagates `std::bad_alloc` and orphans an unpublished 24-byte header with only its count initialized; retry allocates a replacement header. Entry failure retains the published sentinel and retry reuses it. Checks cover guest/libc domains, absent guest notifications, source bytes/generations, callback identity, termination and fresh initialization. The fixture reclaims the unpublished orphan only after asserting that library cleanup left it intact. The pre-fix implementation replaces the exception with an unsupported error and fails the regression. - [Legacy copy heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - deeper entry-allocation exceptions during collection insertion and concurrent lock ownership remain unverified. Twelve SDK 5 reference cases cover Value and both Pair constructor aliases with scalar, string and nested Array/Object sources. Failure to allocate the destination callback-map entry propagates `std::bad_alloc` after payload copying. Pair keys remain independently owned; constructor unwinding does not release them. Checks cover exact guest requests, complete payloads, padding, absent copy callbacks, preserved source callbacks/bytes/generations, shared-map invariants, assignment retry and cleanup in both allocator domains. The pre-fix Pair catch frees the key and fails the retained-allocation regression. Fresh initialization succeeds after explicit destruction and teardown. + +- [Array child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - analogous Object and Value-wrapped construction exceptions remain unverified. Six SDK 5 reference cases cover both Array constructor aliases, failure at either child callback-map entry and success. Construction publishes the initial empty destination before copying a temporary list. Entry exceptions leave that destination empty and orphan the temporary sentinel/admitted nodes; completed child callback records remain published until module teardown. Retry replaces the active destination without reclaiming orphaned links. Checks cover exact requests, retained generations, source bytes, map invariants, child recovery, teardown and fresh initialization. Fixture reclamation of orphaned links occurs separately. The pre-fix rollback/unpublished-destination behavior fails the regression. From eb9d57c81d076e09ad61d651ba335c41b5abe56f Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:09:11 +0200 Subject: [PATCH 083/318] test(libSceJson2): verify wrapped Array callback exception state --- core/libs/prx/libSceJson2/CMakeLists.txt | 3 ++- core/libs/tests/GuestJsonLegacyHandlers.cpp | 28 +++++++++++++++------ docs/dev/debt/libSceJson2.md | 2 +- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 124fe9e0c3..1bc7e74dd9 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -378,8 +378,9 @@ if(BUILD_TESTING) foreach(position RANGE 0 2) foreach(alias RANGE 0 1) add_test(NAME guest_json_legacy_array_heap_exception_${position}_${alias} COMMAND guest_json_legacy_handlers_tests array-heap ${position} ${alias}) + add_test(NAME guest_json_legacy_value_array_heap_exception_${position}_${alias} COMMAND guest_json_legacy_handlers_tests value-array-heap ${position} ${alias}) if(WIN32) - set_tests_properties(guest_json_legacy_array_heap_exception_${position}_${alias} PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_json_legacy_array_heap_exception_${position}_${alias} guest_json_legacy_value_array_heap_exception_${position}_${alias} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endforeach() diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index e6daca16aa..32c63758dd 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -44,6 +44,8 @@ bool APS5_VABI _ZNK3sce4Json5ValuecvbEv(const Value*); int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(Value*, Handler, void*); int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullHandler, void*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_5ArrayE(Value*, const Array*); void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); void APS5_VABI _ZN3sce4Json5ArrayC1ERKS1_(Array*, const Array*); void APS5_VABI _ZN3sce4Json5ArrayC2ERKS1_(Array*, const Array*); @@ -637,7 +639,7 @@ static int CheckCopyHeapExceptions(unsigned shape, unsigned alias) { return 0; } -static int CheckArrayHeapExceptions(unsigned budget, unsigned alias) { +static int CheckArrayHeapExceptions(unsigned budget, unsigned alias, bool wrapped = false) { Require(budget <= 2 && alias < 2, "Array-copy schedules cover both child entries and constructor aliases"); LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); @@ -658,14 +660,21 @@ static int CheckArrayHeapExceptions(unsigned budget, unsigned alias) { const auto* bytes = static_cast(pointer); snapshots.push_back({pointer, generation, {bytes, bytes + trace.guest.at(pointer)}}); } - Array copy; std::memset(©, 0xa5, sizeof(copy)); const auto requests = trace.requests.size(); trace.heapBudget = static_cast(budget); + Array directCopy; std::memset(&directCopy, 0xa5, sizeof(directCopy)); Value wrappedCopy; std::memset(&wrappedCopy, 0xa5, sizeof(wrappedCopy)); + const auto requests = trace.requests.size(); trace.heapBudget = static_cast(budget); bool caught = false; try { - if (alias) _ZN3sce4Json5ArrayC2ERKS1_(©, &array); - else _ZN3sce4Json5ArrayC1ERKS1_(©, &array); + if (wrapped) { + if (alias) _ZN3sce4Json5ValueC2ERKNS0_5ArrayE(&wrappedCopy, &array); + else _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(&wrappedCopy, &array); + } else if (alias) _ZN3sce4Json5ArrayC2ERKS1_(&directCopy, &array); + else _ZN3sce4Json5ArrayC1ERKS1_(&directCopy, &array); } catch (const std::bad_alloc&) { caught = true; } - std::vector expected{24, 48, 48, 48}; if (budget) expected.push_back(48); - Require(caught == (budget < 2) && trace.guest.contains(copy.storage) && trace.guest.size() == baseline.size() + (budget == 1 ? 5u : 4u) && + if (wrapped) Require(wrappedCopy.type == 6 && !wrappedCopy.parent && !wrappedCopy.auxiliary && trace.guest.contains(reinterpret_cast(wrappedCopy.payload)), + "Value wrappers must publish their independent Array handle and type before callback entry failure"); + Array& copy = wrapped ? *reinterpret_cast(wrappedCopy.payload) : directCopy; + std::vector expected{24, 48, 48, 48}; if (wrapped) expected.insert(expected.begin(), 8); if (budget) expected.push_back(48); + Require(caught == (budget < 2) && trace.guest.contains(copy.storage) && trace.guest.size() == baseline.size() + (budget == 1 ? 5u : 4u) + unsigned(wrapped) && trace.heap.size() == heapBaseline.size() + budget && !trace.notifications && std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, "Interrupted Array construction must publish empty destination storage and retain the temporary list without rollback"); auto* copyHeader = static_cast(copy.storage); @@ -673,7 +682,7 @@ static int CheckArrayHeapExceptions(unsigned budget, unsigned alias) { "Only completed list copying may replace the initial empty destination"); std::map orphans; if (caught) for (const auto& [pointer, generation] : trace.guestGenerations) { - if (!baseline.contains(pointer) && pointer != copy.storage && pointer != copyHeader->head) { + if (!baseline.contains(pointer) && pointer != copy.storage && pointer != copyHeader->head && (!wrapped || pointer != reinterpret_cast(wrappedCopy.payload))) { Require(trace.guest.at(pointer) == 48, "Only temporary list links become orphaned"); orphans.emplace(pointer, generation); } } @@ -693,7 +702,9 @@ static int CheckArrayHeapExceptions(unsigned budget, unsigned alias) { const auto* child = _ZNK3sce4Json5Array5frontEv(©); Require(_ZNK3sce4Json5Value10getBooleanEv(child) == reinterpret_cast(&context.replacement.payload) && context.source == child && !context.parent, "Retried children must dispatch through their own callback records"); - _ZN3sce4Json5ArrayD1Ev(©); _ZN3sce4Json5ArrayD1Ev(&array); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ValueD1Ev(&context.replacement); + if (wrapped) _ZN3sce4Json5ValueD1Ev(&wrappedCopy); + else _ZN3sce4Json5ArrayD1Ev(©); + _ZN3sce4Json5ArrayD1Ev(&array); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ValueD1Ev(&context.replacement); Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.heap.empty() && trace.heapGenerations.empty() && trace.guestGenerations == orphans, "Module teardown reclaims callback records but cannot reclaim orphaned caller list nodes"); for (const auto& [pointer, generation] : orphans) FreeLegacy(nullptr, pointer, &trace); @@ -704,6 +715,7 @@ static int CheckArrayHeapExceptions(unsigned budget, unsigned alias) { } int main(int argc, char** argv) { + if (argc == 4 && std::strcmp(argv[1], "value-array-heap") == 0) return CheckArrayHeapExceptions(static_cast(std::strtoul(argv[2], nullptr, 10)), static_cast(std::strtoul(argv[3], nullptr, 10)), true); if (argc == 4 && std::strcmp(argv[1], "array-heap") == 0) return CheckArrayHeapExceptions(static_cast(std::strtoul(argv[2], nullptr, 10)), static_cast(std::strtoul(argv[3], nullptr, 10))); if (argc == 3) return CheckCopyHeapExceptions(static_cast(std::strtoul(argv[1], nullptr, 10)), static_cast(std::strtoul(argv[2], nullptr, 10))); if (argc == 2) return CheckHeapExceptions(static_cast(std::strtoul(argv[1], nullptr, 10))); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index d683946a8f..68a62c4278 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -103,4 +103,4 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Legacy copy heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - deeper entry-allocation exceptions during collection insertion and concurrent lock ownership remain unverified. Twelve SDK 5 reference cases cover Value and both Pair constructor aliases with scalar, string and nested Array/Object sources. Failure to allocate the destination callback-map entry propagates `std::bad_alloc` after payload copying. Pair keys remain independently owned; constructor unwinding does not release them. Checks cover exact guest requests, complete payloads, padding, absent copy callbacks, preserved source callbacks/bytes/generations, shared-map invariants, assignment retry and cleanup in both allocator domains. The pre-fix Pair catch frees the key and fails the retained-allocation regression. Fresh initialization succeeds after explicit destruction and teardown. -- [Array child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - analogous Object and Value-wrapped construction exceptions remain unverified. Six SDK 5 reference cases cover both Array constructor aliases, failure at either child callback-map entry and success. Construction publishes the initial empty destination before copying a temporary list. Entry exceptions leave that destination empty and orphan the temporary sentinel/admitted nodes; completed child callback records remain published until module teardown. Retry replaces the active destination without reclaiming orphaned links. Checks cover exact requests, retained generations, source bytes, map invariants, child recovery, teardown and fresh initialization. Fixture reclamation of orphaned links occurs separately. The pre-fix rollback/unpublished-destination behavior fails the regression. +- [Array child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - analogous Object construction exceptions remain unverified. Twelve SDK 5 reference cases cover both direct Array and Value-wrapped constructor aliases, failure at either child map entry and success. Construction publishes the initial empty destination before copying a temporary list; Value wrappers also publish their type and independent handle. Entry exceptions orphan the temporary sentinel/admitted nodes while completed callback records survive until module teardown. Retry replaces active storage without reclaiming orphaned links. Checks cover requests, generations, source bytes, map invariants, recovery, teardown and fresh initialization. Fixture orphan reclamation is separate. Pre-fix rollback and a mutation clearing the wrapper type both fail regressions. From 089ca096add3ce1f24589cd1c42d02e704e948f3 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:12:57 +0200 Subject: [PATCH 084/318] fix(libSceJson2): preserve Object state on child callback exceptions --- core/libs/prx/libSceJson2/CMakeLists.txt | 11 +++ core/libs/prx/libSceJson2/Export.cpp | 43 +++++------ core/libs/tests/GuestJsonLegacyHandlers.cpp | 85 +++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 116 insertions(+), 27 deletions(-) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 1bc7e74dd9..43ba327c07 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -375,6 +375,17 @@ if(BUILD_TESTING) target_link_libraries(guest_json_legacy_handlers_tests PRIVATE libkernel libc) configure_windows_unwind(guest_json_legacy_handlers_tests) add_test(NAME guest_json_legacy_handlers COMMAND guest_json_legacy_handlers_tests) + foreach(position RANGE 0 2) + foreach(alias RANGE 0 1) + foreach(wrapper direct wrapped) + add_test(NAME guest_json_legacy_object_heap_exception_${position}_${alias}_${wrapper} COMMAND guest_json_legacy_handlers_tests object-heap ${position} ${alias} ${wrapper}) + if(WIN32) + set_tests_properties(guest_json_legacy_object_heap_exception_${position}_${alias}_${wrapper} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() + endforeach() + endforeach() + endforeach() foreach(position RANGE 0 2) foreach(alias RANGE 0 1) add_test(NAME guest_json_legacy_array_heap_exception_${position}_${alias} COMMAND guest_json_legacy_handlers_tests array-heap ${position} ${alias}) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 9ebe215c03..48186f49c8 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1043,28 +1043,22 @@ ObjectLink* CloneObjectTree(ObjectLink* source, ObjectLink* parent, ObjectLink* node->pair.key.text = nullptr; Construct(node->pair.value); node->color = source->color; node->nil = 0; - try { - String key{&node->key}; AssignText(key, {source->key.characters, source->key.length}, true); - Construct(node->pair.key, source->pair.key); - std::memcpy(&node->pair.reserved, &source->pair.reserved, sizeof(std::uint32_t)); - CopyInto(node->pair.value, source->pair.value); - if (!source->left->nil) node->left = CloneObjectTree(source->left, node, head); - if (!source->right->nil) node->right = CloneObjectTree(source->right, node, head); - } catch (...) { - ReleaseObjectTree(node->left); ReleaseObjectTree(node->right); DestroyObjectLink(node); throw; - } + String key{&node->key}; AssignText(key, {source->key.characters, source->key.length}, true); + Construct(node->pair.key, source->pair.key); + std::memcpy(&node->pair.reserved, &source->pair.reserved, sizeof(std::uint32_t)); + CopyInto(node->pair.value, source->pair.value); + if (!source->left->nil) node->left = CloneObjectTree(source->left, node, head); + if (!source->right->nil) node->right = CloneObjectTree(source->right, node, head); return node; } void CopyObjectStorage(ObjectStorage& destination, const ObjectStorage& source) { auto* replacement = NewObjectHead(); ObjectStorage pending{replacement, 0, 0}; - try { - if (replacement && source.head && !source.head->parent->nil) { - replacement->parent = CloneObjectTree(source.head->parent, replacement, replacement); - if (replacement->parent != replacement) { pending.count = source.count; pending.RefreshEndpoints(); } - } - } catch (...) { pending.clear(); Release(replacement); throw; } + if (replacement && source.head && !source.head->parent->nil) { + replacement->parent = CloneObjectTree(source.head->parent, replacement, replacement); + if (replacement->parent != replacement) { pending.count = source.count; pending.RefreshEndpoints(); } + } destination.clear(); Release(destination.head); destination.head = replacement; destination.count = pending.count; @@ -1308,16 +1302,13 @@ void CopyArray(Array& destination, const Array& source) { } void CopyObject(Object& destination, const Object& source) { - Object pending{}; - if (!source.items) { destination = pending; return; } - pending.items = static_cast(Allocate(ModernValues() ? 32 : 24)); - if (!pending.items) { destination = pending; return; } - pending.items->count = 0; - pending.items->head = NewObjectHead(); - pending.items->SetOwner(nullptr); - try { CopyObjectStorage(*pending.items, *source.items); } - catch (...) { Destroy(pending); throw; } - destination = pending; + if (!source.items) { destination = {}; return; } + destination.items = static_cast(Allocate(ModernValues() ? 32 : 24)); + if (!destination.items) { destination = {}; return; } + destination.items->count = 0; + destination.items->head = NewObjectHead(); + destination.items->SetOwner(nullptr); + CopyObjectStorage(*destination.items, *source.items); } void AssignReal(Node& n, double value); diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index 32c63758dd..c9b8907acb 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -13,6 +13,7 @@ struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint32_t reserved; std::int32_t type; }; struct Array { void* storage; }; +struct Object { void* storage; }; struct String { void* storage; }; struct Pair { String key; std::uint64_t reserved; Value value; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; }; @@ -20,6 +21,12 @@ using Handler = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Valu using ReadCallback = int (APS5_VABI*)(char&, void*); using NullHandler = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); extern "C" { +void APS5_VABI _ZN3sce4Json6ObjectC1ERKS1_(Object*, const Object*); +void APS5_VABI _ZN3sce4Json6ObjectC2ERKS1_(Object*, const Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Object* APS5_VABI _ZN3sce4Json6ObjectaSERKS1_(Object*, const Object*); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value*, const Object*); +void APS5_VABI _ZN3sce4Json5ValueC2ERKNS0_6ObjectE(Value*, const Object*); void* APS5_VABI _ZN3sce4Json6MallocEm(std::size_t); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(Value*, ReadCallback, void*); void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); @@ -714,7 +721,85 @@ static int CheckArrayHeapExceptions(unsigned budget, unsigned alias, bool wrappe return 0; } +static int CheckObjectHeapExceptions(unsigned budget, unsigned alias, bool wrapped) { + Require(budget <= 2 && alias < 2, "Object-copy schedules cover both child entries and constructor aliases"); + LegacyTrace trace; HeapTrace = &trace; LegacyAllocator allocator; Parameter parameter{&allocator, &trace, 16384}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&owner, ¶meter) == 0, "Initialize tree-copy exception resources"); + Context context; _ZN3sce4Json5ValueC1Eb(&context.replacement, true); Value source{}, root{}; + ParseLegacy(source); ParseLegacy(root, "{\"a\":9,\"b\":9}"); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, RecoverLegacy, &context) == 0, "Prepare source callback records"); + auto* object = reinterpret_cast(root.payload); + auto* sourceHeader = static_cast(object->storage); + std::string ignored; std::vector values; CollectionTreeShape(sourceHeader->head->parent, &root, ignored, values); + std::map entries{{&source, &context}}; + for (const auto* value : values) { + _ZN3sce4Json5ValueaSERKS1_(const_cast(value), &source); entries.emplace(value, &context); + } + CheckMap(trace, entries); + const auto baseline = trace.guestGenerations, heapBaseline = trace.heapGenerations; + const Value sourceValue = source, rootValue = root; + std::vector snapshots; + for (const auto& [pointer, generation] : baseline) { + if (trace.guest.at(pointer) == 24 && trace.heap.contains(*static_cast(pointer))) continue; + const auto* bytes = static_cast(pointer); + snapshots.push_back({pointer, generation, {bytes, bytes + trace.guest.at(pointer)}}); + } + Object direct; std::memset(&direct, 0xa5, sizeof(direct)); Value value; std::memset(&value, 0xa5, sizeof(value)); + const auto requests = trace.requests.size(); trace.heapBudget = static_cast(budget); bool caught = false; + try { + if (wrapped) { + if (alias) _ZN3sce4Json5ValueC2ERKNS0_6ObjectE(&value, object); + else _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(&value, object); + } else if (alias) _ZN3sce4Json6ObjectC2ERKS1_(&direct, object); + else _ZN3sce4Json6ObjectC1ERKS1_(&direct, object); + } catch (const std::bad_alloc&) { caught = true; } + if (wrapped) Require(value.type == 7 && !value.parent && !value.auxiliary && trace.guest.contains(reinterpret_cast(value.payload)), "Value constructors must retain the published Object type and independent handle"); + Object& copy = wrapped ? *reinterpret_cast(value.payload) : direct; + std::vector expected{24, 104, 104, 104, 2, 24, 2}; + if (budget) expected.insert(expected.end(), {104, 2, 24, 2}); + if (wrapped) expected.insert(expected.begin(), 8); + Require(caught == (budget < 2) && trace.guest.contains(copy.storage) && trace.guest.size() == baseline.size() + (budget == 0 ? 7u : budget == 1 ? 11u : 10u) + unsigned(wrapped) && + trace.heap.size() == heapBaseline.size() + budget && !trace.notifications && std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, + "Interrupted tree construction must retain the published empty destination and unfinished replacement tree without rollback"); + auto* header = static_cast(copy.storage); + Require(header->count == (caught ? 0u : 2u) && header->head && (!caught || header->head->parent == header->head), "Only complete tree copying may publish the replacement root"); + std::map orphans; + if (caught) for (const auto& [pointer, generation] : trace.guestGenerations) { + if (!baseline.contains(pointer) && pointer != copy.storage && pointer != header->head && (!wrapped || pointer != reinterpret_cast(value.payload))) orphans.emplace(pointer, generation); + } + Require(orphans.size() == (caught ? budget == 0 ? 5u : 9u : 0u), "Unfinished trees retain their sentinel, nodes and independently allocated internal/public keys"); + for (const auto& [pointer, generation] : trace.heapGenerations) if (!heapBaseline.contains(pointer)) { + const auto* node = static_cast(pointer); + Require(!node->nil && node->callback == RecoverLegacy && node->context == &context && node->key->type == 3 && node->key->payload == 9, + "Completed replacement-tree child records remain published after the next entry allocation fails"); + entries.emplace(node->key, &context); + } + CheckMap(trace, entries); trace.heapBudget = -1; + Require(_ZN3sce4Json6ObjectaSERKS1_(©, object) == ©, "Independent assignment retry must publish a complete tree"); + Value view{nullptr, nullptr, reinterpret_cast(©), 0, 7}; std::string actual; std::vector copiedValues; + CollectionShape(view, nullptr, actual, copiedValues); + Require(actual == "7:0o2{k=a/q=a/3:0=9;k=b/q=b/3:0=9;};", "Retried trees must preserve both keys, scalar contents and parent links"); + const auto* child = &static_cast(copy.storage)->head->left->pair.value; + Require(_ZNK3sce4Json5Value10getBooleanEv(child) == reinterpret_cast(&context.replacement.payload) && context.source == child && !context.parent, + "Retried tree entries must dispatch through their own callback records"); + for (const auto& [pointer, generation] : orphans) Require(trace.guestGenerations.at(pointer) == generation, "Retry must not reclaim the orphaned replacement tree"); + for (const auto& snapshot : snapshots) Require(trace.guestGenerations.at(snapshot.pointer) == snapshot.generation && std::memcmp(snapshot.pointer, snapshot.bytes.data(), snapshot.bytes.size()) == 0, + "Failed tree copies and retry must preserve every original allocation and source byte"); + Require(std::memcmp(&source, &sourceValue, sizeof(source)) == 0 && std::memcmp(&root, &rootValue, sizeof(root)) == 0, "Both original Values must retain their metadata ownership"); + if (wrapped) _ZN3sce4Json5ValueD1Ev(&value); else _ZN3sce4Json6ObjectD1Ev(©); + _ZN3sce4Json5ValueD1Ev(&root); _ZN3sce4Json5ValueD1Ev(&source); _ZN3sce4Json5ValueD1Ev(&context.replacement); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.heap.empty() && trace.heapGenerations.empty() && trace.guestGenerations == orphans, + "Module teardown reclaims all callback records while leaving the orphaned caller tree intact"); + for (const auto& [pointer, generation] : orphans) FreeLegacy(nullptr, pointer, &trace); + Require(trace.guest.empty() && _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(&owner, ¶meter) == 0 && _ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && trace.guest.empty() && trace.heap.empty(), + "Separate orphan reclamation must preserve fresh module initialization and cleanup"); + HeapTrace = nullptr; + return 0; +} + int main(int argc, char** argv) { + if (argc == 5 && std::strcmp(argv[1], "object-heap") == 0) return CheckObjectHeapExceptions(static_cast(std::strtoul(argv[2], nullptr, 10)), static_cast(std::strtoul(argv[3], nullptr, 10)), std::strcmp(argv[4], "wrapped") == 0); if (argc == 4 && std::strcmp(argv[1], "value-array-heap") == 0) return CheckArrayHeapExceptions(static_cast(std::strtoul(argv[2], nullptr, 10)), static_cast(std::strtoul(argv[3], nullptr, 10)), true); if (argc == 4 && std::strcmp(argv[1], "array-heap") == 0) return CheckArrayHeapExceptions(static_cast(std::strtoul(argv[2], nullptr, 10)), static_cast(std::strtoul(argv[3], nullptr, 10))); if (argc == 3) return CheckCopyHeapExceptions(static_cast(std::strtoul(argv[1], nullptr, 10)), static_cast(std::strtoul(argv[2], nullptr, 10))); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 68a62c4278..fe6bf5e050 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -103,4 +103,6 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Legacy copy heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - deeper entry-allocation exceptions during collection insertion and concurrent lock ownership remain unverified. Twelve SDK 5 reference cases cover Value and both Pair constructor aliases with scalar, string and nested Array/Object sources. Failure to allocate the destination callback-map entry propagates `std::bad_alloc` after payload copying. Pair keys remain independently owned; constructor unwinding does not release them. Checks cover exact guest requests, complete payloads, padding, absent copy callbacks, preserved source callbacks/bytes/generations, shared-map invariants, assignment retry and cleanup in both allocator domains. The pre-fix Pair catch frees the key and fails the retained-allocation regression. Fresh initialization succeeds after explicit destruction and teardown. -- [Array child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - analogous Object construction exceptions remain unverified. Twelve SDK 5 reference cases cover both direct Array and Value-wrapped constructor aliases, failure at either child map entry and success. Construction publishes the initial empty destination before copying a temporary list; Value wrappers also publish their type and independent handle. Entry exceptions orphan the temporary sentinel/admitted nodes while completed callback records survive until module teardown. Retry replaces active storage without reclaiming orphaned links. Checks cover requests, generations, source bytes, map invariants, recovery, teardown and fresh initialization. Fixture orphan reclamation is separate. Pre-fix rollback and a mutation clearing the wrapper type both fail regressions. +- [Array child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - concurrent publication and additional thrown-callback combinations remain unverified. Twelve SDK 5 reference cases cover both direct Array and Value-wrapped constructor aliases, failure at either child map entry and success. Construction publishes the initial empty destination before copying a temporary list; Value wrappers also publish their type and independent handle. Entry exceptions orphan the temporary sentinel/admitted nodes while completed callback records survive until module teardown. Retry replaces active storage without reclaiming orphaned links. Checks cover requests, generations, source bytes, map invariants, recovery, teardown and fresh initialization. Fixture orphan reclamation is separate. Pre-fix rollback and a mutation clearing the wrapper type both fail regressions. + +- [Object child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - deeper tree shapes, guest allocator throws during cloning and concurrent ownership remain unverified. Twelve SDK 5 reference cases cover both direct Object and Value-wrapped constructor aliases, either child entry exception and success. The initial empty tree remains published while unfinished replacement nodes, keys and sentinel become orphaned; completed callback records remain until module teardown. Retry publishes a complete tree without reclaiming the orphaned replacement. Checks cover exact requests, source bytes/generations, tree contents, key storage, callback identity, published wrappers, retry, cleanup and fresh initialization. The fixture separately reclaims orphaned caller storage. The pre-fix recursive rollback and delayed destination publication fail the regression. From 023578881d551e74e5b8a3dac47b1c189dd1e5ee Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:17:14 +0200 Subject: [PATCH 085/318] test(libSceJson2): verify allocator-owned iterator fallback identity --- core/libs/tests/GuestJsonArrayAllocator.cpp | 16 ++++++++++++++++ docs/dev/debt/libSceJson2.md | 6 +++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/core/libs/tests/GuestJsonArrayAllocator.cpp b/core/libs/tests/GuestJsonArrayAllocator.cpp index 8fddc2acd8..fe407899f1 100644 --- a/core/libs/tests/GuestJsonArrayAllocator.cpp +++ b/core/libs/tests/GuestJsonArrayAllocator.cpp @@ -42,6 +42,8 @@ Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEi(Iterator*, int); void APS5_VABI _ZN3sce4Json5Array8iterator7advanceEm(Iterator*, std::size_t); const Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +const Value* APS5_VABI _ZNK3sce4Json5Array8iteratorptEv(const Iterator*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); @@ -189,6 +191,20 @@ int main() { Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Array8iteratordeEv(&begin)) == 30, "Advance must preserve circular traversal"); trace.budget = 0; Iterator failed{}; _ZNK3sce4Json5Array5beginEv(&failed, &array); Require(!failed.state, "Iterator allocation failure must leave an empty outer handle"); + const auto fallbackRequests = trace.requests.size(); const auto fallbackNotices = trace.notifications; + const auto* fallback = _ZNK3sce4Json5Array8iteratordeEv(&failed); + Require(fallback && trace.live.contains(const_cast(fallback)) && trace.live.at(const_cast(fallback)) == 32 && + fallback->type == 0 && !fallback->parent && !fallback->auxiliary, + "Missing iterator state must return the allocator-owned initialized module fallback"); + const Value fallbackValue = *fallback; + Iterator missing{}; Value lookup{}; _ZN3sce4Json5ValueC1El(&lookup, 17); + Require(_ZNK3sce4Json5Array8iteratordeEv(&missing) == fallback && _ZNK3sce4Json5Array8iteratorptEv(&failed) == fallback && + _ZNK3sce4Json5Array8iteratorptEv(&missing) == fallback && + trace.requests.size() == fallbackRequests && trace.notifications == fallbackNotices && std::memcmp(fallback, &fallbackValue, sizeof(fallbackValue)) == 0, + "Independent missing iterators must share the exact fallback without allocations, notices or mutation"); + Require(_ZNK3sce4Json5ValueixEPKc(&lookup, "missing") == fallback && std::memcmp(fallback, &fallbackValue, sizeof(fallbackValue)) == 0, + "Failed keyed lookup must return the same module fallback without mutating its bytes"); + _ZN3sce4Json5ValueD1Ev(&lookup); trace.budget = -1; _ZN3sce4Json5Array8iteratorD1Ev(©); _ZN3sce4Json5Array8iteratorD1Ev(&begin); _ZN3sce4Json5Array8iteratorD1Ev(&end); const auto beforeCopies = trace.live.size(); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index fe6bf5e050..293ee6f955 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -8,7 +8,7 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [sce::Json::Array::pop_front](../../../core/libs/prx/libSceJson2/Export.cpp) - the console result for an empty array remains unverified; the existing host behavior throws `std::out_of_range`. Missing storage throws unsupported. Nonempty removal releases the allocator-owned node without allocating and preserves surviving element/iterator identity. - [sce::Json::Parser::parse](../../../core/libs/prx/libSceJson2/Export.cpp) - null text with nonzero size throws; decimal overflow follows the configured real-conversion mode. Numeric-token validation, more complex parser allocation failures and cross-thread operation behavior remain incomplete. Parsing mutates the supplied destination, following `0x6940`, `0x6840` and `0x5d30`: failed arrays/objects retain admitted entries, strings append their decoded prefix, and incomplete scalar literals preserve the previous value. Existing collections retain their entries across later parses. Local tests cover nested parent links, duplicate-member failure, type transitions and actual allocator cleanup; the same contract passes the reference module. These checks do not establish allocator-failure equivalence or Linux/Windows guest execution. - [sce::Json::InitParameter2, InitParameterRtti and InitParameterRtti2](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - initialization owns its allocator-backed context, default resources, module fallback and kernel synchronization handles. Allocation admission, rollback and retry follow the observed default-resource protocol; file-buffer size is snapshotted and used by file parsing. Remaining kernel-status and callback/concurrent lifecycle paths, plus full guest execution, remain unverified. The active-byte and enum validation rules follow the inspected initializer at `0x920` and `0x12b0`–`0x13f7`, with local allocation-failure coverage. The module firmware version is unverified; these are library observations, not console measurements. -- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. The callback regressions exercise SDK 6 behavior; console execution and older-SDK callback compatibility are unverified. +- [sce::Json::Value auxiliary ownership](../../../core/libs/prx/libSceJson2/Export.cpp) - values use the inspected 32-byte inline payload and parent layout, and allocator-backed auxiliary ownership. Outer value handles, string headers and characters, array and object storage and their iterator states use the supplied allocator; module defaults are allocated during initialization, including admitted partial default resources. Request sizes, failure admission and SDK-dependent ownership below SDK 6 remain incomplete. Modern callbacks and SDK 5 mismatch-map copying have reference-backed contracts below; legacy special-float/element registration and full guest execution remain incomplete. - [InitParameter2 special-float configuration](../../../core/libs/prx/libSceJson2/Export.cpp) follows assignment helpers at `0x40a0` and `0x41a0` and formatting at `0x26a0`: mode 0 converts non-finite numbers to strings, mode 1 to null, and mode 2 retains real values and emits lowercase non-JSON tokens. Parser acceptance of those tokens, floating-point exception flags, signaling-NaN behavior and concurrent reconfiguration remain unverified. Legacy initialization does not enable the conversion; its internal formatter is now reproduced through the inspected float32 conversion, six truncating fractional digits, range saturation and NaN integer result. The regression covers signed zero, precision boundaries, large values, infinities and NaNs; floating-point rounding modes and exception flags remain unverified. `Value::set(double)` and `toString(String&)` expose no consistent return value and are declared `void`. - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and inherited older-SDK failure scopes still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. @@ -17,11 +17,11 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. - [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object, callback metadata and file-buffer allocation failures dispatch the supplied allocator's notifier. - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. -- [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; a missing iterator state uses the module default value, whose allocator-backed identity remains incomplete. +- [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; missing-state Array dereference/arrow share the initialized 32-byte allocator-owned fallback with keyed lookup. Iterator access allocates nothing and preserves its bytes under allocation failure. - [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - untested legacy callback combinations remain incomplete. Modern root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. - [sce::Json::Parser file input](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator failures outside the file buffer and callback-read failure combinations remain incomplete. Buffer ownership and the read adapter follow `0x69c0` and `0x6b20`; local contract tests substitute the kernel file boundary and do not prove guest filesystem integration on Linux or Windows. -- [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. +- [sce::Json::Array storage and iterators](../../../core/libs/prx/libSceJson2/Export.cpp) - nested value-copy allocator failures, shared concurrent allocation-operation accounting and legacy parent binding remain incomplete. Header, sentinel, node and iterator allocation follow `0xb2d0`, `0xb410`, `0xb540`, `0xb6f0` and `0xb130`; local contract tests cover scalar/nested copies and partial ownership. Failed node allocation in `push_back` removes the existing last entry, following `0xb880`; this behavior is verified against the inspected module only. Circular traversal follows `0xb250` and `0xb290`; large advances reduce complete cycles without changing the resulting node. `front` on a partial header with no sentinel throws instead of dereferencing null. Full Linux/Windows execution is unverified. - [sce::Json::Object storage and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy ownership, callback-bearing copies and concurrent operation accounting remain incomplete. Allocator-backed tree nodes can retain incomplete entries or a source count exceeding their live nodes after copy failure (`0xca40`, `0xe000`). Modern assignment (`0xd400`) preserves the original on initial allocation failure but can replace it with partial entries after later failures; self-assignment rebuilds the tree. Legacy insertion copies the pair directly. SDK 5 boundary tests do not establish cross-firmware compatibility. Erasure allocates nothing (`0xd350`). Missing-state iterator dereference throws if its temporary default iterator cannot be allocated; the reference can recurse. Full Linux/Windows execution remains unverified. - [sce::Json::Parser embedded zero characters](../../../core/libs/prx/libSceJson2/Export.cpp) - decoded Unicode zero is discarded through the inspected character-append behavior. Binary string values serialize using their stored length, while object keys serialize their C-string prefix. Broader malformed Unicode handling remains incomplete. From 5fe57f183ccf5e6fe5d85bffd575cc887e134dc8 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:21:37 +0200 Subject: [PATCH 086/318] test(libSceJson2): verify partial element fallback recovery --- core/libs/tests/GuestJsonElementRecovery.cpp | 95 ++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/core/libs/tests/GuestJsonElementRecovery.cpp b/core/libs/tests/GuestJsonElementRecovery.cpp index cd3894b467..54093c019f 100644 --- a/core/libs/tests/GuestJsonElementRecovery.cpp +++ b/core/libs/tests/GuestJsonElementRecovery.cpp @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include struct Value { Value* parent; void* auxiliary; std::uintptr_t payload; std::uint32_t reserved; std::int32_t type; }; struct String { void* text; }; @@ -64,6 +67,97 @@ static void Event(const Context& trace, int calls, unsigned error, const Value* Require(trace.calls == calls && trace.error == error && trace.value == value && trace.parent == parent, "Recovery must receive the exact error, source value, parent and context"); } +struct PartialElementTrace { + std::map live; + std::map generations; + std::vector requests; + unsigned nextGeneration = 0; + int budget = -1; + unsigned notifications = 0; + unsigned calls = 0; + const Value* expectedValue = nullptr; + const Value* expectedParent = nullptr; + const Value* expectedFallback = nullptr; + bool reenter = false; +}; +static void* APS5_VABI AllocatePartialElement(void*, std::size_t size, void* data) { + auto& trace = *static_cast(data); trace.requests.push_back(size); + if (!trace.budget) return nullptr; + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(size); + Require(memory && trace.live.emplace(memory, size).second && trace.generations.emplace(memory, ++trace.nextGeneration).second, "Partial metadata requires independent allocation generations"); + std::memset(memory, 0xa5, size); return memory; +} +static void APS5_VABI FreePartialElement(void*, void* memory, void* data) { + if (!memory) return; + auto& trace = *static_cast(data); + Require(trace.live.erase(memory) == 1 && trace.generations.erase(memory) == 1, "Partial recovery metadata must release through its owning allocator exactly once"); + std::free(memory); +} +static void APS5_VABI NotifyPartialElement(void*, int error, std::size_t size, void* data) { + auto& trace = *static_cast(data); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == size, "Failure notices must identify the rejected metadata allocation"); + ++trace.notifications; +} +static void APS5_VABI RecoverPartialElement(int error, const String* key, const std::size_t* index, const Value* parent, const Value* value, void* data) { + auto& trace = *static_cast(data); ++trace.calls; + Require(static_cast(error) == 0x80848118 && !key && index && *index == std::numeric_limits::max() && value == trace.expectedValue && parent == trace.expectedParent, + "Partial recovery must preserve error, all index bits, source, immediate parent and context"); + if (trace.reenter) { + const auto calls = trace.calls; + Require(_ZNK3sce4Json5Value8getValueEm(value, *index) == trace.expectedFallback && trace.calls == calls, "Reentry must share the fallback while suppressing recursive callbacks"); + _ZN3sce4Json5Value3setEl(const_cast(trace.expectedFallback), 77); + } +} +static void* const PartialElementTable[]{nullptr, nullptr, reinterpret_cast(AllocatePartialElement), reinterpret_cast(FreePartialElement), reinterpret_cast(NotifyPartialElement)}; +struct PartialElementAllocator { void* const* table = PartialElementTable; }; +static void CheckPartialElementFallbacks() { + for (unsigned budget = 0; budget <= 3; ++budget) for (bool reenter : {false, true}) { + PartialElementTrace trace; PartialElementAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0, "Initialize partial element recovery"); + Value root{}; _ZN3sce4Json5ValueC1Ev(&root); + const auto* moduleFallback = _ZNK3sce4Json5Value8getValueEm(&root, std::numeric_limits::max()); + Require(moduleFallback && trace.live.at(const_cast(moduleFallback)) == 32, "Initial recovery must expose the initialized module fallback"); + const auto requests = trace.requests.size(); trace.budget = static_cast(budget); + const int status = _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, RecoverPartialElement, &trace); + std::vector expected{72}; if (budget) expected.push_back(32); if (budget >= 2) expected.push_back(16); + Require(status == (budget < 3 ? static_cast(0x80848102) : 0) && trace.notifications == unsigned(budget < 3) && + std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected, + "Partial handler admission must preserve exact auxiliary, fallback and record request order"); + void* originalAuxiliary = root.auxiliary; const auto retryRequests = trace.requests.size(); trace.budget = -1; + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, RecoverPartialElement, &trace) == 0, "Independent registration retry must succeed"); + const auto retryExpected = budget == 0 ? std::vector{72, 32, 16} : budget < 3 ? std::vector{16} : std::vector{}; + Require(std::vector(trace.requests.begin() + retryRequests, trace.requests.end()) == retryExpected && (!originalAuxiliary || root.auxiliary == originalAuxiliary), + "Registration retry must reuse admitted metadata without recreating a missing fallback"); + auto* auxiliary = static_cast(root.auxiliary); auto* fallback = static_cast(auxiliary[2]); + Require(bool(fallback) == (budget != 1) && auxiliary[5], "Failed fallback admission must remain missing after successful handler retry"); + const auto auxiliaryGeneration = trace.generations.at(auxiliary), recordGeneration = trace.generations.at(auxiliary[5]); + const auto fallbackGeneration = fallback ? trace.generations.at(fallback) : 0; + trace.expectedFallback = fallback ? fallback : moduleFallback; trace.reenter = reenter; + const auto dispatch = [&](const Value* source, const Value* parent, unsigned calls) { + trace.expectedValue = source; trace.expectedParent = parent; + const auto before = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = 0; + Require(_ZNK3sce4Json5Value8getValueEm(source, std::numeric_limits::max()) == trace.expectedFallback && trace.calls == calls && + trace.requests.size() == before && trace.notifications == notices, + "Partial fallback dispatch and callback reentry must preserve identity without allocation or notifications"); + trace.budget = -1; + Require(!reenter || (trace.expectedFallback->type == 2 && trace.expectedFallback->payload == 77), "Callback mutation must remain visible through the returned fallback"); + }; + dispatch(&root, nullptr, 1); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, "[1]", 3) == 0, "Parsing must retain partial root recovery metadata"); + const auto* child = _ZNK3sce4Json5Value8getValueEm(&root, 0); + Require(child && child->parent == &root, "The parsed child must retain its actual parent"); dispatch(child, &root, 2); + Require(root.auxiliary == auxiliary && auxiliary[2] == fallback && trace.generations.at(auxiliary) == auxiliaryGeneration && trace.generations.at(auxiliary[5]) == recordGeneration && + (!fallback || (trace.generations.at(fallback) == fallbackGeneration && fallback->parent == &root)) && + static_cast(auxiliary[5])[0] == reinterpret_cast(&RecoverPartialElement) && static_cast(auxiliary[5])[1] == &trace, + "Parsing and dispatch must preserve auxiliary, fallback and handler generations, callback ABI and ownership"); + _ZN3sce4Json5ValueD1Ev(&root); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.live.empty() && trace.generations.empty(), "Every partial admission and retry must reclaim real allocation ownership"); + } +} + int main() { Value root{}; _ZN3sce4Json5ValueC1Ev(&root); @@ -125,5 +219,6 @@ int main() { _ZN3sce4Json6StringD1Ev(&missing); _ZN3sce4Json5ValueD1Ev(&root); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate element recovery"); + CheckPartialElementFallbacks(); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 293ee6f955..2a23af48e3 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -13,7 +13,7 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and inherited older-SDK failure scopes still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - libc-heap failures and untested legacy callback combinations remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Additional legacy compatibility remains unverified. -- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback-value failure behavior and compatibility below SDK 6 remain incomplete. +- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Complex fallback-payload allocation failures and compatibility below SDK 6 remain incomplete. - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. - [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object, callback metadata and file-buffer allocation failures dispatch the supplied allocator's notifier. - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. @@ -106,3 +106,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Array child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - concurrent publication and additional thrown-callback combinations remain unverified. Twelve SDK 5 reference cases cover both direct Array and Value-wrapped constructor aliases, failure at either child map entry and success. Construction publishes the initial empty destination before copying a temporary list; Value wrappers also publish their type and independent handle. Entry exceptions orphan the temporary sentinel/admitted nodes while completed callback records survive until module teardown. Retry replaces active storage without reclaiming orphaned links. Checks cover requests, generations, source bytes, map invariants, recovery, teardown and fresh initialization. Fixture orphan reclamation is separate. Pre-fix rollback and a mutation clearing the wrapper type both fail regressions. - [Object child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - deeper tree shapes, guest allocator throws during cloning and concurrent ownership remain unverified. Twelve SDK 5 reference cases cover both direct Object and Value-wrapped constructor aliases, either child entry exception and success. The initial empty tree remains published while unfinished replacement nodes, keys and sentinel become orphaned; completed callback records remain until module teardown. Retry publishes a complete tree without reclaiming the orphaned replacement. Checks cover exact requests, source bytes/generations, tree contents, key storage, callback identity, published wrappers, retry, cleanup and fresh initialization. The fixture separately reclaims orphaned caller storage. The pre-fix recursive rollback and delayed destination publication fail the regression. + +- [Partial element-recovery fallback](../../../core/libs/prx/libSceJson2/Export.cpp) - complex callback-created fallback payload failures and older SDK dispatch remain unverified. Eight SDK 6 reference cases cover failure of auxiliary/fallback/handler admission and success, with and without callback reentry. Registration retry reuses admitted metadata without repairing a missing fallback. Parsing preserves that missing state; root/child failures invoke the handler and return the module fallback instead. Checks cover request order, error/index/source/parent ABI, suppression of recursive dispatch, visible scalar mutation, zero-allocation dispatch under failure, stable auxiliary/fallback/record generations and complete teardown. A mutation recreating the missing fallback fails the retry regression. From 9887e2a97520e6979a7ee4e693a4fa3ee95c75b8 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:29:17 +0200 Subject: [PATCH 087/318] fix(libSceJson2): preserve C-string setter allocation and ABI contracts --- core/libs/prx/libSceJson2/Export.cpp | 8 +-- core/libs/tests/GuestJson.cpp | 20 +++--- core/libs/tests/GuestJsonElementRecovery.cpp | 68 +++++++++++++++++++- docs/dev/debt/libSceJson2.md | 2 + 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 48186f49c8..b994fae39b 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2966,12 +2966,12 @@ int APS5_VABI _ZN3sce4Json5Value3setEb(Value* self, bool value) { SetType(NodeOf int APS5_VABI _ZN3sce4Json5Value3setEl(Value* self, std::int64_t value) { SetType(NodeOf(*self), TypeInteger); self->integer = value; return 0; } int APS5_VABI _ZN3sce4Json5Value3setEm(Value* self, std::uint64_t value) { SetType(NodeOf(*self), TypeUInteger); self->uinteger = value; return 0; } void APS5_VABI _ZN3sce4Json5Value3setEd(Value* self, double value) { AssignReal(NodeOf(*self), value); } -int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { +void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value::set: null text"); - const std::string copy(text); + ScopedString temporary{std::string_view{text}}; + if (!temporary.string.text) return; SetType(NodeOf(*self), TypeString); - if (self->type == TypeString) AssignText(*self->string, copy); - return 0; + if (self->type == TypeString) AssignText(*self->string, View(temporary.string), true); } int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { const std::string copy(View(*text)); diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index db3130ac87..6085b73027 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -106,7 +106,7 @@ void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5Value5clearEv(Value*); int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); int APS5_VABI _ZN3sce4Json5Value3setEm(Value*, std::uint64_t); -int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value*, const char*); +void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value*, const char*); std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); const bool* APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value*); const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); @@ -309,7 +309,7 @@ static void ObjectsAndArrays() { Require(!_ZNK3sce4Json6Object5emptyEv(&object)); Require(_ZNK3sce4Json5Value7getTypeEv(first) == TypeNull); Require(_ZN3sce4Json5Value3setEl(first, 1) == 0); - Require(_ZN3sce4Json5Value3setEPKc(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &beta), "b") == 0); + _ZN3sce4Json5Value3setEPKc(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &beta), "b"); Require(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha) == first); std::vector keys; @@ -384,7 +384,7 @@ static void ObjectPair() { _ZN3sce4Json6StringC1EPKc(&key, "answer"); Value value{}; _ZN3sce4Json5ValueC1Ev(&value); - Require(_ZN3sce4Json5Value3setEPKc(&value, "forty-two") == 0); + _ZN3sce4Json5Value3setEPKc(&value, "forty-two"); Pair pair{}; std::memset(&pair, 0x5a, sizeof(pair)); @@ -479,7 +479,7 @@ static void ObjectFind() { Require(_ZN3sce4Json5Value3setEl(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha), 1) == 0); Value* second = _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &beta); - Require(_ZN3sce4Json5Value3setEPKc(second, "b") == 0); + _ZN3sce4Json5Value3setEPKc(second, "b"); _ZNK3sce4Json6Object3endEv(&end, &object); _ZN3sce4Json6Object8iteratorD1Ev(&found); Iterator* returned = _ZNK3sce4Json6Object4findERKNS0_6StringE(&found, &object, &beta); @@ -602,7 +602,7 @@ static void ObjectInsert() { _ZN3sce4Json6StringC1EPKc(&otherKey, "other"); Value otherValue{}; _ZN3sce4Json5ValueC1Ev(&otherValue); - Require(_ZN3sce4Json5Value3setEPKc(&otherValue, "str") == 0); + _ZN3sce4Json5Value3setEPKc(&otherValue, "str"); Pair otherPair{}; _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&otherPair, &otherKey, &otherValue); Iterator otherInserted{}; @@ -741,7 +741,7 @@ static void ValueSwap() { _ZN3sce4Json5ValueC1Ev(&v1); _ZN3sce4Json5ValueC1Ev(&v2); Require(_ZN3sce4Json5Value3setEl(&v1, 123) == 0); - Require(_ZN3sce4Json5Value3setEPKc(&v2, "hello") == 0); + _ZN3sce4Json5Value3setEPKc(&v2, "hello"); _ZN3sce4Json5Value4swapERS1_(&v1, &v2); Require(*_ZNK3sce4Json5Value10getIntegerEv(&v2) == 123); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&v1)), "hello") == 0); @@ -1057,7 +1057,7 @@ static void ValueClear() { _ZN3sce4Json5Value5clearEv(&value); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); - Require(_ZN3sce4Json5Value3setEPKc(&value, "test string") == 0); + _ZN3sce4Json5Value3setEPKc(&value, "test string"); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeString); _ZN3sce4Json5Value5clearEv(&value); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); @@ -1236,7 +1236,7 @@ static void ValueTypeMismatchHandler() { Value other{}; _ZN3sce4Json5ValueC1Ev(&other); - Require(_ZN3sce4Json5Value3setEPKc(&other, "other") == 0); + _ZN3sce4Json5Value3setEPKc(&other, "other"); Require(_ZN3sce4Json5ValueaSERKS1_(&root, &other) == &root); Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 42); RequireMismatch(7, TypeInteger, &root, &context); @@ -1247,7 +1247,7 @@ static void ValueTypeMismatchHandler() { Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 42); RequireMismatch(8, TypeInteger, &root, &context); _ZN3sce4Json5Value5clearEv(&root); - Require(_ZN3sce4Json5Value3setEPKc(&root, "cleared") == 0); + _ZN3sce4Json5Value3setEPKc(&root, "cleared"); Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 42); RequireMismatch(9, TypeInteger, &root, &context); Value copy{}; @@ -1358,7 +1358,7 @@ static void ValueOperatorBool() { Require(_ZNK3sce4Json5ValuecvbEv(&scalar) == (value != 0.0)); } for (const char* text : {"", "0", "false", " "}) { - Require(_ZN3sce4Json5Value3setEPKc(&scalar, text) == 0); + _ZN3sce4Json5Value3setEPKc(&scalar, text); Require(_ZNK3sce4Json5ValuecvbEv(&scalar) == (*text != 0)); } _ZN3sce4Json5ValueD1Ev(&scalar); diff --git a/core/libs/tests/GuestJsonElementRecovery.cpp b/core/libs/tests/GuestJsonElementRecovery.cpp index 54093c019f..9d9e051839 100644 --- a/core/libs/tests/GuestJsonElementRecovery.cpp +++ b/core/libs/tests/GuestJsonElementRecovery.cpp @@ -22,6 +22,7 @@ void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5Value5clearEv(Value*); int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value*, const char*); int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, Handler, void*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); @@ -79,6 +80,8 @@ struct PartialElementTrace { const Value* expectedParent = nullptr; const Value* expectedFallback = nullptr; bool reenter = false; + bool assignString = false; + unsigned expectedError = 0x80848118; }; static void* APS5_VABI AllocatePartialElement(void*, std::size_t size, void* data) { auto& trace = *static_cast(data); trace.requests.push_back(size); @@ -101,12 +104,13 @@ static void APS5_VABI NotifyPartialElement(void*, int error, std::size_t size, v } static void APS5_VABI RecoverPartialElement(int error, const String* key, const std::size_t* index, const Value* parent, const Value* value, void* data) { auto& trace = *static_cast(data); ++trace.calls; - Require(static_cast(error) == 0x80848118 && !key && index && *index == std::numeric_limits::max() && value == trace.expectedValue && parent == trace.expectedParent, + Require(static_cast(error) == trace.expectedError && !key && index && *index == std::numeric_limits::max() && value == trace.expectedValue && parent == trace.expectedParent, "Partial recovery must preserve error, all index bits, source, immediate parent and context"); - if (trace.reenter) { + if (trace.reenter || trace.assignString) { const auto calls = trace.calls; Require(_ZNK3sce4Json5Value8getValueEm(value, *index) == trace.expectedFallback && trace.calls == calls, "Reentry must share the fallback while suppressing recursive callbacks"); - _ZN3sce4Json5Value3setEl(const_cast(trace.expectedFallback), 77); + if (trace.assignString) _ZN3sce4Json5Value3setEPKc(const_cast(trace.expectedFallback), "recovery"); + else _ZN3sce4Json5Value3setEl(const_cast(trace.expectedFallback), 77); } } static void* const PartialElementTable[]{nullptr, nullptr, reinterpret_cast(AllocatePartialElement), reinterpret_cast(FreePartialElement), reinterpret_cast(NotifyPartialElement)}; @@ -158,6 +162,63 @@ static void CheckPartialElementFallbacks() { } } +static void CheckElementStringFailures() { + for (bool missingFallback : {false, true}) for (bool nested : {false, true}) for (unsigned budget = 0; budget <= 5; ++budget) { + PartialElementTrace trace; PartialElementAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0, "Initialize fallback String failure resources"); + Value root{}; _ZN3sce4Json5ValueC1Ev(&root); + const auto* moduleFallback = _ZNK3sce4Json5Value8getValueEm(&root, std::numeric_limits::max()); + if (missingFallback) { + trace.budget = 1; + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, RecoverPartialElement, &trace) == static_cast(0x80848102), "Prepare a root without its auxiliary fallback"); + trace.budget = -1; + } + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, RecoverPartialElement, &trace) == 0 && + _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, "[1]", 3) == 0, "Prepare nested recovery with admitted callback metadata"); + auto* auxiliary = static_cast(root.auxiliary); + auto* fallback = static_cast(auxiliary[2]); + Require(bool(fallback) != missingFallback, "Parsing must preserve the intended fallback admission state"); + trace.expectedFallback = fallback ? fallback : moduleFallback; + trace.expectedValue = nested ? _ZNK3sce4Json5Value8getValueEm(&root, 0) : &root; + trace.expectedParent = nested ? &root : nullptr; trace.expectedError = nested ? 0x80848118 : 0x80848116; trace.assignString = true; + const auto generations = trace.generations; + std::map> snapshots; + for (const auto& [pointer, size] : trace.live) { + if (pointer == trace.expectedFallback) continue; + const auto* bytes = static_cast(pointer); snapshots.emplace(pointer, std::vector{bytes, bytes + size}); + } + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = static_cast(budget); + Require(_ZNK3sce4Json5Value8getValueEm(trace.expectedValue, std::numeric_limits::max()) == trace.expectedFallback && trace.calls == 1, + "Callback String allocation failure must preserve the exact returned fallback and suppress reentry"); + std::vector expected{24}; if (budget) { expected.push_back(9); expected.push_back(8); } if (budget >= 3) expected.push_back(24); if (budget >= 4) expected.push_back(9); + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected && trace.notifications == notices + (budget == 1 ? 2u : unsigned(budget < 5)), + "Callback String assignment must preserve temporary and destination handle/header/character request order and actual failure notices"); + Require(trace.expectedFallback->type == (budget < 4 ? 0 : 5) && trace.expectedFallback->parent == (missingFallback ? nullptr : &root), + "Failed String assignment must retain the admitted partial payload and fallback owner"); + if (budget >= 4) Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(trace.expectedFallback->payload)), budget == 4 ? "" : "recovery") == 0, + "Character failure must retain an empty String while successful admission exposes complete text"); + trace.budget = -1; const auto retryRequests = trace.requests.size(); + _ZN3sce4Json5Value3setEPKc(const_cast(trace.expectedFallback), "recovery"); + const auto retryExpected = budget < 4 ? std::vector{24, 9, 8, 24, 9} : std::vector{24, 9, 9}; + Require(std::vector(trace.requests.begin() + retryRequests, trace.requests.end()) == retryExpected && trace.expectedFallback->type == 5 && + std::strcmp(_ZNK3sce4Json6String5c_strEv(reinterpret_cast(trace.expectedFallback->payload)), "recovery") == 0, + "Independent retry must reuse admitted String storage and complete only missing payload allocations"); + const auto retryGenerations = trace.generations; const auto repeatedRequests = trace.requests.size(); const auto repeatedNotices = trace.notifications; + trace.budget = 0; + Require(_ZNK3sce4Json5Value8getValueEm(trace.expectedValue, std::numeric_limits::max()) == trace.expectedFallback && trace.calls == 2 && + trace.generations == retryGenerations && trace.requests.size() == repeatedRequests + 1 && trace.requests.back() == 24 && trace.notifications == repeatedNotices + 1, + "Recovery must resume after allocation failure and preserve the completed payload when temporary admission fails"); + trace.budget = -1; + Require(root.auxiliary == auxiliary && auxiliary[2] == fallback && auxiliary[5], "Payload failure and retry must not replace callback metadata or repair a missing fallback"); + for (const auto& [pointer, generation] : generations) Require(trace.generations.at(pointer) == generation, "Payload recovery must preserve all preexisting allocation identities"); + for (const auto& [pointer, bytes] : snapshots) Require(std::memcmp(pointer, bytes.data(), bytes.size()) == 0, "Payload recovery must preserve source, module and callback-record bytes"); + _ZN3sce4Json5ValueD1Ev(&root); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.live.empty() && trace.generations.empty(), "Root and module fallbacks must release their real String payload ownership on teardown"); + } +} + int main() { Value root{}; _ZN3sce4Json5ValueC1Ev(&root); @@ -220,5 +281,6 @@ int main() { _ZN3sce4Json5ValueD1Ev(&root); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate element recovery"); CheckPartialElementFallbacks(); + CheckElementStringFailures(); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 2a23af48e3..806000da59 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -108,3 +108,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Object child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - deeper tree shapes, guest allocator throws during cloning and concurrent ownership remain unverified. Twelve SDK 5 reference cases cover both direct Object and Value-wrapped constructor aliases, either child entry exception and success. The initial empty tree remains published while unfinished replacement nodes, keys and sentinel become orphaned; completed callback records remain until module teardown. Retry publishes a complete tree without reclaiming the orphaned replacement. Checks cover exact requests, source bytes/generations, tree contents, key storage, callback identity, published wrappers, retry, cleanup and fresh initialization. The fixture separately reclaims orphaned caller storage. The pre-fix recursive rollback and delayed destination publication fail the regression. - [Partial element-recovery fallback](../../../core/libs/prx/libSceJson2/Export.cpp) - complex callback-created fallback payload failures and older SDK dispatch remain unverified. Eight SDK 6 reference cases cover failure of auxiliary/fallback/handler admission and success, with and without callback reentry. Registration retry reuses admitted metadata without repairing a missing fallback. Parsing preserves that missing state; root/child failures invoke the handler and return the module fallback instead. Checks cover request order, error/index/source/parent ABI, suppression of recursive dispatch, visible scalar mutation, zero-allocation dispatch under failure, stable auxiliary/fallback/record generations and complete teardown. A mutation recreating the missing fallback fails the retry regression. + +- [Element fallback String allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - collection payloads and thrown allocator callbacks during fallback mutation remain unverified. Twenty-four SDK 6 reference cases cover root/module fallbacks, root/child dispatch, all C-string setter allocation positions and success. The setter first allocates a temporary String; failed temporary admission preserves the destination. Later failures retain null or empty payloads and exact notices, including two notices after temporary character failure. Retry reuses admitted handles/headers while recreating temporary storage. Checks cover callback reentry, request order, visible text, metadata generations, source bytes, repeated recovery and complete teardown. The previous host-string temporary skipped guest requests and fails the regression. The setter at `0x1c00` leaves guard residue in the return register; it is declared `void` and tests check state rather than fabricated status. From 8fec8f9243dc7aca4fbccdab592efd7080c76cb3 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:32:42 +0200 Subject: [PATCH 088/318] fix(libSceJson2): declare Value setter result contracts accurately --- core/libs/prx/libSceJson2/Export.cpp | 14 +++--- core/libs/tests/GuestJson.cpp | 46 ++++++++++---------- core/libs/tests/GuestJsonCallbackCopies.cpp | 2 +- core/libs/tests/GuestJsonElementRecovery.cpp | 2 +- core/libs/tests/GuestJsonValueLayout.cpp | 2 +- core/libs/tests/GuestJsonValueOwnership.cpp | 4 +- core/libs/tests/JsonCopyFailure.cpp | 2 +- docs/dev/debt/libSceJson2.md | 2 + 8 files changed, 37 insertions(+), 37 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index b994fae39b..4dcf4f4b78 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2961,10 +2961,10 @@ void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value* self, Value* other) { Destroy(temporary); } -int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value* self, ValueType type) { SetType(NodeOf(*self), type); return 0; } -int APS5_VABI _ZN3sce4Json5Value3setEb(Value* self, bool value) { SetType(NodeOf(*self), TypeBoolean); self->boolean = value; return 0; } -int APS5_VABI _ZN3sce4Json5Value3setEl(Value* self, std::int64_t value) { SetType(NodeOf(*self), TypeInteger); self->integer = value; return 0; } -int APS5_VABI _ZN3sce4Json5Value3setEm(Value* self, std::uint64_t value) { SetType(NodeOf(*self), TypeUInteger); self->uinteger = value; return 0; } +void APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value* self, ValueType type) { SetType(NodeOf(*self), type); } +void APS5_VABI _ZN3sce4Json5Value3setEb(Value* self, bool value) { SetType(NodeOf(*self), TypeBoolean); self->boolean = value; } +void APS5_VABI _ZN3sce4Json5Value3setEl(Value* self, std::int64_t value) { SetType(NodeOf(*self), TypeInteger); self->integer = value; } +void APS5_VABI _ZN3sce4Json5Value3setEm(Value* self, std::uint64_t value) { SetType(NodeOf(*self), TypeUInteger); self->uinteger = value; } void APS5_VABI _ZN3sce4Json5Value3setEd(Value* self, double value) { AssignReal(NodeOf(*self), value); } void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { if (text == nullptr) throw std::invalid_argument("sce::Json::Value::set: null text"); @@ -2973,11 +2973,10 @@ void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { SetType(NodeOf(*self), TypeString); if (self->type == TypeString) AssignText(*self->string, View(temporary.string), true); } -int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { const std::string copy(View(*text)); SetType(NodeOf(*self), TypeString); if (self->type == TypeString) AssignText(*self->string, copy, true); - return 0; } void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { if (!array->items) return; @@ -2989,9 +2988,8 @@ void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value* self, const Object* SetType(*self, TypeObject); if (self->object) AssignObject(*self->object, *object); } -int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value* self, const Value* other) { +void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value* self, const Value* other) { if (self != other) AssignNode(NodeOf(*self), NodeOf(*other)); - return 0; } ValueType APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value* self) { return NodeOf(*self).type; } diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index 6085b73027..1f0703fd1d 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -42,7 +42,7 @@ int APS5_VABI _ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEP int APS5_VABI _ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(Value*, SpecialFloatHandler, void*); int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, ElementAccessFailureHandler, void*); void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); -int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json6StringC1Ev(String*); @@ -104,8 +104,8 @@ void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value*, const Object*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5Value5clearEv(Value*); -int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); -int APS5_VABI _ZN3sce4Json5Value3setEm(Value*, std::uint64_t); +void APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5Value3setEm(Value*, std::uint64_t); void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value*, const char*); std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); const bool* APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value*); @@ -308,7 +308,7 @@ static void ObjectsAndArrays() { Value* first = _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha); Require(!_ZNK3sce4Json6Object5emptyEv(&object)); Require(_ZNK3sce4Json5Value7getTypeEv(first) == TypeNull); - Require(_ZN3sce4Json5Value3setEl(first, 1) == 0); + _ZN3sce4Json5Value3setEl(first, 1); _ZN3sce4Json5Value3setEPKc(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &beta), "b"); Require(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha) == first); @@ -395,7 +395,7 @@ static void ObjectPair() { Require(_ZNK3sce4Json5Value7getTypeEv(&pair.value) == TypeString); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&pair.value)), "forty-two") == 0); - Require(_ZN3sce4Json5Value3setEl(&value, 7) == 0); + _ZN3sce4Json5Value3setEl(&value, 7); _ZN3sce4Json6StringD1Ev(&key); _ZN3sce4Json5ValueD1Ev(&value); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), "answer") == 0); @@ -410,7 +410,7 @@ static void ArrayIteratorCopies() { for (std::int64_t i = 1; i <= 3; ++i) { Value item{}; _ZN3sce4Json5ValueC1Ev(&item); - Require(_ZN3sce4Json5Value3setEl(&item, i) == 0); + _ZN3sce4Json5Value3setEl(&item, i); _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &item); _ZN3sce4Json5ValueD1Ev(&item); } @@ -477,7 +477,7 @@ static void ObjectFind() { _ZN3sce4Json6Object8iteratorD1Ev(&found); _ZN3sce4Json6Object8iteratorD1Ev(&end); - Require(_ZN3sce4Json5Value3setEl(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha), 1) == 0); + _ZN3sce4Json5Value3setEl(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha), 1); Value* second = _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &beta); _ZN3sce4Json5Value3setEPKc(second, "b"); _ZNK3sce4Json6Object3endEv(&end, &object); @@ -524,8 +524,8 @@ static void ObjectIteratorCopies() { Value val1{}, val2{}; _ZN3sce4Json5ValueC1Ev(&val1); _ZN3sce4Json5ValueC1Ev(&val2); - Require(_ZN3sce4Json5Value3setEl(&val1, 10) == 0); - Require(_ZN3sce4Json5Value3setEl(&val2, 20) == 0); + _ZN3sce4Json5Value3setEl(&val1, 10); + _ZN3sce4Json5Value3setEl(&val2, 20); Pair p1{}, p2{}; _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&p1, &key1, &val1); _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&p2, &key2, &val2); @@ -577,7 +577,7 @@ static void ObjectInsert() { _ZN3sce4Json6StringC1EPKc(&key, "score"); Value value{}; _ZN3sce4Json5ValueC1Ev(&value); - Require(_ZN3sce4Json5Value3setEl(&value, 100) == 0); + _ZN3sce4Json5Value3setEl(&value, 100); Pair pair{}; _ZN3sce4Json6Object4PairC1ERKNS0_6StringERKNS0_5ValueE(&pair, &key, &value); @@ -590,7 +590,7 @@ static void ObjectInsert() { Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(&p->key), "score") == 0); Require(*_ZNK3sce4Json5Value10getIntegerEv(&p->value) == 100); - Require(_ZN3sce4Json5Value3setEl(&pair.value, 200) == 0); + _ZN3sce4Json5Value3setEl(&pair.value, 200); Iterator duplicate{}; Require(_ZN3sce4Json6Object6insertERKNS1_4PairE(&duplicate, &object, &pair) == &duplicate); Require(_ZNK3sce4Json6Object8iteratoreqERKS2_(&duplicate, &inserted)); @@ -740,12 +740,12 @@ static void ValueSwap() { Value v1{}, v2{}; _ZN3sce4Json5ValueC1Ev(&v1); _ZN3sce4Json5ValueC1Ev(&v2); - Require(_ZN3sce4Json5Value3setEl(&v1, 123) == 0); + _ZN3sce4Json5Value3setEl(&v1, 123); _ZN3sce4Json5Value3setEPKc(&v2, "hello"); _ZN3sce4Json5Value4swapERS1_(&v1, &v2); Require(*_ZNK3sce4Json5Value10getIntegerEv(&v2) == 123); Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&v1)), "hello") == 0); - Require(_ZN3sce4Json5Value3setEl(&v1, -456) == 0); + _ZN3sce4Json5Value3setEl(&v1, -456); const auto* firstStorage = _ZNK3sce4Json5Value10getIntegerEv(&v1); const auto* secondStorage = _ZNK3sce4Json5Value10getIntegerEv(&v2); _ZN3sce4Json5Value4swapERS1_(&v1, &v2); @@ -871,7 +871,7 @@ static void NullAccess(SetNullAccessCallback setCallback) { int context = 0; Require(setCallback(initializer, OnNullAccess, &context) == 0); _ZN3sce4Json5ValueC1Ev(&fallback); - Require(_ZN3sce4Json5Value3setEl(&fallback, 99) == 0); + _ZN3sce4Json5Value3setEl(&fallback, 99); Value root{}; _ZN3sce4Json5ValueC1Ev(&root); const char text[] = "{\"present\":\"text\",\"none\":null}"; @@ -927,7 +927,7 @@ static void ValueAccess() { Value* mutableList = _ZN3sce4Json5Value10referValueERKNS0_6StringE(&root, &key); Value* element = _ZN3sce4Json5Value10referValueEm(mutableList, 0); Require(element != nullptr); - Require(_ZN3sce4Json5Value3setEl(element, 15) == 0); + _ZN3sce4Json5Value3setEl(element, 15); Require(*_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5Value8getValueEm(list, 0)) == 15); Require(_ZN3sce4Json5Value10referValueEm(mutableList, 3) == nullptr); Require(_ZN3sce4Json5Value10referValueEm(&root, 0) != nullptr); @@ -1052,7 +1052,7 @@ static void ValueClear() { _ZN3sce4Json5Value5clearEv(&value); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); - Require(_ZN3sce4Json5Value3setEl(&value, 12345) == 0); + _ZN3sce4Json5Value3setEl(&value, 12345); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeInteger); _ZN3sce4Json5Value5clearEv(&value); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); @@ -1076,7 +1076,7 @@ static void ValueClear() { _ZN3sce4Json6StringD1Ev(&key); _ZN3sce4Json6ObjectD1Ev(&obj); - Require(_ZN3sce4Json5Value3setEl(&value, 777) == 0); + _ZN3sce4Json5Value3setEl(&value, 777); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeInteger); Require(*_ZNK3sce4Json5Value10getIntegerEv(&value) == 777); @@ -1112,7 +1112,7 @@ static void IntegerRepresentation() { } Require(*_ZNK3sce4Json5Value10getIntegerEv(&value) == -1); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeUInteger); - Require(_ZN3sce4Json5Value3setEl(&value, -1) == 0); + _ZN3sce4Json5Value3setEl(&value, -1); Require(*_ZNK3sce4Json5Value11getUIntegerEv(&value) == std::numeric_limits::max()); Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeInteger); for (const char* text : {"-0", "-1", "-9223372036854775808"}) { @@ -1160,7 +1160,7 @@ static void ValueNullAccess() { lastContext = nullptr; int context = 0; _ZN3sce4Json5ValueC1Ev(&fallback); - Require(_ZN3sce4Json5Value3setEl(&fallback, 99) == 0); + _ZN3sce4Json5Value3setEl(&fallback, 99); Value root{}; _ZN3sce4Json5ValueC1Ev(&root); const char text[] = "{\"present\":\"text\",\"none\":null}"; @@ -1201,7 +1201,7 @@ static void RequireMismatch(int calls, std::int32_t requested, const Value* valu static void ValueTypeMismatchHandler() { _ZN3sce4Json5ValueC1Ev(&replacement); - Require(_ZN3sce4Json5Value3setEl(&replacement, 42) == 0); + _ZN3sce4Json5Value3setEl(&replacement, 42); int context = 0; Value root{}; _ZN3sce4Json5ValueC1Ev(&root); @@ -1243,7 +1243,7 @@ static void ValueTypeMismatchHandler() { Require(*_ZNK3sce4Json5Value10getIntegerEv(&other) == 0); Require(mismatchCalls == 7); _ZN3sce4Json5Value3setEd(&other, 1.5); - Require(_ZN3sce4Json5Value3setERKS1_(&root, &other) == 0); + _ZN3sce4Json5Value3setERKS1_(&root, &other); Require(*_ZNK3sce4Json5Value10getIntegerEv(&root) == 42); RequireMismatch(8, TypeInteger, &root, &context); _ZN3sce4Json5Value5clearEv(&root); @@ -1346,11 +1346,11 @@ static void ValueOperatorBool() { Value scalar{}; _ZN3sce4Json5ValueC1Ev(&scalar); for (std::int64_t value : {std::int64_t{0}, std::int64_t{1}, std::int64_t{-1}}) { - Require(_ZN3sce4Json5Value3setEl(&scalar, value) == 0); + _ZN3sce4Json5Value3setEl(&scalar, value); Require(_ZNK3sce4Json5ValuecvbEv(&scalar) == (value != 0)); } for (std::uint64_t value : {std::uint64_t{0}, std::uint64_t{1}, std::uint64_t{0xffffffffffffffff}}) { - Require(_ZN3sce4Json5Value3setEm(&scalar, value) == 0); + _ZN3sce4Json5Value3setEm(&scalar, value); Require(_ZNK3sce4Json5ValuecvbEv(&scalar) == (value != 0)); } for (double value : {0.0, -0.0, 1.0, -1.0, std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { diff --git a/core/libs/tests/GuestJsonCallbackCopies.cpp b/core/libs/tests/GuestJsonCallbackCopies.cpp index 5e9e8b6f3e..473cead6c1 100644 --- a/core/libs/tests/GuestJsonCallbackCopies.cpp +++ b/core/libs/tests/GuestJsonCallbackCopies.cpp @@ -27,7 +27,7 @@ void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); -int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); diff --git a/core/libs/tests/GuestJsonElementRecovery.cpp b/core/libs/tests/GuestJsonElementRecovery.cpp index 9d9e051839..dd06f7294e 100644 --- a/core/libs/tests/GuestJsonElementRecovery.cpp +++ b/core/libs/tests/GuestJsonElementRecovery.cpp @@ -21,7 +21,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5Value5clearEv(Value*); -int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value*, const char*); int APS5_VABI _ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(Value*, Handler, void*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); diff --git a/core/libs/tests/GuestJsonValueLayout.cpp b/core/libs/tests/GuestJsonValueLayout.cpp index 02b619bc67..59b8652d51 100644 --- a/core/libs/tests/GuestJsonValueLayout.cpp +++ b/core/libs/tests/GuestJsonValueLayout.cpp @@ -24,7 +24,7 @@ void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); void APS5_VABI _ZN3sce4Json5Value5clearEv(Value*); -int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +void APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); const std::uint64_t* APS5_VABI _ZNK3sce4Json5Value11getUIntegerEv(const Value*); const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value*); diff --git a/core/libs/tests/GuestJsonValueOwnership.cpp b/core/libs/tests/GuestJsonValueOwnership.cpp index e03f3e0edf..1fec01a761 100644 --- a/core/libs/tests/GuestJsonValueOwnership.cpp +++ b/core/libs/tests/GuestJsonValueOwnership.cpp @@ -19,8 +19,8 @@ void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); -int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); -int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); +void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value*, Value*); const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value*); diff --git a/core/libs/tests/JsonCopyFailure.cpp b/core/libs/tests/JsonCopyFailure.cpp index 2b19fd9d47..306fd9d2f9 100644 --- a/core/libs/tests/JsonCopyFailure.cpp +++ b/core/libs/tests/JsonCopyFailure.cpp @@ -37,7 +37,7 @@ void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value*, const Array*); void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value*, const Object*); -int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); +void APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value*); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 806000da59..fbbb768b06 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -110,3 +110,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Partial element-recovery fallback](../../../core/libs/prx/libSceJson2/Export.cpp) - complex callback-created fallback payload failures and older SDK dispatch remain unverified. Eight SDK 6 reference cases cover failure of auxiliary/fallback/handler admission and success, with and without callback reentry. Registration retry reuses admitted metadata without repairing a missing fallback. Parsing preserves that missing state; root/child failures invoke the handler and return the module fallback instead. Checks cover request order, error/index/source/parent ABI, suppression of recursive dispatch, visible scalar mutation, zero-allocation dispatch under failure, stable auxiliary/fallback/record generations and complete teardown. A mutation recreating the missing fallback fails the retry regression. - [Element fallback String allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - collection payloads and thrown allocator callbacks during fallback mutation remain unverified. Twenty-four SDK 6 reference cases cover root/module fallbacks, root/child dispatch, all C-string setter allocation positions and success. The setter first allocates a temporary String; failed temporary admission preserves the destination. Later failures retain null or empty payloads and exact notices, including two notices after temporary character failure. Retry reuses admitted handles/headers while recreating temporary storage. Checks cover callback reentry, request order, visible text, metadata generations, source bytes, repeated recovery and complete teardown. The previous host-string temporary skipped guest requests and fails the regression. The setter at `0x1c00` leaves guard residue in the return register; it is declared `void` and tests check state rather than fabricated status. + +- [Value setter return ABI](../../../core/libs/prx/libSceJson2/Export.cpp) - String source alias/missing-header semantics and Value self-setting still require runtime audits. Type, Boolean, signed/unsigned integer, String and Value setters expose no defined status result. The type helper at `0x580` ends with guard verification; scalar wrappers at `0x1a90`, `0x1ae0` and `0x1b30` do not establish another result. The String overload at `0x1cc0` can return without setting a result; Value setting at `0x1e40` also leaves guard residue. Definitions and test declarations use `void`; tests retain content, layout, ownership and callback assertions instead of testing fabricated zero returns. The reference scalar-layout contract passes with the corrected declaration. Full guest integration remains unverified. From e237f1bfe64031cf5f3679cc759d82def1f78d6f Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:36:20 +0200 Subject: [PATCH 089/318] fix(libSceJson2): preserve String setter aliases and missing headers --- core/libs/prx/libSceJson2/Export.cpp | 4 +- core/libs/tests/GuestJsonValueAllocator.cpp | 56 +++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 4dcf4f4b78..5f776ebb84 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2974,9 +2974,9 @@ void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { if (self->type == TypeString) AssignText(*self->string, View(temporary.string), true); } void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { - const std::string copy(View(*text)); + if (!text->text) return; SetType(NodeOf(*self), TypeString); - if (self->type == TypeString) AssignText(*self->string, copy, true); + if (self->type == TypeString) AssignText(*self->string, View(*text), true); } void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { if (!array->items) return; diff --git a/core/libs/tests/GuestJsonValueAllocator.cpp b/core/libs/tests/GuestJsonValueAllocator.cpp index 7b8d40de52..07c2ef4784 100644 --- a/core/libs/tests/GuestJsonValueAllocator.cpp +++ b/core/libs/tests/GuestJsonValueAllocator.cpp @@ -14,6 +14,7 @@ struct Value { Value* parent; void* auxiliary; std::uint64_t payload; std::uint3 struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; struct Array { void* storage; }; struct Object { void* storage; }; +struct String { void* storage; }; using NullCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); using MismatchCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, const Value*, void*); using FloatCallback = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); @@ -25,6 +26,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); void APS5_VABI _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(Value*, std::int32_t); void APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value*, std::int32_t); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value*, const String*); void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); void APS5_VABI _ZN3sce4Json5Value3setEd(Value*, double); void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); @@ -427,6 +429,59 @@ static void CheckNestedCopies(ValueAllocationTrace& trace) { Require(trace.live.size() == baseline, "Nested partial-state teardown must release all admitted ownership"); } } +static void CheckStringSetterAliases(ValueAllocationTrace& trace) { + const auto baseline = trace.live; + for (unsigned kind = 0; kind < 4; ++kind) for (bool registered : {false, true}) for (bool fail : {false, true}) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + if (!kind) _ZN3sce4Json5ValueC1El(&value, 9); + else _ZN3sce4Json5ValueC1EPKc(&value, "original"); + if (registered) Require(Register(value, 0, &trace) == 0, "Prepare callback metadata before String setter alias testing"); + String missing{}, shared{}; + if (kind >= 2) shared.storage = *reinterpret_cast(value.payload); + const String* source = kind < 2 ? &missing : kind == 2 ? reinterpret_cast(value.payload) : &shared; + const Value before = value; const auto live = trace.live; + std::map> snapshots; + for (const auto& [pointer, size] : live) { + const auto* bytes = static_cast(pointer); snapshots.emplace(pointer, std::vector{bytes, bytes + size}); + } + const auto requests = trace.requests.size(); const auto frees = trace.frees.size(); const auto notices = trace.notifications; const auto nullFrees = trace.nullFrees; + trace.budget = fail ? 0 : -1; + _ZN3sce4Json5Value3setERKNS0_6StringE(&value, source); + trace.budget = -1; + Require(std::memcmp(&value, &before, sizeof(value)) == 0 && trace.live == live && trace.requests.size() == requests && trace.frees.size() == frees && trace.notifications == notices && trace.nullFrees == nullFrees, + "Missing String headers and shared-header self-setting must preserve metadata, payload and allocation effects"); + for (const auto& [pointer, bytes] : snapshots) Require(std::memcmp(pointer, bytes.data(), bytes.size()) == 0, "Ignored or aliased String setting must preserve all original allocation bytes"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live == baseline, "String setter alias cases must reclaim their actual payload and callback resources"); + } + for (unsigned budget = 0; budget <= 1; ++budget) { + Value sourceValue, destination; std::memset(&sourceValue, 0xa5, sizeof(sourceValue)); std::memset(&destination, 0xa5, sizeof(destination)); + _ZN3sce4Json5ValueC1EPKc(&sourceValue, "replacement"); _ZN3sce4Json5ValueC1EPKc(&destination, "old"); + const Value originalSource = sourceValue; + const auto* source = reinterpret_cast(sourceValue.payload); + const auto* sourceHeader = source->storage; const auto* sourceCharacters = *static_cast(sourceHeader); + const auto handle = destination.payload; const auto* header = *reinterpret_cast(handle); void* oldCharacters = *static_cast(header); + const auto requests = trace.requests.size(); const auto frees = trace.frees.size(); const auto notices = trace.notifications; + trace.budget = static_cast(budget); + _ZN3sce4Json5Value3setERKNS0_6StringE(&destination, source); + trace.budget = -1; + Require(destination.type == 5 && destination.payload == handle && *reinterpret_cast(handle) == header && + std::strcmp(*static_cast(header), budget ? "replacement" : "") == 0 && + trace.requests.size() == requests + 1 && trace.requests.back() == 12 && trace.notifications == notices + unsigned(!budget) && + trace.frees.size() == frees + 1 && trace.frees.back() == oldCharacters, + "Independent String sources must copy through the existing handle/header and retain empty text after character allocation failure"); + Require(std::memcmp(&sourceValue, &originalSource, sizeof(sourceValue)) == 0 && source->storage == sourceHeader && *static_cast(sourceHeader) == sourceCharacters && + std::strcmp(sourceCharacters, "replacement") == 0, "Distinct-source assignment must preserve all source ownership and text bytes"); + const auto retryRequests = trace.requests.size(); + _ZN3sce4Json5Value3setERKNS0_6StringE(&destination, source); + Require(trace.requests.size() == retryRequests + 1 && trace.requests.back() == 12 && destination.payload == handle && + std::strcmp(*static_cast(header), "replacement") == 0, "Independent retry must complete the copy using the retained destination header"); + _ZN3sce4Json5ValueD1Ev(&destination); _ZN3sce4Json5ValueD1Ev(&sourceValue); + Require(trace.live == baseline, "Distinct String source cases must reclaim both real payloads"); + } + +} + int main() { ValueAllocationTrace trace; ValueAllocationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 0, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); @@ -492,6 +547,7 @@ int main() { _ZN3sce4Json5ValueD1Ev(&value); Require(trace.live.size() == baseline, "Float conversion must release temporary and destination allocations"); } + CheckStringSetterAliases(trace); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate value allocation contract"); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index fbbb768b06..26e2cfb589 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -111,4 +111,6 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Element fallback String allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - collection payloads and thrown allocator callbacks during fallback mutation remain unverified. Twenty-four SDK 6 reference cases cover root/module fallbacks, root/child dispatch, all C-string setter allocation positions and success. The setter first allocates a temporary String; failed temporary admission preserves the destination. Later failures retain null or empty payloads and exact notices, including two notices after temporary character failure. Retry reuses admitted handles/headers while recreating temporary storage. Checks cover callback reentry, request order, visible text, metadata generations, source bytes, repeated recovery and complete teardown. The previous host-string temporary skipped guest requests and fails the regression. The setter at `0x1c00` leaves guard residue in the return register; it is declared `void` and tests check state rather than fabricated status. -- [Value setter return ABI](../../../core/libs/prx/libSceJson2/Export.cpp) - String source alias/missing-header semantics and Value self-setting still require runtime audits. Type, Boolean, signed/unsigned integer, String and Value setters expose no defined status result. The type helper at `0x580` ends with guard verification; scalar wrappers at `0x1a90`, `0x1ae0` and `0x1b30` do not establish another result. The String overload at `0x1cc0` can return without setting a result; Value setting at `0x1e40` also leaves guard residue. Definitions and test declarations use `void`; tests retain content, layout, ownership and callback assertions instead of testing fabricated zero returns. The reference scalar-layout contract passes with the corrected declaration. Full guest integration remains unverified. +- [Value setter return ABI](../../../core/libs/prx/libSceJson2/Export.cpp) - Value self-setting and String references invalidated by destination type changes still require runtime audits. Type, Boolean, signed/unsigned integer, String and Value setters expose no defined status result. The type helper at `0x580` ends with guard verification; scalar wrappers at `0x1a90`, `0x1ae0` and `0x1b30` do not establish another result. The String overload at `0x1cc0` can return without setting a result; Value setting at `0x1e40` also leaves guard residue. Definitions and test declarations use `void`; tests retain content, layout, ownership and callback assertions instead of testing fabricated zero returns. The reference scalar-layout contract passes with the corrected declaration. Full guest integration remains unverified. + +- [Value String-argument setter](../../../core/libs/prx/libSceJson2/Export.cpp) - sources nested inside a destination being replaced, binary aliases and thrown callbacks remain unverified. Eighteen SDK 6 reference cases cover missing headers on numeric/String destinations, direct/shared-header aliases, optional root callback metadata and allocation disabled/enabled. Those paths preserve all Value/allocation bytes and perform no allocator effects. Independent-source controls copy text through the existing handle/header, clear old characters before failed character admission and retry successfully. The old host-string snapshot changed missing-header destinations and reallocated self-aliases; it fails the regression. Checks retain source text/ownership and complete cleanup. The implementation uses the String header and underlying text alias check directly. From f8e4d8acb8521922bc40ad24da7d049773b51e46 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:43:05 +0200 Subject: [PATCH 090/318] fix(libSceJson2): preserve collection self-setting behavior --- core/libs/prx/libSceJson2/Export.cpp | 4 +- core/libs/tests/GuestJsonValueAllocator.cpp | 82 +++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 5f776ebb84..50b5d43a6a 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2945,7 +2945,7 @@ void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value* self, const Object* ob void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value* self, const Value* other) { CopyInto(*self, *other); } void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value* self) { Destroy(*self); } Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value* self, const Value* other) { - if (self != other) AssignNode(NodeOf(*self), NodeOf(*other)); + AssignNode(NodeOf(*self), NodeOf(*other)); return self; } void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value* self, Value* other) { @@ -2989,7 +2989,7 @@ void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value* self, const Object* if (self->object) AssignObject(*self->object, *object); } void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value* self, const Value* other) { - if (self != other) AssignNode(NodeOf(*self), NodeOf(*other)); + AssignNode(NodeOf(*self), NodeOf(*other)); } ValueType APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value* self) { return NodeOf(*self).type; } diff --git a/core/libs/tests/GuestJsonValueAllocator.cpp b/core/libs/tests/GuestJsonValueAllocator.cpp index 07c2ef4784..dc04d752ca 100644 --- a/core/libs/tests/GuestJsonValueAllocator.cpp +++ b/core/libs/tests/GuestJsonValueAllocator.cpp @@ -34,6 +34,8 @@ void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value*, const char*); void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value*, const Value*); Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); +void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value*, const Array*); void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value*, const Object*); void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value*, const Array*); @@ -482,6 +484,85 @@ static void CheckStringSetterAliases(ValueAllocationTrace& trace) { } +static void CheckValueSelfSetting(ValueAllocationTrace& trace) { + const auto baseline = trace.live; + for (int type = 0; type <= 7; ++type) for (bool setter : {false, true}) for (unsigned budget = 0; budget <= 2; ++budget) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + if (type == 5) _ZN3sce4Json5ValueC1EPKc(&value, "self"); + else _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&value, type); + Require(Register(value, 0, &trace, true) == 0, "Prepare callback ownership for Value self-setting"); + const Value before = value; const auto live = trace.live; + std::map> snapshots; + for (const auto& [pointer, size] : live) { + const auto* bytes = static_cast(pointer); snapshots.emplace(pointer, std::vector{bytes, bytes + size}); + } + void* header = type >= 6 ? *reinterpret_cast(value.payload) : nullptr; + void* sentinel = header ? *static_cast(header) : nullptr; + const auto requests = trace.requests.size(); const auto frees = trace.frees.size(); const auto notices = trace.notifications; const auto nullFrees = trace.nullFrees; + trace.budget = static_cast(budget); + if (setter) _ZN3sce4Json5Value3setERKS1_(&value, &value); + else Require(_ZN3sce4Json5ValueaSERKS1_(&value, &value) == &value, "Self-assignment must return its original receiver"); + trace.budget = -1; + Require(std::memcmp(&value, &before, sizeof(value)) == 0, "Every self-operation must preserve the inline Value, handle, callbacks and inactive bytes"); + const bool rebuild = type >= 6; + if (!rebuild) { + Require(trace.live == live && trace.requests.size() == requests && trace.frees.size() == frees && trace.notifications == notices && trace.nullFrees == nullFrees, + "Scalar and String self-operations must preserve allocation ownership without effects"); + } else { + std::vector expected{32}; if (budget) expected.push_back(type == 6 ? 48 : 104); + Require(std::vector(trace.requests.begin() + requests, trace.requests.end()) == expected && trace.notifications == notices + unsigned(budget < 2) && + trace.nullFrees == nullFrees + unsigned(!budget) && trace.frees.size() == frees + budget && trace.live.size() == live.size(), + "Collection self-setting must attempt replacement storage with exact requests, null frees and rollback effects"); + auto* replacement = *reinterpret_cast(value.payload); + Require((replacement == header) == (budget < 2) && static_cast(replacement)[1] == 0 && + static_cast(replacement)[3] == reinterpret_cast(&value), + "Self-setting must preserve failed replacement storage or publish a fresh empty collection bound to its original owner"); + if (budget == 2) Require(trace.frees[frees] == sentinel && trace.frees[frees + 1] == header, "Completed self-setting must reclaim the original sentinel before its header"); + } + for (const auto& [pointer, bytes] : snapshots) { + if (rebuild && budget == 2 && (pointer == header || pointer == sentinel || pointer == reinterpret_cast(value.payload))) continue; + Require(trace.live.contains(pointer) && std::memcmp(pointer, bytes.data(), bytes.size()) == 0, + "Self-setting must preserve every unaffected allocation, callback record and fallback byte"); + } + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live == baseline, "Self-operations must preserve complete destruction of original or replacement resources"); + } + for (bool array : {false, true}) for (bool setter : {false, true}) { + Value value; std::memset(&value, 0xa5, sizeof(value)); + if (array) PrepareNestedSource(value); + else { + _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(&value, 7); + Value key{}, scalar{}; _ZN3sce4Json5ValueC1EPKc(&key, "key"); _ZN3sce4Json5ValueC1El(&scalar, 9); + auto* member = _ZN3sce4Json6ObjectixERKNS0_6StringE(reinterpret_cast(value.payload), reinterpret_cast(key.payload)); + _ZN3sce4Json5ValueaSERKS1_(member, &scalar); + _ZN3sce4Json5ValueD1Ev(&scalar); _ZN3sce4Json5ValueD1Ev(&key); + } + Require(Register(value, 0, &trace, true) == 0, "Prepare metadata for nonempty collection self-use"); + const Value before = value; const auto* header = *reinterpret_cast(value.payload); + const auto state = array ? NestedState(value) : std::string{}; + const auto requests = trace.requests.size(); + if (setter) _ZN3sce4Json5Value3setERKS1_(&value, &value); + else Require(_ZN3sce4Json5ValueaSERKS1_(&value, &value) == &value, "Nonempty assignment must return its original receiver"); + Require(std::memcmp(&value, &before, sizeof(value)) == 0 && *reinterpret_cast(value.payload) != header && trace.requests.size() > requests, + "Nonempty collection self-use must rebuild storage while retaining the Value handle and callback metadata"); + if (array) Require(state == "6po3[5p:abc;2p42;6p!1[5p:def;];]" && NestedState(value) == "6po3[5p:abc;2p42;6po1[5p:def;];]", "Nested self-use must preserve contents and parent links while rebinding the copied nested header owner"); + else { + auto* replacement = *reinterpret_cast(value.payload); + auto* head = *static_cast(replacement); + auto* node = *static_cast(head); + const auto* bytes = static_cast(node); + const auto* child = reinterpret_cast(bytes + 64); + const auto* keyHeader = *reinterpret_cast(bytes + 48); + Require(static_cast(replacement)[1] == 1 && child->type == 2 && child->payload == 9 && child->parent == &value && + std::strcmp(*reinterpret_cast(bytes + 24), "key") == 0 && std::strcmp(*static_cast(keyHeader), "key") == 0, + "Object self-use must preserve ordered internal/public keys, child contents and owner binding"); + } + _ZN3sce4Json5ValueD1Ev(&value); + Require(trace.live == baseline, "Nonempty self-use must reclaim both replaced and final collection storage"); + } + +} + int main() { ValueAllocationTrace trace; ValueAllocationAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 0, {}}; std::uint8_t initializer = 0; _ZN3sce4Json11InitializerC1Ev(&initializer); @@ -548,6 +629,7 @@ int main() { Require(trace.live.size() == baseline, "Float conversion must release temporary and destination allocations"); } CheckStringSetterAliases(trace); + CheckValueSelfSetting(trace); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), "Terminate value allocation contract"); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 26e2cfb589..f684904806 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -111,6 +111,8 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Element fallback String allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - collection payloads and thrown allocator callbacks during fallback mutation remain unverified. Twenty-four SDK 6 reference cases cover root/module fallbacks, root/child dispatch, all C-string setter allocation positions and success. The setter first allocates a temporary String; failed temporary admission preserves the destination. Later failures retain null or empty payloads and exact notices, including two notices after temporary character failure. Retry reuses admitted handles/headers while recreating temporary storage. Checks cover callback reentry, request order, visible text, metadata generations, source bytes, repeated recovery and complete teardown. The previous host-string temporary skipped guest requests and fails the regression. The setter at `0x1c00` leaves guard residue in the return register; it is declared `void` and tests check state rather than fabricated status. -- [Value setter return ABI](../../../core/libs/prx/libSceJson2/Export.cpp) - Value self-setting and String references invalidated by destination type changes still require runtime audits. Type, Boolean, signed/unsigned integer, String and Value setters expose no defined status result. The type helper at `0x580` ends with guard verification; scalar wrappers at `0x1a90`, `0x1ae0` and `0x1b30` do not establish another result. The String overload at `0x1cc0` can return without setting a result; Value setting at `0x1e40` also leaves guard residue. Definitions and test declarations use `void`; tests retain content, layout, ownership and callback assertions instead of testing fabricated zero returns. The reference scalar-layout contract passes with the corrected declaration. Full guest integration remains unverified. +- [Value setter return ABI](../../../core/libs/prx/libSceJson2/Export.cpp) - String references invalidated by destination type changes still require runtime audits. Type, Boolean, signed/unsigned integer, String and Value setters expose no defined status result. The type helper at `0x580` ends with guard verification; scalar wrappers at `0x1a90`, `0x1ae0` and `0x1b30` do not establish another result. The String overload at `0x1cc0` can return without setting a result; Value setting at `0x1e40` also leaves guard residue. Definitions and test declarations use `void`; tests retain content, layout, ownership and callback assertions instead of testing fabricated zero returns. The reference scalar-layout contract passes with the corrected declaration. Full guest integration remains unverified. - [Value String-argument setter](../../../core/libs/prx/libSceJson2/Export.cpp) - sources nested inside a destination being replaced, binary aliases and thrown callbacks remain unverified. Eighteen SDK 6 reference cases cover missing headers on numeric/String destinations, direct/shared-header aliases, optional root callback metadata and allocation disabled/enabled. Those paths preserve all Value/allocation bytes and perform no allocator effects. Independent-source controls copy text through the existing handle/header, clear old characters before failed character admission and retry successfully. The old host-string snapshot changed missing-header destinations and reallocated self-aliases; it fails the regression. Checks retain source text/ownership and complete cleanup. The implementation uses the String header and underlying text alias check directly. + +- [Value self-setting and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - older SDKs, non-finite recovery and interrupted nonempty self-copies remain unverified. Fifty-two SDK 6 reference cases cover all eight types through both entry points, empty collection admission failures and healthy nonempty Array/Object copies. Assignment at `0x20e0` unconditionally calls the setter and returns its receiver. Both operations rebuild collections while retaining the Value handle and callback metadata. Scalars and Strings naturally preserve their storage. Checks cover replacement/null-free sequences, rollback, inactive bytes, unaffected allocations, internal/public keys, nested contents and the observed nested-header owner rebinding. The old self-guards suppress replacement and fail the regression. All original/replacement storage is reclaimed on destruction. From de8f64de05821f37b70fb02af3a46719fa7a422d Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:49:26 +0200 Subject: [PATCH 091/318] fix(libSceJson2): preserve legacy missing-sentinel retry effects --- core/libs/prx/libSceJson2/CMakeLists.txt | 9 ++ core/libs/prx/libSceJson2/Export.cpp | 3 +- core/libs/tests/GuestJsonLegacySelf.cpp | 105 +++++++++++++++++++++++ docs/dev/debt/libSceJson2.md | 4 +- 4 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 core/libs/tests/GuestJsonLegacySelf.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 43ba327c07..76fc757623 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -4,6 +4,15 @@ target_include_directories(libSceJson2 PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceJson2) if(BUILD_TESTING) + add_test_executable(guest_json_legacy_self_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonLegacySelf.cpp Export.cpp) + target_include_directories(guest_json_legacy_self_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_legacy_self_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_json_legacy_self_tests) + add_test(NAME guest_json_legacy_self COMMAND guest_json_legacy_self_tests) + if(WIN32) + set_tests_properties(guest_json_legacy_self PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() add_test_executable(guest_json_termination_exceptions_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonTerminationExceptions.cpp) target_include_directories(guest_json_termination_exceptions_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_json_termination_exceptions_tests PRIVATE libSceJson2 libkernel libc) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 50b5d43a6a..0296558fba 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1059,7 +1059,8 @@ void CopyObjectStorage(ObjectStorage& destination, const ObjectStorage& source) replacement->parent = CloneObjectTree(source.head->parent, replacement, replacement); if (replacement->parent != replacement) { pending.count = source.count; pending.RefreshEndpoints(); } } - destination.clear(); Release(destination.head); + destination.clear(); + if (destination.head) Release(destination.head); destination.head = replacement; destination.count = pending.count; } diff --git a/core/libs/tests/GuestJsonLegacySelf.cpp b/core/libs/tests/GuestJsonLegacySelf.cpp new file mode 100644 index 0000000000..c69e9c5aaa --- /dev/null +++ b/core/libs/tests/GuestJsonLegacySelf.cpp @@ -0,0 +1,105 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct Value { Value* parent; void* auxiliary; std::uintptr_t payload; unsigned reserved; int type; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; int mode; unsigned reserved[3]; }; +using NullHandler = const Value& (APS5_VABI*)(int, const Value*, void*); +extern "C" { +void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value*, const Value*); +Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value*, const Value*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(Value*, NullHandler, void*); +} +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + static const std::uint32_t words[]{0, 0, 0, 0, 0x05000000}; + return words; +} +static void Require(bool condition, const char* message) { + if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} +struct LegacySelfTrace { + std::map live; + std::map generations; + std::vector requests; + std::vector releases; + unsigned nextGeneration = 0; + unsigned notices = 0; + unsigned nullFrees = 0; + int budget = -1; +}; +static void* APS5_VABI AllocateSelf(void*, std::size_t size, void* data) { + auto& trace = *static_cast(data); trace.requests.push_back(size); + if (!trace.budget) return nullptr; + if (trace.budget > 0) --trace.budget; + void* memory = std::malloc(size); + Require(memory && trace.live.emplace(memory, size).second && trace.generations.emplace(memory, ++trace.nextGeneration).second, "Self-setting allocations require distinct live generations"); + std::memset(memory, 0xa5, size); return memory; +} +static void APS5_VABI ReleaseSelf(void*, void* memory, void* data) { + auto& trace = *static_cast(data); + if (!memory) { ++trace.nullFrees; return; } + Require(trace.live.erase(memory) == 1 && trace.generations.erase(memory) == 1, "Legacy self-setting must release each owned allocation exactly once"); + trace.releases.push_back(memory); std::free(memory); +} +static void APS5_VABI NotifySelf(void*, int error, std::size_t size, void* data) { + auto& trace = *static_cast(data); + Require(static_cast(error) == 0x80848102 && trace.requests.back() == size, "Legacy self-setting failure must identify the rejected sentinel"); + ++trace.notices; +} +static void* const SelfTable[]{nullptr, nullptr, reinterpret_cast(AllocateSelf), reinterpret_cast(ReleaseSelf), reinterpret_cast(NotifySelf)}; +struct LegacySelfAllocator { void* const* table = SelfTable; }; +int main() { + for (bool object : {false, true}) for (bool setter : {false, true}) for (unsigned budget = 0; budget <= 2; ++budget) { + LegacySelfTrace trace; LegacySelfAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0 && trace.live.size() == 11, "Initialize the eleven-resource legacy module"); + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, object ? "{}" : "[]", 2) == 0 && + _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&value, nullptr, &trace) == 0, "Prepare legacy collection ownership and auxiliary callback context"); + const Value before = value; const auto generations = trace.generations; + void* header = *reinterpret_cast(value.payload); void* sentinel = *static_cast(header); + Require(trace.live.at(header) == 24 && trace.live.at(sentinel) == (object ? 104u : 48u), "Legacy collections must retain their 24-byte header and actual sentinel"); + std::map> snapshots; + for (const auto& [pointer, size] : trace.live) { + if (pointer == header || pointer == sentinel) continue; + const auto* bytes = static_cast(pointer); snapshots.emplace(pointer, std::vector{bytes, bytes + size}); + } + const auto requests = trace.requests.size(); const auto releases = trace.releases.size(); const auto notices = trace.notices; const auto nullFrees = trace.nullFrees; + trace.budget = static_cast(budget); + if (setter) _ZN3sce4Json5Value3setERKS1_(&value, &value); + else Require(_ZN3sce4Json5ValueaSERKS1_(&value, &value) == &value, "Legacy self-assignment must return its receiver"); + trace.budget = -1; + auto* words = static_cast(header); + Require(std::memcmp(&value, &before, sizeof(value)) == 0 && *reinterpret_cast(value.payload) == header && words[1] == 0 && + trace.requests.size() == requests + 1 && trace.requests.back() == (object ? 104u : 48u) && trace.notices == notices + unsigned(!budget) && + trace.releases.size() == releases + 1 && trace.releases.back() == sentinel && trace.nullFrees == nullFrees, + "Legacy self-use must keep its header and metadata while releasing the original sentinel even when replacement admission fails"); + Require((words[0] != 0) == bool(budget), "Failed legacy replacement must retain a missing sentinel rather than restore the old one"); + if (budget) Require(reinterpret_cast(words[0]) != sentinel && trace.live.at(reinterpret_cast(words[0])) == (object ? 104u : 48u), "Successful legacy replacement must publish a fresh sentinel"); + for (const auto& [pointer, generation] : generations) if (pointer != sentinel) Require(trace.generations.at(pointer) == generation, "Self-use must preserve all non-sentinel allocation generations"); + for (const auto& [pointer, bytes] : snapshots) Require(std::memcmp(pointer, bytes.data(), bytes.size()) == 0, "Self-use must preserve callback, fallback, handle and module bytes"); + const auto retryRequests = trace.requests.size(); const auto retryReleases = trace.releases.size(); const auto retryNullFrees = trace.nullFrees; const auto retryNotices = trace.notices; + const auto previousSentinel = words[0]; + if (setter) _ZN3sce4Json5Value3setERKS1_(&value, &value); + else Require(_ZN3sce4Json5ValueaSERKS1_(&value, &value) == &value, "Legacy self-use retry must return its original receiver"); + Require(words[0] && words[0] != previousSentinel && words[1] == 0 && std::memcmp(&value, &before, sizeof(value)) == 0 && + trace.requests.size() == retryRequests + 1 && trace.requests.back() == (object ? 104u : 48u) && trace.releases.size() == retryReleases + unsigned(previousSentinel != 0) && + trace.nullFrees == retryNullFrees && trace.notices == retryNotices, + "Same-Value retry must repair a missing sentinel without fabricating a null free or replacing its header and metadata"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.live.empty() && trace.generations.empty(), "Partial legacy collections and module resources must remain destructible"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0 && _ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && trace.live.empty(), "A completed legacy self-use case must permit a fresh module generation"); + } + return 0; +} diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index f684904806..7a5e022d16 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -115,4 +115,6 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Value String-argument setter](../../../core/libs/prx/libSceJson2/Export.cpp) - sources nested inside a destination being replaced, binary aliases and thrown callbacks remain unverified. Eighteen SDK 6 reference cases cover missing headers on numeric/String destinations, direct/shared-header aliases, optional root callback metadata and allocation disabled/enabled. Those paths preserve all Value/allocation bytes and perform no allocator effects. Independent-source controls copy text through the existing handle/header, clear old characters before failed character admission and retry successfully. The old host-string snapshot changed missing-header destinations and reallocated self-aliases; it fails the regression. Checks retain source text/ownership and complete cleanup. The implementation uses the String header and underlying text alias check directly. -- [Value self-setting and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - older SDKs, non-finite recovery and interrupted nonempty self-copies remain unverified. Fifty-two SDK 6 reference cases cover all eight types through both entry points, empty collection admission failures and healthy nonempty Array/Object copies. Assignment at `0x20e0` unconditionally calls the setter and returns its receiver. Both operations rebuild collections while retaining the Value handle and callback metadata. Scalars and Strings naturally preserve their storage. Checks cover replacement/null-free sequences, rollback, inactive bytes, unaffected allocations, internal/public keys, nested contents and the observed nested-header owner rebinding. The old self-guards suppress replacement and fail the regression. All original/replacement storage is reclaimed on destruction. +- [Value self-setting and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nonempty older-SDK self-copies, non-finite recovery and interrupted nonempty self-copies remain unverified. Fifty-two SDK 6 reference cases cover all eight types through both entry points, empty collection admission failures and healthy nonempty Array/Object copies. Assignment at `0x20e0` unconditionally calls the setter and returns its receiver. Both operations rebuild collections while retaining the Value handle and callback metadata. Scalars and Strings naturally preserve their storage. Checks cover replacement/null-free sequences, rollback, inactive bytes, unaffected allocations, internal/public keys, nested contents and the observed nested-header owner rebinding. The old self-guards suppress replacement and fail the regression. All original/replacement storage is reclaimed on destruction. + +- [Legacy collection self-use](../../../core/libs/prx/libSceJson2/Export.cpp) - nonempty/child-callback self-copies and concurrent ownership remain unverified. Twelve SDK 5 reference cases cover empty Array/Object setters and assignment, sentinel allocation failure and success. Both entry points retain the 24-byte collection header and Value/auxiliary ownership while replacing the sentinel. Failure still releases the old sentinel and leaves a destructible missing-sentinel header. Same-Value retry repairs it without forwarding a null free. Checks cover exact requests/releases, live generations, unchanged callback/fallback/module bytes, partial destruction and fresh module generations. The previous Object replacement helper forwards an extra null free and fails the retry regression. From 10905c0a31aed3f428a58e5ad7a3fdff8ea9a0c3 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 19:56:51 +0200 Subject: [PATCH 092/318] test(libSceJson2): verify nonempty legacy self-use failures --- core/libs/tests/GuestJsonLegacySelf.cpp | 119 +++++++++++++++++++++++- docs/dev/debt/libSceJson2.md | 6 +- 2 files changed, 122 insertions(+), 3 deletions(-) diff --git a/core/libs/tests/GuestJsonLegacySelf.cpp b/core/libs/tests/GuestJsonLegacySelf.cpp index c69e9c5aaa..cf36125a0a 100644 --- a/core/libs/tests/GuestJsonLegacySelf.cpp +++ b/core/libs/tests/GuestJsonLegacySelf.cpp @@ -6,6 +6,7 @@ #include #include #include +#include struct Value { Value* parent; void* auxiliary; std::uintptr_t payload; unsigned reserved; int type; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; int mode; unsigned reserved[3]; }; @@ -37,10 +38,12 @@ struct LegacySelfTrace { unsigned notices = 0; unsigned nullFrees = 0; int budget = -1; + bool once = false; + std::vector releaseIds; }; static void* APS5_VABI AllocateSelf(void*, std::size_t size, void* data) { auto& trace = *static_cast(data); trace.requests.push_back(size); - if (!trace.budget) return nullptr; + if (!trace.budget) { if (trace.once) trace.budget = -1; return nullptr; } if (trace.budget > 0) --trace.budget; void* memory = std::malloc(size); Require(memory && trace.live.emplace(memory, size).second && trace.generations.emplace(memory, ++trace.nextGeneration).second, "Self-setting allocations require distinct live generations"); @@ -49,6 +52,7 @@ static void* APS5_VABI AllocateSelf(void*, std::size_t size, void* data) { static void APS5_VABI ReleaseSelf(void*, void* memory, void* data) { auto& trace = *static_cast(data); if (!memory) { ++trace.nullFrees; return; } + trace.releaseIds.push_back(trace.generations.at(memory)); Require(trace.live.erase(memory) == 1 && trace.generations.erase(memory) == 1, "Legacy self-setting must release each owned allocation exactly once"); trace.releases.push_back(memory); std::free(memory); } @@ -59,7 +63,120 @@ static void APS5_VABI NotifySelf(void*, int error, std::size_t size, void* data) } static void* const SelfTable[]{nullptr, nullptr, reinterpret_cast(AllocateSelf), reinterpret_cast(ReleaseSelf), reinterpret_cast(NotifySelf)}; struct LegacySelfAllocator { void* const* table = SelfTable; }; +struct SelfListLink { SelfListLink* previous; SelfListLink* next; Value value; }; +struct SelfTreeLink { + SelfTreeLink* parent; SelfTreeLink* left; SelfTreeLink* right; + const char* key; unsigned length; unsigned capacity; std::uint64_t reserved; + void* publicKey; std::uint64_t padding; Value value; unsigned char color; unsigned char nil; unsigned char tail[6]; +}; +static_assert(sizeof(SelfListLink) == 48 && sizeof(SelfTreeLink) == 104); +static std::string SelfItemState(const Value& value, const Value* owner) { + return std::to_string(value.type) + (value.parent == owner ? "p" : value.parent ? "x" : "n") + std::to_string(value.payload); +} +static void SelfTreeState(const SelfTreeLink* node, const Value* owner, std::string& output, unsigned depth = 0) { + Require(depth < 8, "Replacement tree traversal must remain bounded"); + if (node->nil) return; + SelfTreeState(node->left, owner, output, depth + 1); + output += std::string(node->key) + "/" + (node->publicKey ? *static_cast(node->publicKey) : "?") + "=" + SelfItemState(node->value, owner) + ";"; + SelfTreeState(node->right, owner, output, depth + 1); +} +static std::string SelfState(const Value& value, bool object) { + auto* words = *reinterpret_cast(value.payload); + std::string output = std::to_string(words[1]) + "["; + if (!words[0]) return output + "missing]"; + if (object) { + auto* head = reinterpret_cast(words[0]); SelfTreeState(head->parent, &value, output); + } else { + auto* head = reinterpret_cast(words[0]); unsigned count = 0; + for (auto* node = head->next; node != head; node = node->next) { + Require(++count < 8, "Replacement list traversal must remain bounded"); output += SelfItemState(node->value, &value) + ";"; + } + } + return output + "]"; +} +template +static std::uint64_t SelfOrder(const std::vector& values, std::size_t begin) { + std::uint64_t hash = UINT64_C(14695981039346656037); + for (auto i = begin; i < values.size(); ++i) { hash ^= values[i]; hash *= UINT64_C(1099511628211); } + return hash; +} +struct NonemptySelfCase { + bool object; bool once; unsigned budget; + std::size_t requests; std::uint64_t requestOrder; unsigned notices; std::size_t releases; std::uint64_t releaseOrder; const char* state; + std::size_t retryRequests; std::uint64_t retryRequestOrder; std::size_t retryReleases; std::uint64_t retryReleaseOrder; const char* retryState; +}; +static const NonemptySelfCase NonemptySelfCases[]{ + {0, 0, 0, 1, UINT64_C(12638135523509116079), 1, 3, UINT64_C(6554792551179442466), "0[missing]", 1, UINT64_C(12638135523509116079), 0, UINT64_C(14695981039346656037), "0[]"}, + {0, 0, 1, 2, UINT64_C(575361273772712493), 1, 3, UINT64_C(6554792551179442466), "0[]", 1, UINT64_C(12638135523509116079), 1, UINT64_C(12638167409346334198), "0[]"}, + {0, 0, 2, 3, UINT64_C(5656224367409952583), 1, 3, UINT64_C(6554792551179442466), "1[3n11;]", 2, UINT64_C(575361273772712493), 2, UINT64_C(613651766217591568), "1[3n11;]"}, + {0, 0, 3, 3, UINT64_C(5656224367409952583), 0, 3, UINT64_C(6554792551179442466), "2[3n11;3n22;]", 3, UINT64_C(5656224367409952583), 3, UINT64_C(12213400556975252055), "2[3n11;3n22;]"}, + {0, 1, 0, 1, UINT64_C(12638135523509116079), 1, 3, UINT64_C(6554792551179442466), "0[missing]", 1, UINT64_C(12638135523509116079), 0, UINT64_C(14695981039346656037), "0[]"}, + {0, 1, 1, 2, UINT64_C(575361273772712493), 1, 3, UINT64_C(6554792551179442466), "0[]", 1, UINT64_C(12638135523509116079), 1, UINT64_C(12638167409346334198), "0[]"}, + {0, 1, 2, 3, UINT64_C(5656224367409952583), 1, 3, UINT64_C(6554792551179442466), "1[3n11;]", 2, UINT64_C(575361273772712493), 2, UINT64_C(613651766217591568), "1[3n11;]"}, + {0, 1, 3, 3, UINT64_C(5656224367409952583), 0, 3, UINT64_C(6554792551179442466), "2[3n11;3n22;]", 3, UINT64_C(5656224367409952583), 3, UINT64_C(12213400556975252055), "2[3n11;3n22;]"}, + {1, 0, 0, 1, UINT64_C(12638197096160295895), 1, 9, UINT64_C(7899594896565220821), "0[missing]", 1, UINT64_C(12638197096160295895), 0, UINT64_C(14695981039346656037), "0[]"}, + {1, 0, 1, 2, UINT64_C(628920684195530125), 1, 9, UINT64_C(7899594896565220821), "0[]", 1, UINT64_C(12638197096160295895), 1, UINT64_C(12638126727416090391), "0[]"}, + {1, 0, 2, 5, UINT64_C(17406812458522649141), 3, 9, UINT64_C(7899594896565220821), "2[/?=3n11;]", 2, UINT64_C(628920684195530125), 2, UINT64_C(566743301632497700), "2[/?=3n11;]"}, + {1, 0, 3, 5, UINT64_C(17406812458522649141), 2, 9, UINT64_C(7899594896565220821), "2[a/?=3n11;]", 3, UINT64_C(3708358159552328189), 3, UINT64_C(1887292020941694844), "2[a/?=3n11;]"}, + {1, 0, 4, 6, UINT64_C(10307126579844094685), 2, 9, UINT64_C(7899594896565220821), "2[a/=3n11;]", 4, UINT64_C(8967656197913659423), 4, UINT64_C(12385604588108498361), "2[a/=3n11;]"}, + {1, 0, 5, 6, UINT64_C(10307126579844094685), 1, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;]", 5, UINT64_C(17406713502476110151), 5, UINT64_C(16291384495448556647), "2[a/a=3n11;]"}, + {1, 0, 6, 8, UINT64_C(8732740249579296847), 2, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;/?=3n22;]", 6, UINT64_C(10307126579844094685), 6, UINT64_C(6796161341782386540), "2[a/a=3n11;/?=3n22;]"}, + {1, 0, 7, 8, UINT64_C(8732740249579296847), 1, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;b/?=3n22;]", 7, UINT64_C(12001570155012663533), 7, UINT64_C(16058915569980617336), "2[a/a=3n11;b/?=3n22;]"}, + {1, 0, 8, 9, UINT64_C(15869276825689737943), 1, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;b/=3n22;]", 8, UINT64_C(8732740249579296847), 8, UINT64_C(13124253386643929941), "2[a/a=3n11;b/=3n22;]"}, + {1, 0, 9, 9, UINT64_C(15869276825689737943), 0, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;b/b=3n22;]", 9, UINT64_C(15869276825689737943), 9, UINT64_C(5256107193438867751), "2[a/a=3n11;b/b=3n22;]"}, + {1, 1, 0, 1, UINT64_C(12638197096160295895), 1, 9, UINT64_C(7899594896565220821), "0[missing]", 1, UINT64_C(12638197096160295895), 0, UINT64_C(14695981039346656037), "0[]"}, + {1, 1, 1, 2, UINT64_C(628920684195530125), 1, 9, UINT64_C(7899594896565220821), "0[]", 1, UINT64_C(12638197096160295895), 1, UINT64_C(12638126727416090391), "0[]"}, + {1, 1, 2, 9, UINT64_C(15869276825689737943), 1, 9, UINT64_C(7899594896565220821), "2[/a=3n11;b/b=3n22;]", 8, UINT64_C(16025805325695402919), 8, UINT64_C(13124253386643929941), "2[/a=3n11;b/b=3n22;]"}, + {1, 1, 3, 8, UINT64_C(10981429471021977615), 1, 9, UINT64_C(7899594896565220821), "2[a/?=3n11;b/b=3n22;]", 7, UINT64_C(12136851201209200173), 7, UINT64_C(16058915569980617336), "2[a/?=3n11;b/b=3n22;]"}, + {1, 1, 4, 9, UINT64_C(15869276825689737943), 1, 9, UINT64_C(7899594896565220821), "2[a/=3n11;b/b=3n22;]", 8, UINT64_C(10981429471021977615), 8, UINT64_C(13124253386643929941), "2[a/=3n11;b/b=3n22;]"}, + {1, 1, 5, 6, UINT64_C(10307126579844094685), 1, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;]", 5, UINT64_C(17406713502476110151), 5, UINT64_C(16291384495448556647), "2[a/a=3n11;]"}, + {1, 1, 6, 9, UINT64_C(15869276825689737943), 1, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;/b=3n22;]", 8, UINT64_C(8707858301437801767), 8, UINT64_C(13124253386643929941), "2[a/a=3n11;/b=3n22;]"}, + {1, 1, 7, 8, UINT64_C(8732740249579296847), 1, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;b/?=3n22;]", 7, UINT64_C(12001570155012663533), 7, UINT64_C(16058915569980617336), "2[a/a=3n11;b/?=3n22;]"}, + {1, 1, 8, 9, UINT64_C(15869276825689737943), 1, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;b/=3n22;]", 8, UINT64_C(8732740249579296847), 8, UINT64_C(13124253386643929941), "2[a/a=3n11;b/=3n22;]"}, + {1, 1, 9, 9, UINT64_C(15869276825689737943), 0, 9, UINT64_C(7899594896565220821), "2[a/a=3n11;b/b=3n22;]", 9, UINT64_C(15869276825689737943), 9, UINT64_C(5256107193438867751), "2[a/a=3n11;b/b=3n22;]"}, +}; + +static void CheckNonemptySelfUse() { + for (const auto& expected : NonemptySelfCases) for (bool setter : {false, true}) { + LegacySelfTrace trace; LegacySelfAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0, "Initialize nonempty legacy self-use"); + const auto module = trace.generations; + Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + const char* text = expected.object ? "{\"a\":11,\"b\":22}" : "[11,22]"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, text, std::strlen(text)) == 0 && + _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&value, nullptr, &trace) == 0, "Prepare nonempty collection and callback context"); + const Value original = value; void* header = *reinterpret_cast(value.payload); + auto protectedGenerations = module; + for (void* pointer : {reinterpret_cast(value.payload), value.auxiliary, static_cast(value.auxiliary)[2]}) protectedGenerations.emplace(pointer, trace.generations.at(pointer)); + std::map> snapshots; + for (const auto& [pointer, generation] : protectedGenerations) { + const auto* bytes = static_cast(pointer); snapshots.emplace(pointer, std::vector{bytes, bytes + trace.live.at(pointer)}); + } + const auto requests = trace.requests.size(); const auto releases = trace.releaseIds.size(); const auto notices = trace.notices; const auto nullFrees = trace.nullFrees; + trace.budget = static_cast(expected.budget); trace.once = expected.once; + if (setter) _ZN3sce4Json5Value3setERKS1_(&value, &value); + else Require(_ZN3sce4Json5ValueaSERKS1_(&value, &value) == &value, "Nonempty legacy assignment must retain its receiver identity"); + trace.budget = -1; trace.once = false; + Require(trace.requests.size() == requests + expected.requests && SelfOrder(trace.requests, requests) == expected.requestOrder && trace.notices == notices + expected.notices && + trace.releaseIds.size() == releases + expected.releases && SelfOrder(trace.releaseIds, releases) == expected.releaseOrder && trace.nullFrees == nullFrees && SelfState(value, expected.object) == expected.state, + "Legacy self-use must preserve exact admission/release ordering, partial contents, key failures, parent links and retained counts"); + const auto retryRequests = trace.requests.size(); const auto retryReleases = trace.releaseIds.size(); const auto retryNotices = trace.notices; + if (setter) _ZN3sce4Json5Value3setERKS1_(&value, &value); + else Require(_ZN3sce4Json5ValueaSERKS1_(&value, &value) == &value, "Partial self-use retry must retain its receiver identity"); + Require(trace.requests.size() == retryRequests + expected.retryRequests && SelfOrder(trace.requests, retryRequests) == expected.retryRequestOrder && trace.notices == retryNotices && + trace.releaseIds.size() == retryReleases + expected.retryReleases && SelfOrder(trace.releaseIds, retryReleases) == expected.retryReleaseOrder && trace.nullFrees == nullFrees && SelfState(value, expected.object) == expected.retryState, + "Independent retry must clone the actual partial collection without inventing lost entries or repairing observed count/key differences"); + Require(std::memcmp(&value, &original, sizeof(value)) == 0 && *reinterpret_cast(value.payload) == header, "Both self-use attempts must retain Value, header and callback ownership"); + for (const auto& [pointer, generation] : protectedGenerations) Require(trace.generations.at(pointer) == generation && std::memcmp(pointer, snapshots.at(pointer).data(), snapshots.at(pointer).size()) == 0, + "Self-use failure and retry must preserve every module, handle, callback and fallback generation and byte"); + _ZN3sce4Json5ValueD1Ev(&value); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.live.empty() && trace.generations.empty(), "All partial nonempty collections and module allocations must be reclaimed"); + } +} + int main() { + CheckNonemptySelfUse(); for (bool object : {false, true}) for (bool setter : {false, true}) for (unsigned budget = 0; budget <= 2; ++budget) { LegacySelfTrace trace; LegacySelfAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 2, {}}; std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 7a5e022d16..d0aee4bd0a 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -115,6 +115,8 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Value String-argument setter](../../../core/libs/prx/libSceJson2/Export.cpp) - sources nested inside a destination being replaced, binary aliases and thrown callbacks remain unverified. Eighteen SDK 6 reference cases cover missing headers on numeric/String destinations, direct/shared-header aliases, optional root callback metadata and allocation disabled/enabled. Those paths preserve all Value/allocation bytes and perform no allocator effects. Independent-source controls copy text through the existing handle/header, clear old characters before failed character admission and retry successfully. The old host-string snapshot changed missing-header destinations and reallocated self-aliases; it fails the regression. Checks retain source text/ownership and complete cleanup. The implementation uses the String header and underlying text alias check directly. -- [Value self-setting and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nonempty older-SDK self-copies, non-finite recovery and interrupted nonempty self-copies remain unverified. Fifty-two SDK 6 reference cases cover all eight types through both entry points, empty collection admission failures and healthy nonempty Array/Object copies. Assignment at `0x20e0` unconditionally calls the setter and returns its receiver. Both operations rebuild collections while retaining the Value handle and callback metadata. Scalars and Strings naturally preserve their storage. Checks cover replacement/null-free sequences, rollback, inactive bytes, unaffected allocations, internal/public keys, nested contents and the observed nested-header owner rebinding. The old self-guards suppress replacement and fail the regression. All original/replacement storage is reclaimed on destruction. +- [Value self-setting and assignment](../../../core/libs/prx/libSceJson2/Export.cpp) - nested older-SDK self-copies, non-finite recovery and interrupted modern nonempty self-copies remain unverified. Fifty-two SDK 6 reference cases cover all eight types through both entry points, empty collection admission failures and healthy nonempty Array/Object copies. Assignment at `0x20e0` unconditionally calls the setter and returns its receiver. Both operations rebuild collections while retaining the Value handle and callback metadata. Scalars and Strings naturally preserve their storage. Checks cover replacement/null-free sequences, rollback, inactive bytes, unaffected allocations, internal/public keys, nested contents and the observed nested-header owner rebinding. The old self-guards suppress replacement and fail the regression. All original/replacement storage is reclaimed on destruction. -- [Legacy collection self-use](../../../core/libs/prx/libSceJson2/Export.cpp) - nonempty/child-callback self-copies and concurrent ownership remain unverified. Twelve SDK 5 reference cases cover empty Array/Object setters and assignment, sentinel allocation failure and success. Both entry points retain the 24-byte collection header and Value/auxiliary ownership while replacing the sentinel. Failure still releases the old sentinel and leaves a destructible missing-sentinel header. Same-Value retry repairs it without forwarding a null free. Checks cover exact requests/releases, live generations, unchanged callback/fallback/module bytes, partial destruction and fresh module generations. The previous Object replacement helper forwards an extra null free and fails the retry regression. +- [Legacy collection self-use](../../../core/libs/prx/libSceJson2/Export.cpp) - nested/child-callback self-copies and concurrent ownership remain unverified. Twelve SDK 5 reference cases cover empty Array/Object setters and assignment, sentinel allocation failure and success. Both entry points retain the 24-byte collection header and Value/auxiliary ownership while replacing the sentinel. Failure still releases the old sentinel and leaves a destructible missing-sentinel header. Same-Value retry repairs it without forwarding a null free. Checks cover exact requests/releases, live generations, unchanged callback/fallback/module bytes, partial destruction and fresh module generations. The previous Object replacement helper forwards an extra null free and fails the retry regression. + +- [Nonempty legacy self-use failures](../../../core/libs/prx/libSceJson2/Export.cpp) - nested/child-callback payloads, thrown callbacks and concurrent accounting remain unverified. Fifty-six SDK 5 reference cases cover two-element Arrays/Objects through setters and assignment, persistent/one-shot failure at every sentinel/node/key admission position and success. Exact request and released-generation fingerprints accompany checks of partial values, parent links, missing sentinels, independent internal/public keys and counts exceeding live nodes. Retry clones the actual partial state without restoring lost entries or inventing key/count repairs. Checks preserve module, handle, auxiliary and fallback bytes/generations and verify complete teardown. A mutation clamping a partial tree count to its live nodes fails the regression. From 771f3eebfd9a689f206e6e07fe8f42bbffc3caf6 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 20:06:36 +0200 Subject: [PATCH 093/318] test(libSceJson2): verify collection fallback allocation recovery --- core/libs/tests/GuestJsonElementRecovery.cpp | 198 ++++++++++++++++++- docs/dev/debt/libSceJson2.md | 8 +- 2 files changed, 200 insertions(+), 6 deletions(-) diff --git a/core/libs/tests/GuestJsonElementRecovery.cpp b/core/libs/tests/GuestJsonElementRecovery.cpp index dd06f7294e..d7e97f4abb 100644 --- a/core/libs/tests/GuestJsonElementRecovery.cpp +++ b/core/libs/tests/GuestJsonElementRecovery.cpp @@ -12,9 +12,13 @@ struct Value { Value* parent; void* auxiliary; std::uintptr_t payload; std::uint32_t reserved; std::int32_t type; }; struct String { void* text; }; +struct Array { void* storage; }; +struct Object { void* storage; }; struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; using Handler = void (APS5_VABI*)(int, const String*, const std::size_t*, const Value*, const Value*, void*); extern "C" { +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value*, const Array*); +void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value*, const Object*); void APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void*, const Parameter*); int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); @@ -81,11 +85,13 @@ struct PartialElementTrace { const Value* expectedFallback = nullptr; bool reenter = false; bool assignString = false; + const Value* collectionSource = nullptr; + bool once = false; unsigned expectedError = 0x80848118; }; static void* APS5_VABI AllocatePartialElement(void*, std::size_t size, void* data) { auto& trace = *static_cast(data); trace.requests.push_back(size); - if (!trace.budget) return nullptr; + if (!trace.budget) { if (trace.once) trace.budget = -1; return nullptr; } if (trace.budget > 0) --trace.budget; void* memory = std::malloc(size); Require(memory && trace.live.emplace(memory, size).second && trace.generations.emplace(memory, ++trace.nextGeneration).second, "Partial metadata requires independent allocation generations"); @@ -106,10 +112,13 @@ static void APS5_VABI RecoverPartialElement(int error, const String* key, const auto& trace = *static_cast(data); ++trace.calls; Require(static_cast(error) == trace.expectedError && !key && index && *index == std::numeric_limits::max() && value == trace.expectedValue && parent == trace.expectedParent, "Partial recovery must preserve error, all index bits, source, immediate parent and context"); - if (trace.reenter || trace.assignString) { + if (trace.reenter || trace.assignString || trace.collectionSource) { const auto calls = trace.calls; Require(_ZNK3sce4Json5Value8getValueEm(value, *index) == trace.expectedFallback && trace.calls == calls, "Reentry must share the fallback while suppressing recursive callbacks"); - if (trace.assignString) _ZN3sce4Json5Value3setEPKc(const_cast(trace.expectedFallback), "recovery"); + if (trace.collectionSource) { + if (trace.collectionSource->type == 6) _ZN3sce4Json5Value3setERKNS0_5ArrayE(const_cast(trace.expectedFallback), reinterpret_cast(trace.collectionSource->payload)); + else _ZN3sce4Json5Value3setERKNS0_6ObjectE(const_cast(trace.expectedFallback), reinterpret_cast(trace.collectionSource->payload)); + } else if (trace.assignString) _ZN3sce4Json5Value3setEPKc(const_cast(trace.expectedFallback), "recovery"); else _ZN3sce4Json5Value3setEl(const_cast(trace.expectedFallback), 77); } } @@ -219,6 +228,188 @@ static void CheckElementStringFailures() { } } +struct ElementListLink { ElementListLink* previous; ElementListLink* next; Value value; }; +struct ElementTreeLink { + ElementTreeLink* parent; ElementTreeLink* left; ElementTreeLink* right; + const char* key; unsigned length; unsigned capacity; std::uint64_t reserved; + void* publicKey; std::uint64_t padding; Value value; unsigned char color; unsigned char nil; unsigned char tail[6]; +}; +static_assert(sizeof(ElementListLink) == 48 && sizeof(ElementTreeLink) == 104); +static std::string ElementCollectionState(const Value& value, const Value* parent); +static void ElementTreeState(const ElementTreeLink* node, const Value* owner, std::string& output, unsigned depth = 0) { + Require(depth < 8, "Fallback tree traversal must remain bounded"); + if (node->nil) return; + ElementTreeState(node->left, owner, output, depth + 1); + output += std::string(node->key) + "/" + (node->publicKey ? *static_cast(node->publicKey) : "?") + "=" + ElementCollectionState(node->value, owner) + ";"; + ElementTreeState(node->right, owner, output, depth + 1); +} +static std::string ElementCollectionState(const Value& value, const Value* parent) { + std::string output = std::to_string(value.type) + (value.parent == parent ? "p" : value.parent ? "x" : "n"); + if (!value.type) return output; + if (value.type == 2 || value.type == 3) return output + std::to_string(value.payload); + Require(value.type == 6 || value.type == 7, "Fallback contents must have their admitted scalar or collection type"); + if (!value.payload) return output + "!"; + auto* words = *reinterpret_cast(value.payload); + if (!words) return output + "?"; + output += words[3] == reinterpret_cast(&value) ? "o" : "x"; + output += std::to_string(words[1]) + "["; + if (!words[0]) return output + "missing]"; + if (value.type == 7) { + auto* head = reinterpret_cast(words[0]); ElementTreeState(head->parent, &value, output); + } else { + auto* head = reinterpret_cast(words[0]); unsigned count = 0; + for (auto* node = head->next; node != head; node = node->next) { + Require(++count < 8, "Fallback list traversal must remain bounded"); output += ElementCollectionState(node->value, &value) + ";"; + } + } + return output + "]"; +} +static std::uint64_t ElementRequestOrder(const std::vector& requests, std::size_t begin) { + std::uint64_t hash = UINT64_C(14695981039346656037); + for (auto i = begin; i < requests.size(); ++i) { hash ^= requests[i]; hash *= UINT64_C(1099511628211); } + return hash; +} +struct ElementCollectionCase { + bool object; bool once; unsigned budget; std::size_t requests; std::uint64_t requestOrder; unsigned notices; const char* state; + std::size_t retryRequests; std::uint64_t retryRequestOrder; const char* retryState; +}; +static const ElementCollectionCase ElementCollectionCases[]{ + {0, 0, 0, 1, UINT64_C(12638161911788193143), 1, "0p", 7, UINT64_C(8729915328826252127), "6po2[3p11;3p22;]"}, + {0, 0, 1, 2, UINT64_C(598301484379165397), 1, "0p", 7, UINT64_C(8729915328826252127), "6po2[3p11;3p22;]"}, + {0, 0, 2, 4, UINT64_C(4035778052887317517), 2, "6po0[missing]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 0, 3, 4, UINT64_C(4035778052887317517), 1, "6po0[]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 0, 4, 5, UINT64_C(15412030556972272551), 1, "6po0[]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 0, 5, 7, UINT64_C(8729915328826252127), 2, "6po0[]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 0, 6, 7, UINT64_C(8729915328826252127), 1, "6po1[3p11;]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 0, 7, 7, UINT64_C(8729915328826252127), 0, "6po2[3p11;3p22;]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 1, 0, 1, UINT64_C(12638161911788193143), 1, "0p", 7, UINT64_C(8729915328826252127), "6po2[3p11;3p22;]"}, + {0, 1, 1, 2, UINT64_C(598301484379165397), 1, "0p", 7, UINT64_C(8729915328826252127), "6po2[3p11;3p22;]"}, + {0, 1, 2, 7, UINT64_C(8729915328826252127), 1, "6po2[3p11;3p22;]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 1, 3, 4, UINT64_C(4035778052887317517), 1, "6po0[]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 1, 4, 5, UINT64_C(15412030556972272551), 1, "6po0[]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 1, 5, 7, UINT64_C(8729915328826252127), 1, "6po1[3p22;]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 1, 6, 7, UINT64_C(8729915328826252127), 1, "6po1[3p11;]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {0, 1, 7, 7, UINT64_C(8729915328826252127), 0, "6po2[3p11;3p22;]", 4, UINT64_C(12494627435050714437), "6po2[3p11;3p22;]"}, + {1, 0, 0, 1, UINT64_C(12638161911788193143), 1, "0p", 29, UINT64_C(16556326738605743255), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 1, 2, UINT64_C(598301484379165397), 1, "0p", 29, UINT64_C(16556326738605743255), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 2, 3, UINT64_C(2174132616866993959), 1, "0p", 29, UINT64_C(16556326738605743255), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 3, 4, UINT64_C(3997444679488938213), 1, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 4, 5, UINT64_C(8830793754451562391), 1, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 5, 11, UINT64_C(1847692082414021751), 6, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 6, 13, UINT64_C(11795418786358859533), 7, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 7, 13, UINT64_C(11795418786358859533), 6, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 8, 15, UINT64_C(8198587246079178223), 7, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 9, 15, UINT64_C(8198587246079178223), 6, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 10, 16, UINT64_C(18047407352626022791), 6, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 11, 16, UINT64_C(18047407352626022791), 5, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 12, 17, UINT64_C(72191830074401783), 5, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 13, 17, UINT64_C(72191830074401783), 4, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 14, 18, UINT64_C(8609619750561420695), 4, "7po1[/?=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 15, 19, UINT64_C(7887639270042689613), 4, "7po1[a/?=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 16, 20, UINT64_C(15090866843038583229), 4, "7po1[a/=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 17, 20, UINT64_C(15090866843038583229), 3, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 18, 22, UINT64_C(16504299194552701231), 4, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 19, 22, UINT64_C(16504299194552701231), 3, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 20, 24, UINT64_C(3929529969927096549), 4, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 21, 24, UINT64_C(3929529969927096549), 3, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 22, 25, UINT64_C(4075476546177097205), 3, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 23, 25, UINT64_C(4075476546177097205), 2, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 24, 26, UINT64_C(7916499978844114973), 2, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 25, 26, UINT64_C(7916499978844114973), 1, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 26, 28, UINT64_C(14474737658796973967), 2, "7po2[a/a=3p11;/?=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 27, 28, UINT64_C(14474737658796973967), 1, "7po2[a/a=3p11;b/?=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 28, 29, UINT64_C(16556326738605743255), 1, "7po2[a/a=3p11;b/=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 0, 29, 29, UINT64_C(16556326738605743255), 0, "7po2[a/a=3p11;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 0, 1, UINT64_C(12638161911788193143), 1, "0p", 29, UINT64_C(16556326738605743255), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 1, 2, UINT64_C(598301484379165397), 1, "0p", 29, UINT64_C(16556326738605743255), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 2, 3, UINT64_C(2174132616866993959), 1, "0p", 29, UINT64_C(16556326738605743255), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 3, 4, UINT64_C(3997444679488938213), 1, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 4, 5, UINT64_C(8830793754451562391), 1, "7po0[]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 5, 22, UINT64_C(16085087199939238845), 1, "7po2[a/?=3p11;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 6, 26, UINT64_C(12187800176058535535), 1, "7po2[a/=3p11;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 7, 24, UINT64_C(14200822062155788271), 1, "7po2[a/?=3p11;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 8, 27, UINT64_C(16994698599719519703), 1, "7po2[a/=3p11;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 9, 27, UINT64_C(7401576235132465975), 1, "7po2[/a=3n0;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 10, 28, UINT64_C(11183815068046375183), 1, "7po2[/a=3n0;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 11, 26, UINT64_C(6368825009497694205), 1, "7po2[a/?=3n0;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 12, 28, UINT64_C(9933439070053794599), 1, "7po2[a/=3n0;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 13, 26, UINT64_C(18330066241628227221), 1, "7po1[b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 14, 29, UINT64_C(16556326738605743255), 1, "7po2[/a=3n0;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 15, 28, UINT64_C(9933439070053794599), 1, "7po2[a/?=3n0;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 16, 29, UINT64_C(16556326738605743255), 1, "7po2[a/=3n0;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 17, 22, UINT64_C(235942740017013557), 1, "7po2[a/a=3p11;b/?=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 18, 26, UINT64_C(8706565262351133383), 1, "7po2[a/a=3p11;b/=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 19, 24, UINT64_C(3883726514528165511), 1, "7po2[a/a=3p11;b/?=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 20, 27, UINT64_C(14424272583777125367), 1, "7po2[a/a=3p11;b/=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 21, 27, UINT64_C(9861833381400848727), 1, "7po2[/b=3n0;a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 22, 28, UINT64_C(14449855710655478887), 1, "7po2[/b=3n0;a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 23, 26, UINT64_C(7880037974236572341), 1, "7po2[a/a=3p11;b/?=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 24, 28, UINT64_C(14474737658796973967), 1, "7po2[a/a=3p11;b/=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 25, 26, UINT64_C(7916499978844114973), 1, "7po1[a/a=3p11;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 26, 29, UINT64_C(16556326738605743255), 1, "7po2[a/a=3p11;/b=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 27, 28, UINT64_C(14474737658796973967), 1, "7po2[a/a=3p11;b/?=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 28, 29, UINT64_C(16556326738605743255), 1, "7po2[a/a=3p11;b/=3n0;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, + {1, 1, 29, 29, UINT64_C(16556326738605743255), 0, "7po2[a/a=3p11;b/b=3p22;]", 26, UINT64_C(3004510956633589525), "7po2[a/a=3p11;b/b=3p22;]"}, +}; + +static void AssignElementCollection(PartialElementTrace& trace) { + if (trace.collectionSource->type == 6) _ZN3sce4Json5Value3setERKNS0_5ArrayE(const_cast(trace.expectedFallback), reinterpret_cast(trace.collectionSource->payload)); + else _ZN3sce4Json5Value3setERKNS0_6ObjectE(const_cast(trace.expectedFallback), reinterpret_cast(trace.collectionSource->payload)); +} +static void CheckElementCollectionFailures() { + for (const auto& expected : ElementCollectionCases) for (bool missingFallback : {false, true}) { + PartialElementTrace trace; PartialElementAllocator allocator; Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, ¶meter) == 0, "Initialize collection fallback recovery"); + Value root{}, replacement{}; _ZN3sce4Json5ValueC1Ev(&root); _ZN3sce4Json5ValueC1Ev(&replacement); + const auto* moduleFallback = _ZNK3sce4Json5Value8getValueEm(&root, std::numeric_limits::max()); + if (missingFallback) { + trace.budget = 1; + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, RecoverPartialElement, &trace) == static_cast(0x80848102), "Prepare a missing root-owned fallback"); trace.budget = -1; + } + Require(_ZN3sce4Json5Value30setElementAccessFailureHandlerEPFviPKNS0_6StringEPKmPKS1_S8_PvES9_(&root, RecoverPartialElement, &trace) == 0 && + _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, "[1]", 3) == 0, "Prepare nested callback dispatch"); + const char* text = expected.object ? "{\"a\":11,\"b\":22}" : "[11,22]"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&replacement, text, std::strlen(text)) == 0, "Prepare the independent callback replacement source"); + const Value originalSource = replacement; + auto* auxiliary = static_cast(root.auxiliary); + trace.expectedFallback = auxiliary[2] ? static_cast(auxiliary[2]) : moduleFallback; + trace.expectedValue = _ZNK3sce4Json5Value8getValueEm(&root, 0); trace.expectedParent = &root; trace.collectionSource = &replacement; + Require(bool(auxiliary[2]) != missingFallback, "Parsing must retain the intended fallback admission state"); + const Value* fallbackParent = missingFallback ? nullptr : &root; + const auto generations = trace.generations; + std::map> snapshots; + for (const auto& [pointer, size] : trace.live) { + if (pointer == trace.expectedFallback) continue; + const auto* bytes = static_cast(pointer); snapshots.emplace(pointer, std::vector{bytes, bytes + size}); + } + const auto requests = trace.requests.size(); const auto notices = trace.notifications; + trace.budget = static_cast(expected.budget); trace.once = expected.once; + Require(_ZNK3sce4Json5Value8getValueEm(trace.expectedValue, std::numeric_limits::max()) == trace.expectedFallback && trace.calls == 1, + "Callback collection mutation must preserve fallback identity and suppress recursive recovery"); + trace.budget = -1; trace.once = false; + Require(trace.requests.size() == requests + expected.requests && ElementRequestOrder(trace.requests, requests) == expected.requestOrder && trace.notifications == notices + expected.notices && + ElementCollectionState(*trace.expectedFallback, fallbackParent) == expected.state, + "Fallback collection failures must preserve exact requests/notices, partial values, keys, counts, parent links and header ownership"); + const auto retryRequests = trace.requests.size(); const auto retryNotices = trace.notifications; + AssignElementCollection(trace); + Require(trace.requests.size() == retryRequests + expected.retryRequests && ElementRequestOrder(trace.requests, retryRequests) == expected.retryRequestOrder && trace.notifications == retryNotices && + ElementCollectionState(*trace.expectedFallback, fallbackParent) == expected.retryState, + "Independent fallback retry must retain metadata and publish the complete replacement with the reference allocation protocol"); + const auto handle = trace.expectedFallback->payload; + Require(_ZNK3sce4Json5Value8getValueEm(trace.expectedValue, std::numeric_limits::max()) == trace.expectedFallback && trace.calls == 2 && + trace.expectedFallback->payload == handle && ElementCollectionState(*trace.expectedFallback, fallbackParent) == expected.retryState, + "Recovery must resume after failed allocation and retain its completed collection handle"); + Require(root.auxiliary == auxiliary && bool(auxiliary[2]) != missingFallback && static_cast(auxiliary[5])[0] == reinterpret_cast(&RecoverPartialElement) && + static_cast(auxiliary[5])[1] == &trace && std::memcmp(&replacement, &originalSource, sizeof(replacement)) == 0, + "Collection recovery must preserve the registered callback, context, source Value and missing-fallback state"); + for (const auto& [pointer, generation] : generations) Require(trace.generations.at(pointer) == generation, "Collection recovery must retain every preexisting allocation generation"); + for (const auto& [pointer, bytes] : snapshots) Require(std::memcmp(pointer, bytes.data(), bytes.size()) == 0, "Collection recovery must preserve source, module and callback metadata bytes"); + _ZN3sce4Json5ValueD1Ev(&replacement); _ZN3sce4Json5ValueD1Ev(&root); + Require(_ZN3sce4Json11Initializer9terminateEv(&owner) == 0 && !owner && trace.live.empty() && trace.generations.empty(), "All source, partial/retried fallback and module resources must be reclaimed"); + } +} + int main() { Value root{}; _ZN3sce4Json5ValueC1Ev(&root); @@ -282,5 +473,6 @@ int main() { Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Terminate element recovery"); CheckPartialElementFallbacks(); CheckElementStringFailures(); + CheckElementCollectionFailures(); return 0; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index d0aee4bd0a..34b886a055 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -13,7 +13,7 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [sce::Json::Object::Pair, sce::Json::Array::iterator and sce::Json::Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - Pair allocator-callback reentry and inherited older-SDK failure scopes still require audit. Both Pair constructors preserve the eight bytes between key and value and inactive Value bytes, following `0xc230` and `0xc280`. Reference tests cover both constructor/destructor aliases, all value tags, independent copies and allocator cleanup. Array and object postfix increment use `this` and the dummy integer argument and advance in place, following `0xb270` and `0xc4a0`; they do not take a hidden result pointer. This ABI is verified against the inspected module, not every firmware version. - [sce::Json::Value handler registration](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float/element registration and libc-heap failure handling remain incomplete. Modern auxiliary records, fallback values and callback records use the supplied allocator. The inspected setters at `0x39c0`, `0x3b00`, `0x3e90` and `0x3fa0` reject owned children with `0x80848104`. The value null-access setter is a three-argument member returning status, rather than a static callback setter. Legacy mismatch-map storage is covered separately below; other SDK-dependent legacy combinations remain unverified. - [sce::Json::Value type-mismatch dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - libc-heap failures and untested legacy callback combinations remain incomplete. Global registration follows `0x1480`; root-local mismatch precedes global null recovery, followed by global mismatch. Current root lookup and borrowed-result dispatch follow `0x22b0` and `0x2930`, including requested type, source, immediate parent, callback precedence and a shared reentry guard. Additional legacy compatibility remains unverified. -- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Complex fallback-payload allocation failures and compatibility below SDK 6 remain incomplete. +- [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback payload failures and compatibility below SDK 6 remain incomplete. - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. - [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object, callback metadata and file-buffer allocation failures dispatch the supplied allocator's notifier. - [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. @@ -107,9 +107,9 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Object child callback heap exceptions](../../../core/libs/prx/libSceJson2/Export.cpp) - deeper tree shapes, guest allocator throws during cloning and concurrent ownership remain unverified. Twelve SDK 5 reference cases cover both direct Object and Value-wrapped constructor aliases, either child entry exception and success. The initial empty tree remains published while unfinished replacement nodes, keys and sentinel become orphaned; completed callback records remain until module teardown. Retry publishes a complete tree without reclaiming the orphaned replacement. Checks cover exact requests, source bytes/generations, tree contents, key storage, callback identity, published wrappers, retry, cleanup and fresh initialization. The fixture separately reclaims orphaned caller storage. The pre-fix recursive rollback and delayed destination publication fail the regression. -- [Partial element-recovery fallback](../../../core/libs/prx/libSceJson2/Export.cpp) - complex callback-created fallback payload failures and older SDK dispatch remain unverified. Eight SDK 6 reference cases cover failure of auxiliary/fallback/handler admission and success, with and without callback reentry. Registration retry reuses admitted metadata without repairing a missing fallback. Parsing preserves that missing state; root/child failures invoke the handler and return the module fallback instead. Checks cover request order, error/index/source/parent ABI, suppression of recursive dispatch, visible scalar mutation, zero-allocation dispatch under failure, stable auxiliary/fallback/record generations and complete teardown. A mutation recreating the missing fallback fails the retry regression. +- [Partial element-recovery fallback](../../../core/libs/prx/libSceJson2/Export.cpp) - nested callback-created fallback payload failures and older SDK dispatch remain unverified. Eight SDK 6 reference cases cover failure of auxiliary/fallback/handler admission and success, with and without callback reentry. Registration retry reuses admitted metadata without repairing a missing fallback. Parsing preserves that missing state; root/child failures invoke the handler and return the module fallback instead. Checks cover request order, error/index/source/parent ABI, suppression of recursive dispatch, visible scalar mutation, zero-allocation dispatch under failure, stable auxiliary/fallback/record generations and complete teardown. A mutation recreating the missing fallback fails the retry regression. -- [Element fallback String allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - collection payloads and thrown allocator callbacks during fallback mutation remain unverified. Twenty-four SDK 6 reference cases cover root/module fallbacks, root/child dispatch, all C-string setter allocation positions and success. The setter first allocates a temporary String; failed temporary admission preserves the destination. Later failures retain null or empty payloads and exact notices, including two notices after temporary character failure. Retry reuses admitted handles/headers while recreating temporary storage. Checks cover callback reentry, request order, visible text, metadata generations, source bytes, repeated recovery and complete teardown. The previous host-string temporary skipped guest requests and fails the regression. The setter at `0x1c00` leaves guard residue in the return register; it is declared `void` and tests check state rather than fabricated status. +- [Element fallback String allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - nested collection payloads and thrown allocator callbacks during fallback mutation remain unverified. Twenty-four SDK 6 reference cases cover root/module fallbacks, root/child dispatch, all C-string setter allocation positions and success. The setter first allocates a temporary String; failed temporary admission preserves the destination. Later failures retain null or empty payloads and exact notices, including two notices after temporary character failure. Retry reuses admitted handles/headers while recreating temporary storage. Checks cover callback reentry, request order, visible text, metadata generations, source bytes, repeated recovery and complete teardown. The previous host-string temporary skipped guest requests and fails the regression. The setter at `0x1c00` leaves guard residue in the return register; it is declared `void` and tests check state rather than fabricated status. - [Value setter return ABI](../../../core/libs/prx/libSceJson2/Export.cpp) - String references invalidated by destination type changes still require runtime audits. Type, Boolean, signed/unsigned integer, String and Value setters expose no defined status result. The type helper at `0x580` ends with guard verification; scalar wrappers at `0x1a90`, `0x1ae0` and `0x1b30` do not establish another result. The String overload at `0x1cc0` can return without setting a result; Value setting at `0x1e40` also leaves guard residue. Definitions and test declarations use `void`; tests retain content, layout, ownership and callback assertions instead of testing fabricated zero returns. The reference scalar-layout contract passes with the corrected declaration. Full guest integration remains unverified. @@ -120,3 +120,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Legacy collection self-use](../../../core/libs/prx/libSceJson2/Export.cpp) - nested/child-callback self-copies and concurrent ownership remain unverified. Twelve SDK 5 reference cases cover empty Array/Object setters and assignment, sentinel allocation failure and success. Both entry points retain the 24-byte collection header and Value/auxiliary ownership while replacing the sentinel. Failure still releases the old sentinel and leaves a destructible missing-sentinel header. Same-Value retry repairs it without forwarding a null free. Checks cover exact requests/releases, live generations, unchanged callback/fallback/module bytes, partial destruction and fresh module generations. The previous Object replacement helper forwards an extra null free and fails the retry regression. - [Nonempty legacy self-use failures](../../../core/libs/prx/libSceJson2/Export.cpp) - nested/child-callback payloads, thrown callbacks and concurrent accounting remain unverified. Fifty-six SDK 5 reference cases cover two-element Arrays/Objects through setters and assignment, persistent/one-shot failure at every sentinel/node/key admission position and success. Exact request and released-generation fingerprints accompany checks of partial values, parent links, missing sentinels, independent internal/public keys and counts exceeding live nodes. Retry clones the actual partial state without restoring lost entries or inventing key/count repairs. Checks preserve module, handle, auxiliary and fallback bytes/generations and verify complete teardown. A mutation clamping a partial tree count to its live nodes fails the regression. + +- [Element fallback collection allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - nested payloads, thrown callbacks and concurrent lifecycle changes remain unverified. One hundred fifty-two SDK 6 reference cases cover two-element Array/Object assignment into root-owned and module fallbacks, persistent/one-shot failure at every allocation position and success. Checks include reentry suppression, exact request/notice fingerprints, missing sentinels, partial values/keys/counts, parent links, header ownership and unchanged source/module/callback allocation generations and bytes. Independent retry completes the replacement; subsequent recovery retains its handle. Teardown reclaims all source, partial and replacement payloads. All production probe results match the reference. A mutation dropping the fallback collection owner fails the regression. From 3f16cb656dbc9e922fdc35fb2c248d9de745b565 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 20:16:22 +0200 Subject: [PATCH 094/318] docs(libSceJson2): clarify legacy callback safety limits --- docs/dev/debt/libSceJson2.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 34b886a055..abc6149542 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -28,7 +28,7 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [sce::Json::Value handle admission and type changes](../../../core/libs/prx/libSceJson2/Export.cpp) - collection copy/assignment failure ordering, nested recovery failures and older SDK compatibility remain incomplete. The supplied allocator owns each eight-byte complex-value handle. Typed construction and type changes follow `0x580`, `0x10d0`, `0x1170` and `0x1210`: the old payload is destroyed before replacement admission; string/header failure leaves null, an array can retain a header without its sentinel, and object construction reclaims a failed sentinel. Scalar changes preserve inactive payload bytes. Special-float string conversion follows the temporary string at `0x40a0`; failure of its header preserves the prior value, while later failures can leave null or an empty string. Local tests cover SDK 6 process metadata, not every firmware version. -- [sce::Json::Value auxiliary admission and retry](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float and element registration remain unsupported and throw. Modern admission follows `0x3280`: a failed fallback allocation retains the 72-byte auxiliary record; later registration reuses it without recreating the missing fallback. Local tests cover all four callback setters, their allocation failures, replacement and null callbacks, child rejection, parent preservation and teardown through the original allocator. Legacy auxiliary records use 24 bytes; mismatch callbacks use a separate map whose header uses the supplied allocator and whose 56-byte nodes use libc new/delete. +- [sce::Json::Value auxiliary admission and retry](../../../core/libs/prx/libSceJson2/Export.cpp) - legacy special-float and element registration remain unsupported and throw. The helper at `0x3280` allocates 24 bytes below application SDK 6 (`0x34d1`), while the setters at `0x3e90` and `0x3fa0` read/write handler slots at offsets `0x20` and `0x28` without an SDK check. Those accesses exceed the legacy allocation, so this implementation throws. Modern admission follows `0x3280`: a failed fallback allocation retains the 72-byte auxiliary record; later registration reuses it without recreating the missing fallback. Local tests cover all four callback setters, their allocation failures, replacement and null callbacks, child rejection, parent preservation and teardown through the original allocator. Legacy auxiliary records use 24 bytes; mismatch callbacks use a separate map whose header uses the supplied allocator and whose 56-byte nodes use libc new/delete. - [sce::Json::Value scalar and string copying](../../../core/libs/prx/libSceJson2/Export.cpp) - nested collection assignment failures remain incomplete. Scalar and string dispatch follow `0x1e40`: null and boolean copies preserve inactive destination bytes, numeric values retain their exact payloads, and string assignment changes type before copying characters into the existing header. A character-allocation failure can empty an existing string; it does not restore its old contents. Copy construction initializes ownership metadata directly, following `0x1e20`. Local allocator tests cover null, boolean, signed, unsigned, finite real and string sources across all destination types and failure budgets. @@ -122,3 +122,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Nonempty legacy self-use failures](../../../core/libs/prx/libSceJson2/Export.cpp) - nested/child-callback payloads, thrown callbacks and concurrent accounting remain unverified. Fifty-six SDK 5 reference cases cover two-element Arrays/Objects through setters and assignment, persistent/one-shot failure at every sentinel/node/key admission position and success. Exact request and released-generation fingerprints accompany checks of partial values, parent links, missing sentinels, independent internal/public keys and counts exceeding live nodes. Retry clones the actual partial state without restoring lost entries or inventing key/count repairs. Checks preserve module, handle, auxiliary and fallback bytes/generations and verify complete teardown. A mutation clamping a partial tree count to its live nodes fails the regression. - [Element fallback collection allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - nested payloads, thrown callbacks and concurrent lifecycle changes remain unverified. One hundred fifty-two SDK 6 reference cases cover two-element Array/Object assignment into root-owned and module fallbacks, persistent/one-shot failure at every allocation position and success. Checks include reentry suppression, exact request/notice fingerprints, missing sentinels, partial values/keys/counts, parent links, header ownership and unchanged source/module/callback allocation generations and bytes. Independent retry completes the replacement; subsequent recovery retains its handle. Teardown reclaims all source, partial and replacement payloads. All production probe results match the reference. A mutation dropping the fallback collection owner fails the regression. + +- [Initializer allocator-info callback](../../../core/libs/prx/libSceJson2/Export.cpp) - the inspected 19-byte function at `0x16e0` ignores the receiver, callback and context. It returns `0x80848110` when the allocation context is absent and zero when present, without storing or invoking the callback. The implementation intentionally follows that behavior; it is not evidence of allocation telemetry support on other library versions. From 9eb0b5f2638ec961372d688351384c36bf09c4d6 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 23 Oct 2026 20:17:54 +0200 Subject: [PATCH 095/318] test(libSceJson2): set DLL paths for lifecycle regressions --- core/libs/prx/libSceJson2/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 76fc757623..de5ed8c55d 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -330,6 +330,10 @@ if(BUILD_TESTING) ENVIRONMENT "LOCPATH=${numeric_locale_dir}") endif() if(WIN32) + get_property(json2_module_exception_tests DIRECTORY PROPERTY TESTS) + list(FILTER json2_module_exception_tests INCLUDE REGEX "^guest_json_(modern_termination_|initialization_exception_)") + set_tests_properties(${json2_module_exception_tests} guest_json_composite_destruction PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") set_tests_properties(guest_json guest_json_initialization_publication guest_json_callback_lifecycle guest_json_callback_copies guest_json_pair_allocator guest_json_pair_layout guest_json_inherited_recovery guest_json_recovery_allocator guest_json_parser_metadata guest_json_decimal_range guest_json_serialization_callback guest_json_key_lookup guest_json_array_insertion guest_json_array_mutation guest_json_internal_initializer guest_json_allocation_entry guest_json_string_search guest_json_references guest_json_aliases guest_json_parser_state guest_json_numeric_locale guest_json_float_settings guest_json_allocator_settings guest_json_value_layout guest_json_value_ownership guest_json_special_float guest_json_element_recovery guest_json_null_recovery guest_json_global_recovery guest_json_string_allocator guest_json_array_allocator guest_json_object_allocator guest_json_value_allocator guest_json_allocator PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() From c8b8d24aef93eabfa11b1afabdc9013830e8da0c Mon Sep 17 00:00:00 2001 From: ntamas94 Date: Sat, 24 Oct 2026 02:07:58 +0200 Subject: [PATCH 096/318] fix(libSceAgcDriver): size a PS4 depth target's HTILE in 512x512 blocks A PS4 (GCN, 8 pipes) depth target's HTILE spans 16 KiB per 512x512 pixel block, not the 32 KiB per 1024x512 of the PS5 layout. The two agree only for widths that are a multiple of 1024. Gang Beasts (CUSA04865) clears each 512x512 spot-light shadow map by filling 16 KiB of its HTILE; measured as 32 KiB, that fill did not cover the HTILE, the bind read 16 KiB past it and saw no clear, and the shadow casters drew over the previous frames' depth. Co-Authored-By: Claude Opus 5.5 --- .../libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp | 7 +++++-- core/libs/prx/libSceAgcDriver/tests/Graphics.cpp | 5 +++++ docs/dev/debt/libSceAgcDriver.md | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index caae231995..034baf7146 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -10,6 +10,7 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" #include "RdnaDecoder/include/RdnaDecoder/RdnaDescriptorFormat.hpp" #include #include @@ -2321,8 +2322,10 @@ VkImageAspectFlags HtileFillClears(std::uint32_t pattern, bool stencilInHtile) { } std::uint64_t HtileMetadataBytes(VkExtent2D extent) { - const auto blocks = static_cast((extent.width + 1023u) / 1024u) * ((extent.height + 511u) / 512u); - return blocks * 32768u; + const bool gcn = ShaderRecompiler::GcnGuestShaders(); + const std::uint32_t blockWidth = gcn ? 512u : 1024u; + const auto blocks = static_cast((extent.width + blockWidth - 1u) / blockWidth) * ((extent.height + 511u) / 512u); + return blocks * (gcn ? 16384u : 32768u); } bool HtileFillCovers(std::uint64_t htile, VkExtent2D extent, std::uint64_t address, std::size_t bytes) { diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index 9f97985dcf..e07aa51eec 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -1,3 +1,4 @@ +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" #include "BdaTests.hpp" #include "GraphicsTests.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -2251,6 +2252,10 @@ void DepthStencilTests() { // 260, AMD PAL), so a fill of the words a linear layout would hold covers nothing. Require(!HtileFillCovers(0x10000, {16, 16}, 0x10000, 16) && !HtileFillCovers(0x10000, {16, 16}, 0x10000, 264), "a fill of the linear HTILE size of a 16x16 surface misses its pipe-aligned words"); Require(AgcDriver::Graphics::HtileMetadataBytes({1024, 512}) == 0x8000u && AgcDriver::Graphics::HtileMetadataBytes({1025, 512}) == 0x10000u && AgcDriver::Graphics::HtileMetadataBytes({3840, 2160}) == 20u * 0x8000u, "the HTILE metadata is 32 KiB per 1024x512 block"); + ShaderRecompiler::SetGcnGuestShaders(true); + Require(AgcDriver::Graphics::HtileMetadataBytes({512, 512}) == 0x4000u && AgcDriver::Graphics::HtileMetadataBytes({256, 256}) == 0x4000u && AgcDriver::Graphics::HtileMetadataBytes({2048, 1152}) == 0x30000u && AgcDriver::Graphics::HtileMetadataBytes({4096, 4096}) == 0x100000u, "a PS4 HTILE is 16 KiB per 512x512 block"); + Require(HtileFillCovers(0x10000, {512, 512}, 0x10000, 0x4000) && !HtileFillCovers(0x10000, {512, 512}, 0x10000, 0x3ffc), "a 16 KiB fill covers the HTILE of a 512x512 PS4 depth target"); + ShaderRecompiler::SetGcnGuestShaders(false); } Require(state.renderExtent.width == 4 && state.renderExtent.height == 2, "render extent ignores the depth surface"); Require(!state.depthTest && !state.depthWrite && state.stencilTest, "depth/stencil enables changed"); diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index b7e0107a5d..ed8b4ada4d 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -52,7 +52,7 @@ - [Registered shader types](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - types 0, 1, 2, 4, 6 and 7 are prepared and type 5 is skipped. Any other type, such as type 8, is logged (`unsupported registered shader type 8; it is not prepared at registration`) and left unprepared at registration, and throws when it is used later. Hello Neighbor 2 (PPSA07426), Ad Infinitum (PPSA10600) and Ghostrunner (PPSA03685) register type 8 headers and run on past them; what a type 8 header describes is unknown, #1892 found no context registers in it. - [NGG vertex workgroups](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - type-2 PRIMGEN programs using DS instructions or `s_barrier` are prepared with the draw's routing and primitive registers on the existing mesh path. The workgroup requirement follows [Mesa's NGG lowering](https://github.com/mihawk-99/PS5_Mesa/blob/b3588f78dd7fd46fbadc7ecd487e5f14d4213e4b/src/amd/common/nir/ac_nir_lower_ngg.c#L1135); the mesh path supplies LDS and two guest lanes per host invocation for wave64 on a 32-lane host subgroup. Console routing has not been measured. Passthrough and tessellation keep their existing paths; unsupported mesh assembly and subgroup budgets throw. - [Shader header copies](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a header bound for a draw or dispatch at another address than the registered one is accepted when its fixed fields match the registered header, and every array it points at still holds the registered bytes; the registered header's metadata is used. `user_data` is not compared (PPSA21567 binds such a copy of a vertex shader header with `user_data` set to null; whether the console reads the copy's `user_data` is unknown). A copy whose registers, context registers, specials or semantics differ, or whose arrays are unreadable, throws `... refers to a replaced shader header`; what the console does with a header a title patches while relocating it is not measured. PPSA01325 (#2914) resolves the ABI with such a copy -- [HtileFillCovers](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - a fill is taken to cover a depth surface's whole HTILE when it spans 32 KiB per 1024x512 pixel block from DB_HTILE_DATA_BASE, the pipe-aligned SW_64KB_Z_X metadata extent AMD PAL's gfx10 addrlib gives for GB_ADDR_CONFIG 0x00100044 (1-8 samples, D16/D32); other metadata layouts are not decoded. A partial fill is ignored (the surface keeps its contents); not measured on the console. +- [HtileFillCovers](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - a fill is taken to cover a depth surface's whole HTILE when it spans 32 KiB per 1024x512 pixel block from DB_HTILE_DATA_BASE, the pipe-aligned SW_64KB_Z_X metadata extent AMD PAL's gfx10 addrlib gives for GB_ADDR_CONFIG 0x00100044 (1-8 samples, D16/D32). A PS4 depth target's HTILE spans 16 KiB per 512x512 pixel block, the macro tile addrlib's CI `ComputeTileDataWidthAndHeight` gives for 32-bit HTILE words, 16384 cache bits and the 8 pipes of P8_32x32_16x16, and the size the clears of Gang Beasts (CUSA04865) fill for 512x512, 2048x1152 and 4096x4096 targets; other metadata layouts are not decoded. A partial fill is ignored (the surface keeps its contents); not measured on the console. - [Depth fast clear through HTILE](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - a surface cleared by an HTILE fill is cleared to DB_DEPTH_CLEAR, also when it is next sampled. For a depth-only HTILE the clear word also carries a value (radv writes `minz = maxz = round(depth * 0x3fff)`, `HTILE_Z_CLEAR_REG` in Mesa's `src/amd/common/ac_descriptors.h`, and allows TC-compatible fast clears only to 0.0 or 1.0); which of the two the texture unit returns for a cleared tile was not measured. - [Depth-only and stencil HTILE fill](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - in a whole-HTILE fill, ZMask (bits 0-3) 0 clears the depth and SMem (bits 8-9) 0 clears the stencil, while SMem 3 leaves the stencil alone (`HtileFillClears`); other SMem values are taken to leave it alone too. The bit meanings follow the gfx10 HTILE layout as the code reads it, not a measurement; `agc_driver_depth_fast_clear_submit` pins them for the paths the Driver takes. - [Depth and stencil plane fills](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - a fill that covers a known depth surface's whole stencil plane (`DepthSliceBytes` of its extent at one byte per texel) with four equal bytes clears the host stencil to that byte the next time the surface is bound, sampled or stored to, and a fill that covers the whole depth plane (two or four bytes per texel) clears the depth when the dword is a float in 0.0-1.0, or two equal 16-bit halves read as UNORM for D16 (`NoteDepthMetadataFill`). A partial fill, a dword whose bytes differ, a depth value outside 0.0-1.0 (0xff000000 is seen), a multisampled surface, or a fill made before the surface exists leave the host image as it was. An HTILE fill or a recognized HTILE clear kernel noted after a plane fill replaces its value with DB_DEPTH_CLEAR/DB_STENCIL_CLEAR. On the console the fill writes the plane the depth block reads; the surfaces this was seen on keep the stencil out of HTILE (PPSA26525 Heretic + Hexen and PPSA21444 DOOM + DOOM II reset the Cohtml UI stencil plane to 1 with a 0x01010101 fill every frame, then draw with stencil Equal 1 and 2), and what the console returns for a plane fill under a compressed HTILE was not measured. `agc_driver_depth_plane_fill` pins the rules. From 1bee5f35b7ccf8fea236da95d179e998e069ca53 Mon Sep 17 00:00:00 2001 From: wail-arch <116732676+wail-arch@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:23:08 +0000 Subject: [PATCH 097/318] fix(relinker): bind guest function exports at address 0 on Linux glibc's loader skips a defined symbol whose value is 0, and the ps5-payload-dev SDK links a shared object's code at address 0, so the first function of every SDK module could not be imported. Export such a function through a jmp in the module's added segment. --- core/relinker/CMakeLists.txt | 4 + .../src/linux/LinuxGuestModuleWriter.cpp | 11 +++ .../test_linux_guest_zero_address_exports.py | 90 +++++++++++++++++++ docs/dev/debt/relinker.md | 1 + 4 files changed, 106 insertions(+) create mode 100644 core/relinker/relinker/tests/test_linux_guest_zero_address_exports.py diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 41bbcc59f0..4be4fc4f17 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -630,6 +630,10 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_absolute_symbols.py $) add_test(NAME guest_module_identity COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_module_identity.py $) + add_test(NAME linux_guest_zero_address_exports COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_guest_zero_address_exports.py + $ $) + add_asan_test_options(linux_guest_zero_address_exports verify_asan_link_order=0) add_test(NAME guest_unused_import_dependencies COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_unused_import_dependencies.py $ $) diff --git a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp index 76b91f97dc..4e8c2fcfdb 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp @@ -62,6 +62,17 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma needsTlsResolver = true; } } + for (std::size_t index = 1; index < image.Symbols.size(); ++index) { + const auto& symbol = image.Symbols[index]; + if (symbol.Value != 0 || symbol.Section == 0 || symbol.Section >= 0xff00 || (symbol.Info >> 4) == 0 || + (symbol.Info & 15) != 2 || symbol.Visibility == 1 || symbol.Visibility == 2) continue; + const auto thunk = address(); + if (thunk > static_cast(std::numeric_limits::max()) - 5) + throw Domain::RelinkerException("Guest export at address 0 exceeds relative branch range"); + bytes.push_back(0xe9); + Io::AppendU32(bytes, static_cast(-static_cast(thunk + 5))); + Io::WriteU64(symbols, index * 24 + 8, thunk); + } std::vector needed; for (const auto& dependency : dependencies) needed.push_back(addString(dependency)); if (needsTlsResolver) needed.push_back(addString("ld-linux-x86-64.so.2")); diff --git a/core/relinker/relinker/tests/test_linux_guest_zero_address_exports.py b/core/relinker/relinker/tests/test_linux_guest_zero_address_exports.py new file mode 100644 index 0000000000..55ee15765e --- /dev/null +++ b/core/relinker/relinker/tests/test_linux_guest_zero_address_exports.py @@ -0,0 +1,90 @@ +import os +from pathlib import Path +import platform +import shutil +import struct +import subprocess +import sys +import tempfile + +from test_guest_deferred_dependencies import page_aligned +from test_guest_intel_trampolines import main_fixture +from test_guest_symbol_names import dynamic_tags, file_offset, symbol_name +from test_windows_import_modules import consumer, provider + + +def at_address_zero(image, info=0x12): + struct.pack_into(" Date: Sat, 24 Oct 2026 10:02:59 +0200 Subject: [PATCH 098/318] test(agc): run the non-offset 2D derivative cases everywhere The maintenance8 gate skips the whole 2D shader, but only its image_sample_*_o cases sample with computed texel offsets. Split the shader into a non-offset and an offsets part and gate only the second, so the seven other 2D cases keep running on GPUs without VK_KHR_maintenance8. --- .../execution/ImageSampleDerivatives.cpp | 157 +++++++++++------- 1 file changed, 94 insertions(+), 63 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleDerivatives.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleDerivatives.cpp index 51e039534e..f40e29c4bd 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleDerivatives.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleDerivatives.cpp @@ -80,7 +80,7 @@ alignas(256) constexpr std::array Input{ alignas(256) std::array Buffer{}; alignas(256) std::array Texels{}; -alignas(256) constexpr std::array Code2D{ +alignas(256) constexpr std::array Code2D{ 0x160200b0, 0x34060086, 0xe0381000, 0x80000401, 0xe0381010, 0x80002801, 0xe0381020, 0x80003201, 0xbe900404, 0xbe920406, 0xbe940408, 0xbe96040a, 0xbe98040c, 0xbe9a040e, 0x7e140280, 0x7e160280, 0x7e180280, 0x7e1a0280, 0x7e1c0280, 0x7e1e0280, 0x7e200280, 0x7e220280, 0x7e240280, 0x7e260280, @@ -93,15 +93,25 @@ alignas(256) constexpr std::array Code2D{ 0xbe9e037e, 0xbefe03ff, 0xaeeeffff, 0xf1a00108, 0x00c40c40, 0xbefe031e, 0x7e80032c, 0x7e82032e, 0x7e840304, 0x7e860305, 0xf0880109, 0x00c40d40, 0x7e80032c, 0x7e82032e, 0x7e840304, 0x7e860305, 0xf1a00109, 0x00c40e40, 0x7e800328, 0x7e820329, 0x7e840332, 0x7e860333, 0x7e880330, 0xf0880108, - 0x40c40f40, 0x7e80032c, 0x7e82032e, 0x7e840330, 0xf0880109, 0x40c41040, 0x7e80033c, 0x7e820328, - 0x7e840329, 0x7e860332, 0x7e880333, 0x7e8a0304, 0x7e8c0305, 0xf1b00108, 0x00c41140, 0x7e80033e, - 0x7e82032c, 0x7e84032e, 0x7e860304, 0x7e880305, 0xf0c80109, 0x00c41240, 0x7e80033c, 0x7e82032c, - 0x7e84032e, 0x7e860304, 0x7e880305, 0xf1b00109, 0x00c41340, 0x7e80033c, 0x7e82032c, 0x7e84032e, - 0x7e860330, 0xf0c80109, 0x40c41440, 0x7e80033c, 0x7e82032c, 0x7e84032e, 0x7e860304, 0x7e880305, - 0xbe9e037e, 0xbefe03ff, 0xaeeeffff, 0xf1b00109, 0x00c41540, 0xbefe031e, 0xbf8c3f70, 0xe0701600, + 0x40c40f40, 0x7e80032c, 0x7e82032e, 0x7e840330, 0xf0880109, 0x40c41040, 0xbf8c3f70, 0xe0701600, 0x80000a03, 0xe0701604, 0x80000b03, 0xe0701608, 0x80000c03, 0xe070160c, 0x80000d03, 0xe0701610, - 0x80000e03, 0xe0701614, 0x80000f03, 0xe0701618, 0x80001003, 0xe070161c, 0x80001103, 0xe0701620, - 0x80001203, 0xe0701624, 0x80001303, 0xe0701628, 0x80001403, 0xe070162c, 0x80001503, 0xbf810000, + 0x80000e03, 0xe0701614, 0x80000f03, 0xe0701618, 0x80001003, 0xbf810000, +}; + +alignas(256) constexpr std::array Code2DOffsets{ + 0x160200b0, 0x34060086, 0xe0381000, 0x80000401, 0xe0381010, 0x80002801, 0xe0381020, 0x80003201, + 0xbe900404, 0xbe920406, 0xbe940408, 0xbe96040a, 0xbe98040c, 0xbe9a040e, 0x7e140280, 0x7e160280, + 0x7e180280, 0x7e1a0280, 0x7e1c0280, 0x7e1e0280, 0x7e200280, 0x7e220280, 0x7e240280, 0x7e260280, + 0x7e280280, 0x7e2a0280, 0x7e2c0280, 0x7e2e0280, 0x7e300280, 0x7e320280, 0xbf8c3f70, 0x5e585328, + 0xd52f002d, 0x0201012a, 0x5e5c6732, 0xd52f002f, 0x02010134, 0x5e600b04, 0xd52f0031, 0x02010106, + 0xd52f0036, 0x02010128, 0xd52f0037, 0x02010132, 0xd52f003b, 0x02010104, 0x7e780281, 0x7e7a02f6, + 0x7e7c02ff, 0x00000101, 0x7e80033c, 0x7e820328, 0x7e840329, 0x7e860332, 0x7e880333, 0x7e8a0304, + 0x7e8c0305, 0xf1b00108, 0x00c41140, 0x7e80033e, 0x7e82032c, 0x7e84032e, 0x7e860304, 0x7e880305, + 0xf0c80109, 0x00c41240, 0x7e80033c, 0x7e82032c, 0x7e84032e, 0x7e860304, 0x7e880305, 0xf1b00109, + 0x00c41340, 0x7e80033c, 0x7e82032c, 0x7e84032e, 0x7e860330, 0xf0c80109, 0x40c41440, 0x7e80033c, + 0x7e82032c, 0x7e84032e, 0x7e860304, 0x7e880305, 0xbe9e037e, 0xbefe03ff, 0xaeeeffff, 0xf1b00109, + 0x00c41540, 0xbefe031e, 0xbf8c3f70, 0xe0701600, 0x80001103, 0xe0701604, 0x80001203, 0xe0701608, + 0x80001303, 0xe070160c, 0x80001403, 0xe0701610, 0x80001503, 0xbf810000, }; alignas(256) constexpr std::array Code3D{ @@ -137,57 +147,77 @@ alignas(256) constexpr std::array Code1D{ 0xe0701608, 0x80000c03, 0xe070160c, 0x80000d03, 0xe0701610, 0x80000e03, 0xbf810000, }; -constexpr std::array Names2D{"image_sample_d", "image_sample_cd", "image_sample_cd (exec)", "image_sample_d_g16", "image_sample_cd_g16", "image_sample_d a16", "image_sample_d_g16 a16", "image_sample_cd_o", "image_sample_d_o_g16", "image_sample_cd_o_g16", "image_sample_d_o_g16 a16", "image_sample_cd_o_g16 (exec)"}; -constexpr std::array Masked2D{false, false, true, false, false, false, false, false, false, false, false, true}; -constexpr std::array Expected2D{ - 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x40100000, - 0x42cc8000, 0x40100000, 0x40100000, 0x40100000, 0x42cc0000, 0x42cc8000, 0x42cc8000, 0x42cc0000, - 0x42cc8000, 0x42cc0000, 0x42cc0000, 0x42ce8000, 0x434b0000, 0x42ce8000, 0x42ce0000, 0x42ce8000, - 0x434b0000, 0x434b4000, 0x434b4000, 0x434b0000, 0x434b4000, 0x434b0000, 0x434b0000, 0x434c4000, - 0x43980000, 0x434c4000, 0x434c0000, 0x434c4000, 0x43980000, 0x43982000, 0x43982000, 0x43980000, - 0x43982000, 0x43980000, 0x43980000, 0x4398a000, 0x43988000, 0x4398a000, 0x43988000, 0x4398a000, - 0x40a00000, 0x40a00000, 0x40a00000, 0x40a00000, 0x40a00000, 0x40a00000, 0x40a00000, 0x40c00000, - 0x42d40000, 0x40c00000, 0x40c00000, 0x40c00000, 0x42d48000, 0x42d40000, 0x42d40000, 0x42d48000, - 0x42d40000, 0x42d48000, 0x42d48000, 0x42d60000, 0x434f4000, 0x42d60000, 0x42d68000, 0x42d60000, - 0x434f0000, 0x434f0000, 0x434f0000, 0x434f0000, 0x434f0000, 0x434f0000, 0x434f0000, 0x43500000, - 0x439a0000, 0x43500000, 0x43500000, 0x43500000, 0x439a0000, 0x439a0000, 0x439a0000, 0x439a0000, - 0x439a0000, 0x439a0000, 0x439a0000, 0x439a8000, 0x439a8000, 0x439a8000, 0x439a8000, 0x439a8000, - 0x41100000, 0x41100000, 0x41100000, 0x41100000, 0x41100000, 0x41100000, 0x41100000, 0x41200000, - 0x42dc0000, 0x41200000, 0x41200000, 0x41200000, 0x42dc0000, 0x42dc0000, 0x42dc0000, 0x42dc0000, - 0x42dc0000, 0x42dc0000, 0x42dc0000, 0x42de0000, 0x43530000, 0x42de0000, 0x42de0000, 0x42de0000, - 0x43534000, 0x43530000, 0x43530000, 0x43534000, 0x43530000, 0x43534000, 0x43534000, 0x43540000, - 0x439c2000, 0x43540000, 0x43544000, 0x43540000, 0x439c0000, 0x439c0000, 0x439c0000, 0x439c0000, - 0x439c0000, 0x439c0000, 0x439c0000, 0x439c8000, 0x439c8000, 0x439c8000, 0x439c8000, 0x439c8000, - 0x41500000, 0x41500000, 0x41500000, 0x41500000, 0x41500000, 0x41500000, 0x41500000, 0x41600000, - 0x42e40000, 0x41600000, 0x41600000, 0x41600000, 0x42e40000, 0x42e40000, 0x42e40000, 0x42e40000, - 0x42e40000, 0x42e40000, 0x42e40000, 0x42e60000, 0x43570000, 0x42e60000, 0x42e60000, 0x42e60000, - 0x43570000, 0x43570000, 0x43570000, 0x43570000, 0x43570000, 0x43570000, 0x43570000, 0x43580000, - 0x439e0000, 0x43580000, 0x43580000, 0x43580000, 0x439e2000, 0x439e0000, 0x439e0000, 0x439e2000, - 0x439e0000, 0x439e2000, 0x439e2000, 0x439e8000, 0x439ea000, 0x439e8000, 0x439ea000, 0x439e8000, - 0x41880000, 0x41880000, 0x00000000, 0x41880000, 0x41880000, 0x41880000, 0x41880000, 0x41900000, - 0x42ec0000, 0x41900000, 0x41900000, 0x00000000, 0x42ec8000, 0x42ec0000, 0x42ec8000, 0x42ec8000, - 0x42ec0000, 0x42ec8000, 0x42ec8000, 0x42ee0000, 0x435b4000, 0x42ee0000, 0x42ee8000, 0x42ee8000, - 0x435b0000, 0x435b0000, 0x435b4000, 0x435b0000, 0x435b0000, 0x435b0000, 0x435b0000, 0x435c0000, - 0x43a00000, 0x435c0000, 0x435c0000, 0x435c4000, 0x43a00000, 0x43a00000, 0x43a02000, 0x43a00000, - 0x43a00000, 0x43a00000, 0x43a00000, 0x43a08000, 0x43a08000, 0x43a08000, 0x43a08000, 0x43a0a000, - 0x41a80000, 0x41a80000, 0x00000000, 0x41a80000, 0x41a80000, 0x41a80000, 0x41a80000, 0x41b00000, - 0x42f40000, 0x41b00000, 0x41b00000, 0x00000000, 0x42f40000, 0x42f40000, 0x42f40000, 0x42f40000, - 0x42f40000, 0x42f40000, 0x42f40000, 0x42f60000, 0x435f0000, 0x42f60000, 0x42f60000, 0x42f60000, - 0x435f4000, 0x435f0000, 0x435f0000, 0x435f4000, 0x435f0000, 0x435f4000, 0x435f4000, 0x43600000, - 0x43a22000, 0x43600000, 0x43604000, 0x43600000, 0x43a20000, 0x43a20000, 0x43a20000, 0x43a20000, - 0x43a20000, 0x43a20000, 0x43a20000, 0x43a28000, 0x43a28000, 0x43a28000, 0x43a28000, 0x43a28000, - 0x41c80000, 0x41c80000, 0x00000000, 0x41c80000, 0x41c80000, 0x41c80000, 0x41c80000, 0x41d00000, - 0x42fc0000, 0x41d00000, 0x41d00000, 0x00000000, 0x42fc0000, 0x42fc0000, 0x42fc0000, 0x42fc0000, - 0x42fc0000, 0x42fc0000, 0x42fc0000, 0x42fe0000, 0x43630000, 0x42fe0000, 0x42fe0000, 0x42fe0000, - 0x43630000, 0x43630000, 0x43630000, 0x43630000, 0x43630000, 0x43630000, 0x43630000, 0x43640000, - 0x43a40000, 0x43640000, 0x43640000, 0x43640000, 0x43a42000, 0x43a40000, 0x43a40000, 0x43a42000, - 0x43a40000, 0x43a42000, 0x43a42000, 0x43a48000, 0x43a4a000, 0x43a48000, 0x43a4a000, 0x43a48000, - 0x41e80000, 0x41e80000, 0x00000000, 0x41e80000, 0x41e80000, 0x41e80000, 0x41e80000, 0x41f00000, - 0x43020000, 0x41f00000, 0x41f00000, 0x00000000, 0x43024000, 0x43020000, 0x43024000, 0x43024000, - 0x43020000, 0x43024000, 0x43024000, 0x43030000, 0x43674000, 0x43030000, 0x43034000, 0x43034000, - 0x43670000, 0x43670000, 0x00000000, 0x43670000, 0x43670000, 0x43670000, 0x43670000, 0x43680000, - 0x43a60000, 0x43680000, 0x43680000, 0x00000000, 0x43a60000, 0x43a60000, 0x43a62000, 0x43a60000, - 0x43a60000, 0x43a60000, 0x43a60000, 0x43a68000, 0x43a68000, 0x43a68000, 0x43a68000, 0x43a6a000, +constexpr std::array Names2D{"image_sample_d", "image_sample_cd", "image_sample_cd (exec)", "image_sample_d_g16", "image_sample_cd_g16", "image_sample_d a16", "image_sample_d_g16 a16"}; +constexpr std::array Masked2D{false, false, true, false, false, false, false}; +constexpr std::array Names2DOffsets{"image_sample_cd_o", "image_sample_d_o_g16", "image_sample_cd_o_g16", "image_sample_d_o_g16 a16", "image_sample_cd_o_g16 (exec)"}; +constexpr std::array Masked2DOffsets{false, false, false, false, true}; +constexpr std::array Expected2D{ + 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x3fa00000, 0x3fa00000, + 0x42cc0000, 0x42cc8000, 0x42cc8000, 0x42cc0000, 0x42cc8000, 0x42cc0000, 0x42cc0000, + 0x434b0000, 0x434b4000, 0x434b4000, 0x434b0000, 0x434b4000, 0x434b0000, 0x434b0000, + 0x43980000, 0x43982000, 0x43982000, 0x43980000, 0x43982000, 0x43980000, 0x43980000, + 0x40a00000, 0x40a00000, 0x40a00000, 0x40a00000, 0x40a00000, 0x40a00000, 0x40a00000, + 0x42d48000, 0x42d40000, 0x42d40000, 0x42d48000, 0x42d40000, 0x42d48000, 0x42d48000, + 0x434f0000, 0x434f0000, 0x434f0000, 0x434f0000, 0x434f0000, 0x434f0000, 0x434f0000, + 0x439a0000, 0x439a0000, 0x439a0000, 0x439a0000, 0x439a0000, 0x439a0000, 0x439a0000, + 0x41100000, 0x41100000, 0x41100000, 0x41100000, 0x41100000, 0x41100000, 0x41100000, + 0x42dc0000, 0x42dc0000, 0x42dc0000, 0x42dc0000, 0x42dc0000, 0x42dc0000, 0x42dc0000, + 0x43534000, 0x43530000, 0x43530000, 0x43534000, 0x43530000, 0x43534000, 0x43534000, + 0x439c0000, 0x439c0000, 0x439c0000, 0x439c0000, 0x439c0000, 0x439c0000, 0x439c0000, + 0x41500000, 0x41500000, 0x41500000, 0x41500000, 0x41500000, 0x41500000, 0x41500000, + 0x42e40000, 0x42e40000, 0x42e40000, 0x42e40000, 0x42e40000, 0x42e40000, 0x42e40000, + 0x43570000, 0x43570000, 0x43570000, 0x43570000, 0x43570000, 0x43570000, 0x43570000, + 0x439e2000, 0x439e0000, 0x439e0000, 0x439e2000, 0x439e0000, 0x439e2000, 0x439e2000, + 0x41880000, 0x41880000, 0x00000000, 0x41880000, 0x41880000, 0x41880000, 0x41880000, + 0x42ec8000, 0x42ec0000, 0x42ec8000, 0x42ec8000, 0x42ec0000, 0x42ec8000, 0x42ec8000, + 0x435b0000, 0x435b0000, 0x435b4000, 0x435b0000, 0x435b0000, 0x435b0000, 0x435b0000, + 0x43a00000, 0x43a00000, 0x43a02000, 0x43a00000, 0x43a00000, 0x43a00000, 0x43a00000, + 0x41a80000, 0x41a80000, 0x00000000, 0x41a80000, 0x41a80000, 0x41a80000, 0x41a80000, + 0x42f40000, 0x42f40000, 0x42f40000, 0x42f40000, 0x42f40000, 0x42f40000, 0x42f40000, + 0x435f4000, 0x435f0000, 0x435f0000, 0x435f4000, 0x435f0000, 0x435f4000, 0x435f4000, + 0x43a20000, 0x43a20000, 0x43a20000, 0x43a20000, 0x43a20000, 0x43a20000, 0x43a20000, + 0x41c80000, 0x41c80000, 0x00000000, 0x41c80000, 0x41c80000, 0x41c80000, 0x41c80000, + 0x42fc0000, 0x42fc0000, 0x42fc0000, 0x42fc0000, 0x42fc0000, 0x42fc0000, 0x42fc0000, + 0x43630000, 0x43630000, 0x43630000, 0x43630000, 0x43630000, 0x43630000, 0x43630000, + 0x43a42000, 0x43a40000, 0x43a40000, 0x43a42000, 0x43a40000, 0x43a42000, 0x43a42000, + 0x41e80000, 0x41e80000, 0x00000000, 0x41e80000, 0x41e80000, 0x41e80000, 0x41e80000, + 0x43024000, 0x43020000, 0x43024000, 0x43024000, 0x43020000, 0x43024000, 0x43024000, + 0x43670000, 0x43670000, 0x00000000, 0x43670000, 0x43670000, 0x43670000, 0x43670000, + 0x43a60000, 0x43a60000, 0x43a62000, 0x43a60000, 0x43a60000, 0x43a60000, 0x43a60000, +}; +constexpr std::array Expected2DOffsets{ + 0x40100000, 0x42cc8000, 0x40100000, 0x40100000, 0x40100000, + 0x42ce8000, 0x434b0000, 0x42ce8000, 0x42ce0000, 0x42ce8000, + 0x434c4000, 0x43980000, 0x434c4000, 0x434c0000, 0x434c4000, + 0x4398a000, 0x43988000, 0x4398a000, 0x43988000, 0x4398a000, + 0x40c00000, 0x42d40000, 0x40c00000, 0x40c00000, 0x40c00000, + 0x42d60000, 0x434f4000, 0x42d60000, 0x42d68000, 0x42d60000, + 0x43500000, 0x439a0000, 0x43500000, 0x43500000, 0x43500000, + 0x439a8000, 0x439a8000, 0x439a8000, 0x439a8000, 0x439a8000, + 0x41200000, 0x42dc0000, 0x41200000, 0x41200000, 0x41200000, + 0x42de0000, 0x43530000, 0x42de0000, 0x42de0000, 0x42de0000, + 0x43540000, 0x439c2000, 0x43540000, 0x43544000, 0x43540000, + 0x439c8000, 0x439c8000, 0x439c8000, 0x439c8000, 0x439c8000, + 0x41600000, 0x42e40000, 0x41600000, 0x41600000, 0x41600000, + 0x42e60000, 0x43570000, 0x42e60000, 0x42e60000, 0x42e60000, + 0x43580000, 0x439e0000, 0x43580000, 0x43580000, 0x43580000, + 0x439e8000, 0x439ea000, 0x439e8000, 0x439ea000, 0x439e8000, + 0x41900000, 0x42ec0000, 0x41900000, 0x41900000, 0x00000000, + 0x42ee0000, 0x435b4000, 0x42ee0000, 0x42ee8000, 0x42ee8000, + 0x435c0000, 0x43a00000, 0x435c0000, 0x435c0000, 0x435c4000, + 0x43a08000, 0x43a08000, 0x43a08000, 0x43a08000, 0x43a0a000, + 0x41b00000, 0x42f40000, 0x41b00000, 0x41b00000, 0x00000000, + 0x42f60000, 0x435f0000, 0x42f60000, 0x42f60000, 0x42f60000, + 0x43600000, 0x43a22000, 0x43600000, 0x43604000, 0x43600000, + 0x43a28000, 0x43a28000, 0x43a28000, 0x43a28000, 0x43a28000, + 0x41d00000, 0x42fc0000, 0x41d00000, 0x41d00000, 0x00000000, + 0x42fe0000, 0x43630000, 0x42fe0000, 0x42fe0000, 0x42fe0000, + 0x43640000, 0x43a40000, 0x43640000, 0x43640000, 0x43640000, + 0x43a48000, 0x43a4a000, 0x43a48000, 0x43a4a000, 0x43a48000, + 0x41f00000, 0x43020000, 0x41f00000, 0x41f00000, 0x00000000, + 0x43030000, 0x43674000, 0x43030000, 0x43034000, 0x43034000, + 0x43680000, 0x43a60000, 0x43680000, 0x43680000, 0x00000000, + 0x43a68000, 0x43a68000, 0x43a68000, 0x43a68000, 0x43a6a000, }; constexpr std::array Names3D{"image_sample_d 3D", "image_sample_cd 3D", "image_sample_d_g16 3D", "image_sample_cd_g16 3D", "image_sample_d a16 3D", "image_sample_d_g16 a16 3D"}; constexpr std::array Masked3D{false, false, false, false, false, false}; @@ -329,12 +359,13 @@ int main() { if (device->Target().subgroupSize < Threads) { std::printf("EXEC-masked cases skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); } - const bool computedOffsets = device->Target().nonConstantImageOffsets; - if (!computedOffsets) std::puts("2D cases skipped, their computed texel offsets need VK_KHR_maintenance8"); + const bool nonConstantOffsets = device->Target().nonConstantImageOffsets; + if (!nonConstantOffsets) std::puts("image_sample_*_o cases skipped without VK_KHR_maintenance8"); std::vector waves{Threads}; if (device->Target().subgroupSize >= Threads) waves.push_back(Wave64Threads); for (const auto threads : waves) { - if (computedOffsets) Run(*device, Code2D, threads, 9u, Height, 1u, Names2D, Masked2D, Expected2D); + Run(*device, Code2D, threads, 9u, Height, 1u, Names2D, Masked2D, Expected2D); + if (nonConstantOffsets) Run(*device, Code2DOffsets, threads, 9u, Height, 1u, Names2DOffsets, Masked2DOffsets, Expected2DOffsets); Run(*device, Code3D, threads, 10u, Height, Depth, Names3D, Masked3D, Expected3D); Run(*device, Code1D, threads, 8u, 1u, 1u, Names1D, Masked1D, Expected1D); } From 90850dd955adaa32e75df11099ffa4287a8acc2a Mon Sep 17 00:00:00 2001 From: Kyon Date: Sat, 24 Oct 2026 10:13:29 +0200 Subject: [PATCH 099/318] test(libSceJson2): retain public object type after missing value --- core/libs/tests/GuestJsonParserState.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/libs/tests/GuestJsonParserState.cpp b/core/libs/tests/GuestJsonParserState.cpp index e6b6edb5bb..0dc8040951 100644 --- a/core/libs/tests/GuestJsonParserState.cpp +++ b/core/libs/tests/GuestJsonParserState.cpp @@ -21,6 +21,7 @@ void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value*); +std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); const Value* APS5_VABI _ZNK3sce4Json5Array5frontEv(const Array*); const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); @@ -161,6 +162,14 @@ int main() { Require(Parse(value, "{}") == 0 && _ZNK3sce4Json5Value5countEv(&value) == 1, "Parsing an empty object into an existing object must retain its members"); _ZN3sce4Json5ValueD1Ev(&value); } + Value missingMember{}; _ZN3sce4Json5ValueC1El(&missingMember, 77); + Require(static_cast(Parse(missingMember, "{\"a\":}")) == 0x80848101 && + _ZNK3sce4Json5Value7getTypeEv(&missingMember) == 7, + "A missing member value must report parse failure while retaining the object type"); + Require(Parse(missingMember, "{}") == 0 && _ZNK3sce4Json5Value7getTypeEv(&missingMember) == 7, + "The retained object must remain usable after a missing-value parse error"); + _ZN3sce4Json5ValueD1Ev(&missingMember); + Require(live.size() == baseline, "The missing-value error and retry must reclaim admitted storage"); Value nested{}; _ZN3sce4Json5ValueC1El(&nested, 77); Require(static_cast(Parse(nested, "[[1,")) == 0x80848101 && nested.type == 6, "Nested failures must retain the outer collection"); const auto* inner = _ZNK3sce4Json5Array5frontEv(reinterpret_cast(nested.payload)); From 98a12e2aa5d4996df436d76ff16c8bfb79fe86d4 Mon Sep 17 00:00:00 2001 From: nozzzd <215746657+nozzzd@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:22:01 +0000 Subject: [PATCH 100/318] feat(libSceNpManager): declare 189 PS5 pending exports --- core/libs/prx/libSceNpManager/CMakeLists.txt | 2 +- .../prx/libSceNpManager/PendingExports.cpp | 765 ++++++++++++++++++ docs/dev/debt/libSceNpManager.md | 2 + 3 files changed, 768 insertions(+), 1 deletion(-) create mode 100644 core/libs/prx/libSceNpManager/PendingExports.cpp diff --git a/core/libs/prx/libSceNpManager/CMakeLists.txt b/core/libs/prx/libSceNpManager/CMakeLists.txt index cd99c36b34..9669381a0a 100644 --- a/core/libs/prx/libSceNpManager/CMakeLists.txt +++ b/core/libs/prx/libSceNpManager/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(libSceNpManager SHARED EXCLUDE_FROM_ALL Export.cpp) +add_library(libSceNpManager SHARED EXCLUDE_FROM_ALL Export.cpp PendingExports.cpp) target_include_directories(libSceNpManager PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceNpManager) diff --git a/core/libs/prx/libSceNpManager/PendingExports.cpp b/core/libs/prx/libSceNpManager/PendingExports.cpp new file mode 100644 index 0000000000..46736416fc --- /dev/null +++ b/core/libs/prx/libSceNpManager/PendingExports.cpp @@ -0,0 +1,765 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI _ZN3sce2np10NpOnlineId13GetNpOnlineIdERKNS0_4UserEP13SceNpOnlineId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineId13GetNpOnlineIdERKNS0_4UserEPS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineId16SetNpOnlineIdStrEPKc(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineId5ClearEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdC1ERK13SceNpOnlineId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdC1ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdC1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdC2ERK13SceNpOnlineId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdC2ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdC2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np10NpOnlineIdD2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np11NpHttpTransC2EP16SceNpAllocatorEx(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np11NpHttpTransD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np11NpHttpTransD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np11NpHttpTransD2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpHttpClient4InitEii(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpHttpClientC1EP16SceNpAllocatorEx(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpHttpClientC2EP16SceNpAllocatorEx(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpHttpClientD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpHttpClientD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpHttpClientD2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpTitleToken5ClearEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpTitleTokenC1ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpTitleTokenC1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpTitleTokenC2ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpTitleTokenC2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpTitleTokenD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpTitleTokenD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np12NpTitleTokenD2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessToken5ClearEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenC1ERK16SceNpAccessToken(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenC1ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenC1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenC2ERK16SceNpAccessToken(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenC2ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenC2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np13NpAccessTokenD2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client10EndRequestEii(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client11InitServiceEi(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client11TermServiceEi(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client11WaitRequestEiij(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client12AbortRequestEii(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client12BeginRequestEii(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client13CreateRequestEimPKvPi(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client13DeleteRequestEii(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client13GetIpmiClientEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client13PollEventFlagEijmjPm(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np3ipc14service_client14PollEventQueueEiPvm(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpId5ClearEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdC1ERK7SceNpId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdC1ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdC1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdC2ERK7SceNpId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdC2ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdC2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4NpIdD2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4User5ClearEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4User7GetUserEiPS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserC1ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserC1Ei(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserC1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserC2ERKS1_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserC2Ei(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserC2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np4UserD2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np8NpTicket5ClearEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np8NpTicketD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np8NpTicketD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2np8NpTicketD2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npeqERK13SceNpOnlineIdRKNS0_10NpOnlineIdE(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npeqERKNS0_10NpOnlineIdERK13SceNpOnlineId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npeqERKNS0_10NpOnlineIdES3_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npeqERKNS0_4UserERKi(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npeqERKNS0_4UserES3_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npneERK13SceNpOnlineIdRKNS0_10NpOnlineIdE(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npneERKNS0_10NpOnlineIdERK13SceNpOnlineId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npneERKNS0_10NpOnlineIdES3_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npneERKNS0_4UserERKi(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZN3sce2npneERKNS0_4UserES3_(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np10NpOnlineId7IsEmptyEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np12NpTitleToken6GetStrEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np13NpAccessToken7IsEmptyEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np4User10IsLoggedInEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np4User12GetAccountIdEPm(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np4User12HasAccountIdEPb(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np4User25GetAccountIdFromRegistoryEPm(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np4User7IsEmptyEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np4User7IsGuestEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np4User9GetUserIdEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np8NpTicket13GetTicketDataEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZNK3sce2np8NpTicket13GetTicketSizeEv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZThn16_N3sce2np11NpHttpTransD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZThn16_N3sce2np11NpHttpTransD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZThn8_N3sce2np11NpHttpTransD0Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _ZThn8_N3sce2np11NpHttpTransD1Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpIpcCreateMemoryFromKernel(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpIpcCreateMemoryFromPool(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpIpcDestroyMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpIpcFreeImpl(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpIpcGetNpMemAllocator(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpIpcMallocImpl(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpIpcReallocImpl(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpManagerCreateMemoryFromKernel(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpManagerCreateMemoryFromPool(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpManagerDestroyMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpManagerFreeImpl(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpManagerGetNpMemAllocator(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpManagerMallocImpl(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI _sceNpManagerReallocImpl(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntGetOnlineIdByAccountId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginGetAccessTokenViaImplicitFlow(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginGetAccountId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginGetAuthenticateResponse(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginGetDeviceCodeInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginGetOnlineId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginGetUserId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginResetSsoToken(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginRevalidatePassword(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginSetUserId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginSignin(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginValidateKratosAuthCode(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntLoginVerifyDeviceCode(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntPfAuth(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntRemoveActiveSigninStateCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntRemoveOnlineIdChangeCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntRemovePlusMemberTypeCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntRemoveSigninStateCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntRevalidatePassword(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntSetPlusMemberTypeNB(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntSetRequestHeader(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntSetTimeout(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntSignout(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntSubmitUserCode(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntTemporarySignout(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntUpdateBcAccessToken(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerIntUpdateVshAccessToken(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerPrxStartVsh(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpManagerPrxStopVsh(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2CreateUserContext(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2DeleteUserContext(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2Init(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2IsInit(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2OptionalCheckCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2RegisterDataType(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2RegisterExtendedDataFilter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2RegisterNotificationExCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2SendPushStatisticsDataSystemTelemetry(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2SetGlobalMutex(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2SetNpCommunicationId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2Term(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2TriggerEmptyUserEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2UnregisterDataType(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2UnregisterExtendedDataFilter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2UnregisterNotificationExCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2UnsetNpCommunicationId(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPush2WaitCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushCheckCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushInit(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushIntBeginInactive(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushIntEndInactive(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushIntGetBindUserState(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushIntGetConnectionState(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushIntRegisterNotificationPacketCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushIntUnregisterNotificationPacketCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushRegisterExtendedDataFilter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushRegisterNotificationExCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushStartNotification(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushStartNotificationA(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushStopNotification(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushStopNotificationA(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushTerm(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushUnregisterExtendedDataFilter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpPushUnregisterNotificationCallback(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpRegisterStateCallbackForToolkit(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpServiceClientInit(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpServiceClientTerm(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpSetAdditionalScope(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpSetGamePresenceOnlineA(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpSetNpTitleIdVsh(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpUnregisterGamePresenceCallbackA(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +int APS5_VABI sceNpUnregisterStateCallbackForToolkit(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/docs/dev/debt/libSceNpManager.md b/docs/dev/debt/libSceNpManager.md index 0595fc92c4..a549870dcd 100644 --- a/docs/dev/debt/libSceNpManager.md +++ b/docs/dev/debt/libSceNpManager.md @@ -6,6 +6,8 @@ ### Unknown function info +- [libSceNpManager pending exports](../../../core/libs/prx/libSceNpManager/PendingExports.cpp) - 189 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceNpManager.c). Their signatures and behavior are unverified; the no-parameter declarations exist for import resolution and throw through `NotImplemented_nid_no_patch`. + - [libSceNpManager pending exports](../../../core/libs/prx/libSceNpManager/Export.cpp) - 143 named callable exports are present in the local `libSceNpManager.sprx` export table and the public PS5 3.20 symbol list. Their signatures and behavior are unverified; each declaration uses the no-parameter unknown-signature convention and throws through `NotImplemented_nid_no_patch`. - [sceNpGetSandboxInfo](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager, NID `ZVZ++68nNI8`) - the name hashes to the imported NID (found with nidhunt, confirmed with `tools/nid_names.py`). STAR WARS: Galactic Racer (PPSA28416) imports it; its OnlineSubsystemPS5 calls it with one pointer to a 0x50-byte stack buffer and reads the buffer as a string (discussions/3086). The returned string, the exact parameter struct and console errors are unknown, so every call throws without reading or writing caller memory. A working implementation belongs with that title's bring-up. From fbf7004b7d3fc94c7c2107b4cd49cfb5871d3b33 Mon Sep 17 00:00:00 2001 From: Retr0 Date: Sat, 24 Oct 2026 03:07:16 +0300 Subject: [PATCH 101/318] fix(libc): reopen a stream onto /dev/random or /dev/urandom with freopen --- core/libs/prx/libc/src/FileAndHeap.cpp | 33 +++++++++++++++++++++----- core/libs/tests/GuestFilesystem.cpp | 20 ++++++++++++++++ docs/dev/debt/libc.md | 2 +- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libc/src/FileAndHeap.cpp b/core/libs/prx/libc/src/FileAndHeap.cpp index c5a44fa29e..136aa74fcc 100644 --- a/core/libs/prx/libc/src/FileAndHeap.cpp +++ b/core/libs/prx/libc/src/FileAndHeap.cpp @@ -334,6 +334,20 @@ FileStream* APS5_VABI fdopen_nid_postfix(int descriptor, const char* mode) { } catch (const std::bad_alloc&) { errno = 12; return nullptr; } } +static bool IsRandomDevicePath(std::string_view path) { + return path == "/dev/random" || path == "/dev/urandom"; +} + +#ifdef _WIN32 +static const std::filesystem::path NullDevicePath = "NUL"; +#else +static const std::filesystem::path NullDevicePath = "/dev/null"; +#endif + +static std::FILE* OpenNullDevice(const std::filesystem::path& = {}, const std::string& = {}) { + return std::fopen(NullDevicePath.string().c_str(), "rb"); +} + static int OpenFailure(const std::filesystem::path& path, bool writes, int hostError) { const int error = GuestErrnoFromHost_nid_no_patch(hostError); std::error_code status; @@ -387,6 +401,17 @@ FileStream* APS5_VABI freopen_nid_postfix(const char* filename, const char* mode const auto parsed = ParseFileMode(mode); if (!parsed) { errno = 22; return nullptr; } try { + if (IsRandomDevicePath(filename)) { + if (parsed->writes) throw std::runtime_error(std::string(__func__) + ": writing to " + filename + " is not implemented"); + if (!stream->Reopen(NullDevicePath, "rb", OpenNullDevice)) { + const int error = errno; + if (stream->IsDynamic()) delete stream; + errno = error; + return nullptr; + } + stream->MarkRandomDevice(); + return stream; + } std::filesystem::path path; if (const int error = *filename ? ResolvePathError_nid_no_patch(filename, path) : 2) { if (stream->IsHostStandard()) stream->Detach(); @@ -425,13 +450,9 @@ FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode) errno = EINVAL; return nullptr; } - if (const std::string_view path(filename); path == "/dev/random" || path == "/dev/urandom") { + if (IsRandomDevicePath(filename)) { if (parsed->writes) throw std::runtime_error(std::string(__func__) + ": writing to " + filename + " is not implemented"); -#ifdef _WIN32 - std::unique_ptr device(std::fopen("NUL", "rb"), std::fclose); -#else - std::unique_ptr device(std::fopen("/dev/null", "rb"), std::fclose); -#endif + std::unique_ptr device(OpenNullDevice(), std::fclose); if (!device) return nullptr; auto stream = std::make_unique(device.get(), true); device.release(); diff --git a/core/libs/tests/GuestFilesystem.cpp b/core/libs/tests/GuestFilesystem.cpp index 720a8e8188..7650f8d44a 100644 --- a/core/libs/tests/GuestFilesystem.cpp +++ b/core/libs/tests/GuestFilesystem.cpp @@ -1551,6 +1551,26 @@ int main() { } Require(rejected); } + auto* reopened = fopen_nid_postfix(presentName.c_str(), "rb"); + Require(reopened != nullptr && freopen_nid_postfix(device, "r", reopened) == reopened); + std::memset(first, 0, sizeof(first)); + std::memset(second, 0, sizeof(second)); + Require(fread_nid_postfix(first, 1, sizeof(first), reopened) == sizeof(first)); + Require(fread_nid_postfix(second, 1, sizeof(second), reopened) == sizeof(second)); + Require(std::memcmp(first, second, sizeof(first)) != 0); + Require(feof_nid_postfix(reopened) == 0 && ferror_nid_postfix(reopened) == 0); + Require(freopen_nid_postfix(presentName.c_str(), "rb", reopened) == reopened); + Require(fread_nid_postfix(first, 1, sizeof(first), reopened) == std::filesystem::file_size(present) && fgetc_nid_postfix(reopened) == -1); + Require(freopen_nid_postfix(device, "rb", reopened) == reopened && fclose_nid_postfix(reopened) == 0); + reopened = fopen_nid_postfix(presentName.c_str(), "rb"); + Require(reopened != nullptr); + bool rejected = false; + try { + freopen_nid_postfix(device, "r+", reopened); + } catch (const std::runtime_error&) { + rejected = true; + } + Require(rejected && fclose_nid_postfix(reopened) == 0); } Require(unlink_nid_postfix(presentName.c_str()) == 0 && !std::filesystem::exists(present)); const auto empty = root / "empty"; diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 82e7b65c99..23f2519d2d 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -122,7 +122,7 @@ - [`fopen` and `freopen`](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) on Windows: `freopen` onto a directory in a read mode fails with `EACCES`, where FreeBSD opens it. Reading a directory stream from `fopen` fails with `EISDIR` and sets the error indicator on every host, as FreeBSD 11 tmpfs does; FreeBSD 11 FFS returns the directory's raw blocks instead, and what the console's filesystems return is unverified - [readdir and readdir_r](../../../core/libs/prx/libc/src/Directory.cpp) (libc) on Windows report, on a file system whose directory listing carries no file ID (the entry's ID is 0, or the volume refuses `FileIdBothDirectoryInfo`), a nonzero `d_fileno` hashed from the entry name and the directory's file index, which differs from `lstat`'s `st_ino`. Not reproduced: NTFS lists an ID for every entry - [Writes on an update stream](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) on Windows go through the UCRT, which drops a write that directly follows a read without `fflush`, `fseek`, `fsetpos` or `rewind` unless the read reached end of file: `fwrite` returns 0, and `fputc` and `fprintf` report success, without the error indicator. FreeBSD 11.4's [`__swsetup`](https://github.com/freebsd/freebsd-src/blob/d106002967eff712894ab8b5337bbbff106ce5e0/lib/libc/stdio/wsetup.c#L59-L74) discards the read buffer and writes at the descriptor's offset, past a read-ahead of `st_blksize` bytes for a regular file, so where the console writes depends on its block size. Reads directly after writes follow FreeBSD's `__srefill` -- [`fopen` of /dev/random and /dev/urandom](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) in a read mode returns a stream over the host null device whose `fread` fills the buffer with host OS entropy, as the libkernel [random device](../../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) descriptors do. Only the exact paths are recognised, the stream has no descriptor, `setvbuf`, `feof`, `ferror`, `clearerr`, `flockfile`, `ftrylockfile`, `funlockfile` and `fclose` work on it, `fflush` returns 0 as on a FreeBSD read-only stream, and every other stream function throws, as does opening the paths for writing. `freopen` onto these paths still fails with `ENOENT`. ProsperoTV (PPSA99003) reads its phone pairing code this way +- [`fopen` and `freopen` of /dev/random and /dev/urandom](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) in a read mode return a stream over the host null device whose `fread` fills the buffer with host OS entropy, as the libkernel [random device](../../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) descriptors do. Only the exact paths are recognised, the stream has no descriptor, `setvbuf`, `feof`, `ferror`, `clearerr`, `flockfile`, `ftrylockfile`, `funlockfile` and `fclose` work on it, `fflush` returns 0 as on a FreeBSD read-only stream, and every other stream function throws, as does opening the paths for writing. ProsperoTV (PPSA99003) reads its phone pairing code this way - [RemoveTree](../../../core/libs/prx/libc/include/RemoveTree.hpp) (libc) removes a host tree itself on Windows instead of calling `std::filesystem::remove_all`: MinGW libstdc++ up to GCC 16 does not treat a reparse point as a link, so `remove_all` recurses through it and never terminates, and any tree that holds a symlink hangs. It builds the entry list from `GetFileAttributesW`, removes a reparse point as the link it is rather than its target, clears the read-only attribute and retries as `remove_all` does, and reports the first native error. The libstdc++ fix is in GCC 17 (commit 88856ad855ae); other hosts call `remove_all`. - The [`_Unwind_Context`](../../../core/libs/prx/libc/include/exceptions/Unwind.hpp) that `_Unwind_Backtrace` passes to its callback starts with a table laid out like LLVM libunwind's `AbstractUnwindCursor` virtual table, because homebrew built with the ps5-payload-dev SDK links libunwind statically and reads the frames with its own `_Unwind_GetIP` (`__unw_get_reg`), which calls through that table. Only register reads and writes (`validReg`, `getReg`, `setReg`, with `UNW_REG_IP` and `UNW_REG_SP`), `validFloatReg` (always false) and `isSignalFrame` are answered; the other slots throw. The layout follows libunwind's `UnwindCursor.hpp`, not the console's own `libSceLibcInternal`, which is not checked - The Windows libc [exception unwinder](../../../core/libs/prx/libc/src/exception/Unwind.cpp) cannot run arbitrary Windows exception or termination handlers. It accepts GCC C and C++ handlers identified by a project's `.sehgcc` section and interprets their LSDA with the guest personalities. A handler in an unmarked module, such as a C++ cleanup through `__gxx_personality_seh0` in `libstdc++-6.dll` or a system DLL function like `qsort` in `ucrtbase.dll` (Windows 10 22H2), still stops unwinding and the exception is reported as unhandled. SjLj exceptions are unsupported; Scoop GCC 15.2.0 selects them when SEH is disabled and lacks the System V ABI SEH compiler fix carried by MSYS2. From 69a7bd2c19dd68c1b269705ef33bfbf35609452c Mon Sep 17 00:00:00 2001 From: Dom3005 Date: Sat, 24 Oct 2026 15:29:09 +0200 Subject: [PATCH 102/318] feat(libkernel_sys): create symlink targets with parent-directory components Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libkernel_sys/Export.cpp | 38 ++++++++++++++++++++++---- core/libs/tests/GuestKernelSys.cpp | 34 +++++++++++++++++++++-- docs/dev/debt/libkernel_sys.md | 1 + 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libkernel_sys/Export.cpp b/core/libs/prx/libkernel_sys/Export.cpp index cd60e58230..c042d2944f 100644 --- a/core/libs/prx/libkernel_sys/Export.cpp +++ b/core/libs/prx/libkernel_sys/Export.cpp @@ -243,6 +243,33 @@ bool HasLongComponent(std::string_view path) { return false; } +bool HostFollowsGuestTarget(const char* created, const char* target, const std::filesystem::path& native) { + auto directory = native.parent_path(); + for (const auto& part : std::filesystem::path(target)) { + if (part.empty() || part == ".") continue; + if (part != "..") { + directory /= part; + continue; + } + std::error_code error; + if (!std::filesystem::is_directory(directory, error) || IsSymbolicLink(directory)) return false; + directory = directory.parent_path(); + } + const std::string_view link(created); + const auto separator = link.find_last_of("/\\"); + std::string guest = separator == std::string_view::npos ? std::string(".") : std::string(link.substr(0, separator == 0 ? 1 : separator)); + guest += '/'; + guest += target; + std::filesystem::path resolved; + if (ResolvePathError_nid_no_patch(guest.c_str(), resolved)) return false; + const auto trimmed = [](std::filesystem::path path) { + path = path.lexically_normal(); + if (!path.has_filename() && path.has_relative_path()) path = path.parent_path(); + return path; + }; + return trimmed(resolved) == trimmed(directory); +} + bool IsDotOrDotDot(std::string_view path) { while (!path.empty() && (path.back() == '/' || path.back() == '\\')) path.remove_suffix(1); const auto separator = path.find_last_of("/\\"); @@ -518,12 +545,6 @@ int APS5_VABI symlink_nid_postfix(const char* target, const char* created) { NotImplemented_nid_no_patch("symlink: absolute targets"); return -1; } - for (const auto& part : contents) { - if (part == "..") { - NotImplemented_nid_no_patch("symlink: parent-directory targets"); - return -1; - } - } std::filesystem::path native; if (const int error = ResolvePathError_nid_no_patch(created, native)) return static_cast(Fail(error)); std::error_code error; @@ -533,6 +554,11 @@ int APS5_VABI symlink_nid_postfix(const char* target, const char* created) { if (std::filesystem::exists(std::filesystem::symlink_status(native, error))) return static_cast(Fail(GuestEexist)); #endif if (!std::filesystem::is_directory(native.parent_path(), error)) return static_cast(Fail(HasNonDirectoryPrefix(native) ? GuestEnotdir : GuestEnoent)); + if (std::any_of(contents.begin(), contents.end(), [](const std::filesystem::path& part) { return part == ".."; }) && + !HostFollowsGuestTarget(created, target, native)) { + NotImplemented_nid_no_patch("symlink: a parent-directory target the host would not follow to the guest path it names"); + return -1; + } const bool directory = std::filesystem::is_directory(native.parent_path() / contents, error); #ifdef _WIN32 DWORD flags = SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; diff --git a/core/libs/tests/GuestKernelSys.cpp b/core/libs/tests/GuestKernelSys.cpp index 1650d050d4..c57b34ee8c 100644 --- a/core/libs/tests/GuestKernelSys.cpp +++ b/core/libs/tests/GuestKernelSys.cpp @@ -61,6 +61,8 @@ int APS5_VABI dup2_nid_postfix(int, int); int APS5_VABI setsid_nid_postfix(void); int APS5_VABI link_nid_postfix(const char*, const char*); int APS5_VABI symlink_nid_postfix(const char*, const char*); +void AddPathAlias_nid_no_patch(const char*, const char*); +void RemovePathAlias_nid_no_patch(const char*); int APS5_VABI openat_nid_postfix(int, const char*, int, ...); int APS5_VABI _openat_nid_postfix(int, const char*, int, ...); int APS5_VABI unlinkat_nid_postfix(int, const char*, int); @@ -213,14 +215,14 @@ bool RemoveSymlink(const std::string& link) { void CheckSymlinkTargetRestrictions(const std::filesystem::path& root) { const auto created = (root / "rejected").string(); - for (const auto& target : {std::string(".."), std::string("../x"), std::string("a/../b"), std::string("a/../../x"), std::string("/abs")}) + for (const auto& target : {std::string("a/../b"), std::string("a/../../x"), std::string("/abs")}) Require(SymlinkTargetThrows(target.c_str(), created)); #ifdef _WIN32 for (const auto& target : {std::string("C:/"), std::string("\\\\server\\share")}) Require(SymlinkTargetThrows(target.c_str(), created)); #endif const auto links = [](const std::string& created, const char* target) { - char buffer[16]; + char buffer[64]; const auto length = std::strlen(target); return readlink_nid_postfix(created.c_str(), buffer, sizeof(buffer)) == static_cast(length) && std::memcmp(buffer, target, length) == 0; @@ -234,6 +236,34 @@ void CheckSymlinkTargetRestrictions(const std::filesystem::path& root) { const auto ellipsis = (root / "accepted-ellipsis").string(); Require(symlink_nid_postfix("...", ellipsis.c_str()) == 0 && links(ellipsis, "...")); Require(RemoveSymlink(nested) && RemoveSymlink(dots) && RemoveSymlink(ellipsis)); + const auto climb = root / "climb"; + Require(std::filesystem::create_directories(climb / "deeper")); + const auto readsFile = [](const std::string& link) { + const int fd = sceKernelOpen(link.c_str(), SCE_KERNEL_O_RDONLY, 0); + char byte = 0; + const bool read = fd >= 0 && sceKernelRead(fd, &byte, 1) == 1 && byte == 'x'; + return fd >= 0 && sceKernelClose(fd) == 0 && read; + }; + const auto up = (climb / "up").string(); + Require(symlink_nid_postfix("../file.txt", up.c_str()) == 0 && links(up, "../file.txt") && readsFile(up)); + const auto inner = (climb / "inner").string(); + Require(symlink_nid_postfix("deeper/../../file.txt", inner.c_str()) == 0 && links(inner, "deeper/../../file.txt") && readsFile(inner)); + const auto parent = (climb / "parent").string(); + Require(symlink_nid_postfix("..", parent.c_str()) == 0 && links(parent, "..") && readsFile((climb / "parent" / "file.txt").string())); + const auto parentDangling = (climb / "dangling").string(); + Require(symlink_nid_postfix("../missing/x", parentDangling.c_str()) == 0 && links(parentDangling, "../missing/x")); + const auto through = (root / "through").string(); + Require(symlink_nid_postfix("climb", through.c_str()) == 0); + Require(SymlinkTargetThrows("../file.txt", (root / "through" / "via").string())); + Require(!std::filesystem::exists(std::filesystem::symlink_status(climb / "via"))); + const auto mount = "/anyps5-symlink-climb-mount"; + AddPathAlias_nid_no_patch(mount, std::filesystem::absolute(climb).string().c_str()); + Require(SymlinkTargetThrows("../file.txt", std::string(mount) + "/escape")); + Require(symlink_nid_postfix("deeper/../up", (std::string(mount) + "/kept").c_str()) == 0 && readsFile((climb / "kept").string())); + RemovePathAlias_nid_no_patch(mount); + Require(RemoveSymlink(up) && RemoveSymlink(inner) && RemoveSymlink(parent) && RemoveSymlink(parentDangling)); + Require(RemoveSymlink((climb / "kept").string()) && RemoveSymlink(through)); + Require(std::filesystem::remove(climb / "deeper") && std::filesystem::remove(climb)); } else { Require(link == -1 && Error() == 1); std::fprintf(stderr, "symbolic links unavailable: the symlink target success cases not tested\n"); diff --git a/docs/dev/debt/libkernel_sys.md b/docs/dev/debt/libkernel_sys.md index 5f015be839..a49f4deb39 100644 --- a/docs/dev/debt/libkernel_sys.md +++ b/docs/dev/debt/libkernel_sys.md @@ -19,6 +19,7 @@ - [mkdirat](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) passes `AT_FDCWD` (-100) and absolute paths to `mkdir`. Any other relative path is joined to the descriptor's host directory, found the same way as for `fchdir`, so guest path aliases and case-insensitive name matching do not apply to it, and a `..` component is not kept inside the guest root. It returns `EBADF` for a descriptor that is not open and `ENOTDIR` for one that is not a directory. On Windows, `mode` is ignored. Console behaviour is unverified - [readlink](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) on Windows reads a symbolic link's target from its reparse data and returns it with `/` separators, only when the target is relative, as `symlink` creates them. A symbolic link with an absolute target returns its print name, or its substitute name when the print name is empty, without a `\??\` or `\\?\` prefix. A directory junction returns the same kind of name when it is a drive path (`X:\...`), using the print name when that is a drive path and the substitute name otherwise. A mounted volume (`\??\Volume{GUID}\`), any other name surrogate and every reparse point without the name surrogate bit (WOF compression, deduplication, cloud files) return `EINVAL`, as FreeBSD does for a file that is not a link and as CPython's `os.readlink`, which reads only symbolic links and junctions, does - [aio_read, aio_write, aio_fsync, aio_error, aio_return, aio_suspend, aio_waitcomplete, aio_cancel](../../../core/libs/prx/libkernel_sys/Aio.cpp) (libkernel_sys) - a request with `SIGEV_SIGNAL`, `SIGEV_THREAD_ID` or `SIGEV_KEVENT` completion, or on a socket, throws, because delivering a guest signal or kevent and socket aio are not modelled. One worker thread runs the process's requests one at a time in submission order, so a request is cancellable only until that thread takes it. A queued request keeps the descriptor number, not the open file: closing or reusing the descriptor before it runs changes what it reaches. Pipes fail with `EOPNOTSUPP`, as FreeBSD 11 does with `vfs.aio.enable_unsafe` 0; the console's setting is unverified. Waits are not interrupted by signals (`EINTR`), and timeouts under 1 ms round up to 1 ms (hz 1000). Otherwise the rules follow FreeBSD 11 `vfs_aio.c` +- [symlink](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) stores the target unchanged in a host symbolic link, which the host follows later, not the guest path resolver. It throws for an absolute target, which would need a host path in the link and a translation back in `readlink`. A target with a `..` component is created only when the host and the guest would reach the same place: every directory a `..` leaves must exist and not be a symbolic link, so the host's physical `..` matches the guest resolver's lexical one, and the target joined to the link's host directory must equal the guest path it names, resolved through the guest mounts. A `..` that climbs out of a mount such as `/savedata0`, or past a directory reached through a symbolic link, throws, where FreeBSD 11.4's `namei` follows it from the link's directory. Console behaviour is unverified - [readlinkat](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) passes `AT_FDCWD` (-100) and absolute paths to `readlink`. Any other relative path is joined to the descriptor's host directory, found the same way as for `fchdir`, so guest path aliases and case-insensitive name matching do not apply to it, and a `..` component is not kept inside the guest root. It returns `EBADF` for a descriptor that is not open and `ENOTDIR` for one that is not a directory. Console behaviour is unverified - [link](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) on Windows resolves a symbolic link source through the host, which follows longer chains than FreeBSD: a chain of 40 links is linked, where FreeBSD 11.4 returns `ELOOP` after 32 (`MAXSYMLINKS`). A loop returns `ELOOP` as on FreeBSD - [fstatat](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) passes `AT_FDCWD` (-100) and absolute paths to `stat`, or to `lstat` with `AT_SYMLINK_NOFOLLOW` (0x200). Any other relative path is joined to the descriptor's host directory, found the same way as for `fchdir`, so guest path aliases and case-insensitive name matching do not apply to it, and a `..` component is not kept inside the guest root. It returns `EBADF` for a descriptor that is not open and `ENOTDIR` for one that is not a directory, and flags other than `AT_SYMLINK_NOFOLLOW` return `EINVAL`. PPSA74126 imports it. Console behaviour is unverified From 33fad67fd99b1fb828b73ac390de033813a36ecb Mon Sep 17 00:00:00 2001 From: nozzzd <215746657+nozzzd@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:29:18 +0000 Subject: [PATCH 103/318] docs(libSceNpManager): list the pending declarations after the existing entries --- docs/dev/debt/libSceNpManager.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/dev/debt/libSceNpManager.md b/docs/dev/debt/libSceNpManager.md index a549870dcd..33dad19e2d 100644 --- a/docs/dev/debt/libSceNpManager.md +++ b/docs/dev/debt/libSceNpManager.md @@ -6,8 +6,6 @@ ### Unknown function info -- [libSceNpManager pending exports](../../../core/libs/prx/libSceNpManager/PendingExports.cpp) - 189 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceNpManager.c). Their signatures and behavior are unverified; the no-parameter declarations exist for import resolution and throw through `NotImplemented_nid_no_patch`. - - [libSceNpManager pending exports](../../../core/libs/prx/libSceNpManager/Export.cpp) - 143 named callable exports are present in the local `libSceNpManager.sprx` export table and the public PS5 3.20 symbol list. Their signatures and behavior are unverified; each declaration uses the no-parameter unknown-signature convention and throws through `NotImplemented_nid_no_patch`. - [sceNpGetSandboxInfo](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager, NID `ZVZ++68nNI8`) - the name hashes to the imported NID (found with nidhunt, confirmed with `tools/nid_names.py`). STAR WARS: Galactic Racer (PPSA28416) imports it; its OnlineSubsystemPS5 calls it with one pointer to a 0x50-byte stack buffer and reads the buffer as a string (discussions/3086). The returned string, the exact parameter struct and console errors are unknown, so every call throws without reading or writing caller memory. A working implementation belongs with that title's bring-up. @@ -16,6 +14,7 @@ - [sceNpSetTimeout](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager) - the signature follows [orbital](https://github.com/AlexAltea/orbital/blob/b321fcdbd700cc02da1cb1ad25412781d5ec73c6/tools/ida/db_types.json), [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/0fe263a4760dfbfa973366890061749b4af0de97/src/core/libraries/np/np_manager.cpp) and [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Np/NpAccount.cs); the rejected values (all five 0, negative retries, a resolve timeout under 1 s or another timeout under 10 s, 0 meaning the default) are shadPS4's, not checked on a console. An id that was never created or was deleted returns `REQUEST_NOT_FOUND`, as in shadPS4; there is no network, so the timeouts are not stored - [sceNpGetUserIdByAccountId](../../../core/libs/prx/libSceNpManager/Export.cpp) - the `(account id, user id output)` signature and its errors follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/np/np_manager.cpp): account id 0 or a null output returns `SCE_NP_ERROR_INVALID_ARGUMENT`, and a signed-out NP returns `SCE_NP_ERROR_SIGNED_OUT` without writing the output. NP is always signed out here, so no account id resolves to a user - [sceNpNotifyPlusFeature](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager) - PS4 titles import it; its parameter block and what the console does with it for a signed-out user are unknown ([shadPS4](https://github.com/shadps4-emu/shadPS4/blob/81be949167d0060b08f26dc3e055a0b640d129c8/src/core/libraries/np/np_manager.cpp#L1170) returns 0 without reading it), so every call throws. +- [libSceNpManager pending exports](../../../core/libs/prx/libSceNpManager/PendingExports.cpp) - 189 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceNpManager.c). Their signatures and behavior are unverified; the no-parameter declarations exist for import resolution and throw through `NotImplemented_nid_no_patch`. ### Functional From c01486aaf778db5da1019c61a765c453ed4ee5ad Mon Sep 17 00:00:00 2001 From: wail-arch <116732676+wail-arch@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:38:00 +0000 Subject: [PATCH 104/318] test(relinker): cover each export kind at address 0 on Linux Test weak, protected and aliased function exports at address 0, which each get their own jmp, next to internal, hidden and object exports, which stay at 0. Record that aliases get separate addresses, that the jmp must lie within 2 GiB of address 0 and that a data import at 0 is reported only by the host loader. Label the test and give it a timeout like its neighbours. --- core/relinker/CMakeLists.txt | 2 + .../test_linux_guest_zero_address_exports.py | 45 +++++++++++++++---- docs/dev/debt/relinker.md | 2 +- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 4be4fc4f17..e309d58eb8 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -633,6 +633,8 @@ if(BUILD_TESTING) add_test(NAME linux_guest_zero_address_exports COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_guest_zero_address_exports.py $ $) + set_property(TEST linux_guest_zero_address_exports APPEND PROPERTY LABELS relinker) + set_tests_properties(linux_guest_zero_address_exports PROPERTIES TIMEOUT 30) add_asan_test_options(linux_guest_zero_address_exports verify_asan_link_order=0) add_test(NAME guest_unused_import_dependencies COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_unused_import_dependencies.py diff --git a/core/relinker/relinker/tests/test_linux_guest_zero_address_exports.py b/core/relinker/relinker/tests/test_linux_guest_zero_address_exports.py index 55ee15765e..c6ca0bcbde 100644 --- a/core/relinker/relinker/tests/test_linux_guest_zero_address_exports.py +++ b/core/relinker/relinker/tests/test_linux_guest_zero_address_exports.py @@ -8,13 +8,13 @@ import sys import tempfile from test_guest_deferred_dependencies import page_aligned +from test_guest_export_lookup import guest_symbols from test_guest_intel_trampolines import main_fixture from test_guest_symbol_names import dynamic_tags, file_offset, symbol_name from test_windows_import_modules import consumer, provider -def at_address_zero(image, info=0x12): - struct.pack_into(" Date: Sat, 24 Oct 2026 13:40:49 +0200 Subject: [PATCH 105/318] feat(ngs2): implement the custom Reverb module --- .../libs/prx/libSceNgs2.native/CMakeLists.txt | 2 +- .../libSceNgs2.native/include/Ngs2Types.hpp | 1 + .../libs/prx/libSceNgs2.native/src/Custom.cpp | 21 +++ .../libSceNgs2.native/src/Ngs2Internal.hpp | 10 +- .../libs/prx/libSceNgs2.native/src/Reverb.cpp | 170 +++++++++++++++--- core/libs/prx/libSceNgs2.native/src/Voice.cpp | 2 + .../libSceNgs2.native/tests/Ngs2Custom.cpp | 4 +- .../tests/Ngs2CustomReverb.cpp | 170 ++++++++++++++++++ docs/dev/debt/libSceNgs2.native.md | 4 +- 9 files changed, 355 insertions(+), 29 deletions(-) create mode 100644 core/libs/prx/libSceNgs2.native/tests/Ngs2CustomReverb.cpp diff --git a/core/libs/prx/libSceNgs2.native/CMakeLists.txt b/core/libs/prx/libSceNgs2.native/CMakeLists.txt index 93bb6c445e..cd116b79fa 100644 --- a/core/libs/prx/libSceNgs2.native/CMakeLists.txt +++ b/core/libs/prx/libSceNgs2.native/CMakeLists.txt @@ -2,7 +2,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/Ngs2Library.cmake) add_sce_ngs2_library(libSceNgs2.native) if(BUILD_TESTING) - foreach(ngs2Test IN ITEMS Render Atrac9 Filter Custom Parse Stream UserFx Report Geom VoiceIndex HandleErrors VoiceHandles) + foreach(ngs2Test IN ITEMS Render Atrac9 Filter Custom CustomReverb Parse Stream UserFx Report Geom VoiceIndex HandleErrors VoiceHandles) string(TOLOWER ${ngs2Test} testName) add_test_executable(ngs2_${testName}_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/Ngs2${ngs2Test}.cpp) target_include_directories(ngs2_${testName}_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/LibAtrac9/C/src) diff --git a/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp b/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp index eda5e07ef3..5d62570f3d 100644 --- a/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp +++ b/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp @@ -92,6 +92,7 @@ static constexpr std::uint32_t SCE_NGS2_CUSTOM_MASTERING_VOICE_PARAM_SETUP = 0x4 static constexpr std::uint32_t SCE_NGS2_CUSTOM_MASTERING_VOICE_PARAM_GAIN = 0x40030004; static constexpr std::uint32_t SCE_NGS2_CUSTOM_MASTERING_VOICE_PARAM_OUTPUT = 0x40030005; static constexpr std::uint32_t SCE_NGS2_CUSTOM_VOICE_PARAM_FILTER = 0x40001300; +static constexpr std::uint32_t SCE_NGS2_CUSTOM_VOICE_PARAM_REVERB = 0x40001b00; static constexpr std::uint32_t SCE_NGS2_CUSTOM_VOICE_PARAM_GAIN = 0x40001500; static constexpr std::uint32_t SCE_NGS2_CUSTOM_VOICE_PARAM_USER_FX = 0x40001900; static constexpr std::uint32_t SCE_NGS2_CUSTOM_VOICE_PARAM_DELAY = 0x40001d00; diff --git a/core/libs/prx/libSceNgs2.native/src/Custom.cpp b/core/libs/prx/libSceNgs2.native/src/Custom.cpp index 5d066d889a..8dfdb1efe2 100644 --- a/core/libs/prx/libSceNgs2.native/src/Custom.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Custom.cpp @@ -93,6 +93,7 @@ int Ngs2CheckCustomRack(std::uint32_t rackId, const Ngs2CustomRackOption& option for (std::uint32_t m = 0; m < option.num_modules; m++) { const auto moduleId = option.module[m].module_id; if (moduleId == SCE_NGS2_CUSTOM_MODULE_ID_GAIN && option.module[m].option != nullptr) throw std::runtime_error("NGS2: an option for the custom Gain module is not implemented"); + if (moduleId == SCE_NGS2_CUSTOM_MODULE_ID_REVERB) continue; if (moduleId == SCE_NGS2_CUSTOM_MODULE_ID_GAIN && rackId == SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER) continue; if (moduleId == SCE_NGS2_CUSTOM_MODULE_ID_GAIN) throw std::runtime_error("NGS2: the custom Gain module on a custom sampler rack is not implemented"); const char* name = BuiltInModuleName(moduleId); @@ -119,11 +120,17 @@ void Ngs2SetupUserFx(Ngs2Rack& rack, const Ngs2CustomRackOption& option) { voice.userFx.resize(option.num_modules); voice.customUserFx.resize(option.num_modules); voice.customGain.assign(option.num_modules, {}); + voice.customReverb.resize(option.num_modules); } for (std::uint32_t m = 0; m < option.num_modules; m++) { rack.customModuleIds[m] = option.module[m].module_id; const auto& info = option.module[m]; rack.customModules[m] = {info.module_id, info.source_buffer_id, info.extra_buffer_id, info.dest_buffer_id, info.state_offset, info.state_size, 0, 0}; + if (info.module_id == SCE_NGS2_CUSTOM_MODULE_ID_REVERB) { + const auto size = info.option == nullptr ? 1u : OptionWord(*info.option); + for (auto& voice : rack.voices) voice.customReverb[m] = Ngs2CreateCustomReverb(size); + continue; + } if (option.module[m].module_id == SCE_NGS2_CUSTOM_MODULE_ID_FILTER) continue; if (option.module[m].module_id == SCE_NGS2_CUSTOM_MODULE_ID_USER_FX2) { const auto& module = *reinterpret_cast(option.module[m].option); @@ -164,6 +171,7 @@ static void SetupCustomSubmixer(Ngs2Voice& voice, const Ngs2CustomSubmixerVoiceS if (setup.flags != 0) throw std::runtime_error("NGS2: custom submixer setup flags are not implemented"); if (setup.num_input_channels != setup.num_output_channels) throw std::runtime_error("NGS2: custom submixer channel conversion is not implemented"); if (setup.num_input_channels == 0 || setup.num_input_channels > voice.rack->maxChannels) APS5_INVALID_ARG_EX; + Ngs2ResetCustomReverb(voice); voice.channels = setup.num_input_channels; } @@ -243,6 +251,15 @@ void Ngs2ApplyCustomParam(Ngs2Voice& voice, const Ngs2VoiceParamHeader& param) { Ngs2SetFilter(voice, filter); return; } + if ((param.id & ~0xffu) == SCE_NGS2_CUSTOM_VOICE_PARAM_REVERB) { + if (rackId != SCE_NGS2_RACK_ID_CUSTOM_SAMPLER && rackId != SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER) { + throw std::invalid_argument("NGS2: voice param " + Ngs2Hex(param.id) + " belongs to another rack"); + } + const auto index = param.id & 0x1f; + if (index >= voice.customReverb.size() || !voice.customReverb[index]) APS5_INVALID_ARG_EX; + Ngs2SetCustomReverbParams(voice.customReverb[index], ParamAs(param).i3dl2); + return; + } if ((param.id & ~0x1fu) == SCE_NGS2_CUSTOM_VOICE_PARAM_GAIN) { if (rackId != SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER) throw std::invalid_argument("NGS2: voice param " + Ngs2Hex(param.id) + " belongs to another rack"); SetGainParam(voice, param.id & 0x1f, ParamAs(param)); @@ -269,6 +286,10 @@ void Ngs2ProcessUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t samp std::vector channels(voice.channels); for (std::uint32_t c = 0; c < voice.channels; c++) channels[c] = voice.samples.data() + static_cast(c) * grain; for (std::size_t m = 0; m < voice.rack->customModuleIds.size(); m++) { + if (voice.rack->customModuleIds[m] == SCE_NGS2_CUSTOM_MODULE_ID_REVERB) { + voice.hasSamples = Ngs2ProcessCustomReverb(voice, m, grain, sampleRate); + continue; + } if (voice.rack->customModuleIds[m] == SCE_NGS2_CUSTOM_MODULE_ID_DELAY) continue; if (voice.rack->customModuleIds[m] == SCE_NGS2_CUSTOM_MODULE_ID_GAIN) { ProcessGain(voice, voice.customGain[m], channels, grain); diff --git a/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp b/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp index 73270afab9..6fa48a2779 100644 --- a/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp +++ b/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp @@ -69,6 +69,8 @@ struct Ngs2ReverbDeleter { void operator()(Ngs2ReverbState* reverb) const; }; +using Ngs2Reverb = std::unique_ptr; + struct Ngs2Voice; struct Ngs2FilterHistory { @@ -122,7 +124,8 @@ struct Ngs2Voice { std::uint32_t sampleRate = 0; std::uint32_t waveformType = 0; Ngs2Atrac9 atrac9; - std::unique_ptr reverb; + Ngs2Reverb reverb; + std::vector customReverb; float pitch = 1.0f; std::uint64_t phase = 0; std::deque blocks; @@ -245,7 +248,12 @@ void Ngs2ApplyCustomParam(Ngs2Voice& voice, const Ngs2VoiceParamHeader& param); void Ngs2ProcessUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate); void Ngs2SetReverbParams(Ngs2Voice& voice, const Ngs2ReverbI3DL2Param& params); void Ngs2SetupReverb(Ngs2Voice& voice); +Ngs2Reverb Ngs2CreateCustomReverb(std::uint32_t size); +void Ngs2SetCustomReverbParams(Ngs2Reverb& reverb, const Ngs2ReverbI3DL2Param& params); +bool Ngs2ProcessCustomReverb(Ngs2Voice& voice, std::size_t module, std::uint32_t grain, std::uint32_t sampleRate); void Ngs2ClearReverb(Ngs2Voice& voice); +void Ngs2ClearCustomReverb(Ngs2Voice& voice); +void Ngs2ResetCustomReverb(Ngs2Voice& voice); bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate); void Ngs2ProcessLegacyUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate); void Ngs2RenderSystem(Ngs2System& system, const Ngs2RenderBufferInfo* bufferInfo, std::uint32_t numBufferInfo); diff --git a/core/libs/prx/libSceNgs2.native/src/Reverb.cpp b/core/libs/prx/libSceNgs2.native/src/Reverb.cpp index ab3849fbdd..8cd877fd46 100644 --- a/core/libs/prx/libSceNgs2.native/src/Reverb.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Reverb.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -72,6 +73,9 @@ struct Ngs2ReverbChannel { struct Ngs2ReverbState { Ngs2ReverbI3DL2Param params{}; bool reset = true; + bool custom = false; + bool active = false; + std::uint32_t ringSamples = TAP_RING_SAMPLES; bool changed = true; std::uint32_t sampleRate = 0; std::vector ring; @@ -82,6 +86,15 @@ struct Ngs2ReverbState { float density = NAN; std::array combs; std::array channels; + std::vector input; + std::vector mapped; + std::vector mono; + std::vector early; + std::vector late; + std::vector oldEarly; + std::vector oldLate; + std::vector combSamples; + std::vector tail; bool roomFilter = false; float roomB0 = 0.0f; float roomA1 = 0.0f; @@ -190,7 +203,7 @@ static void SetupComb(Ngs2Comb& comb, std::uint32_t length, const Ngs2ReverbI3DL } static void Reset(Ngs2ReverbState& reverb) { - reverb.ring.assign(TAP_RING_SAMPLES, 0.0f); + reverb.ring.assign(reverb.ringSamples, 0.0f); reverb.ringCursor = 0; reverb.taps = {}; reverb.earlyGain = 1.0f; @@ -239,6 +252,46 @@ static void UpdateRoomFilter(Ngs2ReverbState& reverb, float rate) { reverb.roomA2 = pole * -pole; } +static void RunCustomRoomFilter(const Ngs2ReverbState& reverb, Ngs2ReverbChannel& state, float* samples, std::uint32_t grain) { + const float b = reverb.roomB0; + const float a1 = reverb.roomA1; + const float a2 = reverb.roomA2; + const float h1 = b * a1; + const float h2 = b * a2 + h1 * a1; + const float h3 = h1 * a2 + h2 * a1; + const std::array x0{b, h1, h2, h3}; + const std::array x1{0, b, h1, h2}; + const std::array x2{0, 0, b, h1}; + const std::array x3{0, 0, 0, b}; + const float feedback1 = a2 + a1 * a1; + const float feedback2 = a1 * a2 + feedback1 * a1; + const float feedback3 = feedback1 * a2 + feedback2 * a1; + const float previous1 = a2 * a1; + const float previous2 = a2 * a2 + previous1 * a1; + const float previous3 = previous1 * a2 + previous2 * a1; + const std::array y1{a1, feedback1, feedback2, feedback3}; + const std::array y2{a2, previous1, previous2, previous3}; + std::uint32_t i = 0; + for (; i + 4 <= grain; i += 4) { + const std::array input{samples[i], samples[i + 1], samples[i + 2], samples[i + 3]}; + for (std::uint32_t lane = 0; lane < 4; lane++) { + float odd = std::fma(x1[lane], input[1], x3[lane] * input[3]); + odd = std::fma(y1[lane], state.y1, odd); + float even = std::fma(x0[lane], input[0], x2[lane] * input[2]); + even = std::fma(y2[lane], state.y2, even); + samples[i + lane] = even + odd; + } + state.y2 = samples[i + 2]; + state.y1 = samples[i + 3]; + } + for (; i < grain; i++) { + const float y = std::fma(a2, state.y2, std::fma(a1, state.y1, b * samples[i])); + state.y2 = state.y1; + state.y1 = y; + samples[i] = y; + } +} + static void RunComb(Ngs2Comb& comb, const float* input, float* output, std::uint32_t grain) { const auto length = static_cast(comb.buffer.size()); for (std::uint32_t i = 0; i < grain; i++) { @@ -264,7 +317,20 @@ static void RunAllpass(Ngs2Allpass& allpass, float gain, float* samples, std::ui } } -static void MixDown(const float* input, std::uint32_t channels, std::uint32_t grain, float* mono) { +static void MixDown(const float* input, std::uint32_t channels, std::uint32_t grain, float* mono, const Ngs2ReverbState& reverb) { + if (reverb.custom) { + const auto mask = reverb.params.reserved[0]; + const auto count = std::popcount(((1u << channels) - 1) & ~mask); + const float gain = count == 0 ? 0.0f : (count <= 2 ? 1.0f : 0.707106769f) / std::sqrt(static_cast(count)); + bool first = true; + for (std::uint32_t c = 0; c < channels; c++) { + if ((mask & (1u << c)) != 0) continue; + const float* channel = input + static_cast(c) * grain; + for (std::uint32_t i = 0; i < grain; i++) mono[i] = first ? channel[i] * gain : mono[i] + channel[i] * gain; + first = false; + } + return; + } const float* channel[8]; for (std::uint32_t c = 0; c < channels; c++) channel[c] = input + static_cast(c) * grain; if (channels == 1) { @@ -279,8 +345,7 @@ static void MixDown(const float* input, std::uint32_t channels, std::uint32_t gr } } -bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate) { - auto& reverb = *voice.reverb; +static bool ProcessReverb(Ngs2Voice& voice, Ngs2ReverbState& reverb, std::uint32_t grain, std::uint32_t sampleRate) { const float rate = static_cast(sampleRate); const std::uint32_t channels = voice.channels; if (reverb.sampleRate != sampleRate) { @@ -293,10 +358,16 @@ bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t samp reverb.changed = true; } const std::uint32_t inputChannels = voice.InputChannels(); - const std::vector input = voice.samples; - std::vector mono(grain), early(grain), late(grain); - MixDown(input.data(), inputChannels, grain, mono.data()); - std::vector mapped; + reverb.input = voice.samples; + const auto& input = reverb.input; + auto& mono = reverb.mono; + auto& early = reverb.early; + auto& late = reverb.late; + mono.assign(grain, 0.0f); + early.resize(grain); + late.resize(grain); + MixDown(input.data(), inputChannels, grain, mono.data(), reverb); + auto& mapped = reverb.mapped; if (inputChannels != channels) { mapped.assign(static_cast(grain) * channels, 0.0f); for (std::uint32_t dst = 0; dst < channels; dst++) { @@ -310,7 +381,10 @@ bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t samp } const std::vector& direct = inputChannels != channels ? mapped : input; if (reverb.changed) { - std::vector oldEarly(grain), oldLate(grain); + auto& oldEarly = reverb.oldEarly; + auto& oldLate = reverb.oldLate; + oldEarly.resize(grain); + oldLate.resize(grain); RunTaps(reverb, reverb.taps, mono.data(), grain, oldEarly.data(), oldLate.data()); ApplyParams(reverb, rate); RunTaps(reverb, reverb.taps, mono.data(), grain, early.data(), late.data()); @@ -325,7 +399,8 @@ bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t samp reverb.ring[reverb.ringCursor] = mono[i]; reverb.ringCursor = reverb.ringCursor + 1 == reverb.ring.size() ? 0 : reverb.ringCursor + 1; } - std::vector combs(static_cast(COMBS) * grain); + auto& combs = reverb.combSamples; + combs.resize(static_cast(COMBS) * grain); for (std::uint32_t i = 0; i < COMBS; i++) RunComb(reverb.combs[i], late.data(), combs.data() + static_cast(i) * grain, grain); UpdateRoomFilter(reverb, rate); const auto& params = reverb.params; @@ -334,10 +409,12 @@ bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t samp auto& state = reverb.channels[channel]; float* out = voice.samples.data() + static_cast(channel) * grain; const float* in = direct.data() + static_cast(channel) * grain; - if (channel == LFE_CHANNEL) { + const bool excluded = reverb.custom ? (params.reserved[0] & (1u << channel)) != 0 : channel == LFE_CHANNEL; + if (excluded) { std::copy(in, in + grain, out); } else { - std::vector tail(grain, 0.0f); + auto& tail = reverb.tail; + tail.assign(grain, 0.0f); std::uint32_t selected = 0; for (std::uint32_t i = 0; i < COMBS; i++) { if ((CHANNEL_COMBS[channel] >> i & 1) == 0) continue; @@ -348,14 +425,18 @@ bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t samp } for (auto& allpass : state.allpasses) RunAllpass(allpass, params.diffusion * 0.005f, tail.data(), grain); for (std::uint32_t n = 0; n < grain; n++) out[n] = early[n] * reverb.earlyGain + tail[n] * reverb.lateGain; - for (std::uint32_t n = 0; n < grain; n++) { - const float y = reverb.roomFilter ? reverb.roomB0 * out[n] + reverb.roomA1 * state.y1 + reverb.roomA2 * state.y2 : out[n]; - state.y2 = state.y1; - state.y1 = y; - out[n] = y; + if (reverb.custom && reverb.roomFilter) { + RunCustomRoomFilter(reverb, state, out, grain); + } else { + for (std::uint32_t n = 0; n < grain; n++) { + const float y = reverb.roomFilter ? reverb.roomB0 * out[n] + reverb.roomA1 * state.y1 + reverb.roomA2 * state.y2 : out[n]; + state.y2 = state.y1; + state.y1 = y; + out[n] = y; + } } } - const float wet = channel == LFE_CHANNEL ? 0.0f : params.wet; + const float wet = excluded ? 0.0f : params.wet; Ramp(out, reset ? wet : state.wet, wet, grain, out, false); Ramp(out, reset ? params.dry : state.dry, params.dry, grain, in, true); state.wet = wet; @@ -367,16 +448,16 @@ bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t samp return audible; } -void Ngs2SetReverbParams(Ngs2Voice& voice, const Ngs2ReverbI3DL2Param& params) { +static void SetReverbParams(Ngs2Reverb& reverb, const Ngs2ReverbI3DL2Param& params) { if (std::isnan(params.wet) || std::isnan(params.dry) || std::isnan(params.decay_time) || std::isnan(params.decay_hf_ratio) || std::isnan(params.reflections_delay) || std::isnan(params.reverb_delay) || std::isnan(params.diffusion) || std::isnan(params.density) || std::isnan(params.hf_reference)) { APS5_INVALID_ARG_EX; } - if (!voice.reverb) voice.reverb.reset(new Ngs2ReverbState{}); - auto& stored = voice.reverb->params; + if (!reverb) reverb.reset(new Ngs2ReverbState{}); + auto& stored = reverb->params; stored = params; - stored.wet = Clamp(params.wet, 0.0f, 32.0f); - stored.dry = Clamp(params.dry, 0.0f, 32.0f); + stored.wet = reverb->custom ? params.wet : Clamp(params.wet, 0.0f, 32.0f); + stored.dry = reverb->custom ? params.dry : Clamp(params.dry, 0.0f, 32.0f); stored.reflection_pattern = params.reflection_pattern < 16 ? params.reflection_pattern : 0; stored.decay_time = Clamp(params.decay_time, 0.1f, 20.0f); stored.decay_hf_ratio = Clamp(params.decay_hf_ratio, 0.00999999978f, 2.0f); @@ -385,7 +466,32 @@ void Ngs2SetReverbParams(Ngs2Voice& voice, const Ngs2ReverbI3DL2Param& params) { stored.diffusion = Clamp(params.diffusion, 0.0f, 100.0f); stored.density = Clamp(params.density, 0.0f, 100.0f); stored.hf_reference = Clamp(params.hf_reference, 20.0f, 20000.0f); - voice.reverb->changed = true; + reverb->changed = true; +} + +bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate) { + return ProcessReverb(voice, *voice.reverb, grain, sampleRate); +} + +void Ngs2SetReverbParams(Ngs2Voice& voice, const Ngs2ReverbI3DL2Param& params) { + SetReverbParams(voice.reverb, params); +} + +Ngs2Reverb Ngs2CreateCustomReverb(std::uint32_t size) { + Ngs2Reverb reverb(new Ngs2ReverbState{}); + reverb->custom = true; + reverb->ringSamples = size == 0 ? 4800 : 14400; + return reverb; +} + +void Ngs2SetCustomReverbParams(Ngs2Reverb& reverb, const Ngs2ReverbI3DL2Param& params) { + SetReverbParams(reverb, params); + reverb->active = true; +} + +bool Ngs2ProcessCustomReverb(Ngs2Voice& voice, std::size_t module, std::uint32_t grain, std::uint32_t sampleRate) { + auto& reverb = *voice.customReverb[module]; + return reverb.active ? ProcessReverb(voice, reverb, grain, sampleRate) : voice.hasSamples; } void Ngs2SetupReverb(Ngs2Voice& voice) { @@ -393,6 +499,22 @@ void Ngs2SetupReverb(Ngs2Voice& voice) { Ngs2SetReverbParams(voice, SETUP_PARAMS); } +void Ngs2ClearCustomReverb(Ngs2Voice& voice) { + for (auto& reverb : voice.customReverb) { + if (reverb) reverb->reset = true; + } +} + +void Ngs2ResetCustomReverb(Ngs2Voice& voice) { + for (auto& reverb : voice.customReverb) { + if (reverb) { + reverb->active = false; + reverb->reset = true; + } + } +} + void Ngs2ClearReverb(Ngs2Voice& voice) { if (voice.reverb) voice.reverb->reset = true; + Ngs2ClearCustomReverb(voice); } diff --git a/core/libs/prx/libSceNgs2.native/src/Voice.cpp b/core/libs/prx/libSceNgs2.native/src/Voice.cpp index 5ff13687d9..c572ef473b 100644 --- a/core/libs/prx/libSceNgs2.native/src/Voice.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Voice.cpp @@ -24,6 +24,7 @@ void Ngs2Voice::SetEvent(std::uint32_t eventId) { switch (eventId) { case SCE_NGS2_VOICE_EVENT_PLAY: if (state == Ngs2PlayState::Empty || state == Ngs2PlayState::Stopped) { + Ngs2ClearCustomReverb(*this); state = Ngs2PlayState::Playing; stateFlags |= SCE_NGS2_VOICE_STATE_FLAG_INUSE; if (userFxHandler) userFxFlags = 1; @@ -50,6 +51,7 @@ void Ngs2Voice::SetEvent(std::uint32_t eventId) { void Ngs2Voice::ResetSetup() { ++waveformRevision; SetEvent(SCE_NGS2_VOICE_EVENT_STOP_IMM); + Ngs2ResetCustomReverb(*this); std::fill(ports.begin(), ports.end(), Ngs2Port{}); for (auto& matrix : matrices) matrix.clear(); filters.clear(); diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2Custom.cpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2Custom.cpp index 80d9d2cb34..76d81d71ae 100644 --- a/core/libs/prx/libSceNgs2.native/tests/Ngs2Custom.cpp +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2Custom.cpp @@ -461,8 +461,8 @@ static void TestCustomRackChecks() { const ModuleOptionWords reverbOff{SCE_NGS2_CUSTOM_REVERB_OPTION_SIZE, {0}}; const ModuleOptionWords reverbSize{16, {1}}; const ModuleOptionWords reverbWord{SCE_NGS2_CUSTOM_REVERB_OPTION_SIZE, {2}}; - Require(CreateThrows(BuiltInRackOption(userFx2, SCE_NGS2_CUSTOM_MODULE_ID_REVERB, &reverb, 0))); - Require(CreateThrows(BuiltInRackOption(userFx2, SCE_NGS2_CUSTOM_MODULE_ID_REVERB, &reverbOff, 0))); + Require(!CreateThrows(BuiltInRackOption(userFx2, SCE_NGS2_CUSTOM_MODULE_ID_REVERB, &reverb, 0))); + Require(!CreateThrows(BuiltInRackOption(userFx2, SCE_NGS2_CUSTOM_MODULE_ID_REVERB, &reverbOff, 0))); Require(create(BuiltInRackOption(userFx2, SCE_NGS2_CUSTOM_MODULE_ID_REVERB, &reverbSize, 0)) == SCE_NGS2_ERROR_INVALID_OPTION_SIZE); Require(create(BuiltInRackOption(userFx2, SCE_NGS2_CUSTOM_MODULE_ID_REVERB, &reverbWord, 0)) == SCE_NGS2_ERROR_INVALID_REVERB_OPTION); Require(create(BuiltInRackOption(userFx2, SCE_NGS2_CUSTOM_MODULE_ID_REVERB, &reverb, 4)) == SCE_NGS2_ERROR_INVALID_STATE_SIZE); diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2CustomReverb.cpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2CustomReverb.cpp new file mode 100644 index 0000000000..c8b0bd2d5d --- /dev/null +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2CustomReverb.cpp @@ -0,0 +1,170 @@ +#include "Ngs2Test.hpp" + +#include +#include +#include + +static constexpr std::uint32_t RenderGrain = 256; + +static int APS5_VABI Impulse(Ngs2UserFxProcessContext* context) { + auto& calls = *reinterpret_cast(context->user_data0); + Require(context->num_channels == 2 && context->num_grain_samples == RenderGrain); + for (std::uint32_t channel = 0; channel < 2; channel++) { + std::fill_n(context->channel_data[channel], RenderGrain, 0.0f); + if (calls == 0) context->channel_data[channel][3] = channel == 0 ? .5f : .25f; + } + calls++; + return SCE_NGS2_OK; +} + +static void TestReverb(std::uint32_t size, bool strongHf = false) { + const auto firstBuffer = usedBuffers; + const auto system = CreateSystem(); + Require(sceNgs2SystemSetGrainSamples(system, RenderGrain) == SCE_NGS2_OK); + const auto master = Mastering(system, 2); + const std::uint32_t moduleOption[]{12, size, 0}; + Ngs2CustomSubmixerRackOption option{}; + auto& common = option.custom_rack_option.rack_option; + common.size = sizeof(option); + common.max_grain_samples = 512; + common.max_voices = 2; + common.max_matrices = 1; + common.max_ports = 1; + option.custom_rack_option.state_size = 16; + option.custom_rack_option.num_buffers = 1; + option.custom_rack_option.num_modules = 2; + option.max_channels = 2; + option.max_inputs = 1; + for (auto index : {0u, 1u}) { + auto& module = option.custom_rack_option.module[index]; + module.module_id = SCE_NGS2_CUSTOM_MODULE_ID_REVERB; + module.option = reinterpret_cast(moduleOption); + } + Ngs2ContextBufferInfo query{}; + Require(sceNgs2RackQueryBufferSize(SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER, &common, &query) == SCE_NGS2_OK); + const auto storage = Buffer(query); + uintptr_t rack = 0; + Require(sceNgs2RackCreate(system, SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER, &common, &storage, &rack) == SCE_NGS2_OK); + std::array voices{}; + for (auto index : {0u, 1u}) { + Require(sceNgs2RackGetVoiceHandle(rack, index, &voices[index]) == SCE_NGS2_OK); + Control(voices[index], SCE_NGS2_CUSTOM_SUBMIXER_VOICE_PARAM_SETUP, Ngs2CustomSubmixerVoiceSetupParam{{}, 2, 2, 0, 0}); + Patch(voices[index], master); + Event(voices[index], SCE_NGS2_VOICE_EVENT_PLAY); + } + Ngs2ReverbI3DL2Param cave{1, 0, -1000, 0, 0, 2.91f, 1.3f, -602, .25f, -302, .022f, 100, 100, 5000, {}}; + if (strongHf) { + cave.reflections_delay = .015f; + cave.room_hf = -12000; + } + const auto source = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SUBMIXER)); + Control(source, SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP, Ngs2SubmixerVoiceSetupParam{{}, 2, 0}); + std::uint32_t calls = 0; + Control(source, SCE_NGS2_SUBMIXER_VOICE_PARAM_USER_FX, + Ngs2SubmixerVoiceUserFxParam{{}, Impulse, reinterpret_cast(&calls), 0, 0}); + Patch(source, voices[0]); + Event(source, SCE_NGS2_VOICE_EVENT_PLAY); + std::vector out(RenderGrain * 2); + const Ngs2RenderBufferInfo output{out.data(), out.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, 2}; + Require(sceNgs2SystemRender(system, &output, 1) == SCE_NGS2_OK); + for (std::uint32_t i = 0; i < out.size(); i++) Require(out[i] == (i == 6 ? .5f : i == 7 ? .25f : 0.0f)); + calls = 0; + Control(voices[0], SCE_NGS2_CUSTOM_VOICE_PARAM_REVERB | 0x20, Ngs2ReverbVoiceI3DL2Param{{}, cave}); + auto dry = cave; + dry.wet = 0; + dry.dry = 1; + Control(voices[0], SCE_NGS2_CUSTOM_VOICE_PARAM_REVERB | 1, Ngs2ReverbVoiceI3DL2Param{{}, dry}); + for (std::uint32_t grain = 0; grain < 80; grain++) { + Require(sceNgs2SystemRender(system, &output, 1) == SCE_NGS2_OK); + for (const auto frame : {6000u, 8192u, 12000u, 16000u}) { + if (frame / RenderGrain != grain) continue; + const float expected[3][4][2]{ + {{5.78283021e-9f, 2.58373547e-7f}, {-8.08179466e-5f, 8.2077866e-5f}, {-7.49083818e-4f, -3.76510725e-4f}, {3.18058097e-4f, 6.42870436e-4f}}, + {{0, 0}, {0, 0}, {0, 0}, {8.11674454e-6f, -8.95900448e-5f}}, + {{-1.07959904e-5f, -1.03797774e-5f}, {2.38732318e-5f, 2.75960247e-5f}, {3.50958362e-5f, 3.04597361e-5f}, {2.78984935e-5f, 2.99852363e-5f}}, + }; + const auto sample = frame % RenderGrain; + const auto check = frame == 6000 ? 0 : frame == 8192 ? 1 : frame == 12000 ? 2 : 3; + Require(std::abs(out[sample * 2] - expected[strongHf ? 2 : size][check][0]) < 1e-7f); + Require(std::abs(out[sample * 2 + 1] - expected[strongHf ? 2 : size][check][1]) < 1e-7f); + } + } + Control(voices[0], SCE_NGS2_CUSTOM_SUBMIXER_VOICE_PARAM_SETUP, Ngs2CustomSubmixerVoiceSetupParam{{}, 2, 2, 0, 0}); + Patch(voices[0], master); + calls = 0; + Event(voices[0], SCE_NGS2_VOICE_EVENT_PLAY); + Require(sceNgs2SystemRender(system, &output, 1) == SCE_NGS2_OK); + for (std::uint32_t i = 0; i < out.size(); i++) Require(out[i] == (i == 6 ? .5f : i == 7 ? .25f : 0.0f)); + Event(voices[0], SCE_NGS2_VOICE_EVENT_STOP_IMM); + Event(voices[0], SCE_NGS2_VOICE_EVENT_PLAY); + Require(sceNgs2SystemRender(system, &output, 1) == SCE_NGS2_OK); + for (float sample : out) Require(sample == 0); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); + usedBuffers = firstBuffer; +} + +static void TestSamplerRestart() { + const auto firstBuffer = usedBuffers; + const auto system = CreateSystem(); + Require(sceNgs2SystemSetGrainSamples(system, RenderGrain) == SCE_NGS2_OK); + const auto master = Mastering(system, 2); + Ngs2CustomSamplerRackOption option{}; + auto& common = option.custom_rack_option.rack_option; + common.size = sizeof(option); + common.max_grain_samples = 512; + common.max_voices = 1; + common.max_matrices = 1; + common.max_ports = 1; + option.custom_rack_option.state_size = 48; + option.custom_rack_option.num_buffers = 1; + option.custom_rack_option.num_modules = 1; + option.custom_rack_option.module[0].module_id = SCE_NGS2_CUSTOM_MODULE_ID_REVERB; + option.max_channel_works = 2; + option.max_waveform_blocks = 4; + Ngs2ContextBufferInfo query{}; + Require(sceNgs2RackQueryBufferSize(SCE_NGS2_RACK_ID_CUSTOM_SAMPLER, &common, &query) == SCE_NGS2_OK); + const auto storage = Buffer(query); + uintptr_t rack = 0; + Require(sceNgs2RackCreate(system, SCE_NGS2_RACK_ID_CUSTOM_SAMPLER, &common, &storage, &rack) == SCE_NGS2_OK); + const auto voice = Voice(rack); + struct SamplerSetup { + Ngs2VoiceParamHeader header; + Ngs2WaveformFormat format; + std::uint32_t reserved[2]; + }; + Control(voice, SCE_NGS2_CUSTOM_SAMPLER_VOICE_PARAM_SETUP, + SamplerSetup{{}, {SCE_NGS2_WAVEFORM_TYPE_PCM_I16L, 2, 48000, 0, 0, 0}, {}}); + const Ngs2ReverbI3DL2Param cave{1, 0, -1000, 0, 0, 2.91f, 1.3f, -602, .015f, -302, .022f, 100, 100, 5000, {}}; + Control(voice, SCE_NGS2_CUSTOM_VOICE_PARAM_REVERB, Ngs2ReverbVoiceI3DL2Param{{}, cave}); + Patch(voice, master); + std::vector pcm(RenderGrain * 2); + pcm[6] = 16384; + pcm[7] = 8192; + Ngs2WaveformBlock block{0, pcm.size() * sizeof(std::int16_t), 0, 0, RenderGrain, 0, 0}; + Control(voice, SCE_NGS2_CUSTOM_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, pcm.data(), 0, 1, &block}); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + std::vector out(RenderGrain * 2); + const Ngs2RenderBufferInfo output{out.data(), out.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, 2}; + for (std::uint32_t i = 0; i < 8; i++) Require(sceNgs2SystemRender(system, &output, 1) == SCE_NGS2_OK); + Require(Flags(voice) == 0); + std::vector silence(RenderGrain * 2 * 20); + block.data_size = silence.size() * sizeof(std::int16_t); + block.num_samples = silence.size() / 2; + Control(voice, SCE_NGS2_CUSTOM_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, + Ngs2SamplerVoiceWaveformBlocksParam{{}, silence.data(), SCE_NGS2_WAVEFORM_BLOCKS_FLAG_RESET, 1, &block}); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + for (std::uint32_t i = 0; i < 20; i++) { + Require(sceNgs2SystemRender(system, &output, 1) == SCE_NGS2_OK); + for (float sample : out) Require(sample == 0); + } + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); + usedBuffers = firstBuffer; +} + +int main() { + TestReverb(0); + TestReverb(1); + TestReverb(1, true); + TestSamplerRestart(); + return 0; +} diff --git a/docs/dev/debt/libSceNgs2.native.md b/docs/dev/debt/libSceNgs2.native.md index 5997a09d48..a41bc3a018 100644 --- a/docs/dev/debt/libSceNgs2.native.md +++ b/docs/dev/debt/libSceNgs2.native.md @@ -26,10 +26,12 @@ - [NGS2 custom submixer rack](../../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (libSceNgs2.native, rack 0x4002) - UserFx2 process flags and setup/process/cleanup order follow only [KytyPS5](https://github.com/KytyPS5/KytyPS5), not a console measurement; layouts follow [shadPS4](https://github.com/shadps4-emu/shadPS4)'s `ngs2_custom.h`. A module control handler, unequal input and output channels, setup flags and default rack options throw. Module state is a separate buffer, not the `state_offset` slice of voice state; `sceNgs2VoiceGetState` throws for this rack. UserFx2 runs only on grains with input samples, so an effect tail stops with the input - [NGS2 custom UserFx](../../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (libSceNgs2.native) - the 48-byte process context, process flags, callback error handling and null-handler behavior retain the existing submixer callback model and are not console-verified. Non-null module options throw because their console error is unknown. UserFx on custom sampler rack 0x4001 throws: its parameter indexing and routing are not implemented. Module id 0x19, the 40-byte voice parameter (handler and three user-data words), parameter id `0x40001900 | index`, and lack of option and module state follow [fma-rc's PS5 12.70 module reading](https://github.com/boykopovar/AnyPS5/issues/8039#issuecomment-6319332943), including parameter dispatch at 0x25ec4. - [NGS2 custom Delay](../../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (libSceNgs2.native, 0x1d) - processing and defaults remain unverified on PS5. The [agreed compatibility path](https://github.com/boykopovar/AnyPS5/pull/9337#issuecomment-6351205140) preserves main's PCM pass-through for Void Terrarium (PPSA03061) until any Delay parameter is set; every `0x40001dxx` parameter throws. Only one-buffer custom submixers with no module state and a 20-byte option are accepted. Option type 0/1, taps 1..8, length 1..20000 and reserved 0 follow [fma-rc's 12.70 module reading](https://github.com/boykopovar/AnyPS5/issues/8039#issuecomment-6319332943). Other shapes and values throw because their console errors are unknown. This preserves host behavior, not console-equivalent Delay processing -- [NGS2 custom module ids](../../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (libSceNgs2.native) - Reverb (0x1b), PitchShift (0x20) and Convolution (0x23) are not processed: a rack that passes their creation checks throws. Gain (0x15) on custom sampler racks also throws. Per [fma-rc's 12.70 module reading](https://github.com/boykopovar/AnyPS5/issues/8039#issuecomment-6294323150), creation returns `0x804A8302` for a wrong non-zero state size, and `0x804A8013`, `0x804A8099` and `0x804A8098` for a bad Reverb option size or word and PitchShift word. Unknown: check order, errors for a Gain option, missing Reverb or PitchShift options, other PitchShift and Convolution option sizes, and Convolution's state (64 bytes from SDK 3.0). PPSA04540 uses 0x15, 0x19, 0x1b and 0x23 (#8039), PPSA35169 0x20 (#4246) and PPSA33655 0x15, 0x19 and 0x1b +- [NGS2 custom module ids](../../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (libSceNgs2.native) - PitchShift (0x20) and Convolution (0x23) are not processed: a rack that passes their creation checks throws. Gain (0x15) on custom sampler racks also throws. Per [fma-rc's 12.70 module reading](https://github.com/boykopovar/AnyPS5/issues/8039#issuecomment-6294323150), creation returns `0x804A8302` for a wrong non-zero state size, and `0x804A8013`, `0x804A8099` and `0x804A8098` for a bad Reverb option size or word and PitchShift word. Unknown: check order, errors for a Gain option, missing PitchShift options, other PitchShift and Convolution option sizes, and Convolution's state (64 bytes from SDK 3.0). PPSA04540 uses 0x15, 0x19, 0x1b and 0x23 (#8039), PPSA35169 0x20 (#4246) and PPSA33655 0x15, 0x19 and 0x1b - [NGS2 custom Gain module](../../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (libSceNgs2.native, rack 0x4002) - follows [fma-rc's reading of the 12.70 module](https://github.com/boykopovar/AnyPS5/issues/8039#issuecomment-6319332943): parameter `0x40001500 | module index` holds 37 linear levels, one per channel, default 0, and the module writes source × level into dest, ramping linearly over the grain from the previous level. Unverified: where the ramp starts and ends (here the grain's first sample is one step past the previous level and its last sample is at the new level), whether stopping or setting up the voice again resets the levels, and what the library does with a level outside -32 to 32 (here it throws). On a custom sampler rack, whose parameter index is one higher per fma-rc, Gain throws. A Gain module with an option throws: fma-rc reads Gain as taking none, and the error the library returns for one is unknown. The extra buffer is ignored, as fma-rc reads it - [NGS2 custom mastering rack](../../../core/libs/prx/libSceNgs2.native/src/Rack.cpp) (libSceNgs2.native, rack 0x4003) - follows [fma-rc's reading of the 12.70 module](https://github.com/boykopovar/AnyPS5/issues/8039#issuecomment-6319332943): option size is 0x508, matching the custom submixer rack layout. Creation returns `0x804A8052` above 37 channels; channels 9 to 37 throw because voices hold at most 8 channels. Mastering racks with modules throw: attached modules and non-default option fields are unverified on PS5. Voice control parameters 0x40030000 (setup), 0x40030004 (gain) and 0x40030005 (output) configure channels, levels and destination buffer. `sceNgs2VoiceGetState` fills common voice state flags. PPSA10278 and PPSA04540 create rack 0x4003 without modules (#8039) - [NGS2 custom rack buffers and channels](../../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (libSceNgs2.native, racks 0x4001 and 0x4002) - racks with 2 to 4 buffers, custom submixers with 9 to 37 channels and submixer `max_inputs` other than 1 or 2 throw: how modules and ports route several buffers is unknown, and voices hold at most 8 channels. Per [fma-rc's reading of the 12.70 module](https://github.com/boykopovar/AnyPS5/issues/8039#issuecomment-6294323150), creation returns `0x804A830B` for a buffer count outside 1 to 4 or below the submixer's `max_inputs`, `0x804A8306` for a module source, extra or dest buffer id at or above the count, and `0x804A8052` for more than 37 submixer channels; the check order is not known. `sceNgs2RackQueryBufferSize` throws for these options, as no source says whether it checks them. A port buffer id other than 0 throws. PPSA04540 creates racks with up to 4 buffers and a 16-channel custom submixer (#8039) +- [NGS2 custom Reverb](../../../core/libs/prx/libSceNgs2.native/src/Reverb.cpp) (libSceNgs2.native, module 0x1b) - NaN controls, low-rate/half-rate processing, multiple custom buffers and custom mastering modules remain unsupported. Numerical equivalence is not bit-exact: 3.20 and 13.60 DSP comparisons at 48 kHz cover both creation sizes and a hot parameter update; ordinary stereo differs by less than 2e-8 and strong HF attenuation by less than 8e-8 in those cases. The original 13.60 voice graph confirms initial/setup bypass, parameter retention with history cleared on fresh PLAY, and sampler completion ending the voice. These checks use host adapters and an equivalent SSE4a extraction lowering for Rosetta; they establish neither console execution nor a game unblock. + - [NGS2 reverb rack](../../../core/libs/prx/libSceNgs2.native/src/Reverb.cpp) (libSceNgs2.native, rack 0x2001) - follows the PS4 firmware 11.00 `libSceNgs2.sprx`, read and checked against its output at 11.025 to 48 kHz, including parameter changes during playback (differences at float rounding level); not yet compared with the PS5 library. Structure layouts come from [shadPS4](https://github.com/shadps4-emu/shadPS4)'s `ngs2_reverb.h` and the param ids 0x20010000 (setup) and 0x20010001 (I3DL2) from PPSA22520. Only the default `reverb_size` of 1 is handled, with its 14400-sample reflection buffer; other sizes and the library's half-rate mode throw. System sample rates below about 7.5 kHz, where an allpass delay would have no samples, throw. Setup only accepts equal input and output channel counts - [sceNgs2ReportRegisterHandler, sceNgs2ReportUnregisterHandler](../../../core/libs/prx/libSceNgs2.native/src/System.cpp) (libSceNgs2.native, NIDs `uBIN24Tv2MI`, `nPzb7Ly-VjE`) keep a registry of report handlers. Register rejects a null handler with an invalid argument and returns `INVALID_OUT_ADDRESS` for a null out handle, then stores the report type, handler and user data under a freshly allocated opaque handle; unregister removes and frees the matching record and throws for a handle that is not registered. The signatures (report type, handler, user data, out handle) and (report handle) follow [SharpProspero](https://github.com/SvenGDK/SharpProspero) and [shadPS4](https://github.com/shadps4-emu/shadPS4)'s `ngs2_report.h`, and the 24-byte `Ngs2ReportDataHeader` passed to a handler is its `OrbisNgs2ReportDataHeader`. No report is produced yet, so no registered handler is ever invoked and PPSA04540, which imports both and registers a CPU load handler (type 5), still receives none. - [sceNgs2SystemSetUserData, sceNgs2SystemGetUserData, sceNgs2SystemLock, sceNgs2SystemUnlock](../../../core/libs/prx/libSceNgs2.native/src/System.cpp) (libSceNgs2.native) - shadPS4 only checks the system handle (`INVALID_SYSTEM_HANDLE`) and SharpProspero names what each call does; neither gives the other limits. A null user data pointer throws. `sceNgs2SystemLock` and `sceNgs2SystemUnlock` follow [prosper's notes](https://github.com/mattias800/prosper/blob/d61eb2c0ad2de879ef74df9c4aa050c512804a18/prosper/src/hle/audio/hle_audio.cpp#L4367-L4384) on the PS5 module: Lock takes a per-system lock and counts nested holds, Unlock releases one hold from any thread and returns 0 without a hold, and `sceNgs2SystemRender` only tries the lock: while another thread holds it, Render clears one grain of each output, leaves the render count and voices unchanged and returns 0. The other calls do not wait for the system lock. Peace Walker (PPSA18216) holds it while it spins on a flag that its audio thread clears only after `sceNgs2SystemRender` and eight `sceNgs2VoiceGetStateFlags` calls, so on the console neither can wait for it. Whether the console's voice and rack calls take the lock at all is not known. `sceNgs2RackLock` still holds the library's lock until `sceNgs2RackUnlock`, as before From 3a425fa1ee2af22977e876d6af2a33113a7dc776 Mon Sep 17 00:00:00 2001 From: Kyon Date: Sat, 24 Oct 2026 16:06:38 +0200 Subject: [PATCH 106/318] fix(libSceJson2): avoid repeated metadata lookup exceptions --- core/libs/prx/libSceJson2/Export.cpp | 4 ++++ docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 0296558fba..9086b7be9e 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -397,11 +398,14 @@ std::string_view View(const String& string) { } std::uint32_t CompiledSdk() { + static std::atomic lookupFailed{false}; std::uint32_t sdk = std::numeric_limits::max(); + if (lookupFailed.load(std::memory_order_relaxed)) return sdk; try { const auto* process = static_cast(ApplicationProcessParameters_nid_no_patch()); if (process) std::memcpy(&sdk, process + 0x10, sizeof(sdk)); } catch (const std::runtime_error&) { + lookupFailed.store(true, std::memory_order_relaxed); } return sdk; } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index abc6149542..a94736622e 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -16,7 +16,7 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [sce::Json::Value element-access failure dispatch](../../../core/libs/prx/libSceJson2/Export.cpp) - SDK-dependent dispatch remains incomplete. Global registration and fallback selection follow `0x1670` and `0x2c70`. Root-owned fallback identity, reentry suppression and errors `0x80848115` through `0x80848118` follow `0x2c70`, `0x4e80` and `0x5070`. Nested fallback payload failures and compatibility below SDK 6 remain incomplete. - [sce::Json::Value special-float recovery](../../../core/libs/prx/libSceJson2/Export.cpp) - allocator-backed value failures remain incomplete. Global registration follows `0x14f0`, including legacy-initializer rejection and duplicate registration; root handlers precede global handlers. Root recovery follows `0x2b90`, `0x2d90` and `0x41a0`, including the hidden result object, temporary cloning and teardown, function types 0/1, optional error flag and reentry suppression. SDK-dependent registration below SDK 6, signaling NaNs and floating-point exception flags remain unverified. - [sce::Json::MemAllocator::notifyError](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the base class's virtual does nothing; whether the console's base implementation logs is unknown. String, array, object, callback metadata and file-buffer allocation failures dispatch the supplied allocator's notifier. -- [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Appending null text with a nonzero length throws instead of dereferencing it. +- [sce::Json::String size, capacity, resize, append(const char*, size_t), operator+=](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - headers and characters use the supplied allocator, with the inspected 24-byte header and 32-bit capacity. Constructors, clear, reserve, resize, copy, assignment and append failure contracts follow `0xa100`–`0xb0a0`, `0xde50` and the instructions at `0xdf20`. Overlapping assignment, exact substring temporary allocations, allocator failure propagation through every value/parser operation and native assertion behavior for sizes above 32 bits remain incomplete. SDK-dependent growth reads process parameters; missing metadata uses the reference query-failure rule. Failed host metadata lookup is remembered to avoid repeated allocating exceptions; successful SDK queries remain live. Appending null text with a nonzero length throws instead of dereferencing it. - [sce::Json::Array::iterator::advance and Object::iterator members](../../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - the source-level return type of `advance` remains unverified; array advancement is declared `void` and tested for its side effects. Object iterator advancement past the end throws instead of asserting. Array iterators expose the inspected node payload address, including the sentinel; missing-state Array dereference/arrow share the initialized 32-byte allocator-owned fallback with keyed lookup. Iterator access allocates nothing and preserves its bytes under allocation failure. - [sce::Json::Value::operator bool](../../../core/libs/prx/libSceJson2/Export.cpp) - untested legacy callback combinations remain incomplete. Modern root-local mismatch precedes global mismatch; ordinary boolean values bypass both. Root-local mismatch dispatch precedes ordinary conversion, following `0x2100`; the returned value is read through its boolean getter under the shared callback guard. From fb6d1df782c6426e60c678f59757bc4fcb44a7e5 Mon Sep 17 00:00:00 2001 From: Kyon Date: Sat, 24 Oct 2026 17:16:28 +0200 Subject: [PATCH 107/318] test(libSceJson2): cover parser entry allocation cleanup --- core/libs/tests/GuestJsonParserState.cpp | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/core/libs/tests/GuestJsonParserState.cpp b/core/libs/tests/GuestJsonParserState.cpp index 0dc8040951..c3b12b4055 100644 --- a/core/libs/tests/GuestJsonParserState.cpp +++ b/core/libs/tests/GuestJsonParserState.cpp @@ -19,6 +19,7 @@ int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); void APS5_VABI _ZN3sce4Json5ValueC1El(Value*, std::int64_t); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(Value*, int (APS5_VABI*)(char&, void*), void*); const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value*); std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); @@ -57,6 +58,44 @@ static void APS5_VABI NotifyTracked(void*, int error, std::size_t, void* context static int Parse(Value& value, const char* text) { return _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, text, std::strlen(text)); } +static int APS5_VABI ReadNext(char& character, void* context) { + auto& cursor = *static_cast(context); + if (!*cursor) return -1; + character = *cursor++; + return 0; +} +static void CheckArrayEntryFailures(ParserTrace& live, std::size_t baseline) { + for (bool callback : {false, true}) for (bool valid : {false, true}) { + bool completed = false; + unsigned failures = 0; + for (int budget = 0; budget < 32 && !completed; ++budget) { + Value value{}; _ZN3sce4Json5ValueC1El(&value, 77); + Require(Parse(value, "[]") == 0, "Prepare an owned empty array before element admission"); + const char* document = valid ? "[1]" : "[1,"; + const char* cursor = document; + const auto notifications = live.notifications; + live.budget = budget; + const int status = callback ? _ZN3sce4Json6Parser5parseERNS0_5ValueEPFiRcPvES5_(&value, ReadNext, &cursor) : Parse(value, document); + live.budget = -1; + const bool failed = live.notifications != notifications; + const int expected = failed ? static_cast(0x80848102) : valid ? 0 : callback ? -1 : static_cast(0x80848101); + Require(status == expected, "Array element allocation and reader failures must preserve their public status"); + if (!failed) { + completed = true; + Require(value.type == 6 && _ZNK3sce4Json5Value5countEv(&value) == (valid ? 1 : 2), "Complete and malformed array parsing must retain the admitted entry count"); + const auto* first = _ZNK3sce4Json5Array5frontEv(reinterpret_cast(value.payload)); + Require(first->type == 3 && first->payload == 1 && first->parent == &value, "The parsed element must remain owned by its destination array"); + } else { + ++failures; + Require(Parse(value, "false") == 0 && Parse(value, "[]") == 0 && Parse(value, "[1]") == 0 && _ZNK3sce4Json5Value5countEv(&value) == 1, + "A failed array parse must permit cleanup and successful reuse of the same value"); + } + _ZN3sce4Json5ValueD1Ev(&value); + Require(live.size() == baseline, "Element admission and callback parsing must leave no unreachable guest allocations after cleanup"); + } + Require(completed && failures != 0, "Every reader and syntax case must exercise guest allocation failure and complete admission"); + } +} int main() { JsonTest::Allocator allocator; void* const vtable[]{nullptr, nullptr, reinterpret_cast(AllocateTracked), reinterpret_cast(FreeTracked), reinterpret_cast(NotifyTracked)}; @@ -67,6 +106,7 @@ int main() { _ZN3sce4Json11InitializerC1Ev(initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(initializer, ¶meter) == 0, "Initialize parser state tests"); const auto baseline = live.size(); + CheckArrayEntryFailures(live, baseline); struct FailureCase { bool once; int budget; int type; unsigned notifications; std::size_t requests; const char* text; }; const FailureCase failures[]{ {false, 0, 0, 2, 2, ""}, {false, 1, 0, 2, 3, ""}, From 4f15f29b4215be2105bbad01a9447ffbf130e6be Mon Sep 17 00:00:00 2001 From: milescoviello Date: Sat, 24 Oct 2026 16:56:46 +0000 Subject: [PATCH 108/318] fix(agc): share the code of registrations of one program Every registration keeps a snapshot of its program's code, and a program registered through several headers kept one copy per header: Ghost of Tsushima (PPSA02225) had 94,000 snapshots at 14,321 code addresses 172 s in, holding 185.8 MiB of code of which 43.0 MiB was distinct. A snapshot's code is now a shared, immutable buffer, and a registration takes the buffer of a snapshot of the same program that already holds the same words, so the copies of one program become one. Comparing two snapshots' code first compares the buffers. Co-Authored-By: Claude Opus 5.5 --- .../include/Driver/Shaders/ShaderRegistry.hpp | 40 ++++++++++++++++++- .../src/Driver/Shaders/ShaderRegistry.cpp | 8 ++-- .../tests/execution/PreparedGraphics.cpp | 3 +- .../tests/execution/PreparedPublication.cpp | 2 +- .../tests/execution/PreparedShaders.cpp | 1 + 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index b1907257c4..510424e3ec 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -171,6 +171,44 @@ struct PreparedShaders : PreparedShaderState { std::vector convertedSamples; }; +class SharedCode { +public: + SharedCode() = default; + SharedCode(std::vector values) : words(std::make_shared>(std::move(values))) {} + SharedCode(std::initializer_list values) : SharedCode(std::vector(values)) {} + template + SharedCode(TIterator first, TIterator last) : SharedCode(std::vector(first, last)) {} + + operator const std::vector&() const { return Words(); } + [[nodiscard]] const std::vector& Words() const { return words != nullptr ? *words : none(); } + [[nodiscard]] const std::uint32_t* data() const { return Words().data(); } + [[nodiscard]] std::size_t size() const { return Words().size(); } + [[nodiscard]] bool empty() const { return Words().empty(); } + [[nodiscard]] const std::uint32_t* begin() const { return Words().data(); } + [[nodiscard]] const std::uint32_t* end() const { return Words().data() + Words().size(); } + [[nodiscard]] std::uint32_t operator[](std::size_t index) const { return Words()[index]; } + [[nodiscard]] bool SharesWith(const SharedCode& other) const { return words != nullptr && words == other.words; } + bool operator==(const SharedCode& other) const { return words == other.words || Words() == other.Words(); } + bool operator==(const std::vector& other) const { return Words() == other; } + + std::vector& Mutable() { + if (words == nullptr) words = std::make_shared>(); + else if (words.use_count() > 1) words = std::make_shared>(*words); + return *words; + } + void resize(std::size_t count) { Mutable().resize(count); } + template + void assign(TIterator first, TIterator last) { Mutable().assign(first, last); } + +private: + static const std::vector& none() { + static const std::vector none; + return none; + } + + std::shared_ptr> words; +}; + struct RegisteredShaderState { Registers shader; Registers context; @@ -183,7 +221,7 @@ struct ShaderSnapshot { std::uint64_t codeAddress; std::uint64_t headerAddress; std::uint8_t type; - std::vector code; + SharedCode code; std::vector header; std::shared_ptr prepared = std::make_shared(); std::shared_ptr registeredState; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 03db5c7985..9c8e68a3c2 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -119,7 +119,7 @@ std::shared_ptr ReadRawShader(std::uint64_t address) { snapshot.code.resize(std::min(available, std::max(64, previous * 2))); drain(address + previous * sizeof(std::uint32_t), (snapshot.code.size() - previous) * sizeof(std::uint32_t)); GuestMemory::Read(address + previous * sizeof(std::uint32_t), - std::as_writable_bytes(std::span(snapshot.code).subspan(previous)), alignof(std::uint32_t)); + std::as_writable_bytes(std::span(snapshot.code.Mutable()).subspan(previous)), alignof(std::uint32_t)); try { const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(snapshot.code); if (const auto tail = ShaderRecompiler::FindUserDataTail(decoded, 0u, 106u)) snapshot.userDataTailRegister = tail->userDataRegister; @@ -2324,7 +2324,7 @@ void Driver::RegisterShader(const Shader* shader) { require(!RawGraphicsDrawnIn(reinterpret_cast(code), reinterpret_cast(code) + fields.shader_size), "a shader was registered over a graphics program that was drawn without registration"); ShaderSnapshot snapshot{reinterpret_cast(code), reinterpret_cast(shader), fields.type, {}, {}}; snapshot.code.resize(fields.shader_size / sizeof(std::uint32_t)); - std::memcpy(snapshot.code.data(), code, fields.shader_size); + std::memcpy(snapshot.code.Mutable().data(), code, fields.shader_size); snapshot.header.resize(fields.header_size); std::memcpy(snapshot.header.data(), static_cast(shader), fields.header_size); @@ -2359,7 +2359,9 @@ void Driver::RegisterShader(const Shader* shader) { if (shaders != nullptr) { if (const auto found = shaders->find(snapshot.codeAddress); found != shaders->end()) { for (const auto& current : found->second) { - if (current->registeredState == nullptr || current->type != snapshot.type || current->code != snapshot.code || *current->registeredState != *snapshot.registeredState) continue; + if (current->code != snapshot.code) continue; + snapshot.code = current->code; + if (current->registeredState == nullptr || current->type != snapshot.type || *current->registeredState != *snapshot.registeredState) continue; snapshot.registeredState = current->registeredState; break; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp index 56ea345d1f..199febbf1b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp @@ -84,7 +84,8 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path fragment.Initialize(1u); const std::array pixelCode{0xc8020002u, 0xc8060102u, 0xc80a0202u, 0xc80e0302u, 0xf800180fu, 0x03020100u, 0xbf810000u}; fragment.snapshot->code.resize(64); - fragment.snapshot->code.insert(fragment.snapshot->code.end(), pixelCode.begin(), pixelCode.end()); + auto& fragmentCode = fragment.snapshot->code.Mutable(); + fragmentCode.insert(fragmentCode.end(), pixelCode.begin(), pixelCode.end()); ShaderRegistry registry; for (const auto* fixture : {&front, &back, &domain, &fragment}) registry.emplace(fixture->snapshot->codeAddress, std::vector>{fixture->snapshot}); AgcDriver::QueueState queue{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedPublication.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedPublication.cpp index 1273ff05c7..bbcd01b7e0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedPublication.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedPublication.cpp @@ -338,7 +338,7 @@ void Registration(AgcDriver::VulkanDevice& device) { Require(invocation.Materialize(*capture)->spirv.data() == result->spirv.data(), "prepared handle did not retain its artifact"); auto changed = makeSnapshot(); changed->headerAddress = replacement->headerAddress; - changed->code[0] ^= 1; + changed->code.Mutable()[0] ^= 1; PublishRegisteredShader(registry, changed); Require(registry->at(0x10000).back() == changed && registry->at(0x10000).size() == 1 && old.expired(), "changed shader code retained obsolete headers"); auto metadata = makeSnapshot(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp index 88ecd918f8..13637c800f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -446,6 +446,7 @@ void SharedCodeHeaders() { }; const auto [vertexSnapshot, meshSnapshot] = snapshots(); Require(vertexSnapshot->registeredState != meshSnapshot->registeredState, "a header with other registers shared another header's registered state"); + Require(vertexSnapshot->code.SharesWith(meshSnapshot->code), "two headers of one program keep separate copies of its code"); Shader copy = vertex.shader; copy.user_data = nullptr; AgcDriverResolveShaderAbi_nid_postfix(©, {}, primitive); From f9b5cc27e1636cbc26db30ae82269b2714570cc0 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:50:36 -0700 Subject: [PATCH 109/318] feat(agc): add APS5_NO_IDLE_STORE to leave pending storage results to their readers Queue 0's worker stores every pending storage-image result once the queues have stayed idle for 50 ms. A test that orders a pending result against a later consumer races that store: it can land before the consumer is queued, and the order under test is never exercised. APS5_NO_IDLE_STORE=1 turns the idle store off, so pending results stay pending until a reader, a label or an explicit flush stores them. The default is unchanged. --- .../Execution/src/Driver/Queues/QueueWorker.cpp | 3 ++- docs/dev/ENVIRONMENT.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index 546be83644..c4ac6c1bb9 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -144,6 +144,7 @@ void Driver::run(std::uint32_t id) noexcept { if (id == 0) StartWorkerSampler(); Submission submission; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static const bool idleStore = std::getenv("APS5_NO_IDLE_STORE") == nullptr; auto& costs = submissionCosts(id); try { if (PerformanceTrace::Configured()) PerformanceTrace::BindThread(role); @@ -183,7 +184,7 @@ void Driver::run(std::uint32_t id) noexcept { && std::all_of(workers.begin(), workers.end(), [](const auto& item) { return item.second.pending.empty(); }); }; const auto idleResultsPending = [&] { - return id == 0 && Graphics::StorageTexture::AnyPending() && allQueuesIdle() && !completionsPending(); + return idleStore && id == 0 && Graphics::StorageTexture::AnyPending() && allQueuesIdle() && !completionsPending(); }; const auto poll = [&](const auto& done) { #ifdef _WIN32 diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 93ed160653..92435e2354 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -256,6 +256,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_HOST_IMPORT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | disables importing guest memory, so address-based shaders copy instead of using it in place | | `APS5_NO_HW_F32_FLUSH` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | skips the startup f32 denormal flush probe and keeps the emulated flush in every shader | | `APS5_NO_HW_F32_PRESERVE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | stops requesting DenormPreserve for 32-bit floats, so the device keeps its default f32 denormal handling | +| `APS5_NO_IDLE_STORE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp` | | stops queue 0's worker from storing pending storage-image results after the queues stay idle for 50 ms, so they stay pending until a reader, label or flush stores them | | `APS5_NO_IMAGE_PAGE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp` | | leaves the main guest image's page-protection table uncached | | `APS5_NO_INDIRECT_DRAW` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp` | | rejects indirect draw packets at submission validation | | `APS5_NO_JOIN_WAW_CHECK` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | lets a contiguous store join the pending one without the write-after-write check | From 5138b1b1d25664561055b3b5b5810d9780dd93cb Mon Sep 17 00:00:00 2001 From: Maximilian Fleckenstein <166870690+maximilianfv@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:47:17 -0300 Subject: [PATCH 110/318] fix(agc): order wave LDS accesses with memory barriers on MoltenVK SPIRV-Cross widens a barrier's execution scope to its memory scope, so the subgroup-scope OpControlBarrier with workgroup memory semantics that orders one wave's LDS accesses becomes a threadgroup_barrier. Emitted inside a branch only some waves take, those barriers paired with other waves' barriers, and agc_driver_lds_tree_reduction and agc_driver_lds_subword_store_branch read LDS before it was written. MoltenVK now takes the memory-barrier path AMD's proprietary driver already uses. Co-authored-by: Himansh Raj <67262056+iamthehimansh@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../Execution/include/SubgroupBarrier.hpp | 2 +- .../libSceAgcDriver/tests/SubgroupBarrier.cpp | 23 ++++++++++--------- docs/dev/debt/shader.md | 1 + 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp index 8675650a20..352855e6df 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp @@ -6,7 +6,7 @@ namespace AgcDriver { inline bool SubgroupControlBarrierUsable(VkDriverId driver) { - return driver != VK_DRIVER_ID_AMD_PROPRIETARY; + return driver != VK_DRIVER_ID_AMD_PROPRIETARY && driver != VK_DRIVER_ID_MOLTENVK; } inline bool WorkgroupScopeFenceUsable(VkDriverId driver) { diff --git a/core/libs/prx/libSceAgcDriver/tests/SubgroupBarrier.cpp b/core/libs/prx/libSceAgcDriver/tests/SubgroupBarrier.cpp index 0578ee283d..696e85a7af 100644 --- a/core/libs/prx/libSceAgcDriver/tests/SubgroupBarrier.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/SubgroupBarrier.cpp @@ -16,18 +16,19 @@ namespace { struct Case { VkDriverId driver; - bool usable; + bool controlBarrier; + bool workgroupFence; }; constexpr Case Cases[]{ - {VK_DRIVER_ID_AMD_PROPRIETARY, false}, - {VK_DRIVER_ID_AMD_OPEN_SOURCE, true}, - {VK_DRIVER_ID_MESA_RADV, true}, - {VK_DRIVER_ID_NVIDIA_PROPRIETARY, true}, - {VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS, true}, - {VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA, true}, - {VK_DRIVER_ID_MESA_LLVMPIPE, true}, - {VK_DRIVER_ID_MOLTENVK, true}, + {VK_DRIVER_ID_AMD_PROPRIETARY, false, false}, + {VK_DRIVER_ID_AMD_OPEN_SOURCE, true, true}, + {VK_DRIVER_ID_MESA_RADV, true, true}, + {VK_DRIVER_ID_NVIDIA_PROPRIETARY, true, true}, + {VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS, true, true}, + {VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA, true, true}, + {VK_DRIVER_ID_MESA_LLVMPIPE, true, true}, + {VK_DRIVER_ID_MOLTENVK, false, true}, }; constexpr std::array BranchedLdsStep{ @@ -120,8 +121,8 @@ Barriers recompile(std::uint32_t threads, bool waveLdsMemoryBarriers) { void verifyDriverQuirk() { for (const auto& item : Cases) { - require(AgcDriver::SubgroupControlBarrierUsable(item.driver) == item.usable, "driver " + std::to_string(static_cast(item.driver)) + ": expected subgroup control barriers to be " + (item.usable ? "usable" : "unusable")); - require(AgcDriver::WorkgroupScopeFenceUsable(item.driver) == item.usable, "driver " + std::to_string(static_cast(item.driver)) + ": expected workgroup-scope fences to be " + (item.usable ? "usable" : "unusable")); + require(AgcDriver::SubgroupControlBarrierUsable(item.driver) == item.controlBarrier, "driver " + std::to_string(static_cast(item.driver)) + ": expected subgroup control barriers to be " + (item.controlBarrier ? "usable" : "unusable")); + require(AgcDriver::WorkgroupScopeFenceUsable(item.driver) == item.workgroupFence, "driver " + std::to_string(static_cast(item.driver)) + ": expected workgroup-scope fences to be " + (item.workgroupFence ? "usable" : "unusable")); } } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index db3130a1f6..93351290df 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -196,4 +196,5 @@ - [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it) those loads keep their selections; LDS loads, stores, atomics, formatted and physical-address (BDA) accesses always do. LDS keeps the branch because NVIDIA's compiler predicates a branched LDS load at no cost, while the select form costs about 2.5 instructions per dword and splits a `ds_read_b128` into four loads (RTX 4060 measurements by @karanvora2599 in #4048); Intel's static counts favour the select only for wide reads, and run time was not measured on either. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. - [Merged hull programs](../../../core/shader/recompiler/Optimization/src/TessellationLowering.cpp) (registered binary type 3) run whole in the hull stage, one invocation per output control point. On the console hull lane i keeps local lane i's registers, which with fewer input than output control points stop matching the patch's control points from i = inputs on; here invocation i holds control point min(i, inputs - 1) of its own patch. A wave-mask branch (`s_cbranch_exec[n]z`, `s_cbranch_vcc[n]z`) tests the invocation's own lane, so it can skip a block, scalar instructions included, that the console runs for the wave; `v_readfirstlane_b32` returns the invocation's own value, not the first active lane's, and another lane's register throws. The register layout (`s3`, `v0`-`v5`, as for the separate halves) is not measured on hardware. s[0:1] hold the hull user-data address (SH 0x102/0x103) when a draw sets it and zero otherwise, so a program reading it unset faults at the null address. - [Wave LDS ordering on AMD's proprietary driver](../../../core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp) (shader recompiler) - there ([SubgroupBarrier.hpp](../../../core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp)) one guest wave's LDS accesses are ordered by a subgroup-scope `OpMemoryBarrier` with no control barrier, relying on a 32-lane subgroup being one hardware wave whose lanes run in lockstep and issue LDS instructions in order, which Vulkan does not guarantee; for LDS kept in device memory the same is assumed of global loads and stores, unmeasured. Driver 32.0.21042.62 (Radeon 0x13c0, RDNA 2) compiled the subgroup-scope `OpControlBarrier` to `s_barrier`, which waves in a branch paired with other waves' barriers (PS5LM Plus). Measured on that driver only ([LdsTreeReduction.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsTreeReduction.cpp)); a Radeon 860M on 32.0.13058.8001 reportedly did not show the fault. +- [Wave LDS ordering on MoltenVK](../../../core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp) (shader recompiler) - there ([SubgroupBarrier.hpp](../../../core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp)) one guest wave's LDS accesses are ordered by a subgroup-scope `OpMemoryBarrier`, which becomes a simdgroup-scope `atomic_thread_fence` with no execution barrier, relying on a 32-lane simdgroup running in lockstep, which Metal does not guarantee. The subgroup-scope `OpControlBarrier` other drivers get cannot be used: SPIRV-Cross widens a barrier's execution scope to its memory scope, so it became a `threadgroup_barrier`, which waves in a branch paired with other waves' barriers. Measured on an M1 Pro (MoltenVK 1.4.2) only ([LdsTreeReduction.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsTreeReduction.cpp), [LdsSubwordStoreBranch.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsSubwordStoreBranch.cpp)). - [LDS atomic fences on AMD's proprietary driver](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp) (shader recompiler) - there ([SubgroupBarrier.hpp](../../../core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp)) the fence after an LDS atomic or a sub-dword or misaligned LDS store is a device-scope `OpMemoryBarrier`, which that driver compiles to GL0 and GL1 cache invalidations; the cost is unmeasured. Driver 32.0.21042.62 (Radeon 0x13c0, RDNA 2) compiled the workgroup-scope fence to `s_barrier`, which waves in a branch paired with other waves' barriers (PS5LM Plus with flash attention). Measured on that driver only ([LdsSubwordStoreBranch.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsSubwordStoreBranch.cpp)); a Radeon 860M (RDNA 3.5, 32.0.13058.8001) reportedly gave correct flash-attention output, but the gate covers every version. From 290f9eed01224dd071752c4ad0cd0657d772602b Mon Sep 17 00:00:00 2001 From: Maximilian Fleckenstein <166870690+maximilianfv@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:47:18 -0300 Subject: [PATCH 111/318] fix(agc): keep sRGB storage images in their UNORM twin on every driver StorageFormatOrUndefined kept an 8-bit sRGB storage texture in its sRGB format whenever the driver reported storage support for it, and fell back to the UNORM twin only otherwise. Shader stores to an sRGB storage image are sRGB-encoded, so a guest store of a raw code landed encoded. MoltenVK 1.4.2 reports storage support for these formats: agc_driver_image_srgb_sample_copy and its forced variant sampled the encoded value, and agc_driver_recover_srgb_allocation failed because CanSrgbExpandFrom refused a storage format that differs from the sampled UNORM format. The five 8-bit sRGB formats now always use their UNORM twin for storage. Co-authored-by: Ycaro Gabriel <280482597+ycarogabriel-lgtm@users.noreply.github.com> Co-authored-by: Himansh Raj <67262056+iamthehimansh@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/Graphics/src/Texture.cpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index d8394b6c5e..ee2f31e5b3 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1234,21 +1234,21 @@ VkFormat StorageFormatOrUndefined(const Context& context, VkFormat format) { }; VkFormat storage = VK_FORMAT_UNDEFINED; const auto packed = PackedStorageViewFormat(format); + VkFormat linear = VK_FORMAT_UNDEFINED; + switch (format) { + case VK_FORMAT_R8G8B8A8_SRGB: linear = VK_FORMAT_R8G8B8A8_UNORM; break; + case VK_FORMAT_B8G8R8A8_SRGB: linear = VK_FORMAT_B8G8R8A8_UNORM; break; + case VK_FORMAT_A8B8G8R8_SRGB_PACK32: linear = VK_FORMAT_A8B8G8R8_UNORM_PACK32; break; + case VK_FORMAT_R8_SRGB: linear = VK_FORMAT_R8_UNORM; break; + case VK_FORMAT_R8G8_SRGB: linear = VK_FORMAT_R8G8_UNORM; break; + default: break; + } if (packed != VK_FORMAT_UNDEFINED) { if (supports(packed)) storage = format; + } else if (linear != VK_FORMAT_UNDEFINED) { + if (supports(linear)) storage = linear; } else if (supports(format)) { storage = format; - } else { - VkFormat linear = VK_FORMAT_UNDEFINED; - switch (format) { - case VK_FORMAT_R8G8B8A8_SRGB: linear = VK_FORMAT_R8G8B8A8_UNORM; break; - case VK_FORMAT_B8G8R8A8_SRGB: linear = VK_FORMAT_B8G8R8A8_UNORM; break; - case VK_FORMAT_A8B8G8R8_SRGB_PACK32: linear = VK_FORMAT_A8B8G8R8_UNORM_PACK32; break; - case VK_FORMAT_R8_SRGB: linear = VK_FORMAT_R8_UNORM; break; - case VK_FORMAT_R8G8_SRGB: linear = VK_FORMAT_R8G8_UNORM; break; - default: break; - } - if (linear != VK_FORMAT_UNDEFINED && supports(linear)) storage = linear; } std::lock_guard lock(table.mutex); table.formats.emplace(key, storage); From d57418e1b28727f1ac83383fa2f9e27730b0a9e4 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:50:37 -0700 Subject: [PATCH 112/318] test(agc): keep queued_index_read_order's image results pending past the idle store Since #7233, Driver::WaitIdle also syncs the recorder and waits for the device under the GPU mutex. On lavapipe that takes longer than the queue worker's 50 ms idle quiet period, so the idle store flushes the image write's results as soon as WaitIdle returns. The test then fails its setup check ("the image write left no pending result to order against") on most runs on GitHub's Linux runner. Under load the store can instead land after that check but before the draw holds its read, and the test fails at "the queued draw's commit stored the image's results over its index buffer" although the commit never stored them. The test now sets APS5_NO_IDLE_STORE=1, as CopiedWriterLabelOrder sets APS5_NO_OPPORTUNISTIC_REAP, so only the queued draw's held read decides when the results are stored. The idle store keeps its own tests (agc_idle_publish_wait, agc_driver_store_idle_results_wake). --- .../libSceAgcDriver/tests/execution/QueuedIndexReadOrder.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/QueuedIndexReadOrder.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/QueuedIndexReadOrder.cpp index acc2b1617a..6c76aa88d8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/QueuedIndexReadOrder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/QueuedIndexReadOrder.cpp @@ -344,9 +344,11 @@ int main() { #ifdef _WIN32 _putenv_s("APS5_PIPELINED_DRAWS", "1"); _putenv_s("APS5_PIPELINE_DMA", "1"); + _putenv_s("APS5_NO_IDLE_STORE", "1"); #else setenv("APS5_PIPELINED_DRAWS", "1", 1); setenv("APS5_PIPELINE_DMA", "1", 1); + setenv("APS5_NO_IDLE_STORE", "1", 1); #endif try { { From 433e43e8d19c5140f53e4ccb7309f90254293e16 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:33:21 -0700 Subject: [PATCH 113/318] fix(agc): let queue 0 wait with APS5_NO_IDLE_STORE while storage results stay pending With the idle store off, idleResultsPending() is always false, so queue 0's retry after the idle check saw no stored results and pending ones on every pass. Once every queue was idle with results pending and no completions, it went round the loop without reaching changed.wait(), unlocking and relocking the driver mutex until another submission came. The retry for pending results now applies only when the idle store is on. The retry on an idle epoch change and the wait's wake on a submission, an epoch change or a completion are unchanged. --- .../libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp | 2 +- docs/dev/ENVIRONMENT.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index c4ac6c1bb9..a5c5bba2cf 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -222,7 +222,7 @@ void Driver::run(std::uint32_t id) noexcept { lock.lock(); if (ready()) break; if (id == 0 && allQueuesIdle() && !completionsPending() - && ((!storeIdleResults && Graphics::StorageTexture::AnyPending()) || idleEpoch.load(std::memory_order_acquire) != idleEpochAtCheck)) continue; + && ((idleStore && !storeIdleResults && Graphics::StorageTexture::AnyPending()) || idleEpoch.load(std::memory_order_acquire) != idleEpochAtCheck)) continue; if (id == 0) queue0Dormant.store(true, std::memory_order_relaxed); const auto idleEpochAtWait = idleEpoch.load(std::memory_order_acquire); changed.wait(lock, [&] { return ready() || (id == 0 && (idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait || completionsPending())); }); diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 92435e2354..4741463855 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -256,7 +256,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_HOST_IMPORT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | disables importing guest memory, so address-based shaders copy instead of using it in place | | `APS5_NO_HW_F32_FLUSH` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | skips the startup f32 denormal flush probe and keeps the emulated flush in every shader | | `APS5_NO_HW_F32_PRESERVE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | stops requesting DenormPreserve for 32-bit floats, so the device keeps its default f32 denormal handling | -| `APS5_NO_IDLE_STORE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp` | | stops queue 0's worker from storing pending storage-image results after the queues stay idle for 50 ms, so they stay pending until a reader, label or flush stores them | +| `APS5_NO_IDLE_STORE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp` | | stops queue 0's worker from storing pending storage-image results after the queues stay idle for 50 ms, so they stay pending until a reader, label or flush stores them; queue 0 still waits for work while they are pending | | `APS5_NO_IMAGE_PAGE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp` | | leaves the main guest image's page-protection table uncached | | `APS5_NO_INDIRECT_DRAW` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp` | | rejects indirect draw packets at submission validation | | `APS5_NO_JOIN_WAW_CHECK` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | lets a contiguous store join the pending one without the write-after-write check | From 5f0c5f0bab256571002ecce24f4e00192150e683 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:33:22 -0700 Subject: [PATCH 114/318] test(agc): queue 0 waits and wakes with the idle store off agc_driver_store_idle_results_disabled runs the idle store fixture with APS5_NO_IDLE_STORE=1. Once a dispatch leaves a storage result pending and every queue is idle, queue 0 must wait within 2 s, and 200 ms later the result must still be pending and not in guest memory. A second dispatch, to another texture, must then run: its result is pending within 10 s, and queue 0 waits again within 2 s. --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 5 +++ .../tests/execution/StoreIdleResults.cpp | 34 ++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 5ab641b0d4..230d3136cb 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2214,11 +2214,16 @@ if(BUILD_TESTING) add_test(NAME agc_driver_store_idle_results_wake COMMAND agc_driver_store_idle_results_tests --idle-epoch-wake) set_tests_properties(agc_driver_store_idle_results_wake PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_store_idle_results_disabled COMMAND agc_driver_store_idle_results_tests --disabled-idle-store) + set_tests_properties(agc_driver_store_idle_results_disabled PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_IDLE_STORE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) set_tests_properties(agc_driver_store_pending_identity PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") set_tests_properties(agc_driver_store_idle_results_wake PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_driver_store_idle_results_disabled PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StoreIdleResults.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StoreIdleResults.cpp index c9c8e08c3a..491abfd4e1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/StoreIdleResults.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StoreIdleResults.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" @@ -13,6 +14,7 @@ #include #include #include +#include #include namespace { @@ -86,8 +88,9 @@ int main(int argc, char** argv) { } std::vector storage((BlockBytes + 0x10000u) / 4u); const bool wake = argc == 2 && std::string_view(argv[1]) == "--idle-epoch-wake"; - std::vector nextStorage(wake ? (BlockBytes + 0x10000u) / 4u : 0u); - auto* nextTexels = wake ? reinterpret_cast((reinterpret_cast(nextStorage.data()) + 0xffffu) & ~std::uintptr_t{0xffffu}) : nullptr; + const bool disabled = argc == 2 && std::string_view(argv[1]) == "--disabled-idle-store"; + std::vector nextStorage(wake || disabled ? (BlockBytes + 0x10000u) / 4u : 0u); + auto* nextTexels = wake || disabled ? reinterpret_cast((reinterpret_cast(nextStorage.data()) + 0xffffu) & ~std::uintptr_t{0xffffu}) : nullptr; struct ShutdownGuard { ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} } } shutdown; @@ -127,9 +130,30 @@ int main(int argc, char** argv) { cpu = nextTexels; Store(nextTexels); } - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); - while (!stored() && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); - Require(stored(), "the queue-idle store did not bring the result to guest memory without a direct flush or further submission"); + if (disabled) { + const auto pendingDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!AgcDriver::Graphics::StorageTexture::AnyPending() && std::chrono::steady_clock::now() < pendingDeadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(AgcDriver::Graphics::StorageTexture::AnyPending(), "disabled idle store fixture did not observe pending storage results"); + const auto dormant = [] { return AgcDriver::DriverDetail::queue0Dormant.load(std::memory_order_relaxed); }; + const auto dormantDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!dormant() && std::chrono::steady_clock::now() < dormantDeadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(dormant(), "with APS5_NO_IDLE_STORE queue 0 kept polling over pending storage results with every queue idle instead of waiting"); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + Require(AgcDriver::Graphics::StorageTexture::AnyPending() && !stored(), "with APS5_NO_IDLE_STORE the idle store still brought the result to guest memory"); + const std::array, 1> nextRange{{{AddressOf(nextTexels), AddressOf(nextTexels) + bytes}}}; + Require(!AgcDriver::Graphics::StorageTexture::AnyPendingOverlaps(nextRange), "disabled idle store fixture found a result pending over the second texture before its dispatch"); + Store(nextTexels); + const auto wakeDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!AgcDriver::Graphics::StorageTexture::AnyPendingOverlaps(nextRange) && std::chrono::steady_clock::now() < wakeDeadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(AgcDriver::Graphics::StorageTexture::AnyPendingOverlaps(nextRange), "queue 0 did not run a submission made after it waited with APS5_NO_IDLE_STORE"); + const auto parkedAgainDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!dormant() && std::chrono::steady_clock::now() < parkedAgainDeadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(dormant(), "with APS5_NO_IDLE_STORE queue 0 kept polling after running the second dispatch instead of waiting again"); + } else { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!stored() && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(stored(), "the queue-idle store did not bring the result to guest memory without a direct flush or further submission"); + } AgcDriverShutdown_nid_postfix(); { From ffe4691999acc55f907310d339d22367ebfef0c6 Mon Sep 17 00:00:00 2001 From: Felipebmaitan Date: Sun, 25 Oct 2026 01:59:07 -0300 Subject: [PATCH 115/318] fix(ult): serialize ulthread publication and join ownership --- core/libs/prx/libSceUlt/CMakeLists.txt | 17 ++- core/libs/prx/libSceUlt/Export.cpp | 112 +++++++------- core/libs/prx/libSceUlt/SceUltTypes.hpp | 1 + core/libs/tests/GuestUltThread.cpp | 85 +++++++++++ core/libs/tests/UltThreadLifecycle.cpp | 192 ++++++++++++++++++++++++ docs/dev/debt/libSceUlt.md | 3 +- 6 files changed, 355 insertions(+), 55 deletions(-) create mode 100644 core/libs/tests/UltThreadLifecycle.cpp diff --git a/core/libs/prx/libSceUlt/CMakeLists.txt b/core/libs/prx/libSceUlt/CMakeLists.txt index ea7e2f4a31..e1fd7666dd 100644 --- a/core/libs/prx/libSceUlt/CMakeLists.txt +++ b/core/libs/prx/libSceUlt/CMakeLists.txt @@ -3,6 +3,19 @@ target_include_directories(libSceUlt PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceUlt) if(BUILD_TESTING) + add_test_executable(guest_ult_thread_lifecycle_tests ${LIBS_INCLUDE_DIR}/tests/UltThreadLifecycle.cpp Export.cpp src/ReaderWriterLock.cpp) + target_include_directories(guest_ult_thread_lifecycle_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_ult_thread_lifecycle_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_ult_thread_lifecycle_tests) + foreach(testCase IN ITEMS create-error create-exception published-exception join-owner tryjoin-owner join-error join-exception) + add_test(NAME guest_ult_thread_lifecycle_${testCase} COMMAND guest_ult_thread_lifecycle_tests ${testCase}) + set_tests_properties(guest_ult_thread_lifecycle_${testCase} PROPERTIES TIMEOUT 10) + if(WIN32) + set_tests_properties(guest_ult_thread_lifecycle_${testCase} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() + add_test_executable(guest_ult_condition_tests ${LIBS_INCLUDE_DIR}/tests/GuestUltCondition.cpp) target_include_directories(guest_ult_condition_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_ult_condition_tests PRIVATE libSceUlt libc) @@ -31,9 +44,9 @@ if(BUILD_TESTING) add_test_executable(guest_ult_thread_tests ${LIBS_INCLUDE_DIR}/tests/GuestUltThread.cpp) target_include_directories(guest_ult_thread_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_ult_thread_tests PRIVATE libSceUlt libc) + target_link_libraries(guest_ult_thread_tests PRIVATE libSceUlt libkernel libc) configure_windows_unwind(guest_ult_thread_tests) - foreach(testCase IN ITEMS outstanding validation tryjoin destroy underscoredcreate) + foreach(testCase IN ITEMS outstanding validation tryjoin destroy underscoredcreate cancel-blocked cancel-pending self-join) add_test(NAME guest_ult_thread_${testCase} COMMAND guest_ult_thread_tests ${testCase}) set_tests_properties(guest_ult_thread_${testCase} PROPERTIES TIMEOUT 20) if(WIN32) diff --git a/core/libs/prx/libSceUlt/Export.cpp b/core/libs/prx/libSceUlt/Export.cpp index 88ac5556cc..6e9a414b6b 100644 --- a/core/libs/prx/libSceUlt/Export.cpp +++ b/core/libs/prx/libSceUlt/Export.cpp @@ -1,4 +1,5 @@ #include "prx/libkernel/Pthread/include/ThreadLifecycle.hpp" +#include "prx/libkernel/Pthread/include/Cancel.hpp" #include "prx/libkernel/include/Api.hpp" #include "prx/libSceUlt/include/Api.hpp" #include @@ -127,6 +128,51 @@ void* APS5_VABI ulthreadRunner(void* arg) { return reinterpret_cast(static_cast(result)); } +void APS5_VABI releaseUlthreadJoin(void* argument) { + auto& state = *static_cast*>(argument); + { + std::lock_guard lock(gMutex); + state->_joining = false; + } + state.reset(); +} + +int joinUlthread(void* ulthread, std::int32_t* status, bool blocking) { + if (ulthread == nullptr) return ULT_ERROR_NULL; + std::shared_ptr state; + { + std::lock_guard lock(gMutex); + const auto it = gUlthreads.find(ulthread); + if (it == gUlthreads.end()) return ULT_ERROR_STATE; + state = it->second; + if (!blocking && !state->_exited.load(std::memory_order_acquire)) return ULT_ERROR_AGAIN; + if (state->_joining || state.get() == gCurrentUlthread) return ULT_ERROR_STATE; + state->_joining = true; + } + CleanupHandler cleanup{}; + void* const mark = ThreadCancel::CleanupMark(); + __pthread_cleanup_push_imp_nid_postfix(releaseUlthreadJoin, &state, &cleanup); + void* result = nullptr; + int joined; + try { + joined = scePthreadJoin(state->_thread, &result); + } catch (...) { + ThreadCancel::CleanupRestore(mark); + releaseUlthreadJoin(&state); + throw; + } + ThreadCancel::CleanupRestore(mark); + if (joined != 0) { + releaseUlthreadJoin(&state); + return ULT_ERROR_STATE; + } + if (status != nullptr) *status = static_cast(reinterpret_cast(result)); + std::lock_guard lock(gMutex); + const auto it = gUlthreads.find(ulthread); + if (it != gUlthreads.end() && it->second == state) gUlthreads.erase(it); + return ULT_OK; +} + } bool UltWaitingPoolExists(void* pool) { @@ -804,67 +850,29 @@ int APS5_VABI sceUltUlthreadCreate(void* ulthread, const char* name, UltUlthread state->_arg = arg; state->_object = ulthread; state->_runtime = runtime; - { - std::lock_guard lock(gMutex); - if (gRuntimes.find(runtime) == gRuntimes.end()) { - return ULT_ERROR_STATE; - } - if (gUlthreads.find(ulthread) != gUlthreads.end()) { - return ULT_ERROR_STATE; - } - std::memset(ulthread, 0, 512); - gUlthreads[ulthread] = state; - } const std::string threadName = name != nullptr ? std::string(name).substr(0, 31) : std::string(); - const int result = scePthreadCreate(&state->_thread, nullptr, ulthreadRunner, state.get(), threadName.c_str()); - if (result != 0) { - std::lock_guard lock(gMutex); - gUlthreads.erase(ulthread); - return ULT_ERROR_AGAIN; + std::lock_guard lock(gMutex); + if (!gRuntimes.contains(runtime) || gUlthreads.contains(ulthread)) return ULT_ERROR_STATE; + gUlthreads.emplace(ulthread, state); + std::memset(ulthread, 0, 512); + try { + if (scePthreadCreate(&state->_thread, nullptr, ulthreadRunner, state.get(), threadName.c_str()) != 0) { + gUlthreads.erase(ulthread); + return ULT_ERROR_AGAIN; + } + } catch (...) { + if (state->_thread == nullptr) gUlthreads.erase(ulthread); + throw; } return ULT_OK; } int APS5_VABI sceUltUlthreadJoin(void* ulthread, std::int32_t* status) { - if (ulthread == nullptr) { - return ULT_ERROR_NULL; - } - std::shared_ptr state; - { - std::lock_guard lock(gMutex); - auto it = gUlthreads.find(ulthread); - if (it == gUlthreads.end()) { - return ULT_ERROR_STATE; - } - state = it->second; - } - void* result = nullptr; - if (scePthreadJoin(state->_thread, &result) != 0) { - return ULT_ERROR_STATE; - } - if (status != nullptr) { - *status = static_cast(reinterpret_cast(result)); - } - std::lock_guard lock(gMutex); - gUlthreads.erase(ulthread); - return ULT_OK; + return joinUlthread(ulthread, status, true); } int APS5_VABI sceUltUlthreadTryJoin(void* ulthread, std::int32_t* status) { - if (ulthread == nullptr) { - return ULT_ERROR_NULL; - } - { - std::lock_guard lock(gMutex); - auto it = gUlthreads.find(ulthread); - if (it == gUlthreads.end()) { - return ULT_ERROR_STATE; - } - if (!it->second->_exited.load(std::memory_order_acquire)) { - return ULT_ERROR_AGAIN; - } - } - return sceUltUlthreadJoin(ulthread, status); + return joinUlthread(ulthread, status, false); } int APS5_VABI sceUltUlthreadGetSelf(void** ulthread) { diff --git a/core/libs/prx/libSceUlt/SceUltTypes.hpp b/core/libs/prx/libSceUlt/SceUltTypes.hpp index 4c77af1649..0534b0f8cb 100644 --- a/core/libs/prx/libSceUlt/SceUltTypes.hpp +++ b/core/libs/prx/libSceUlt/SceUltTypes.hpp @@ -342,6 +342,7 @@ struct UltUlthreadState { Pthread _thread = nullptr; void* _runtime = nullptr; std::atomic _exited{false}; + bool _joining = false; }; #endif diff --git a/core/libs/tests/GuestUltThread.cpp b/core/libs/tests/GuestUltThread.cpp index 50fef67d4c..e3dda88b5e 100644 --- a/core/libs/tests/GuestUltThread.cpp +++ b/core/libs/tests/GuestUltThread.cpp @@ -1,4 +1,6 @@ #include "prx/libSceUlt/include/Api.hpp" +#include "prx/libkernel/include/Api.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" #include "SceTypes.hpp" #include #include @@ -8,6 +10,7 @@ #include #include #include +#include struct UltConditionVariableInfo; struct UltWaitingThreadInfo; @@ -46,6 +49,85 @@ static std::int32_t APS5_VABI Immediate(std::uint64_t arg) { struct alignas(8) Runtime { std::uint8_t bytes[4096]{}; }; struct alignas(8) Thread { std::uint8_t bytes[512]{}; }; +static std::binary_semaphore gTargetStarted{0}; +static std::binary_semaphore gTargetRelease{0}; +static std::binary_semaphore gJoinStart{0}; + +static std::int32_t APS5_VABI GatedTarget(std::uint64_t argument) { + gTargetStarted.release(); + CHECK(gTargetRelease.try_acquire_for(std::chrono::seconds(4))); + return static_cast(argument); +} + +static void* APS5_VABI JoinTarget(void* argument) { + CHECK(gJoinStart.try_acquire_for(std::chrono::seconds(4))); + std::int32_t status = -1; + CHECK(sceUltUlthreadJoin(argument, &status) == Ok && status == 91); + return nullptr; +} + +static void AwaitJoinWait(Pthread caller) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + for (;;) { + { + std::lock_guard lock(caller->cancelLock); + if (caller->cancelWait != nullptr) return; + } + CHECK(std::chrono::steady_clock::now() < deadline); + std::this_thread::yield(); + } +} + +static void CanceledJoin(bool pending) { + Runtime runtime; + Thread thread; + CHECK(sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == Ok); + CHECK(sceUltUlthreadCreate(&thread, "target", GatedTarget, 91, nullptr, 0, &runtime, nullptr, 0) == Ok); + CHECK(gTargetStarted.try_acquire_for(std::chrono::seconds(2))); + Pthread caller = nullptr; + CHECK(scePthreadCreate(&caller, nullptr, JoinTarget, &thread, "joiner") == 0); + if (!pending) { + gJoinStart.release(); + AwaitJoinWait(caller); + } + CHECK(scePthreadCancel(caller) == 0); + if (pending) gJoinStart.release(); + void* canceled = nullptr; + CHECK(scePthreadJoin(caller, &canceled) == 0); + CHECK(canceled == reinterpret_cast(std::uintptr_t{1})); + Pthread replacement = nullptr; + CHECK(scePthreadCreate(&replacement, nullptr, JoinTarget, &thread, "replacement") == 0); + gJoinStart.release(); + AwaitJoinWait(replacement); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == State); + CHECK(sceUltFinalize() == Busy); + gTargetRelease.release(); + CHECK(scePthreadJoin(replacement, nullptr) == 0); + CHECK(sceUltUlthreadJoin(&thread, nullptr) == State); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == Ok); + CHECK(sceUltFinalize() == Ok); +} + +static std::int32_t APS5_VABI SelfJoin(std::uint64_t) { + void* self = nullptr; + CHECK(sceUltUlthreadGetSelf(&self) == Ok); + std::int32_t status = -1; + CHECK(sceUltUlthreadJoin(self, &status) == State && status == -1); + CHECK(sceUltUlthreadTryJoin(self, &status) == Again && status == -1); + return 91; +} + +static void SelfJoin() { + Runtime runtime; + Thread thread; + CHECK(sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == Ok); + CHECK(sceUltUlthreadCreate(&thread, "self", SelfJoin, 0, nullptr, 0, &runtime, nullptr, 0) == Ok); + std::int32_t status = -1; + CHECK(sceUltUlthreadJoin(&thread, &status) == Ok && status == 91); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == Ok); + CHECK(sceUltFinalize() == Ok); +} + static void Outstanding() { Runtime runtime; Thread thread; @@ -276,5 +358,8 @@ int main(int argc, char** argv) { else if (std::strcmp(argv[1], "tryjoin") == 0) TryJoin(); else if (std::strcmp(argv[1], "destroy") == 0) Destroy(); else if (std::strcmp(argv[1], "underscoredcreate") == 0) UnderscoredCreate(); + else if (std::strcmp(argv[1], "cancel-blocked") == 0) CanceledJoin(false); + else if (std::strcmp(argv[1], "cancel-pending") == 0) CanceledJoin(true); + else if (std::strcmp(argv[1], "self-join") == 0) SelfJoin(); else CHECK(false); } diff --git a/core/libs/tests/UltThreadLifecycle.cpp b/core/libs/tests/UltThreadLifecycle.cpp new file mode 100644 index 0000000000..b54ec8325d --- /dev/null +++ b/core/libs/tests/UltThreadLifecycle.cpp @@ -0,0 +1,192 @@ +#include "prx/libSceUlt/include/Api.hpp" +#include "prx/libkernel/include/Api.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void Require(bool value, int line) { + if (value) return; + std::fprintf(stderr, "ULT lifecycle check failed at line %d\n", line); + std::abort(); +} + +#define CHECK(value) Require((value), __LINE__) + +struct alignas(8) Runtime { std::byte bytes[4096]{}; }; +struct alignas(8) Thread { std::byte bytes[512]{}; }; + +struct HostThread { + std::thread thread; + void* result = nullptr; +}; + +std::atomic entryFinished{false}; +std::atomic joinCalls{0}; +std::atomic hostThreads{0}; +std::binary_semaphore joining{0}; +std::binary_semaphore releaseJoin{0}; +bool gateJoin = false; +int createFailure = 0; +int joinFailure = 0; + +void Await(std::binary_semaphore& gate) { + CHECK(gate.try_acquire_for(std::chrono::seconds(2))); +} + +std::int32_t APS5_VABI Entry(std::uint64_t argument) { + void* self = nullptr; + CHECK(sceUltUlthreadGetSelf(&self) == ULT_OK && self != nullptr); + CHECK(sceUltUlthreadYield() == ULT_OK); + return static_cast(argument); +} + +void Create(Runtime& runtime, Thread& thread) { + CHECK(sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == ULT_OK); + CHECK(sceUltUlthreadCreate(&thread, "thread", Entry, 73, nullptr, 0, &runtime, nullptr, 0) == ULT_OK); + entryFinished.wait(false); +} + +void Finish(Runtime& runtime, Thread& thread) { + std::int32_t status = -1; + CHECK(sceUltUlthreadJoin(&thread, &status) == ULT_OK && status == 73); + CHECK(hostThreads.load() == 0); + CHECK(sceUltUlthreadJoin(&thread, &status) == ULT_ERROR_STATE && status == 73); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == ULT_OK); + CHECK(sceUltFinalize() == ULT_OK); +} + +void CreateFailure(int failure) { + Runtime runtime; + Thread thread; + CHECK(sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == ULT_OK); + createFailure = failure; + bool threw = false; + int result = ULT_OK; + try { + result = sceUltUlthreadCreate(&thread, "thread", Entry, 73, nullptr, 0, &runtime, nullptr, 0); + } catch (const std::runtime_error&) { + threw = true; + } + CHECK(threw == (failure != 1)); + if (failure == 1) CHECK(result == ULT_ERROR_AGAIN); + if (failure == 3) { + CHECK(sceUltFinalize() == ULT_ERROR_BUSY); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == ULT_ERROR_STATE); + Finish(runtime, thread); + return; + } + CHECK(hostThreads.load() == 0 && !entryFinished.load()); + CHECK(sceUltUlthreadTryJoin(&thread, nullptr) == ULT_ERROR_STATE); + CHECK(sceUltUlthreadCreate(&thread, "retry", Entry, 73, nullptr, 0, &runtime, nullptr, 0) == ULT_OK); + Finish(runtime, thread); +} + +void Ownership(bool nonblocking) { + Runtime runtime; + Thread thread; + Create(runtime, thread); + gateJoin = true; + std::int32_t firstStatus = -1; + int firstResult = ULT_ERROR_STATE; + std::thread first([&] { + firstResult = nonblocking ? sceUltUlthreadTryJoin(&thread, &firstStatus) : sceUltUlthreadJoin(&thread, &firstStatus); + }); + Await(joining); + std::int32_t secondStatus = -2; + CHECK(sceUltUlthreadJoin(&thread, &secondStatus) == ULT_ERROR_STATE && secondStatus == -2); + CHECK(sceUltUlthreadTryJoin(&thread, &secondStatus) == ULT_ERROR_STATE && secondStatus == -2); + CHECK(joinCalls.load() == 1); + CHECK(sceUltUlthreadCreate(&thread, "replacement", Entry, 74, nullptr, 0, &runtime, nullptr, 0) == ULT_ERROR_STATE); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == ULT_ERROR_STATE); + CHECK(sceUltFinalize() == ULT_ERROR_BUSY); + releaseJoin.release(); + first.join(); + CHECK(firstResult == ULT_OK && firstStatus == 73 && hostThreads.load() == 0); + CHECK(sceUltUlthreadTryJoin(&thread, &secondStatus) == ULT_ERROR_STATE && secondStatus == -2); + gateJoin = false; + entryFinished.store(false); + CHECK(sceUltUlthreadCreate(&thread, "replacement", Entry, 73, nullptr, 0, &runtime, nullptr, 0) == ULT_OK); + Finish(runtime, thread); +} + +void JoinFailure(int failure) { + Runtime runtime; + Thread thread; + Create(runtime, thread); + joinFailure = failure; + std::int32_t status = -1; + bool threw = false; + int result = ULT_OK; + try { + result = sceUltUlthreadJoin(&thread, &status); + } catch (const std::runtime_error&) { + threw = true; + } + CHECK(threw == (failure == 2)); + if (!threw) CHECK(result == ULT_ERROR_STATE); + CHECK(status == -1 && hostThreads.load() == 1); + Finish(runtime, thread); +} + +} + +extern "C" { + +int APS5_VABI scePthreadCreate(Pthread* value, const PthreadAttr*, PthreadEntry entry, void* argument, const char*) { + const int failure = createFailure; + createFailure = 0; + if (failure == 1) return 11; + if (failure == 2) throw std::runtime_error("Injected prepublication creation failure"); + auto state = std::make_unique(); + auto* raw = state.get(); + state->thread = std::thread([=] { + raw->result = entry(argument); + entryFinished.store(true); + entryFinished.notify_all(); + }); + ++hostThreads; + *value = reinterpret_cast(state.release()); + if (failure == 3) throw std::runtime_error("Injected postpublication creation failure"); + return 0; +} + +int APS5_VABI scePthreadJoin(Pthread value, void** result) { + const int call = joinCalls.fetch_add(1); + if (gateJoin) { + if (call != 0) return 22; + joining.release(); + Await(releaseJoin); + } + const int failure = joinFailure; + joinFailure = 0; + if (failure == 1) return 22; + if (failure == 2) throw std::runtime_error("Injected precommit join failure"); + CHECK(value != nullptr); + std::unique_ptr state(reinterpret_cast(value)); + state->thread.join(); + if (result != nullptr) *result = state->result; + --hostThreads; + return 0; +} + +} + +int main(int argc, char** argv) { + CHECK(argc == 2); + if (std::strcmp(argv[1], "create-error") == 0) CreateFailure(1); + else if (std::strcmp(argv[1], "create-exception") == 0) CreateFailure(2); + else if (std::strcmp(argv[1], "published-exception") == 0) CreateFailure(3); + else if (std::strcmp(argv[1], "join-owner") == 0) Ownership(false); + else if (std::strcmp(argv[1], "tryjoin-owner") == 0) Ownership(true); + else if (std::strcmp(argv[1], "join-error") == 0) JoinFailure(1); + else if (std::strcmp(argv[1], "join-exception") == 0) JoinFailure(2); + else CHECK(false); +} diff --git a/docs/dev/debt/libSceUlt.md b/docs/dev/debt/libSceUlt.md index d539ce9772..cba64e972b 100644 --- a/docs/dev/debt/libSceUlt.md +++ b/docs/dev/debt/libSceUlt.md @@ -4,7 +4,8 @@ - [Bound condition variables](../../../core/libs/prx/libSceUlt/Export.cpp) use the creation-time mutex, inferred from PPSA28061 callers of `_sceUltConditionVariableCreate`; the wait caller reloads only the condition argument. Error precedence, busy/state errors, FIFO signal selection and unsupported recursive waits have not been measured on PS5 hardware. Non-null options and waits at recursion depth above one throw. A live condition prevents destroying its bound mutex. Finalization follows the existing host mutex lifecycle: it invalidates conditions and wakes queued or reacquiring waiters with a state error. Scheduling priority and build-version validation are not implemented. -- [libSceUlt ulthreads](../../../core/libs/prx/libSceUlt/Export.cpp) - Runtime option defaults still omit the native header, stack size, worker affinity and priority fields; affinity depends on unresolved process configuration queries. Concurrent join ownership is not modeled: the native runtime distinguishes an ordinary running thread from one with another joiner. Host threads do not use the guest context or worker scheduler. +- [libSceUlt ulthreads](../../../core/libs/prx/libSceUlt/Export.cpp) - Runtime option defaults still omit the native header, stack size, worker affinity and priority fields; affinity depends on unresolved process configuration queries. Host threads do not use the guest context or worker scheduler. Native concurrent-join error values and precedence remain unverified; the host retains running TryJoin `AGAIN` and competing committed-join `STATE` behavior. Guest TLS, cancellation state and cleanup handlers remain host-thread-affine; context migration has no established contract. +- [Ulthread completion](../../../core/libs/prx/libSceUlt/Export.cpp) is published before pthread destructors and native teardown finish, so TryJoin can still block in kernel join. Cancellation cleanup releases a pending join claim, but exceptions after kernel join commits cannot restore native joinability. Creation exceptions before handle publication remove the registration; after publication they retain it because a runner may already use its state. These are host lifecycle guarantees, not console measurements. - [libSceUlt queues](../../../core/libs/prx/libSceUlt/Export.cpp) - Queue-data pool destruction and shared slot limits still require comparison with the native node allocator. - [`sceUltConditionVariableCreate` and `sceUltConditionVariableOptParamInitialize`](../../../core/libs/prx/libSceUlt/Export.cpp) (libSceUlt) are absent from the console export list, where only the underscored variants `_sceUltConditionVariableCreate` and `_sceUltConditionVariableOptParamInitialize` appear; their non-underscored signatures are assumed. Condition variable option parameter initialization only zeroes 128 bytes, and its layout without the common header remains unverified on hardware. - [Ult factory and creation exports](../../../core/libs/prx/libSceUlt/Export.cpp) without a leading underscore are absent from the supplied module; their signatures remain unresolved. Diagnostic signatures and synchronization argument lists are recovered. From 2388d8a11a689941e4fb4ac6c23a906a6dd8148c Mon Sep 17 00:00:00 2001 From: Felipebmaitan Date: Sun, 25 Oct 2026 02:17:46 -0300 Subject: [PATCH 116/318] fix(ult): roll back cleanup setup and exercise registry races --- core/libs/prx/libSceUlt/CMakeLists.txt | 11 +- core/libs/prx/libSceUlt/Export.cpp | 11 +- core/libs/tests/UltThreadLifecycle.cpp | 155 +++++++++++++++++++++++++ docs/dev/debt/libSceUlt.md | 2 +- 4 files changed, 172 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceUlt/CMakeLists.txt b/core/libs/prx/libSceUlt/CMakeLists.txt index e1fd7666dd..ba901396bd 100644 --- a/core/libs/prx/libSceUlt/CMakeLists.txt +++ b/core/libs/prx/libSceUlt/CMakeLists.txt @@ -3,11 +3,18 @@ target_include_directories(libSceUlt PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceUlt) if(BUILD_TESTING) - add_test_executable(guest_ult_thread_lifecycle_tests ${LIBS_INCLUDE_DIR}/tests/UltThreadLifecycle.cpp Export.cpp src/ReaderWriterLock.cpp) + add_test_executable(guest_ult_thread_lifecycle_tests ${LIBS_INCLUDE_DIR}/tests/UltThreadLifecycle.cpp src/ReaderWriterLock.cpp) target_include_directories(guest_ult_thread_lifecycle_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_ult_thread_lifecycle_tests PRIVATE libkernel libc) configure_windows_unwind(guest_ult_thread_lifecycle_tests) - foreach(testCase IN ITEMS create-error create-exception published-exception join-owner tryjoin-owner join-error join-exception) + set(ultLifecycleCases create-error create-exception published-exception join-owner tryjoin-owner join-error join-exception) + if(NOT APPLE AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_definitions(guest_ult_thread_lifecycle_tests PRIVATE ULT_LIFECYCLE_INTERPOSE=1) + target_link_options(guest_ult_thread_lifecycle_tests PRIVATE + LINKER:--wrap=pthread_mutex_lock LINKER:--wrap=pthread_mutex_unlock LINKER:--wrap=_ZN12ThreadCancel11CleanupMarkEv) + list(APPEND ultLifecycleCases publication generation cleanup-setup) + endif() + foreach(testCase IN LISTS ultLifecycleCases) add_test(NAME guest_ult_thread_lifecycle_${testCase} COMMAND guest_ult_thread_lifecycle_tests ${testCase}) set_tests_properties(guest_ult_thread_lifecycle_${testCase} PROPERTIES TIMEOUT 10) if(WIN32) diff --git a/core/libs/prx/libSceUlt/Export.cpp b/core/libs/prx/libSceUlt/Export.cpp index 6e9a414b6b..65edfb4e65 100644 --- a/core/libs/prx/libSceUlt/Export.cpp +++ b/core/libs/prx/libSceUlt/Export.cpp @@ -150,14 +150,17 @@ int joinUlthread(void* ulthread, std::int32_t* status, bool blocking) { state->_joining = true; } CleanupHandler cleanup{}; - void* const mark = ThreadCancel::CleanupMark(); - __pthread_cleanup_push_imp_nid_postfix(releaseUlthreadJoin, &state, &cleanup); + void* mark = nullptr; + bool cleanupRegistered = false; void* result = nullptr; int joined; try { + mark = ThreadCancel::CleanupMark(); + __pthread_cleanup_push_imp_nid_postfix(releaseUlthreadJoin, &state, &cleanup); + cleanupRegistered = true; joined = scePthreadJoin(state->_thread, &result); } catch (...) { - ThreadCancel::CleanupRestore(mark); + if (cleanupRegistered) ThreadCancel::CleanupRestore(mark); releaseUlthreadJoin(&state); throw; } @@ -850,9 +853,9 @@ int APS5_VABI sceUltUlthreadCreate(void* ulthread, const char* name, UltUlthread state->_arg = arg; state->_object = ulthread; state->_runtime = runtime; - const std::string threadName = name != nullptr ? std::string(name).substr(0, 31) : std::string(); std::lock_guard lock(gMutex); if (!gRuntimes.contains(runtime) || gUlthreads.contains(ulthread)) return ULT_ERROR_STATE; + const std::string threadName = name != nullptr ? std::string(name).substr(0, 31) : std::string(); gUlthreads.emplace(ulthread, state); std::memset(ulthread, 0, 512); try { diff --git a/core/libs/tests/UltThreadLifecycle.cpp b/core/libs/tests/UltThreadLifecycle.cpp index b54ec8325d..ebc654adcb 100644 --- a/core/libs/tests/UltThreadLifecycle.cpp +++ b/core/libs/tests/UltThreadLifecycle.cpp @@ -9,6 +9,7 @@ #include #include #include +#include "../prx/libSceUlt/Export.cpp" namespace { @@ -26,6 +27,7 @@ struct alignas(8) Thread { std::byte bytes[512]{}; }; struct HostThread { std::thread thread; void* result = nullptr; + std::uint64_t generation = 0; }; std::atomic entryFinished{false}; @@ -37,6 +39,22 @@ bool gateJoin = false; int createFailure = 0; int joinFailure = 0; +#ifdef ULT_LIFECYCLE_INTERPOSE +thread_local int actor = 0; +std::atomic pauseSelection{false}; +std::atomic failCleanupSetup{false}; +std::atomic originalGeneration{0}; +std::binary_semaphore selected{0}; +std::binary_semaphore releaseSelection{0}; +std::binary_semaphore creating{0}; +std::binary_semaphore releaseCreate{0}; +std::binary_semaphore registryAttempt{0}; +std::binary_semaphore releaseRegistryAttempt{0}; +std::binary_semaphore replacementStarted{0}; +std::binary_semaphore releaseReplacement{0}; +bool gateCreate = false; +#endif + void Await(std::binary_semaphore& gate) { CHECK(gate.try_acquire_for(std::chrono::seconds(2))); } @@ -136,16 +154,140 @@ void JoinFailure(int failure) { Finish(runtime, thread); } +#ifdef ULT_LIFECYCLE_INTERPOSE +void Publication() { + Runtime runtime; + Thread thread; + CHECK(sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == ULT_OK); + gateCreate = true; + std::thread creator([&] { + CHECK(sceUltUlthreadCreate(&thread, "thread", Entry, 73, nullptr, 0, &runtime, nullptr, 0) == ULT_OK); + }); + Await(creating); + std::int32_t status = -1; + std::thread joiner([&] { + actor = 2; + CHECK(sceUltUlthreadJoin(&thread, &status) == ULT_OK); + }); + Await(registryAttempt); + const bool registryAvailable = gMutex.try_lock(); + if (registryAvailable) gMutex.unlock(); + CHECK(!registryAvailable && joinCalls.load() == 0); + releaseRegistryAttempt.release(); + releaseCreate.release(); + creator.join(); + joiner.join(); + CHECK(status == 73 && hostThreads.load() == 0); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == ULT_OK); + CHECK(sceUltFinalize() == ULT_OK); +} + +std::int32_t APS5_VABI Replacement(std::uint64_t argument) { + replacementStarted.release(); + Await(releaseReplacement); + return static_cast(argument); +} + +void Generation() { + Runtime runtime; + Thread thread; + Create(runtime, thread); + pauseSelection.store(true); + std::int32_t originalStatus = -1; + int originalResult = ULT_ERROR_STATE; + std::thread original([&] { + actor = 1; + originalResult = sceUltUlthreadTryJoin(&thread, &originalStatus); + }); + Await(selected); + std::int32_t contenderStatus = -2; + const int contender = sceUltUlthreadJoin(&thread, &contenderStatus); + if (contender == ULT_OK) { + CHECK(contenderStatus == 73); + CHECK(sceUltUlthreadCreate(&thread, "replacement", Replacement, 92, nullptr, 0, &runtime, nullptr, 0) == ULT_OK); + Await(replacementStarted); + } else { + CHECK(contender == ULT_ERROR_STATE && contenderStatus == -2); + CHECK(sceUltUlthreadCreate(&thread, "replacement", Replacement, 92, nullptr, 0, &runtime, nullptr, 0) == ULT_ERROR_STATE); + } + releaseSelection.release(); + original.join(); + CHECK(originalGeneration.load() == 73 && originalResult == ULT_OK && originalStatus == 73); + CHECK(contender == ULT_ERROR_STATE && hostThreads.load() == 0); + CHECK(sceUltUlthreadCreate(&thread, "replacement", Replacement, 92, nullptr, 0, &runtime, nullptr, 0) == ULT_OK); + Await(replacementStarted); + CHECK(sceUltUlthreadTryJoin(&thread, &contenderStatus) == ULT_ERROR_AGAIN && contenderStatus == -2); + releaseReplacement.release(); + CHECK(sceUltUlthreadJoin(&thread, &contenderStatus) == ULT_OK && contenderStatus == 92); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == ULT_OK); + CHECK(sceUltFinalize() == ULT_OK); +} + +void CleanupSetup() { + Runtime runtime; + Thread thread; + Create(runtime, thread); + failCleanupSetup.store(true); + std::int32_t status = -1; + bool threw = false; + try { + sceUltUlthreadJoin(&thread, &status); + } catch (const std::runtime_error&) { + threw = true; + } + CHECK(threw && status == -1 && joinCalls.load() == 0); + std::thread replacement([&] { Finish(runtime, thread); }); + replacement.join(); +} +#endif + } extern "C" { +#ifdef ULT_LIFECYCLE_INTERPOSE +int __real_pthread_mutex_lock(pthread_mutex_t* mutex); +int __real_pthread_mutex_unlock(pthread_mutex_t* mutex); +void* __real__ZN12ThreadCancel11CleanupMarkEv(); + +int __wrap_pthread_mutex_lock(pthread_mutex_t* mutex) { + if (actor == 2 && mutex == gMutex.native_handle()) { + actor = 0; + registryAttempt.release(); + Await(releaseRegistryAttempt); + } + return __real_pthread_mutex_lock(mutex); +} + +int __wrap_pthread_mutex_unlock(pthread_mutex_t* mutex) { + const int result = __real_pthread_mutex_unlock(mutex); + if (actor == 1 && mutex == gMutex.native_handle() && pauseSelection.exchange(false)) { + selected.release(); + Await(releaseSelection); + } + return result; +} + +void* __wrap__ZN12ThreadCancel11CleanupMarkEv() { + if (failCleanupSetup.exchange(false)) throw std::runtime_error("Injected cleanup setup failure"); + return __real__ZN12ThreadCancel11CleanupMarkEv(); +} +#endif + int APS5_VABI scePthreadCreate(Pthread* value, const PthreadAttr*, PthreadEntry entry, void* argument, const char*) { const int failure = createFailure; createFailure = 0; if (failure == 1) return 11; if (failure == 2) throw std::runtime_error("Injected prepublication creation failure"); +#ifdef ULT_LIFECYCLE_INTERPOSE + if (gateCreate) { + gateCreate = false; + creating.release(); + Await(releaseCreate); + } +#endif auto state = std::make_unique(); + state->generation = static_cast(argument)->_arg; auto* raw = state.get(); state->thread = std::thread([=] { raw->result = entry(argument); @@ -160,6 +302,14 @@ int APS5_VABI scePthreadCreate(Pthread* value, const PthreadAttr*, PthreadEntry int APS5_VABI scePthreadJoin(Pthread value, void** result) { const int call = joinCalls.fetch_add(1); +#ifdef ULT_LIFECYCLE_INTERPOSE + if (actor == 1) { + CHECK(value != nullptr); + const auto generation = reinterpret_cast(value)->generation; + originalGeneration.store(generation); + if (generation != 73) return 22; + } +#endif if (gateJoin) { if (call != 0) return 22; joining.release(); @@ -188,5 +338,10 @@ int main(int argc, char** argv) { else if (std::strcmp(argv[1], "tryjoin-owner") == 0) Ownership(true); else if (std::strcmp(argv[1], "join-error") == 0) JoinFailure(1); else if (std::strcmp(argv[1], "join-exception") == 0) JoinFailure(2); +#ifdef ULT_LIFECYCLE_INTERPOSE + else if (std::strcmp(argv[1], "publication") == 0) Publication(); + else if (std::strcmp(argv[1], "generation") == 0) Generation(); + else if (std::strcmp(argv[1], "cleanup-setup") == 0) CleanupSetup(); +#endif else CHECK(false); } diff --git a/docs/dev/debt/libSceUlt.md b/docs/dev/debt/libSceUlt.md index cba64e972b..3fe232a93c 100644 --- a/docs/dev/debt/libSceUlt.md +++ b/docs/dev/debt/libSceUlt.md @@ -5,7 +5,7 @@ - [Bound condition variables](../../../core/libs/prx/libSceUlt/Export.cpp) use the creation-time mutex, inferred from PPSA28061 callers of `_sceUltConditionVariableCreate`; the wait caller reloads only the condition argument. Error precedence, busy/state errors, FIFO signal selection and unsupported recursive waits have not been measured on PS5 hardware. Non-null options and waits at recursion depth above one throw. A live condition prevents destroying its bound mutex. Finalization follows the existing host mutex lifecycle: it invalidates conditions and wakes queued or reacquiring waiters with a state error. Scheduling priority and build-version validation are not implemented. - [libSceUlt ulthreads](../../../core/libs/prx/libSceUlt/Export.cpp) - Runtime option defaults still omit the native header, stack size, worker affinity and priority fields; affinity depends on unresolved process configuration queries. Host threads do not use the guest context or worker scheduler. Native concurrent-join error values and precedence remain unverified; the host retains running TryJoin `AGAIN` and competing committed-join `STATE` behavior. Guest TLS, cancellation state and cleanup handlers remain host-thread-affine; context migration has no established contract. -- [Ulthread completion](../../../core/libs/prx/libSceUlt/Export.cpp) is published before pthread destructors and native teardown finish, so TryJoin can still block in kernel join. Cancellation cleanup releases a pending join claim, but exceptions after kernel join commits cannot restore native joinability. Creation exceptions before handle publication remove the registration; after publication they retain it because a runner may already use its state. These are host lifecycle guarantees, not console measurements. +- [Ulthread completion](../../../core/libs/prx/libSceUlt/Export.cpp) is published before pthread destructors and native teardown finish, so TryJoin can still block in kernel join. ULT releases its join claim on cancellation or exception, but kernel wait allocation can leave native join ownership and wait registration stranded before commitment; exceptions after commitment cannot restore native joinability either. Mocked precommit failures test only ULT rollback. Creation exceptions before handle publication remove the registration; after publication they retain it because a runner may already use its state. These are host lifecycle guarantees, not console measurements. - [libSceUlt queues](../../../core/libs/prx/libSceUlt/Export.cpp) - Queue-data pool destruction and shared slot limits still require comparison with the native node allocator. - [`sceUltConditionVariableCreate` and `sceUltConditionVariableOptParamInitialize`](../../../core/libs/prx/libSceUlt/Export.cpp) (libSceUlt) are absent from the console export list, where only the underscored variants `_sceUltConditionVariableCreate` and `_sceUltConditionVariableOptParamInitialize` appear; their non-underscored signatures are assumed. Condition variable option parameter initialization only zeroes 128 bytes, and its layout without the common header remains unverified on hardware. - [Ult factory and creation exports](../../../core/libs/prx/libSceUlt/Export.cpp) without a leading underscore are absent from the supplied module; their signatures remain unresolved. Diagnostic signatures and synchronization argument lists are recovered. From 7385907173648e9f6c9f12d72decc8a718ab5ce4 Mon Sep 17 00:00:00 2001 From: Felipebmaitan Date: Sun, 25 Oct 2026 02:37:23 -0300 Subject: [PATCH 117/318] fix(ult): use patchable cleanup and retain consumed handler state --- core/libs/prx/libSceUlt/CMakeLists.txt | 4 ++-- core/libs/prx/libSceUlt/Export.cpp | 11 +++++----- core/libs/tests/UltThreadLifecycle.cpp | 28 +++++++++++++++++++++----- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/core/libs/prx/libSceUlt/CMakeLists.txt b/core/libs/prx/libSceUlt/CMakeLists.txt index ba901396bd..4718f1657d 100644 --- a/core/libs/prx/libSceUlt/CMakeLists.txt +++ b/core/libs/prx/libSceUlt/CMakeLists.txt @@ -7,11 +7,11 @@ if(BUILD_TESTING) target_include_directories(guest_ult_thread_lifecycle_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_ult_thread_lifecycle_tests PRIVATE libkernel libc) configure_windows_unwind(guest_ult_thread_lifecycle_tests) - set(ultLifecycleCases create-error create-exception published-exception join-owner tryjoin-owner join-error join-exception) + set(ultLifecycleCases create-error create-exception published-exception join-owner tryjoin-owner join-error join-exception join-cleanup-exception) if(NOT APPLE AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") target_compile_definitions(guest_ult_thread_lifecycle_tests PRIVATE ULT_LIFECYCLE_INTERPOSE=1) target_link_options(guest_ult_thread_lifecycle_tests PRIVATE - LINKER:--wrap=pthread_mutex_lock LINKER:--wrap=pthread_mutex_unlock LINKER:--wrap=_ZN12ThreadCancel11CleanupMarkEv) + LINKER:--wrap=pthread_mutex_lock LINKER:--wrap=pthread_mutex_unlock LINKER:--wrap=__pthread_cleanup_push_imp_nid_postfix) list(APPEND ultLifecycleCases publication generation cleanup-setup) endif() foreach(testCase IN LISTS ultLifecycleCases) diff --git a/core/libs/prx/libSceUlt/Export.cpp b/core/libs/prx/libSceUlt/Export.cpp index 65edfb4e65..77431e20d6 100644 --- a/core/libs/prx/libSceUlt/Export.cpp +++ b/core/libs/prx/libSceUlt/Export.cpp @@ -130,6 +130,7 @@ void* APS5_VABI ulthreadRunner(void* arg) { void APS5_VABI releaseUlthreadJoin(void* argument) { auto& state = *static_cast*>(argument); + if (!state) return; { std::lock_guard lock(gMutex); state->_joining = false; @@ -150,21 +151,21 @@ int joinUlthread(void* ulthread, std::int32_t* status, bool blocking) { state->_joining = true; } CleanupHandler cleanup{}; - void* mark = nullptr; bool cleanupRegistered = false; void* result = nullptr; int joined; try { - mark = ThreadCancel::CleanupMark(); __pthread_cleanup_push_imp_nid_postfix(releaseUlthreadJoin, &state, &cleanup); cleanupRegistered = true; joined = scePthreadJoin(state->_thread, &result); } catch (...) { - if (cleanupRegistered) ThreadCancel::CleanupRestore(mark); - releaseUlthreadJoin(&state); + if (state) { + if (cleanupRegistered) __pthread_cleanup_pop_imp_nid_postfix(1); + else releaseUlthreadJoin(&state); + } throw; } - ThreadCancel::CleanupRestore(mark); + __pthread_cleanup_pop_imp_nid_postfix(0); if (joined != 0) { releaseUlthreadJoin(&state); return ULT_ERROR_STATE; diff --git a/core/libs/tests/UltThreadLifecycle.cpp b/core/libs/tests/UltThreadLifecycle.cpp index ebc654adcb..c111d2ce48 100644 --- a/core/libs/tests/UltThreadLifecycle.cpp +++ b/core/libs/tests/UltThreadLifecycle.cpp @@ -1,5 +1,6 @@ #include "prx/libSceUlt/include/Api.hpp" #include "prx/libkernel/include/Api.hpp" +#include "prx/libkernel/Pthread/include/Cancel.hpp" #include #include #include @@ -38,6 +39,11 @@ std::binary_semaphore releaseJoin{0}; bool gateJoin = false; int createFailure = 0; int joinFailure = 0; +int sentinelExecutions = 0; + +void APS5_VABI Sentinel(void*) { + ++sentinelExecutions; +} #ifdef ULT_LIFECYCLE_INTERPOSE thread_local int actor = 0; @@ -139,6 +145,8 @@ void JoinFailure(int failure) { Runtime runtime; Thread thread; Create(runtime, thread); + void* const before = ThreadCancel::CleanupMark(); + if (failure == 3) pthread_cleanup_push_nid_postfix(Sentinel, nullptr); joinFailure = failure; std::int32_t status = -1; bool threw = false; @@ -148,10 +156,14 @@ void JoinFailure(int failure) { } catch (const std::runtime_error&) { threw = true; } - CHECK(threw == (failure == 2)); + CHECK(threw == (failure >= 2)); if (!threw) CHECK(result == ULT_ERROR_STATE); CHECK(status == -1 && hostThreads.load() == 1); - Finish(runtime, thread); + if (failure == 3) { + CHECK(sentinelExecutions == 1 && ThreadCancel::CleanupMark() == before); + std::thread replacement([&] { Finish(runtime, thread); }); + replacement.join(); + } else Finish(runtime, thread); } #ifdef ULT_LIFECYCLE_INTERPOSE @@ -248,7 +260,7 @@ extern "C" { #ifdef ULT_LIFECYCLE_INTERPOSE int __real_pthread_mutex_lock(pthread_mutex_t* mutex); int __real_pthread_mutex_unlock(pthread_mutex_t* mutex); -void* __real__ZN12ThreadCancel11CleanupMarkEv(); +void APS5_VABI __real___pthread_cleanup_push_imp_nid_postfix(void (APS5_VABI* routine)(void*), void* argument, void* info); int __wrap_pthread_mutex_lock(pthread_mutex_t* mutex) { if (actor == 2 && mutex == gMutex.native_handle()) { @@ -268,9 +280,9 @@ int __wrap_pthread_mutex_unlock(pthread_mutex_t* mutex) { return result; } -void* __wrap__ZN12ThreadCancel11CleanupMarkEv() { +void APS5_VABI __wrap___pthread_cleanup_push_imp_nid_postfix(void (APS5_VABI* routine)(void*), void* argument, void* info) { if (failCleanupSetup.exchange(false)) throw std::runtime_error("Injected cleanup setup failure"); - return __real__ZN12ThreadCancel11CleanupMarkEv(); + __real___pthread_cleanup_push_imp_nid_postfix(routine, argument, info); } #endif @@ -319,6 +331,11 @@ int APS5_VABI scePthreadJoin(Pthread value, void** result) { joinFailure = 0; if (failure == 1) return 22; if (failure == 2) throw std::runtime_error("Injected precommit join failure"); + if (failure == 3) { + __pthread_cleanup_pop_imp_nid_postfix(1); + __pthread_cleanup_pop_imp_nid_postfix(1); + throw std::runtime_error("Injected failure after cancellation cleanup"); + } CHECK(value != nullptr); std::unique_ptr state(reinterpret_cast(value)); state->thread.join(); @@ -338,6 +355,7 @@ int main(int argc, char** argv) { else if (std::strcmp(argv[1], "tryjoin-owner") == 0) Ownership(true); else if (std::strcmp(argv[1], "join-error") == 0) JoinFailure(1); else if (std::strcmp(argv[1], "join-exception") == 0) JoinFailure(2); + else if (std::strcmp(argv[1], "join-cleanup-exception") == 0) JoinFailure(3); #ifdef ULT_LIFECYCLE_INTERPOSE else if (std::strcmp(argv[1], "publication") == 0) Publication(); else if (std::strcmp(argv[1], "generation") == 0) Generation(); From 38288a49f61648cb4670178ca70c75dc8d1bfb78 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:49:28 +0000 Subject: [PATCH 118/318] fix(agc): bind a written TYPE 0 or 1 static image slot as a null storage image --- .../src/Driver/Memory/WrittenBuffers.cpp | 2 +- .../tests/GuestTextureResource.cpp | 3 ++ .../libSceAgcDriver/tests/ShaderMemory.cpp | 38 ++++++++++++++----- .../tests/execution/DrawWriteRanges.cpp | 17 +++++++++ .../Optimization/src/ResourceMaterializer.cpp | 2 +- docs/dev/debt/libSceAgcDriver.md | 2 +- 6 files changed, 51 insertions(+), 13 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index 58fa8648a3..f78422cef4 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -113,7 +113,7 @@ std::vector> DrawWriteRanges(const Graph const bool written = element >= binding.imageWritten.size() || binding.imageWritten[element]; if (!written || binding.guestDescriptor.size() < (static_cast(element) + 1) * 8) continue; const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8, 8); - if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; + if (Graphics::IsNullTextureResource(words)) continue; const auto resource = Graphics::DecodeTextureResource(words); if (resource.baseAddress == 0) continue; const auto surfaceBytes = Graphics::DescribeSurface(resource).guestBytes; diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp index ab7db93b7a..efb04d47ac 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp @@ -155,6 +155,9 @@ void RunGuestTextureResourceTests() { Require(IsNullTextureResource(pack(nullType)), "a type-1 texture with a nonzero base was not classified as null"); nullType.typeRaw = 2; Require(!IsNullTextureResource(pack(nullType)), "a reserved type-2 texture was classified as null"); + const std::array reportedStorage{0x04efd000u, 0x00040011u, 0x00017a00u, 0x00005204u, 0x0af00000u, 0u, 0x00000612u, 0x00fff000u}; + Require(IsNullTextureResource(reportedStorage), "the type-0 words of a written storage slot were not classified as null"); + reject([&] { static_cast(DecodeTextureResource(reportedStorage)); }, "guest texture descriptor"); auto result = DecodeTextureResource(pack(base)); Require(result.baseAddress == (base.base40 << 8u), "decoded base address changed"); Require(result.width == 16 && result.height == 16, "decoded width or height changed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 4d7e4dc551..e1a2e7bc57 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1075,7 +1075,7 @@ void verifySignedSrtComparison() { void verifyTypeZeroImageMaterialization() { using namespace ShaderRecompiler; - for (const std::uint32_t type : {0u, 1u}) { + for (const std::uint32_t type : {0u, 1u, 2u}) { IrResourcePlan plan; plan.srtPlanComplete = true; plan.resourceTrackingComplete = true; @@ -1103,19 +1103,37 @@ void verifyTypeZeroImageMaterialization() { plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); ResourceSnapshot snapshot; - ResourceMaterializer{}.Materialize(plan, SrtRuntime{}, snapshot); - require(snapshot.images.size() == 1u && snapshot.images[0].dwordCount == 8u && snapshot.images[0].dwords == words, - type == 0u ? "TYPE=0 image descriptor with a nonzero base was rejected or altered during materialization" : "TYPE=1 image descriptor with a nonzero base was rejected or altered during materialization"); + const auto materialize = [&] { ResourceMaterializer{}.Materialize(plan, SrtRuntime{}, snapshot); }; + if (type == 2u) { + expectFailure(materialize, "runtime image descriptor is invalid", "a reserved TYPE=2 read-only image descriptor was accepted"); + } else { + materialize(); + require(snapshot.images.size() == 1u && snapshot.images[0].dwordCount == 8u && snapshot.images[0].dwords == words, + type == 0u ? "TYPE=0 image descriptor with a nonzero base was rejected or altered during materialization" : "TYPE=1 image descriptor with a nonzero base was rejected or altered during materialization"); + } - for (const bool atomic : {false, true}) { + struct Access { + bool read; + bool written; + bool atomic; + }; + for (const auto access : {Access{false, true, false}, Access{true, true, false}, Access{false, false, true}, Access{false, true, true}}) { image.resourceClass = ImageResourceClass::Storage; - image.read = false; - image.written = !atomic; - image.atomic = atomic; + image.read = access.read; + image.written = access.written; + image.atomic = access.atomic; plan.info.images[0] = image; plan.info.runtimeImageModes[0] = ResourceMaterializer::RuntimeImageModes(image); - expectFailure([&] { ResourceMaterializer{}.Materialize(plan, SrtRuntime{}, snapshot); }, "runtime image descriptor is invalid", - atomic ? "TYPE=0 or 1 atomic image descriptor was accepted without hardware evidence" : "TYPE=0 or 1 writable image descriptor was accepted without hardware evidence"); + if (type == 2u) { + expectFailure(materialize, "runtime image descriptor is invalid", "a reserved TYPE=2 storage image descriptor was accepted"); + } else if (access.atomic) { + expectFailure(materialize, "runtime image descriptor is invalid", "TYPE=0 or 1 atomic image descriptor was accepted without hardware evidence"); + } else { + snapshot = {}; + materialize(); + require(snapshot.images.size() == 1u && snapshot.images[0].dwordCount == 8u && snapshot.images[0].dwords == words, + access.read ? "TYPE=0 or 1 read-write storage image descriptor was not materialized as a null storage image" : "TYPE=0 or 1 written storage image descriptor was not materialized as a null storage image"); + } } } } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp index d45a4cc045..a9be10492e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp @@ -87,6 +87,23 @@ int main() { Require(bounded, "a dispatch whose written images are a real one and a null one is not bounded"); Require(Contains(dispatchRanges, written, written + surfaceBytes), "a written image next to a null one is not a dispatch write range"); Require(!Touches(dispatchRanges, 0, surfaceBytes), "a null storage image is a dispatch write range"); + auto typeZero = ImageWords(0x500000000ull, 256, 128); + typeZero[3] &= 0x0fffffffu; + auto typeOne = typeZero; + typeOne[3] |= 1u << 28u; + const std::array reported{0x04efd000u, 0x00040011u, 0x00017a00u, 0x00005204u, 0x0af00000u, 0u, 0x00000612u, 0x00fff000u}; + for (const auto& [words, base] : {std::pair{typeZero, 0x500000000ull}, std::pair{typeOne, 0x500000000ull}, std::pair{reported, 0x1104efd00000ull}}) { + std::copy(words.begin(), words.end(), program.bindings[0].guestDescriptor.begin() + 8); + for (const bool exact : {false, true}) { + const auto ranges = DrawWriteRanges(state, stages, exact); + Require(Contains(ranges, written, written + surfaceBytes), "a written image next to a TYPE 0 or 1 one is not a write range"); + Require(!Touches(ranges, base, base + surfaceBytes), "a TYPE 0 or 1 storage image is a write range"); + } + bool typedBounded = false; + const auto typedRanges = AgcDriver::DriverDetail::DispatchWriteRanges(program, typedBounded); + Require(typedBounded && Contains(typedRanges, written, written + surfaceBytes), "a dispatch whose written images are a real one and a TYPE 0 or 1 one lost the real one"); + Require(!Touches(typedRanges, base, base + surfaceBytes), "a TYPE 0 or 1 storage image is a dispatch write range"); + } AgcDriver::Graphics::State targets{}; targets.hasColorTarget = true; auto& color = targets.color; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 921b9bb653..e0b0e17b25 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -82,7 +82,7 @@ ImageType rawImageType(const DescriptorValue& descriptor) { bool nullImageDescriptor(const ImageResource& image, const DescriptorValue& descriptor) { if (nullImageDescriptor(descriptor)) return true; - return static_cast(rawImageType(descriptor)) <= 1u && image.table == NoTable && image.read && !image.written && !image.atomic; + return static_cast(rawImageType(descriptor)) <= 1u && image.table == NoTable && (image.read || image.written) && !image.atomic; } IrBufferFormat rawImageFormat(const DescriptorValue& descriptor) { diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 608bedf97c..602e104ecc 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -85,7 +85,7 @@ - [Initial stencil state](../../../core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp) uses zero for `DB_STENCIL_CONTROL`, `DB_STENCILREFMASK` and `DB_STENCILREFMASK_BF` (context `0x10b`-`0x10d`), including after `CLEAR_STATE`. The public V0, V4, V5 and V7-V11 [register-default tables](../../../core/libs/prx/libSceAgc/Command/include/RegisterDefaultsData.hpp) supply these values. Linux's [GFX10 clear-state table](https://github.com/torvalds/linux/blob/master/drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h) agrees on the control register but uses `0x01000000` for the reference masks; the console reset values have not been measured. Planet Coaster (PPSA01736 v01.017) enables stencil tests without writing the control register. - [Multisampled DCC comp-to-single clears](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) (AGC driver) - RADV's multisampled comp-to-single clear dispatch ([`radv_clear_dcc_comp_to_single`](https://github.com/mihawk-99/PS5_Mesa/blob/1ba9874f478657ccf6da72ce55f0260164cc0bd0/src/amd/vulkan/meta/radv_meta_clear.c#L954)) is recognized by its code and not run. Its clear color (user data 5 to 8, which the shader stores as the four dwords of sample 0's texel) is recorded for the surface its image descriptor names, and DCC keys of 0x10 on a multisampled target of that surface, pipe-bank XOR, size and sample count import as a clear to it, for texels of 8 to 128 bits. A record is dropped when an unmap or allocation removal covers its surface's base. The guest's color memory doesn't get the color at the blocks' first pixels, and the recorded color is assumed to be the one the keys refer to: RADV writes 0x10 keys only in a fast clear that runs this dispatch for the same image right after, and initializes DCC to 0x00 or 0xff on an image's first use ([`radv_init_color_image_metadata`](https://github.com/mihawk-99/PS5_Mesa/blob/1ba9874f478657ccf6da72ce55f0260164cc0bd0/src/amd/vulkan/radv_cmd_buffer.c#L16894)). A new image on memory reused within one mapping, with the same address, XOR, size and sample count, whose clear runs a comp-to-single shader other than the recognized one, would still import the old color. - [Initial depth and stencil planes](../../../core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp) - the console's reset values of `DB_Z_INFO` and `DB_STENCIL_INFO` (context `0x010`-`0x011`) are not measured. Both start, and return after `CLEAR_STATE`, at zero (formats `INVALID`), the value of Linux's [GFX10 clear-state table](https://github.com/torvalds/linux/blob/master/drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h). The public [register-default tables](../../../core/libs/prx/libSceAgc/Command/include/RegisterDefaultsData.hpp) write `0x80000180` and `0x20000180` instead, also with formats `INVALID`; of their other bits, only `TILE_STENCIL_DISABLE` would change a result: the HTILE layout of a multisampled depth plane whose `DB_STENCIL_INFO` was never written. -- [Type-0 and type-1 texture descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (`TYPE` = 0 or 1 in dword 3) with a nonzero base are bound as null images, like a zero base. TYPE 0 comes from a gfx1036 (RDNA2) measurement in issue #4139 (image sample, load and resinfo return zero, no fault). TYPE 1 comes from a gfx1010 (RDNA1) measurement in #4139 and #5828 (`image_sample_lz`, `image_sample_b`, `image_load` and `image_get_resinfo` return zero, no fault) with format 22 only; TYPE 1 is not measured on RDNA2, with other formats or with depth-compare reads (`image_sample_c*`). Neither type was measured on the console's gfx1013. +- [Type-0 and type-1 texture descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (`TYPE` = 0 or 1 in dword 3) with a nonzero base are bound as null images, like a zero base, in [static slots](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) with no image table that are read or written; atomic slots throw. Reads are measured: TYPE 0 on gfx1036 (RDNA2) in #4139 (sample, load and resinfo return zero, no fault), TYPE 1 on gfx1010 (RDNA1) in #4139 and #5828 (`image_sample_lz`, `image_sample_b`, `image_load` and `image_get_resinfo`) with format 22 only, not on RDNA2, with other formats or with `image_sample_c*`. Stores are not: a written slot is a null storage image, whose stores `nullDescriptor` discards, pending [`probe_type0_store.py`](https://github.com/EagleBearerYS/AnyPS5/tree/test/hw-oracle-type0-store/tools/hw-oracle) on `test/hw-oracle-type0-store`. Neither type was measured on the console's gfx1013. - [Multisampled color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) lose their guest-read guards and cannot be reused after a successful allocation removal or unmap covers their full guarded range. The next target, sampled or storage lookup removes retired cache owners; recorded work and sampled views keep their images alive until their last reference is released. Partial unmaps keep the guard; partial allocation reuse and guest access to live multisampled data remain unsupported. - [Stencil reference export replay](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) on devices without `VK_EXT_shader_stencil_export` replays the low eight bits of the values proved from constant, select and phi exports, with an export proved on every kept path. An export the prover cannot bound replays all 256 eight-bit values, one pass per value per triangle (logged once per process). It records instances, then their triangles, then the reference classes of each triangle. One pipeline filters fragments at shader termination against a dynamic push value before late stencil and depth tests. The [Vulkan stencil-reference replacement rule](https://docs.vulkan.org/refpages/latest/refpages/source/FragStencilRefEXT.html) discards bits above the attachment's stencil width; supported stencil attachments have eight bits. Numeric subgroup observations throw except bounded own-lane mask plumbing and group control with kept-lane and derivative-helper coverage proved. WQM inputs must exclude helper contributions (a ballot of the active lanes, as in an EXEC copy, covers whole quads and is accepted), since [non-quad subgroup operations may treat helpers as inactive](https://docs.vulkan.org/glslext/latest/glslext/khr/GL_KHR_shader_subgroup.html). Readonly guarded SRT reads may report internal host faults, and read-only BDA reads may report faults; these draws wait and check faults before publishing results, and loops throw. Indirect draws, restart, strips, fans, tessellation, mesh, multisampling, early tests, ordered pixel execution, guest shader memory writes, other fault-producing memory access, clocks, primitive ID inputs, attachment feedback and pipeline face/channel splits throw. An exhausted stage push range throws. Equivalence to console ordering has not been measured on console hardware. From 0d41b11bc190e630d89ddb7d5b52b0bd511c33e8 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:29:02 -0700 Subject: [PATCH 119/318] docs(libSceAgcDriver): record the gfx1030 store measurement for written TYPE 0 and 1 slots --- docs/dev/debt/libSceAgcDriver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 602e104ecc..fccef933ac 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -85,7 +85,7 @@ - [Initial stencil state](../../../core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp) uses zero for `DB_STENCIL_CONTROL`, `DB_STENCILREFMASK` and `DB_STENCILREFMASK_BF` (context `0x10b`-`0x10d`), including after `CLEAR_STATE`. The public V0, V4, V5 and V7-V11 [register-default tables](../../../core/libs/prx/libSceAgc/Command/include/RegisterDefaultsData.hpp) supply these values. Linux's [GFX10 clear-state table](https://github.com/torvalds/linux/blob/master/drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h) agrees on the control register but uses `0x01000000` for the reference masks; the console reset values have not been measured. Planet Coaster (PPSA01736 v01.017) enables stencil tests without writing the control register. - [Multisampled DCC comp-to-single clears](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) (AGC driver) - RADV's multisampled comp-to-single clear dispatch ([`radv_clear_dcc_comp_to_single`](https://github.com/mihawk-99/PS5_Mesa/blob/1ba9874f478657ccf6da72ce55f0260164cc0bd0/src/amd/vulkan/meta/radv_meta_clear.c#L954)) is recognized by its code and not run. Its clear color (user data 5 to 8, which the shader stores as the four dwords of sample 0's texel) is recorded for the surface its image descriptor names, and DCC keys of 0x10 on a multisampled target of that surface, pipe-bank XOR, size and sample count import as a clear to it, for texels of 8 to 128 bits. A record is dropped when an unmap or allocation removal covers its surface's base. The guest's color memory doesn't get the color at the blocks' first pixels, and the recorded color is assumed to be the one the keys refer to: RADV writes 0x10 keys only in a fast clear that runs this dispatch for the same image right after, and initializes DCC to 0x00 or 0xff on an image's first use ([`radv_init_color_image_metadata`](https://github.com/mihawk-99/PS5_Mesa/blob/1ba9874f478657ccf6da72ce55f0260164cc0bd0/src/amd/vulkan/radv_cmd_buffer.c#L16894)). A new image on memory reused within one mapping, with the same address, XOR, size and sample count, whose clear runs a comp-to-single shader other than the recognized one, would still import the old color. - [Initial depth and stencil planes](../../../core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp) - the console's reset values of `DB_Z_INFO` and `DB_STENCIL_INFO` (context `0x010`-`0x011`) are not measured. Both start, and return after `CLEAR_STATE`, at zero (formats `INVALID`), the value of Linux's [GFX10 clear-state table](https://github.com/torvalds/linux/blob/master/drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h). The public [register-default tables](../../../core/libs/prx/libSceAgc/Command/include/RegisterDefaultsData.hpp) write `0x80000180` and `0x20000180` instead, also with formats `INVALID`; of their other bits, only `TILE_STENCIL_DISABLE` would change a result: the HTILE layout of a multisampled depth plane whose `DB_STENCIL_INFO` was never written. -- [Type-0 and type-1 texture descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (`TYPE` = 0 or 1 in dword 3) with a nonzero base are bound as null images, like a zero base, in [static slots](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) with no image table that are read or written; atomic slots throw. Reads are measured: TYPE 0 on gfx1036 (RDNA2) in #4139 (sample, load and resinfo return zero, no fault), TYPE 1 on gfx1010 (RDNA1) in #4139 and #5828 (`image_sample_lz`, `image_sample_b`, `image_load` and `image_get_resinfo`) with format 22 only, not on RDNA2, with other formats or with `image_sample_c*`. Stores are not: a written slot is a null storage image, whose stores `nullDescriptor` discards, pending [`probe_type0_store.py`](https://github.com/EagleBearerYS/AnyPS5/tree/test/hw-oracle-type0-store/tools/hw-oracle) on `test/hw-oracle-type0-store`. Neither type was measured on the console's gfx1013. +- [Type-0 and type-1 texture descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (`TYPE` = 0 or 1 in dword 3) with a nonzero base are bound as null images, like a zero base, in [static slots](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) with no image table that are read or written; atomic slots throw. Reads are measured: TYPE 0 on gfx1036 (RDNA2) in #4139 (sample, load and resinfo return zero, no fault), TYPE 1 on gfx1010 (RDNA1) in #4139 and #5828 (`image_sample_lz`, `image_sample_b`, `image_load` and `image_get_resinfo`) with format 22 only, not on RDNA2, with other formats or with `image_sample_c*`. Stores are measured on gfx1030 (RDNA2) in #9749: an `image_store` through the PPSA01474 words with TYPE 0 or 1 changed no memory around the base and didn't fault, so a written slot is a null storage image, whose stores `nullDescriptor` discards. Neither type was measured on the console's gfx1013. - [Multisampled color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) lose their guest-read guards and cannot be reused after a successful allocation removal or unmap covers their full guarded range. The next target, sampled or storage lookup removes retired cache owners; recorded work and sampled views keep their images alive until their last reference is released. Partial unmaps keep the guard; partial allocation reuse and guest access to live multisampled data remain unsupported. - [Stencil reference export replay](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) on devices without `VK_EXT_shader_stencil_export` replays the low eight bits of the values proved from constant, select and phi exports, with an export proved on every kept path. An export the prover cannot bound replays all 256 eight-bit values, one pass per value per triangle (logged once per process). It records instances, then their triangles, then the reference classes of each triangle. One pipeline filters fragments at shader termination against a dynamic push value before late stencil and depth tests. The [Vulkan stencil-reference replacement rule](https://docs.vulkan.org/refpages/latest/refpages/source/FragStencilRefEXT.html) discards bits above the attachment's stencil width; supported stencil attachments have eight bits. Numeric subgroup observations throw except bounded own-lane mask plumbing and group control with kept-lane and derivative-helper coverage proved. WQM inputs must exclude helper contributions (a ballot of the active lanes, as in an EXEC copy, covers whole quads and is accepted), since [non-quad subgroup operations may treat helpers as inactive](https://docs.vulkan.org/glslext/latest/glslext/khr/GL_KHR_shader_subgroup.html). Readonly guarded SRT reads may report internal host faults, and read-only BDA reads may report faults; these draws wait and check faults before publishing results, and loops throw. Indirect draws, restart, strips, fans, tessellation, mesh, multisampling, early tests, ordered pixel execution, guest shader memory writes, other fault-producing memory access, clocks, primitive ID inputs, attachment feedback and pipeline face/channel splits throw. An exhausted stage push range throws. Equivalence to console ordering has not been measured on console hardware. From db1c8af872d76522e32611a41cb88f44711b48c8 Mon Sep 17 00:00:00 2001 From: TechLevelZero <35297505+TechLevelZero@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:53:49 +0100 Subject: [PATCH 120/318] fix(shader): sign-extend 32-bit literals of signed 64-bit integer operands A 32-bit literal read as a 64-bit integer was always zero-extended. RDNA2 sign-extends it when the operand is a signed 64-bit integer: the i64 compares, v_ashrrev_i64, the v_mad_i64_i32 addend, s_ashr_i64, s_bfe_i64 and s_flbit_i32_i64. Unsigned and bitwise 64-bit operands still zero-extend, as measured on RDNA2 (gfx1036) with the hardware oracle. Co-Authored-By: Claude Opus 5.5 --- .../tests/execution/Signed64BitLiteral.cpp | 120 ++++++++++++++++++ .../include/RdnaDecoder/RdnaOpcode.hpp | 1 + .../recompiler/RdnaDecoder/src/RdnaOpcode.cpp | 26 ++++ .../src/TranslationContext/SubwordPacking.cpp | 2 +- 4 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/Signed64BitLiteral.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Signed64BitLiteral.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Signed64BitLiteral.cpp new file mode 100644 index 0000000000..dac8631412 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Signed64BitLiteral.cpp @@ -0,0 +1,120 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34040086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0xd4a10008u, 0x000100ffu, 0x80000000u, + 0xd501000au, 0x00210280u, 0xd4a40008u, 0x0001fe80u, 0xff00ff00u, 0xd501000bu, 0x00210280u, 0xd4a20008u, + 0x000208ffu, 0xfffffffbu, 0xd501000cu, 0x00210280u, 0xbe8918ffu, 0x80000000u, 0x7e1a0209u, 0xd701000eu, + 0x0001fe84u, 0x80000000u, 0xd7000010u, 0x0001fe84u, 0x80000000u, 0xd5770a12u, 0x03fd0481u, 0xfffffffbu, + 0xd5760a14u, 0x03fd0481u, 0xfffffffbu, 0x918c84ffu, 0x80000000u, 0x7e2c020cu, 0x7e2e020du, 0xbe9003ffu, + 0x00280010u, 0x950e10ffu, 0x80000000u, 0x7e30020eu, 0x7e32020fu, 0xe0781000u, 0x80010a02u, 0xe0781010u, + 0x80010e02u, 0xe0781020u, 0x80011202u, 0xe0781030u, 0x80011602u, 0xbf810000u, +}; + +constexpr std::array Names{ + "v_cmp_lt_i64 0x80000000, 0", + "v_cmp_gt_i64 0, 0xff00ff00", + "v_cmp_eq_i64 0xfffffffb, -5", + "s_flbit_i32_i64 0x80000000", + "v_ashrrev_i64 4, 0x80000000 (low)", + "v_ashrrev_i64 4, 0x80000000 (high)", + "v_lshrrev_b64 4, 0x80000000 (low)", + "v_lshrrev_b64 4, 0x80000000 (high)", + "v_mad_i64_i32 1, 2, 0xfffffffb (low)", + "v_mad_i64_i32 1, 2, 0xfffffffb (high)", + "v_mad_u64_u32 1, 2, 0xfffffffb (low)", + "v_mad_u64_u32 1, 2, 0xfffffffb (high)", + "s_ashr_i64 0x80000000, 4 (low)", + "s_ashr_i64 0x80000000, 4 (high)", + "s_bfe_i64 0x80000000, offset 16 width 40 (low)", + "s_bfe_i64 0x80000000, offset 16 width 40 (high)", +}; + +constexpr std::array Expected{ + 0x00000001u, 0x00000001u, 0x00000001u, 0x00000021u, 0xf8000000u, 0xffffffffu, 0x08000000u, 0x00000000u, + 0xfffffffdu, 0xffffffffu, 0xfffffffdu, 0x00000000u, 0xf8000000u, 0xffffffffu, 0xffff8000u, 0xffffffffu, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Run(AgcDriver::VulkanDevice& device) { + Input.fill(0u); + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + Input[lane * Inputs] = 0xfffffffbu; + Input[lane * Inputs + 1u] = 0xffffffffu; + } + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + for (std::uint32_t column = 0; column < Results; ++column) { + const auto value = Output[lane * Results + column]; + Require(value == Expected[column], std::string(Names[column]) + " in thread " + std::to_string(lane) + " is " + Hex(value) + ", expected " + Hex(Expected[column])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("signed 64-bit literal tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp index 0b83c3ae6d..8575428009 100644 --- a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp +++ b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp @@ -1100,6 +1100,7 @@ enum class RdnaOpcode : std::uint16_t { [[nodiscard]] bool IsBufferMemoryOpcode(RdnaOpcode opcode); [[nodiscard]] bool IsImageOpcode(RdnaOpcode opcode); [[nodiscard]] bool IsExportOpcode(RdnaOpcode opcode); +[[nodiscard]] bool IsSigned64BitIntegerOpcode(RdnaOpcode opcode); bool IsConditionalBranchOpcode(RdnaOpcode opcode); bool IsDirectBranchOpcode(RdnaOpcode opcode); diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp index 374315cb1a..39eda0547b 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp @@ -904,6 +904,32 @@ bool IsExportOpcode(RdnaOpcode opcode) { } } +bool IsSigned64BitIntegerOpcode(RdnaOpcode opcode) { + requireClassifiableOpcode(opcode); + switch (opcode) { + case RdnaOpcode::VCmpLtI64: + case RdnaOpcode::VCmpEqI64: + case RdnaOpcode::VCmpLeI64: + case RdnaOpcode::VCmpGtI64: + case RdnaOpcode::VCmpNeI64: + case RdnaOpcode::VCmpGeI64: + case RdnaOpcode::VCmpxLtI64: + case RdnaOpcode::VCmpxEqI64: + case RdnaOpcode::VCmpxLeI64: + case RdnaOpcode::VCmpxGtI64: + case RdnaOpcode::VCmpxNeI64: + case RdnaOpcode::VCmpxGeI64: + case RdnaOpcode::VAshrrevI64: + case RdnaOpcode::VMadI64I32: + case RdnaOpcode::SAshrI64: + case RdnaOpcode::SBfeI64: + case RdnaOpcode::SFlbitI32I64: + return true; + default: + return false; + } +} + bool IsConditionalBranchOpcode(RdnaOpcode opcode) { requireClassifiableOpcode(opcode); switch (opcode) { diff --git a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp index 3ba6021ff2..166ef8d921 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp @@ -14,7 +14,7 @@ IrU32 TranslationContext::readU32(const RdnaOperand& operand) { std::array TranslationContext::readU32Pair(const RdnaOperand& operand) { switch (operand.kind) { - case RdnaOperandKind::LiteralConstant: return {readU32(operand), IrU32(ir.Constant(0u))}; + case RdnaOperandKind::LiteralConstant: return {readU32(operand), IrU32(ir.Constant(IsSigned64BitIntegerOpcode(currentOpcode) && (operand.value & 0x80000000u) != 0u ? 0xffffffffu : 0u))}; case RdnaOperandKind::IntegerInlineConstant: return {readU32(operand), IrU32(ir.Constant(operand.signedVal < 0 ? 0xffffffffu : 0u))}; case RdnaOperandKind::FloatInlineConstant: { const auto bits = std::bit_cast(static_cast(std::bit_cast(operand.value))); From 01f2e70cd6865744c80636a67f4c8ee7dd290bf6 Mon Sep 17 00:00:00 2001 From: TechLevelZero <35297505+TechLevelZero@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:49:49 +0100 Subject: [PATCH 121/318] fix(agc): keep a device's idle stores and storage write-backs on its own recorder A device's idle, label and flip stores flushed every device's pending storage images, and a storage-image write-back recorded into the active recorder whatever device it belonged to. The active recorder is one per process and is switched without the GPU mutex, so with two live devices the driver's idle store recorded the other device's buffers and images on its own command buffer, which the validation layer reports as commonparent errors and which ended in a lost device. A device's own stores now take only its images, and a write-back uses the active recorder only when it belongs to the image's device. Co-Authored-By: Claude Opus 5.5 --- .../Execution/src/VulkanDevice.cpp | 6 +- .../Graphics/include/Texture.hpp | 4 +- .../libSceAgcDriver/Graphics/src/Texture.cpp | 22 +-- .../tests/execution/TwoDevicePendingStore.cpp | 132 ++++++++++++++++++ docs/dev/debt/libSceAgcDriver.md | 1 + 5 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/TwoDevicePendingStore.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index b2f732936e..b411383d80 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -2367,7 +2367,7 @@ void VulkanDevice::StoreIdleResults(bool reapFirst) { GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); std::lock_guard gpuLock(GuestMemory::GpuMutex()); const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); - Graphics::StorageTexture::FlushAllPending("idle"); + Graphics::StorageTexture::FlushAllPending("idle", state->device); SubmitRecorded(reapFirst); } @@ -2444,7 +2444,7 @@ bool VulkanDevice::StoresPendingOver(std::uint64_t address, std::size_t bytes) c int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) { static const bool storeAtLabel = Graphics::StorageTexture::StoreAtLabelRequested(std::getenv("APS5_STORE_AT_LABEL")); - if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label"); + if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label", state->device); Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before label"); if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4; auto& recorder = *state->recorder; @@ -3333,7 +3333,7 @@ FrameDumps& Dumps() { bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { static const bool storeAtFlip = Graphics::StorageTexture::StoreAtFlipRequested(std::getenv("APS5_STORE_AT_FLIP")); - if (storeAtFlip) Graphics::StorageTexture::FlushAllPending("flip"); + if (storeAtFlip) Graphics::StorageTexture::FlushAllPending("flip", state->device); else Graphics::StorageTexture::FlushDumpSelected("flip"); auto& dumps = Dumps(); const bool passthrough = state->surfaceColorSpace == VK_COLOR_SPACE_HDR10_ST2084_EXT; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 0c6aec2abf..16c75f792a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -213,8 +213,8 @@ public: // any unit was: a reader of the import must sync for a publish as for a store. Only images // with a pending unit inside the range are stored (APS5_NO_FLUSH_PRETEST=1 lists every // overlapping pending image, as before). - static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access", PublishScope scope = PublishScope::Whole, bool* published = nullptr, bool waited = false); - static void FlushAllPending(const char* reason); + static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access", PublishScope scope = PublishScope::Whole, bool* published = nullptr, bool waited = false, VkDevice device = VK_NULL_HANDLE); + static void FlushAllPending(const char* reason, VkDevice device = VK_NULL_HANDLE); static bool WaitForFlushing(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr); static void SaveBeforeUnwatch(std::uint64_t address, std::size_t bytes); static constexpr std::uint64_t SmallStoreBytes = 64 * 1024; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index d8394b6c5e..476fca0d09 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -56,6 +56,11 @@ void OwnedImage::Allocate(const Context& device, const VkMemoryAllocateInfo& all namespace { +Recorder* recorderFor(const Context& context) { + auto* recorder = Recorder::Active(); + return recorder != nullptr && recorder->Device() == context.device ? recorder : nullptr; +} + // APS5_PROFILE_DRAW: accumulate texture setup phases and report every 200 textures. struct TextureProfile { double allocate = 0, read = 0, gpu = 0, view = 0; @@ -3529,7 +3534,7 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se const bool scratchUsed = std::any_of(directRanges.begin(), directRanges.end(), [](const auto& range) { return range.first == range.second; }) || std::any_of(slabPieces.begin(), slabPieces.end(), [](const SlabPieces& pieces) { return !pieces.copies.empty(); }) || std::any_of(importCopies.begin(), importCopies.end(), [](const auto& copies) { return !copies.empty(); }) || !saves.empty() || !padding.copies.empty() || !seeds.empty(); auto tiledScratch = scratchUsed ? std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT) : nullptr; detiler.BeginBatch(); - auto* recorder = Recorder::Active(); + auto* recorder = recorderFor(context); std::unique_ptr batch; VkCommandBuffer commands = VK_NULL_HANDLE; VkAccessFlags covered = 0; @@ -3911,7 +3916,7 @@ void waitForIdlePublish(const char* reason) { } -bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason, PublishScope scope, bool* published, bool waited) { +bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason, PublishScope scope, bool* published, bool waited, VkDevice device) { static_cast(FlushDepthPlanes(address, bytes)); if (published != nullptr) *published = false; if (AgcDriver::DriverDetail::DrawPipeline::HeldReads(address, bytes)) { @@ -3927,7 +3932,7 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons const auto* exempt = refreshing; for (auto it = pending.textures.MayOverlap(address, bytes) ? pending.textures.begin() : pending.textures.end(); it != pending.textures.end();) { auto* texture = *it; - if (texture != except && texture != exempt && texture->overlaps(address, bytes)) { + if (texture != except && texture != exempt && (device == VK_NULL_HANDLE || texture->context.device == device) && texture->overlaps(address, bytes)) { if (!texture->pendingUnitInside(address, bytes)) { pretestSkipped.fetch_add(1, std::memory_order_relaxed); ++it; @@ -4114,7 +4119,7 @@ void StorageTexture::commitStoredSave(const std::vector& units) { if (units.empty()) return; const auto ticket = ++lastSaveTicket; for (const auto unit : units) savedTicket[unit] = ticket; - auto* recorder = Recorder::Active(); + auto* recorder = recorderFor(context); if (recorder == nullptr) { applyStoredSave(ticket, units); return; @@ -4155,7 +4160,8 @@ void StorageTexture::awaitStoredSave(const std::vector* layers) { last = std::max(last, layerBegin(unit) + layerBytes(unit)); } if (last <= first) return; - auto* recorder = Recorder::Active(); + auto* recorder = recorderFor(context); + if (recorder == nullptr) recorder = context.recorder; Require(recorder != nullptr, "a write-back's saved guest bytes are pending without a recorder"); recorder->SyncThrough(first, static_cast(last - first)); const auto outstanding = [&] { @@ -4220,8 +4226,8 @@ void StorageTexture::FlushDumpSelected(const char* reason) { for (const auto& [address, bytes] : ranges) static_cast(FlushPending(address, bytes, nullptr, reason)); } -void StorageTexture::FlushAllPending(const char* reason) { - static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason)); +void StorageTexture::FlushAllPending(const char* reason, VkDevice device) { + static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason, PublishScope::Whole, nullptr, false, device)); } std::size_t StorageTexture::StoreSmallPending(const char* reason) { @@ -5646,7 +5652,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { for (const auto& [from, to] : keep) copied.push_back({from - descriptor.baseAddress, to - descriptor.baseAddress, from - descriptor.baseAddress}); const auto padding = paddingSeeds(segments, copied); detiler.BeginBatch(); - auto* recorder = Recorder::Active(); + auto* recorder = recorderFor(context); std::unique_ptr batch; VkCommandBuffer commands = VK_NULL_HANDLE; auto timing = Recorder::NoTiming; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/TwoDevicePendingStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/TwoDevicePendingStore.cpp new file mode 100644 index 0000000000..37d7585295 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/TwoDevicePendingStore.cpp @@ -0,0 +1,132 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver { + +struct VulkanDeviceTestAccess { + static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); } +}; + +} + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 8; +constexpr std::uint32_t Side = 256; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t BlockBytes = 0x60000; +constexpr std::uint32_t Untouched = 0xdeadbeefu; +constexpr std::uint32_t Stored = 0x12345678u; + +alignas(256) constexpr std::array StoreCode{ + 0x7e040300u, 0x7e060280u, 0x7e0202ffu, Stored, 0xf0200108u, 0x00020102u, 0xbf810000u, +}; + +std::array TextureDescriptor(const std::uint32_t* texels) { + const auto address = static_cast(reinterpret_cast(texels)); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (TileR64KBX << 20u) | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +void Store(AgcDriver::VulkanDevice& device, const std::uint32_t* texels) { + std::vector userData(16, 0u); + const auto texture = TextureDescriptor(texels); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + const std::span code(StoreCode); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void FlushAll(const char* reason) { + std::lock_guard gpuLock(AgcDriver::GuestMemory::GpuMutex()); + AgcDriver::Graphics::StorageTexture::FlushAllPending(reason); +} + +} + +int main() { + try { + const auto first = OpenVulkanTestDevice(); + if (!first) return VulkanTestSkipped; + std::vector storage((BlockBytes + 0x10000u) / 4u); + auto* texels = reinterpret_cast((reinterpret_cast(storage.data()) + 0xffffu) & ~std::uintptr_t{0xffffu}); + std::fill(texels, texels + BlockBytes / 4u, Untouched); + { + GuestAllocations::Mutation mutation; + mutation.Add(texels, BlockBytes, true, true, true); + } + const auto bytes = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(TextureDescriptor(texels))).guestBytes; + const volatile std::uint32_t* cpu = texels; + const auto stored = [&] { + for (std::size_t index = 0; index < bytes / 4u; ++index) { + if (cpu[index] == Stored) return true; + } + return false; + }; + + Store(*first, texels); + Require(AgcDriver::Graphics::StorageTexture::AnyPending(), "two devices: the first device's image store left no pending results"); + { + const auto second = OpenVulkanTestDevice(); + Require(second != nullptr, "two devices: the second device did not open"); + Require(AgcDriver::Graphics::Recorder::Active() == AgcDriver::VulkanDeviceTestAccess::ContextOf(*second).recorder, "two devices: the second device did not activate its recorder"); + second->StoreIdleResults(false); + second->WaitIdle(); + Require(AgcDriver::Graphics::StorageTexture::AnyPending(), "two devices: the second device's idle store took the first device's pending results"); + Require(!stored(), "two devices: the first device's results reached guest memory through the second device's idle store"); + FlushAll("memory access"); + first->WaitIdle(); + Require(!AgcDriver::Graphics::StorageTexture::AnyPending(), "two devices: a flush with the second device active left the first device's results pending"); + Require(stored(), "two devices: a flush with the second device active did not bring the first device's results to guest memory"); + second->WaitIdle(); + } + { + GuestAllocations::Mutation mutation; + mutation.Remove(texels); + } + std::puts("two device pending store tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 98b9dd1278..4d8103a7ee 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -124,6 +124,7 @@ - On a device without VK_EXT_robustness2's `nullDescriptor`, such as MoltenVK, a null image descriptor in a typed heap is a [placeholder image](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PlaceholderImages.cpp) of that heap's type, 1×1 and cleared to zero, which every null descriptor of the heap shares. Reads give zeros, as `nullDescriptor` allows, but stores and atomics to it are kept, so a later read of a null storage image can return them, and a depth comparison compares against depth 0. A heap whose placeholder the device cannot create, for example 1D or 3D depth, 64-bit atomics without 64-bit image atomics, or multisampled storage without `shaderStorageImageMultisample`, throws when a null descriptor reaches it. - [Draw input snapshots](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver): `PrepareDrawBindings` caps read-only input buffer snapshots at 64 MiB (overridable via `APS5_DRAW_SNAPSHOT_MAX_MIB`). Larger inputs (streaming heaps, bindless pools) or draws exceeding the cumulative budget skip the per-draw snapshot and bind in place, avoiding multi-gigabyte CPU copy stalls (such as Ratchet & Clank: Rift Apart PPSA01474 stalling at frame ~380 copying 36 GB every 5 s). If the title rewrites those large ranges on the CPU before GPU completion, the in-place binding observes the modification, as verified by `drawSnapshotCapTests`. - [Pending storage-image results](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) are stored when something reads their memory, not when the GPU work that wrote them completes. A title that waits for that work, frees the image and reuses its memory can have the new data overwritten by the late store (#1970). A title's own CPU reads are not seen either, so a title that reads the results after waiting for a label reads stale memory. Explicit-size file writes (`write`, `pwrite`, `writev`, `pwritev`, AIO and `fwrite`) synchronize their source ranges through the [host-read hook](../../../core/libs/prx/libc/src/GuestArena.cpp); other host readers are not covered. `APS5_STORE_AT_FLIP=1` stores every pending result at each flip and `APS5_STORE_AT_LABEL=1` before each label the GPU writes; both store every pending image, not only the ones the waited-for work wrote, and neither is on by default. +- [The active recorder](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp) (libSceAgcDriver) is one per process, and `Recorder::Activate` switches it without the GPU mutex. With two live devices (tests that open their own device next to the driver's), a device's idle, label and flip stores take only its own pending storage images, and a storage-image write-back records into the active recorder only when that recorder is its device's, otherwise in a submission of its own. Other users of `Recorder::Active()` don't check the device, so they can still record one device's work on the other's command buffer. - [`APS5_SKIP_REFUSED_DRAWS`](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp) (libSceAgcDriver) drops a draw whose graphics stage the recompiler refuses instead of failing the queue, so the frame lacks that draw, and the stage is compiled and refused again at each such draw. Compute refusals still fail. - [Pending storage-image results over memory outside the write watch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) leave to the CPU the 64 KiB blocks whose bytes differ from the ones saved at the upload, the fill, the image copy, the previous store or the moment their range left the watch (`Unwatch`, or a host import that does not stay watched), and the blocks the watch saw the CPU write before that moment. A store made on the GPU saves its bytes through a host-visible copy that becomes the saved bytes when its batch completes; a store or a refresh that needs them earlier waits for that batch. A unit without a write generation saves the guest bytes when results land in it outside the watch. One whose range leaves the watch while its results are pending saves its bytes through the unwatch hook, which also runs when a failed collect takes a range out of the watch. A range that leaves the watch without the hook running over it (an import fallback that unwatched without saving) leaves its pending results without saved bytes; storing such results throws. A unit whose results were stored while its blocks were watched keeps only a generation; when its blocks leave the watch while nothing is pending, the hook saves nothing, and results that land in it afterwards have neither saved bytes nor a watch, so storing them throws too. - [Storage images over uncommitted guest memory](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) - a storage image whose guest range has no committed page when it is created throws in the `StorageTexture` constructor, which aborts the guest. Several titles reach this intermittently in runs of a few minutes, on AMD and NVIDIA alike; in two of the three logs, lines about storage images over released or rewritten memory (`memory no longer registered`, `guest memory changed while GPU results were pending`) come shortly before it. The log does not name the image's address. What the console does with a storage descriptor over memory that is not committed is unknown, and which allocation the guest recreates there was not found. From a22a46931e80c6c53792f482f22e25813e269c13 Mon Sep 17 00:00:00 2001 From: TechLevelZero <35297505+TechLevelZero@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:29:12 +0000 Subject: [PATCH 122/318] fix(shader): round v_div_fmas_f32 once and keep its denormal inputs On RDNA2 v_div_fmas_f32 never flushes its inputs, rounds the scaled fused result once, and flushes a denormal result to a zero of its sign when the f32 denormal mode flushes outputs. With Float64 the sum is now computed in f64 and rounded to odd, so the scaling into the denormal range no longer rounds twice and the hardware f32 flush no longer reaches the inputs; mode 1 now flushes the result. Co-Authored-By: Claude Opus 5.5 --- .../tests/execution/Division.cpp | 68 +++++++++++++++++++ core/shader/recompiler/Recompiler.cpp | 1 + .../Translation/InstructionTranslator.hpp | 1 + .../Translation/TranslationContext.hpp | 3 + .../Translation/src/FloatInstructions.cpp | 45 ++++++++++++ .../Translation/src/InstructionTranslator.cpp | 1 + .../Translation/src/VectorInstructions.cpp | 2 + docs/dev/debt/shader.md | 3 +- 8 files changed, 123 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp index 9cbfb1997d..78466f98f0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,11 @@ alignas(256) constexpr std::array ScaleCode{ 0xe0702004u, 0x80010b01u, 0xbf810000u, }; +alignas(256) constexpr std::array FmasCode{ + 0x34020082u, 0xe0302000u, 0x80000401u, 0xe0302004u, 0x80000501u, 0xe0302008u, 0x80000601u, 0xe030200cu, + 0x80000701u, 0xbf8c3f70u, 0x7d8a0e80u, 0xd56f000au, 0x041a0b04u, 0xe0702000u, 0x80010a01u, 0xbf810000u, +}; + struct ScaleVector { std::uint32_t s0, s1, s2, flushed, kept, vcc; }; @@ -621,6 +627,39 @@ constexpr DivisionVector DivisionVectors[] = { {0x2210592fu, 0x0ac8fdd3u, 0x56b7dabeu}, }; +struct FmasDenormalVector { + std::uint32_t s0, s1, s2, vcc, flushed, kept; +}; + +constexpr FmasDenormalVector FmasDenormalVectors[] = { + {0x00000001u, 0x80800000u, 0x1fea6146u, 1u, 0x00000000u, 0x003a9851u}, + {0x8da51d6fu, 0x96a2a07au, 0x2004bac1u, 1u, 0x00000000u, 0x00425d61u}, + {0xbec6b7ecu, 0x20000000u, 0x00800000u, 1u, 0x80000000u, 0x8018d6fdu}, + {0x00000001u, 0x6a2932fau, 0x00400000u, 1u, 0x00000000u, 0x002a4cbfu}, + {0x0becd7b0u, 0x2535a659u, 0x80000001u, 0u, 0x80000000u, 0x80000001u}, + {0x20000000u, 0x08fe0996u, 0x807fffffu, 0u, 0x80000000u, 0x807fffffu}, + {0x9d1996e9u, 0x03feab54u, 0x00400000u, 0u, 0x00000000u, 0x00400000u}, + {0x80000000u, 0x9d47e6bfu, 0x00000001u, 0u, 0x00000000u, 0x00000001u}, + {0x32d351d2u, 0x1f800000u, 0x9f800000u, 1u, 0x80000000u, 0x80200000u}, + {0x1452d51eu, 0xb9246da0u, 0x9682e357u, 1u, 0x80000000u, 0x80000008u}, + {0x84b5a818u, 0x2de2cedcu, 0x1f800000u, 1u, 0x00000000u, 0x00200000u}, + {0x149e259bu, 0x80800000u, 0x1a3dc795u, 1u, 0x00000000u, 0x000005eeu}, + {0xff800000u, 0x00000001u, 0x1e8b9c7bu, 0u, 0xff800000u, 0xff800000u}, + {0x096fea1fu, 0x40000000u, 0x00400000u, 0u, 0x09efea2fu, 0x09efea2fu}, + {0xb8c9817au, 0x0adcc633u, 0x007fffffu, 0u, 0x842cc747u, 0x842cc747u}, + {0x807fffffu, 0xff7fffffu, 0x3f000000u, 0u, 0x408fffffu, 0x408fffffu}, + {0xff800000u, 0x00000001u, 0x1e8b9c7bu, 1u, 0xff800000u, 0xff800000u}, + {0x807fffffu, 0xff7fffffu, 0x3f000000u, 1u, 0x208fffffu, 0x208fffffu}, + {0x802927c2u, 0x7f7fffffu, 0x80000000u, 1u, 0x9fa49f07u, 0x9fa49f07u}, + {0x007fffffu, 0xde01282au, 0x25a1ba53u, 1u, 0x05a1b64au, 0x05a1b64au}, + {0x00400000u, 0x14afa671u, 0x807fffffu, 0u, 0x80000000u, 0x807fffffu}, + {0x95928216u, 0x80000001u, 0x807fffffu, 0u, 0x80000000u, 0x807fffffu}, + {0x807fffffu, 0x3f800000u, 0x00000000u, 0u, 0x80000000u, 0x807fffffu}, + {0x9e2d4ac9u, 0x80000000u, 0x80000001u, 0u, 0x80000000u, 0x80000001u}, + {0x007fffffu, 0xff800000u, 0x15e81bf3u, 0u, 0xff800000u, 0xff800000u}, + {0x007fffffu, 0xff800000u, 0x15e81bf3u, 1u, 0xff800000u, 0xff800000u}, +}; + struct FmasNanVector { std::uint32_t s0, s1, s2, noIeee, ieee; }; @@ -775,6 +814,30 @@ void CheckFmasNan(AgcDriver::VulkanDevice& device, const std::optional(spv::CapabilityFloat64)) != capabilities.end(); +} + +void CheckFmasDenormals(AgcDriver::VulkanDevice& device) { + static_assert(std::size(FmasDenormalVectors) <= Threads); + for (const std::uint32_t mode : {0xc0u, 0xd0u, 0xe0u, 0xf0u}) { + Shader shader(device, FmasCode, ShaderRecompiler::ShaderFloatMode{mode, true, false, false}); + Input.fill(0u); + for (std::uint32_t lane = 0; lane < std::size(FmasDenormalVectors); ++lane) { + const auto& vector = FmasDenormalVectors[lane]; + std::copy_n(std::array{vector.s0, vector.s1, vector.s2, vector.vcc}.begin(), 4, Input.begin() + lane * Stride); + } + shader.Run(); + for (std::uint32_t lane = 0; lane < std::size(FmasDenormalVectors); ++lane) { + const auto& vector = FmasDenormalVectors[lane]; + const auto actual = Output[lane * Stride]; + const auto expected = ((mode >> 4u) & 2u) != 0u ? vector.kept : vector.flushed; + Require(actual == expected, "division fmas denormals: FLOAT_MODE " + Hex(mode) + " (" + Hex(vector.s0) + ", " + Hex(vector.s1) + ", " + Hex(vector.s2) + ") vcc " + std::to_string(vector.vcc) + ": v_div_fmas_f32 is " + Hex(actual) + ", expected " + Hex(expected)); + } + } +} + void CheckDivision(AgcDriver::VulkanDevice& device) { Shader shader(device, DivisionCode); constexpr std::uint32_t count = sizeof(DivisionVectors) / sizeof(DivisionVectors[0]); @@ -807,6 +870,11 @@ int main() { CheckFmasNan(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); CheckFmasNan(*device, ShaderRecompiler::ShaderFloatMode{0xf0u, false, false, false}); CheckFmasNan(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, false, true, false}); + if (HasFloat64(*device)) { + CheckFmasDenormals(*device); + } else { + std::puts("the device has no Float64, so v_div_fmas_f32 denormal and scaling rows are not checked"); + } if (fused) { CheckDivision(*device); } else { diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index e7ddf63820..42e94903d1 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -247,6 +247,7 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparat translateOptions.float32FlushToZero = HardwareFloat32Flush(request); translateOptions.inputInfo = inputInfo; translateOptions.float32DenormPreserve = request.target.float32DenormPreserve; + translateOptions.float64 = supports(spv::CapabilityFloat64); constexpr InstructionTranslator translator; diff --git a/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp b/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp index 21f6bc7e0e..b3273a4078 100644 --- a/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp +++ b/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp @@ -37,6 +37,7 @@ struct TranslateOptions { bool nativeFloat16 = false; bool float32FlushToZero = false; bool float32DenormPreserve = false; + bool float64 = false; bool gcnLegacyVertexInputs = false; std::optional floatMode; ShaderStageInputInfo inputInfo; diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 00da5a7db5..0043b0432c 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -27,6 +27,7 @@ public: void SetNativeFloat16(bool enabled) { nativeFloat16 = enabled; } void SetFloat32DenormPreserve(bool enabled) { float32DenormPreserve = enabled; } void SetFloat32FlushToZero(bool enabled) { float32FlushToZero = enabled; } + void SetFloat64(bool enabled) { float64 = enabled; } void AddBranchCondition(const BasicBlock& source, BlockInfo& info); void TranslateCodeTableLoad(const RdnaInstruction& instruction, const ControlFlowGraph::CodeTableLoad& table); void TranslateUnlinkedCall(const RdnaInstruction& instruction); @@ -224,6 +225,7 @@ private: bool minMaxF16(const RdnaInstruction& inst, IrOpcode opcode); bool vDivScaleF32(const RdnaInstruction& inst); bool vDivFmasF32(const RdnaInstruction& inst); + IrValue& fmaScaledF32(IrValue& lhs, IrValue& rhs, IrValue& addend, IrValue& powerHigh); bool vDivFixupF32(const RdnaInstruction& inst); bool vFrexpMantF32(const RdnaInstruction& inst); bool vFmaLegacyF32(const RdnaInstruction& inst, bool accumulate); @@ -389,6 +391,7 @@ private: std::uint32_t f32DenormalFlush = 0u; bool float32FlushToZero = false; bool float32DenormPreserve = false; + bool float64 = false; std::uint32_t currentProgramCounter = 0; std::uint32_t currentVectorLimit = 1; }; diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index 0632342acb..07737d9082 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -554,6 +554,11 @@ bool TranslationContext::vDivFmasF32(const RdnaInstruction& inst) { return IrU32(ir.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&ir.BitCastU32(value), &ir.Constant(23u), &ir.Constant(8u)})); }; const IrU1 up(ir.Emit(IrOpcode::UGreaterThanEqual32, IrType::U1, {&exponent(*addend).Value(), &ir.Constant(128u)})); + if (float64) { + IrValue& powerHigh = ir.Select(scale.Value(), ir.Select(up.Value(), ir.Constant(0x43f00000u), ir.Constant(0x3bf00000u)), ir.Constant(0x3ff00000u)); + writeOperand(inst.destination, nanResultF32({lhs, rhs, addend}, &fmaScaledF32(*lhs, *rhs, *addend, powerHigh), &invalidProductF32(lhs, rhs))); + return true; + } IrValue& power = ir.Emit(IrOpcode::SelectF32, IrType::F32, {&up.Value(), &ir.ConstantF32(18446744073709551616.0f), &ir.ConstantF32(5.42101086242752217e-20f)}); IrValue& plain = ir.Emit(IrOpcode::FPFma32, IrType::F32, {lhs, rhs, addend}); const IrU1 plainInfinite(ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(plain), ir.Constant(0x7fffffffu)), ir.Constant(0x7f800000u))); @@ -572,6 +577,46 @@ bool TranslationContext::vDivFmasF32(const RdnaInstruction& inst) { return true; } +IrValue& TranslationContext::fmaScaledF32(IrValue& lhs, IrValue& rhs, IrValue& addend, IrValue& powerHigh) { + const auto low = [&](IrValue& value) -> IrValue& { return ir.CompositeExtract(value, 0u); }; + const auto high = [&](IrValue& value) -> IrValue& { return ir.CompositeExtract(value, 1u); }; + const auto select = [&](IrValue& condition, IrValue& yes, IrValue& no) -> IrValue& { + return ir.ConstructU64(ir.Select(condition, low(yes), low(no)), ir.Select(condition, high(yes), high(no))); + }; + const auto add = [&](IrValue& a, IrValue& b) -> IrValue& { return ir.Emit(IrOpcode::FPAdd64, IrType::U64, {&a, &b}); }; + const auto multiply = [&](IrValue& a, IrValue& b) -> IrValue& { return ir.Emit(IrOpcode::FPMul64, IrType::U64, {&a, &b}); }; + const auto negate = [&](IrValue& value) -> IrValue& { return ir.ConstructU64(low(value), ir.BitwiseXor(high(value), ir.Constant(0x80000000u))); }; + const auto below = [&](IrValue& value, std::uint64_t limit) -> IrValue& { return ir.Emit(IrOpcode::ULessThan64, IrType::U1, {&value, &ir.ConstantU64(limit)}); }; + const auto widen = [&](IrValue& value) -> IrValue& { + IrValue& bits = ir.BitCastU32(value); + IrValue& magnitude = ir.BitwiseAnd(bits, ir.Constant(0x7fffffffu)); + IrValue& denormal = multiply(ir.Emit(IrOpcode::ConvertF64U32, IrType::U64, {&magnitude}), ir.ConstantU64(0x36a0000000000000ull)); + IrValue& signedDenormal = ir.ConstructU64(low(denormal), ir.BitwiseOr(high(denormal), ir.BitwiseAnd(bits, ir.Constant(0x80000000u)))); + return select(ir.ULessThan(magnitude, ir.Constant(0x00800000u)), signedDenormal, ir.Emit(IrOpcode::ConvertF64F32, IrType::U64, {&value})); + }; + IrValue& product = multiply(widen(lhs), widen(rhs)); + IrValue& wideAddend = widen(addend); + IrValue& sum = add(product, wideAddend); + IrValue& productPart = add(sum, negate(wideAddend)); + IrValue& addendPart = add(sum, negate(productPart)); + IrValue& error = add(add(product, negate(productPart)), add(wideAddend, negate(addendPart))); + IrValue& finite = ir.INotEqual(ir.BitwiseAnd(high(sum), ir.Constant(0x7ff00000u)), ir.Constant(0x7ff00000u)); + IrValue& inexact = ir.INotEqual(ir.BitwiseOr(low(error), ir.BitwiseAnd(high(error), ir.Constant(0x7fffffffu))), ir.Constant(0u)); + IrValue& even = ir.IEqual(ir.BitwiseAnd(low(sum), ir.Constant(1u)), ir.Constant(0u)); + IrValue& outward = ir.IEqual(ir.BitwiseAnd(ir.BitwiseXor(high(error), high(sum)), ir.Constant(0x80000000u)), ir.Constant(0u)); + IrValue& odd = select(outward, ir.Emit(IrOpcode::IAdd64, IrType::U64, {&sum, &ir.ConstantU64(1u)}), ir.Emit(IrOpcode::ISub64, IrType::U64, {&sum, &ir.ConstantU64(1u)})); + IrValue& rounded = select(ir.LogicalAnd(finite, ir.LogicalAnd(inexact, even)), odd, sum); + IrValue& scaled = multiply(rounded, ir.ConstructU64(ir.Constant(0u), powerHigh)); + IrValue& sign = ir.BitwiseAnd(high(scaled), ir.Constant(0x80000000u)); + IrValue& magnitude = ir.ConstructU64(low(scaled), ir.BitwiseAnd(high(scaled), ir.Constant(0x7fffffffu))); + IrValue& denormal = ir.Emit(IrOpcode::ConvertU32F64, IrType::U32, {&ir.Emit(IrOpcode::FPRoundEven64, IrType::U64, {&multiply(magnitude, ir.ConstantU64(0x4940000000000000ull))})}); + IrValue& overflow = ir.LogicalAnd(ir.LogicalNot(below(magnitude, 0x47effffff0000000ull)), below(magnitude, 0x7ff0000000000000ull)); + IrValue* bits = &ir.BitCastU32(ir.Emit(IrOpcode::ConvertF32F64, IrType::F32, {&scaled})); + bits = &ir.Select(overflow, ir.BitwiseOr(sign, ir.Constant(0x7f800000u)), *bits); + bits = &ir.Select(below(magnitude, 0x3810000000000000ull), ir.BitwiseOr(sign, denormal), *bits); + return ir.BitCastF32(*bits); +} + bool TranslationContext::vDivFixupF32(const RdnaInstruction& inst) { const IrU32 quotient = readU32(sourceAt(inst, 0u)); const IrU32 denominator = flushF32Denormal(readU32(sourceAt(inst, 1u))); diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index 08393a3942..df753b05a3 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -601,6 +601,7 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con context.SetNativeFloat16(options.nativeFloat16); context.SetFloat32DenormPreserve(options.float32DenormPreserve); context.SetFloat32FlushToZero(options.float32FlushToZero); + context.SetFloat64(options.float64); context.SetEntryRounding(entryRounding.at(cfgBlock.id)); for (std::uint32_t index = cfgBlock.instructionBegin; index < cfgBlock.instructionEnd; index++) { const auto& instruction = decoded.instructions[index]; diff --git a/core/shader/recompiler/Translation/src/VectorInstructions.cpp b/core/shader/recompiler/Translation/src/VectorInstructions.cpp index 804535419f..aeb5fca431 100644 --- a/core/shader/recompiler/Translation/src/VectorInstructions.cpp +++ b/core/shader/recompiler/Translation/src/VectorInstructions.cpp @@ -63,6 +63,8 @@ std::uint32_t TranslationContext::f32DenormalFlushFor(const RdnaInstruction& ins case RdnaOpcode::VMullitF32: flush = 3u; break; + case RdnaOpcode::VDivFmasF32: + return floatMode.has_value() && ((floatMode->floatMode >> 4u) & 2u) != 0u ? 0u : 2u; case RdnaOpcode::VMinF32: case RdnaOpcode::VMaxF32: case RdnaOpcode::VMin3F32: diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 1e2974aba2..26653919d2 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -126,7 +126,8 @@ - [f16 unary, frexp and conversion denormals](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follow the `FLOAT_MODE` f16 denormal field: `v_ceil_f16`, `v_floor_f16`, `v_fract_f16`, `v_rcp_f16`, `v_rsq_f16`, `v_sqrt_f16`, `v_log_f16`, `v_exp_f16`, `v_sin_f16`, `v_frexp_mant_f16`, `v_frexp_exp_i16_f16`, `v_cvt_norm_i16_f16`, `v_cvt_norm_u16_f16` and `v_cvt_f32_f16` flush a denormal half input to its signed zero when bit 0 is clear (0 and 2), and the f16-result ones flush a denormal half result to its signed zero when bit 1 is clear (0 and 1), after the host rounds it to half. Measured on RDNA2 (gfx1036) at IEEE=0 and IEEE=1 with denorm16 0..3 over every non-NaN f16 input (63,490 rows per mode and instruction): the only mismatches are the 1-ULP driver differences of `v_exp_f16` (2 inputs) and `v_sin_f16` (14), which are the same with denormals kept. NaN inputs are not compared. `v_ldexp_f16` shares the input rule. `v_cvt_pknorm_i16_f16` and `v_cvt_pknorm_u16_f16` flush both half inputs by the same bit, measured on RDNA2 (gfx1030) over every f16 input (each half also negated in the other source) at IEEE=0 and IEEE=1 with denorm16 0..3 and at f16 rounding 1..3: every row matches. Shaders without a known float mode keep the previous lowering. - [v_div_fixup_f16](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) flushes its src1 and src2 to a zero of the same sign when bit 0 of the `FLOAT_MODE` f16 denormal field is clear (0 and 2) and keeps a denormal src0 and result in every mode, as `v_div_fixup_f32` does with the f32 field. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) over 6859 triples of 19 halves (zeros, denormals, minimum normals, ±0.5, ±1, the largest finite value, infinities and quiet and signaling NaNs) at IEEE=0 and IEEE=1 with denorm16 0..3 and FP16_OVFL 0 and 1, also with neg and abs source modifiers and with op_sel high halves. Shaders without a known float mode keep the previous lowering. The PS5's gfx1013 is not measured. - [Packed f16 NaN results](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (`v_pk_add/mul/fma/min/max_f16`) follow the hardware measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1 with f16 round-to-nearest-even and f16 denormals kept: with IEEE=1 a signaling NaN input is quieted and min/max return the other operand only for a quiet NaN; with IEEE=0 NaN inputs pass through unquieted and min/max ignore signaling. In `v_pk_fma_f16` an invalid product (inf * 0) gives the default NaN even when src2 is a NaN. A shader without a known float mode is translated as IEEE=0, the mode titles use; a mode with another f16 rounding throws. In the f16 denormal modes that flush, each denormal input lane is flushed to a zero of its sign when bit 0 of the field is clear (0 and 2), so `inf * 0x0001` is an invalid product there, and when bit 1 is clear (0 and 1) a `v_pk_add_f16` result below 2^-14, or a `v_pk_mul_f16`, `v_pk_fma_f16` or `v_pk_fmac_f16` result whose exact value rounded to 11 significant bits with an unbounded exponent is below 2^-14, is flushed to a zero of its sign; `v_pk_min/max_f16` flush only their inputs. The native f16x2 add and multiply is not used while results are flushed. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with denorm16 0..3, over 19,192 to 162,204 lanes per instruction with op_sel, op_sel_hi, neg_lo, neg_hi and clamp variants and products and sums at the 2^-14 boundary. -- [f32 denormals](../../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/mullit/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign (`v_mullit_f32` measured on RDNA2 (gfx1036) over 6907 rows per mode at IEEE=0 and IEEE=1, so a positive denormal src2 counts as zero); the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. `v_div_scale_f32` flushes all three inputs to signed zeros, measured on RDNA2 (gfx1036) over 2744 operand triples per mode at IEEE=0 and IEEE=1 with f32 denormals flushed and kept; its scale-down behaviour and host denormal preservation remain incomplete. `v_div_fixup_f32` flushes its src1 and src2 and keeps a denormal src0 and result, measured on RDNA2 (gfx1036) over 6859 rows per mode at IEEE=0 and IEEE=1. Tininess is detected after rounding, measured on RDNA2 (gfx1036) for `v_mul_f32` and `v_mullit_f32` at IEEE=0 and IEEE=1: a product is flushed when its value rounded to 24 bits with an unbounded exponent is below 2^-126, so `0x3f7ffffe * 0x00800001` (2^-126 * (1 - 2^-46)) gives `0x00800000` and `0x3f7fffff * 0x00800000` (2^-126 * (1 - 2^-24)) gives zero. The check rounds in the f32 round mode of `FLOAT_MODE` (bits 1:0), measured in all four modes, while the products themselves are still rounded to nearest even. This applies to `v_mul_f32`, `v_mul_legacy_f32`, `v_mullit_f32`, and to `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. In the three directed modes the check is computed exactly from the significands. To nearest even (0, the mode of shaders without a known float mode) one factor is multiplied by 2^64, which is exact below 2^64, and the product with the other factor is compared with 2^-62. That multiply is a plain `OpFMul` like the product, and no f32 rounding mode is requested in the SPIR-V, so the check relies on the driver rounding f32 to nearest even by default, as the products do, and a product that rounds up to 2^-126 is the emitted product. `v_fma_mix_f32` follows this rule with mixed f16/f32 sources too, measured on RDNA2 (gfx1036) with f16 in either multiplicand slot, low and high half selection, and an f32 or f16 zero addend: f32 `0x00801002` times f16 `0x3bff` gives `0x00800000` to nearest at IEEE=0 and IEEE=1, while the neighbours `0x00801001` and `0x00801003` give zero and `0x00800001`. The tiny-product boundary was also measured for both signs in all four f32 round modes at IEEE=0; Its f16 sources are flushed to a zero of their sign when bit 0 of the f16/f64 denormal field is clear (0 and 2), whatever the f32 field, measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with f32 denormal modes 0 and 3 and f16 denormal modes 0 to 3, over 2744 triples of 14 halves with all three sources f16 and 1960 rows with an f32 multiplicand. `v_fma_mixlo_f16` and `v_fma_mixhi_f16` flush their f32 sources too and keep a product below 2^-126, measured on RDNA2 (gfx1036). `v_mad_legacy_f32` and `v_mac_legacy_f32` flush their inputs, their product and their result in every mode, see its entry; their product is still checked for tininess on its exact value, so one below 2^-126 gives zero even when it rounds to 2^-126, as the console's separately rounded encoding is not measured at this boundary. Denormal modes 1 and 2 throw for the other instructions except the f32 compares and the f32 conversions in their entries. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_dot2_f32_f16` and `v_div_fmas_f32` (on hardware `v_div_fmas_f32` flushes only its result), f32 interpolation and `ds_add_f32` keep the previous lowering; `ds_min/max/cmpst_f32` and the image, buffer, global and flat `fmin/fmax/fcmpswap` flush both operands by the field, see the float atomics entries. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested; `v_ldexp_f32` and the f32 compares are the exceptions. `v_cmp_X_f32` and `v_cmpx_X_f32` compare the operands as sign-magnitude integers with the NaN cases handled apart, so a denormal is not read as zero, measured on RDNA2 (gfx1036) over 256 operand pairs per opcode at IEEE=0 and IEEE=1 with f32 denormals kept and flushed. `v_ldexp_f32` preserves subnormal inputs and rounds a subnormal result in the f32 round mode of `FLOAT_MODE`, measured on RDNA2 (gfx1036) over 2163 rows in each of the four round modes at IEEE=0 and IEEE=1. An overflow rounds like any other result (infinity, or the largest finite value when the mode rounds that sign toward zero). With f32 denormals flushed it flushes a subnormal input and result, measured over the same rows in each round mode at IEEE=0 only. A device whose driver splits `Fma` into a multiply and an add flushes a product below 2^-126 before adding a non-zero addend in `v_fma_f32` and `v_fmac_f32`: `0x1fffffff * 0x1fffffff + 0x00800000` gives `0x00800000` instead of the fused `0x00ffffff` on RADV RENOIR (Mesa 26.2.4), whose compiled shader has `v_mad_f32` and no `v_fma_f32`. [F32DenormalFlush](../../../core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp) accepts that sum on such a device. +- [f32 denormals](../../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/mullit/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign (`v_mullit_f32` measured on RDNA2 (gfx1036) over 6907 rows per mode at IEEE=0 and IEEE=1, so a positive denormal src2 counts as zero); the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. `v_div_scale_f32` flushes all three inputs to signed zeros, measured on RDNA2 (gfx1036) over 2744 operand triples per mode at IEEE=0 and IEEE=1 with f32 denormals flushed and kept; its scale-down behaviour and host denormal preservation remain incomplete. `v_div_fixup_f32` flushes its src1 and src2 and keeps a denormal src0 and result, measured on RDNA2 (gfx1036) over 6859 rows per mode at IEEE=0 and IEEE=1. Tininess is detected after rounding, measured on RDNA2 (gfx1036) for `v_mul_f32` and `v_mullit_f32` at IEEE=0 and IEEE=1: a product is flushed when its value rounded to 24 bits with an unbounded exponent is below 2^-126, so `0x3f7ffffe * 0x00800001` (2^-126 * (1 - 2^-46)) gives `0x00800000` and `0x3f7fffff * 0x00800000` (2^-126 * (1 - 2^-24)) gives zero. The check rounds in the f32 round mode of `FLOAT_MODE` (bits 1:0), measured in all four modes, while the products themselves are still rounded to nearest even. This applies to `v_mul_f32`, `v_mul_legacy_f32`, `v_mullit_f32`, and to `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. In the three directed modes the check is computed exactly from the significands. To nearest even (0, the mode of shaders without a known float mode) one factor is multiplied by 2^64, which is exact below 2^64, and the product with the other factor is compared with 2^-62. That multiply is a plain `OpFMul` like the product, and no f32 rounding mode is requested in the SPIR-V, so the check relies on the driver rounding f32 to nearest even by default, as the products do, and a product that rounds up to 2^-126 is the emitted product. `v_fma_mix_f32` follows this rule with mixed f16/f32 sources too, measured on RDNA2 (gfx1036) with f16 in either multiplicand slot, low and high half selection, and an f32 or f16 zero addend: f32 `0x00801002` times f16 `0x3bff` gives `0x00800000` to nearest at IEEE=0 and IEEE=1, while the neighbours `0x00801001` and `0x00801003` give zero and `0x00800001`. The tiny-product boundary was also measured for both signs in all four f32 round modes at IEEE=0; Its f16 sources are flushed to a zero of their sign when bit 0 of the f16/f64 denormal field is clear (0 and 2), whatever the f32 field, measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with f32 denormal modes 0 and 3 and f16 denormal modes 0 to 3, over 2744 triples of 14 halves with all three sources f16 and 1960 rows with an f32 multiplicand. `v_fma_mixlo_f16` and `v_fma_mixhi_f16` flush their f32 sources too and keep a product below 2^-126, measured on RDNA2 (gfx1036). `v_mad_legacy_f32` and `v_mac_legacy_f32` flush their inputs, their product and their result in every mode, see its entry; their product is still checked for tininess on its exact value, so one below 2^-126 gives zero even when it rounds to 2^-126, as the console's separately rounded encoding is not measured at this boundary. Denormal modes 1 and 2 throw for the other instructions except the f32 compares and the f32 conversions in their entries. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_dot2_f32_f16`, f32 interpolation and `ds_add_f32` keep the previous lowering; `ds_min/max/cmpst_f32` and the image, buffer, global and flat `fmin/fmax/fcmpswap` flush both operands by the field, see the float atomics entries. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested; `v_ldexp_f32` and the f32 compares are the exceptions. `v_cmp_X_f32` and `v_cmpx_X_f32` compare the operands as sign-magnitude integers with the NaN cases handled apart, so a denormal is not read as zero, measured on RDNA2 (gfx1036) over 256 operand pairs per opcode at IEEE=0 and IEEE=1 with f32 denormals kept and flushed. `v_ldexp_f32` preserves subnormal inputs and rounds a subnormal result in the f32 round mode of `FLOAT_MODE`, measured on RDNA2 (gfx1036) over 2163 rows in each of the four round modes at IEEE=0 and IEEE=1. An overflow rounds like any other result (infinity, or the largest finite value when the mode rounds that sign toward zero). With f32 denormals flushed it flushes a subnormal input and result, measured over the same rows in each round mode at IEEE=0 only. A device whose driver splits `Fma` into a multiply and an add flushes a product below 2^-126 before adding a non-zero addend in `v_fma_f32` and `v_fmac_f32`: `0x1fffffff * 0x1fffffff + 0x00800000` gives `0x00800000` instead of the fused `0x00ffffff` on RADV RENOIR (Mesa 26.2.4), whose compiled shader has `v_mad_f32` and no `v_fma_f32`. [F32DenormalFlush](../../../core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp) accepts that sum on such a device. +- [v_div_fmas_f32](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) on a device without the Float64 capability keeps the earlier f32 lowering, so there it differs from the hardware: a result scaled into the denormal range is rounded twice and can be one unit in the last place off, and in mode 0 with the hardware f32 flush the driver also flushes denormal inputs and can flush a result to +0 instead of a zero of its sign. `agc_driver_division` skips its denormal rows there. The hardware rule, measured on RDNA2 (gfx1036) over 728 operand triples with VCC clear and set at IEEE=0 in each f32 denormal mode: inputs are never flushed; `src0 * src1 + src2`, scaled by 2^64 or 2^-64 when VCC is set, is rounded once; a denormal result is flushed to a zero of its sign when bit 1 of the f32 denormal field is clear (modes 0 and 1). With Float64 the translation follows it. Directed f32 rounding is not modelled. The PS5's gfx1013 is not measured. - [v_rcp, v_rsq, v_sqrt, v_log, v_exp, v_sin and v_cos](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (f32 and f16) use the driver's approximation for finite results, not the console's: the low bits can differ. Only NaN, invalid, zero and infinite results and the exact sin/cos zeros are chosen on the bits. Measured on RDNA2 (gfx1036) over every f16 input and 32,832 f32 inputs: RADV returns the hardware's bits for every input of rcp, rsq, sqrt and log and all but one of exp (f16 0x11c5, 1 ULP); NVIDIA (615.71.09) stays within 2 ULP except `v_log_f32` near 1.0 (next entry). [v_sin and v_cos](../../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take the driver's sine or cosine of the radians of the exact cycle, except within 2^-12 of a turn from a zero of the function, where the result is the linear term of the exact distance to the zero: the hardware's constant 6.2831826 (0x40c90fd5) at the integers, which the hardware applies exactly to inputs below 2^-12 of a turn, and the f32 2π at the half and quarter turns, where the hardware is within 7 ULP of it. Outside those bands, a driver result more than 4 ULP from a Taylor series in turns of the exact cycle is replaced by the series, which is within 2 ULP of the true sine and cosine; the hardware is up to 9 ULP from them (gfx1035, 243,092 f32 inputs), so results stay within 13 ULP of the hardware on any driver. RADV returns the hardware's bits for every f16 input and for 32,422 sine and 32,486 cosine results of the 32,832 f32 inputs; the rest are 1 ULP off, except up to 14 ULP on either side of non-zero integers and up to 15 ULP beyond ±256 cycles, where the hardware's own reduction differs from the exact fraction, and up to 6 ULP inside the half- and quarter-turn bands. NVIDIA's own sine is within 1 ULP on 93% of the f32 inputs and up to 1,166 ULP off just outside the bands, its cosine within 163 ULP; the series replaces those results. A denormal f32 input with f32 denormals kept gives the hardware the input times 6.2831826; both hosts flush it. - [v_rcp_f32 and v_rcp_iflag_f32 with `APS5_EXACT_RCP`](../../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterReciprocal.cpp) return the hardware reciprocal for normal inputs with normal results: a cubic over 32 mantissa segments, the linear term rounded to nearest even, plus 2,761 one-ULP corrections. It matches all 2^23 mantissas of an RDNA2 (gfx1031) sweep; inputs or results that are not normal keep the driver's 1/x. It is off by default because the coefficient and correction tables add about 13k SPIR-V words to each shader that uses it. The switch also reaches v_rcp_f16 (rounded from the f32 reciprocal) and the pixel shader's position W input. - [v_ldexp_f32 in denormal modes 1 and 2](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the `FLOAT_MODE` f32 denormal field: mode 1 keeps a subnormal input and flushes a result below 2^-126 to a zero of its sign in every f32 round mode, also one that rounds to 2^-126 at subnormal precision (`0x3fffffff` scaled by 2^-127); mode 2 flushes a subnormal input to a zero of its sign and rounds a subnormal result in the f32 round mode of `FLOAT_MODE`, as mode 3 does. Measured on RDNA2 (gfx1030, Radeon RX 6800, AMD Windows driver 32.0.21045.5002 through HIP) over 10,260 rows (54 inputs of both signs: zeros, subnormals, normals near 2^-126 and 1.0, the largest finite value, infinities and NaNs, each scaled by 190 exponents from -2^31 to 2^31-1) at IEEE=0 and IEEE=1 with each f32 denormal mode 0 to 3 and f32 round mode 0 to 3, DX10_CLAMP=1, f16 denormals kept, f16 round to nearest even and FP16_OVFL=0; with the `clamp` modifier the result is clamped as in the other modes. The PS5's gfx1013 is not measured. From 6a16714ec1d090a552e66567f7cc60551fb1b4c6 Mon Sep 17 00:00:00 2001 From: xXPBSXx <20614131+xXPBSXx@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:36:05 +0100 Subject: [PATCH 123/318] fix(libkernel): fail pipe with EFAULT and close both descriptors when the descriptor array is not writable guest memory FreeBSD 11.4 sys_pipe2 copies the descriptors out after kern_pipe and, if that fails, closes both and returns EFAULT (sys/kern/sys_pipe.c:479-483). pipe checked the array with HostWrite::Open, which accepts any non-arena host pointer, and wrote it with memcpy, so a pointer into unmapped memory crashed the host. Copy out with the checked write and close both descriptors on failure. Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libkernel/CMakeLists.txt | 10 ++++++++ core/libs/prx/libkernel/File/src/Stdio.cpp | 8 +++--- core/libs/tests/GuestPipeFault.cpp | 30 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 core/libs/tests/GuestPipeFault.cpp diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 9da2728d50..e4f31948b9 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -1885,6 +1885,16 @@ if(BUILD_TESTING) set_tests_properties(guest_fstat_fault PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pipe_fault_tests ${LIBS_INCLUDE_DIR}/tests/GuestPipeFault.cpp) + target_include_directories(guest_pipe_fault_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pipe_fault_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pipe_fault_tests) + add_test(NAME guest_pipe_fault COMMAND guest_pipe_fault_tests) + set_tests_properties(guest_pipe_fault PROPERTIES TIMEOUT 15) + if(WIN32) + set_tests_properties(guest_pipe_fault PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_rename_dot_tests ${LIBS_INCLUDE_DIR}/tests/GuestRenameDot.cpp) target_include_directories(guest_rename_dot_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_rename_dot_tests PRIVATE libkernel libc) diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 486416f762..db5a42bbfb 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -582,8 +582,6 @@ static int NativeFstatfs(int descriptor, KernelStatfs* fs) { extern "C" int APS5_VABI pipe_nid_postfix(int* descriptors) { if (!descriptors) return PosixFailure(GUEST_EFAULT); - const GuestArena::HostWrite destination(descriptors, 2 * sizeof(int)); - if (!destination.Open()) return PosixFailure(GUEST_EFAULT); int native[2]; #ifdef _WIN32 const int result = ::_pipe(native, 65536, _O_BINARY); @@ -593,7 +591,11 @@ extern "C" int APS5_VABI pipe_nid_postfix(int* descriptors) { if (result != 0) return PosixFailure(SceErrorFromErrno(errno) & 0xffff); File::RememberPipe(native[0]); File::RememberPipe(native[1]); - std::memcpy(descriptors, native, sizeof(native)); + if (!GuestCopyOut_nid_no_patch(descriptors, native, sizeof(native))) { + close_nid_postfix(native[0]); + close_nid_postfix(native[1]); + return PosixFailure(GUEST_EFAULT); + } return 0; } diff --git a/core/libs/tests/GuestPipeFault.cpp b/core/libs/tests/GuestPipeFault.cpp new file mode 100644 index 0000000000..e36f8f8cc2 --- /dev/null +++ b/core/libs/tests/GuestPipeFault.cpp @@ -0,0 +1,30 @@ +#include "prx/libc/include/Api.hpp" +#include "prx/libkernel/include/Api.hpp" + +#include +#include +#include +#include + +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "pipe fault check failed at line %d\n", line); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) + +static constexpr int GuestEfault = 14; + +int main() { + int first[2]{-1, -1}; + Require(pipe_nid_postfix(first) == 0); + Require(close_nid_postfix(first[0]) == 0 && close_nid_postfix(first[1]) == 0); + for (const std::uintptr_t address : {std::uintptr_t{1}, ~std::uintptr_t{0xfff}}) { + *__error_nid_postfix() = 0; + Require(pipe_nid_postfix(reinterpret_cast(address)) == -1 && *__error_nid_postfix() == GuestEfault); + } + int second[2]{-1, -1}; + Require(pipe_nid_postfix(second) == 0 && second[0] == first[0] && second[1] == first[1]); + Require(close_nid_postfix(second[0]) == 0 && close_nid_postfix(second[1]) == 0); +} From f88835a37e5fc0e408344cdfd7ed2e252d724233 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:26:25 +0000 Subject: [PATCH 124/318] fix(libSceNetCtl): report sceNetCtlGetInfoV6 on an IPv6 host as not implemented On a host with an IPv6 address, sceNetCtlGetInfoV6 threw a bare runtime_error for every valid code, so progress.py counted it as done. The layout of each answer is unverified, so that path now goes through NotImplemented_nid_no_patch. The INVALID_ADDR, INVALID_CODE and IPv6-less NOT_CONNECTED answers are unchanged. --- core/libs/prx/libSceNetCtl/Export.cpp | 3 ++- core/libs/tests/GuestNetCtl.cpp | 9 +++++++++ docs/dev/debt/libSceNetCtl.md | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceNetCtl/Export.cpp b/core/libs/prx/libSceNetCtl/Export.cpp index d3e153e34a..99fd55536b 100644 --- a/core/libs/prx/libSceNetCtl/Export.cpp +++ b/core/libs/prx/libSceNetCtl/Export.cpp @@ -618,7 +618,8 @@ int APS5_VABI sceNetCtlGetInfoV6(std::uint32_t code, NetCtlInfoV6* info) { if (!info) return SCE_NET_CTL_ERROR_INVALID_ADDR; if (code < NET_CTL_INFO_V6_IP_ADDRESS || code > NET_CTL_INFO_V6_SECONDARY_DNS) return SCE_NET_CTL_ERROR_INVALID_CODE; if (!HostHasAddress(AF_INET6)) return SCE_NET_CTL_ERROR_NOT_CONNECTED; - throw std::runtime_error("sceNetCtlGetInfoV6: info code " + std::to_string(code) + " is not implemented"); + NotImplemented_nid_no_patch(__func__); + return 0; } void APS5_VABI sceNetCtlTerm(void) { diff --git a/core/libs/tests/GuestNetCtl.cpp b/core/libs/tests/GuestNetCtl.cpp index 9a0e595f2b..129b09979d 100644 --- a/core/libs/tests/GuestNetCtl.cpp +++ b/core/libs/tests/GuestNetCtl.cpp @@ -301,6 +301,15 @@ int main() { Require(sceNetCtlGetInfoV6(6, reinterpret_cast(infoV6)) == kErrInvalidCode); Require(sceNetCtlGetInfoV6(1, nullptr) == kErrInvalidAddr); if (stateV6 != kStateIpObtained) Require(sceNetCtlGetInfoV6(1, reinterpret_cast(infoV6)) == kErrNotConnected); + if (stateV6 == kStateIpObtained) { + bool notImplemented = false; + try { + sceNetCtlGetInfoV6(1, reinterpret_cast(infoV6)); + } catch (const std::runtime_error& error) { + notImplemented = std::strcmp(error.what(), "sceNetCtlGetInfoV6 not implemented") == 0; + } + Require(notImplemented); + } const int result = sceNetCtlGetInfo(14, &info); if (result == kErrNotConnected) { diff --git a/docs/dev/debt/libSceNetCtl.md b/docs/dev/debt/libSceNetCtl.md index add05f78df..f6547299a6 100644 --- a/docs/dev/debt/libSceNetCtl.md +++ b/docs/dev/debt/libSceNetCtl.md @@ -9,7 +9,7 @@ - [sceNetCtlScanIpcInt](../../../core/libs/prx/libSceNetCtl/Export.cpp) (libSceNetCtl) - argument registers and IPC transfer directions are recovered for the info, connection, scan, NAT, bandwidth and Wi-Fi queries. The connection configuration is 1464 bytes, scan options 88, BSSID output 96, scan info 112, IPv6 info 256, internal NAT info 196, bandwidth info 32, bandwidth-estimation info 28 and Wi-Fi info 48. Their field semantics still require the receiving service; incomplete declarations preserve this gap. Scan queries transfer a six-byte BSSID and use a 64-bit in/out buffer size. - [sceNetCtlGetInfo](../../../core/libs/prx/libSceNetCtl/Export.cpp) (libSceNetCtl) - answers from the host interface that holds an IPv4 address, preferring the one that carries the default route: its address, netmask, gateway, MTU and first two IPv4 DNS servers. The device is reported as wired, the link as connected, the IP configuration as DHCP and the HTTP proxy as off, whatever the host uses, and the Ethernet address is the one `sceNetGetMacAddress` reports. A gateway or DNS server the interface does not have is reported as `0.0.0.0`; what a console reports then is unknown. The DHCP host name (12), the PPPoE user name (13), the HTTP proxy server (20) and the Wi-Fi BSSID and SSID (5, 6) are empty strings, as that configuration sets none of them; the remaining Wi-Fi codes (7-10) and the HTTP proxy port (21) report 0; this follows [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/network.cpp) and is not measured on hardware -- [sceNetCtlGetInfoV6](../../../core/libs/prx/libSceNetCtl/Export.cpp) (libSceNetCtl, NID `Jy1EO5GdlcM`) - imported by PPSA08587. `(int code, SceNetCtlInfoV6* info)` with codes 1-5 (IP address, link-local address, default route, primary and secondary DNS) comes only from [SharpProspero](https://github.com/SvenGDK/SharpProspero), whose source is unknown; shadPS4 and fpPS4 only stub it. A null `info` answers `INVALID_ADDR`, a code outside 1-5 `INVALID_CODE`, and a host without an IPv6 address `NOT_CONNECTED`, as `sceNetCtlGetStateV6` reports it. The layout of each answer is unverified, so every code throws on a host with an IPv6 address +- [sceNetCtlGetInfoV6](../../../core/libs/prx/libSceNetCtl/Export.cpp) (libSceNetCtl, NID `Jy1EO5GdlcM`) - imported by PPSA08587. `(int code, SceNetCtlInfoV6* info)` with codes 1-5 (IP address, link-local address, default route, primary and secondary DNS) comes only from [SharpProspero](https://github.com/SvenGDK/SharpProspero), whose source is unknown; shadPS4 and fpPS4 only stub it. A null `info` answers `INVALID_ADDR`, a code outside 1-5 `INVALID_CODE`, and a host without an IPv6 address `NOT_CONNECTED`, as `sceNetCtlGetStateV6` reports it. The layout of each answer is unverified, so on a host with an IPv6 address every code goes through `NotImplemented_nid_no_patch` - [sceNetCtlGetResult](../../../core/libs/prx/libSceNetCtl/Export.cpp) (libSceNetCtl) - `(int eventType, int* errorCode)` reports no error: event types 1-3 (disconnected to IP obtained, the numbering `sceNetCtlCheckCallback` delivers) write 0 to `errorCode`, as `sceNetCtlGetResultV6` does. The error code a console reports for each event, a disconnection's cause in particular, is unconfirmed. Callers found: PPSA20052 (Worms Armageddon: Anniversary Edition 01.003.000) calls it 2 s into boot, and PPSA24191 (Monster Hunter Wilds) without a network, with event 1 (disconnected) as its first call. A null `errorCode` answers `INVALID_ADDR` and other event types throw, as their error code on a console is unknown - [sceNetCtlGetResultV6](../../../core/libs/prx/libSceNetCtl/Export.cpp) (libSceNetCtl, NID `H5yARg37U5g`) - `(int eventType, int* errorCode)` comes only from [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Net/NetCtl.cs#L246-L251), whose source is unknown; shadPS4 only stubs it. Callbacks registered with `sceNetCtlRegisterCallbackV6` receive no event, so event types 1-3 write 0 to `errorCode`; what a console writes before the first event is unmeasured. A null `errorCode` answers `INVALID_ADDR`, as `sceNetCtlGetResult` does, and other event types throw, as their error code on a console is unknown - [sceNetCtlRegisterCallbackForNpToolkit, sceNetCtlUnregisterCallbackForNpToolkit, sceNetCtlCheckCallbackForNpToolkit](../../../core/libs/prx/libSceNetCtl/Export.cpp) (libSceNetCtl) - a table of eight callbacks apart from the IPv4 and IPv6 ones, with the `(int eventType, void* arg)` callback, follows [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/net_ctl_obj.h), whose NpToolkit check delivers the same events to that table. The three share the register, unregister and dispatch of `sceNetCtlRegisterCallback`, `sceNetCtlUnregisterCallback` and `sceNetCtlCheckCallback`, so they return the same initialization, address, slot-limit and ID errors, and the check reports the state `sceNetCtlCheckCallback` does. Whether the console keeps this table apart, and what `sceNetCtlCheckCallback` does with it, is unverified on hardware From 7bfa9e76743ad3ad9cde2f7c565794ea48fa8a10 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:26:56 +0000 Subject: [PATCH 125/318] fix(libSceRudp): report context activation as not implemented Past its initialization and argument errors, sceRudpActivate threw a bare runtime_error for every valid call, so progress.py counted it as done. Transport mode checks, address conversion and connection activation are not implemented, so the valid call now goes through NotImplemented_nid_no_patch. The NOT_INITIALIZED and INVALID_ARGUMENT answers are unchanged. --- core/libs/prx/libSceRudp/Export.cpp | 3 ++- core/libs/tests/GuestRudpContextLifecycle.cpp | 5 +++-- docs/dev/debt/libSceRudp.md | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceRudp/Export.cpp b/core/libs/prx/libSceRudp/Export.cpp index dd4dd2c260..95b70c2e9f 100644 --- a/core/libs/prx/libSceRudp/Export.cpp +++ b/core/libs/prx/libSceRudp/Export.cpp @@ -128,7 +128,8 @@ int APS5_VABI sceRudpActivate(std::int32_t contextId, const void* address, std:: if (!g_inited) return RUDP_ERROR_NOT_INITIALIZED; if (contextId < 0 || !address) return RUDP_ERROR_INVALID_ARGUMENT; static_cast(addressLength); - throw std::runtime_error(__func__); + NotImplemented_nid_no_patch(__func__); + return 0; } int APS5_VABI sceRudpGetStatus(void* status, std::size_t size) { diff --git a/core/libs/tests/GuestRudpContextLifecycle.cpp b/core/libs/tests/GuestRudpContextLifecycle.cpp index 766e84b8dd..362e628fbb 100644 --- a/core/libs/tests/GuestRudpContextLifecycle.cpp +++ b/core/libs/tests/GuestRudpContextLifecycle.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include extern "C" { @@ -166,8 +167,8 @@ int main() { try { sceRudpActivate(ctx, address, sizeof(address)); - } catch (const std::runtime_error&) { - activationUnsupported = true; + } catch (const std::runtime_error& error) { + activationUnsupported = std::string(error.what()) == "sceRudpActivate not implemented"; } Require(activationUnsupported); diff --git a/docs/dev/debt/libSceRudp.md b/docs/dev/debt/libSceRudp.md index 6b667bd48d..01557aecc9 100644 --- a/docs/dev/debt/libSceRudp.md +++ b/docs/dev/debt/libSceRudp.md @@ -4,7 +4,7 @@ - [sceRudpProcessEvents, sceRudpNetFlush and sceRudpNetReceived](../../../core/libs/prx/libSceRudp/Export.cpp) (libSceRudp) - there is no reliable-UDP stack on the host, and every `sceRudpWrite`, `sceRudpRead` and `sceRudpGetStatus` call here reports a reset or idle context without queueing data, so no context ever holds pending data to process, send or receive; the three calls return success after `sceRudpInit` and `RUDP_ERROR_NOT_INITIALIZED` before it, as [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/rudp/rudp.cpp) returns OK from its stubs. What the console's `sceRudpNetReceived` returns is unverified. PPSA03490 calls `sceRudpProcessEvents(0)` followed by `sceRudpNetFlush()` from its P2P service thread and ignores the first result. - [sceRudpEnableInternalIOThread2](../../../core/libs/prx/libSceRudp/Export.cpp) (libSceRudp) takes `(uint32_t stack_size, uint32_t priority, uint64_t affinity)`, taken from the managed `SceRudp.Rudp.EnableInternalIOThread(uint stackSize, uint priority, ulong affinity)` in PPSA06888's Unity metadata; the first two match `sceRudpEnableInternalIOThread`. It returns success after `sceRudpInit` and `RUDP_ERROR_NOT_INITIALIZED` before it, as `sceRudpEnableInternalIOThread` does, without starting any thread. Whether the console reports other errors, and whether it applies the affinity, is unverified. -- [sceRudpActivate](../../../core/libs/prx/libSceRudp/Export.cpp) takes a context ID, socket address and 32-bit address length. Initialization and initial argument checks match the native implementation; transport mode checks, address conversion and connection activation remain unimplemented and throw. +- [sceRudpActivate](../../../core/libs/prx/libSceRudp/Export.cpp) takes a context ID, socket address and 32-bit address length. Initialization and initial argument checks match the native implementation; transport mode checks, address conversion and connection activation remain unimplemented, so a call that passes the initial checks goes through `NotImplemented_nid_no_patch`. ### Unknown function info From 79d7265aa953cc56e2e4f587fbcb2561ce5a1dd5 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:27:27 +0000 Subject: [PATCH 126/318] fix(libkernel): report the internal pool query, mlockall and clock_settime as not implemented sceKernelInternalMemoryGetAvailableSize with an output, mlockall with valid flags and clock_settime with a valid CLOCK_REALTIME request threw bare exceptions on every such call, so progress.py counted them as done. Internal pool statistics, locking every mapping and setting the clock are not implemented, so those calls now go through NotImplemented_nid_no_patch. The -1, EINVAL and EFAULT answers before them are unchanged. --- core/libs/prx/libkernel/DirectMemory/Export.cpp | 3 ++- core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp | 4 ++-- core/libs/prx/libkernel/Time/Time.cpp | 5 ++--- core/libs/tests/GuestFreebsdClocks.cpp | 10 +++++----- core/libs/tests/GuestMemory.cpp | 4 ++-- core/libs/tests/GuestMlock.cpp | 4 ++-- docs/dev/debt/libkernel.md | 6 +++--- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 93101cb207..0c54de321b 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -468,7 +468,8 @@ int APS5_VABI sceKernelConfiguredFlexibleMemorySize(size_t* size) { int APS5_VABI sceKernelInternalMemoryGetAvailableSize(std::size_t* size) { if (!size) return -1; - throw std::logic_error("sceKernelInternalMemoryGetAvailableSize: internal pool statistics are not implemented"); + NotImplemented_nid_no_patch(__func__); + return 0; } int APS5_VABI sceKernelSetVirtualRangeName(const void* addr, uint64_t len, const char* name) { diff --git a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp index 24ad4c0331..436894af6b 100644 --- a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp @@ -308,8 +308,8 @@ int APS5_VABI mlockall_nid_postfix(int flags) { SetError(GuestInvalid); return -1; } - if ((flags & GuestLockFuture) != 0) throw std::runtime_error("mlockall: MCL_FUTURE is not implemented: later mappings are not locked"); - throw std::runtime_error("mlockall: MCL_CURRENT is not implemented: the process also holds host stacks, heaps, modules and driver mappings"); + NotImplemented_nid_no_patch("mlockall"); + return 0; } int APS5_VABI munlockall_nid_postfix() { diff --git a/core/libs/prx/libkernel/Time/Time.cpp b/core/libs/prx/libkernel/Time/Time.cpp index 4e3d97ebca..35bbbbcdc0 100644 --- a/core/libs/prx/libkernel/Time/Time.cpp +++ b/core/libs/prx/libkernel/Time/Time.cpp @@ -619,9 +619,8 @@ int APS5_VABI clock_settime_nid_postfix(int clockId, const KernelTimespec* tp) { *__error_nid_postfix() = GUEST_EINVAL; return -1; } - char requested[48]; - std::snprintf(requested, sizeof(requested), "%lld.%09lld", static_cast(tp->tv_sec), static_cast(tp->tv_nsec)); - throw std::runtime_error(std::string("clock_settime: setting CLOCK_REALTIME to ") + requested + " is not supported, the host clock is not changed"); + NotImplemented_nid_no_patch("clock_settime"); + return 0; } // --------------------------------------------------------------------------- diff --git a/core/libs/tests/GuestFreebsdClocks.cpp b/core/libs/tests/GuestFreebsdClocks.cpp index 1ae0f71258..f5c79ab62b 100644 --- a/core/libs/tests/GuestFreebsdClocks.cpp +++ b/core/libs/tests/GuestFreebsdClocks.cpp @@ -232,12 +232,12 @@ static void RequireSettimeError(int clockId, const KernelTimespec* time, int err Require(clock_settime_nid_postfix(clockId, time) == -1 && *__error_nid_postfix() == error); } -static bool SettimeThrows(const KernelTimespec& time, const char* requested) { +static bool SettimeNotImplemented(const KernelTimespec& time) { try { clock_settime_nid_postfix(GUEST_CLOCK_REALTIME, &time); } catch (const std::runtime_error& error) { - return std::strstr(error.what(), requested) != nullptr; + return std::strcmp(error.what(), "clock_settime not implemented") == 0; } return false; } @@ -256,9 +256,9 @@ static void SettimeValidatesLikeFreebsd() { RequireSettimeError(GUEST_CLOCK_REALTIME, &wholeSecondOfNanoseconds, 22); RequireSettimeError(GUEST_CLOCK_REALTIME, &negativeSeconds, 22); RequireSettimeError(GUEST_CLOCK_MONOTONIC, &negativeSeconds, 22); - Require(SettimeThrows(valid, "1767225600.000000005")); - Require(SettimeThrows(KernelTimespec{0, 0}, "0.000000000")); - Require(SettimeThrows(KernelTimespec{1, NANOS_PER_SECOND - 1}, "1.999999999")); + Require(SettimeNotImplemented(valid)); + Require(SettimeNotImplemented(KernelTimespec{0, 0})); + Require(SettimeNotImplemented(KernelTimespec{1, NANOS_PER_SECOND - 1})); } static std::int32_t RuntimeDaylight(std::int32_t daylight) { diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 7f78313a05..dc45e9d8f2 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -136,8 +136,8 @@ static void CheckInternalMemoryAvailableSize() { try { sceKernelInternalMemoryGetAvailableSize(&available); - } catch (const std::logic_error&) { - threw = true; + } catch (const std::runtime_error& error) { + threw = std::string(error.what()) == "sceKernelInternalMemoryGetAvailableSize not implemented"; } Require(threw && available == 1); Require(sceKernelInternalMemoryGetAvailableSize(nullptr) == -1); diff --git a/core/libs/tests/GuestMlock.cpp b/core/libs/tests/GuestMlock.cpp index 71883b2b4c..1327f4030f 100644 --- a/core/libs/tests/GuestMlock.cpp +++ b/core/libs/tests/GuestMlock.cpp @@ -29,8 +29,8 @@ static void RequireLockAllUnsupported(int flags) { try { mlockall_nid_postfix(flags); - } catch (const std::runtime_error&) { - rejected = true; + } catch (const std::runtime_error& error) { + rejected = std::string(error.what()) == "mlockall not implemented"; } Require(rejected); } diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 4e1851fb4e..8925ea04d4 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -9,7 +9,7 @@ - [sceCoredumpSetUserDataType](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel, NID `Uxqkdta7wEg`) - `(int type)` follows the PS5 12.00 `libkernel.sprx` at 0x138B0: the sign-extended `type` goes to syscall 601 (command 29, third argument 0), and the raw result comes back with no SCE code mapping, so a kernel error is -1 with `errno` set. Which types the kernel accepts is not measured. No dump is produced here, so every type returns 0. Palworld (PPSA23126, #2598) and Hello Neighbor (PPSA35154, discussion #6991) import it; #4144 left it throwing because no signature was known - [sceKernelGetMainSocId and sceKernelGetCpumode](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) both return 0, and [sceKernelGetCpuFrequency](../../../core/libs/prx/libkernel/Time/Time.cpp) returns the host TSC frequency measured for `sceKernelGetTscFrequency`, not the console CPU clock. The PS5's SoC id and CPU mode are unknown; shadPS4's PS4 values are [0x710f10/0x740f30 and 0, 2 or 5 from the param.sfo CPU-mode flags](https://github.com/shadps4-emu/shadPS4/blob/e0a466753a1a/src/core/libraries/kernel/process.cpp). PPSA99203 only logs them (`[console] main SoC id: result=%#x out=%#x`, `CPU: %llu Hz, cpumode %d`), passing a pointer that `sceKernelGetMainSocId` leaves untouched - [sceKernelEnableDmemAliasing](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) returns 0 and changes nothing: direct memory can always be mapped more than once here, as `DoMapDirect` in [DirectMemory.cpp](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) only checks that the physical pages are allocated, so there is no aliasing to turn on. [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/e0a466753a1aacbc8db8253be685bf7ab4d9daae/src/core/libraries/kernel/memory.cpp#L39-L43) (PS4) sets a flag there that lifts the `EBUSY` its [`sceKernelMapDirectMemory2`](https://github.com/shadps4-emu/shadPS4/blob/e0a466753a1aacbc8db8253be685bf7ab4d9daae/src/core/libraries/kernel/memory.cpp#L299-L301), and [`sceKernelMapDirectMemory` for SDK 2.50 and later](https://github.com/shadps4-emu/shadPS4/blob/e0a466753a1aacbc8db8253be685bf7ab4d9daae/src/core/libraries/kernel/memory.cpp#L237-L242), return for direct memory that is [already mapped](https://github.com/shadps4-emu/shadPS4/blob/e0a466753a1aacbc8db8253be685bf7ab4d9daae/src/core/memory.cpp#L548-L553); that check is not applied here. The no-argument `int` signature follows shadPS4 and ProsperoEden (PPSA99008), which calls it before mapping the same direct memory again. Whether the PS5 refuses such a mapping without the call is unverified -- [sceKernelInternalMemoryGetAvailableSize](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - internal pool allocation and statistics are not implemented, so a non-null query throws without changing its output. Native null output returns -1; an absent pool returns -2, a failed statistics query returns -3, and success writes total pool bytes minus used bytes. +- [sceKernelInternalMemoryGetAvailableSize](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - internal pool allocation and statistics are not implemented, so a non-null query goes through `NotImplemented_nid_no_patch` without changing its output. Native null output returns -1; an absent pool returns -2, a failed statistics query returns -3, and success writes total pool bytes minus used bytes. - [sceKernelIsThreadSanitizerEnabled](../../../core/libs/prx/libkernel/System/src/Sanitizer.cpp) - the native zero-argument query returns bit 3 of the process sanitizer flags loaded from sysctl `{1, 14, 41}`. The host always reports disabled because guest sanitizer activation is not modeled. ### Unknown function info @@ -246,7 +246,7 @@ - [msync](../../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp) (libkernel) follows FreeBSD 11's `kern_msync` and `vm_map_sync`: the address is rounded down to the page, a range that wraps is `EINVAL`, as is `MS_ASYNC` with `MS_INVALIDATE`, and a range whose start or any page is not mapped (as `sceKernelVirtualQuery` reports it, reservations included) is `ENOMEM`; a page unmapped from inside a larger mapping still counts as mapped there, since `sceKernelVirtualQuery` keeps reporting the whole mapping. The guest has no file mappings, so a successful call writes nothing back. `MS_INVALIDATE` on a mapped range fails with `EBUSY`, as it did in a payload process on a PS5 (firmware 13.60) for anonymous memory, locked or not, also with a hole unmapped inside it; a title process, reservations, direct memory and flexible memory were not measured - [madvise](../../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp) (libkernel) accepts `MADV_PROTECT` (10) as a no-op hint without modifying process protection. FreeBSD 11's `kern_madvise` (`sys/vm/vm_mmap.c`) handles it before the address check, ignores the address and length, and calls `kern_procctl(..., PROC_SPROTECT, PPROT_SET)`, which requires `PRIV_VM_MADV_PROTECT` and fails with `EPERM` without it. Returning 0 unconditionally assumes the calling process is privileged. Console behaviour is unverified - [mincore](../../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp) (libkernel) follows FreeBSD 11's `sys_mincore` (`sys/vm/vm_mmap.c`), not measured on the console: the address is rounded down to the page, length 0 returns 0, a null vector over a mapped range returns `EFAULT`, an overflowing range or any unmapped page in the range (as reported by `sceKernelVirtualQuery`) returns `ENOMEM`. For valid ranges, a page reports `MINCORE_INCORE` (0x1) when `sceKernelVirtualQuery` reports it committed and 0 when it is only reserved; actual residency, referenced and modified bits are not modelled. Length 0 returns 0 when the start page is mapped and `ENOMEM` otherwise, and the range and mapping checks come before the null vector `EFAULT`, as in `kern_mincore` Java 25 LAN / DNS (PPSA99869) imports it -- [mlockall, munlockall](../../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp) (libkernel) - `mlockall` returns `EINVAL` for zero or unknown flags. Valid `MCL_CURRENT` throws because it would lock host stacks, heaps, modules and driver mappings along with guest memory; `MCL_FUTURE` throws because later mappings are not tracked for locking. FreeBSD 11.4's `sys_munlockall` (`sys/vm/vm_mmap.c`:1133-1165) checks `priv_check(td, PRIV_VM_MUNLOCK)` first (returning `EPERM` without it) and unwires the user map (`vm_map_unwire` over `vm_map_min`...`vm_map_max` with `VM_MAP_WIRE_HOLESOK`). Returning 0 unconditionally assumes the calling process is privileged. `munlockall` unlocks memory locked by `mlock` (`VirtualUnlock` on Windows, tolerating `ERROR_NOT_LOCKED` for pages never locked, and `munlockall` on Linux). XPSemu (PPSA97358) imports `mlockall` and `munlockall`; their calls in QEMU `os-posix.c` execute only when `-overcommit mem-lock=on` is enabled. The call has not been observed in a run with this implementation. +- [mlockall, munlockall](../../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp) (libkernel) - `mlockall` returns `EINVAL` for zero or unknown flags. Valid flags go through `NotImplemented_nid_no_patch`, as `MCL_CURRENT` would also lock host stacks, heaps, modules and driver mappings, and later mappings are not tracked for `MCL_FUTURE`. FreeBSD 11.4's `sys_munlockall` (`sys/vm/vm_mmap.c`:1133-1165) checks `priv_check(td, PRIV_VM_MUNLOCK)` first (returning `EPERM` without it) and unwires the user map (`vm_map_unwire` over `vm_map_min`...`vm_map_max` with `VM_MAP_WIRE_HOLESOK`). Returning 0 unconditionally assumes the calling process is privileged. `munlockall` unlocks memory locked by `mlock` (`VirtualUnlock` on Windows, tolerating `ERROR_NOT_LOCKED` for pages never locked, and `munlockall` on Linux). XPSemu (PPSA97358) imports both; QEMU's `os-posix.c` calls them only with `-overcommit mem-lock=on`, and no run here has reached them. - [socket](../../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) creates only `SOCK_STREAM` and `SOCK_DGRAM` sockets in `AF_INET` and `AF_INET6`. A type that FreeBSD 11.4's inet domain has no protocol for in any configuration ([`in_proto.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/netinet/in_proto.c) has no `SOCK_RDM` entry and no unknown type) fails with `EPROTOTYPE` when the protocol is 0 and `EPROTONOSUPPORT` otherwise, as [`socreate`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/kern/uipc_socket.c) does. `SOCK_RAW`, which `in_proto.c` always has (`socreate` creates it for a privileged process), and `SOCK_SEQPACKET`, which it has only in a kernel built with `SCTP`, fail with `EPROTONOSUPPORT`; whether the console allows either is not known. `SOCK_DGRAM` with `IPPROTO_UDPLITE` and `AF_UNIX` (`socketpair` supports it) fail with `EPROTONOSUPPORT` and `EAFNOSUPPORT` where FreeBSD creates the socket - [sigtimedwait](../../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) waits on a process-wide pending set instead of throwing: a signal raised while blocked stays pending and is taken by `sigwait`, `sigwaitinfo` or `sigtimedwait`, delivered to the handler when `_sigprocmask` unblocks it, or delivered by `pause`. The set covers all 128 signals and the lowest number that is pending and in the set is selected (FreeBSD specifies the lowest number only among real-time signals). Selection, `EAGAIN` on timeout, `EINVAL` for a `tv_nsec` outside 0 to 1000000000 while nothing is pending and `NULL` timeout blocking indefinitely follow FreeBSD's [sigtimedwait(2)](https://man.freebsd.org/cgi/man.cgi?query=sigtimedwait). `sigwait` returns 0 with the number in its output, as POSIX specifies, and `sigwaitinfo` and `sigtimedwait` return the number, as FreeBSD does. The guest `siginfo_t` follows FreeBSD 11's `sys/sys/siginfo.h`: 80 bytes, `si_signo` at 0, `si_code` at 8, `si_pid` at 12, `si_uid` at 16, `si_addr` at 24, `si_value` at 32 and `si_band` at 40; only `si_signo`, `si_code` (`SI_USER`) and `si_pid` (the guest pid) are written, `si_uid` stays 0 because no uid is modelled, and the console's layout is not verified. PPSA85300 imports `sigtimedwait` and fails to load without it; its only call site is VLC's `vlc_writev` ([filesystem.c](https://github.com/videolan/vlc/blob/6de05adcbaf2e8b85fe86aad4169393098628119/src/posix/filesystem.c#L243-L278)), which polls with a zero timeout to drain a `SIGPIPE` only after `writev` fails with `EPIPE`. The waits are not cancellation points and an unblocked caught signal does not interrupt them with `EINTR`, where FreeBSD's `sigwaitinfo` and `sigtimedwait` may return it. `SIGPIPE` has no host equivalent and is never raised by a host write here (see `read, write and vector I/O`), so VLC's drain finds nothing - [pause, sigsuspend, sigwait, sigwaitinfo](../../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) take the pending set of `sigtimedwait` instead of throwing. `pause` waits for a pending signal that is not blocked and whose disposition can be delivered, runs its handler on the calling thread and returns -1 with `EINTR`; an ignored signal is discarded and the wait continues, and it never returns for a signal whose handler cannot run. A signal whose registered handler has `SA_SIGINFO` still throws, and one whose disposition is the default action and that has no host equivalent (every signal except 2, 4, 6, 8, 11 and 15) is neither delivered nor discarded: it stays pending until a wait takes it, where FreeBSD runs the default action, which terminates the process for most of them. A pending `SIGALRM` with the default disposition exits the process with status 142 once it is unblocked, as FreeBSD's default action does. PPSA68091, PPSA99169, PPSA99420 and PPSA97358 import them and fail to load without them. PPSA68091 calls `pause` at one site (0x8F7266), in a loop that waits while a global holds another thread's `errno` address. PPSA99169 only stores the address of `pause` in a table of symbol addresses (0x30DB1). PPSA99420 calls `pause` at 0x3BD80F6 and 0x3C9E220, `sigwait` at 0x3C9E4C7 and `sigwaitinfo` at 0x3C9E588. PPSA97358 calls `sigwait` from QEMU's `sigwait_compat` thread ([compatfd.c](https://github.com/ZiZc3/XPSemu/blob/8eec07b4e31f66dce8e0b05c1ca1281af303e933/util/compatfd.c#L28-L53)), which `qemu_signal_init` ([main-loop.c](https://github.com/ZiZc3/XPSemu/blob/8eec07b4e31f66dce8e0b05c1ca1281af303e933/util/main-loop.c#L110-L147)) starts for `SIGIO`, `SIGALRM` and `SIGBUS` when the main loop is created, so it reaches the wait at start-up, and from the vCPU thread of the dummy accelerator ([dummy-cpus.c](https://github.com/ZiZc3/XPSemu/blob/8eec07b4e31f66dce8e0b05c1ca1281af303e933/accel/dummy-cpus.c#L48-L56)). `sigsuspend` swaps the process-wide mask of `_sigprocmask` for its argument, waits as `pause` does through the same helper, then restores the mask through `_sigprocmask`, and returns -1 with `EINTR`; a null set throws, as the console faults in libthr instead of returning `EFAULT`. All four words of the argument replace the mask and are restored afterwards, as in `_sigprocmask`. Java 25 LAN / DNS (PPSA99869, #5501) imports it and fails to load without it, after `sysctlnametomib`, `fegetenv` and `mincore`. None of the five waits is a cancellation point @@ -280,7 +280,7 @@ - [sysctl kern.arandom](../../../core/libs/prx/libkernel/System/src/SystemConfiguration.cpp) (libkernel) serves FreeBSD 11's MIB {1, 37} (CTL_KERN, KERN_ARANDOM) and the "kern.arandom" sysctlbyname lookup through std::random_device, returning min(*oldLength, 256) random bytes. Following FreeBSD's sysctl_kern_arnd, a null buffer reports min(*oldLength, 256) without writing, a null length returns 0 without writing, and requests with a new value return EPERM (1) as the node is read-only. Cryptographic quality depends on the host's random device implementation. - [pthread_atfork](../../../core/libs/prx/libkernel/Pthread/Posix/Atfork.cpp) (libkernel) appends the three handlers to a process-wide list and returns 0, or `ENOMEM` when the entry cannot be allocated, as FreeBSD 11.4's [`_pthread_atfork`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libthr/thread/thr_fork.c#L78-L100) does; null handlers are accepted. The handlers never run: `fork` throws and `vfork` is absent, so no child process is created. Handlers of a module that is unloaded stay in the list, where FreeBSD's `__pthread_cxa_finalize` removes them. PPSA97358 has a source-level call from the constructor `rcu_init` ([rcu.c](https://github.com/ZiZc3/XPSemu/blob/8eec07b4e31f66dce8e0b05c1ca1281af303e933/util/rcu.c#L488-L495)), but current-main runtime reachability is unverified - [sceKernelGetPageTableStats](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) reports 8192 entries on each side and counts them as a PS5 (firmware 13.60) counted them for a payload process: one cpu entry for each 2 MiB granule that holds a committed mapping, whatever its protection (none included), and one gpu entry for each 1 GiB region that holds a GPU-accessible mapping, plus one for each 2 MiB granule of such a mapping that is not a whole granule of direct memory at a 2 MiB-aligned physical address. A granule or region shared by several mappings counts once, and reservations do not count. On the console the first GPU mapping of a process also takes three gpu entries and one cpu entry that stay after it is unmapped, which is not modelled, and only the ranges whose protection is recorded here are counted. A null pointer returns `SCE_KERNEL_ERROR_EFAULT`, where the console faults. Resonance: A Plague Tale Legacy (PPSA30601) calls it at start-up (#4067) -- [clock_settime](../../../core/libs/prx/libkernel/Time/Time.cpp) (libkernel) validates like FreeBSD 11.4's [`kern_clock_settime`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/kern_time.c#L396-L412) for a root caller: `EFAULT` for a null time, `EINVAL` for any clock but `CLOCK_REALTIME`, a negative second count or a nanosecond count outside 0 to 999999999. A valid `CLOCK_REALTIME` request throws, naming the requested time, since the host clock is never set and no guest clock offset is kept. FreeBSD checks `PRIV_CLOCK_SETTIME` first, which a jailed process fails with `EPERM`; whether a title holds it on the console is unverified. PS5MC (PPSA99420) imports it; no call site is known +- [clock_settime](../../../core/libs/prx/libkernel/Time/Time.cpp) (libkernel) validates like FreeBSD 11.4's [`kern_clock_settime`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/kern_time.c#L396-L412) for a root caller: `EFAULT` for a null time, `EINVAL` for any clock but `CLOCK_REALTIME`, a negative second count or a nanosecond count outside 0 to 999999999. A valid `CLOCK_REALTIME` request goes through `NotImplemented_nid_no_patch`, since the host clock is never set and no guest clock offset is kept. FreeBSD checks `PRIV_CLOCK_SETTIME` first, which a jailed process fails with `EPERM`; whether a title holds it on the console is unverified. PS5MC (PPSA99420) imports it; no call site is known - [Short sleeps ended by GPU and flip completions](../../../core/libs/prx/libkernel/Time/SleepWake.cpp) (libkernel) - a `sceKernelUsleep`, `usleep`, `nanosleep` or `sceKernelNanosleep` of 1 ms or less returns before the requested time, which FreeBSD and the console do not do: it ends once the AGC driver reports a completed GPU batch or batch completion action, or VideoOut completes a flip, so a title that polls a GPU label or the flip status between sleeps does not sleep the rest. A title that paces frames with such a sleep can wake up to 1 ms early. A thread returns early at most 8 times in a run of sleeps; the run restarts when the thread submits AGC work or did not sleep the requested time since its last sleep, so a loop that counts sleeps to time out still sleeps all but 8 in full. `APS5_NO_SLEEP_WAKE=1` keeps every sleep in full - [chflags](../../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) follows FreeBSD 11.4's [`kern_chflagsat`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/vfs_syscalls.c#L2366-L2383) and UFS's [`ufs_setattr`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/ufs/ufs/ufs_vnops.c#L509-L556) for a root caller: the path lookup reports `EFAULT`, `ENOENT` and `ENOTDIR`, a flag outside the 17 UFS stores (`VNOVAL` included) `EOPNOTSUPP`, and `SF_SNAPSHOT` `EPERM`. Neither host keeps BSD file flags, so every file is taken to have none (`stat` reports `st_flags` 0): flags 0 succeed without touching the file, and any other flag throws, naming the flags. `UF_READONLY`, `UF_HIDDEN`, `UF_SYSTEM` and `UF_ARCHIVE` are FreeBSD's names for the Windows attributes but are not mapped to them: UFS stores them without enforcing them, `FILE_ATTRIBUTE_READONLY` already carries the owner write bit of `chmod` on Windows, and Linux has no equivalent a process can set without `CAP_LINUX_IMMUTABLE`. The console's filesystem types, `EROFS` on read-only mounts and the change-time update are not modelled - [fchflags](../../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) has the gaps of `chflags`: a valid non-zero flag throws, and the console's filesystem types, `EROFS` and the change-time update are not modelled. Character devices, the random device, other host descriptors and files read from app0.zar throw, since their console filesystems and the privilege check for devices are not modelled. On Windows, anonymous shared memory objects are temporary files that are not told apart from regular files, so they are treated as files instead of reporting `EINVAL` as FreeBSD 11.4's [`getvnode`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/vfs_syscalls.c#L3850-L3877) does. From 0e1c23cbd8067091b05bedd3b63351f0db3964d2 Mon Sep 17 00:00:00 2001 From: roboterspinne <229981445+roboterspinne@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:07:13 +0100 Subject: [PATCH 127/318] fix(relinker): move a Linux PT_LOAD whose file offset is not page-congruent with its address The Linux patcher kept a PT_LOAD whose p_offset and p_vaddr differ modulo the 4 KiB page in place, so the output executable could not be mapped: the kernel's elf_map passes p_offset minus the page offset of p_vaddr to vm_mmap, which fails with EINVAL when that is not page aligned. The gABI requires p_offset and p_vaddr to be congruent modulo the page size. Such a segment is now copied to the end of the file at an offset congruent to p_vaddr modulo max(p_align, 4 KiB), through the path that already separates shared load pages. Congruent layouts keep their exact bytes, and the Windows and macOS outputs are unchanged. --- .../elfpatcher/src/linux/LinuxElfPatcher.cpp | 5 ++- .../elfpatcher/tests/LinuxLoadPageTests.cpp | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp index 2c406e732b..30dd64f44e 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp @@ -115,7 +115,8 @@ void SeparateSharedLoadPages(std::vector& buf, std::vector image(static_cast(prefix), 0); @@ -130,7 +131,7 @@ void SeparateSharedLoadPages(std::vector& buf, std::vector(mapped) + index] != image[index]; if (!differs) diff --git a/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp b/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp index d96fb4cebb..0828c906d4 100644 --- a/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp +++ b/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp @@ -162,6 +162,43 @@ void ZeroFileBytesStayInPlace() { Require(OutputOffset(output, 0x2c20) == 0x2c20, "a segment whose page already reads right was moved"); } +void IncongruentSegmentAtPageStart() { + auto source = Fixture(0x2000); + std::fill(source.begin() + 0x1000, source.begin() + 0x1010, std::uint8_t{0x33}); + std::fill(source.begin() + 0x1010, source.begin() + 0x1110, std::uint8_t{0xAA}); + const std::vector headers{ + Load(0x400, 0x400, 0x10, 0x10, 5), + Load(0x1010, 0x1000, 0x100, 0x100, 6), + }; + const auto output = Patcher().Patch(source, headers, {}, 0, "$ORIGIN/libs", true, false, {}); + RequireImage(output, source, headers, "segment at a page start with an offset inside a page"); +} + +void IncongruentSegmentInsideASharedPage() { + auto source = Fixture(0x2000); + std::fill(source.begin() + 0x1000, source.begin() + 0x1100, std::uint8_t{0xAA}); + std::fill(source.begin() + 0x1200, source.begin() + 0x1210, std::uint8_t{0x33}); + std::fill(source.begin() + 0x1210, source.begin() + 0x1310, std::uint8_t{0x5A}); + const std::vector headers{ + Load(0x400, 0x400, 0x10, 0x10, 5), + Load(0x1000, 0x1000, 0x100, 0x220, 6), + Load(0x1210, 0x1220, 0x100, 0x100, 4), + }; + const auto output = Patcher().Patch(source, headers, {}, 0, "$ORIGIN/libs", true, false, {}); + RequireImage(output, source, headers, "segment in a shared page with an offset at another place in its page"); +} + +void IncongruentOffsetBelowThePagePrefix() { + auto source = Fixture(0x1000); + std::fill(source.begin() + 0x100, source.begin() + 0x180, std::uint8_t{0xBB}); + const std::vector headers{ + Load(0x400, 0x400, 0x10, 0x10, 5), + Load(0x100, 0x2300, 0x80, 0x80, 4), + }; + const auto output = Patcher().Patch(source, headers, {}, 0, "$ORIGIN/libs", true, false, {}); + RequireImage(output, source, headers, "segment whose offset lies below its page prefix"); +} + } int main() { @@ -169,6 +206,9 @@ int main() { ZeroFillBelowTheNextSegment(); DataAndZeroFillBelowTheNextSegment(); ZeroFileBytesStayInPlace(); + IncongruentSegmentAtPageStart(); + IncongruentSegmentInsideASharedPage(); + IncongruentOffsetBelowThePagePrefix(); } catch (const std::exception& error) { std::cerr << error.what() << '\n'; return 1; From b0a495f2d70156e5062396f857d2a66863e36c46 Mon Sep 17 00:00:00 2001 From: roboterspinne <229981445+roboterspinne@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:44:32 +0100 Subject: [PATCH 128/318] fix(relinker): reject a needed module name that several bundled modules match only by extension A needed name with no exact filename match was aliased to the first discovered module whose name differed only in its .prx, .sprx, .suprx or .debug_prx extension, so with two such modules the pick followed path order. GuestModuleBuilder now throws "Ambiguous needed module" naming both paths, as the other resolution paths in Build already do for ambiguous dependency names and module identities. Such an input now fails without an output instead of giving the name to one module by path order. A module whose filename or SONAME equals the needed name still wins, and a single extension match is still aliased. --- .../relinker/src/guest/GuestModuleBuilder.cpp | 15 ++++++++++++--- .../tests/test_guest_needed_modules.py | 18 +++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp index 31dc35aa0d..e23b1349e3 100644 --- a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp +++ b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp @@ -109,13 +109,20 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path const auto neededNames = ReadNeededNames(dynamic); std::set missingNeeded; std::map neededAliases; + std::map ambiguousAliases; for (const auto& name : neededNames) { if (excludedModules.contains(name)) continue; if (std::any_of(paths.begin(), paths.end(), [&](const auto& path) { return Io::Utf8Path(path.filename()) == name || (windows && FoldFilename(Io::Utf8Path(path.filename())) == FoldFilename(name)); })) continue; const auto stem = ModuleStem(name, windows); - const auto alias = stem.empty() ? paths.end() : std::find_if(paths.begin(), paths.end(), [&](const auto& path) { return ModuleStem(Io::Utf8Path(path.filename()), windows) == stem; }); - if (alias != paths.end()) neededAliases.emplace(name, *alias); - else missingNeeded.insert(name); + const auto matchesStem = [&](const auto& path) { return ModuleStem(Io::Utf8Path(path.filename()), windows) == stem; }; + const auto alias = stem.empty() ? paths.end() : std::find_if(paths.begin(), paths.end(), matchesStem); + if (alias == paths.end()) { + missingNeeded.insert(name); + continue; + } + neededAliases.emplace(name, *alias); + const auto other = std::find_if(std::next(alias), paths.end(), matchesStem); + if (other != paths.end()) ambiguousAliases.emplace(name, *other); } const auto matchesIdentity = [](const std::string& name, const std::vector& identities) { const auto stem = std::string_view(name).substr(0, name.size() - Domain::ModuleSuffix(name).size()); @@ -168,6 +175,8 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path } for (const auto& [name, path] : neededAliases) { if (guestNames.contains(name)) continue; + if (const auto other = ambiguousAliases.find(name); other != ambiguousAliases.end()) + throw Domain::RelinkerException("Ambiguous needed module: " + name + " (" + Io::Utf8Path(path) + " and " + Io::Utf8Path(other->second) + ")"); const auto image = std::find_if(images.begin(), images.end(), [&](const auto& candidate) { return candidate.SourcePath == path; }); if (image == images.end()) throw Domain::RelinkerException("Needed module alias has no guest image: " + name); const auto index = static_cast(image - images.begin()); diff --git a/core/relinker/relinker/tests/test_guest_needed_modules.py b/core/relinker/relinker/tests/test_guest_needed_modules.py index e2f2839072..0cb30bd985 100644 --- a/core/relinker/relinker/tests/test_guest_needed_modules.py +++ b/core/relinker/relinker/tests/test_guest_needed_modules.py @@ -6,7 +6,7 @@ import sys import tempfile from test_guest_intel_trampolines import PLAIN_SITE, guest_fixture, main_fixture -from test_guest_module_directories import module_with_symbol, needed_libraries +from test_guest_module_directories import module_with_soname, module_with_symbol, needed_libraries NEEDED = b"needed.prx" @@ -287,6 +287,22 @@ def main(): if not windows: assert needed_libraries(output.read_bytes()) == ["needed.prx"] + case = work / f"{windows}-debug-name-soname" + (case / "sce_module").mkdir(parents=True) + (case / "sce_module" / "needed.prx").write_bytes(module_with_symbol(True)) + (case / "sce_module" / "needed.sprx").write_bytes(module_with_soname("needed.debug_prx")) + result, output = convert(case, windows, b"needed.debug_prx") + assert result.returncode == 0, (result.stdout, result.stderr) + assert output.exists(), output + + case = work / f"{windows}-debug-name-same-stem" + (case / "sce_module").mkdir(parents=True) + (case / "sce_module" / "needed.prx").write_bytes(module_with_symbol(True)) + (case / "sce_module" / "needed.sprx").write_bytes(module_with_symbol(True)) + result, output = convert(case, windows, b"needed.debug_prx") + assert result.returncode == 2 and "Ambiguous needed module: needed.debug_prx" in result.stderr, (result.stdout, result.stderr) + assert not output.exists(), output + case = work / f"{windows}-ambiguous" for name in ("first", "second"): (case / name).mkdir(parents=True) From 2f5c591495bce4c13743efbba257af4eacbf2a4b Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:27:13 +0000 Subject: [PATCH 129/318] fix(tools): ask perf(tools), perf(ci) and perf(build) pull requests for no title capture --- tools/pr_labels.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/pr_labels.py b/tools/pr_labels.py index f6f0039090..c4bdd87cc5 100644 --- a/tools/pr_labels.py +++ b/tools/pr_labels.py @@ -14,6 +14,8 @@ UNBLOCKS = re.compile(r'Unblocks:[ \t]*([^\r\n]*(?:\r?\n[ \t]*[-*][ \t]+(?!\[[ x TITLE_ID = re.compile(r"(?" UNBLOCKS_LABEL = "unblocks title" @@ -47,6 +49,11 @@ def is_perf(title): return bool(PERF.match(title or "")) +def needs_capture(title): + scope = PERF_SCOPE.match(title or "") + return is_perf(title) and not (scope and scope.group(1).strip().lower() in UNCAPTURED_SCOPES) + + def has_numbers(body): body = strip_comments(body) candidates = [line.strip() for line in body.splitlines()] + [block.strip() for block in FENCE.findall(body)] @@ -69,7 +76,7 @@ def labels_for(pr): wanted.add(UNBLOCKS_LABEL) if is_perf(pr["title"]): wanted.add(PERF_LABEL) - if not has_numbers(pr["body"]): + if needs_capture(pr["title"]) and not has_numbers(pr["body"]): wanted.add(MISSING_LABEL) current = set(pr["labels"]) return wanted - current, (MANAGED & current) - wanted From f5fd58c4e7e6838911364f635b5e75e6e4107c1b Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:27:34 +0000 Subject: [PATCH 130/318] test(tools): cover perf labels for tooling scopes --- tools/tests/test_pr_labels.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/tests/test_pr_labels.py b/tools/tests/test_pr_labels.py index 76298e7657..2bbfba35aa 100644 --- a/tools/tests/test_pr_labels.py +++ b/tools/tests/test_pr_labels.py @@ -93,6 +93,20 @@ class NumbersTests(unittest.TestCase): class LabelTests(unittest.TestCase): + def test_tooling_perf_is_labeled_without_asking_for_a_capture(self): + for title in ("perf(tools): x", "perf(ci): x", "perf(build): x", "perf(Tools): x", "perf( tools ): x"): + with self.subTest(title=title): + self.assertEqual(pr_labels.labels_for(pr(title, TEMPLATE)), ({"perf"}, set())) + + def test_runtime_perf_still_needs_a_capture(self): + for title in ("perf: x", "perf!: x", "perf(agc): x", "perf(libc): x", "perf(toolsy): x"): + with self.subTest(title=title): + self.assertEqual(pr_labels.labels_for(pr(title, TEMPLATE)), ({"perf", "perf numbers missing"}, set())) + + def test_tooling_perf_drops_an_earlier_missing_label(self): + self.assertEqual(pr_labels.labels_for(pr("perf(tools): x", TEMPLATE, ["perf", "perf numbers missing"])), + (set(), {"perf numbers missing"})) + def test_labels(self): self.assertEqual(pr_labels.labels_for(pr()), (set(), set())) self.assertEqual(pr_labels.labels_for(pr("perf: x", TEMPLATE)), ({"perf", "perf numbers missing"}, set())) From ed3cdc81a6f0438a35568950545a384a06ced724 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:27:34 +0000 Subject: [PATCH 131/318] docs(contributing): say which perf pull requests need no title capture --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d96b4d4c11..5d51fa4e36 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,7 @@ The [pull request template](.github/pull_request_template.md) is the checklist f - Fixes a title needs go to `main` as pull requests. A Game bring-up discussion lists the open pull requests and local changes the title still needs; update the list when one is merged, closed or replaced. - The PR overlap comment lists open pull requests that change the same lines or exports; agree with their authors which one goes first and declare it with `Depends on`. - Run the tests before opening the pull request and describe what was tested (OS, title or homebrew). -- A pull request that claims a change in speed shows the summary lines of `main` and of the change, from the same machine and scene, and attaches the traces, captured as the [Performance report](.github/ISSUE_TEMPLATE/performance.yml) issue form describes. A slow scene is reported with the same form. A pull request titled `perf(...)` or `perf:` is labeled `perf`, and `perf numbers missing` until its description holds two valid summary lines with different commits. +- A pull request that claims a change in speed shows the summary lines of `main` and of the change, from the same machine and scene, and attaches the traces, captured as the [Performance report](.github/ISSUE_TEMPLATE/performance.yml) issue form describes. A slow scene is reported with the same form. A pull request titled `perf(...)` or `perf:` is labeled `perf`, and `perf numbers missing` until its description holds two valid summary lines with different commits. A `perf(tools)`, `perf(ci)` or `perf(build)` pull request has no title to capture, so it is labeled `perf` only and reports its own before and after timings. - Changes to the AGC driver, the shader recompiler or VideoOut run the same focused tests on `main` and the change with `VK_INSTANCE_LAYERS=VK_LAYER_KHRONOS_validation`, `ANYPS5_NO_SHADER_CACHE=1` and `ANYPS5_REQUIRE_VULKAN=1` using `ctest -V` from the same machine and GPU, confirm neither log reports `was not found but was requested by env var VK_INSTANCE_LAYERS`, confirm the change introduces no new `VUID-` validation errors compared with `main`, and name each GPU the GPU tests ran on (vendor, model, driver, OS) on the `GPUs tested` checklist line. CI runs these tests only on lavapipe, and hardware drivers fail where lavapipe and other vendors pass, so a reviewer can ask for a run on another vendor before merging. - Investigation notes, reports, screenshots and logs go in the pull request, not in the repository. Images for documentation go in the [gist](https://gist.github.com/boykopovar/0e53f2e1426f29ecd41e3b51540b8a90) comments. - Say whether the change was written with AI assistance. The author of the pull request is responsible for every line of it. From 8da632a873145a7a8dc6924cf0f7ed984762859d Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:27:34 +0000 Subject: [PATCH 132/318] docs(dev): note the perf scopes that need no capture in game testing --- docs/dev/GAME_TESTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/GAME_TESTING.md b/docs/dev/GAME_TESTING.md index 82afb1ccfc..02398dc555 100644 --- a/docs/dev/GAME_TESTING.md +++ b/docs/dev/GAME_TESTING.md @@ -85,7 +85,7 @@ cmd /c '.\app.exe > run.log 2>&1' - Use the title's default settings, and say so. A changed graphics mode makes a number incomparable with everyone else's. - Measure the same spot every time: load a save, wait for the scene to settle, then measure a fixed window with no input. - Compare builds on the same machine, alternating them (A, B, A, B), at least two runs each. Throw away a run that was disturbed (another program, a sleeping display) and say so. -- `APS5_TRACE_SUMMARY_EVERY=10` writes one line per 10 s to `aps5-summary.jsonl` without slowing the title; `tools/trace_digest.py` turns it into a table, and `tools/performance_compare.py` compares two captures ([BUILD](BUILD.md#performance-trace)). Pull requests labelled `perf` need a capture of `main` and one with the change. +- `APS5_TRACE_SUMMARY_EVERY=10` writes one line per 10 s to `aps5-summary.jsonl` without slowing the title; `tools/trace_digest.py` turns it into a table, and `tools/performance_compare.py` compares two captures ([BUILD](BUILD.md#performance-trace)). Pull requests labelled `perf` need a capture of `main` and one with the change, except `perf(tools)`, `perf(ci)` and `perf(build)`. - `APS5_DUMP_FRAMES=` with `APS5_DUMP_FRAMES_EVERY=` saves frames as `frame_.bmp` to check what was on screen. A second process of the same title writes to the same names and overwrites the first one's frames. While 64 MiB of frames wait to be written, the presenter waits for the writer, so a slow disk or `APS5_DUMP_SCALE=1` slows the title down instead of stopping it. From a0ca793303b119f230209339af6fe31a3f20a86d Mon Sep 17 00:00:00 2001 From: Eduardo Ruiz Date: Sat, 24 Oct 2026 14:53:14 +0200 Subject: [PATCH 133/318] feat(libc): register conditions for broadcast at thread exit Export _Cnd_register_at_thread_exit (DV2AdZFFEh8), _Cnd_unregister_at_thread_exit (wpuIiVoCWcM) and _Cnd_do_broadcast_at_thread_exit (vyLotuB6AS4), which Minecraft Legends (PPSA05510) and Rayman 30th (PPSA33016) import. Registrations live in a process-wide list keyed by the host thread id, as Microsoft's xnotify.cpp does. The broadcast of the current thread sets the supplied flag, broadcasts the condition and unlocks the mutex. It runs when a std::thread exits through _Pad::_Launch and when a title calls _Thrd_exit; threads created directly with scePthreadCreate are recorded in debt. --- core/libs/prx/libc/CMakeLists.txt | 10 ++ core/libs/prx/libc/include/Api.hpp | 3 + core/libs/prx/libc/src/Threads.cpp | 54 ++++++- core/libs/tests/GuestCndAtThreadExit.cpp | 176 +++++++++++++++++++++++ docs/dev/debt/libc.md | 1 + 5 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 core/libs/tests/GuestCndAtThreadExit.cpp diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index bf8e7ab043..56507aaaa7 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -506,6 +506,16 @@ if(BUILD_TESTING) set_tests_properties(guest_libc_threads PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_cnd_at_thread_exit_tests ${LIBS_INCLUDE_DIR}/tests/GuestCndAtThreadExit.cpp) + target_include_directories(guest_cnd_at_thread_exit_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_cnd_at_thread_exit_tests PRIVATE libc) + configure_windows_unwind(guest_cnd_at_thread_exit_tests) + add_test(NAME guest_cnd_at_thread_exit COMMAND guest_cnd_at_thread_exit_tests) + set_tests_properties(guest_cnd_at_thread_exit PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_cnd_at_thread_exit PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_std_thread_hardware_concurrency_tests ${LIBS_INCLUDE_DIR}/tests/GuestStdThreadHardwareConcurrency.cpp) target_include_directories(guest_std_thread_hardware_concurrency_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_std_thread_hardware_concurrency_tests PRIVATE libc) diff --git a/core/libs/prx/libc/include/Api.hpp b/core/libs/prx/libc/include/Api.hpp index d25dbadc88..70aa91fb01 100644 --- a/core/libs/prx/libc/include/Api.hpp +++ b/core/libs/prx/libc/include/Api.hpp @@ -1006,6 +1006,9 @@ unsigned APS5_VABI _ZNSt6thread20hardware_concurrencyEv_nid_postfix(); int APS5_VABI _Thrd_sleep_nid_postfix(const Xtime* target, Xtime*); int APS5_VABI _Thrd_yield_nid_postfix(); int APS5_VABI _Cnd_signal_nid_postfix(void** handle); +void APS5_VABI _Cnd_register_at_thread_exit_nid_postfix(void* condition, void* mutex, int* ready); +void APS5_VABI _Cnd_unregister_at_thread_exit_nid_postfix(void* mutex); +void APS5_VABI _Cnd_do_broadcast_at_thread_exit_nid_postfix() noexcept; void APS5_VABI _ZNSt4_PadC2Ev_nid_postfix(GuestPad* pad); void APS5_VABI _ZNSt4_PadD2Ev_nid_postfix(GuestPad* pad); void APS5_VABI _ZNSt4_Pad8_ReleaseEv_nid_postfix(GuestPad* pad); diff --git a/core/libs/prx/libc/src/Threads.cpp b/core/libs/prx/libc/src/Threads.cpp index ffa49e1ea7..637b7d13aa 100644 --- a/core/libs/prx/libc/src/Threads.cpp +++ b/core/libs/prx/libc/src/Threads.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX @@ -388,6 +390,53 @@ int APS5_VABI _Cnd_signal_nid_postfix(void** handle) { return SignalCondition_nid_no_patch(handle); } +struct AtThreadExitEntry { + std::thread::id thread; + DinkumwareCondition* condition; + DinkumwareMutex* mutex; + int* ready; +}; + +static std::mutex atThreadExitLock; +static std::list atThreadExitEntries; + +void APS5_VABI _Cnd_do_broadcast_at_thread_exit_nid_postfix() noexcept { + const auto thread = std::this_thread::get_id(); + std::list pending; + { + const std::lock_guard lock(atThreadExitLock); + for (auto entry = atThreadExitEntries.begin(); entry != atThreadExitEntries.end();) { + if (entry->thread == thread) { + auto moved = entry++; + pending.splice(pending.end(), atThreadExitEntries, moved); + } else { + ++entry; + } + } + } + for (const auto& entry : pending) { + if (entry.ready) *entry.ready = 1; + void* condition = entry.condition; + _Cnd_broadcast_nid_postfix(&condition); + void* mutex = entry.mutex; + _Mtx_unlock_nid_postfix(&mutex); + } +} + +void APS5_VABI _Cnd_register_at_thread_exit_nid_postfix(void* conditionHandle, void* mutexHandle, int* ready) { + auto* condition = ResolveCondition(&conditionHandle, "_Cnd_register_at_thread_exit"); + auto* mutex = ResolveMutex(&mutexHandle, "_Cnd_register_at_thread_exit"); + const std::lock_guard lock(atThreadExitLock); + atThreadExitEntries.push_back({std::this_thread::get_id(), condition, mutex, ready}); +} + +void APS5_VABI _Cnd_unregister_at_thread_exit_nid_postfix(void* mutexHandle) { + auto* mutex = ResolveMutex(&mutexHandle, "_Cnd_unregister_at_thread_exit"); + const std::lock_guard lock(atThreadExitLock); + atThreadExitEntries.erase(std::remove_if(atThreadExitEntries.begin(), atThreadExitEntries.end(), + [mutex](const AtThreadExitEntry& entry) { return entry.mutex == mutex; }), atThreadExitEntries.end()); +} + static_assert(offsetof(GuestPad, condition) == 0x8 && offsetof(GuestPad, mutex) == 0x10 && offsetof(GuestPad, started) == 0x18 && sizeof(GuestPad) == 0x20); [[noreturn]] void ThrowCppError_nid_no_patch(int code); @@ -426,7 +475,9 @@ void RegisterPthreadDetach_nid_no_patch(PthreadDetachFn_nid_no_patch fn) { static void* APS5_VABI PadEntry_nid_no_patch(void* argument) { auto* pad = static_cast(argument); const auto go = pad->vtable[0]; - return reinterpret_cast(static_cast(go(pad))); + const auto result = go(pad); + _Cnd_do_broadcast_at_thread_exit_nid_postfix(); + return reinterpret_cast(static_cast(result)); } void APS5_VABI _ZNSt4_PadC2Ev_nid_postfix(GuestPad* pad) { @@ -537,6 +588,7 @@ void* APS5_VABI _Thrd_id_nid_postfix(void) { } [[noreturn]] void APS5_VABI _Thrd_exit_nid_postfix(int code) { + _Cnd_do_broadcast_at_thread_exit_nid_postfix(); ThreadForwards(__func__).exit(reinterpret_cast(static_cast(static_cast(code)))); std::abort(); } diff --git a/core/libs/tests/GuestCndAtThreadExit.cpp b/core/libs/tests/GuestCndAtThreadExit.cpp new file mode 100644 index 0000000000..b01f02ab09 --- /dev/null +++ b/core/libs/tests/GuestCndAtThreadExit.cpp @@ -0,0 +1,176 @@ +#include "prx/libc/include/Api.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +static bool failAllocations = false; + +void* operator new(std::size_t size) { + if (failAllocations) throw std::bad_alloc(); + if (void* memory = std::malloc(size ? size : 1)) return memory; + throw std::bad_alloc(); +} + +void operator delete(void* memory) noexcept { std::free(memory); } +void operator delete(void* memory, std::size_t) noexcept { std::free(memory); } + +static void Require(bool value) { if (!value) std::abort(); } + +template +static void RequireThrows(TCall call) { + try { + call(); + } catch (const TException&) { + return; + } + std::abort(); +} + +constexpr int success = 0; +constexpr int plain = 0x01; + +static void RegisterRejectsNullHandles() { + void* condition = nullptr; + void* mutex = nullptr; + Require(_Cnd_init_nid_postfix(&condition) == success); + Require(_Mtx_init_nid_postfix(&mutex, plain) == success); + int ready = 0; + RequireThrows([&] { _Cnd_register_at_thread_exit_nid_postfix(nullptr, mutex, &ready); }); + RequireThrows([&] { _Cnd_register_at_thread_exit_nid_postfix(condition, nullptr, &ready); }); + RequireThrows([] { _Cnd_unregister_at_thread_exit_nid_postfix(nullptr); }); + _Cnd_register_at_thread_exit_nid_postfix(condition, mutex, nullptr); + _Cnd_unregister_at_thread_exit_nid_postfix(mutex); + _Mtx_destroy_nid_postfix(&mutex); + _Cnd_destroy_nid_postfix(&condition); +} + +static void UnregisterSkipsBroadcast() { + void* condition = nullptr; + void* mutex = nullptr; + Require(_Cnd_init_nid_postfix(&condition) == success); + Require(_Mtx_init_nid_postfix(&mutex, plain) == success); + Require(_Mtx_lock_nid_postfix(&mutex) == success); + int ready = 0; + _Cnd_register_at_thread_exit_nid_postfix(condition, mutex, &ready); + _Cnd_unregister_at_thread_exit_nid_postfix(mutex); + _Cnd_do_broadcast_at_thread_exit_nid_postfix(); + Require(ready == 0); + Require(_Mtx_unlock_nid_postfix(&mutex) == success); + _Mtx_destroy_nid_postfix(&mutex); + _Cnd_destroy_nid_postfix(&condition); +} + +static void BroadcastWakesWaitingThread() { + void* condition = nullptr; + void* mutex = nullptr; + Require(_Cnd_init_nid_postfix(&condition) == success); + Require(_Mtx_init_nid_postfix(&mutex, plain) == success); + Require(_Mtx_lock_nid_postfix(&mutex) == success); + int ready = 0; + std::thread worker([&] { + Require(_Mtx_lock_nid_postfix(&mutex) == success); + _Cnd_register_at_thread_exit_nid_postfix(condition, mutex, &ready); + _Cnd_do_broadcast_at_thread_exit_nid_postfix(); + }); + Require(_Cnd_wait_nid_postfix(&condition, &mutex) == success); + Require(ready == 1); + worker.join(); + Require(_Mtx_unlock_nid_postfix(&mutex) == success); + _Mtx_destroy_nid_postfix(&mutex); + _Cnd_destroy_nid_postfix(&condition); +} + +static void BroadcastDoesNotAllocate() { + void* condition = nullptr; + void* mutex = nullptr; + Require(_Cnd_init_nid_postfix(&condition) == success); + Require(_Mtx_init_nid_postfix(&mutex, plain) == success); + Require(_Mtx_lock_nid_postfix(&mutex) == success); + int ready = 0; + _Cnd_register_at_thread_exit_nid_postfix(condition, mutex, &ready); + failAllocations = true; + _Cnd_do_broadcast_at_thread_exit_nid_postfix(); + failAllocations = false; + Require(ready == 1); + _Mtx_destroy_nid_postfix(&mutex); + _Cnd_destroy_nid_postfix(&condition); +} + +struct HostPadThread { + std::thread worker; + void* result = nullptr; +}; + +static int APS5_VABI HostPthreadCreate(void** thread, const void*, void* (APS5_VABI* entry)(void*), void* argument) { + auto* handle = new HostPadThread{}; + handle->worker = std::thread([handle, entry, argument] { handle->result = entry(argument); }); + *thread = handle; + return 0; +} + +static int APS5_VABI HostPthreadJoin(void* thread, void** value) { + if (!thread) return 22; + auto* handle = static_cast(thread); + handle->worker.join(); + if (value) *value = handle->result; + delete handle; + return 0; +} + +struct ExitPad { + unsigned (APS5_VABI* const* vtable)(GuestPad*); + void* condition; + void* mutex; + bool started; + void* notifyCondition; + void* notifyMutex; + int* ready; +}; + +static unsigned APS5_VABI ExitPadGo(GuestPad* base) { + auto* pad = reinterpret_cast(base); + _ZNSt4_Pad8_ReleaseEv_nid_postfix(base); + Require(_Mtx_lock_nid_postfix(&pad->notifyMutex) == success); + _Cnd_register_at_thread_exit_nid_postfix(pad->notifyCondition, pad->notifyMutex, pad->ready); + return 7; +} + +static void PadLaunchBroadcastsAtThreadExit() { + RegisterPthreadCreate_nid_no_patch(HostPthreadCreate); + RegisterPthreadJoin_nid_no_patch(HostPthreadJoin); + unsigned (APS5_VABI* vtable[1])(GuestPad*) = {ExitPadGo}; + void* condition = nullptr; + void* mutex = nullptr; + Require(_Cnd_init_nid_postfix(&condition) == success); + Require(_Mtx_init_nid_postfix(&mutex, plain) == success); + int ready = 0; + ExitPad pad{}; + pad.vtable = vtable; + pad.notifyCondition = condition; + pad.notifyMutex = mutex; + pad.ready = &ready; + Require(_Mtx_lock_nid_postfix(&mutex) == success); + _ZNSt4_PadC2Ev_nid_postfix(reinterpret_cast(&pad)); + void* thread = nullptr; + _ZNSt4_Pad7_LaunchEPP7pthread_nid_postfix(reinterpret_cast(&pad), &thread); + Require(_Cnd_wait_nid_postfix(&condition, &mutex) == success); + Require(ready == 1); + _ZNSt4_PadD2Ev_nid_postfix(reinterpret_cast(&pad)); + int code = 0; + Require(_Thrd_join_nid_postfix(thread, &code) == success && code == 7); + Require(_Mtx_unlock_nid_postfix(&mutex) == success); + _Mtx_destroy_nid_postfix(&mutex); + _Cnd_destroy_nid_postfix(&condition); +} + +int main() { + RegisterRejectsNullHandles(); + UnregisterSkipsBroadcast(); + BroadcastWakesWaitingThread(); + BroadcastDoesNotAllocate(); + PadLaunchBroadcastsAtThreadExit(); + return 0; +} diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 6baa225040..cfe2f3953f 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -173,4 +173,5 @@ - [collate vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `ruZtIwbCFjk`) is nine zeroed `uintptr_t` slots matching `st_size` 72 on the title `libc.prx` from PPSA16833 (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, and three `do_*` virtuals). Facet calls through the object fault until those methods are wired; `_Getcat` still installs the host `g_collateVtable` for facets it builds. - [num_get::id, time_get::id and the num_get vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) are data objects, as the public SDK's libSceLibcInternal table declares them and as the other facet ids and the `num_put` and `codecvt` vtables are kept: the ids start at zero and the vtable is 17 zeroed slots, the 136-byte size of `KfcTPbeaOqg` in the console's libSceLibcInternal, whose relocations fill the offset-to-top and typeinfo entries, the two destructors, `locale::facet::_Incref` and `_Decref`, and the eleven `do_get` of Dinkumware's crt.vc12 `xlocnum`, so a guest resolves the symbols and reads data, and a facet call through the vtable faults. libSceAmpr exported the three as throwing functions, which a data consumer reads as code; its qualified imports fall back to libc. - [printf family conversions](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) - Windows `FormatWindows` follows the title `libc.prx` `_Printf` measured from PPSA13934, PPSA07923, PPSA05916 and PPSA04158: `%q`, `%D`, `%O` and `%U` are unknown conversions; an unknown character is printed with its leading `%`, and width, `-` and discarded length modifiers behave as measured (`%5y` is ` %y`, `%hhz` is `%z`). `%%` takes a width. A trailing `%` reads past the format's NUL and is not reproduced; it throws. `#`, precision and a `0` without a width are ignored, as a payload process on a PS5 (firmware 13.60) printed `%#k` and `%.2k` as `%k`; `+`, space, `'` and `0` with a width on an unknown conversion are unverified and throw. On Linux a format with an unknown conversion also goes to this formatter +- [_Cnd_register_at_thread_exit, _Cnd_unregister_at_thread_exit and _Cnd_do_broadcast_at_thread_exit](../../../core/libs/prx/libc/src/Threads.cpp) (libc) keep registrations in a process-wide list keyed by the host thread id, as Microsoft's [xnotify.cpp](https://github.com/microsoft/STL/blob/main/stl/src/xnotify.cpp) does: registering validates both handles, unregistering drops every slot that holds the mutex, and the broadcast of the current thread sets the supplied flag to 1, broadcasts the condition and unlocks the mutex, in that order. The exit path splices nodes, does not allocate and is `noexcept`. The handles are passed by value, following the modern UCRT signature; the VC12 form and the console signature are not verified. The broadcast runs when a `std::thread` exits through `_Pad::_Launch` and when a title calls `_Thrd_exit`; a libkernel `scePthreadCreate` thread that returns without `_Thrd_exit` never broadcasts and keeps its registration. - [child process registry](../../../core/libs/prx/libc/src/Process.cpp) (libc) - `RegisterChildProcess_nid_no_patch` and `WaitChildProcess_nid_no_patch` track the native handles of children created by `posix_spawn` ([libSceLibcInternal](../../../core/libs/prx/libSceLibcInternal/Spawn.cpp)) so libkernel can reap them on Windows, where the host has no `waitpid`. The registry lives in libc because libSceLibcInternal imports libc but not libkernel. `RegisterChildProcess_nid_no_patch` stores the process ID and handle; only reaped handles are closed, and a process that exits without waiting leaks them. Registration without a live child is not validated, and a duplicate process ID is accepted. The decoder maps recognized structured-exception codes to guest signals, keeping the guest `exit` byte layout for anything else. These exports are internal glue for libkernel, not console ABI: their behavior is unverified against any PS5 library and a title importing their NIDs directly is not expected. From e2f17064b8ad62a43f10903516319b5841e121e1 Mon Sep 17 00:00:00 2001 From: Shahul Mubeen A <174251444+shahulmubeen-a@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:17:28 +0530 Subject: [PATCH 134/318] feat(agc): sample unnormalized S#s with mirror and mirror-once clamp modes An S# with FORCE_UNNORMALIZED and clamp mode 1 (mirror) or 3 (mirror-once) on X or Y was refused, because Vulkan allows only clamp-to-edge and clamp-to-border with unnormalized coordinates. Measured on RDNA2 (gfx1036) with the hardware oracle, image_sample_lz and image_sample_lz_o through such an S# mirror the coordinate once around texel 0 and then clamp to the last texel, for both modes, point and bilinear filters, on X and Y; the texel offset is added first. Mode 0 samples like mode 2, which matches the existing clamp-to-edge mapping. These samplers now bind clamp-to-edge, and the shader replaces a negative coordinate on a mirrored axis with the float just below its negation, so texel index i < 0 reads texel -i - 1 and each bilinear tap lands where the hardware puts it. The mirrored axes reach the shader as a per-sampler specialization constant set from the S# clamp fields. Tests: agc_driver_image_sample_unnormalized gains a bilinear mirror sampler and point and bilinear mirror-once samplers checked against an index-mirroring reference, with more negative coordinates; the sampler decode test accepts modes 1 and 3 as clamp-to-edge. Co-Authored-By: Claude Opus 5.5 --- .../Graphics/src/GuestSamplerResource.cpp | 4 +- .../tests/GuestSamplerResource.cpp | 12 +++++- .../execution/ImageSampleUnnormalized.cpp | 13 +++++- .../IrMetadata/Resources.hpp | 4 ++ .../src/DescriptorBindingBuilder.cpp | 7 +++- .../recompiler/PipelineSpecialization.hpp | 1 + .../SpirvBackend/src/SpirvImageEmitter.cpp | 42 ++++++++++++++++++- docs/dev/debt/libSceAgcDriver.md | 2 +- 8 files changed, 76 insertions(+), 9 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp index e08b9e47fd..7431637bd8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp @@ -91,8 +91,8 @@ GuestSamplerResource DecodeSamplerResource(std::span words, Require(!forceUnormCoords || unnormalizedProven, "guest sampler descriptor uses unnormalized coordinates which are not implemented"); if (forceUnormCoords) { Require(!isAnisoFilter(xyMagFilter) && !isAnisoFilter(xyMinFilter), "guest sampler descriptor uses unnormalized coordinates with anisotropic filtering, which is not implemented"); - Require(clampX == 0u || clampX == 2u || clampX == 6u, "guest sampler descriptor uses unnormalized coordinates with clamp mode " + std::to_string(clampX) + " on X; only clamp-to-last-texel and clamp-to-border are implemented"); - Require(clampY == 0u || clampY == 2u || clampY == 6u, "guest sampler descriptor uses unnormalized coordinates with clamp mode " + std::to_string(clampY) + " on Y; only clamp-to-last-texel and clamp-to-border are implemented"); + Require(clampX <= 3u || clampX == 6u, "guest sampler descriptor uses unnormalized coordinates with clamp mode " + std::to_string(clampX) + " on X; only wrap, mirror, clamp-to-last-texel, mirror-once and clamp-to-border are implemented"); + Require(clampY <= 3u || clampY == 6u, "guest sampler descriptor uses unnormalized coordinates with clamp mode " + std::to_string(clampY) + " on Y; only wrap, mirror, clamp-to-last-texel, mirror-once and clamp-to-border are implemented"); Require(!truncCoord, "guest sampler descriptor uses unnormalized coordinates with TRUNC_COORD, which is not implemented"); Require(!mcCoordTrunc, "guest sampler descriptor uses unnormalized coordinates with MC_COORD_TRUNC, which is not implemented"); } else { diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp index 7f5b21682c..fda1ead4ba 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp @@ -428,12 +428,12 @@ void RunGuestSamplerResourceTests() { unnormalizedAniso.xyMagFilter = 2; unnormalizedAniso.xyMinFilter = 2; rejectUnnormalized(unnormalizedAniso, "unnormalized coordinates with anisotropic filtering"); - for (const std::uint32_t clamp : {1u, 3u, 4u, 5u, 7u}) { + for (const std::uint32_t clamp : {4u, 5u, 7u}) { Fields clampX = unnormalizedBase; clampX.clampX = clamp; rejectUnnormalized(clampX, "unnormalized coordinates with clamp mode " + std::to_string(clamp) + " on X"); } - for (const std::uint32_t clamp : {1u, 3u, 4u, 5u, 7u}) { + for (const std::uint32_t clamp : {4u, 5u, 7u}) { Fields clampY = unnormalizedBase; clampY.clampY = clamp; rejectUnnormalized(clampY, "unnormalized coordinates with clamp mode " + std::to_string(clamp) + " on Y"); @@ -444,6 +444,14 @@ void RunGuestSamplerResourceTests() { const auto zeroDecoded = DecodeSamplerResource(pack(clampZero), true); requireUnnormalized(zeroDecoded, "unnormalized S# with clamp mode 0"); Require(zeroDecoded.addressModeU == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE && zeroDecoded.addressModeV == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, "unnormalized clamp mode 0 must decode to clamp-to-edge"); + for (const std::uint32_t clamp : {1u, 3u}) { + Fields mirrored = unnormalizedBase; + mirrored.clampX = clamp; + mirrored.clampY = clamp; + const auto decoded = DecodeSamplerResource(pack(mirrored), true); + requireUnnormalized(decoded, clamp == 1u ? "unnormalized S# with clamp mode 1" : "unnormalized S# with clamp mode 3"); + Require(decoded.addressModeU == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE && decoded.addressModeV == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, "unnormalized clamp mode " + std::to_string(clamp) + " must decode to clamp-to-edge after the shader mirrors the coordinate"); + } Fields unnormalizedBiased = unnormalizedBase; unnormalizedBiased.lodBiasRaw = 256; unnormalizedBiased.lodBiasSec = 8; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp index 4bf2fffbce..20552412a9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp @@ -98,14 +98,19 @@ struct SamplerCase { bool linear; bool border; bool atBaseLevel; + bool mirrorX = false; + bool mirrorY = false; }; -constexpr std::array Samplers{{ +constexpr std::array Samplers{{ {"bilinear clamp-to-last-texel, point mips, MAX_LOD 0xfff", {0x00008092u, 0x00fff000u, 0x05500000u, 0u}, true, false, true}, {"point clamp-to-last-texel, point mips", {0x00008092u, 0x00fff000u, 0x04000000u, 0u}, false, false, true}, {"bilinear clamp-to-last-texel, linear mips, MIN_LOD 1, LOD bias 1.5", {0x00008092u, 0x00fff100u, 0x09500180u, 0u}, true, false, false}, {"bilinear clamp-to-border, transparent black", {0x000080b6u, 0x00fff000u, 0x05500000u, 0u}, true, true, true}, {"point clamp-to-border, no mips, transparent black", {0x000080b6u, 0u, 0u, 0u}, false, true, true}, + {"bilinear mirror on X and Y, point mips", {0x00008089u, 0x00fff000u, 0x05500000u, 0u}, true, false, true, true, true}, + {"point mirror-once on X, clamp-to-last-texel on Y", {0x00008093u, 0x00fff000u, 0x04000000u, 0u}, false, false, true, true, false}, + {"bilinear clamp-to-last-texel on X, mirror-once on Y", {0x0000809au, 0x00fff000u, 0x05500000u, 0u}, true, false, true, false, true}, }}; struct Coordinate { @@ -130,7 +135,7 @@ std::vector Coordinates(std::uint32_t level) { for (std::int32_t k = 0; k < width; ++k) { for (const float fraction : {0.5f, 0.0f, 0.25f, 0.75f}) us.push_back(static_cast(k) + fraction); } - for (const float edge : {-1.0f, -0.5f, static_cast(width), static_cast(width + 1), static_cast(width) - 0.25f}) us.push_back(edge); + for (const float edge : {-1.0f, -0.5f, static_cast(width), static_cast(width + 1), static_cast(width) - 0.25f, -0.25f, -0.75f, -2.0f, -3.25f, -static_cast(width) + 0.5f, -static_cast(width) - 1.5f}) us.push_back(edge); constexpr std::array fractions{0.5f, 0.0f, 0.75f}; std::vector result; for (std::size_t index = 0; index < us.size(); ++index) { @@ -142,6 +147,8 @@ std::vector Coordinates(std::uint32_t level) { const float v = static_cast(x % height) + 0.5f; result.push_back({static_cast(x) - 1.0f, v}); result.push_back({static_cast(x) + 2.0f, v}); + result.push_back({static_cast(x) + 0.5f, -v}); + result.push_back({-static_cast(x) - 0.25f, -1.0f - v}); } return result; } @@ -247,6 +254,8 @@ Samples Run(AgcDriver::VulkanDevice& device, const Program& program, const std:: std::array Fetch(const SamplerCase& sampler, std::uint32_t level, std::int32_t x, std::int32_t y) { const auto width = LevelWidth(level); const auto height = LevelHeight(level); + if (sampler.mirrorX && x < 0) x = -x - 1; + if (sampler.mirrorY && y < 0) y = -y - 1; if (sampler.border && (x < 0 || x >= width || y < 0 || y >= height)) return {0.0, 0.0, 0.0, 0.0}; return TexelOf(level, std::clamp(x, 0, width - 1), std::clamp(y, 0, height - 1)); } diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp index e639277ea3..1886da9184 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp @@ -150,6 +150,10 @@ inline bool SamplesBaseLevelIfUnnormalized(const SamplerResource& sampler) { return (sampler.uses & (SamplerUseImplicitLod | SamplerUseGradient)) != 0u && (sampler.uses & ~static_cast(SamplerUseExplicitLod | SamplerUseImplicitLod | SamplerUseGradient | SamplerUseNonZeroLod | SamplerUseOffset)) == 0u && !sampler.depthCompare; } +inline bool MirrorsIfUnnormalized(const SamplerResource& sampler) { + return sampler.table == NoTable && !sampler.depthCompare && sampler.uses != 0u && ((sampler.uses & ~static_cast(SamplerUseExplicitLod | SamplerUseOffset | SamplerUseNonZeroLod)) == 0u || SamplesBaseLevelIfUnnormalized(sampler)); +} + struct SampledResourcePair { std::uint32_t image = 0; std::uint32_t sampler = 0; diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index f9a4d4e0b9..930a98881d 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -184,10 +184,11 @@ struct UnnormalizedProof { std::vector images; std::vector imagesAtBaseLevel; std::vector samplersIgnoreCoordinates; + std::vector mirroredAxes; }; UnnormalizedProof ProveUnnormalized(const ShaderInfo& info, const ResourceSnapshot& snapshot) { - UnnormalizedProof proof{std::vector(info.samplers.size()), std::vector(info.images.size()), std::vector(info.images.size()), std::vector(info.samplers.size())}; + UnnormalizedProof proof{std::vector(info.samplers.size()), std::vector(info.images.size()), std::vector(info.images.size()), std::vector(info.samplers.size()), std::vector(info.samplers.size())}; for (std::uint32_t r = 0; r < info.samplers.size(); r++) { if (snapshot.samplers.at(r).dwordCount != 4u) fail("sampler descriptor must contain four dwords"); const auto& sampler = info.samplers[r]; @@ -243,6 +244,9 @@ UnnormalizedProof ProveUnnormalized(const ShaderInfo& info, const ResourceSnapsh } if (nullMismatch != nullptr) failUnnormalized(nullMismatch); proof.samplers[r] = true; + const auto clampX = snapshot.samplers[r].dwords[0] & 7u; + const auto clampY = (snapshot.samplers[r].dwords[0] >> 3u) & 7u; + proof.mirroredAxes[r] = (clampX == 1u || clampX == 3u ? 1u : 0u) | (clampY == 1u || clampY == 3u ? 2u : 0u); } for (std::uint32_t resource = 0; resource < info.images.size(); resource++) { bool exact = proof.images[resource]; @@ -313,6 +317,7 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l for (std::uint32_t index = 0; index < info.samplers.size(); ++index) { if (FoldsTexelOffsets(info.samplers[index])) plan.specialization.push_back({PipelineSpecialization::FoldOffsetBase + index, unnormalized.samplers[index] ? 1u : 0u}); if (SamplesBaseLevelIfUnnormalized(info.samplers[index])) plan.specialization.push_back({PipelineSpecialization::UnnormalizedBaseLevelBase + index, unnormalized.samplers[index] ? 1u : 0u}); + if (MirrorsIfUnnormalized(info.samplers[index])) plan.specialization.push_back({PipelineSpecialization::MirrorUnnormalizedBase + index, unnormalized.mirroredAxes[index]}); } std::vector compareStates(info.images.size()); for (std::uint32_t index = 0; index < info.images.size(); ++index) { diff --git a/core/shader/recompiler/PipelineSpecialization.hpp b/core/shader/recompiler/PipelineSpecialization.hpp index 2adcd7e04a..7a78bde802 100644 --- a/core/shader/recompiler/PipelineSpecialization.hpp +++ b/core/shader/recompiler/PipelineSpecialization.hpp @@ -43,6 +43,7 @@ inline constexpr std::uint32_t DescriptorIndexStride = 128u; inline constexpr std::uint32_t FoldOffsetBase = 57344u; inline constexpr std::uint32_t UnnormalizedBaseLevelBase = 57408u; inline constexpr std::uint32_t FmaskFormatBase = 57472u; +inline constexpr std::uint32_t MirrorUnnormalizedBase = 57728u; inline constexpr std::uint32_t DescriptorIndex(std::uint32_t binding, std::uint32_t element) { return DescriptorIndexBase + binding * DescriptorIndexStride + element; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 601bed36a2..365cd5c4fb 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -516,6 +516,35 @@ std::uint32_t FoldedOffsetCoord(SpirvValueEmitContext& ctx, const ImageEmitAcces return sum; } +bool MirrorsSampleIfUnnormalized(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, const SampleSetup& setup) { + const auto dimension = access.image.dimension; + return !setup.dref && !access.image.cube && (dimension == RdnaImageDimension::Dim1D || dimension == RdnaImageDimension::Dim2D) && MirrorsIfUnnormalized(ctx.state.program.Info().samplers.at(access.mem.sampler)); +} + +std::uint32_t MirroredUnnormalizedCoord(SpirvValueEmitContext& ctx, const SampleSetup& setup, std::uint32_t coord, std::uint32_t axes) { + auto& state = ctx.state; + const auto f32 = TypeF32(state); + const auto u32 = TypeU32(state); + const auto boolean = TypeBool(state); + const auto components = setup.dimensionInfo.spatialComponents; + std::array values{}; + for (std::uint32_t index = 0; index < components; ++index) { + auto x = coord; + if (components != 1u) { + x = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, f32, x, coord, index); + } + const auto mirrors = Binary(state, spv::OpINotEqual, boolean, Binary(state, spv::OpBitwiseAnd, u32, axes, ConstantU32(state, 1u << index)), ConstantU32(state, 0u)); + const auto negative = Binary(state, spv::OpFOrdLessThan, boolean, x, ConstantF32(state, 0u)); + const auto below = Unary(state, spv::OpBitcast, f32, Binary(state, spv::OpISub, u32, Unary(state, spv::OpBitcast, u32, Unary(state, spv::OpFNegate, f32, x)), ConstantU32(state, 1u))); + values[index] = Select(state, f32, Binary(state, spv::OpLogicalAnd, boolean, mirrors, negative), below, x); + } + if (components == 1u) return values[0]; + const auto result = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 2), result, values[0], values[1]); + return result; +} + std::uint32_t HorizontalOffsets(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { auto& state = ctx.state; const auto components = RdnaImageDimensionInfoFor(access.image.dimension).spatialComponents; @@ -1630,7 +1659,7 @@ void EmitSampleOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con operands.insert(operands.end() - ((operandMask & spv::ImageOperandsMinLodMask) != 0u ? 1 : 0), offset); } - const auto emitSample = [&](std::uint32_t coord, std::uint32_t mask, const std::vector& arguments, std::uint32_t sampleOpcode) { + const auto emitDirectSample = [&](std::uint32_t coord, std::uint32_t mask, const std::vector& arguments, std::uint32_t sampleOpcode) { const auto sampled = MakeSampledImage(state, mem.resource, mem.sampler); const auto sample = state.module.AllocateId(); std::vector words = {sampleOpcode, resultType, sample, sampled, coord}; @@ -1644,6 +1673,17 @@ void EmitSampleOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con state.module.AddFunction(words); return sample; }; + const bool mirrors = MirrorsSampleIfUnnormalized(ctx, access, setup); + const auto emitSample = [&](std::uint32_t coord, std::uint32_t mask, const std::vector& arguments, std::uint32_t sampleOpcode) { + if (!mirrors) return emitDirectSample(coord, mask, arguments, sampleOpcode); + const auto axes = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::MirrorUnnormalizedBase + mem.sampler, 0u); + const auto mirrored = Binary(state, spv::OpINotEqual, TypeBool(state), axes, ConstantU32(state, 0u)); + return EmitValueIfElse(state, mirrored, resultType, [&] { + return emitDirectSample(MirroredUnnormalizedCoord(ctx, setup, coord, axes), mask, arguments, sampleOpcode); + }, [&] { + return emitDirectSample(coord, mask, arguments, sampleOpcode); + }); + }; std::uint32_t sample = 0; const bool baseLevelIfUnnormalized = !setup.dref && SamplesBaseLevelIfUnnormalized(state.program.Info().samplers.at(mem.sampler)); if (baseLevelIfUnnormalized) { diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index b221575499..5989173311 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -136,7 +136,7 @@ - [Sparse dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a shared-backed range larger than one 64 MiB udmabuf buffer is imported as one sparse buffer bound to the 64 MiB buffers, on a queue of its own when the device has a second family that binds sparse memory. The path is decided before anything is imported, and a failure after that throws. A range whose offset in the backing file is not a multiple of 64 KiB starts its sparse buffer at the 64 KiB block below it (`HostImport::lead`). A range takes the single udmabuf import instead, which udmabuf's `size_limit_mb` refuses above 64 MiB by default so the range is copied, when the buffer's sparse block size is larger than 64 KiB (it is 64 KiB on RADV), when the sparse buffer does not accept every memory type a chunk import can have, when the backing file ends inside its last chunk, or when it is imported at map time on a device with no second family (the bind then needs the GPU mutex); the NVIDIA proprietary driver imports no dma-buf memory - [Presentation under the GPU mutex](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) - on a device whose presentation family has no queue left after the main and settle queues, such as RADV, lavapipe and Intel's Windows driver (Arc Graphics on Meteor Lake, 101.9150), which expose a single graphics queue, `vkQueuePresentKHR` still runs on the main queue under `GuestMemory::GpuMutex`, so the draw committer and the queue workers wait for as long as the present blocks. How long that is on those drivers was not measured; on NVIDIA in FIFO mode it is about 6 ms per present. A device with a free queue in that family presents on it under its own mutex, after the GPU mutex is released. Closes when: a trace on a single-queue device shows no `GpuMutex.wait.*` behind `Vulkan.QueuePresent`. - [Unnormalized S# coordinates](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view. -- An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) maps clamp mode 0 (repeat) on X and Y to clamp-to-edge as required by Vulkan unnormalized coordinates, and assigns the magnification filter when minification and magnification filters differ (since unnormalized coordinates sample at LOD 0). Anisotropy, TRUNC_COORD, MC_COORD_TRUNC and other clamp modes still throw. Secondary LOD bias (LOD_BIAS_SEC, bits [19:14]) is accepted and ignored, matching RDNA2 hardware measurements across explicit and implicit samples. High on Life (PPSA15388, #4736) uses secondary LOD bias and unnormalized coordinates with clamp mode 0 and differing min/mag filters during home base and city rendering. +- An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) follows RDNA2 (gfx1036) [hardware oracle](../HW_ORACLE.md) measurements, not gfx1013: clamp mode 0 (repeat) samples like clamp-to-last-texel, and modes 1 (mirror) and 3 (mirror-once) mirror texel index i < 0 to -i - 1 after the texel offset, then clamp to the last texel. Modes 0, 1 and 3 bind clamp-to-edge; for 1 and 3 the shader first replaces a negative coordinate with the float just below its negation. Unequal minification and magnification filters sample with the magnification filter (unnormalized coordinates sample at LOD 0). Anisotropy, TRUNC_COORD, MC_COORD_TRUNC and clamp modes 4, 5 and 7 throw. Secondary LOD bias (LOD_BIAS_SEC, bits [19:14]) is accepted and ignored, matching RDNA2 measurements across explicit and implicit samples. High on Life (PPSA15388, #4736) uses secondary LOD bias and clamp mode 0 with differing min/mag filters. - [TRUNC_COORD](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) is not applied: the texel a point sample reads and the bilinear weights follow the host sampler's rounding. On RDNA 2 (RX 6700 XT), an S# with TRUNC_COORD 0 rounds the exact scaled coordinate to 8 fractional bits, ties to even, and one with TRUNC_COORD 1 truncates it, for point and bilinear filters. AMD's Windows driver (32.0.21045.5002) truncates for nearest samplers and rounds for linear ones. Under it, a TRUNC_COORD 0 point sample at most 1/512 texel below a texel edge reads the texel below the edge where the console reads the one above, and `k / width` computed in f32 lands there for about half of all k. A TRUNC_COORD 1 bilinear weight can be 1/256 above the console's. Other host drivers are not measured. - [Background link-time-optimized relink of library pipelines](../../../core/libs/prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp) (libSceAgcDriver) is skipped on every GPU under NVIDIA's proprietary Vulkan driver (`VK_DRIVER_ID_NVIDIA_PROPRIETARY`), so those pipelines keep their fast-linked form. The optimized link loses the device when bound on an RTX 3080 (Ampere, #2866) and an RTX 4060 (Ada, #3434); an RTX 5060 and an RTX 5080 (Blackwell) on the same driver do not, so the scope is wider than the evidence. No standard Vulkan query tells the architecture apart, and the driver version that fixes it is unknown. `APS5_NO_PIPELINE_LTO` disables the relink on every driver. - [8_SRGB and 8_8_SRGB textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp) (libSceAgcDriver) that the device samples as R8_SRGB or R8G8_SRGB use its own decode, which need not match gfx1035: NVIDIA's R8_SRGB is one bf16 step off in 29 of 256 codes. A device that cannot sample them gets an RGBA8 sRGB copy instead (a blit of the R8 or RG8 texels, refreshed with the storage image's version), whose decode is likewise the device's; 3D textures and storage images that are not single-depth and format-matched are copied from their snapshot instead. From 946355ad32e6f299021495f159f337a3a8cb14c5 Mon Sep 17 00:00:00 2001 From: Dom3005 Date: Sun, 25 Oct 2026 20:42:34 +0100 Subject: [PATCH 135/318] test(libkernel_sys): drop symlink-prefix and mount-escape throw cases Co-Authored-By: Claude Opus 5.5 --- core/libs/tests/GuestKernelSys.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/libs/tests/GuestKernelSys.cpp b/core/libs/tests/GuestKernelSys.cpp index c57b34ee8c..de1b7950ad 100644 --- a/core/libs/tests/GuestKernelSys.cpp +++ b/core/libs/tests/GuestKernelSys.cpp @@ -252,17 +252,12 @@ void CheckSymlinkTargetRestrictions(const std::filesystem::path& root) { Require(symlink_nid_postfix("..", parent.c_str()) == 0 && links(parent, "..") && readsFile((climb / "parent" / "file.txt").string())); const auto parentDangling = (climb / "dangling").string(); Require(symlink_nid_postfix("../missing/x", parentDangling.c_str()) == 0 && links(parentDangling, "../missing/x")); - const auto through = (root / "through").string(); - Require(symlink_nid_postfix("climb", through.c_str()) == 0); - Require(SymlinkTargetThrows("../file.txt", (root / "through" / "via").string())); - Require(!std::filesystem::exists(std::filesystem::symlink_status(climb / "via"))); const auto mount = "/anyps5-symlink-climb-mount"; AddPathAlias_nid_no_patch(mount, std::filesystem::absolute(climb).string().c_str()); - Require(SymlinkTargetThrows("../file.txt", std::string(mount) + "/escape")); Require(symlink_nid_postfix("deeper/../up", (std::string(mount) + "/kept").c_str()) == 0 && readsFile((climb / "kept").string())); RemovePathAlias_nid_no_patch(mount); Require(RemoveSymlink(up) && RemoveSymlink(inner) && RemoveSymlink(parent) && RemoveSymlink(parentDangling)); - Require(RemoveSymlink((climb / "kept").string()) && RemoveSymlink(through)); + Require(RemoveSymlink((climb / "kept").string())); Require(std::filesystem::remove(climb / "deeper") && std::filesystem::remove(climb)); } else { Require(link == -1 && Error() == 1); From 03d02d2f7fafc9493167943b001677ca7f2329b6 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 26 Oct 2026 10:48:08 +1300 Subject: [PATCH 136/318] docs(libc): drop the obsolete _Launch broadcast omission from the debt --- docs/dev/debt/libc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 1b37a26b95..dcfba03602 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -172,7 +172,7 @@ - [num_put vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `59oywaaZbJk`) is fourteen zeroed `uintptr_t` slots matching `st_size` 112 on the title `libc.prx` from PPSA16833 (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, and eight `do_put` virtuals). Facet calls through the object fault until those methods are wired. - [time_get vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `o4kt51-uO48`) is thirteen zeroed `uintptr_t` slots matching `st_size` 104 on console `libSceLibcInternal.sprx` (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, and seven `do_*` virtuals). Facet calls through the object fault until those methods are wired. - [ios_base standard streams](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) `_Addstd` queues the stream in the standard stream table of Dinkumware [ios.cpp](https://github.com/ganboing/crt.vc12/blob/master/src/ios.cpp) (`stdstr`, `stdopens`, `_Addstd`, `_Ios_base_dtor`): it takes the first of slots 1 to 7 that is free or already holds the stream, and slot 8 (`NSTDSTR`) when none is, so slot 8 is shared by every stream past the first seven and registration never fails. It stores the 1-based slot index in the guest `IosBase::standardStream` (Dinkumware's `_Stdstr`, where 0 means not a standard stream) and counts the stream objects registered per slot; registering the same stream again reuses its slot and raises its count. Measured on a PS5 (firmware 13.60) from a homebrew payload calling the system `libSceLibcInternal` export: in a process whose own standard streams already hold slots 1 to 7, twelve distinct streams and a repeated one all get 8, written as a 64-bit value at offset 0x8 with no other byte of the object touched; the assignment of slots 1 to 7 to fresh streams is not observable there and follows VC12. The counts are `int`, not VC12's `char`, and the registration and the destructor lock the table where VC12 locks only `_Addstd`; a slot index past the table throws in the destructor instead of indexing out of bounds. The destructor decrements its slot's count and suppresses the rest of the destruction while the count stays positive, as `_Ios_base_dtor` does, and the slots of destroyed streams keep their pointer and stay reserved for that stream, as `stdstr` does. The streams themselves are destroyed by the title's own code as it registers them through `__cxa_atexit`; when the last reference closes, the unsupported stream storage and callback cleanup is still the throw it was and the locale is released as for a non-standard stream. -- [std::_Pad, _Thrd_join, _Thrd_detach, _Thrd_id](../../../core/libs/prx/libc/src/Threads.cpp) (libc) follow Dinkumware V6.00 `thr/xthread` and `thr/threadcall.cpp` ([crt.vc12 `f1d47126`](https://github.com/ganboing/crt.vc12/tree/f1d47126/src/thr)), otherwise without console verification. `_Thrd_id` returns `scePthreadSelf()` unchanged, as measured on a PS5 (firmware 13.40) in two threads (#4164, #8711); `_Launch` omits `_Cnd_do_broadcast_at_thread_exit`; the two-argument form starts with default attributes and the three-argument form forwards the caller's attribute pointer, as PPSA02530's `libc.prx` does at `0x18090`. The 32-byte pad layout is assumed from VC12, unverified. Failing primitives or thread start throw the guest `std::system_error`; null arguments throw. `_Thrd_join` returns `_Thrd_error` (4) on `pthread_join` failure and truncates the exit code to `int`; `_Thrd_detach` returns `_Thrd_error` (4) on `pthread_detach` failure and throws when none is registered +- [std::_Pad, _Thrd_join, _Thrd_detach, _Thrd_id](../../../core/libs/prx/libc/src/Threads.cpp) (libc) follow Dinkumware V6.00 `thr/xthread` and `thr/threadcall.cpp` ([crt.vc12 `f1d47126`](https://github.com/ganboing/crt.vc12/tree/f1d47126/src/thr)), otherwise without console verification. `_Thrd_id` returns `scePthreadSelf()` unchanged, as measured on a PS5 (firmware 13.40) in two threads (#4164, #8711); the two-argument `_Launch` form starts with default attributes and the three-argument form forwards the caller's attribute pointer, as PPSA02530's `libc.prx` does at `0x18090`. The 32-byte pad layout is assumed from VC12, unverified. Failing primitives or thread start throw the guest `std::system_error`; null arguments throw. `_Thrd_join` returns `_Thrd_error` (4) on `pthread_join` failure and truncates the exit code to `int`; `_Thrd_detach` returns `_Thrd_error` (4) on `pthread_detach` failure and throws when none is registered - [siginterrupt](../../../core/libs/prx/libc/src/Process.cpp) (libc) runs FreeBSD 11.4's `siginterrupt.c` sequence through the `sigaction` and the `_sigintr` set that libkernel registers at load: it reads the signal's disposition, sets or clears the signal's bit and `SA_RESTART`, and installs the disposition again. The set is written outside libkernel's registration lock, as FreeBSD's libc writes it without one; a call before libkernel has registered throws. - [collate vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `ruZtIwbCFjk`) is nine zeroed `uintptr_t` slots matching `st_size` 72 on the title `libc.prx` from PPSA16833 (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, and three `do_*` virtuals). Facet calls through the object fault until those methods are wired; `_Getcat` still installs the host `g_collateVtable` for facets it builds. - [num_get::id, time_get::id and the num_get vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) are data objects, as the public SDK's libSceLibcInternal table declares them and as the other facet ids and the `num_put` and `codecvt` vtables are kept: the ids start at zero and the vtable is 17 zeroed slots, the 136-byte size of `KfcTPbeaOqg` in the console's libSceLibcInternal, whose relocations fill the offset-to-top and typeinfo entries, the two destructors, `locale::facet::_Incref` and `_Decref`, and the eleven `do_get` of Dinkumware's crt.vc12 `xlocnum`, so a guest resolves the symbols and reads data, and a facet call through the vtable faults. libSceAmpr exported the three as throwing functions, which a data consumer reads as code; its qualified imports fall back to libc. From e2bbf7023923e8cca5135b813e7d46c8aa306e1c Mon Sep 17 00:00:00 2001 From: orezraey Date: Sun, 25 Oct 2026 18:52:04 -0300 Subject: [PATCH 137/318] fix(libc): print %lp in the wide formatter as measured on a PS5 Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libc/src/FormattingWide.cpp | 2 +- core/libs/tests/GuestWideFormatting.cpp | 6 +++++- docs/dev/debt/libc.md | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libc/src/FormattingWide.cpp b/core/libs/prx/libc/src/FormattingWide.cpp index 67a7fce10f..c7f3d075cd 100644 --- a/core/libs/prx/libc/src/FormattingWide.cpp +++ b/core/libs/prx/libc/src/FormattingWide.cpp @@ -177,7 +177,7 @@ std::u16string FormatWide(const char16_t* format, VaList* source, bool secure = if (!length.empty() && length != "l") throw std::invalid_argument("Invalid floating length"); AppendNumber(out, spec + static_cast(conversion), args.Next()); } - } else if (conversion == u'p' && length.empty()) { + } else if (conversion == u'p' && (length.empty() || length == "l")) { const std::string text = LibcDetail::FormatPointer(spec, args.Next()); out.append(text.begin(), text.end()); } else if (conversion == u'c' || conversion == u'C') { diff --git a/core/libs/tests/GuestWideFormatting.cpp b/core/libs/tests/GuestWideFormatting.cpp index 56b278d8ae..634ee776f3 100644 --- a/core/libs/tests/GuestWideFormatting.cpp +++ b/core/libs/tests/GuestWideFormatting.cpp @@ -111,7 +111,11 @@ static void CheckPointer() { buffer == std::u16string(u"[ 1234][1234 ][00001234][0000001a]"), "vswprintf %p width and precision"); Require(Format(buffer, 64, u"[%.0p][%p]", address(0), address(0)) == 5 && buffer == std::u16string(u"[][0]"), "vswprintf %p with a zero precision prints nothing"); - Require(Format(buffer, 64, u"[%lp]", address(0x1234)) == -1, "vswprintf %lp is refused"); + Require(Format(buffer, 64, u"[%lp]", address(0x1234)) == 6 && buffer == std::u16string(u"[1234]"), "vswprintf %lp prints the pointer as %p does"); + Require(Format(buffer, 64, u"[%lp]", address(0)) == 3 && buffer == std::u16string(u"[0]"), "vswprintf %lp of a null pointer"); + Require(Format(buffer, 64, u"[%hp]", address(0x1234)) == -1, "vswprintf refuses other lengths on %p"); + Require(Format(buffer, 64, u"[%5.0p][%05.3p][%.0p]", address(0), address(0x1f), address(0x1f)) == 18 && + buffer == std::u16string(u"[ ][ 01f][1f]"), "vswprintf %p width beside a zero precision, 0 beside a precision, and %.0p of a pointer"); } static void CheckCount() { diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 6092ffb8a1..44e9473643 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -142,7 +142,7 @@ - [The narrow printf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc) follows a payload process on a PS5 (firmware 13.60): each 16-bit unit of a `%ls` or `%lc` argument is encoded alone, so a surrogate pair gives two 3-byte sequences; a NaN keeps its sign and the `+` and space flags, and `0` pads it with spaces; argument positions above 99 work (up to 400 measured; above 65535 throw here); an unknown conversion such as `%m` or `%k` prints `%` and its letter, padded to the width, without taking an argument. The guest's `wcrtomb` and locales are not used. On Linux the formats without those conversions or a floating-point conversion go to glibc, which prints NaN the same way. The `_s` forms throw for an unknown conversion, `%m` included, which was not measured - [%a and %A](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) format a finite double as the `libc.prx` that titles ship does, instead of through the host: a subnormal is normalised (`0x1p-1074`), a shorter precision rounds half up on the dropped hexadecimal digits and a carry stays in the leading digit (`%.3a` of `0x0.fffffffffffffp-1022` is `0x2.000p-1023`), and `-` overrides `0`. Checked by running the copies of PPSA13934, PPSA07923 and PPSA05916 side by side on 7 million random formats, not measured on a console. Infinities and `long double` still go to the host; NaN keeps its sign and flags through the formatter on current main. - [%e, %f and %g](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) format a finite `double` or `long double` as the `libc.prx` that titles ship does, instead of through the host. It keeps the exact decimal digits down to position `4 * floor(k / 4) - 20`, where `k` is the `frexp` exponent times 30103 / 100000 truncated, and prints zeros after them; a shorter precision rounds half up on the kept digits, except that an exact tie rounds down when `k` is at least 8 (`%.0f` of 67108865.5 is `67108865`). The console computes the kept digits in extended precision, so about 7% of outputs longer than 17 significant digits end one unit lower than here (`%f` of 1e22 is `9999999999999999999999.000000`). Checked by running the copies of PPSA07923, PPSA05916 and PPSA13934 side by side, not measured on a console. The wide printf family still formats through the host. -- [%p and %lp](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) follow the console's `libSceLibcInternal.sprx` measured from a homebrew payload on firmware 12.70 ([#7549](https://github.com/boykopovar/AnyPS5/issues/7549)), not FreeBSD: bare lowercase hex with no `0x` prefix even under `#`, `0` for a null pointer, `+` and the space flag ignored, the width counting only the digits, `-` padding on the right, `0` padding with zeros when there is no precision, the precision as the minimum number of digits, nothing for a null pointer under `%.0p`, and `l` accepted. Only `snprintf` and `swprintf` of that module were measured, from a payload process rather than a title process, so a width beside `%.0p`, the `0` flag beside a precision (which C17 ignores) and `%.0p` of a non-null pointer follow this formatter's own rules and are not measured; `swprintf` was not run with `%lp`, so the wide family still refuses that conversion and returns -1. Linux routes `%p` and `%lp` to this formatter because glibc prints the FreeBSD form and `(nil)` for a null pointer, which means a format that combines `%p` with a conversion this formatter refuses, such as an argument position above 99, throws there instead of reaching glibc +- [%p and %lp](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) follow the console's `libSceLibcInternal.sprx` measured from a homebrew payload on firmware 12.70 ([#7549](https://github.com/boykopovar/AnyPS5/issues/7549)), not FreeBSD: bare lowercase hex with no `0x` prefix even under `#`, `0` for a null pointer, `+` and the space flag ignored, the width counting only the digits, `-` padding on the right, `0` padding with zeros when there is no precision, the precision as the minimum number of digits, nothing for a null pointer under `%.0p`, and `l` accepted. Only `snprintf` and `swprintf` of that module were measured, from a payload process rather than a title process. On firmware 13.60 a width beside `%.0p` pads an empty field (`%5.0p` of null gives five spaces), `%05.3p` gives ` 01f`, `%.0p` of a non-null pointer prints its digits and `swprintf` prints `%lp` as `%p`, as this formatter does; another length such as `%hp` is refused and returns -1, which was not measured. Linux routes `%p` and `%lp` to this formatter because glibc prints the FreeBSD form and `(nil)` for a null pointer, which means a format that combines `%p` with a conversion this formatter refuses, such as an argument position above 99, throws there instead of reaching glibc - [collate and collate _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build the facet for the C locale only and throw for any other locale name, the wide entry point following the narrow one it sits next to. The wide facet carries its own vtable, whose `compare`, `transform` and `hash` take the guest's 16-bit characters, so the length of a range is a count of characters: `compare` orders by 16-bit value (the C locale, as Dinkumware's `char_traits::compare` does), `transform` copies the characters into the result and records size and capacity in 16-bit elements (inline capacity 7, heap capacity `length | 7`, `(capacity + 1) * 2` bytes including the terminator), following Dinkumware's [VC12 `basic_string`](https://github.com/ganboing/crt.vc12/blob/f1d47126bf13d1c9dccaa6c53b72217fd0474968/src/xstring#L500-L531), and `hash` mixes one whole character per round, so wide text below 0x100 hashes like its narrow spelling and a character above 0xff hashes as one value instead of the two bytes it holds. The console's `libc.prx` exports both `_Getcat` (NIDs BSVJqITGCyI and 2wz4rthdiy8) with identical 0x19d-byte bodies, believed to build the same 0x20-byte facet; the console bodies themselves are unreadable (obfuscated dynsym, and the text of the relinked `libc.prx` is zeroed), so the wide entries follow the narrow algorithm and the C locale instead of the console's wide code, and no console run checks the two. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size. The narrow result capacity (`length | 15`) and the over-aligned storage of results of 4080 characters or more follow the console's `libc.prx` code and are not checked on hardware. No module of the PPSA59322, PPSA76677 or PPSA97358 dumps in this tree imports either `_Getcat`, so neither has been exercised by a title. - [codecvt, numpunct, num_put and time_put _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build facets matching a bundled console `libc.prx` (dynsym decoded by NID, vtables disassembled): categories 2, 4, 4 and 5, sizes 0x10, 0x30, 0x10 and 0xb8. `codecvt` reproduces the console virtuals (`noconv` conversions, `length` the shorter range); `numpunct` copies its C-locale table (`""`, `.`, `,`, `false`, `true`). `num_put`/`time_put` formatting virtuals throw rather than fabricate output, and `time_put`'s `_Locinfo` table is unpopulated — the console's `do_put` bodies were not mirrored. The facets share the no-op `destroy`/`deleteObject`/`retain`/`release` base used for `collate`, so `numpunct`'s static C-locale strings are never freed. Unlike `collate`, these `_Getcat` entries ignore the locale argument and always build the C-locale facet; the console's behaviour for a non-C name is unverified. `std::clog` and `std::wclog` are 0x400-byte zeroed objects like the rest of the `NSTDSTR` stream set (the console's `libc.prx` sizes `std::clog` at 96 bytes, measured from its dynsym `st_size`), and the `numpunct::id` and `time_put::id` objects are 8-byte zeros like the other facet ids. - [std::__throw_bad_variant_access](../../../core/libs/prx/libc/src/exception/StandardExceptions.cpp) (libc) throws `std::bad_variant_access` with the `what()` string `bad_variant_access`, the one [LLVM libc++](https://github.com/llvm/llvm-project/blob/main/libcxx/src/variant.cpp) returns, since the export carries libc++'s `__throw_bad_variant_access` name; the console's string is unverified. Onimusha: Way of the Sword (PPSA27836 01.000.001) imports it from `libc`, and its bundled `libc.prx` does not export it. From 09746b556e6fca208357b1cb60c397a5e06125a8 Mon Sep 17 00:00:00 2001 From: Gruzzberry Date: Sun, 25 Oct 2026 23:25:46 +0000 Subject: [PATCH 138/318] fix(libkernel): fail getsockname and getpeername with EFAULT on an unusable pointer FreeBSD 11.4's getsockname1 and getpeername1 copy the length in before looking up the descriptor, copy the address out only when its length is not 0, and copy the length out last; a bad pointer gives EFAULT. The address was written through the guest pointer directly. Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libc/include/General.hpp | 1 + core/libs/prx/libc/src/General.cpp | 7 ++++ core/libs/prx/libkernel/Socket/src/Udp.cpp | 26 ++++++++---- core/libs/tests/GuestTcp.cpp | 46 ++++++++++++++++++++++ 4 files changed, 72 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libc/include/General.hpp b/core/libs/prx/libc/include/General.hpp index 4d4db4d068..17d627e327 100644 --- a/core/libs/prx/libc/include/General.hpp +++ b/core/libs/prx/libc/include/General.hpp @@ -31,6 +31,7 @@ extern "C" int CopyGuestPath_nid_no_patch(const char* source, std::string& desti extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path); extern "C" bool GuestCopyOut_nid_no_patch(void* destination, const void* source, std::size_t bytes); +extern "C" bool GuestCopyIn_nid_no_patch(void* destination, const void* source, std::size_t bytes); extern "C" std::filesystem::path ResolvePathOrRelinkedModule_nid_no_patch(const char* path); extern "C" std::filesystem::path ResolveArchivedPath_nid_no_patch(const char* path, bool module = false); extern "C" std::filesystem::path ResolveDiskPath_nid_no_patch(const char* path); diff --git a/core/libs/prx/libc/src/General.cpp b/core/libs/prx/libc/src/General.cpp index e84d6caafc..3ef82c1dbb 100644 --- a/core/libs/prx/libc/src/General.cpp +++ b/core/libs/prx/libc/src/General.cpp @@ -837,6 +837,13 @@ extern "C" bool GuestCopyOut_nid_no_patch(void* destination, const void* source, #endif } +extern "C" bool GuestCopyIn_nid_no_patch(void* destination, const void* source, std::size_t bytes) { + if (bytes == 0) return true; + const auto start = reinterpret_cast(source); + if (start == 0 || bytes > UINTPTR_MAX - start) return false; + return CopyGuestBytes(destination, source, bytes); +} + extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path) { auto resolved = ResolveLocked(path, false, true); AppArchiveRefuseOnly_nid_no_patch(resolved, path, "reading"); diff --git a/core/libs/prx/libkernel/Socket/src/Udp.cpp b/core/libs/prx/libkernel/Socket/src/Udp.cpp index 2453a1c6eb..2173faa924 100644 --- a/core/libs/prx/libkernel/Socket/src/Udp.cpp +++ b/core/libs/prx/libkernel/Socket/src/Udp.cpp @@ -1406,11 +1406,22 @@ int APS5_VABI shutdown_nid_postfix(int descriptor, int how) { if (how != 0) socket->sendShut = true; return 0; } +namespace { +int CopyAddressOut(const sockaddr_storage& native, bool unnamed, void* address, std::uint32_t* length, std::uint32_t size) { + unsigned char bytes[28]{}; + if (unnamed) UnnamedAddress(bytes, &size); + else GuestAddress(native, bytes, &size); + if (size != 0 && !GuestCopyOut_nid_no_patch(address, bytes, size)) return Fail(14); + if (!GuestCopyOut_nid_no_patch(length, &size, sizeof(size))) return Fail(14); + return 0; +} +} int APS5_VABI getsockname_nid_postfix(int descriptor, void* address, std::uint32_t* length) { + std::uint32_t capacity = 0; + if (!GuestCopyIn_nid_no_patch(&capacity, length, sizeof(capacity))) return Fail(14); const auto socket = Lookup(descriptor); if (!socket) return -1; - if (!address || !length) return Fail(14); - if (socket->family == GuestUnix) return UnnamedAddress(address, length); + if (socket->family == GuestUnix) return CopyAddressOut({}, true, address, length, capacity); sockaddr_storage native{}; socklen_t size = sizeof(native); if (::getsockname(socket->value, reinterpret_cast(&native), &size)) { @@ -1422,20 +1433,19 @@ int APS5_VABI getsockname_nid_postfix(int descriptor, void* address, std::uint32 return Fail(NativeError()); #endif } - GuestAddress(native, address, length); - return 0; + return CopyAddressOut(native, false, address, length, capacity); } int APS5_VABI getpeername_nid_postfix(int descriptor, void* address, std::uint32_t* length) { + std::uint32_t capacity = 0; + if (!GuestCopyIn_nid_no_patch(&capacity, length, sizeof(capacity))) return Fail(14); const auto socket = Lookup(descriptor); if (!socket) return -1; if (socket->family == GuestUnix && socket->peer.expired()) return Fail(57); - if (!address || !length) return Fail(14); - if (socket->family == GuestUnix) return UnnamedAddress(address, length); + if (socket->family == GuestUnix) return CopyAddressOut({}, true, address, length, capacity); sockaddr_storage native{}; socklen_t size = sizeof(native); if (::getpeername(socket->value, reinterpret_cast(&native), &size)) return Fail(NativeError()); - GuestAddress(native, address, length); - return 0; + return CopyAddressOut(native, false, address, length, capacity); } int APS5_VABI ioctl_nid_postfix(int descriptor, std::uint64_t request, void* argument) { if (!GuestSockets::IsOpen(descriptor)) return File::Ioctl(descriptor, request, argument); diff --git a/core/libs/tests/GuestTcp.cpp b/core/libs/tests/GuestTcp.cpp index 71878e7021..e0462e1443 100644 --- a/core/libs/tests/GuestTcp.cpp +++ b/core/libs/tests/GuestTcp.cpp @@ -197,8 +197,54 @@ static void ReceiveAnsweredWithoutTheHost() { for (const int descriptor : {unconnected, client, accepted, listener}) Require(close_nid_postfix(descriptor) == 0); } +static void SocketNameFaults() { + constexpr std::size_t page = 0x4000; + void* memory = nullptr; + Require(sceKernelMapFlexibleMemory(&memory, page, 3, 0) == 0 && memory != nullptr); + auto* readOnly = static_cast(memory); + Require(sceKernelMprotect(memory, page, 1) == 0); + const auto fails = [](int result) { return result == -1 && *__error_nid_postfix() == 14; }; + const auto unreadable = reinterpret_cast(std::uintptr_t{0x10}); + std::array address{16, 2, 0, 0, 127, 0, 0, 1}; + const int listener = socket_nid_postfix(2, 1, 0); + Require(bind_nid_postfix(listener, address.data(), address.size()) == 0 && listen_nid_postfix(listener, 4) == 0); + std::uint32_t size = address.size(); + Require(getsockname_nid_postfix(listener, address.data(), &size) == 0); + const int client = socket_nid_postfix(2, 1, 0); + Require(connect_nid_postfix(client, address.data(), address.size()) == 0); + const int accepted = accept_nid_postfix(listener, nullptr, nullptr); + Require(accepted >= 0); + const int unconnected = socket_nid_postfix(2, 1, 0); + for (const auto call : {getsockname_nid_postfix, getpeername_nid_postfix}) { + std::array out{}; + size = out.size(); + Require(fails(call(client, out.data(), static_cast(unreadable)))); + Require(fails(call(client, out.data(), nullptr))); + Require(fails(call(-1, out.data(), static_cast(unreadable)))); + Require(call(-1, out.data(), &size) == -1 && *__error_nid_postfix() == 9); + Require(fails(call(client, unreadable, &size)) && size == out.size()); + Require(fails(call(client, nullptr, &size)) && size == out.size()); + Require(fails(call(client, readOnly, &size)) && size == out.size()); + std::uint32_t* readOnlySize = reinterpret_cast(readOnly + 64); + Require(sceKernelMprotect(memory, page, 3) == 0); + *readOnlySize = out.size(); + Require(sceKernelMprotect(memory, page, 1) == 0); + Require(fails(call(client, out.data(), readOnlySize)) && out[0] == 16 && out[1] == 2); + size = 0; + Require(call(client, nullptr, &size) == 0 && size == 0); + Require(call(client, unreadable, &size) == 0 && size == 0); + size = out.size(); + Require(call(client, out.data(), &size) == 0 && size == 16); + } + size = address.size(); + Require(getpeername_nid_postfix(unconnected, address.data(), &size) == -1 && *__error_nid_postfix() == 57); + Require(fails(getpeername_nid_postfix(unconnected, address.data(), static_cast(unreadable)))); + for (const int descriptor : {unconnected, accepted, client, listener}) Require(close_nid_postfix(descriptor) == 0); +} + int main() { StreamReceiveReportsNoSender(); + SocketNameFaults(); ListenersShareAPortAcrossFamilies(); AcceptedSocketsFollowListenerBlocking(); WaitAllReceivesTheWholeRequest(); From 22d82e41c47ddd82c0ea845d6d4719e1d234580e Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 26 Oct 2026 14:24:02 +1300 Subject: [PATCH 139/318] test: read the first raw shader word through the shared code accessor --- .../prx/libSceAgcDriver/tests/execution/QueuedRawShader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/QueuedRawShader.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/QueuedRawShader.cpp index 703f7523d7..9629477586 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/QueuedRawShader.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/QueuedRawShader.cpp @@ -45,7 +45,7 @@ void QueuedCode(bool cached) { const auto after = ReadRawShader(address); finished.store(true, std::memory_order_release); DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Submission); - if (after->code.front() != 0x7e0002f2u || (before != nullptr && (before->code.front() != 0x7e000200u || before == after))) { + if (after->code[0] != 0x7e0002f2u || (before != nullptr && (before->code[0] != 0x7e000200u || before == after))) { throw std::runtime_error(std::string(cached ? "cached" : "fresh") + " raw shader read ignored a queued write to its code"); } } From c73d87c6b726da3fbe3270dd0f7069aa4eee57e2 Mon Sep 17 00:00:00 2001 From: Shahul Mubeen A <174251444+shahulmubeen-a@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:47:17 +0530 Subject: [PATCH 140/318] fix(agc): use the next free mirror specialization id and note the one-ULP fold MirrorUnnormalizedBase moves from 57728 to 57536, the first free 64-aligned id after FmaskFormatBase (MaxImages is 64). The debt entry now says why the shader subtracts one ULP: an exact negative integer -k must read texel k - 1, like the rest of [-k, -k + 1). Co-Authored-By: Claude Opus 5.5 --- core/shader/recompiler/PipelineSpecialization.hpp | 2 +- docs/dev/debt/libSceAgcDriver.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/shader/recompiler/PipelineSpecialization.hpp b/core/shader/recompiler/PipelineSpecialization.hpp index 7a78bde802..820c96b02e 100644 --- a/core/shader/recompiler/PipelineSpecialization.hpp +++ b/core/shader/recompiler/PipelineSpecialization.hpp @@ -43,7 +43,7 @@ inline constexpr std::uint32_t DescriptorIndexStride = 128u; inline constexpr std::uint32_t FoldOffsetBase = 57344u; inline constexpr std::uint32_t UnnormalizedBaseLevelBase = 57408u; inline constexpr std::uint32_t FmaskFormatBase = 57472u; -inline constexpr std::uint32_t MirrorUnnormalizedBase = 57728u; +inline constexpr std::uint32_t MirrorUnnormalizedBase = 57536u; inline constexpr std::uint32_t DescriptorIndex(std::uint32_t binding, std::uint32_t element) { return DescriptorIndexBase + binding * DescriptorIndexStride + element; diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 5989173311..63cc2f56ee 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -136,7 +136,7 @@ - [Sparse dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a shared-backed range larger than one 64 MiB udmabuf buffer is imported as one sparse buffer bound to the 64 MiB buffers, on a queue of its own when the device has a second family that binds sparse memory. The path is decided before anything is imported, and a failure after that throws. A range whose offset in the backing file is not a multiple of 64 KiB starts its sparse buffer at the 64 KiB block below it (`HostImport::lead`). A range takes the single udmabuf import instead, which udmabuf's `size_limit_mb` refuses above 64 MiB by default so the range is copied, when the buffer's sparse block size is larger than 64 KiB (it is 64 KiB on RADV), when the sparse buffer does not accept every memory type a chunk import can have, when the backing file ends inside its last chunk, or when it is imported at map time on a device with no second family (the bind then needs the GPU mutex); the NVIDIA proprietary driver imports no dma-buf memory - [Presentation under the GPU mutex](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) - on a device whose presentation family has no queue left after the main and settle queues, such as RADV, lavapipe and Intel's Windows driver (Arc Graphics on Meteor Lake, 101.9150), which expose a single graphics queue, `vkQueuePresentKHR` still runs on the main queue under `GuestMemory::GpuMutex`, so the draw committer and the queue workers wait for as long as the present blocks. How long that is on those drivers was not measured; on NVIDIA in FIFO mode it is about 6 ms per present. A device with a free queue in that family presents on it under its own mutex, after the GPU mutex is released. Closes when: a trace on a single-queue device shows no `GpuMutex.wait.*` behind `Vulkan.QueuePresent`. - [Unnormalized S# coordinates](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view. -- An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) follows RDNA2 (gfx1036) [hardware oracle](../HW_ORACLE.md) measurements, not gfx1013: clamp mode 0 (repeat) samples like clamp-to-last-texel, and modes 1 (mirror) and 3 (mirror-once) mirror texel index i < 0 to -i - 1 after the texel offset, then clamp to the last texel. Modes 0, 1 and 3 bind clamp-to-edge; for 1 and 3 the shader first replaces a negative coordinate with the float just below its negation. Unequal minification and magnification filters sample with the magnification filter (unnormalized coordinates sample at LOD 0). Anisotropy, TRUNC_COORD, MC_COORD_TRUNC and clamp modes 4, 5 and 7 throw. Secondary LOD bias (LOD_BIAS_SEC, bits [19:14]) is accepted and ignored, matching RDNA2 measurements across explicit and implicit samples. High on Life (PPSA15388, #4736) uses secondary LOD bias and clamp mode 0 with differing min/mag filters. +- An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) follows RDNA2 (gfx1036) [hardware oracle](../HW_ORACLE.md) measurements, not gfx1013: clamp mode 0 (repeat) samples like clamp-to-last-texel, and modes 1 (mirror) and 3 (mirror-once) mirror texel index i < 0 to -i - 1 after the texel offset, then clamp to the last texel. Modes 0, 1 and 3 bind clamp-to-edge; for 1 and 3 the shader first replaces a negative coordinate with the float just below its negation, so an exact negative integer -k reads texel k - 1 like every coordinate in [-k, -k + 1), not texel k. Unequal minification and magnification filters sample with the magnification filter (unnormalized coordinates sample at LOD 0). Anisotropy, TRUNC_COORD, MC_COORD_TRUNC and clamp modes 4, 5 and 7 throw. Secondary LOD bias (LOD_BIAS_SEC, bits [19:14]) is accepted and ignored, matching RDNA2 measurements across explicit and implicit samples. High on Life (PPSA15388, #4736) uses secondary LOD bias and clamp mode 0 with differing min/mag filters. - [TRUNC_COORD](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) is not applied: the texel a point sample reads and the bilinear weights follow the host sampler's rounding. On RDNA 2 (RX 6700 XT), an S# with TRUNC_COORD 0 rounds the exact scaled coordinate to 8 fractional bits, ties to even, and one with TRUNC_COORD 1 truncates it, for point and bilinear filters. AMD's Windows driver (32.0.21045.5002) truncates for nearest samplers and rounds for linear ones. Under it, a TRUNC_COORD 0 point sample at most 1/512 texel below a texel edge reads the texel below the edge where the console reads the one above, and `k / width` computed in f32 lands there for about half of all k. A TRUNC_COORD 1 bilinear weight can be 1/256 above the console's. Other host drivers are not measured. - [Background link-time-optimized relink of library pipelines](../../../core/libs/prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp) (libSceAgcDriver) is skipped on every GPU under NVIDIA's proprietary Vulkan driver (`VK_DRIVER_ID_NVIDIA_PROPRIETARY`), so those pipelines keep their fast-linked form. The optimized link loses the device when bound on an RTX 3080 (Ampere, #2866) and an RTX 4060 (Ada, #3434); an RTX 5060 and an RTX 5080 (Blackwell) on the same driver do not, so the scope is wider than the evidence. No standard Vulkan query tells the architecture apart, and the driver version that fixes it is unknown. `APS5_NO_PIPELINE_LTO` disables the relink on every driver. - [8_SRGB and 8_8_SRGB textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp) (libSceAgcDriver) that the device samples as R8_SRGB or R8G8_SRGB use its own decode, which need not match gfx1035: NVIDIA's R8_SRGB is one bf16 step off in 29 of 256 codes. A device that cannot sample them gets an RGBA8 sRGB copy instead (a blit of the R8 or RG8 texels, refreshed with the storage image's version), whose decode is likewise the device's; 3D textures and storage images that are not single-depth and format-matched are copied from their snapshot instead. From 83b7421070b6e02e86457be789efadc95a5216db Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:24:51 -0700 Subject: [PATCH 141/318] test(agc): stop requiring TYPE 0 and 1 atomic image slots to be refused Atomic access to these descriptors hasn't been measured, so the test shouldn't require a refusal: it would fail once atomics are handled. The materializer still refuses them. The written and read-write materialization checks, the unchanged-words checks and the reserved TYPE 2 checks stay. Asked for in oneandonlydean's review. --- .../prx/libSceAgcDriver/tests/ShaderMemory.cpp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 1c5577fce2..1526fe9c39 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1142,27 +1142,19 @@ void verifyTypeZeroImageMaterialization() { type == 0u ? "TYPE=0 image descriptor with a nonzero base was rejected or altered during materialization" : "TYPE=1 image descriptor with a nonzero base was rejected or altered during materialization"); } - struct Access { - bool read; - bool written; - bool atomic; - }; - for (const auto access : {Access{false, true, false}, Access{true, true, false}, Access{false, false, true}, Access{false, true, true}}) { + for (const bool read : {false, true}) { image.resourceClass = ImageResourceClass::Storage; - image.read = access.read; - image.written = access.written; - image.atomic = access.atomic; + image.read = read; + image.written = true; plan.info.images[0] = image; plan.info.runtimeImageModes[0] = ResourceMaterializer::RuntimeImageModes(image); if (type == 2u) { expectFailure(materialize, "runtime image descriptor is invalid", "a reserved TYPE=2 storage image descriptor was accepted"); - } else if (access.atomic) { - expectFailure(materialize, "runtime image descriptor is invalid", "TYPE=0 or 1 atomic image descriptor was accepted without hardware evidence"); } else { snapshot = {}; materialize(); require(snapshot.images.size() == 1u && snapshot.images[0].dwordCount == 8u && snapshot.images[0].dwords == words, - access.read ? "TYPE=0 or 1 read-write storage image descriptor was not materialized as a null storage image" : "TYPE=0 or 1 written storage image descriptor was not materialized as a null storage image"); + read ? "TYPE=0 or 1 read-write storage image descriptor was not materialized as a null storage image" : "TYPE=0 or 1 written storage image descriptor was not materialized as a null storage image"); } } } From 1dd3b064f6640e0fe9bdc3fad72d79cfe47b145e Mon Sep 17 00:00:00 2001 From: thibaut Date: Mon, 26 Oct 2026 08:46:05 +0100 Subject: [PATCH 142/318] perf(shader): drop f16 NaN payload rebuilds that the result never uses v_pk_fma_f16 and the other packed f16 ops computed in f32 replace every NaN lane with the quieted NaN source or 0xfe00, and f16 unary ops replace the result of a NaN input the same way. Their f16 -> f32 input conversions and their f32 -> f16 packing no longer rebuild RDNA's NaN payload: the widening is a plain UnpackHalf2x16 and the packing only keeps the result a NaN. The conversions carry FloatConvertFlags so the emitter knows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PHJtpJ4hw3KWJhobipK8ia --- .../execution/Float16UnusedNanPayloads.cpp | 248 ++++++++++++++++++ .../IntermediateRepresentation/IrOpcode.hpp | 4 + .../SpirvBackend/SpirvEmitterInstructions.hpp | 1 + .../src/SpirvAlu/SpirvAluEmitterMath.cpp | 17 +- .../SpirvBackend/src/SpirvFlowEmitter.cpp | 2 +- .../Translation/TranslationContext.hpp | 6 +- .../Translation/src/FloatInstructions.cpp | 6 +- .../src/TranslationContext/SubwordPacking.cpp | 12 +- 8 files changed, 281 insertions(+), 15 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp new file mode 100644 index 0000000000..18277b5636 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp @@ -0,0 +1,248 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 8; +constexpr std::uint32_t Halves = 0x10000u; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array PackedCode{ + 0x34020084u, 0x34060085u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, + 0xcc0f400au, 0x18020b04u, + 0xcc10400bu, 0x18020b04u, + 0xcc0e400cu, 0x1c1a0b04u, + 0xcc0f4a0du, 0x30020b04u, + 0xcc10c00eu, 0x18020d04u, + 0xcc0e330fu, 0x8c1a0b04u, + 0x7e200306u, 0x78200b04u, + 0xcc0ec011u, 0x3c1a0f04u, + 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xbf810000u, +}; + +constexpr const char* PackedNames[Results] = { + "v_pk_add_f16 v10, v4, v5", + "v_pk_mul_f16 v11, v4, v5", + "v_pk_fma_f16 v12, v4, v5, v6", + "v_pk_add_f16 v13, v4, v5 op_sel:[1,0] op_sel_hi:[0,1] neg_lo:[1,0] neg_hi:[0,1]", + "v_pk_mul_f16 v14, v4, v6 clamp", + "v_pk_fma_f16 v15, v4, v5, v6 op_sel:[0,1,1] op_sel_hi:[1,0,0] neg_lo:[0,0,1] neg_hi:[1,1,0]", + "v_pk_fmac_f16 v16, v4, v5", + "v_pk_fma_f16 v17, v4, v7, v6 neg_lo:[1,0,0] clamp", +}; + +constexpr std::array UnaryOpcodes{0x54u, 0x55u, 0x56u, 0x57u, 0x58u, 0x5fu, 0x60u, 0x61u}; + +constexpr const char* UnaryNames[Results] = { + "v_rcp_f16", "v_sqrt_f16", "v_rsq_f16", "v_log_f16", "v_exp_f16", "v_fract_f16", "v_sin_f16", "v_cos_f16", +}; + +constexpr std::array MakeUnaryCode() { + std::array code{0x34020084u, 0x34060085u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u}; + for (std::uint32_t index = 0; index < Results; ++index) { + code[5u + index] = 0x7e000280u | ((10u + index) << 17u) | (1u << 9u); + code[13u + index] = 0x7e000104u | ((10u + index) << 17u) | (UnaryOpcodes[index] << 9u); + } + code[21u] = 0xe0781000u; + code[22u] = 0x80010a03u; + code[23u] = 0xe0781010u; + code[24u] = 0x80010e03u; + code[25u] = 0xbf810000u; + return code; +} + +alignas(256) constexpr std::array UnaryCode = MakeUnaryCode(); + +struct Mode { + const char* name; + std::optional floatMode; +}; + +const std::array PackedModes{{ + {"no FLOAT_MODE", std::nullopt}, + {"dx10 ieee ovfl", ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, true}}, + {"dx10 ieee", ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}}, + {"dx10 ovfl", ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, true}}, + {"dx10", ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}}, + {"ieee ovfl", ShaderRecompiler::ShaderFloatMode{0xc0u, false, true, true}}, + {"ieee", ShaderRecompiler::ShaderFloatMode{0xc0u, false, true, false}}, + {"ovfl", ShaderRecompiler::ShaderFloatMode{0xc0u, false, false, true}}, + {"f16 denormals flushed", ShaderRecompiler::ShaderFloatMode{0x00u, true, true, false}}, +}}; + +const std::array UnaryModes{{ + {"no FLOAT_MODE", std::nullopt}, + {"dx10 ieee", ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}}, + {"dx10", ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}}, + {"f16 denormals flushed", ShaderRecompiler::ShaderFloatMode{0x00u, true, true, false}}, +}}; + +constexpr std::array Specials{ + 0x0000u, 0x8000u, 0x0001u, 0x8001u, 0x03ffu, 0x0400u, 0x3c00u, 0xbc00u, 0x3555u, 0x4000u, 0xc400u, 0x7bffu, 0xfbffu, + 0x7c00u, 0xfc00u, 0x7c01u, 0xfc01u, 0x7d55u, 0xfdffu, 0x7e00u, 0xfe00u, 0x7e55u, 0xfeaau, 0x7fffu, 0xffffu, 0x5bffu, +}; + +std::uint64_t Seed = 0x9e3779b97f4a7c15ull; + +std::uint32_t Next() { + Seed += 0x9e3779b97f4a7c15ull; + std::uint64_t z = Seed; + z = (z ^ (z >> 30u)) * 0xbf58476d1ce4e5b9ull; + z = (z ^ (z >> 27u)) * 0x94d049bb133111ebull; + return static_cast(z ^ (z >> 31u)); +} + +std::uint32_t Other() { + const std::uint32_t bits = Next(); + return (bits & 1u) != 0u ? Specials[(bits >> 1u) % Specials.size()] : (bits >> 16u); +} + +bool IsNan(std::uint32_t half) { + return (half & 0x7fffu) > 0x7c00u; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::uint32_t CountOpcode(std::span spirv, std::uint32_t wanted) { + std::uint32_t count = 0u; + for (std::size_t offset = 5u; offset < spirv.size();) { + const std::uint32_t words = spirv[offset] >> 16u; + Require(words != 0u && offset + words <= spirv.size(), "f16 unused NaN payloads: malformed SPIR-V"); + if ((spirv[offset] & 0xffffu) == wanted) ++count; + offset += words; + } + return count; +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto target = device->Target(); + const auto supports = [&](spv::Capability capability) { + return std::ranges::find(target.supportedCapabilities, static_cast(capability)) != target.supportedCapabilities.end(); + }; + if (!supports(spv::CapabilityFloat16) || !supports(spv::CapabilityRoundingModeRTE) || !supports(spv::CapabilityDenormPreserve) || !target.float16DenormPreserve) { + std::puts("f16 unused NaN payloads: skipped, the device has no native f16 conversions with RTE rounding and kept f16 denormals"); + return VulkanTestSkipped; + } + std::vector withoutFloat16; + std::ranges::copy_if(target.supportedCapabilities, std::back_inserter(withoutFloat16), [](std::uint32_t capability) { return capability != spv::CapabilityFloat16; }); + auto softwareTarget = target; + softwareTarget.supportedCapabilities = withoutFloat16; + softwareTarget.float16DenormPreserve = false; + + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + const auto compile = [&](std::span code, const ShaderRecompiler::SpirvTarget& shaderTarget, const Mode& mode) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + shaderTarget, + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = mode.floatMode; + return ShaderRecompiler::Recompile(request); + }; + const auto run = [&](const auto& result, std::span code) { + Output.fill(0xdeadbeefu); + device->Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device->WaitIdle(); + return Output; + }; + const auto compare = [&](const char* group, const Mode& mode, const char* const* names, const auto& fast, const auto& software) { + std::uint64_t nanResults = 0u; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* row = &Input[tid * Inputs]; + for (std::uint32_t index = 0; index < Results; ++index) { + const std::uint32_t actual = fast[tid * Results + index]; + const std::uint32_t expected = software[tid * Results + index]; + Require(actual == expected, std::string("f16 unused NaN payloads: ") + group + ", " + mode.name + ", " + names[index] + " with v4=" + Hex(row[0]) + " v5=" + Hex(row[1]) + " v6=" + Hex(row[2]) + " v7=" + Hex(row[3]) + " is " + Hex(actual) + ", the software conversions give " + Hex(expected)); + nanResults += (IsNan(expected & 0xffffu) ? 1u : 0u) + (IsNan(expected >> 16u) ? 1u : 0u); + } + } + return nanResults; + }; + + std::uint64_t compared = 0u; + std::uint64_t nanResults = 0u; + const std::span packedCode(PackedCode); + for (const auto& mode : PackedModes) { + const auto fast = compile(packedCode, target, mode); + const auto software = compile(packedCode, softwareTarget, mode); + if (mode.floatMode.has_value() && mode.floatMode->dx10Clamp) { + Require(CountOpcode(fast.spirv.Words(), spv::OpIsNan) == 0u, std::string("f16 unused NaN payloads: packed f16, ") + mode.name + " still rebuilds the NaN of an f16 input whose payload the result never uses"); + } + for (std::uint32_t exhaustive = 0; exhaustive < Inputs; ++exhaustive) { + for (std::uint32_t first = 0; first < Halves; first += Threads * 2u) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t index = 0; index < Inputs; ++index) { + const std::uint32_t low = index == exhaustive ? first + tid * 2u : Other(); + const std::uint32_t high = index == exhaustive ? first + tid * 2u + 1u : Other(); + Input[tid * Inputs + index] = low | (high << 16u); + } + } + const auto fastOutput = run(fast, packedCode); + const auto softwareOutput = run(software, packedCode); + nanResults += compare("packed f16", mode, PackedNames, fastOutput, softwareOutput); + compared += Threads * Results * 2u; + } + } + } + const std::span unaryCode(UnaryCode); + for (const auto& mode : UnaryModes) { + const auto fast = compile(unaryCode, target, mode); + const auto software = compile(unaryCode, softwareTarget, mode); + for (std::uint32_t first = 0; first < Halves; first += Threads) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Input[tid * Inputs] = (first + tid) | (Next() & 0xffff0000u); + for (std::uint32_t index = 1; index < Inputs; ++index) Input[tid * Inputs + index] = Next(); + } + const auto fastOutput = run(fast, unaryCode); + const auto softwareOutput = run(software, unaryCode); + nanResults += compare("unary f16", mode, UnaryNames, fastOutput, softwareOutput); + compared += Threads * Results; + } + } + std::printf("f16 unused NaN payloads tests passed (%llu results, %llu NaN halves)\n", static_cast(compared), static_cast(nanResults)); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index c6b34841ba..5de0ad066d 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -518,6 +518,10 @@ struct PermlaneFlags { bool boundControl = false; }; +struct FloatConvertFlags { + bool nanPayloadUnused = false; +}; + struct MemoryFlags { std::uint32_t index = 0; std::uint32_t pc = 0; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index c442047480..b81fe6d633 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -171,6 +171,7 @@ std::uint32_t EmitLogicalNotBool(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitFNegateValue(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitFAbsValue(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitPackHalf2x16(SpirvEmitterState& state, std::uint32_t arg0); +std::uint32_t EmitPackHalf2x16Value(SpirvEmitterState& state, const IrValue& inst, std::uint32_t arg0); std::uint32_t EmitF32ToF16BitsRte(SpirvEmitterState& state, std::uint32_t value); std::uint32_t EmitFPFma32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2); std::uint32_t EmitFPMad32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index abf7b6fce4..03cc328db2 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -1554,7 +1554,7 @@ static void EmitNativeF16Modes(SpirvEmitterState& state) { } } -static std::uint32_t EmitNativeF32ToF16BitsRte(SpirvEmitterState& state, std::uint32_t value) { +static std::uint32_t EmitNativeF32ToF16BitsRte(SpirvEmitterState& state, std::uint32_t value, bool nanPayloadUnused = false) { EmitNativeF16Modes(state); const auto u32 = TypeU32(state); const auto f16 = state.module.Type(spv::OpTypeFloat, 16u); @@ -1564,6 +1564,7 @@ static std::uint32_t EmitNativeF32ToF16BitsRte(SpirvEmitterState& state, std::ui const auto bits = Unary(state, spv::OpBitcast, u32, pair); const auto source = Unary(state, spv::OpBitcast, u32, value); const auto magnitude = Binary(state, spv::OpBitwiseAnd, u32, source, ConstantU32(state, 0x7fffffffu)); + if (nanPayloadUnused) return Select(state, u32, Binary(state, spv::OpUGreaterThan, TypeBool(state), magnitude, ConstantU32(state, 0x7f800000u)), ConstantU32(state, 0x7e00u), bits); const auto sign = Binary(state, spv::OpBitwiseAnd, u32, Binary(state, spv::OpShiftRightLogical, u32, source, ConstantU32(state, 16u)), ConstantU32(state, 0x8000u)); const auto payload = Binary(state, spv::OpShiftRightLogical, u32, Binary(state, spv::OpBitwiseAnd, u32, magnitude, ConstantU32(state, 0x7fffffu)), ConstantU32(state, 13u)); const auto nan = Binary(state, spv::OpBitwiseOr, u32, ConstantU32(state, 0x7e00u), payload); @@ -1620,16 +1621,24 @@ std::uint32_t EmitF32ToF16BitsRte(SpirvEmitterState& state, std::uint32_t value) return op(spv::OpBitwiseOr, sign, result); } -std::uint32_t EmitPackHalf2x16(SpirvEmitterState& state, std::uint32_t arg0) { +static std::uint32_t PackHalf2x16(SpirvEmitterState& state, std::uint32_t arg0, bool nanPayloadUnused) { const auto component = [&](std::uint32_t index) { const auto value = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, TypeF32(state), value, arg0, index); - return EmitF32ToF16BitsRte(state, value); + return nanPayloadUnused && NativeF16Rte(state) ? EmitNativeF32ToF16BitsRte(state, value, true) : EmitF32ToF16BitsRte(state, value); }; const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), component(1u), ConstantU32(state, 16u)); return Binary(state, spv::OpBitwiseOr, TypeU32(state), component(0u), high); } +std::uint32_t EmitPackHalf2x16(SpirvEmitterState& state, std::uint32_t arg0) { + return PackHalf2x16(state, arg0, false); +} + +std::uint32_t EmitPackHalf2x16Value(SpirvEmitterState& state, const IrValue& inst, std::uint32_t arg0) { + return PackHalf2x16(state, arg0, inst.Flags().nanPayloadUnused); +} + std::uint32_t EmitFPFma32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2) { return Exact(state, EmitGlsl(state, arg0, arg1, arg2)); } @@ -1717,7 +1726,7 @@ static bool OnlyCompared(const IrValue& value) { std::uint32_t EmitConvertF32F16(SpirvEmitterState& state, const IrValue& inst, std::uint32_t bits) { if (NarrowF16Field(*inst.Argument(0))) return EmitF16BitsToF32Decoded(state, bits); - return OnlyCompared(inst) ? EmitF16BitsToF32AnyNan(state, bits) : EmitF16BitsToF32(state, bits); + return inst.Flags().nanPayloadUnused || OnlyCompared(inst) ? EmitF16BitsToF32AnyNan(state, bits) : EmitF16BitsToF32(state, bits); } std::uint32_t EmitCompositeExtractU32x2(SpirvEmitterState& state, std::uint32_t arg0, const IrValue* arg1) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 60c9ac8779..92881a716f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -378,7 +378,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::CompositeExtractU32x2: return Invoke(EmitCompositeExtractU32x2, ctx, inst); case IrOpcode::CompositeExtractU32x3: return Invoke(EmitCompositeExtractU32x3, ctx, inst); case IrOpcode::CompositeExtractU32x4: return Invoke(EmitCompositeExtractU32x4, ctx, inst); - case IrOpcode::PackHalf2x16: return Invoke(EmitPackHalf2x16, ctx, inst); + case IrOpcode::PackHalf2x16: return Invoke(EmitPackHalf2x16Value, ctx, inst); case IrOpcode::PackFloat2x16Rtz: return Invoke(EmitPackFloat2x16Rtz, ctx, inst); case IrOpcode::FPAdd16x2: return Invoke(EmitFPAdd16x2, ctx, inst); case IrOpcode::FPMul16x2: return Invoke(EmitFPMul16x2, ctx, inst); diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 00da5a7db5..dc830b7893 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -69,7 +69,7 @@ private: void rejectHalfOrDoubleOutputModifier(const RdnaOperand& operand) const; IrU32 clampF16Bits(const RdnaOperand& operand, IrU32 bits); void writeOperand(const RdnaOperand& operand, IrValue* value); - IrU32 packHalf2x16(IrF32 low, IrF32 high); + IrU32 packHalf2x16(IrF32 low, IrF32 high, bool nanPayloadUnused = false); void write16Bits(const RdnaOperand& operand, IrU32 value); void writeF16(const RdnaOperand& operand, IrF32 value, std::initializer_list sources, IrValue* invalid = nullptr); bool directedF16Rounding() const; @@ -96,8 +96,8 @@ private: std::array flushF64Bits(const std::array& bits, std::uint32_t keepBit); IrValue& flushTinyF64Result(IrOpcode opcode, IrValue& lhs, IrValue& rhs, IrValue& result); IrU64 readF64(const RdnaOperand& operand); - IrF32 readF16LaneAsF32(const RdnaOperand& operand, bool highLane, bool packed = false); - IrF32 readF16AsF32(const RdnaOperand& operand); + IrF32 readF16LaneAsF32(const RdnaOperand& operand, bool highLane, bool packed = false, bool nanPayloadUnused = false); + IrF32 readF16AsF32(const RdnaOperand& operand, bool nanPayloadUnused = false); IrU32 readF16SourceBits(const RdnaOperand& operand); IrF32 readMixF32(const RdnaOperand& operand); IrU32 readU16LaneRaw(const RdnaOperand& operand, bool highLane); diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index 0632342acb..047aa73763 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -21,7 +21,7 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc const IrU1 tiny(ir.ULessThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu)), ir.Constant(opcode == IrOpcode::FPAdd32 ? 0x38800000u : 0x387ff000u))); return IrF32(ir.BitCastF32(ir.Select(tiny.Value(), ir.BitwiseAnd(bits.Value(), ir.Constant(0x80000000u)), bits.Value()))); }; - const auto readLane = [&](const RdnaOperand& operand, bool high) { return flushF16InputValue(readF16LaneAsF32(operand, high, true)); }; + const auto readLane = [&](const RdnaOperand& operand, bool high) { return flushF16InputValue(readF16LaneAsF32(operand, high, true, true)); }; const auto translateLane = [&](bool high) -> IrF32 { const IrF32 lhs = readLane(sourceAt(inst, 0u), high); const IrF32 rhs = readLane(sourceAt(inst, 1u), high); @@ -65,7 +65,7 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc const IrU32 rhs = packU16Lanes(sourceLaneBits(1u, false), sourceLaneBits(1u, true)); return IrU32(ir.Emit(opcode == IrOpcode::FPAdd32 ? IrOpcode::FPAdd16x2 : IrOpcode::FPMul16x2, IrType::U32, {&lhs.Value(), &rhs.Value()})); }; - IrU32 result = quietSnan ? packU16Lanes(minMaxLane(false), minMaxLane(true)) : native ? nativeResult() : packHalf2x16(translateLane(false), translateLane(true)); + IrU32 result = quietSnan ? packU16Lanes(minMaxLane(false), minMaxLane(true)) : native ? nativeResult() : packHalf2x16(translateLane(false), translateLane(true), true); if (native || !inst.destination.clamp || !dx10Clamp() || quietSnan) { const auto isNan = [&](const IrU32& bits) { return IrU1(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); @@ -122,7 +122,7 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc bool TranslationContext::float16Unary(const RdnaInstruction& inst, IrOpcode opcode) { const RdnaOperand& operand = sourceAt(inst, 0u); - const IrF32 argument = flushF16InputValue(readF16AsF32(operand)); + const IrF32 argument = flushF16InputValue(readF16AsF32(operand, true)); const auto [result, invalid] = unaryFloatSpecials(opcode, argument, IrF32(ir.Emit(opcode, IrType::F32, {&argument.Value()}))); const IrF16 rounded(ir.Emit(IrOpcode::ConvertF16F32, IrType::F16, {&applyF16ResultModifiers(inst.destination, clampF16Overflow(result, {&argument.Value()})).Value()})); IrU32 bits(ir.Emit(IrOpcode::ConvertU32U16, IrType::U32, {&ir.Emit(IrOpcode::BitCastU16F16, IrType::U16, {&rounded.Value()})})); diff --git a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp index 3ba6021ff2..4a7df53850 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp @@ -67,13 +67,14 @@ void TranslationContext::writeU32Pair(const RdnaOperand& operand, const std::arr writeRawU32(offsetOperand(operand, 1u), value[1]); } -IrF32 TranslationContext::readF16LaneAsF32(const RdnaOperand& operand, bool highLane, bool packed) { +IrF32 TranslationContext::readF16LaneAsF32(const RdnaOperand& operand, bool highLane, bool packed, bool nanPayloadUnused) { const bool selectHigh = packed ? (highLane ? operand.opSelHi : operand.opSel) : highLane; const IrU32 source = readF16SourceBits(operand); const IrU32 raw(ir.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&source.Value(), &ir.Constant(selectHigh ? 16u : 0u), &ir.Constant(16u)})); const IrU16 bits(ir.Emit(IrOpcode::ConvertU16U32, IrType::U16, {&raw.Value()})); const IrF16 half(ir.Emit(IrOpcode::BitCastF16U16, IrType::F16, {&bits.Value()})); IrF32 value(ir.Emit(IrOpcode::ConvertF32F16, IrType::F32, {&half.Value()})); + if (nanPayloadUnused) value.Value().SetFlags(FloatConvertFlags{true}); if (operand.absolute) { value = IrF32(ir.Emit(IrOpcode::FPAbs32, IrType::F32, {&value.Value()})); } @@ -104,12 +105,13 @@ IrU32 TranslationContext::readF16SourceBits(const RdnaOperand& operand) { return applyBitSourceModifiers(operand, IrU32(ir.Constant(half))); } -IrF32 TranslationContext::readF16AsF32(const RdnaOperand& operand) { +IrF32 TranslationContext::readF16AsF32(const RdnaOperand& operand, bool nanPayloadUnused) { const IrU32 source = readF16SourceBits(operand); const IrU32 raw = operand.opSel ? IrU32(ir.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&source.Value(), &ir.Constant(16u), &ir.Constant(16u)})) : source; const IrU16 bits(ir.Emit(IrOpcode::ConvertU16U32, IrType::U16, {&raw.Value()})); const IrF16 half(ir.Emit(IrOpcode::BitCastF16U16, IrType::F16, {&bits.Value()})); IrF32 value(ir.Emit(IrOpcode::ConvertF32F16, IrType::F32, {&half.Value()})); + if (nanPayloadUnused) value.Value().SetFlags(FloatConvertFlags{true}); if (operand.absolute) { value = IrF32(ir.Emit(IrOpcode::FPAbs32, IrType::F32, {&value.Value()})); } @@ -154,9 +156,11 @@ IrU32 TranslationContext::readU16AsU32(const RdnaOperand& operand, bool signExte return IrU32(ir.Emit(IrOpcode::BitFieldSExtract, IrType::U32, {&raw.Value(), &ir.Constant(0u), &ir.Constant(16u)})); } -IrU32 TranslationContext::packHalf2x16(IrF32 low, IrF32 high) { +IrU32 TranslationContext::packHalf2x16(IrF32 low, IrF32 high, bool nanPayloadUnused) { IrValue& pair = ir.Emit(IrOpcode::CompositeConstructF32x2, IrType::F32x2, {&low.Value(), &high.Value()}); - return IrU32(ir.Emit(IrOpcode::PackHalf2x16, IrType::U32, {&pair})); + IrValue& packed = ir.Emit(IrOpcode::PackHalf2x16, IrType::U32, {&pair}); + if (nanPayloadUnused) packed.SetFlags(FloatConvertFlags{true}); + return IrU32(packed); } void TranslationContext::write16Bits(const RdnaOperand& operand, IrU32 value) { From 103cf586a9fb6cca773d1b77a696b148006f853c Mon Sep 17 00:00:00 2001 From: thibaut Date: Mon, 26 Oct 2026 10:15:58 +0100 Subject: [PATCH 143/318] test(agc): compare the unused f16 NaN payload programs in one dispatch per pass Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PHJtpJ4hw3KWJhobipK8ia --- .../execution/Float16UnusedNanPayloads.cpp | 79 ++++++++++--------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp index 18277b5636..c9d98ff1a5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp @@ -22,11 +22,14 @@ constexpr std::uint32_t Threads = 64; constexpr std::uint32_t Inputs = 4; constexpr std::uint32_t Results = 8; constexpr std::uint32_t Halves = 0x10000u; -alignas(256) std::array Input{}; -alignas(256) std::array Output{}; +constexpr std::uint32_t PackedGroups = Halves / (Threads * 2u); +constexpr std::uint32_t UnaryGroups = Halves / Threads; +constexpr std::uint32_t Rows = UnaryGroups * Threads; +std::vector Input(Rows * Inputs); +std::vector Output(Rows * Results); -alignas(256) constexpr std::array PackedCode{ - 0x34020084u, 0x34060085u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, +alignas(256) constexpr std::array PackedCode{ + 0x34020084u, 0x34060085u, 0x8f098a08u, 0x8f0a8b08u, 0x4a020209u, 0x4a06060au, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0xcc0f400au, 0x18020b04u, 0xcc10400bu, 0x18020b04u, 0xcc0e400cu, 0x1c1a0b04u, @@ -56,16 +59,16 @@ constexpr const char* UnaryNames[Results] = { }; constexpr std::array MakeUnaryCode() { - std::array code{0x34020084u, 0x34060085u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u}; + std::array code{0x34020084u, 0x34060085u, 0x8f098a08u, 0x8f0a8b08u, 0x4a020209u, 0x4a06060au, 0xe0381000u, 0x80000401u, 0xbf8c3f70u}; for (std::uint32_t index = 0; index < Results; ++index) { - code[5u + index] = 0x7e000280u | ((10u + index) << 17u) | (1u << 9u); - code[13u + index] = 0x7e000104u | ((10u + index) << 17u) | (UnaryOpcodes[index] << 9u); + code[9u + index] = 0x7e000280u | ((10u + index) << 17u) | (1u << 9u); + code[17u + index] = 0x7e000104u | ((10u + index) << 17u) | (UnaryOpcodes[index] << 9u); } - code[21u] = 0xe0781000u; - code[22u] = 0x80010a03u; - code[23u] = 0xe0781010u; - code[24u] = 0x80010e03u; - code[25u] = 0xbf810000u; + code[25u] = 0xe0781000u; + code[26u] = 0x80010a03u; + code[27u] = 0xe0781010u; + code[28u] = 0x80010e03u; + code[29u] = 0xbf810000u; return code; } @@ -166,7 +169,7 @@ int main() { const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); std::copy(input.begin(), input.end(), userData.begin()); std::copy(output.begin(), output.end(), userData.begin() + 4); - const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {true, false, false}, false, 1}; const auto compile = [&](std::span code, const ShaderRecompiler::SpirvTarget& shaderTarget, const Mode& mode) { const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; ShaderRecompiler::RecompileRequest request{ @@ -179,20 +182,21 @@ int main() { request.context.floatMode = mode.floatMode; return ShaderRecompiler::Recompile(request); }; - const auto run = [&](const auto& result, std::span code) { - Output.fill(0xdeadbeefu); - device->Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + const auto run = [&](const auto& result, std::span code, std::uint32_t groups) { + std::ranges::fill(Output, 0xdeadbeefu); + device->Dispatch(result, groups, 1, 1, {}, reinterpret_cast(code.data())); device->WaitIdle(); return Output; }; - const auto compare = [&](const char* group, const Mode& mode, const char* const* names, const auto& fast, const auto& software) { + const auto compare = [&](const char* group, const Mode& mode, const char* const* names, const auto& fast, const auto& software, std::uint32_t rows) { std::uint64_t nanResults = 0u; - for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t tid = 0; tid < rows; ++tid) { const std::uint32_t* row = &Input[tid * Inputs]; + if (std::ranges::all_of(std::span(&software[tid * Results], Results), [](std::uint32_t word) { return word == 0xdeadbeefu; })) Require(false, std::string("f16 unused NaN payloads: ") + group + ", " + mode.name + ", invocation " + std::to_string(tid) + " wrote no result"); for (std::uint32_t index = 0; index < Results; ++index) { const std::uint32_t actual = fast[tid * Results + index]; const std::uint32_t expected = software[tid * Results + index]; - Require(actual == expected, std::string("f16 unused NaN payloads: ") + group + ", " + mode.name + ", " + names[index] + " with v4=" + Hex(row[0]) + " v5=" + Hex(row[1]) + " v6=" + Hex(row[2]) + " v7=" + Hex(row[3]) + " is " + Hex(actual) + ", the software conversions give " + Hex(expected)); + if (actual != expected) Require(false, std::string("f16 unused NaN payloads: ") + group + ", " + mode.name + ", " + names[index] + " with v4=" + Hex(row[0]) + " v5=" + Hex(row[1]) + " v6=" + Hex(row[2]) + " v7=" + Hex(row[3]) + " is " + Hex(actual) + ", the software conversions give " + Hex(expected)); nanResults += (IsNan(expected & 0xffffu) ? 1u : 0u) + (IsNan(expected >> 16u) ? 1u : 0u); } } @@ -208,36 +212,33 @@ int main() { if (mode.floatMode.has_value() && mode.floatMode->dx10Clamp) { Require(CountOpcode(fast.spirv.Words(), spv::OpIsNan) == 0u, std::string("f16 unused NaN payloads: packed f16, ") + mode.name + " still rebuilds the NaN of an f16 input whose payload the result never uses"); } + constexpr std::uint32_t rows = PackedGroups * Threads; for (std::uint32_t exhaustive = 0; exhaustive < Inputs; ++exhaustive) { - for (std::uint32_t first = 0; first < Halves; first += Threads * 2u) { - for (std::uint32_t tid = 0; tid < Threads; ++tid) { - for (std::uint32_t index = 0; index < Inputs; ++index) { - const std::uint32_t low = index == exhaustive ? first + tid * 2u : Other(); - const std::uint32_t high = index == exhaustive ? first + tid * 2u + 1u : Other(); - Input[tid * Inputs + index] = low | (high << 16u); - } + for (std::uint32_t tid = 0; tid < rows; ++tid) { + for (std::uint32_t index = 0; index < Inputs; ++index) { + const std::uint32_t low = index == exhaustive ? tid * 2u : Other(); + const std::uint32_t high = index == exhaustive ? tid * 2u + 1u : Other(); + Input[tid * Inputs + index] = low | (high << 16u); } - const auto fastOutput = run(fast, packedCode); - const auto softwareOutput = run(software, packedCode); - nanResults += compare("packed f16", mode, PackedNames, fastOutput, softwareOutput); - compared += Threads * Results * 2u; } + const auto fastOutput = run(fast, packedCode, PackedGroups); + const auto softwareOutput = run(software, packedCode, PackedGroups); + nanResults += compare("packed f16", mode, PackedNames, fastOutput, softwareOutput, rows); + compared += rows * Results * 2u; } } const std::span unaryCode(UnaryCode); for (const auto& mode : UnaryModes) { const auto fast = compile(unaryCode, target, mode); const auto software = compile(unaryCode, softwareTarget, mode); - for (std::uint32_t first = 0; first < Halves; first += Threads) { - for (std::uint32_t tid = 0; tid < Threads; ++tid) { - Input[tid * Inputs] = (first + tid) | (Next() & 0xffff0000u); - for (std::uint32_t index = 1; index < Inputs; ++index) Input[tid * Inputs + index] = Next(); - } - const auto fastOutput = run(fast, unaryCode); - const auto softwareOutput = run(software, unaryCode); - nanResults += compare("unary f16", mode, UnaryNames, fastOutput, softwareOutput); - compared += Threads * Results; + for (std::uint32_t tid = 0; tid < Rows; ++tid) { + Input[tid * Inputs] = tid | (Next() & 0xffff0000u); + for (std::uint32_t index = 1; index < Inputs; ++index) Input[tid * Inputs + index] = Next(); } + const auto fastOutput = run(fast, unaryCode, UnaryGroups); + const auto softwareOutput = run(software, unaryCode, UnaryGroups); + nanResults += compare("unary f16", mode, UnaryNames, fastOutput, softwareOutput, Rows); + compared += Rows * Results; } std::printf("f16 unused NaN payloads tests passed (%llu results, %llu NaN halves)\n", static_cast(compared), static_cast(nanResults)); return 0; From 2a9175b6a6a2cea63eb7dce6aa62a4d77ea9bee1 Mon Sep 17 00:00:00 2001 From: thibaut Date: Mon, 26 Oct 2026 10:31:12 +0100 Subject: [PATCH 144/318] test(agc): give each compiled form its own unwritten-result marker An output word that neither program stores kept the same sentinel in both buffers and compared equal, so a missing store could pass. The device and software runs now start from different words. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PHJtpJ4hw3KWJhobipK8ia --- .../tests/execution/Float16UnusedNanPayloads.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp index c9d98ff1a5..190a33208a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16UnusedNanPayloads.cpp @@ -182,8 +182,8 @@ int main() { request.context.floatMode = mode.floatMode; return ShaderRecompiler::Recompile(request); }; - const auto run = [&](const auto& result, std::span code, std::uint32_t groups) { - std::ranges::fill(Output, 0xdeadbeefu); + const auto run = [&](const auto& result, std::span code, std::uint32_t groups, std::uint32_t unwritten) { + std::ranges::fill(Output, unwritten); device->Dispatch(result, groups, 1, 1, {}, reinterpret_cast(code.data())); device->WaitIdle(); return Output; @@ -192,7 +192,6 @@ int main() { std::uint64_t nanResults = 0u; for (std::uint32_t tid = 0; tid < rows; ++tid) { const std::uint32_t* row = &Input[tid * Inputs]; - if (std::ranges::all_of(std::span(&software[tid * Results], Results), [](std::uint32_t word) { return word == 0xdeadbeefu; })) Require(false, std::string("f16 unused NaN payloads: ") + group + ", " + mode.name + ", invocation " + std::to_string(tid) + " wrote no result"); for (std::uint32_t index = 0; index < Results; ++index) { const std::uint32_t actual = fast[tid * Results + index]; const std::uint32_t expected = software[tid * Results + index]; @@ -221,8 +220,8 @@ int main() { Input[tid * Inputs + index] = low | (high << 16u); } } - const auto fastOutput = run(fast, packedCode, PackedGroups); - const auto softwareOutput = run(software, packedCode, PackedGroups); + const auto fastOutput = run(fast, packedCode, PackedGroups, 0xdeadbeefu); + const auto softwareOutput = run(software, packedCode, PackedGroups, 0xfeedfaceu); nanResults += compare("packed f16", mode, PackedNames, fastOutput, softwareOutput, rows); compared += rows * Results * 2u; } @@ -235,8 +234,8 @@ int main() { Input[tid * Inputs] = tid | (Next() & 0xffff0000u); for (std::uint32_t index = 1; index < Inputs; ++index) Input[tid * Inputs + index] = Next(); } - const auto fastOutput = run(fast, unaryCode, UnaryGroups); - const auto softwareOutput = run(software, unaryCode, UnaryGroups); + const auto fastOutput = run(fast, unaryCode, UnaryGroups, 0xdeadbeefu); + const auto softwareOutput = run(software, unaryCode, UnaryGroups, 0xfeedfaceu); nanResults += compare("unary f16", mode, UnaryNames, fastOutput, softwareOutput, Rows); compared += Rows * Results; } From 96bb078a0c9d9372232bff1b1a74a5950285e69d Mon Sep 17 00:00:00 2001 From: Evrim Cicek Date: Sun, 25 Oct 2026 00:28:39 +0300 Subject: [PATCH 145/318] fix(libc): drop the kernel state of a descriptor that fclose or freopen closes --- core/libs/prx/libc/include/FileStream.hpp | 7 ++ core/libs/prx/libc/src/FileAndHeap.cpp | 10 ++ core/libs/prx/libkernel/CMakeLists.txt | 8 +- core/libs/prx/libkernel/File/include/File.hpp | 1 + core/libs/prx/libkernel/File/src/Open.cpp | 40 +++++-- core/libs/prx/libkernel/File/src/Stdio.cpp | 6 +- core/libs/tests/GuestStreamClose.cpp | 108 ++++++++++++++++++ 7 files changed, 167 insertions(+), 13 deletions(-) create mode 100644 core/libs/tests/GuestStreamClose.cpp diff --git a/core/libs/prx/libc/include/FileStream.hpp b/core/libs/prx/libc/include/FileStream.hpp index a97f626f49..981f4d7e84 100644 --- a/core/libs/prx/libc/include/FileStream.hpp +++ b/core/libs/prx/libc/include/FileStream.hpp @@ -34,6 +34,8 @@ extern "C" void ForgetDirectoryStream_nid_no_patch(int descriptor); extern "C" int _IO_switch_to_get_mode(std::FILE* stream); #endif extern "C" int DuplicateHostDescriptor_nid_no_patch(int descriptor); +extern "C" void LibcRegisterClosedDescriptors_nid_no_patch(void (*forget)(int descriptor)); +extern "C" void ForgetClosedDescriptor_nid_no_patch(int descriptor); struct GuestFilePrefix { unsigned char* position = nullptr; @@ -158,6 +160,7 @@ public: if (std::exchange(directory, false)) ForgetDirectoryStream_nid_no_patch(replaced); auto* const standard = HostStandardHandle_nid_no_patch(replaced); GuestFileLockForget_nid_no_patch(replaced); + ForgetClosedDescriptor_nid_no_patch(replaced); if (open && !filename.empty()) { if (!IsHostStandard()) { _handle = nullptr; @@ -201,6 +204,7 @@ public: KeepStandardHandle_nid_no_patch(replaced, standard); #else if (::fileno(previous) == 2) KeepHostStandardError_nid_no_patch(); + ForgetClosedDescriptor_nid_no_patch(::fileno(previous)); const int kept = IsHostStandard() ? DuplicateHostDescriptor_nid_no_patch(::fileno(previous)) : -1; if (kept >= 0) ::flockfile(previous); _handle = std::freopen(filename.c_str(), mode, previous); @@ -302,6 +306,9 @@ public: #ifdef _WIN32 GuestFileLockForget_nid_no_patch(::_fileno(_handle)); if (std::exchange(directory, false)) ForgetDirectoryStream_nid_no_patch(::_fileno(_handle)); + ForgetClosedDescriptor_nid_no_patch(::_fileno(_handle)); +#else + ForgetClosedDescriptor_nid_no_patch(::fileno(_handle)); #endif const int result = std::fclose(std::exchange(_handle, nullptr)); _guest.flags = 0; diff --git a/core/libs/prx/libc/src/FileAndHeap.cpp b/core/libs/prx/libc/src/FileAndHeap.cpp index 51550e1dac..f82043ef28 100644 --- a/core/libs/prx/libc/src/FileAndHeap.cpp +++ b/core/libs/prx/libc/src/FileAndHeap.cpp @@ -336,6 +336,16 @@ static int OpenFailure(const std::filesystem::path& path, bool writes, int hostE return writes && error == 13 && std::filesystem::is_directory(path, status) ? 21 : error; } +static void (*g_forgetClosedDescriptor)(int) = nullptr; + +void LibcRegisterClosedDescriptors_nid_no_patch(void (*forget)(int)) { + g_forgetClosedDescriptor = forget; +} + +void ForgetClosedDescriptor_nid_no_patch(int descriptor) { + if (g_forgetClosedDescriptor) g_forgetClosedDescriptor(descriptor); +} + #ifdef _WIN32 static int (*g_openDirectoryDescriptor)(const wchar_t*) = nullptr; static void (*g_forgetDirectoryDescriptor)(int) = nullptr; diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 2a4394d2cb..45ebf6b17b 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -1975,8 +1975,14 @@ if(BUILD_TESTING) set_tests_properties(guest_kernel_fcntl_sync PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_stream_close_tests ${LIBS_INCLUDE_DIR}/tests/GuestStreamClose.cpp) + target_include_directories(guest_stream_close_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_stream_close_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_stream_close_tests) + add_test(NAME guest_stream_close COMMAND guest_stream_close_tests) + set_tests_properties(guest_stream_close PROPERTIES TIMEOUT 15) if(WIN32) - set_tests_properties(guest_kernel_fcntl guest_append_writes PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_kernel_fcntl guest_append_writes guest_stream_close PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") add_test_executable(guest_file_lock_tests ${LIBS_INCLUDE_DIR}/tests/GuestFileLock.cpp) target_include_directories(guest_file_lock_tests PRIVATE ${LIBS_INCLUDE_DIR}) diff --git a/core/libs/prx/libkernel/File/include/File.hpp b/core/libs/prx/libkernel/File/include/File.hpp index 6c8fbfadc2..566f04a3de 100644 --- a/core/libs/prx/libkernel/File/include/File.hpp +++ b/core/libs/prx/libkernel/File/include/File.hpp @@ -33,6 +33,7 @@ namespace File { std::optional WindowsDescriptorPath_nid_no_patch(int descriptor); #endif int HostStandardError(); +void ForgetClosedDescriptor(int descriptor); [[nodiscard]] std::unique_lock LockPositionedTransfer(int descriptor); void SharePositionedTransfer(int from, int to); #ifdef _WIN32 diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp index 7609a30cf4..0d49ee5581 100644 --- a/core/libs/prx/libkernel/File/src/Open.cpp +++ b/core/libs/prx/libkernel/File/src/Open.cpp @@ -7,6 +7,7 @@ #include "prx/libc/include/RenameDot.hpp" #include "prx/libc/include/GuestArena.hpp" #include "prx/libc/include/AppArchive.hpp" +#include "prx/libc/include/FileStream.hpp" #include "prx/libkernel/DirectMemory/DirectMemory.hpp" #include "prx/libkernel/File/include/File.hpp" #include "prx/libkernel/File/include/SharedMemory.hpp" @@ -275,6 +276,37 @@ static int OpenNative(const std::filesystem::path& native, int flags, std::uint1 return fd; } +void File::ForgetClosedDescriptor(int descriptor) { + File::ForgetRandomDevice(descriptor); + File::ForgetDescriptor(descriptor); + ForgetJitSharedMemory(descriptor); +#ifdef _WIN32 + File::ForgetFileLock(descriptor); +#endif + File::ForgetDirectoryDescriptor(descriptor); +} + +namespace { + +void ForgetStreamDescriptor(int descriptor) { + EqueueDescriptorClosed(descriptor); +#ifdef _WIN32 + const auto positioned = File::LockPositionedTransfer(descriptor); +#endif + File::ForgetClosedDescriptor(descriptor); +} + +struct ClosedDescriptorRegistration { + ClosedDescriptorRegistration() { LibcRegisterClosedDescriptors_nid_no_patch(&ForgetStreamDescriptor); } + ~ClosedDescriptorRegistration() { LibcRegisterClosedDescriptors_nid_no_patch(nullptr); } + ClosedDescriptorRegistration(const ClosedDescriptorRegistration&) = delete; + ClosedDescriptorRegistration& operator=(const ClosedDescriptorRegistration&) = delete; +}; + +const ClosedDescriptorRegistration g_closedDescriptorRegistration; + +} + extern "C" { int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) { @@ -321,13 +353,7 @@ int APS5_VABI sceKernelClose(int d) { #ifdef _WIN32 const auto positioned = File::LockPositionedTransfer(d); #endif - File::ForgetRandomDevice(d); - File::ForgetDescriptor(d); - ForgetJitSharedMemory(d); -#ifdef _WIN32 - File::ForgetFileLock(d); -#endif - File::ForgetDirectoryDescriptor(d); + File::ForgetClosedDescriptor(d); if (NativeClose(d) != 0) { if (errno == EBADF) return SCE_KERNEL_ERROR_EBADF; throw std::runtime_error(std::string(__func__) + ": close failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 9068334dea..60e1394e50 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -773,12 +773,8 @@ int APS5_VABI close_nid_postfix(int d) { #ifdef _WIN32 const auto positioned = File::LockPositionedTransfer(d); #endif - File::ForgetRandomDevice(d); - File::ForgetDescriptor(d); - ForgetJitSharedMemory(d); - File::ForgetDirectoryDescriptor(d); + File::ForgetClosedDescriptor(d); #ifdef _WIN32 - File::ForgetFileLock(d); const int shared = File::CloseSharedMemoryDescriptor_nid_no_patch(d); if (shared != -2) return shared; return WithoutParameterHandler([d] { return ::_close(d); }); diff --git a/core/libs/tests/GuestStreamClose.cpp b/core/libs/tests/GuestStreamClose.cpp new file mode 100644 index 0000000000..505bb3f960 --- /dev/null +++ b/core/libs/tests/GuestStreamClose.cpp @@ -0,0 +1,108 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +class FileStream; + +extern "C" { +int APS5_VABI sceKernelOpen(const char*, int, std::uint16_t); +std::int64_t APS5_VABI sceKernelRead(int, void*, std::size_t); +std::int64_t APS5_VABI sceKernelWrite(int, const void*, std::size_t); +std::int64_t APS5_VABI sceKernelPwrite(int, const void*, std::size_t, std::int64_t); +std::int64_t APS5_VABI sceKernelLseek(int, std::int64_t, int); +int APS5_VABI fcntl_nid_postfix(int, int, ...); +FileStream* APS5_VABI fopen_nid_postfix(const char*, const char*); +FileStream* APS5_VABI fdopen_nid_postfix(int, const char*); +FileStream* APS5_VABI freopen_nid_postfix(const char*, const char*, FileStream*); +int APS5_VABI fclose_nid_postfix(FileStream*); +int APS5_VABI fileno_nid_postfix(FileStream*); +} + +static void Check(bool condition, int line) { + if (!condition) { + std::fprintf(stderr, "stream close check failed at line %d\n", line); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) + +static constexpr const char* Appended = "guest_stream_close_appended.txt"; +static constexpr const char* Reused = "guest_stream_close_reused.txt"; +static constexpr int ReadWrite = 0x0002; +static constexpr int Append = 0x0008; +static constexpr int CreateWriteAppend = 0x0209; +static constexpr int Truncate = 0x0400; +static constexpr int CloseOnExec = 0x00100000; +static constexpr int GetDescriptorFlags = 1; +static constexpr int GetStatusFlags = 3; +static constexpr int FromStart = 0; + +static void Store(const char* path, const std::string& contents) { + std::ofstream file(path, std::ios::binary | std::ios::trunc); + file << contents; +} + +static std::string Contents(const char* path) { + std::ifstream file(path, std::ios::binary); + return std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); +} + +static FileStream* Reopen(int closed, const char* mode) { + auto* stream = fopen_nid_postfix(Reused, mode); + Require(stream != nullptr); + Require(fileno_nid_postfix(stream) == closed); + return stream; +} + +int main() { + Store(Reused, "0123456789"); + const int appending = sceKernelOpen(Appended, CreateWriteAppend | Truncate | CloseOnExec, 0644); + Require(appending >= 0); + Require((fcntl_nid_postfix(appending, GetStatusFlags) & Append) != 0); + Require(fcntl_nid_postfix(appending, GetDescriptorFlags) == 1); + auto* stream = fdopen_nid_postfix(appending, "a"); + Require(stream != nullptr); + Require(fclose_nid_postfix(stream) == 0); + + stream = Reopen(appending, "r+"); + Require(fcntl_nid_postfix(appending, GetStatusFlags) == ReadWrite); + Require(fcntl_nid_postfix(appending, GetDescriptorFlags) == 0); + Require(sceKernelPwrite(appending, "AB", 2, 2) == 2); + Require(sceKernelLseek(appending, 4, FromStart) == 4); + Require(sceKernelWrite(appending, "CD", 2) == 2); + Require(fclose_nid_postfix(stream) == 0); + Require(Contents(Reused) == "01ABCD6789"); + + const int random = sceKernelOpen("/dev/urandom", 0, 0); + Require(random >= 0); + stream = fdopen_nid_postfix(random, "r"); + Require(stream != nullptr); + Require(fclose_nid_postfix(stream) == 0); + + stream = Reopen(random, "r"); + char read[16]{}; + Require(sceKernelRead(random, read, sizeof(read)) == 10); + Require(std::string(read, 10) == "01ABCD6789"); + Require(fclose_nid_postfix(stream) == 0); + + const int replaced = sceKernelOpen(Appended, CreateWriteAppend | CloseOnExec, 0644); + Require(replaced >= 0); + stream = fdopen_nid_postfix(replaced, "a"); + Require(stream != nullptr); + stream = freopen_nid_postfix(Reused, "r+", stream); + Require(stream != nullptr); + Require(fileno_nid_postfix(stream) == replaced); + Require(fcntl_nid_postfix(replaced, GetStatusFlags) == ReadWrite); + Require(fcntl_nid_postfix(replaced, GetDescriptorFlags) == 0); + Require(sceKernelPwrite(replaced, "EF", 2, 8) == 2); + Require(fclose_nid_postfix(stream) == 0); + Require(Contents(Reused) == "01ABCD67EF"); + + std::remove(Appended); + std::remove(Reused); +} From d1ab9af5dc4829247f4f34330bf7b957b83ac4d3 Mon Sep 17 00:00:00 2001 From: Evrim Cicek Date: Sun, 25 Oct 2026 01:33:05 +0300 Subject: [PATCH 146/318] fix(libc): hold the positioned-transfer lock until fclose and freopen close the descriptor --- core/libs/prx/libc/include/FileStream.hpp | 43 +++++++++-- core/libs/prx/libc/src/FileAndHeap.cpp | 16 ++-- core/libs/prx/libkernel/File/include/File.hpp | 14 +++- core/libs/prx/libkernel/File/src/Open.cpp | 20 +++-- core/libs/prx/libkernel/File/src/Stdio.cpp | 4 +- core/libs/tests/GuestStreamClose.cpp | 77 +++++++++++++++++++ 6 files changed, 154 insertions(+), 20 deletions(-) diff --git a/core/libs/prx/libc/include/FileStream.hpp b/core/libs/prx/libc/include/FileStream.hpp index 981f4d7e84..b8afcbf94f 100644 --- a/core/libs/prx/libc/include/FileStream.hpp +++ b/core/libs/prx/libc/include/FileStream.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -34,8 +35,34 @@ extern "C" void ForgetDirectoryStream_nid_no_patch(int descriptor); extern "C" int _IO_switch_to_get_mode(std::FILE* stream); #endif extern "C" int DuplicateHostDescriptor_nid_no_patch(int descriptor); -extern "C" void LibcRegisterClosedDescriptors_nid_no_patch(void (*forget)(int descriptor)); -extern "C" void ForgetClosedDescriptor_nid_no_patch(int descriptor); +extern "C" void LibcRegisterClosedDescriptors_nid_no_patch(void* (*begin)(int descriptor), void (*end)(void* closing)); +extern "C" void* BeginClosingDescriptor_nid_no_patch(int descriptor); +extern "C" void EndClosingDescriptor_nid_no_patch(void* closing); + +class ClosingDescriptor { +public: + ClosingDescriptor([[maybe_unused]] std::FILE* stream, int descriptor) +#ifdef _WIN32 + : _stream(stream), _closing(BeginClosingDescriptor_nid_no_patch(descriptor)) {} +#else + : _closing(BeginClosingDescriptor_nid_no_patch(descriptor)) {} +#endif + ~ClosingDescriptor() { EndClosingDescriptor_nid_no_patch(_closing); } + ClosingDescriptor(const ClosingDescriptor&) = delete; + ClosingDescriptor& operator=(const ClosingDescriptor&) = delete; + +private: +#ifdef _WIN32 + struct LockedStream { + explicit LockedStream(std::FILE* value) : stream(value) { ::_lock_file(stream); } + ~LockedStream() { ::_unlock_file(stream); } + LockedStream(const LockedStream&) = delete; + LockedStream& operator=(const LockedStream&) = delete; + std::FILE* stream; + } _stream; +#endif + void* _closing; +}; struct GuestFilePrefix { unsigned char* position = nullptr; @@ -160,11 +187,12 @@ public: if (std::exchange(directory, false)) ForgetDirectoryStream_nid_no_patch(replaced); auto* const standard = HostStandardHandle_nid_no_patch(replaced); GuestFileLockForget_nid_no_patch(replaced); - ForgetClosedDescriptor_nid_no_patch(replaced); + std::optional closing(std::in_place, previous, replaced); if (open && !filename.empty()) { if (!IsHostStandard()) { _handle = nullptr; std::fclose(previous); + closing.reset(); _handle = open(filename, mode); } else { std::wstring resetMode; @@ -204,7 +232,7 @@ public: KeepStandardHandle_nid_no_patch(replaced, standard); #else if (::fileno(previous) == 2) KeepHostStandardError_nid_no_patch(); - ForgetClosedDescriptor_nid_no_patch(::fileno(previous)); + const ClosingDescriptor closing(previous, ::fileno(previous)); const int kept = IsHostStandard() ? DuplicateHostDescriptor_nid_no_patch(::fileno(previous)) : -1; if (kept >= 0) ::flockfile(previous); _handle = std::freopen(filename.c_str(), mode, previous); @@ -306,11 +334,12 @@ public: #ifdef _WIN32 GuestFileLockForget_nid_no_patch(::_fileno(_handle)); if (std::exchange(directory, false)) ForgetDirectoryStream_nid_no_patch(::_fileno(_handle)); - ForgetClosedDescriptor_nid_no_patch(::_fileno(_handle)); + const ClosingDescriptor closing(_handle, ::_fileno(_handle)); #else - ForgetClosedDescriptor_nid_no_patch(::fileno(_handle)); + const ClosingDescriptor closing(_handle, ::fileno(_handle)); #endif - const int result = std::fclose(std::exchange(_handle, nullptr)); + const int result = std::fclose(_handle); + _handle = nullptr; _guest.flags = 0; _guest.descriptor = -1; return result; diff --git a/core/libs/prx/libc/src/FileAndHeap.cpp b/core/libs/prx/libc/src/FileAndHeap.cpp index f82043ef28..17a5ca64a7 100644 --- a/core/libs/prx/libc/src/FileAndHeap.cpp +++ b/core/libs/prx/libc/src/FileAndHeap.cpp @@ -336,14 +336,20 @@ static int OpenFailure(const std::filesystem::path& path, bool writes, int hostE return writes && error == 13 && std::filesystem::is_directory(path, status) ? 21 : error; } -static void (*g_forgetClosedDescriptor)(int) = nullptr; +static void* (*g_beginClosingDescriptor)(int) = nullptr; +static void (*g_endClosingDescriptor)(void*) = nullptr; -void LibcRegisterClosedDescriptors_nid_no_patch(void (*forget)(int)) { - g_forgetClosedDescriptor = forget; +void LibcRegisterClosedDescriptors_nid_no_patch(void* (*begin)(int), void (*end)(void*)) { + g_beginClosingDescriptor = begin; + g_endClosingDescriptor = end; } -void ForgetClosedDescriptor_nid_no_patch(int descriptor) { - if (g_forgetClosedDescriptor) g_forgetClosedDescriptor(descriptor); +void* BeginClosingDescriptor_nid_no_patch(int descriptor) { + return g_beginClosingDescriptor ? g_beginClosingDescriptor(descriptor) : nullptr; +} + +void EndClosingDescriptor_nid_no_patch(void* closing) { + if (g_endClosingDescriptor) g_endClosingDescriptor(closing); } #ifdef _WIN32 diff --git a/core/libs/prx/libkernel/File/include/File.hpp b/core/libs/prx/libkernel/File/include/File.hpp index 566f04a3de..18b1071dd1 100644 --- a/core/libs/prx/libkernel/File/include/File.hpp +++ b/core/libs/prx/libkernel/File/include/File.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -34,7 +35,18 @@ std::optional WindowsDescriptorPath_nid_no_patch(int desc #endif int HostStandardError(); void ForgetClosedDescriptor(int descriptor); -[[nodiscard]] std::unique_lock LockPositionedTransfer(int descriptor); +class PositionedTransferLock { +public: + explicit PositionedTransferLock(std::shared_ptr mutex) : _mutex(std::move(mutex)), _lock(*_mutex) {} + PositionedTransferLock(PositionedTransferLock&&) = default; + PositionedTransferLock& operator=(PositionedTransferLock&&) = delete; + void unlock() { _lock.unlock(); } + +private: + std::shared_ptr _mutex; + std::unique_lock _lock; +}; +[[nodiscard]] PositionedTransferLock LockPositionedTransfer(int descriptor); void SharePositionedTransfer(int from, int to); #ifdef _WIN32 std::int64_t WriteAtEnd(int descriptor, const void* buf, std::size_t nbytes); diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp index 0d49ee5581..7157d7d798 100644 --- a/core/libs/prx/libkernel/File/src/Open.cpp +++ b/core/libs/prx/libkernel/File/src/Open.cpp @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include @@ -288,17 +290,25 @@ void File::ForgetClosedDescriptor(int descriptor) { namespace { -void ForgetStreamDescriptor(int descriptor) { +void* BeginClosingStreamDescriptor(int descriptor) { EqueueDescriptorClosed(descriptor); #ifdef _WIN32 - const auto positioned = File::LockPositionedTransfer(descriptor); -#endif + auto positioned = std::make_unique(File::LockPositionedTransfer(descriptor)); File::ForgetClosedDescriptor(descriptor); + return positioned.release(); +#else + File::ForgetClosedDescriptor(descriptor); + return nullptr; +#endif +} + +void EndClosingStreamDescriptor(void* closing) { + delete static_cast(closing); } struct ClosedDescriptorRegistration { - ClosedDescriptorRegistration() { LibcRegisterClosedDescriptors_nid_no_patch(&ForgetStreamDescriptor); } - ~ClosedDescriptorRegistration() { LibcRegisterClosedDescriptors_nid_no_patch(nullptr); } + ClosedDescriptorRegistration() { LibcRegisterClosedDescriptors_nid_no_patch(&BeginClosingStreamDescriptor, &EndClosingStreamDescriptor); } + ~ClosedDescriptorRegistration() { LibcRegisterClosedDescriptors_nid_no_patch(nullptr, nullptr); } ClosedDescriptorRegistration(const ClosedDescriptorRegistration&) = delete; ClosedDescriptorRegistration& operator=(const ClosedDescriptorRegistration&) = delete; }; diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 60e1394e50..7d4ccba768 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -245,7 +245,7 @@ struct PositionedLock { }; std::mutex g_positionedTableLock; std::unordered_map> g_positionedLocks; -std::unique_lock File::LockPositionedTransfer(int descriptor) { +File::PositionedTransferLock File::LockPositionedTransfer(int descriptor) { std::shared_ptr entry; { std::lock_guard table(g_positionedTableLock); @@ -253,7 +253,7 @@ std::unique_lock File::LockPositionedTransfer(int descriptor) { if (!slot) slot = std::make_shared(); entry = slot; } - return std::unique_lock(entry->mutex); + return PositionedTransferLock(std::shared_ptr(entry, &entry->mutex)); } void File::SharePositionedTransfer(int from, int to) { std::lock_guard table(g_positionedTableLock); diff --git a/core/libs/tests/GuestStreamClose.cpp b/core/libs/tests/GuestStreamClose.cpp index 505bb3f960..4b2119fe69 100644 --- a/core/libs/tests/GuestStreamClose.cpp +++ b/core/libs/tests/GuestStreamClose.cpp @@ -1,4 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include +#include #include #include #include @@ -6,6 +8,8 @@ #include #include #include +#include +#include class FileStream; @@ -21,6 +25,8 @@ FileStream* APS5_VABI fdopen_nid_postfix(int, const char*); FileStream* APS5_VABI freopen_nid_postfix(const char*, const char*, FileStream*); int APS5_VABI fclose_nid_postfix(FileStream*); int APS5_VABI fileno_nid_postfix(FileStream*); +void APS5_VABI flockfile_nid_postfix(FileStream*); +void APS5_VABI funlockfile_nid_postfix(FileStream*); } static void Check(bool condition, int line) { @@ -33,7 +39,10 @@ static void Check(bool condition, int line) { static constexpr const char* Appended = "guest_stream_close_appended.txt"; static constexpr const char* Reused = "guest_stream_close_reused.txt"; +static constexpr const char* Racing = "guest_stream_close_racing.bin"; static constexpr int ReadWrite = 0x0002; +static constexpr int CreateReadWrite = 0x0202; +static constexpr std::int64_t BadDescriptor = static_cast(0x80020009u); static constexpr int Append = 0x0008; static constexpr int CreateWriteAppend = 0x0209; static constexpr int Truncate = 0x0400; @@ -59,6 +68,69 @@ static FileStream* Reopen(int closed, const char* mode) { return stream; } +static void CloseWaitsForPositionedWrites() { + const std::vector data(std::size_t{8} << 20, 'p'); + for (int round = 0; round < 16; ++round) { + const int descriptor = sceKernelOpen(Racing, CreateReadWrite | Truncate, 0644); + Require(descriptor >= 0); + auto* stream = fdopen_nid_postfix(descriptor, "r+"); + Require(stream != nullptr); + std::atomic started = false; + std::int64_t written = 0; + std::thread writer([&] { + started = true; + written = sceKernelPwrite(descriptor, data.data(), data.size(), 0); + }); + while (!started) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(round % 4)); + if (round % 2 != 0) stream = freopen_nid_postfix(Reused, "r", stream); + Require(stream != nullptr && fclose_nid_postfix(stream) == 0); + writer.join(); + Require(written == static_cast(data.size()) || written == BadDescriptor); + } + std::remove(Racing); +} + +static void CloseWaitsForALockedStream() { + const int descriptor = sceKernelOpen(Racing, CreateReadWrite | Truncate, 0644); + Require(descriptor >= 0); + auto* stream = fdopen_nid_postfix(descriptor, "r+"); + Require(stream != nullptr); + std::atomic locked = false; + std::int64_t position = -1; + std::thread holder([&] { + flockfile_nid_postfix(stream); + locked = true; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + position = sceKernelLseek(descriptor, 0, FromStart); + funlockfile_nid_postfix(stream); + }); + while (!locked) std::this_thread::yield(); + Require(fclose_nid_postfix(stream) == 0); + holder.join(); + Require(position == 0); + std::remove(Racing); +} + +static void ReopenWhileStreamsAreCreated() { + std::atomic done = false; + std::thread opener([&] { + while (!done) { + auto* other = fopen_nid_postfix(Appended, "r"); + Require(other != nullptr && fclose_nid_postfix(other) == 0); + } + }); + auto* stream = fopen_nid_postfix(Reused, "r"); + Require(stream != nullptr); + for (int round = 0; round < 500; ++round) { + stream = freopen_nid_postfix(Reused, "r", stream); + Require(stream != nullptr); + } + done = true; + opener.join(); + Require(fclose_nid_postfix(stream) == 0); +} + int main() { Store(Reused, "0123456789"); const int appending = sceKernelOpen(Appended, CreateWriteAppend | Truncate | CloseOnExec, 0644); @@ -103,6 +175,11 @@ int main() { Require(fclose_nid_postfix(stream) == 0); Require(Contents(Reused) == "01ABCD67EF"); + CloseWaitsForPositionedWrites(); + Require(Contents(Reused) == "01ABCD67EF"); + CloseWaitsForALockedStream(); + ReopenWhileStreamsAreCreated(); + std::remove(Appended); std::remove(Reused); } From 833b760148036522616372a9017a8b94429224a7 Mon Sep 17 00:00:00 2001 From: Evrim Cicek Date: Sun, 25 Oct 2026 21:55:20 +0300 Subject: [PATCH 147/318] test(libc): declare the stream close test exports through the public headers --- core/libs/tests/GuestStreamClose.cpp | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/core/libs/tests/GuestStreamClose.cpp b/core/libs/tests/GuestStreamClose.cpp index 4b2119fe69..62d1ecbd7e 100644 --- a/core/libs/tests/GuestStreamClose.cpp +++ b/core/libs/tests/GuestStreamClose.cpp @@ -1,3 +1,7 @@ +#include "prx/libkernel/include/Api.hpp" +#include "prx/libkernel/File/include/File.hpp" +#include "prx/libc/include/Api.hpp" +#include "prx/libc/include/FileStream.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include #include @@ -11,24 +15,6 @@ #include #include -class FileStream; - -extern "C" { -int APS5_VABI sceKernelOpen(const char*, int, std::uint16_t); -std::int64_t APS5_VABI sceKernelRead(int, void*, std::size_t); -std::int64_t APS5_VABI sceKernelWrite(int, const void*, std::size_t); -std::int64_t APS5_VABI sceKernelPwrite(int, const void*, std::size_t, std::int64_t); -std::int64_t APS5_VABI sceKernelLseek(int, std::int64_t, int); -int APS5_VABI fcntl_nid_postfix(int, int, ...); -FileStream* APS5_VABI fopen_nid_postfix(const char*, const char*); -FileStream* APS5_VABI fdopen_nid_postfix(int, const char*); -FileStream* APS5_VABI freopen_nid_postfix(const char*, const char*, FileStream*); -int APS5_VABI fclose_nid_postfix(FileStream*); -int APS5_VABI fileno_nid_postfix(FileStream*); -void APS5_VABI flockfile_nid_postfix(FileStream*); -void APS5_VABI funlockfile_nid_postfix(FileStream*); -} - static void Check(bool condition, int line) { if (!condition) { std::fprintf(stderr, "stream close check failed at line %d\n", line); From 92f4276f28e9643fe7e336e1cc40ee02f30277da Mon Sep 17 00:00:00 2001 From: Evrim Cicek Date: Mon, 26 Oct 2026 16:37:52 +0300 Subject: [PATCH 148/318] test(libc): cover a directory descriptor that fclose closes --- core/libs/tests/GuestKernelAppArchive.cpp | 16 ++++++++++++++++ core/libs/tests/GuestStreamClose.cpp | 12 ++++++++++++ 2 files changed, 28 insertions(+) diff --git a/core/libs/tests/GuestKernelAppArchive.cpp b/core/libs/tests/GuestKernelAppArchive.cpp index d3922949f5..c03d519fb3 100644 --- a/core/libs/tests/GuestKernelAppArchive.cpp +++ b/core/libs/tests/GuestKernelAppArchive.cpp @@ -317,6 +317,21 @@ void Directories() { Check(!DescriptorIsOpen_nid_no_patch(owned)); } +void DirectoryClosedThroughStream(const std::filesystem::path& root) { + std::ofstream(root / "app0" / "StreamClose.txt", std::ios::binary) << "abc"; + const int directory = sceKernelOpen("/app0/sce_sys/Concurrent", SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0); + Check(directory >= 0); + auto* stream = fdopen_nid_postfix(directory, "r"); + Check(stream != nullptr); + Check(fclose_nid_postfix(stream) == 0); + const int file = sceKernelOpen("/app0/StreamClose.txt", SCE_KERNEL_O_RDONLY, 0); + Check(file == directory); + char data[4]{}; + Check(sceKernelRead(file, data, sizeof(data)) == 3); + Check(sceKernelClose(file) == 0); + Check(std::filesystem::remove(root / "app0" / "StreamClose.txt")); +} + void Reads() { const auto expected = AppArchiveFixture::Pattern(BigSize); const int fd = sceKernelOpen("/app0/data/big.BIN", SCE_KERNEL_O_RDONLY, 0); @@ -496,6 +511,7 @@ int main(int argc, char** argv) { Archive(root); FileNumbers(); ConcurrentArchiveDirectoryEntries(); + DirectoryClosedThroughStream(root); Reads(); Status(); Refusals(); diff --git a/core/libs/tests/GuestStreamClose.cpp b/core/libs/tests/GuestStreamClose.cpp index 62d1ecbd7e..bfd1f21311 100644 --- a/core/libs/tests/GuestStreamClose.cpp +++ b/core/libs/tests/GuestStreamClose.cpp @@ -33,6 +33,7 @@ static constexpr int Append = 0x0008; static constexpr int CreateWriteAppend = 0x0209; static constexpr int Truncate = 0x0400; static constexpr int CloseOnExec = 0x00100000; +static constexpr int Directory = 0x00020000; static constexpr int GetDescriptorFlags = 1; static constexpr int GetStatusFlags = 3; static constexpr int FromStart = 0; @@ -148,6 +149,17 @@ int main() { Require(std::string(read, 10) == "01ABCD6789"); Require(fclose_nid_postfix(stream) == 0); + const int directory = sceKernelOpen(".", Directory, 0); + Require(directory >= 0); + stream = fdopen_nid_postfix(directory, "r"); + Require(stream != nullptr); + Require(fclose_nid_postfix(stream) == 0); + + stream = Reopen(directory, "r"); + Require(sceKernelRead(directory, read, sizeof(read)) == 10); + Require(std::string(read, 10) == "01ABCD6789"); + Require(fclose_nid_postfix(stream) == 0); + const int replaced = sceKernelOpen(Appended, CreateWriteAppend | CloseOnExec, 0644); Require(replaced >= 0); stream = fdopen_nid_postfix(replaced, "a"); From e488e5b8f30f69dec196deb62d87445e508dc4b3 Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:58:08 -0700 Subject: [PATCH 149/318] test: stop requiring the five unimplemented network and kernel calls to throw The tests checked the exact "not implemented" message from sceKernelInternalMemoryGetAvailableSize, mlockall, clock_settime, sceNetCtlGetInfoV6 on an IPv6 host and sceRudpActivate. A test that requires a placeholder to throw fails as soon as the call is implemented, so those checks go. The calls still report themselves as not implemented, as their debt entries say, and the argument checks stay: a null size pointer, invalid mlockall flags, an invalid clock_settime time, and a bad RUDP context or address. --- core/libs/tests/GuestFreebsdClocks.cpp | 13 ------------- core/libs/tests/GuestMemory.cpp | 9 --------- core/libs/tests/GuestMlock.cpp | 14 -------------- core/libs/tests/GuestNetCtl.cpp | 9 --------- core/libs/tests/GuestRudpContextLifecycle.cpp | 10 ---------- 5 files changed, 55 deletions(-) diff --git a/core/libs/tests/GuestFreebsdClocks.cpp b/core/libs/tests/GuestFreebsdClocks.cpp index f5c79ab62b..5425b7b433 100644 --- a/core/libs/tests/GuestFreebsdClocks.cpp +++ b/core/libs/tests/GuestFreebsdClocks.cpp @@ -232,16 +232,6 @@ static void RequireSettimeError(int clockId, const KernelTimespec* time, int err Require(clock_settime_nid_postfix(clockId, time) == -1 && *__error_nid_postfix() == error); } -static bool SettimeNotImplemented(const KernelTimespec& time) { - try { - clock_settime_nid_postfix(GUEST_CLOCK_REALTIME, &time); - - } catch (const std::runtime_error& error) { - return std::strcmp(error.what(), "clock_settime not implemented") == 0; - } - return false; -} - static void SettimeValidatesLikeFreebsd() { const KernelTimespec valid{1767225600, 5}; RequireSettimeError(GUEST_CLOCK_REALTIME, nullptr, 14); @@ -256,9 +246,6 @@ static void SettimeValidatesLikeFreebsd() { RequireSettimeError(GUEST_CLOCK_REALTIME, &wholeSecondOfNanoseconds, 22); RequireSettimeError(GUEST_CLOCK_REALTIME, &negativeSeconds, 22); RequireSettimeError(GUEST_CLOCK_MONOTONIC, &negativeSeconds, 22); - Require(SettimeNotImplemented(valid)); - Require(SettimeNotImplemented(KernelTimespec{0, 0})); - Require(SettimeNotImplemented(KernelTimespec{1, NANOS_PER_SECOND - 1})); } static std::int32_t RuntimeDaylight(std::int32_t daylight) { diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 91d8a12ffa..fd3545bf19 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -131,15 +131,6 @@ static void CheckPageTableStats() { } static void CheckInternalMemoryAvailableSize() { - std::size_t available = 1; - bool threw = false; - try { - sceKernelInternalMemoryGetAvailableSize(&available); - - } catch (const std::runtime_error& error) { - threw = std::string(error.what()) == "sceKernelInternalMemoryGetAvailableSize not implemented"; - } - Require(threw && available == 1); Require(sceKernelInternalMemoryGetAvailableSize(nullptr) == -1); } diff --git a/core/libs/tests/GuestMlock.cpp b/core/libs/tests/GuestMlock.cpp index 1327f4030f..2bc476a6e1 100644 --- a/core/libs/tests/GuestMlock.cpp +++ b/core/libs/tests/GuestMlock.cpp @@ -24,17 +24,6 @@ static void RequireLockAllInvalid(int flags) { Require(mlockall_nid_postfix(flags) == -1 && *__error_nid_postfix() == 22); } -static void RequireLockAllUnsupported(int flags) { - bool rejected = false; - try { - mlockall_nid_postfix(flags); - - } catch (const std::runtime_error& error) { - rejected = std::string(error.what()) == "mlockall not implemented"; - } - Require(rejected); -} - #if defined(__linux__) && !defined(ANYPS5_ENABLE_ASAN) #define HAS_LOCKED_KILOBYTES static std::size_t LockedKilobytes() { @@ -146,9 +135,6 @@ int main() { RequireLockAllInvalid(4); RequireLockAllInvalid(1 | 4); RequireLockAllInvalid(-1); - RequireLockAllUnsupported(1); - RequireLockAllUnsupported(2); - RequireLockAllUnsupported(1 | 2); *__error_nid_postfix() = untouched; Require(munlockall_nid_postfix() == 0); diff --git a/core/libs/tests/GuestNetCtl.cpp b/core/libs/tests/GuestNetCtl.cpp index 129b09979d..9a0e595f2b 100644 --- a/core/libs/tests/GuestNetCtl.cpp +++ b/core/libs/tests/GuestNetCtl.cpp @@ -301,15 +301,6 @@ int main() { Require(sceNetCtlGetInfoV6(6, reinterpret_cast(infoV6)) == kErrInvalidCode); Require(sceNetCtlGetInfoV6(1, nullptr) == kErrInvalidAddr); if (stateV6 != kStateIpObtained) Require(sceNetCtlGetInfoV6(1, reinterpret_cast(infoV6)) == kErrNotConnected); - if (stateV6 == kStateIpObtained) { - bool notImplemented = false; - try { - sceNetCtlGetInfoV6(1, reinterpret_cast(infoV6)); - } catch (const std::runtime_error& error) { - notImplemented = std::strcmp(error.what(), "sceNetCtlGetInfoV6 not implemented") == 0; - } - Require(notImplemented); - } const int result = sceNetCtlGetInfo(14, &info); if (result == kErrNotConnected) { diff --git a/core/libs/tests/GuestRudpContextLifecycle.cpp b/core/libs/tests/GuestRudpContextLifecycle.cpp index 362e628fbb..f3a78d6952 100644 --- a/core/libs/tests/GuestRudpContextLifecycle.cpp +++ b/core/libs/tests/GuestRudpContextLifecycle.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include extern "C" { @@ -163,15 +162,6 @@ int main() { Require(sceRudpActivate(ctx, nullptr, 0) == static_cast(0x80770004)); unsigned char address[34]{}; Require(sceRudpActivate(-1, address, sizeof(address)) == static_cast(0x80770004)); - bool activationUnsupported = false; - try { - sceRudpActivate(ctx, address, sizeof(address)); - - } catch (const std::runtime_error& error) { - activationUnsupported = std::string(error.what()) == "sceRudpActivate not implemented"; - } - Require(activationUnsupported); - Require(sceRudpBind(ctx, 0, 0, 0) == 0); Require(sceRudpSetOption(ctx, 0, nullptr, 0) == invalidOption); From ed9c3f6bf27da541fc3f5fbf6010298be0bb0f32 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:03:04 +0000 Subject: [PATCH 150/318] docs(build): note that the EOP frame capture test is Windows only --- docs/dev/BUILD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 377205720b..7a1c4a5f00 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -204,7 +204,7 @@ cmake --build build --target agc_frame_replay build/core/libs/prx/libSceAgcDriver/agc_frame_replay capture ``` -On Windows use `agc_frame_replay.exe` and prepend `build/core/libs/libs/unpatched` to the process `PATH`. With `BUILD_TESTING=ON`, `agc_frame_capture` checks the file format (round trip, zero-block elision, truncated and corrupt files), and `agc_driver_frame_capture_eop` reserves and submits six EOP flips with `APS5_FRAME_CAPTURE_FRAMES=3` and checks that the capture starts after the first flipping submission and holds three frames of one reservation and one submission each, then that a flipping submission on another thread leaves this thread's flip count unchanged, and `agc_driver_frame_replay` captures two frames of CPU writes followed by DMA copies, the second from a block committed during the capture, replays them in `agc_frame_replay` and compares the copies. +On Windows use `agc_frame_replay.exe` and prepend `build/core/libs/libs/unpatched` to the process `PATH`. With `BUILD_TESTING=ON`, `agc_frame_capture` checks the file format (round trip, zero-block elision, truncated and corrupt files), and, on Windows only, `agc_driver_frame_capture_eop` reserves and submits six EOP flips with `APS5_FRAME_CAPTURE_FRAMES=3` and checks that the capture starts after the first flipping submission and holds three frames of one reservation and one submission each, then that a flipping submission on another thread leaves this thread's flip count unchanged, and `agc_driver_frame_replay` captures two frames of CPU writes followed by DMA copies, the second from a block committed during the capture, replays them in `agc_frame_replay` and compares the copies. ## Pipeline statistics From 28a0aaf5de6e9999a5a8a5424c205e9d87f0ee4a Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:03:04 +0000 Subject: [PATCH 151/318] docs(build): say when the frame replay test is registered --- docs/dev/BUILD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 7a1c4a5f00..f803063a60 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -204,7 +204,7 @@ cmake --build build --target agc_frame_replay build/core/libs/prx/libSceAgcDriver/agc_frame_replay capture ``` -On Windows use `agc_frame_replay.exe` and prepend `build/core/libs/libs/unpatched` to the process `PATH`. With `BUILD_TESTING=ON`, `agc_frame_capture` checks the file format (round trip, zero-block elision, truncated and corrupt files), and, on Windows only, `agc_driver_frame_capture_eop` reserves and submits six EOP flips with `APS5_FRAME_CAPTURE_FRAMES=3` and checks that the capture starts after the first flipping submission and holds three frames of one reservation and one submission each, then that a flipping submission on another thread leaves this thread's flip count unchanged, and `agc_driver_frame_replay` captures two frames of CPU writes followed by DMA copies, the second from a block committed during the capture, replays them in `agc_frame_replay` and compares the copies. +On Windows use `agc_frame_replay.exe` and prepend `build/core/libs/libs/unpatched` to the process `PATH`. With `BUILD_TESTING=ON`, `agc_frame_capture` checks the file format (round trip, zero-block elision, truncated and corrupt files), and, on Windows only, `agc_driver_frame_capture_eop` reserves and submits six EOP flips with `APS5_FRAME_CAPTURE_FRAMES=3` and checks that the capture starts after the first flipping submission and holds three frames of one reservation and one submission each, then that a flipping submission on another thread leaves this thread's flip count unchanged, and, on Windows only, `agc_driver_frame_replay` captures two frames of CPU writes followed by DMA copies, the second from a block committed during the capture, replays them in `agc_frame_replay` and compares the copies. ## Pipeline statistics From 92e6167e18fc9671455b5a51003795f85ba3f62a Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:03:04 +0000 Subject: [PATCH 152/318] docs(build): link the RLIMIT_RTTIME note to the AGC driver debt file --- docs/dev/BUILD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index f803063a60..1f537c80f4 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -306,7 +306,7 @@ With `APS5_PROFILE_DRAW` set (any value), the driver prints a profile line on st ## Driver thread priority -Set `APS5_THREAD_PRIORITY=high` to raise the AGC driver's queue workers and the video output presenter above the title's threads. On Linux the threads get nice `APS5_THREAD_NICE` (default -10) through rtkit; on Windows they get `THREAD_PRIORITY_HIGHEST`. `APS5_THREAD_PRIORITY=rt` asks rtkit for real-time priority `APS5_THREAD_RT` (default 10, at most rtkit's `MaxRealtimePriority`) under rtkit's policy, `SCHED_RR` by default; on Windows it sets `THREAD_PRIORITY_TIME_CRITICAL`. rtkit only grants it when the process's hard `RLIMIT_RTTIME` is at most its `RTTimeUSecMax` (200000 us by default), so a hard limit above `APS5_THREAD_RTTIME_US` (default 200000, capped at `RTTimeUSecMax`) is lowered to it and the soft limit to at most half of it. The hard limit cannot be raised again and stays lowered for the whole process even if rtkit then refuses; see [technical debt](TechnicalDebt.md#functional). There is no fallback between the modes: when rtkit or `SetThreadPriority` refuses, a queue worker reports the refusal as a driver failure, which the next submission, wait or shutdown rethrows, and the presenter's start throws it. Each raised thread logs its priority once as `[thread-priority]`. The setting is `off` by default; a value other than `off`, `high` or `rt` is reported the same way as a refusal. +Set `APS5_THREAD_PRIORITY=high` to raise the AGC driver's queue workers and the video output presenter above the title's threads. On Linux the threads get nice `APS5_THREAD_NICE` (default -10) through rtkit; on Windows they get `THREAD_PRIORITY_HIGHEST`. `APS5_THREAD_PRIORITY=rt` asks rtkit for real-time priority `APS5_THREAD_RT` (default 10, at most rtkit's `MaxRealtimePriority`) under rtkit's policy, `SCHED_RR` by default; on Windows it sets `THREAD_PRIORITY_TIME_CRITICAL`. rtkit only grants it when the process's hard `RLIMIT_RTTIME` is at most its `RTTimeUSecMax` (200000 us by default), so a hard limit above `APS5_THREAD_RTTIME_US` (default 200000, capped at `RTTimeUSecMax`) is lowered to it and the soft limit to at most half of it. The hard limit cannot be raised again and stays lowered for the whole process even if rtkit then refuses; see [technical debt](debt/libSceAgcDriver.md#functional). There is no fallback between the modes: when rtkit or `SetThreadPriority` refuses, a queue worker reports the refusal as a driver failure, which the next submission, wait or shutdown rethrows, and the presenter's start throws it. Each raised thread logs its priority once as `[thread-priority]`. The setting is `off` by default; a value other than `off`, `high` or `rt` is reported the same way as a refusal. ## Shader recompiler From c4558d7654c982a8efa1cbd569528f725517957b Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:03:04 +0000 Subject: [PATCH 153/318] docs(build): list every dependency that stays bundled with system dependencies --- docs/dev/BUILD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 1f537c80f4..c3eb0bc7a6 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -132,7 +132,7 @@ Build configuration parameters: SDL and FreeType settings forced by the root `CMakeLists.txt` cannot be overridden with `-D`. -With `ANYPS5_USE_SYSTEM_DEPS=ON` those libraries come from the system (on Debian and Ubuntu: `libsdl2-dev`, `libfreetype-dev`, `libvulkan-dev`, `spirv-headers`, `glslang-dev`, `glslang-tools` and `spirv-tools`). A missing package, Vulkan-Headers older than 1.4.310 (the agc driver uses `VK_KHR_maintenance8`), or an SDL2 without the HIDAPI joystick driver (sdl2-compat over SDL3, or a build with HIDAPI disabled, which breaks the pad and disk-audio tests), fails the configure with an error and never falls back to the submodule. The forced SDL and FreeType settings from the root `CMakeLists.txt` do not apply with the flag ON: the system SDL2 comes with whatever its distribution enabled. The prx libraries are shared objects, so SDL2 is linked from the system as a shared library; the default build keeps its own static SDL2. `ffmpeg-core`, `libjpeg-turbo`, `stb`, `VulkanMemoryAllocator`, `LibAtrac9`, `ogg` and `vorbis` always stay bundled and still need `git submodule update --init`. +With `ANYPS5_USE_SYSTEM_DEPS=ON` those libraries come from the system (on Debian and Ubuntu: `libsdl2-dev`, `libfreetype-dev`, `libvulkan-dev`, `spirv-headers`, `glslang-dev`, `glslang-tools` and `spirv-tools`). A missing package, Vulkan-Headers older than 1.4.310 (the agc driver uses `VK_KHR_maintenance8`), or an SDL2 without the HIDAPI joystick driver (sdl2-compat over SDL3, or a build with HIDAPI disabled, which breaks the pad and disk-audio tests), fails the configure with an error and never falls back to the submodule. The forced SDL and FreeType settings from the root `CMakeLists.txt` do not apply with the flag ON: the system SDL2 comes with whatever its distribution enabled. The prx libraries are shared objects, so SDL2 is linked from the system as a shared library; the default build keeps its own static SDL2. `ffmpeg-core`, `libjpeg-turbo`, `stb`, `VulkanMemoryAllocator`, `LibAtrac9`, `ogg`, `vorbis`, `opus`, `libusb-cmake`, `mbedtls`, `vgmstream`, `zstd`, `ZArchive` and, on Windows, `mimalloc` always stay bundled and still need `git submodule update --init`. ## Quick syntax checks From 776e5c2ea079c601dc3b811bd219917bee085f19 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:03:04 +0000 Subject: [PATCH 154/318] docs(contributing): describe how build jobs merge the selected pull requests --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d96b4d4c11..53579f364e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ For pull requests, full Linux and Windows builds and tests run only when a colla gh workflow run build.yml --repo boykopovar/AnyPS5 --ref main --field pr=1234 ``` -This manual run also builds and tests the relinker on Linux, Windows and macOS when the pull request changes a file the relinker-only build uses, regardless of the author's previous contributions. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main` and it changes a file the relinker-only build uses; full builds never run automatically for pull requests. Every job that runs automatically for pull requests runs only when the pull request changes its files: a `paths` filter on the `pull_request` trigger, or `needs: eligibility` with an `if` on the files output it sets in `build.yml`. The Conventions check fails a workflow change that adds a job without one. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds. +This manual run also builds and tests the relinker on Linux, Windows and macOS when the pull request changes a file the relinker-only build uses, regardless of the author's previous contributions. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main` and it changes a file the relinker-only build uses; full builds never run automatically for pull requests. Every job that runs automatically for pull requests runs only when the pull request changes its files: a `paths` filter on the `pull_request` trigger, or `needs: eligibility` with an `if` on the files output it sets in `build.yml`. The Conventions check fails a workflow change that adds a job without one. All build jobs check out the current `main` and merge the head of each selected pull request into it with `git merge --no-ff`, so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds. With `--field full=false`, the manual run skips the full builds and runs only the relinker matrix on Linux, Windows and macOS, if the pull request changes a file the relinker-only build uses, such as the Python tools and their tests in `tools/`. From 07e7925b24b01e3135c000043fbbf69625270293 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:03:04 +0000 Subject: [PATCH 155/318] docs(contributing): count a merged pull request toward build eligibility --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53579f364e..3ee587b7d7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ For pull requests, full Linux and Windows builds and tests run only when a colla gh workflow run build.yml --repo boykopovar/AnyPS5 --ref main --field pr=1234 ``` -This manual run also builds and tests the relinker on Linux, Windows and macOS when the pull request changes a file the relinker-only build uses, regardless of the author's previous contributions. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main` and it changes a file the relinker-only build uses; full builds never run automatically for pull requests. Every job that runs automatically for pull requests runs only when the pull request changes its files: a `paths` filter on the `pull_request` trigger, or `needs: eligibility` with an `if` on the files output it sets in `build.yml`. The Conventions check fails a workflow change that adds a job without one. All build jobs check out the current `main` and merge the head of each selected pull request into it with `git merge --no-ff`, so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds. +This manual run also builds and tests the relinker on Linux, Windows and macOS when the pull request changes a file the relinker-only build uses, regardless of the author's previous contributions. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit or a merged pull request in `main` and it changes a file the relinker-only build uses; full builds never run automatically for pull requests. Every job that runs automatically for pull requests runs only when the pull request changes its files: a `paths` filter on the `pull_request` trigger, or `needs: eligibility` with an `if` on the files output it sets in `build.yml`. The Conventions check fails a workflow change that adds a job without one. All build jobs check out the current `main` and merge the head of each selected pull request into it with `git merge --no-ff`, so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds. With `--field full=false`, the manual run skips the full builds and runs only the relinker matrix on Linux, Windows and macOS, if the pull request changes a file the relinker-only build uses, such as the Python tools and their tests in `tools/`. From 0cb16de10be4c6aea1af0659b6b4461acc1c0843 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:03:04 +0000 Subject: [PATCH 156/318] docs(agents): list the Fixes line of the pull request template --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 7ded821d28..53162a4f7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ AnyPS5 relinks PS5 executables into native Linux or Windows programs that run ag - New third-party code is a submodule at `3rdparty/`: add its `.gitmodules` entry (`git submodule add 3rdparty/`) and build it from source with CMake. Never `find_package` or copy sources in. - Known gaps go in `docs/dev/debt/.md`; notes, reports and logs go in the pull request. Add no other files. - Commit subjects and PR titles are Conventional Commits. -- PR body: copy [the template](.github/pull_request_template.md), keep every line, tick every box, and fill `What`, `Tested`, `Depends on` (#N or none), `Unblocks` (title ID and discussion, or none), `GPUs tested` (AGC, shader recompiler, VideoOut; otherwise n/a) and `AI-assisted` (yes or no). +- PR body: copy [the template](.github/pull_request_template.md), keep every line, tick every box, and fill `What`, `Tested`, `Depends on` (#N or none), `Fixes` (Fixes #N or none), `Unblocks` (title ID and discussion, or none), `GPUs tested` (AGC, shader recompiler, VideoOut; otherwise n/a) and `AI-assisted` (yes or no). - Before pushing run `python3 tools/check_conventions.py --base origin/main`; before opening run `python3 tools/pr_overlap.py --branch `. - Build and test: [BUILD](docs/dev/BUILD.md); a relinker-only change builds with `-DANYPS5_RELINKER_ONLY=ON`. - A speed claim shows `tools/trace_digest.py` summaries of `main` and of the change from the same machine and scene. From f99e00c5c2516cd7bd8a6e9c117a1ceadd9f58e3 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:03:15 +0000 Subject: [PATCH 157/318] docs(build): list the ANYPS5_RELINKER_CLI switch --- docs/dev/BUILD.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index c3eb0bc7a6..7a667caf86 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -114,6 +114,7 @@ Project switches accept `ON` or `OFF`: | `-DANYPS5_ENABLE_SPIRV_TOOLS=ON` | `OFF` | Enable SPIR-V validation and optimization. | | `-DANYPS5_ENABLE_ASAN=ON` | `OFF` | Build with AddressSanitizer on Linux and move the guest arena out of ASan's shadow gap. | | `-DANYPS5_USE_SYSTEM_DEPS=ON` | `OFF` | Resolve SDL2, freetype, Vulkan-Headers, SPIRV-Headers, glslang and SPIRV-Tools from the system instead of the submodules. | +| `-DANYPS5_RELINKER_CLI=OFF` | `ON` | Skip the native relinker command line (`ANYPS5_BUILD_WASM` turns it off). | | `-DAPS5_ENABLE_TIMING_LOG=ON` | `OFF` | Compile frame timing logging. | | `-DAPS5_ENABLE_TRACY=ON` | `OFF` | Build the [Tracy](#tracy) profiler client into the agc driver. | | `-DAPS5_AGC_CREATE_LOG=OFF` | `ON` | Disable successful `sceAgcCreateShader` logging. | From f636ecd340ddbce94e26a11030ccf25acbb4b9ec Mon Sep 17 00:00:00 2001 From: Shahul Mubeen A Date: Mon, 26 Oct 2026 19:58:19 +0530 Subject: [PATCH 158/318] test(agc): drop the unnormalized clamp mode 4, 5 and 7 refusal loops Modes 4, 5 and 7 are valid S# values that are not supported yet, so the sampler decode test should not require them to throw. The refusal stays in DecodeSamplerResource. Co-Authored-By: Claude Opus 5.5 --- .../prx/libSceAgcDriver/tests/GuestSamplerResource.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp index fda1ead4ba..f5c81190c6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp @@ -428,16 +428,6 @@ void RunGuestSamplerResourceTests() { unnormalizedAniso.xyMagFilter = 2; unnormalizedAniso.xyMinFilter = 2; rejectUnnormalized(unnormalizedAniso, "unnormalized coordinates with anisotropic filtering"); - for (const std::uint32_t clamp : {4u, 5u, 7u}) { - Fields clampX = unnormalizedBase; - clampX.clampX = clamp; - rejectUnnormalized(clampX, "unnormalized coordinates with clamp mode " + std::to_string(clamp) + " on X"); - } - for (const std::uint32_t clamp : {4u, 5u, 7u}) { - Fields clampY = unnormalizedBase; - clampY.clampY = clamp; - rejectUnnormalized(clampY, "unnormalized coordinates with clamp mode " + std::to_string(clamp) + " on Y"); - } Fields clampZero = unnormalizedBase; clampZero.clampX = 0; clampZero.clampY = 0; From 098fc2772588f669f7426a5a26b3681d6867b312 Mon Sep 17 00:00:00 2001 From: Shahul Mubeen A Date: Mon, 26 Oct 2026 20:03:23 +0530 Subject: [PATCH 159/318] docs(agc): keep the unnormalized S# debt entry under 1000 characters Co-Authored-By: Claude Opus 5.5 --- docs/dev/debt/libSceAgcDriver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 63cc2f56ee..79febf6773 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -136,7 +136,7 @@ - [Sparse dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a shared-backed range larger than one 64 MiB udmabuf buffer is imported as one sparse buffer bound to the 64 MiB buffers, on a queue of its own when the device has a second family that binds sparse memory. The path is decided before anything is imported, and a failure after that throws. A range whose offset in the backing file is not a multiple of 64 KiB starts its sparse buffer at the 64 KiB block below it (`HostImport::lead`). A range takes the single udmabuf import instead, which udmabuf's `size_limit_mb` refuses above 64 MiB by default so the range is copied, when the buffer's sparse block size is larger than 64 KiB (it is 64 KiB on RADV), when the sparse buffer does not accept every memory type a chunk import can have, when the backing file ends inside its last chunk, or when it is imported at map time on a device with no second family (the bind then needs the GPU mutex); the NVIDIA proprietary driver imports no dma-buf memory - [Presentation under the GPU mutex](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) - on a device whose presentation family has no queue left after the main and settle queues, such as RADV, lavapipe and Intel's Windows driver (Arc Graphics on Meteor Lake, 101.9150), which expose a single graphics queue, `vkQueuePresentKHR` still runs on the main queue under `GuestMemory::GpuMutex`, so the draw committer and the queue workers wait for as long as the present blocks. How long that is on those drivers was not measured; on NVIDIA in FIFO mode it is about 6 ms per present. A device with a free queue in that family presents on it under its own mutex, after the GPU mutex is released. Closes when: a trace on a single-queue device shows no `GpuMutex.wait.*` behind `Vulkan.QueuePresent`. - [Unnormalized S# coordinates](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view. -- An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) follows RDNA2 (gfx1036) [hardware oracle](../HW_ORACLE.md) measurements, not gfx1013: clamp mode 0 (repeat) samples like clamp-to-last-texel, and modes 1 (mirror) and 3 (mirror-once) mirror texel index i < 0 to -i - 1 after the texel offset, then clamp to the last texel. Modes 0, 1 and 3 bind clamp-to-edge; for 1 and 3 the shader first replaces a negative coordinate with the float just below its negation, so an exact negative integer -k reads texel k - 1 like every coordinate in [-k, -k + 1), not texel k. Unequal minification and magnification filters sample with the magnification filter (unnormalized coordinates sample at LOD 0). Anisotropy, TRUNC_COORD, MC_COORD_TRUNC and clamp modes 4, 5 and 7 throw. Secondary LOD bias (LOD_BIAS_SEC, bits [19:14]) is accepted and ignored, matching RDNA2 measurements across explicit and implicit samples. High on Life (PPSA15388, #4736) uses secondary LOD bias and clamp mode 0 with differing min/mag filters. +- An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) follows RDNA2 (gfx1036) [hardware oracle](../HW_ORACLE.md) measurements, not gfx1013: clamp mode 0 (repeat) samples like clamp-to-last-texel, and modes 1 (mirror) and 3 (mirror-once) mirror texel index i < 0 to -i - 1 after the texel offset, then clamp. Modes 0, 1 and 3 bind clamp-to-edge; for 1 and 3 the shader first maps a negative coordinate c to the float just below -c, so an exact negative integer -k reads texel k - 1, not k. Unequal minification and magnification filters sample with the magnification filter (unnormalized coordinates sample at LOD 0). Anisotropy, TRUNC_COORD, MC_COORD_TRUNC and clamp modes 4, 5 and 7 throw. Secondary LOD bias (LOD_BIAS_SEC, bits [19:14]) is accepted and ignored, matching RDNA2 measurements across explicit and implicit samples. High on Life (PPSA15388, #4736) uses secondary LOD bias and clamp mode 0 with differing min/mag filters. - [TRUNC_COORD](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) is not applied: the texel a point sample reads and the bilinear weights follow the host sampler's rounding. On RDNA 2 (RX 6700 XT), an S# with TRUNC_COORD 0 rounds the exact scaled coordinate to 8 fractional bits, ties to even, and one with TRUNC_COORD 1 truncates it, for point and bilinear filters. AMD's Windows driver (32.0.21045.5002) truncates for nearest samplers and rounds for linear ones. Under it, a TRUNC_COORD 0 point sample at most 1/512 texel below a texel edge reads the texel below the edge where the console reads the one above, and `k / width` computed in f32 lands there for about half of all k. A TRUNC_COORD 1 bilinear weight can be 1/256 above the console's. Other host drivers are not measured. - [Background link-time-optimized relink of library pipelines](../../../core/libs/prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp) (libSceAgcDriver) is skipped on every GPU under NVIDIA's proprietary Vulkan driver (`VK_DRIVER_ID_NVIDIA_PROPRIETARY`), so those pipelines keep their fast-linked form. The optimized link loses the device when bound on an RTX 3080 (Ampere, #2866) and an RTX 4060 (Ada, #3434); an RTX 5060 and an RTX 5080 (Blackwell) on the same driver do not, so the scope is wider than the evidence. No standard Vulkan query tells the architecture apart, and the driver version that fixes it is unknown. `APS5_NO_PIPELINE_LTO` disables the relink on every driver. - [8_SRGB and 8_8_SRGB textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp) (libSceAgcDriver) that the device samples as R8_SRGB or R8G8_SRGB use its own decode, which need not match gfx1035: NVIDIA's R8_SRGB is one bf16 step off in 29 of 256 codes. A device that cannot sample them gets an RGBA8 sRGB copy instead (a blit of the R8 or RG8 texels, refreshed with the storage image's version), whose decode is likewise the device's; 3D textures and storage images that are not single-depth and format-matched are copied from their snapshot instead. From 5780180b94bb8db3dd9b2627e09dacd547564941 Mon Sep 17 00:00:00 2001 From: blu <131330772+ycbuzi@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:34:12 -0700 Subject: [PATCH 160/318] fix(libSceNet): return EFAULT from sceNetSendmsg and sceNetSendto for unreadable guest memory Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceNet/CMakeLists.txt | 12 ++ core/libs/prx/libSceNet/Export.cpp | 26 +++-- core/libs/tests/GuestNetSendmsgFaults.cpp | 132 ++++++++++++++++++++++ 3 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 core/libs/tests/GuestNetSendmsgFaults.cpp diff --git a/core/libs/prx/libSceNet/CMakeLists.txt b/core/libs/prx/libSceNet/CMakeLists.txt index 90129eac83..307fd57ac0 100644 --- a/core/libs/prx/libSceNet/CMakeLists.txt +++ b/core/libs/prx/libSceNet/CMakeLists.txt @@ -39,6 +39,18 @@ if(BUILD_TESTING) set_tests_properties(guest_net_buffer_validation PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_net_sendmsg_fault_tests ${LIBS_INCLUDE_DIR}/tests/GuestNetSendmsgFaults.cpp) + target_include_directories(guest_net_sendmsg_fault_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_net_sendmsg_fault_tests PRIVATE libSceNet libkernel libc) + configure_windows_unwind(guest_net_sendmsg_fault_tests) + foreach(fault IN ITEMS payload iovec header name) + add_test(NAME guest_net_sendmsg_fault_${fault} COMMAND guest_net_sendmsg_fault_tests ${fault}) + set_tests_properties(guest_net_sendmsg_fault_${fault} PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_net_sendmsg_fault_${fault} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() foreach(api IN ITEMS send sendto sendmsg recv recvfrom recvmsg duplex blocking) add_test(NAME net_dontwait_${api} COMMAND net_dontwait_tests ${api}) set_tests_properties(net_dontwait_${api} PROPERTIES TIMEOUT 20) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index ad5ab932d3..99a58fc50f 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -1307,9 +1307,12 @@ int64_t APS5_VABI sceNetSendto(int s, const void* buf, size_t len, int flags, co }, true, static_cast(len)); #endif } else { + std::array address{}; + const auto address_length = std::min(tolen, address.size()); + if (address_length && !GuestSocketReadGuest_nid_no_patch(address.data(), to, address_length)) return fail(NET_EFAULT); sockaddr_storage destination{}; NativeLength destination_length = 0; - if (!guest_to_native_address(to, tolen, destination, destination_length)) return fail(NET_EINVAL); + if (!guest_to_native_address(address.data(), address_length, destination, destination_length)) return fail(NET_EINVAL); if (destination.ss_family != (socket.family == NET_AF_INET ? AF_INET : AF_INET6)) return fail(NET_EAFNOSUPPORT); #ifdef __APPLE__ result = send_stream_in_chunks(*socket.native, flags, static_cast(buf), len, [&](const char* data, std::size_t size) { @@ -1331,9 +1334,18 @@ int64_t APS5_VABI sceNetSendto(int s, const void* buf, size_t len, int flags, co } int64_t APS5_VABI sceNetSendmsg(int s, const NetMsghdr* msg, int flags) { - const auto total = message_length(msg); + if (!msg) return fail(NET_EFAULT); + NetMsghdr message{}; + if (!GuestSocketReadGuest_nid_no_patch(&message, msg, sizeof(message))) return fail(NET_EFAULT); + std::vector vectors; + if (message.iov && message.iov_length > 0 && message.iov_length <= NET_UIO_MAXIOV) { + vectors.resize(static_cast(message.iov_length)); + if (!GuestSocketReadGuest_nid_no_patch(vectors.data(), message.iov, vectors.size() * sizeof(NetIovec))) return fail(NET_EFAULT); + message.iov = vectors.data(); + } + const auto total = message_length(&message); if (total < 0) return total; - if (msg->control && msg->control_length) throw std::runtime_error("sceNetSendmsg: control data is not supported"); + if (message.control && message.control_length) throw std::runtime_error("sceNetSendmsg: control data is not supported"); std::vector buffer; try { buffer.resize(static_cast(total)); @@ -1341,11 +1353,11 @@ int64_t APS5_VABI sceNetSendmsg(int s, const NetMsghdr* msg, int flags) { return fail(55); } std::size_t offset = 0; - for (int i = 0; i < msg->iov_length; ++i) { - if (msg->iov[i].length) std::memcpy(buffer.data() + offset, msg->iov[i].base, msg->iov[i].length); - offset += msg->iov[i].length; + for (const auto& entry : vectors) { + if (entry.length && !GuestSocketReadGuest_nid_no_patch(buffer.data() + offset, entry.base, entry.length)) return fail(NET_EFAULT); + offset += entry.length; } - return sceNetSendto(s, buffer.data(), buffer.size(), flags, msg->name, msg->name ? msg->name_length : 0u); + return sceNetSendto(s, buffer.data(), buffer.size(), flags, message.name, message.name ? message.name_length : 0u); } int64_t APS5_VABI sceNetRecvmsg(int s, NetMsghdr* msg, int flags) { diff --git a/core/libs/tests/GuestNetSendmsgFaults.cpp b/core/libs/tests/GuestNetSendmsgFaults.cpp new file mode 100644 index 0000000000..0bd3c4c8be --- /dev/null +++ b/core/libs/tests/GuestNetSendmsgFaults.cpp @@ -0,0 +1,132 @@ +#include "prx/libkernel/include/Api.hpp" +#include "prx/libSceNet/include/Net.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::size_t Page = 0x4000; +constexpr std::int64_t Efault = static_cast(static_cast(0x8041010eu)); + +void Require(bool condition, std::source_location location = std::source_location::current()) { + if (!condition) { + std::fprintf(stderr, "check failed at line %u\n", location.line()); + std::abort(); + } +} + +struct Sockets { + int sender = -1; + int receiver = -1; + std::array destination{16, 2, 0, 0, 127, 0, 0, 1}; +}; + +Sockets Open() { + Sockets sockets; + sockets.receiver = sceNetSocket("sendmsg-faults-receiver", 2, 2, 17); + sockets.sender = sceNetSocket("sendmsg-faults-sender", 2, 2, 17); + Require(sockets.receiver >= 0 && sockets.sender >= 0); + Require(sceNetBind_nid_postfix(sockets.receiver, sockets.destination.data(), sockets.destination.size()) == 0); + std::uint32_t size = sockets.destination.size(); + Require(sceNetGetsockname(sockets.receiver, sockets.destination.data(), &size) == 0 && size == sockets.destination.size()); + return sockets; +} + +void RequireFault(std::int64_t result) { + Require(result == Efault); + Require(*sceNetErrnoLoc() == 14); +} + +void RequireNothingSent(Sockets& sockets) { + const char marker[] = "marker"; + Require(sceNetSendto(sockets.sender, marker, sizeof(marker), 0, sockets.destination.data(), sockets.destination.size()) == sizeof(marker)); + char received[64]{}; + Require(sceNetRecv(sockets.receiver, received, sizeof(received), 0) == sizeof(marker)); + Require(std::memcmp(received, marker, sizeof(marker)) == 0); +} + +void RequireSend(Sockets& sockets, const NetMsghdr* message) { + Require(sceNetSendmsg(sockets.sender, message, 0) == 8); + unsigned char received[64]{}; + Require(sceNetRecv(sockets.receiver, received, sizeof(received), 0) == 8); + for (std::size_t i = 0; i < 8; ++i) Require(received[i] == 0x5a); +} + +} + +int main(int argc, char** argv) { + Require(argc == 2); + const std::string_view mode = argv[1]; + Require(mode == "payload" || mode == "iovec" || mode == "header" || mode == "name"); + Require(sceNetInit_nid_postfix() == 0); + auto sockets = Open(); + + std::int64_t physical = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, Page * 3, 0, 0, &physical) == 0); + void* mapping = nullptr; + Require(sceKernelMapDirectMemory(&mapping, Page * 3, 3, 0, physical, 0) == 0); + auto* bytes = static_cast(mapping); + std::memset(bytes, 0x5a, Page * 3); + auto* first = bytes; + auto* second = bytes + Page; + auto* third = bytes + Page * 2; + + auto* message = reinterpret_cast(first + 0x100); + auto* vector = reinterpret_cast(first + 0x200); + *vector = NetIovec{third, 8}; + *message = NetMsghdr{sockets.destination.data(), sockets.destination.size(), vector, 1, nullptr, 0, 0}; + RequireSend(sockets, message); + + *sceNetErrnoLoc() = 0; + if (mode == "payload") { + *vector = NetIovec{second + Page - 8, 16}; + Require(sceKernelMprotect(third, Page, 0) == 0); + RequireFault(sceNetSendmsg(sockets.sender, message, 0)); + *sceNetErrnoLoc() = 0; + RequireFault(sceNetSendto(sockets.sender, second + Page - 8, 16, 0, sockets.destination.data(), sockets.destination.size())); + Require(sceKernelMprotect(third, Page, 3) == 0); + *vector = NetIovec{third, 8}; + } else if (mode == "iovec") { + auto* crossing = reinterpret_cast(second - sizeof(void*)); + const unsigned char* payload = third; + std::memcpy(crossing, &payload, sizeof(payload)); + const std::uint64_t length = 8; + std::memcpy(second, &length, sizeof(length)); + message->iov = crossing; + Require(sceKernelMprotect(second, Page, 0) == 0); + RequireFault(sceNetSendmsg(sockets.sender, message, 0)); + Require(sceKernelMprotect(second, Page, 3) == 0); + message->iov = vector; + } else if (mode == "header") { + auto* crossing = reinterpret_cast(second - sizeof(void*)); + std::memcpy(crossing, message, sizeof(*message)); + Require(sceKernelMprotect(second, Page, 0) == 0); + RequireFault(sceNetSendmsg(sockets.sender, crossing, 0)); + Require(sceKernelMprotect(second, Page, 3) == 0); + } else { + *vector = NetIovec{first + 0x400, 8}; + auto* name = third; + std::memcpy(name, sockets.destination.data(), sockets.destination.size()); + message->name = name; + Require(sceKernelMprotect(third, Page, 0) == 0); + RequireFault(sceNetSendmsg(sockets.sender, message, 0)); + *sceNetErrnoLoc() = 0; + const char data[8]{}; + RequireFault(sceNetSendto(sockets.sender, data, sizeof(data), 0, name, sockets.destination.size())); + Require(sceKernelMprotect(third, Page, 3) == 0); + message->name = sockets.destination.data(); + } + RequireNothingSent(sockets); + RequireSend(sockets, message); + + Require(sceNetSocketClose(sockets.sender) == 0); + Require(sceNetSocketClose(sockets.receiver) == 0); + std::printf("guest net sendmsg %s fault test passed\n", argv[1]); + return 0; +} From a344e526ecc71c5a1fbf4bbca3a2416d834bd37a Mon Sep 17 00:00:00 2001 From: frankea Date: Mon, 26 Oct 2026 12:24:28 -0400 Subject: [PATCH 161/318] fix(libSceVideoRecordingP): answer parameter checks and calls without a recorder as a PS5 does --- .../libs/prx/libSceVideoRecordingP/Export.cpp | 65 ++++++--- .../prx/libSceVideoRecordingP/include/Api.hpp | 6 +- core/libs/tests/GuestVideoRecording.cpp | 134 +++++++++++++++--- docs/dev/debt/libSceVideoRecordingP.md | 8 +- 4 files changed, 176 insertions(+), 37 deletions(-) diff --git a/core/libs/prx/libSceVideoRecordingP/Export.cpp b/core/libs/prx/libSceVideoRecordingP/Export.cpp index 6166139756..422dbf4344 100644 --- a/core/libs/prx/libSceVideoRecordingP/Export.cpp +++ b/core/libs/prx/libSceVideoRecordingP/Export.cpp @@ -10,9 +10,13 @@ static_assert(offsetof(VideoRecordingQueryParam, option) == 28); static_assert(sizeof(VideoRecordingSupportInfo) == 18); -static_assert(sizeof(VideoRecordingParam) == 32); -static_assert(offsetof(VideoRecordingParam, duration) == 12); -static_assert(offsetof(VideoRecordingParam, format) == 28); +static_assert(sizeof(VideoRecordingParam) == 64); +static_assert(offsetof(VideoRecordingParam, duration) == 8); +static_assert(offsetof(VideoRecordingParam, mode) == 12); +static_assert(offsetof(VideoRecordingParam, affinityMask) == 16); +static_assert(offsetof(VideoRecordingParam, threadPriority) == 24); +static_assert(offsetof(VideoRecordingParam, option) == 32); +static_assert(offsetof(VideoRecordingParam, format) == 36); static_assert(sizeof(VideoRecordingParam2) == 40); static_assert(offsetof(VideoRecordingParam2, threadPriority) == 16); static_assert(offsetof(VideoRecordingParam2, duration) == 20); @@ -20,6 +24,38 @@ static_assert(offsetof(VideoRecordingParam2, format) == 36); namespace { constexpr int SceVideoRecordingStatusNone = 0; +constexpr int ErrorInvalidArgument = static_cast(0x80a80003); +constexpr int ErrorNotOpen = static_cast(0x80a80006); +constexpr int ErrorNoVideoFormat = static_cast(0x80a80008); +constexpr int MinimumHeapSize = 0x1000; +constexpr std::uint64_t MinimumParamSize = 0x10; +constexpr std::uint32_t MaximumDuration = 3600; + +bool IsVideoFormat(std::uint32_t format) { + switch (format) { + case 0x1: + case 0x2: + case 0x10: + case 0x11: + case 0x28: + case 0x40: + case 0x41: + return true; + default: + return false; + } +} + +int CheckParam(const VideoRecordingParam* param) { + if (param == nullptr || param->size < MinimumParamSize) return ErrorInvalidArgument; + if (param->duration == 0 || param->duration > MaximumDuration || param->mode > 1) return ErrorInvalidArgument; + const std::uint32_t option = param->size > offsetof(VideoRecordingParam, option) ? param->option : 0; + const std::uint32_t format = param->size > offsetof(VideoRecordingParam, format) ? param->format : 0; + if (option == 0 && format == 0) return MinimumHeapSize; + if (option > 1 || (format != 0 && !IsVideoFormat(format))) return ErrorInvalidArgument; + if (format == 0) return ErrorNoVideoFormat; + return 0; +} } extern "C" { @@ -31,8 +67,7 @@ void APS5_VABI _sceVideoRecordingQueryParam_nid_postfix(VideoRecordingQueryParam int APS5_VABI sceVideoRecordingClose(int discard) { (void)discard; - NotImplemented_nid_no_patch(__func__); - return 0; + return ErrorNotOpen; } int APS5_VABI sceVideoRecordingCloseByHandle(VideoRecordingContext* handle, int discard) { @@ -102,10 +137,10 @@ int APS5_VABI sceVideoRecordingInitialize(void* param) { } int APS5_VABI sceVideoRecordingOpen(const char* path, const VideoRecordingParam* param, void* heap, int heapSize) { - (void)path; - (void)param; - (void)heap; - (void)heapSize; + if (path == nullptr || param == nullptr || heap == nullptr || heapSize < 0) return ErrorInvalidArgument; + const int result = CheckParam(param); + if (result < 0) return result; + if (result == MinimumHeapSize && heapSize < MinimumHeapSize) return ErrorInvalidArgument; NotImplemented_nid_no_patch(__func__); return 0; } @@ -126,9 +161,9 @@ std::int64_t APS5_VABI sceVideoRecordingQueryHddSize2(const VideoRecordingParam2 } int APS5_VABI sceVideoRecordingQueryMemSize(const VideoRecordingParam* param) { - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + const int result = CheckParam(param); + if (result == 0) NotImplemented_nid_no_patch("sceVideoRecordingQueryMemSize (heap size for a video format)"); + return result; } int APS5_VABI sceVideoRecordingQueryMemSize2(const VideoRecordingParam2* param) { @@ -170,13 +205,11 @@ int APS5_VABI sceVideoRecordingSetInfo(std::int32_t info, const void* data, std: } int APS5_VABI sceVideoRecordingStart(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return ErrorNotOpen; } int APS5_VABI sceVideoRecordingStop(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return ErrorNotOpen; } int APS5_VABI sceVideoRecordingTerm(void) { diff --git a/core/libs/prx/libSceVideoRecordingP/include/Api.hpp b/core/libs/prx/libSceVideoRecordingP/include/Api.hpp index 2f831a0c7e..c98b1a46b4 100644 --- a/core/libs/prx/libSceVideoRecordingP/include/Api.hpp +++ b/core/libs/prx/libSceVideoRecordingP/include/Api.hpp @@ -6,12 +6,16 @@ struct VideoRecordingContext; struct VideoRecordingParam { std::uint64_t size; - std::int32_t threadPriority; std::uint32_t duration; std::uint32_t mode; + std::uint64_t affinityMask; + std::int32_t threadPriority; std::int32_t outputMode; std::uint32_t option; std::uint32_t format; + std::uint32_t reserved40; + std::uint32_t reserved44; + std::uint8_t reserved48[16]; }; struct VideoRecordingParam2 { diff --git a/core/libs/tests/GuestVideoRecording.cpp b/core/libs/tests/GuestVideoRecording.cpp index ff7b12233e..1ae1d378e6 100644 --- a/core/libs/tests/GuestVideoRecording.cpp +++ b/core/libs/tests/GuestVideoRecording.cpp @@ -10,6 +10,10 @@ namespace { constexpr int StatusNone = 0; +constexpr int InvalidArgument = static_cast(0x80a80003); +constexpr int NotOpen = static_cast(0x80a80006); +constexpr int NoVideoFormat = static_cast(0x80a80008); +constexpr int MinimumHeap = 0x1000; constexpr std::uint8_t Sentinel = 0xA5; void Require(bool condition, const char* message) { @@ -29,33 +33,129 @@ bool Throws(TCall call) { return false; } +VideoRecordingParam RecordingParam(std::uint32_t format) { + VideoRecordingParam param{}; + param.size = sizeof(param); + param.duration = 3600; + param.mode = 0; + param.affinityMask = 0x1fff; + param.threadPriority = 700; + param.option = 1; + param.format = format; + return param; +} + +int Query(const VideoRecordingParam& param) { + return sceVideoRecordingQueryMemSize(¶m); +} + } int main() { Require(sceVideoRecordingGetStatus() == StatusNone, "a recorder that was never opened reported a status"); Require(sceVideoRecordingGetStatus() == StatusNone, "the status changed between two reads"); + Require(sceVideoRecordingStart() == NotOpen, "the encoder started without a recorder"); + Require(sceVideoRecordingStop() == NotOpen, "the encoder stopped without a recorder"); + Require(sceVideoRecordingClose(0) == NotOpen, "a recorder that was never opened closed"); + Require(sceVideoRecordingClose(1) == NotOpen, "a recorder that was never opened closed with discard"); - std::array param{}; - param.fill(Sentinel); - std::array heap{}; + std::array query{}; + query.fill(Sentinel); + Require(Throws([&] { _sceVideoRecordingQueryParam_nid_postfix(reinterpret_cast(query.data())); }), "the default parameters were produced"); + + Require(sceVideoRecordingQueryMemSize(nullptr) == InvalidArgument, "a heap size was reported for no parameters"); + for (const std::uint32_t format : {0x1u, 0x2u, 0x10u, 0x11u, 0x28u, 0x40u, 0x41u}) { + Require(Throws([&] { Query(RecordingParam(format)); }), "a heap size was reported for a video format"); + VideoRecordingParam param = RecordingParam(format); + param.option = 0; + Require(Throws([&] { Query(param); }), "a heap size was reported for a video format without the option"); + param.mode = 1; + Require(Throws([&] { Query(param); }), "a heap size was reported for mode 1"); + } + for (std::uint32_t format = 0x3; format <= 0x44; ++format) { + if (format == 0x10 || format == 0x11 || format == 0x28 || format == 0x40 || format == 0x41) continue; + Require(Query(RecordingParam(format)) == InvalidArgument, "an unknown video format was accepted"); + } + Require(Query(RecordingParam(0)) == NoVideoFormat, "the option without a video format was accepted"); + for (const std::uint64_t size : {0x0ull, 0x8ull, 0xfull}) { + VideoRecordingParam param = RecordingParam(1); + param.size = size; + Require(Query(param) == InvalidArgument, "a block shorter than its duration and mode was accepted"); + } + for (const std::uint64_t size : {0x10ull, 0x11ull, 0x18ull, 0x1cull, 0x1dull, 0x20ull}) { + VideoRecordingParam param = RecordingParam(1); + param.size = size; + Require(Query(param) == MinimumHeap, "a block without option and format did not ask for 0x1000 bytes"); + } + for (const std::uint64_t size : {0x21ull, 0x24ull}) { + VideoRecordingParam param = RecordingParam(1); + param.size = size; + Require(Query(param) == NoVideoFormat, "a block that ends before its format was accepted"); + } + for (const std::uint64_t size : {0x25ull, 0x40ull, 0x1000ull}) { + VideoRecordingParam param = RecordingParam(1); + param.size = size; + Require(Throws([&] { Query(param); }), "a heap size was reported for a block with a format"); + } + for (const std::uint32_t duration : {0u, 3601u, 7200u, 0xffffffffu}) { + VideoRecordingParam param = RecordingParam(1); + param.duration = duration; + Require(Query(param) == InvalidArgument, "a duration outside 1 to 3600 was accepted"); + } + for (const std::uint32_t duration : {1u, 60u, 3599u}) { + VideoRecordingParam param = RecordingParam(1); + param.duration = duration; + Require(Throws([&] { Query(param); }), "a duration inside 1 to 3600 was refused"); + } + for (const std::uint32_t mode : {2u, 3u}) { + VideoRecordingParam param = RecordingParam(1); + param.mode = mode; + Require(Query(param) == InvalidArgument, "a mode above 1 was accepted"); + } + for (const std::uint32_t option : {2u, 3u}) { + VideoRecordingParam param = RecordingParam(1); + param.option = option; + Require(Query(param) == InvalidArgument, "an option above 1 was accepted"); + param.format = 0; + Require(Query(param) == InvalidArgument, "an option above 1 without a format was accepted"); + } + VideoRecordingParam minimal = RecordingParam(0); + minimal.option = 0; + Require(Query(minimal) == MinimumHeap, "a block without option and format did not ask for 0x1000 bytes"); + for (const std::uint32_t value : {0u, 1u, 2u, 3u}) { + VideoRecordingParam param = RecordingParam(0); + param.option = 0; + param.outputMode = static_cast(value); + param.threadPriority = static_cast(value); + param.affinityMask = value; + param.reserved40 = value; + param.reserved44 = value; + Require(Query(param) == MinimumHeap, "fields the heap size does not depend on changed it"); + } + + std::array heap{}; heap.fill(Sentinel); - - Require(Throws([&] { _sceVideoRecordingQueryParam_nid_postfix(reinterpret_cast(param.data())); }), "the default parameters were produced"); - Require(Throws([&] { sceVideoRecordingQueryMemSize(reinterpret_cast(param.data())); }), "a heap size was reported"); - Require(Throws([&] { sceVideoRecordingQueryMemSize(nullptr); }), "a heap size was reported for no parameters"); - Require(Throws([&] { sceVideoRecordingOpen("clip.mp4", reinterpret_cast(param.data()), heap.data(), static_cast(heap.size())); }), "the recorder opened"); - Require(Throws([&] { sceVideoRecordingStart(); }), "the encoder started"); - Require(Throws([&] { sceVideoRecordingStop(); }), "the encoder stopped"); - Require(Throws([&] { sceVideoRecordingClose(0); }), "the recorder closed"); - Require(Throws([&] { sceVideoRecordingClose(1); }), "the recorder closed with discard"); - - for (const std::uint8_t value : param) Require(value == Sentinel, "the parameters were written"); + const VideoRecordingParam video = RecordingParam(1); + VideoRecordingParam refused = RecordingParam(1); + refused.duration = 0; + Require(sceVideoRecordingOpen(nullptr, &video, heap.data(), static_cast(heap.size())) == InvalidArgument, "the recorder opened without a path"); + Require(sceVideoRecordingOpen("clip.mp4", nullptr, heap.data(), static_cast(heap.size())) == InvalidArgument, "the recorder opened without parameters"); + Require(sceVideoRecordingOpen("clip.mp4", &video, nullptr, static_cast(heap.size())) == InvalidArgument, "the recorder opened without a heap"); + Require(sceVideoRecordingOpen("clip.mp4", &video, heap.data(), -1) == InvalidArgument, "the recorder opened with a negative heap size"); + Require(sceVideoRecordingOpen("clip.mp4", &refused, heap.data(), 0x7fffffff) == InvalidArgument, "the recorder opened with parameters QueryMemSize refuses"); + Require(sceVideoRecordingOpen("clip.mp4", &minimal, heap.data(), MinimumHeap - 1) == InvalidArgument, "the recorder opened with a heap smaller than QueryMemSize asks"); + VideoRecordingParam optionOnly = RecordingParam(0); + Require(sceVideoRecordingOpen("clip.mp4", &optionOnly, heap.data(), static_cast(heap.size())) == NoVideoFormat, "the recorder opened with the option and no format"); + Require(Throws([&] { sceVideoRecordingOpen("clip.mp4", &minimal, heap.data(), MinimumHeap); }), "the recorder opened without recording"); + Require(Throws([&] { sceVideoRecordingOpen("clip.mp4", &video, heap.data(), 0x7fffffff); }), "the recorder opened without recording"); for (const std::uint8_t value : heap) Require(value == Sentinel, "the heap was written"); Require(sceVideoRecordingGetStatus() == StatusNone, "a failed open changed the status"); + Require(sceVideoRecordingStart() == NotOpen, "a failed open left a recorder to start"); + Require(sceVideoRecordingClose(1) == NotOpen, "a failed open left a recorder to close"); - Require(sceVideoRecordingGetInfo(0xa001, nullptr, 0) == static_cast(0x80a80006), "info 0xa001 validation"); - Require(sceVideoRecordingGetInfo(0xa005, nullptr, 0) == static_cast(0x80a80006), "info 0xa005 validation"); - Require(sceVideoRecordingGetInfo(0, nullptr, 0) == static_cast(0x80a80003), "unknown info validation"); + Require(sceVideoRecordingGetInfo(0xa001, nullptr, 0) == NotOpen, "info 0xa001 validation"); + Require(sceVideoRecordingGetInfo(0xa005, nullptr, 0) == NotOpen, "info 0xa005 validation"); + Require(sceVideoRecordingGetInfo(0, nullptr, 0) == InvalidArgument, "unknown info validation"); std::puts("VideoRecording tests passed"); return 0; } diff --git a/docs/dev/debt/libSceVideoRecordingP.md b/docs/dev/debt/libSceVideoRecordingP.md index 835fa5eb21..8a8d86cb84 100644 --- a/docs/dev/debt/libSceVideoRecordingP.md +++ b/docs/dev/debt/libSceVideoRecordingP.md @@ -2,17 +2,19 @@ ### Unknown function info -- [sceVideoRecordingGetInfo, sceVideoRecordingSetInfo](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) support native validation and errors before a recording context exists. Open and initialization remain protected unimplemented exports; live recording queries, control and volume changes are unavailable. +- [sceVideoRecordingGetInfo, sceVideoRecordingSetInfo](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) support native validation and errors before a recording context exists. Opening a recorder and initialization remain unimplemented; live recording queries, control and volume changes are unavailable. - [_sceVideoRecordingQueryParam_nid_postfix](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp): the supplied module reads a 64-bit size and writes a 32-bit mode at offset 12 for sizes at least 16, plus a 32-bit option at offset 28 for sizes at least 64. Mode/option meanings remain unresolved; the protected implementation is unchanged. - [sceVideoRecordingCloseByHandle, sceVideoRecordingGetInfoByHandle](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) - native handles are context pointers. Recording state and per-handle queries remain unimplemented. - [sceVideoRecordingCopyBGRAtoNV12](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) - the destination, source, 32-bit width and height arguments are confirmed. Pixel conversion remains unimplemented. - [sceVideoRecordingGetSupportInfo, sceVideoRecordingGetVideoOutMode](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) - GetSupportInfo writes an 18-byte payload whose fields remain unresolved. GetVideoOutMode takes a four-byte output confirmed by the native recording assembly and Avcap2 IPC descriptor; the mode values remain unresolved. Both exports remain unimplemented. - [sceVideoRecordingInitialize](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp): the native initializer consumes no caller arguments; the former optional parameter is not read by the Avcap2 initializer or recording setup. Its unused parameter slot is retained because the protected body references it. -- [sceVideoRecordingOpen, sceVideoRecordingOpen2](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) - the 32-byte and 40-byte parameter variants, four argument registers and 32-bit heap size are recovered. Mode, output mode, option and format semantics remain incomplete. QueryHddSize2 returns a signed 64-bit size or error; recording and sizing exports remain unimplemented. +- [sceVideoRecordingOpen2](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) - the 40-byte parameter variant, four argument registers and 32-bit heap size are recovered. Mode, output mode, option and format semantics remain incomplete. QueryHddSize2 returns a signed 64-bit size or error; recording and sizing exports remain unimplemented. - [Handle and support queries](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) - native recording handles are context pointers, and GetSupportInfo writes 18 bytes. Support payload field semantics remain incomplete. RestModeProcessStart, RestModeProcessGetStatus and SendCommand consume no caller arguments in the supplied native module; their existing unused parameter slots remain because the protected bodies reference those parameter names. ### Functional -- [sceVideoRecordingGetStatus](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) (libSceVideoRecordingP) always returns 0, `None` ("The recorder is not open") in [SharpProspero's `VideoRecordingStatus`](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Video/VideoRecording.cs#L18-L29), also listed in [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/15abf68c9b52c4b04d094f2837719fdb871021f3/src/core/libraries/video_recording/video_recording.h#L32-L36). It is true here because `sceVideoRecordingOpen` throws, so no recorder is ever open. Unverified: whether the console returns 0 or an error before `sceVideoRecordingInit`/`sceVideoRecordingInitialize`, and when it reports `Running` (1) or `Paused` (2) +- [sceVideoRecordingGetStatus, sceVideoRecordingStart, sceVideoRecordingStop, sceVideoRecordingClose](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) (libSceVideoRecordingP) answer as a PS5 does with no recorder open: GetStatus 0, `None` ("The recorder is not open") in [SharpProspero's `VideoRecordingStatus`](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Video/VideoRecording.cs#L18-L29), also listed in [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/15abf68c9b52c4b04d094f2837719fdb871021f3/src/core/libraries/video_recording/video_recording.h#L32-L36), and 0x80A80006 from the other three, measured on firmware 13.40 inside PPSA01317 without `sceVideoRecordingInit`. It is true here because `sceVideoRecordingOpen` never opens one. Unverified: when GetStatus reports `Running` (1) or `Paused` (2) + +- [sceVideoRecordingOpen, sceVideoRecordingQueryMemSize](../../../core/libs/prx/libSceVideoRecordingP/Export.cpp) (libSceVideoRecordingP) never open a recorder: both throw for a block with a video format, and Open throws for any block it accepts, because neither the heap size nor the recording session is implemented. The checks before that follow a PS5 (firmware 13.40, measured inside PPSA01317): QueryMemSize returns 0x80A80003 for a block shorter than 16 bytes, a duration outside 1 to 3600 seconds, a mode or option above 1 and an unknown format, 0x80A80008 for the option without a format and 0x1000 for neither, and Open refuses null arguments and a negative or too small heap with 0x80A80003. Read and not measured: Open checks the affinity mask, thread priority and output mode after the heap size From 0dc6d3829544ec5b4933d59f3d2ef9819a703085 Mon Sep 17 00:00:00 2001 From: hugodearaujo Date: Sat, 24 Oct 2026 11:13:11 -0300 Subject: [PATCH 162/318] fix(shader): decide each resource branch condition once per plan MayDecide recursed into both operands of every LogicalAnd and LogicalOr without keeping what it had decided, and ran a full MayEvaluate at each step. A condition whose operands share their own operands, as when each level's two conjunctions read the level before, took time exponential in its depth. MayDecide now keeps each value's answer, and ExtractPlan keeps one record for all of a plan's blocks. Skyrim (PPSA03747) prepares shader 0x122151a400 when it calls sceAgcCreateInterpolantMapping on its New Game loading screen. Since 1c9a522bef that preparation did not finish: the title's main thread spun in MayDecide, 20 to 29 frames deep, for the rest of a 300 s run, and no frame was presented after about 150 s. agc_shader_replay --prepare of the captured request runs past 60 s on main and takes 18 ms with this change. --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 35 +++++++++++++++++++ .../Optimization/SrtWalker/SrtEvaluator.hpp | 2 ++ .../Optimization/src/ResourceMaterializer.cpp | 3 +- .../src/SrtWalker/SrtEvaluator.cpp | 31 ++++++++++------ 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 2a31594300..3c77388678 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #ifdef _WIN32 @@ -1104,6 +1105,39 @@ void verifyMayEvaluate() { } } +void verifyMayDecideSharedConditions() { + using namespace ShaderRecompiler; + IrResourcePlan plan; + std::vector> values; + const auto make = [&](IrOpcode opcode, IrType type, std::initializer_list arguments = {}) -> IrValue& { + values.push_back(std::make_unique(opcode, type, static_cast(values.size()))); + for (auto* argument : arguments) values.back()->AddArgument(argument); + return *values.back(); + }; + auto& scalar = make(IrOpcode::Void, IrType::ScalarReg); + scalar.SetRegister({RegisterBank::Scalar, 0}); + auto& vector = make(IrOpcode::Void, IrType::VectorReg); + vector.SetRegister({RegisterBank::Vector, 0}); + auto& input = make(IrOpcode::GetUserData, IrType::U32, {&scalar}); + auto& lane = make(IrOpcode::GetRegister, IrType::U32, {&vector}); + const auto varying = [&] { return &make(IrOpcode::IEqual32, IrType::Bool, {&input, &lane}); }; + const auto chain = [&](IrValue* first) { + auto* condition = first; + for (int level = 0; level < 64; ++level) { + auto& left = make(IrOpcode::LogicalAnd, IrType::Bool, {condition, varying()}); + auto& right = make(IrOpcode::LogicalAnd, IrType::Bool, {condition, varying()}); + condition = &make(IrOpcode::LogicalOr, IrType::Bool, {&left, &right}); + } + return condition; + }; + auto* undecided = chain(varying()); + auto* decided = chain(&make(IrOpcode::LogicalAnd, IrType::Bool, {varying(), &make(IrOpcode::IEqual32, IrType::Bool, {&input, &input})})); + require(!Detail::Evaluator::MayDecide(plan, undecided), "MayDecide decided 64 levels of lane conditions that share their operands"); + require(Detail::Evaluator::MayDecide(plan, decided), "MayDecide did not decide 64 levels of shared conditions over a uniform conjunct"); + std::unordered_map memo; + require(!Detail::Evaluator::MayDecide(plan, undecided, memo) && Detail::Evaluator::MayDecide(plan, decided, memo), "MayDecide answered differently through a shared memo"); +} + void verifyEvaluatedValues() { using namespace ShaderRecompiler; std::vector> values; @@ -8670,6 +8704,7 @@ int main(int argc, char** argv) { verifyOutOfRangeConstantReads(); verifySignedMinimum(); verifyMayEvaluate(); + verifyMayDecideSharedConditions(); verifyPureFlatSlots(); verifyPlanConditionPruning(); verifyPlanRetention(); diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp index 776a3c6dd9..9fb9e6a281 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -180,6 +181,7 @@ public: static bool MayEvaluate(const IrResourcePlan& program, IrValue* value, const IrValue* activeMask = nullptr); static bool MayDecide(const IrResourcePlan& program, IrValue* value); + static bool MayDecide(const IrResourcePlan& program, IrValue* value, std::unordered_map& decided); static EvaluatorReads Reads(const IrResourcePlan& program, const IrValue& value); bool Evaluate(TNode value, std::uint32_t& result); bool EvaluateWide(TNode value, std::uint64_t& result); diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 10202666a4..f91d93da37 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -2436,8 +2436,9 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const plan.info = source.info; PrepareImageModes(plan.info); plan.uniformFill = source.uniformFill; + std::unordered_map decided; for (auto& block : plan.controlFlow) { - if (block.condition != nullptr && !Detail::Evaluator::MayDecide(source, block.condition)) block.condition = nullptr; + if (block.condition != nullptr && !Detail::Evaluator::MayDecide(source, block.condition, decided)) block.condition = nullptr; } const auto addSource = [&plan](std::uint32_t index) { if (index >= plan.descriptorSources.size()) { diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp index 75d62f0546..d8ca54d65d 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp @@ -451,18 +451,29 @@ bool BasicEvaluator::MayEvaluate(const IrResourcePlan& program, IrValue* template bool BasicEvaluator::MayDecide(const IrResourcePlan& program, IrValue* value) { + std::unordered_map decided; + return MayDecide(program, value, decided); +} + +template +bool BasicEvaluator::MayDecide(const IrResourcePlan& program, IrValue* value, std::unordered_map& decided) { if (value == nullptr) return false; - if (MayEvaluate(program, value)) return true; value = value->Resolve(); - const auto decides = [&program, value](std::size_t index) { return value->Argument(index) != nullptr && MayDecide(program, value->Argument(index)); }; - switch (value->Opcode()) { - case IrOpcode::LogicalNot: return value->ArgumentCount() == 1u && decides(0); - case IrOpcode::LogicalAnd: - case IrOpcode::LogicalOr: return value->ArgumentCount() == 2u && (decides(0) || decides(1)); - case IrOpcode::ULessThan32: - case IrOpcode::UGreaterThan32: return value->ArgumentCount() == 2u && (MayEvaluate(program, value->Argument(0)) || MayEvaluate(program, value->Argument(1))); - default: return false; - } + if (const auto found = decided.find(value); found != decided.end()) return found->second; + const auto decides = [&program, &decided, value](std::size_t index) { return value->Argument(index) != nullptr && MayDecide(program, value->Argument(index), decided); }; + const auto decidable = [&] { + if (MayEvaluate(program, value)) return true; + switch (value->Opcode()) { + case IrOpcode::LogicalNot: return value->ArgumentCount() == 1u && decides(0); + case IrOpcode::LogicalAnd: + case IrOpcode::LogicalOr: return value->ArgumentCount() == 2u && (decides(0) || decides(1)); + case IrOpcode::ULessThan32: + case IrOpcode::UGreaterThan32: return value->ArgumentCount() == 2u && (MayEvaluate(program, value->Argument(0)) || MayEvaluate(program, value->Argument(1))); + default: return false; + } + }(); + decided.emplace(value, decidable); + return decidable; } template From 5890fc1820ffd074850a2ecf376ababb3a95e567 Mon Sep 17 00:00:00 2001 From: roboterspinne <229981445+roboterspinne@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:53:32 +0100 Subject: [PATCH 163/318] fix(relinker): align moved Linux PT_LOADs to the page, not to p_align The move path of SeparateSharedLoadPages padded the output byte by byte to an offset congruent modulo max(p_align, 0x1000), so a p_align of 0x80000000 grew the output by 2 GiB and 0x8000000000000000 ended in std::bad_alloc. A moved segment now starts at the next offset congruent to its address modulo the page and its p_align becomes at most 0x1000, since Linux maps every PT_LOAD from p_offset - (p_vaddr % 0x1000) and reads p_align only to align the ET_DYN load bias. At a fixed base every input PT_LOAD p_align is checked against the base before any segment is moved, with the message RebaseProgramHeaders uses. --- .../elfpatcher/src/linux/LinuxElfPatcher.cpp | 18 +++-- .../elfpatcher/tests/LinuxLoadPageTests.cpp | 69 +++++++++++++++++-- docs/dev/debt/relinker.md | 1 + 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp index 716cca1c35..a1e61f8ff3 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp @@ -156,14 +156,13 @@ void SeparateSharedLoadPages(std::vector& buf, std::vector(mapped) + index] != image[index]; if (!differs) continue; - const auto alignment = std::max(load.Alignment, kDefaultLoadAlignment); - while ((buf.size() + prefix) % alignment != load.MappedAddress % alignment) - buf.push_back(0); + Io::AlignBuffer(buf, static_cast(kDefaultLoadAlignment)); buf.insert(buf.end(), image.begin(), image.end()); const auto offset = static_cast(buf.size()); buf.resize(static_cast(offset + load.FileSize)); std::copy_n(buf.begin() + static_cast(load.Offset), static_cast(load.FileSize), buf.begin() + static_cast(offset)); load.Offset = offset; + load.Alignment = std::min(load.Alignment, kDefaultLoadAlignment); } } @@ -276,6 +275,11 @@ std::vector TracePrologue(const std::uint64_t vaddr, const std::ui return code; } +void RequireBaseAlignment(const std::uint64_t base, const std::uint64_t alignment) { + if (alignment > 1 && base % alignment != 0) + throw Domain::RelinkerException("Fixed Linux base " + Hex(base) + " is not a multiple of the PT_LOAD alignment " + Hex(alignment)); +} + void RebaseProgramHeaders(std::vector& buf, const std::uint64_t tableOffset, const std::uint16_t entrySize, const std::uint16_t count, const std::uint64_t base) { for (std::uint16_t index = 0; index < count; ++index) { const auto header = static_cast(tableOffset + static_cast(index) * entrySize); @@ -285,9 +289,7 @@ void RebaseProgramHeaders(std::vector& buf, const std::uint64_t ta const auto address = Rebased(base, Io::ReadU64(buf, header + kPhdrVaddrOffset), "Program header address"); const auto physical = Rebased(base, Io::ReadU64(buf, header + kPhdrPaddrOffset), "Program header physical address"); if (type == PT_LOAD) { - const auto alignment = Io::ReadU64(buf, header + kPhdrAlignOffset); - if (alignment > 1 && base % alignment != 0) - throw Domain::RelinkerException("Fixed Linux base " + Hex(base) + " is not a multiple of the PT_LOAD alignment " + Hex(alignment)); + RequireBaseAlignment(base, Io::ReadU64(buf, header + kPhdrAlignOffset)); if (Io::ReadU64(buf, header + kPhdrMemSizeOffset) > kLinuxUserAddressEnd - address) throw Domain::RelinkerException("PT_LOAD at " + Hex(address) + " ends outside the Linux user address space"); } @@ -581,6 +583,10 @@ std::vector LinuxElfPatcher::Patch( const std::uint64_t extraBlockSize = static_cast(buf.size()) - extraBlockOff; auto headers = originalHeaders; + if (_fixedBase) + for (const auto& header : headers) + if (header.Type == PT_LOAD) + RequireBaseAlignment(*_fixedBase, header.Alignment); SeparateSharedLoadPages(buf, headers); ProgramHeaderLayoutRequest layoutRequest{}; diff --git a/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp b/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp index 0828c906d4..df75c963b9 100644 --- a/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp +++ b/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -51,17 +52,18 @@ Bytes Fixture(const std::size_t size) { return bytes; } -Domain::ProgramHeader Load(const std::uint64_t offset, const std::uint64_t vaddr, const std::uint64_t fileSize, const std::uint64_t memorySize, const std::uint32_t flags) { - return {1, flags, offset, vaddr, vaddr, fileSize, memorySize, 0x1000}; +Domain::ProgramHeader Load(const std::uint64_t offset, const std::uint64_t vaddr, const std::uint64_t fileSize, const std::uint64_t memorySize, const std::uint32_t flags, const std::uint64_t alignment = kPage) { + return {1, flags, offset, vaddr, vaddr, fileSize, memorySize, alignment}; } -Elfpatcher::Linux::LinuxElfPatcher Patcher() { +Elfpatcher::Linux::LinuxElfPatcher Patcher(const std::optional fixedBase = std::nullopt) { const auto byteWriter = std::make_shared(); return Elfpatcher::Linux::LinuxElfPatcher( std::make_shared(), std::make_shared(std::make_shared(), byteWriter), std::make_shared(byteWriter), - byteWriter); + byteWriter, + fixedBase); } std::vector OutputLoads(const Bytes& output) { @@ -120,6 +122,12 @@ std::uint64_t OutputOffset(const Bytes& output, const std::uint64_t vaddr) { throw std::runtime_error("linux load pages: no output PT_LOAD at the segment's address"); } +std::uint64_t OutputAlignment(const Bytes& output, const std::uint64_t vaddr) { + for (const auto& load : OutputLoads(output)) + if (load.MappedAddress == vaddr) return load.Alignment; + throw std::runtime_error("linux load pages: no output PT_LOAD at the segment's address"); +} + void ZeroFillBelowTheNextSegment() { auto source = Fixture(0x3000); std::fill(source.begin() + 0x1000, source.begin() + 0x1100, std::uint8_t{0xAA}); @@ -199,6 +207,56 @@ void IncongruentOffsetBelowThePagePrefix() { RequireImage(output, source, headers, "segment whose offset lies below its page prefix"); } +void LargeAlignmentMovesToThePage() { + for (const auto alignment : {std::uint64_t{0x80000000}, std::uint64_t{0x4000}, std::uint64_t{0x8000000000000000}}) { + auto source = Fixture(0x2000); + std::fill(source.begin() + 0x1000, source.begin() + 0x1010, std::uint8_t{0x33}); + std::fill(source.begin() + 0x1010, source.begin() + 0x1110, std::uint8_t{0xAA}); + const std::vector headers{ + Load(0x400, 0x400, 0x10, 0x10, 5), + Load(0x1010, 0x1000, 0x100, 0x100, 6, alignment), + }; + const auto output = Patcher().Patch(source, headers, {}, 0, "$ORIGIN/libs", true, false, {}); + Require(output.size() <= source.size() + 3 * kPage, "output grew to " + std::to_string(output.size()) + " bytes for alignment " + std::to_string(alignment)); + RequireImage(output, source, headers, "incongruent segment with a large alignment"); + Require(OutputAlignment(output, 0x1000) == kPage, "moved segment declares an alignment its offset does not honour"); + } +} + +void LargeAlignmentOnASharedPage() { + auto source = Fixture(0x3000); + std::fill(source.begin() + 0x1000, source.begin() + 0x1100, std::uint8_t{0xAA}); + std::fill(source.begin() + 0x1100, source.begin() + 0x1c20, std::uint8_t{0x55}); + std::fill(source.begin() + 0x1c20, source.begin() + 0x2000, std::uint8_t{0x77}); + const std::vector headers{ + Load(0x400, 0x400, 0x10, 0x10, 5), + Load(0x1000, 0x1000, 0x100, 0x1c20, 6), + Load(0x1c20, 0x2c20, 0x3e0, 0x3e0, 0, std::uint64_t{0x80000000}), + }; + const auto output = Patcher().Patch(source, headers, {}, 0, "$ORIGIN/libs", true, false, {}); + Require(output.size() <= source.size() + 3 * kPage, "output grew to " + std::to_string(output.size()) + " bytes on the shared page path"); + RequireImage(output, source, headers, "shared page with a large alignment"); + Require(OutputAlignment(output, 0x2c20) == kPage, "moved segment declares an alignment its offset does not honour"); +} + +void LargeAlignmentAtAFixedBaseIsRefused() { + for (const auto alignment : {std::uint64_t{0x80000000}, std::uint64_t{0x8000000000000000}}) { + auto source = Fixture(0x2000); + std::fill(source.begin() + 0x1010, source.begin() + 0x1110, std::uint8_t{0xAA}); + const std::vector headers{ + Load(0x400, 0x400, 0x10, 0x10, 5), + Load(0x1010, 0x1000, 0x100, 0x100, 6, alignment), + }; + std::string message; + try { + Patcher(0x400000).Patch(source, headers, {}, 0, "$ORIGIN/libs", false, false, {}); + } catch (const std::exception& error) { + message = error.what(); + } + Require(message.find("is not a multiple of the PT_LOAD alignment") != std::string::npos, "fixed base accepted an alignment it is not a multiple of: " + message); + } +} + } int main() { @@ -209,6 +267,9 @@ int main() { IncongruentSegmentAtPageStart(); IncongruentSegmentInsideASharedPage(); IncongruentOffsetBelowThePagePrefix(); + LargeAlignmentMovesToThePage(); + LargeAlignmentOnASharedPage(); + LargeAlignmentAtAFixedBaseIsRefused(); } catch (const std::exception& error) { std::cerr << error.what() << '\n'; return 1; diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index f2d970f219..45e875c94f 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -14,6 +14,7 @@ - Converted Windows main executables [omit `DYNAMIC_BASE`](../../../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) and [prefer `0x400000`](../../../core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp) for their lowest ELF load page by default, not only with `--windows-fixed-base`: on the console the executable lies inside the 40-bit window that the title's own `sceLibcMspaceCreate` checks (#3043). Where the console loads an executable that does not require an address is not measured; `0x400000` is the base that executables reading their GOT through `app0/imports.txt` assume (#4013). Relocations are kept, so if that range is taken Windows can relocate the image, possibly above that window; no run has observed this, and from this base an image cannot reach the shared user data page at `0x7ffe0000`, since Windows refuses a `SizeOfImage` above `0x77000000` (#9332). Verified on Windows 11 only. - [SDK memory-management syscall traps](../../../core/relinker/relinker/src/analysis/SyscallScanner.cpp) under `--trap-sdk-syscall-stubs` recognize three exact instruction sequences from [ps5-payload-dev SDK libc/mman.c](https://github.com/ps5-payload-dev/sdk/blob/12cf3dda40/libc/mman.c), observed in Nuvio Maxd 0.5.15 (PPSA99288): syscall 74 and the two syscall 477 paths, with their argument preparation and carry-flag error handling. Matching requires a decoded instruction boundary and the complete sequence inside the scanned code range. Other SDK/compiler output remains rejected. The matched syscall becomes `UD2`; this does not implement mmap or mprotect and a call still faults. - The [instruction decoder](../../../core/relinker/codegen/src/x86/X64InstructionDecoder.cpp) reads a 16-bit displacement for `66 E8`, `66 E9` and `66 0F 80`-`8F` without REX.W, as AMD processors, the console included, do (binutils `objdump -M amd64`; REX.W selects the 32-bit form). AMD also truncates the target of every near relative branch with that prefix, rel8, LOOP and JRCXZ forms included, to 16 bits. Instruction metadata marks these branches as truncated and reports no branch target for them, and stubs refuse to move them. Compilers do not emit these forms. +- A `PT_LOAD` that the [Linux patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) copies to the end of the output, because its first page would read other bytes or its `p_offset` and `p_vaddr` differ modulo the page, is placed at an offset congruent to its address modulo the page size and its `p_align` becomes `min(p_align, 0x1000)`: Linux maps a segment from `p_offset - (p_vaddr % 0x1000)` (`elf_map` in `fs/binfmt_elf.c`) and reads `p_align` only to align the `ET_DYN` load bias to the largest power-of-two value among all `PT_LOAD` headers (`maximum_alignment`). While another `PT_LOAD` keeps that largest value, the load bias alignment does not change; when the moved segment alone carried it, as in a single-segment `--lazy-binding` image, it drops to the next largest value, down to the page. Whether a title relies on the larger alignment is not known. A kept segment keeps its `p_align`, and at a fixed base every input `p_align` above 1 must still divide the base. - Converted Linux main executables are [placed at `0x400000`](../../../core/relinker/relinker/src/pipeline/Conversion.cpp) by default, as `ET_EXEC` without ASLR, not only with `--linux-fixed-base`: the console executable lies inside the 40-bit window that the title's own `sceLibcMspaceCreate` checks (#3043). Where the console loads an executable that does not require an address is not measured; `0x400000` is the base that executables reading their GOT through `app0/imports.txt` assume (#4013). The relinker refuses an image that spans 2 GiB or more, so the executable ends below the guest arena at `0x200000000`. With `--lazy-binding` the executable stays position independent. - `--frame-red-zone` frames only leaf functions that start with a chain of pushes of at least five bytes (`push rbp; mov rbp, rsp; push ...`, or pushes alone, which may be followed by `sub $n, %rsp` or `lea -n(%rsp), %rsp`) and leave through exits of at least five bytes that undo that allocation with `add` or `lea`, pop the same registers and `ret` ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). Leaves with a frame pointer that allocate, leaves with neither pushes nor an allocation, shorter prologues and exits, exits that free another size, and functions with indirect jumps or other RSP arithmetic keep their red zone. - In a red-zone leaf without a frame pointer, `--frame-red-zone` lowers RSP after the prologue by the deepest displacement below RSP, rounded up to 16, and moves every RSP operand by the same amount ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). A leaf that takes the address of a slot below RSP, or uses an RSP operand with an index register, is lowered by the whole 128 bytes, since a pointer or an index can reach deeper than any displacement; values below RSP reached through any other register are not seen. Leaves with a frame pointer are lowered by 128 bytes. Inside a moved frame the CFA rules of the guest's unwind tables are off by the shift. From 91382605720d5730ae712a4f0b5a78d978125989 Mon Sep 17 00:00:00 2001 From: Maximilian Fleckenstein <166870690+maximilianfv@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:10:15 -0300 Subject: [PATCH 164/318] fix(agc): load the Vulkan loader and list MoltenVK on macOS The driver loaded libvulkan.so.1, which does not exist on macOS, so it could not create a Vulkan device there. Load libvulkan.1.dylib on macOS. MoltenVK is a portability driver: the loader lists it only for an instance created with VK_KHR_portability_enumeration and VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR, so request both on macOS, with or without a window. A device that advertises VK_KHR_portability_subset must enable it; enable it with the portability features the device reports. Co-authored-by: mugurc <20699965+mugurc@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 1 + .../Execution/src/VulkanDevice.cpp | 21 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 29669383a9..06eb8fe7d7 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -389,6 +389,7 @@ set(agcZlibIncludeDir ${CMAKE_SOURCE_DIR}/3rdparty/libjpeg-turbo/src/spng/zlib) set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/TraceFile.cpp PROPERTIES INCLUDE_DIRECTORIES ${agcZlibIncludeDir} ) +set_property(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/VulkanDevice.cpp APPEND PROPERTY COMPILE_DEFINITIONS VK_ENABLE_BETA_EXTENSIONS) add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL ${agcDriverSources} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 2306dbe33b..a749d564dd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1122,6 +1122,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : APS5_LOG_OUT("VulkanDevice constructor window=%p", static_cast(window)); #ifdef _WIN32 state->library = SDL_LoadObject("vulkan-1.dll"); +#elif defined(__APPLE__) + state->library = SDL_LoadObject("libvulkan.1.dylib"); #else state->library = SDL_LoadObject("libvulkan.so.1"); #endif @@ -1162,9 +1164,13 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : instanceExtensions.push_back(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME); state->swapchainColorSpaces = true; } - create.enabledExtensionCount = static_cast(instanceExtensions.size()); - create.ppEnabledExtensionNames = instanceExtensions.data(); } +#ifdef __APPLE__ + instanceExtensions.push_back(VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME); + create.flags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR; +#endif + create.enabledExtensionCount = static_cast(instanceExtensions.size()); + create.ppEnabledExtensionNames = instanceExtensions.empty() ? nullptr : instanceExtensions.data(); check(state->InstanceFunction("vkCreateInstance")(&create, nullptr, &state->instance), "vkCreateInstance"); APS5_LOG_OUT("Vulkan instance created instance=%p", reinterpret_cast(state->instance)); if (window != nullptr) { @@ -1628,6 +1634,13 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : if (state->nonSeamlessCubeMap) deviceExtensions.push_back(VK_EXT_NON_SEAMLESS_CUBE_MAP_EXTENSION_NAME); } nonSeamlessCubeFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NON_SEAMLESS_CUBE_MAP_FEATURES_EXT}; + VkPhysicalDevicePortabilitySubsetFeaturesKHR portabilityFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PORTABILITY_SUBSET_FEATURES_KHR}; + const bool portabilitySubset = hasExtension(VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME); + if (portabilitySubset) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &portabilityFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + deviceExtensions.push_back(VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME); + } nonSeamlessCubeFeatures.nonSeamlessCubeMap = VK_TRUE; VkPhysicalDeviceCoverageReductionModeFeaturesNV coverageReductionFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COVERAGE_REDUCTION_MODE_FEATURES_NV}; if (hasExtension(VK_NV_FRAMEBUFFER_MIXED_SAMPLES_EXTENSION_NAME) && hasExtension(VK_NV_COVERAGE_REDUCTION_MODE_EXTENSION_NAME)) { @@ -1917,6 +1930,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : nonSeamlessCubeFeatures.pNext = const_cast(deviceInfo.pNext); deviceInfo.pNext = &nonSeamlessCubeFeatures; } + if (portabilitySubset) { + portabilityFeatures.pNext = const_cast(deviceInfo.pNext); + deviceInfo.pNext = &portabilityFeatures; + } if (!state->mixedAttachmentSamples && (state->mergedCoverageSampleCounts != 0 || state->truncatedCoverageSampleCounts != 0)) { coverageReductionFeatures.pNext = const_cast(deviceInfo.pNext); deviceInfo.pNext = &coverageReductionFeatures; From 2a999da21f5650bfb9aacf88c0d0194d3c5d66ad Mon Sep 17 00:00:00 2001 From: Retr0 Date: Mon, 26 Oct 2026 21:07:38 +0200 Subject: [PATCH 165/318] test(libc): drop the freopen check that pins a write mode on a random device to throwing --- core/libs/tests/GuestFilesystem.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/core/libs/tests/GuestFilesystem.cpp b/core/libs/tests/GuestFilesystem.cpp index 7650f8d44a..1675e0d4e7 100644 --- a/core/libs/tests/GuestFilesystem.cpp +++ b/core/libs/tests/GuestFilesystem.cpp @@ -1562,15 +1562,6 @@ int main() { Require(freopen_nid_postfix(presentName.c_str(), "rb", reopened) == reopened); Require(fread_nid_postfix(first, 1, sizeof(first), reopened) == std::filesystem::file_size(present) && fgetc_nid_postfix(reopened) == -1); Require(freopen_nid_postfix(device, "rb", reopened) == reopened && fclose_nid_postfix(reopened) == 0); - reopened = fopen_nid_postfix(presentName.c_str(), "rb"); - Require(reopened != nullptr); - bool rejected = false; - try { - freopen_nid_postfix(device, "r+", reopened); - } catch (const std::runtime_error&) { - rejected = true; - } - Require(rejected && fclose_nid_postfix(reopened) == 0); } Require(unlink_nid_postfix(presentName.c_str()) == 0 && !std::filesystem::exists(present)); const auto empty = root / "empty"; From d847910c69dfebe95049d55fe79552f5f5b6087e Mon Sep 17 00:00:00 2001 From: JJenkx Date: Mon, 26 Oct 2026 16:46:01 -0400 Subject: [PATCH 166/318] feat(http2): deliver redirect callbacks and follow HTTPS redirects --- core/libs/prx/libSceHttp2/CMakeLists.txt | 6 + core/libs/prx/libSceHttp2/Export.cpp | 9 +- core/libs/prx/libSceHttp2/Transport.cpp | 168 ++++++++++++--- core/libs/prx/libSceHttp2/Transport.hpp | 8 +- core/libs/prx/libSceHttp2/include/Http2.hpp | 4 +- core/libs/tests/GuestHttp2.cpp | 16 ++ core/libs/tests/GuestHttp2Transport.cpp | 218 +++++++++++++++++++- docs/dev/debt/libSceHttp2.md | 4 +- 8 files changed, 392 insertions(+), 41 deletions(-) diff --git a/core/libs/prx/libSceHttp2/CMakeLists.txt b/core/libs/prx/libSceHttp2/CMakeLists.txt index 15e7dd4e2f..2bffe9d4bc 100644 --- a/core/libs/prx/libSceHttp2/CMakeLists.txt +++ b/core/libs/prx/libSceHttp2/CMakeLists.txt @@ -55,4 +55,10 @@ if(BUILD_TESTING) set_tests_properties(guest_http2 guest_http2_cookies guest_http2_transport guest_http2_early_response guest_http2_dns_first PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME guest_http2_redirects COMMAND guest_http2_transport_tests --redirects) + set_tests_properties(guest_http2_redirects PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_http2_redirects PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() diff --git a/core/libs/prx/libSceHttp2/Export.cpp b/core/libs/prx/libSceHttp2/Export.cpp index c5ff9a1b43..568db5d509 100644 --- a/core/libs/prx/libSceHttp2/Export.cpp +++ b/core/libs/prx/libSceHttp2/Export.cpp @@ -83,7 +83,7 @@ static std::shared_ptr Transport(int id, bool create) { if (!handle) return nullptr; if (create && !handle->transport) { if (handle->version != 3) NotImplemented_nid_no_patch("sceHttp2SendRequest: HTTP/1.x transport is not implemented"); - handle->transport = Http2::CreateRequest_nid_no_patch(handle->sslContext, handle->capacity, handle->settings, handle->method, handle->url); + handle->transport = Http2::CreateRequest_nid_no_patch(id, handle->sslContext, handle->capacity, handle->settings, handle->method, handle->url); } if (!handle->transport) NotImplemented_nid_no_patch("sceHttp2: request has not been sent"); return handle->transport; @@ -234,8 +234,11 @@ int APS5_VABI sceHttp2SetRecvTimeOut(int id, uint32_t usec) { }); } -int APS5_VABI sceHttp2SetRedirectCallback(int id, void* cb_func, void* user_arg) { - return Update(id, __func__, [&](auto&) { if (cb_func || user_arg) NotImplemented_nid_no_patch("sceHttp2SetRedirectCallback"); }); +int APS5_VABI sceHttp2SetRedirectCallback(int id, Http2RedirectCallback cb_func, void* user_arg) { + return Update(id, __func__, [&](auto& settings) { + settings.redirectCallback = cb_func; + settings.redirectArgument = user_arg; + }); } int APS5_VABI sceHttp2SetRequestContentLength(int id, uint64_t content_length) { diff --git a/core/libs/prx/libSceHttp2/Transport.cpp b/core/libs/prx/libSceHttp2/Transport.cpp index 767e89579e..5357128acc 100644 --- a/core/libs/prx/libSceHttp2/Transport.cpp +++ b/core/libs/prx/libSceHttp2/Transport.cpp @@ -101,6 +101,52 @@ Url Parse(std::string_view text) { return url; } +std::string NormalizePath(std::string_view path) { + std::string result; + while (!path.empty()) { + if (path.starts_with("../")) path.remove_prefix(3); + else if (path.starts_with("./")) path.remove_prefix(2); + else if (path.starts_with("/./")) path.remove_prefix(2); + else if (path == "/.") path = "/"; + else if (path.starts_with("/../") || path == "/..") { + path = path == "/.." ? "/" : path.substr(3); + const auto slash = result.rfind('/'); + result.erase(slash == std::string::npos ? 0 : slash); + } else if (path == "." || path == "..") path = {}; + else { + const auto slash = path.find('/', path.front() == '/' ? 1 : 0); + result.append(path.substr(0, slash)); + path = slash == std::string_view::npos ? std::string_view{} : path.substr(slash); + } + } + return result; +} + +Url RedirectTarget(const Url& base, std::string_view reference) { + reference = reference.substr(0, reference.find('#')); + const auto colon = reference.find(':'); + const auto delimiter = reference.find_first_of("/?"); + Url result; + if (colon != std::string_view::npos && (delimiter == std::string_view::npos || colon < delimiter)) result = Parse(reference); + else if (reference.starts_with("//")) result = Parse("https:" + std::string(reference)); + else { + std::string path; + if (reference.empty()) path = base.path; + else if (reference.starts_with('?')) path = base.path.substr(0, base.path.find('?')) + std::string(reference); + else if (reference.starts_with('/')) path = reference; + else { + path = base.path.substr(0, base.path.find('?')); + path.erase(path.rfind('/') + 1); + path += reference; + } + result = Parse("https://" + base.authority + path); + } + const auto query = result.path.find('?'); + result.path = NormalizePath(std::string_view(result.path).substr(0, query)) + + (query == std::string::npos ? std::string{} : result.path.substr(query)); + return result; +} + struct Address { std::array bytes{}; std::uint32_t size = 0; @@ -231,19 +277,23 @@ void AddHeader(Settings& settings, const char* name, const char* value, std::uin } struct Request::State { + const int id; int sslContext; const std::shared_ptr capacity; bool acquired = false; - const Settings settings; - const std::string method; - const Url url; + Settings settings; + std::string method; + Url url; std::mutex mutex; std::atomic cancelled{false}; bool started = false; + bool callbackActive = false; bool ready = false; bool ended = false; int status = 0; int blockStatus = 0; + std::string location; + bool hasLocation = false; int networkResult = 0; int socket = -1; int ssl = -1; @@ -263,8 +313,8 @@ struct Request::State { std::exception_ptr callbackFailure; Clock::time_point totalDeadline; - State(int context, std::shared_ptr capacity, Settings options, std::string verb, std::string target) - : sslContext(context), capacity(std::move(capacity)), settings(std::move(options)), method(std::move(verb)), url(Parse(target)) { + State(int id, int context, std::shared_ptr capacity, Settings options, std::string verb, std::string target) + : id(id), sslContext(context), capacity(std::move(capacity)), settings(std::move(options)), method(std::move(verb)), url(Parse(target)) { Require(!method.empty() && nghttp2_check_method(reinterpret_cast(method.data()), method.size()), "sceHttp2CreateRequestWithURL: invalid method"); Require(method != "CONNECT", "sceHttp2: CONNECT is not implemented"); @@ -379,6 +429,8 @@ struct Request::State { state.blockStatus = 0; state.hasLength = false; state.contentLength = 0; + state.location.clear(); + state.hasLocation = false; } }); } @@ -394,6 +446,11 @@ struct Request::State { const std::string_view key(reinterpret_cast(name), nameSize); const std::string_view text(reinterpret_cast(value), valueSize); if (key == ":status") state.blockStatus = static_cast(Number(text)); + if (key == "location") { + Require(!state.hasLocation, "sceHttp2: duplicate redirect location; console error mapping is unknown"); + state.location = text; + state.hasLocation = true; + } if (key == "content-length") { state.contentLength = Number(text); state.hasLength = true; @@ -410,8 +467,6 @@ struct Request::State { if (frame->hd.stream_id != state.stream) return; if (frame->hd.type == NGHTTP2_HEADERS && state.blockStatus >= 200) { state.status = state.blockStatus; - Require(!state.settings.redirect || (state.status != 301 && state.status != 302 && state.status != 303 && state.status != 307 && state.status != 308), - "sceHttp2: automatic redirects are not implemented"); Require(!state.settings.auth || (state.status != 401 && state.status != 407), "sceHttp2: authentication is not implemented"); state.ready = true; } @@ -525,6 +580,7 @@ struct Request::State { std::unique_lock Lock() { std::unique_lock lock(mutex, std::try_to_lock); Require(lock.owns_lock(), "sceHttp2: concurrent operation on one request is not implemented"); + Require(!callbackActive, "sceHttp2: operation during redirect callback is not implemented"); if (failure) std::rethrow_exception(failure); Require(!cancelled.load(), "sceHttp2: request cancelled; console error mapping is unknown"); return lock; @@ -537,8 +593,8 @@ struct Request::State { } }; -Request::Request(int context, std::shared_ptr capacity, Settings settings, std::string method, std::string url) - : state(std::make_unique(context, std::move(capacity), std::move(settings), std::move(method), std::move(url))) {} +Request::Request(int id, int context, std::shared_ptr capacity, Settings settings, std::string method, std::string url) + : state(std::make_unique(id, context, std::move(capacity), std::move(settings), std::move(method), std::move(url))) {} Request::~Request() = default; int Request::Send(const void* data, std::size_t size) { @@ -549,24 +605,80 @@ int Request::Send(const void* data, std::size_t size) { state->started = true; state->totalDeadline = Clock::now() + std::chrono::microseconds(state->settings.timeout); try { - const int active = state->capacity->active.fetch_add(1); - state->acquired = true; - Require(active < state->capacity->maximum, "sceHttp2: connection slot waiting is not implemented"); - state->networkResult = state->Connect(); - if (state->networkResult < 0) { state->Close(); return state->networkResult; } - state->Setup(data, size); - const auto sendDeadline = state->Deadline(state->settings.sendTimeout); - auto receiveDeadline = sendDeadline; - bool uploaded = false; - for (;;) { - state->Flush(uploaded ? state->Deadline(state->settings.sendTimeout) : sendDeadline); - if (!uploaded && state->uploadOffset == size) { - uploaded = true; - receiveDeadline = state->Deadline(state->settings.recvTimeout); + const void* payload = data; + std::size_t payloadSize = size; + for (unsigned redirects = 0;; ++redirects) { + const int active = state->capacity->active.fetch_add(1); + state->acquired = true; + Require(active < state->capacity->maximum, "sceHttp2: connection slot waiting is not implemented"); + state->networkResult = state->Connect(); + if (state->networkResult < 0) { state->Close(); return state->networkResult; } + state->Setup(payload, payloadSize); + const auto sendDeadline = state->Deadline(state->settings.sendTimeout); + auto receiveDeadline = sendDeadline; + bool uploaded = false; + for (;;) { + state->Flush(uploaded ? state->Deadline(state->settings.sendTimeout) : sendDeadline); + if (!uploaded && state->uploadOffset == payloadSize) { + uploaded = true; + receiveDeadline = state->Deadline(state->settings.recvTimeout); + } + if (state->ready) break; + state->Receive(uploaded ? receiveDeadline : sendDeadline); + if (state->ready) break; } - if (state->ready) break; - state->Receive(uploaded ? receiveDeadline : sendDeadline); - if (state->ready) break; + const bool redirectStatus = state->status == 301 || state->status == 302 || state->status == 303 || state->status == 307 || state->status == 308; + if (!state->settings.redirect || !redirectStatus || state->location.empty()) break; + const bool retrieve = (state->status == 303 && state->method != "HEAD") || + ((state->status == 301 || state->status == 302) && state->method == "POST"); + std::string method = retrieve ? "GET" : state->method; + if (const auto callback = state->settings.redirectCallback) { + const char* selected = method.c_str(); + state->callbackActive = true; + lock.unlock(); + int result; + try { + result = callback(state->id, state->status, &selected, state->location.c_str(), state->settings.redirectArgument); + } catch (...) { + lock.lock(); + state->callbackActive = false; + throw; + } + lock.lock(); + state->callbackActive = false; + state->Check(state->totalDeadline); + if (result != 0) break; + Require(selected, "sceHttp2: redirect callback returned a null method"); + method = selected; + } + Require(redirects < 6, "sceHttp2: redirect limit exceeded; console error mapping is unknown"); + Require(!method.empty() && nghttp2_check_method(reinterpret_cast(method.data()), method.size()) && method != "CONNECT", + "sceHttp2: redirect callback returned an unsupported method"); + auto target = RedirectTarget(state->url, state->location); + const bool differentOrigin = Lower(target.host) != Lower(state->url.host) || target.port != state->url.port; + const bool dropBody = (retrieve || method != state->method) && (method == "GET" || method == "HEAD"); + std::erase_if(state->settings.headers, [&](const auto& header) { + const auto& key = header.first; + return (differentOrigin && (key == "authorization" || key == "proxy-authorization" || key == "cookie" || key == "origin" || key == "referer")) || + (dropBody && (key.starts_with("content-") || key == "digest" || key == "last-modified")); + }); + if (dropBody) { payload = nullptr; payloadSize = 0; } + state->settings.contentLength = payloadSize; + state->Close(); + state->method = std::move(method); + state->url = std::move(target); + state->ready = state->ended = false; + state->status = state->blockStatus = 0; + state->location.clear(); + state->hasLocation = state->hasLength = false; + state->contentLength = 0; + state->headerBytes = 0; + state->body.clear(); + state->bodyOffset = 0; + state->upload = nullptr; + state->uploadSize = state->uploadOffset = 0; + state->callbackFailure = nullptr; + state->stream = -1; } state->upload = nullptr; if (state->ended && state->body.empty()) state->Close(); @@ -625,8 +737,8 @@ std::shared_ptr CreateCapacity_nid_no_patch(int maximum) { return std::make_shared(maximum); } -std::shared_ptr CreateRequest_nid_no_patch(int sslContext, std::shared_ptr capacity, Settings settings, std::string method, std::string url) { - return std::make_shared(sslContext, std::move(capacity), std::move(settings), std::move(method), std::move(url)); +std::shared_ptr CreateRequest_nid_no_patch(int id, int sslContext, std::shared_ptr capacity, Settings settings, std::string method, std::string url) { + return std::make_shared(id, sslContext, std::move(capacity), std::move(settings), std::move(method), std::move(url)); } } diff --git a/core/libs/prx/libSceHttp2/Transport.hpp b/core/libs/prx/libSceHttp2/Transport.hpp index 5549358bf5..29514a2675 100644 --- a/core/libs/prx/libSceHttp2/Transport.hpp +++ b/core/libs/prx/libSceHttp2/Transport.hpp @@ -1,5 +1,7 @@ #pragma once +#include "prx/libSceHttp2/include/Http2.hpp" + #include #include #include @@ -11,6 +13,8 @@ namespace Http2 { struct Settings { + Http2RedirectCallback redirectCallback = nullptr; + void* redirectArgument = nullptr; std::string userAgent; std::vector> headers; std::uint64_t contentLength = 0; @@ -33,7 +37,7 @@ struct Capacity { class Request { public: - Request(int sslContext, std::shared_ptr capacity, Settings settings, std::string method, std::string url); + Request(int id, int sslContext, std::shared_ptr capacity, Settings settings, std::string method, std::string url); ~Request(); Request(const Request&) = delete; Request& operator=(const Request&) = delete; @@ -50,6 +54,6 @@ private: void AddHeader(Settings& settings, const char* name, const char* value, std::uint32_t mode); std::shared_ptr CreateCapacity_nid_no_patch(int maximum); -std::shared_ptr CreateRequest_nid_no_patch(int sslContext, std::shared_ptr capacity, Settings settings, std::string method, std::string url); +std::shared_ptr CreateRequest_nid_no_patch(int id, int sslContext, std::shared_ptr capacity, Settings settings, std::string method, std::string url); } diff --git a/core/libs/prx/libSceHttp2/include/Http2.hpp b/core/libs/prx/libSceHttp2/include/Http2.hpp index 678aec1f08..4d641dffca 100644 --- a/core/libs/prx/libSceHttp2/include/Http2.hpp +++ b/core/libs/prx/libSceHttp2/include/Http2.hpp @@ -2,6 +2,8 @@ #include "SceTypes.hpp" +using Http2RedirectCallback = int (APS5_VABI*)(int requestId, int statusCode, const char** method, const char* location, void* userArg); + struct Http2MemoryPoolStats { size_t pool_size; size_t max_inuse_size; @@ -46,7 +48,7 @@ int APS5_VABI sceHttp2SetConnectionWaitTimeOut(int id, uint32_t usec); int APS5_VABI sceHttp2SetConnectTimeOut(int id, uint32_t usec); int APS5_VABI sceHttp2SetInflateGZIPEnabled(int id, int enable); int APS5_VABI sceHttp2SetRecvTimeOut(int id, uint32_t usec); -int APS5_VABI sceHttp2SetRedirectCallback(int id, void* cbFunc, void* userArg); +int APS5_VABI sceHttp2SetRedirectCallback(int id, Http2RedirectCallback cbFunc, void* userArg); int APS5_VABI sceHttp2SetRequestContentLength(int id, uint64_t contentLength); int APS5_VABI sceHttp2SetResolveTimeOut(int id, uint32_t usec); int APS5_VABI sceHttp2SetSendTimeOut(int id, uint32_t usec); diff --git a/core/libs/tests/GuestHttp2.cpp b/core/libs/tests/GuestHttp2.cpp index 163760ae30..0d7e4fcd88 100644 --- a/core/libs/tests/GuestHttp2.cpp +++ b/core/libs/tests/GuestHttp2.cpp @@ -16,6 +16,13 @@ static bool Throws(TCall call) { return false; } +static int callbackCalls = 0; + +static int APS5_VABI RejectRedirect(int, int, const char**, const char*, void*) { + ++callbackCalls; + return -1; +} + int main() { Require(sceHttp2CreateCookieBox(0xffff) == static_cast(0x817b121du)); const int box = sceHttp2CreateCookieBox(0x10000); @@ -27,6 +34,15 @@ int main() { Require(Throws([&] { sceHttp2CreateTemplate(context, "agent", 3, -1); })); const int request = sceHttp2CreateRequestWithURL(tmpl, "GET", "https://localhost/", 0); Require(box > 0 && context > 0 && tmpl > 0 && request > 0); + Require(sceHttp2SetRedirectCallback(-1, RejectRedirect, nullptr) == InvalidId); + for (const int id : {tmpl, request}) { + Require(sceHttp2SetRedirectCallback(id, RejectRedirect, &callbackCalls) == 0); + Require(sceHttp2SetRedirectCallback(id, nullptr, &callbackCalls) == 0); + Require(sceHttp2SetRedirectCallback(id, RejectRedirect, nullptr) == 0); + } + Require(callbackCalls == 0); + for (const int id : {context, box}) Require(Throws([&] { sceHttp2SetRedirectCallback(id, RejectRedirect, nullptr); })); + Require(sceHttp2SetCookieBox(tmpl, 0) == 0); Require(Throws([&] { sceHttp2SetCookieBox(tmpl, box); })); Require(sceHttp2CookieFlush(box) == 0); diff --git a/core/libs/tests/GuestHttp2Transport.cpp b/core/libs/tests/GuestHttp2Transport.cpp index b785722d08..8d8b2e40f8 100644 --- a/core/libs/tests/GuestHttp2Transport.cpp +++ b/core/libs/tests/GuestHttp2Transport.cpp @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -48,12 +50,13 @@ bool Throws(TCall call) { return false; } -enum class Mode { Normal, Large, NoLength, Head, NotFound, Redirect, Auth, Reset, Truncate, StallHandshake, StallHeaders, StallBody, NoAlpn, Informational, StallUpload, EarlyFinal, EarlyReset, EarlyEmpty, EarlyLarge, EarlyResetIncomplete }; +enum class Mode { Normal, Large, NoLength, Head, NotFound, Redirect, Auth, Reset, Truncate, StallHandshake, StallHeaders, StallBody, NoAlpn, Informational, StallUpload, RedirectOnce, EarlyFinal, EarlyReset, EarlyEmpty, EarlyLarge, EarlyResetIncomplete }; const char* Protocols[] = {"h2", nullptr}; class Server { public: - explicit Server(Mode mode) : mode(mode) { + explicit Server(Mode mode, std::string location = "/another", int redirectStatus = 302) + : mode(mode), location(std::move(location)), redirectStatus(redirectStatus) { mbedtls_net_init(&listener); mbedtls_entropy_init(&entropy); mbedtls_ctr_drbg_init(&random); @@ -114,6 +117,9 @@ public: bool IsEarly() const { return mode >= Mode::EarlyFinal; } private: const Mode mode; + const std::string location; + const int redirectStatus; + unsigned accepted = 0; mbedtls_net_context listener; mbedtls_entropy_context entropy; mbedtls_ctr_drbg_context random; @@ -185,10 +191,12 @@ private: if (mode == Mode::StallHeaders) { Stage(2); Stall(); return; } if (mode == Mode::Reset) { Require(nghttp2_submit_rst_stream(session, 0, stream, NGHTTP2_REFUSED_STREAM) == 0, "server reset"); return; } body = (mode == Mode::Large || mode == Mode::EarlyLarge) ? std::string(200000, 'x') : "response body"; - std::string status = IsEarly() ? "413" : mode == Mode::NotFound ? "404" : mode == Mode::Redirect ? "302" : mode == Mode::Auth ? "401" : "200"; + const bool redirect = mode == Mode::Redirect || (mode == Mode::RedirectOnce && accepted == 1); + std::string status = IsEarly() ? "413" : mode == Mode::NotFound ? "404" : redirect ? std::to_string(redirectStatus) : mode == Mode::Auth ? "401" : "200"; if (mode == Mode::EarlyEmpty) body.clear(); auto length = std::to_string(body.size()); - if (mode == Mode::Head) body.clear(); + const auto requestMethod = headers.find(":method"); + if (mode == Mode::Head || (requestMethod != headers.end() && requestMethod->second == "HEAD")) body.clear(); if (mode == Mode::Truncate) body = "short"; if (mode == Mode::Informational) { nghttp2_nv early{reinterpret_cast(const_cast(":status")), reinterpret_cast(const_cast("103")), 7, 3, 0}; @@ -196,7 +204,7 @@ private: } std::vector> fields{{":status", status}, {"x-loopback", "yes"}}; if (mode != Mode::NoLength) fields.emplace_back("content-length", length); - if (mode == Mode::Redirect) fields.emplace_back("location", "/another"); + if (redirect) fields.emplace_back("location", location); std::vector response; for (auto& [name, value] : fields) response.push_back({reinterpret_cast(name.data()), reinterpret_cast(value.data()), name.size(), value.size(), 0}); nghttp2_data_provider2 provider{}; @@ -205,6 +213,26 @@ private: } void Serve() { + const unsigned connections = mode == Mode::RedirectOnce ? 2 : 1; + while (accepted < connections) { + for (;;) { + const int ready = mbedtls_net_poll(&listener, MBEDTLS_NET_POLL_READ, 20); + Require(ready >= 0, "server accept poll"); + if (ready) break; + std::lock_guard lock(mutex); + if (released) return; + } + headers.clear(); + upload.clear(); + body.clear(); + offset = 0; + responded = ended = false; + ++accepted; + ServeConnection(); + } + } + + void ServeConnection() { mbedtls_net_context client; mbedtls_net_init(&client); mbedtls_ssl_context ssl; @@ -298,6 +326,174 @@ struct Client { ~Client() { if (context > 0) Require(sceHttp2Term(context) == 0, "client term"); } }; +struct RedirectObservation { + int calls = 0; + int request = 0; + int status = 0; + int result = 0; + std::string method, location; + const char* replacement = nullptr; + bool nullMethod = false; + bool removeRequest = false; + bool throwException = false; +}; + +int APS5_VABI RedirectCallback(int request, int status, const char** method, const char* location, void* argument) { + auto& observed = *static_cast(argument); + ++observed.calls; + observed.request = request; + observed.status = status; + observed.method = *method; + observed.location = location; + if (observed.replacement || observed.nullMethod) *method = observed.replacement; + if (observed.removeRequest) Require(sceHttp2DeleteRequest(request) == 0, "delete from redirect callback"); + if (observed.throwException) throw std::runtime_error("redirect callback failure"); + return observed.result; +} + +void Response(Client& client, int expectedStatus) { + int status = 0; + Require(sceHttp2GetStatusCode(client.request, &status) == 0 && status == expectedStatus, "redirect response status"); + std::string received; + std::array buffer; + for (;;) { + const int count = sceHttp2ReadData(client.request, buffer.data(), buffer.size()); + Require(count >= 0, "redirect response read"); + if (!count) break; + received.append(buffer.data(), static_cast(count)); + } + Require(received == "response body", "redirect preserves final response body"); +} + +void RedirectPolicies(int ssl) { + for (int policy = 0; policy < 6; ++policy) { + Server server(Mode::Redirect, policy == 4 ? "" : "/another"); + Client client(ssl); + RedirectObservation original, changed; + original.result = policy == 5 ? 7 : -1; + if (policy == 0) Require(sceHttp2SetRedirectCallback(client.tmpl, RedirectCallback, &original) == 0, "template redirect callback"); + client.Create(server.Url()); + if (policy == 0) Require(sceHttp2SetRedirectCallback(client.tmpl, RedirectCallback, &changed) == 0, "template callback mutation"); + else Require(sceHttp2SetRedirectCallback(client.request, RedirectCallback, &original) == 0, "request redirect callback"); + if (policy == 2) Require(sceHttp2SetAutoRedirect(client.request, 0) == 0, "manual redirects omit callback"); + if (policy == 3) { + Require(sceHttp2SetRedirectCallback(client.request, nullptr, &changed) == 0, "clear callback with argument"); + Require(sceHttp2SetAutoRedirect(client.request, 0) == 0, "manual cleared callback"); + } + Require(sceHttp2SendRequest(client.request, nullptr, 0) == 0, "redirect refusal response"); + Response(client, 302); + server.Join(); + const int expectedCalls = policy < 2 || policy == 5 ? 1 : 0; + Require(original.calls == expectedCalls && changed.calls == 0, "callback inheritance and policy"); + if (expectedCalls) Require(original.request == client.request && original.status == 302 && original.method == "GET" && original.location == "/another", "redirect callback arguments"); + } +} + +void RedirectFollow(int ssl, int status, const char* replacement = nullptr, bool clearCallback = false, bool schemeRelative = false) { + const bool head = replacement && std::string_view(replacement) == "HEAD"; + Server target(head ? Mode::Head : Mode::Normal); + std::string destination = target.Url(); + if (schemeRelative) destination.erase(0, 6); + Server redirect(Mode::Redirect, destination, status); + Client client(ssl, 1); + RedirectObservation observed; + observed.replacement = replacement; + Require(sceHttp2SetRedirectCallback(client.tmpl, RedirectCallback, &observed) == 0, "inherited follow callback"); + for (const auto* name : {"authorization", "proxy-authorization", "cookie", "origin", "referer"}) + Require(sceHttp2AddRequestHeader(client.tmpl, name, "origin-only", 0) == 0, "origin header"); + Require(sceHttp2AddRequestHeader(client.tmpl, "content-type", "application/octet-stream", 0) == 0, "body metadata"); + Require(sceHttp2AddRequestHeader(client.tmpl, "x-retained", "yes", 0) == 0, "retained header"); + const std::string body = "upload bytes"; + client.Create(redirect.Url(), "POST", body.size()); + if (clearCallback) Require(sceHttp2SetRedirectCallback(client.request, nullptr, &observed) == 0, "clear inherited callback"); + Require(sceHttp2SendRequest(client.request, body.data(), body.size()) == 0, "follow redirect"); + if (head) { + int result = 0; + Require(sceHttp2GetStatusCode(client.request, &result) == 0 && result == 200, "redirect HEAD status"); + char byte; + Require(sceHttp2ReadData(client.request, &byte, 1) == 0, "redirect HEAD has no response body"); + } else Response(client, 200); + redirect.Join(); + target.Join(); + const bool retrieve = status == 301 || status == 302 || status == 303; + const std::string method = replacement ? replacement : retrieve ? "GET" : "POST"; + Require(target.Headers().find(":method")->second == method, "redirect request method"); + Require(target.Headers().find(":path")->second == "/path?query=yes", "redirect fragment removed"); + Require(target.Upload() == (method == "GET" || method == "HEAD" ? "" : body), "redirect upload policy"); + Require(target.Headers().count("content-type") == (method == "GET" || method == "HEAD" ? 0u : 1u), "redirect body headers"); + for (const auto* name : {"authorization", "proxy-authorization", "cookie", "origin", "referer"}) + Require(target.Headers().count(name) == 0, "origin headers excluded at a new origin"); + Require(target.Headers().count("x-retained") == 1, "ordinary header retained"); + Require(target.Headers().find("content-length")->second == std::to_string(target.Upload().size()), "redirect wire content length"); + Require(observed.calls == (clearCallback ? 0 : 1), "follow callback count"); + if (!clearCallback) Require(observed.request == client.request && observed.status == status && observed.method == (retrieve ? "GET" : "POST") && observed.location == destination, "follow callback arguments"); +} + +void RelativeRedirects(int ssl) { + for (const auto& [reference, expected] : std::vector>{ + {"another", "/another"}, {"../another", "/another"}, {"/a/b/../c/.", "/a/c/"}, + {"?next=1", "/path?next=1"}, {"../a//b?x=/../z", "/a//b?x=/../z"}, + {"../../../../c", "/c"}, {".", "/"}, {"..", "/"}, {"./a/./b/../../c", "/c"}, + {"/a/%2e%2e/b", "/a/%2e%2e/b"}, {"g?y/./x", "/g?y/./x"}, {"#tag", "/path?query=yes"}}) { + Server server(Mode::RedirectOnce, reference, 307); + Client client(ssl, 1); + Require(sceHttp2AddRequestHeader(client.tmpl, "authorization", "same-origin", 0) == 0, "same origin authorization"); + client.Create(server.Url()); + Require(sceHttp2SendRequest(client.request, nullptr, 0) == 0, "relative redirect"); + Response(client, 200); + server.Join(); + Require(server.Headers().find(":path")->second == expected, "resolved relative reference"); + Require(server.Headers().count("authorization") == 1, "same origin retains authorization"); + } +} + +void RedirectRequestMethods(int ssl) { + for (const auto& [method, status] : std::vector>{{"PUT", 301}, {"PUT", 303}, {"GET", 307}, {"HEAD", 303}}) { + const bool head = std::string_view(method) == "HEAD"; + Server target(head ? Mode::Head : Mode::Normal); + Server redirect(Mode::Redirect, target.Url(), status); + Client client(ssl, 1); + const std::string body = head ? "" : "retained data"; + client.Create(redirect.Url(), method, body.size()); + Require(sceHttp2SendRequest(client.request, body.data(), body.size()) == 0, "redirect method semantics"); + if (head) { + int result = 0; + char byte; + Require(sceHttp2GetStatusCode(client.request, &result) == 0 && result == 200, "HEAD redirected status"); + Require(sceHttp2ReadData(client.request, &byte, 1) == 0, "HEAD redirected body"); + } else Response(client, 200); + redirect.Join(); + target.Join(); + Require(target.Headers().find(":method")->second == (status == 303 && !head ? "GET" : method), "redirect preserves or rewrites method"); + Require(target.Upload() == (status == 303 && !head ? "" : body), "redirect preserves or drops data"); + } +} + +void RedirectFailures(int ssl) { + for (int mode = 0; mode < 5; ++mode) { + Server server(Mode::Redirect, mode == 4 ? "http://localhost/" : "/another"); + Client client(ssl); + RedirectObservation observed; + observed.nullMethod = mode == 0; + observed.replacement = mode == 1 ? "bad method" : nullptr; + observed.removeRequest = mode == 2; + observed.throwException = mode == 3; + client.Create(server.Url()); + Require(sceHttp2SetRedirectCallback(client.request, RedirectCallback, &observed) == 0, "failing redirect callback"); + Require(Throws([&] { sceHttp2SendRequest(client.request, nullptr, 0); }), "unsupported redirect remains explicit"); + server.Join(); + Require(observed.calls == 1, "failure callback delivered once"); + if (observed.removeRequest) Require(sceHttp2DeleteRequest(client.request) == static_cast(0x817b1100), "callback deletion persists"); + } + std::vector> chain; + for (int index = 0; index < 7; ++index) + chain.push_back(std::make_unique(Mode::Redirect, chain.empty() ? "/unused" : chain.back()->Url())); + Client client(ssl, 1); + client.Create(chain.back()->Url()); + Require(Throws([&] { sceHttp2SendRequest(client.request, nullptr, 0); }), "bounded redirects"); + for (auto& server : chain) server->Join(); +} + void Success(int ssl, Mode mode, bool disableCa = false) { std::fprintf(stderr, "HTTP2 success scenario %d\n", static_cast(mode)); Server server(mode); @@ -533,10 +729,20 @@ int main(int argc, char** argv) { } else if (argc == 2 && std::string_view(argv[1]) == "--dns-first") { FirstNetworkRequest(ssl); #endif + } else if (argc == 2 && std::string_view(argv[1]) == "--redirects") { + RedirectPolicies(ssl); + for (const int status : {301, 302, 303, 307, 308}) RedirectFollow(ssl, status); + RedirectFollow(ssl, 302, "PATCH"); + RedirectFollow(ssl, 303, "HEAD"); + RedirectFollow(ssl, 307, nullptr, true); + RedirectFollow(ssl, 308, nullptr, false, true); + RedirectRequestMethods(ssl); + RelativeRedirects(ssl); + RedirectFailures(ssl); } else { Require(argc == 1, "test arguments"); for (const auto mode : {Mode::Normal, Mode::Large, Mode::NoLength, Mode::Head, Mode::NotFound, Mode::Redirect, Mode::Auth, Mode::Informational}) Success(ssl, mode); - for (const auto mode : {Mode::NoAlpn, Mode::Reset, Mode::Truncate, Mode::Redirect, Mode::Auth}) Failure(ssl, mode); + for (const auto mode : {Mode::NoAlpn, Mode::Reset, Mode::Truncate, Mode::Auth}) Failure(ssl, mode); Failure(ssl, Mode::Normal, true); for (const auto mode : {Mode::StallHandshake, Mode::StallHeaders, Mode::StallBody, Mode::StallUpload}) { Deadline(ssl, mode, false); diff --git a/docs/dev/debt/libSceHttp2.md b/docs/dev/debt/libSceHttp2.md index 633c67724e..36898138ef 100644 --- a/docs/dev/debt/libSceHttp2.md +++ b/docs/dev/debt/libSceHttp2.md @@ -10,7 +10,7 @@ - [libSceHttp2 pending exports](../../../core/libs/prx/libSceHttp2/Export.cpp) - 12 named callable exports were identified in the PS5 3.20 export table and cross-checked against firmware 13.60. Signatures and behavior remain unverified; declarations use the unknown-signature convention and throw through `NotImplemented_nid_no_patch`. Evidence: [3.20 export list](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceHttp2.c). ### Functional -- [libSceHttp2 transport](../../../core/libs/prx/libSceHttp2/Transport.cpp) - only synchronous HTTPS requests with version 3 and ALPN `h2` are supported. HTTP/1.x, cleartext HTTP/2, negotiation fallback, connection reuse, multiplexing across requests, CONNECT, response trailers, automatic redirects and authentication challenges throw. Version values, method strings, header overwrite/append modes, status/read/content-length signatures and timeout units follow [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Net/Http2.cs); wire framing and negotiation follow [RFC 9113 sections 3, 5, 6 and 8](https://www.rfc-editor.org/rfc/rfc9113.html). [prosper](https://github.com/mattias800/prosper/blob/b1bec57186a0/prosper/src/hle/net/hle_http2.cpp) instead records the method argument as an integer; the string signature follows SharpProspero. These are public compatibility contracts, not console measurements. +- [libSceHttp2 transport](../../../core/libs/prx/libSceHttp2/Transport.cpp) - only synchronous HTTPS requests with version 3 and ALPN `h2` are supported. HTTP/1.x, cleartext HTTP/2, negotiation fallback, connection reuse, multiplexing across requests, CONNECT, response trailers and authentication challenges throw. Version values, method strings, header overwrite/append modes, status/read/content-length signatures and timeout units follow [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Net/Http2.cs); wire framing and negotiation follow [RFC 9113 sections 3, 5, 6 and 8](https://www.rfc-editor.org/rfc/rfc9113.html). [prosper](https://github.com/mattias800/prosper/blob/b1bec57186a0/prosper/src/hle/net/hle_http2.cpp) instead records the method argument as an integer; the string signature follows SharpProspero. These are public compatibility contracts, not console measurements. - [libSceHttp2 error paths](../../../core/libs/prx/libSceHttp2/Transport.cpp) - cancellation, deadlines, DNS/TLS/protocol failures, malformed inputs, concurrent calls on one request and exhausted connection slots throw because their console error mappings are unknown. Socket connect errors propagate the libSceNet result, following the module-analysis report in [prosper #2894](https://github.com/mattias800/prosper/issues/2894); the report does not establish every exported error path. A request uses one connection; a context's concurrency cap counts connections until EOF, failure or deletion. Native pooling, scheduling, validation order and teardown semantics remain unverified. DNS uses two host workers and a queue of eight; cancellation/deadlines abandon the caller's wait, not libc's lookup. Library unload can wait for a host lookup to return. - [libSceHttp2 TLS settings](../../../core/libs/prx/libSceHttp2/Export.cpp) - minimum-version selection, nonnull SSL callbacks and flags outside `0x3d` throw. Supported certificate-check bits follow SharpProspero and use libSceSsl's TLS 1.2 backend and CA context; native defaults and error translation are unverified. Settings on object kinds other than templates/requests throw; native object-kind restrictions are unverified. Settings become immutable when transport is created. Zero timeout values, nonzero resolve retries, gzip inflation and connection-slot wait timeouts throw. Host HTTP allocations are not charged to the context pool; `sceHttp2GetMemoryPoolStats` throws for live contexts instead of reporting zero usage. - [sceHttp2GetAllResponseHeaders](../../../core/libs/prx/libSceHttp2/Export.cpp) throws: the serialization, size and lifetime of a decoded HTTP/2 header block have not been established. Status and Content-Length fields are decoded; absent Content-Length reports `NotFound`, following SharpProspero. @@ -20,3 +20,5 @@ - [libSceHttp2 automatic proxy configuration](../../../core/libs/prx/libSceHttp2/Export.cpp) - flags 0 and 1 both use direct connections, matching [libSceNetCtl](../../../core/libs/prx/libSceNetCtl/Export.cpp), which reports proxy configuration off, an empty server and port zero. Other flag values throw. This is the host configuration policy, not a console measurement of proxy discovery; PAC/WPAD and configured proxies remain unsupported. The boolean argument and acceptance of 1 by callers are evidenced by [OliSe Player](https://github.com/Marice/OliSePlayer/blob/fb06d7ac7467a50ba1f2ee299dac20e43213d674/src/netxm.c#L73) and [RetroArch](https://github.com/mihawk-99/PS5_RetroArch/blob/7268510b12fe7461342e0cc4497c87a0910c1055/src/webui_update.cpp#L305); call sites do not establish library behavior. - [libSceHttp2 default trust](../../../core/libs/prx/libSceHttp2/Transport.cpp) - HTTPS currently uses only authorities explicitly loaded into the caller's libSceSsl context; a fresh context cannot validate public servers. Verification remains enabled. Loading host roots is left to the shared per-connection SSL work in [#11134](https://github.com/boykopovar/AnyPS5/pull/11134) and the stacked HTTP2 follow-up [#11145](https://github.com/boykopovar/AnyPS5/pull/11145), rather than adding a second trust-store implementation here. The exact console root set, trust updates and pool accounting remain unverified. This transport alone does not enable stock HTTPS callers that load no roots. + +- [libSceHttp2 redirects](../../../core/libs/prx/libSceHttp2/Transport.cpp) - native caching, callback scheduling, target-reference delivery and method/body rewriting remain unverified. The typed callback, zero-follow/nonzero-refuse behavior and six-redirect cap follow [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Net/Http2.cs#L327-L334), not console measurements. Refusal exposes the received status/body. Method selection follows [RFC 9110 section 15.4](https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4); HTTPS reference resolution follows [RFC 3986 section 5.2](https://www.rfc-editor.org/rfc/rfc3986.html#section-5.2). Downgrade to HTTP, duplicate Location fields, invalid callback methods, redirect-limit exhaustion and changing active-request options throw; native error mappings are unknown. From dbf2107ba6276e2394bc7fddbfcf0d9a665ef68d Mon Sep 17 00:00:00 2001 From: QRMOE <217609482+QRMOE@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:41:43 +0100 Subject: [PATCH 167/318] fix(shader): load and store 16-bit data of converted packed images Converted 11_11_10 UNORM, 10_11_11 FLOAT and 11_11_10 FLOAT images are read and written through an R32_UINT view. A D16 store now widens each half to f32 before encoding it, and a D16 load returns the decoded components as halves instead of refusing the access. Co-Authored-By: Claude Opus 5.5 --- .../tests/execution/ImageConvertedUnorm.cpp | 202 +++++++++++++++++- .../Optimization/src/ResourceMaterializer.cpp | 6 +- .../SpirvBackend/src/SpirvImageEmitter.cpp | 20 +- docs/dev/debt/shader.md | 1 + 4 files changed, 215 insertions(+), 14 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp index bb50f103fd..97dfe2791d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp @@ -51,6 +51,8 @@ constexpr std::uint32_t DivisionUlps = 3; constexpr std::size_t BlockBytes = 65536; constexpr std::size_t RoundTripOffset = 16384; constexpr std::size_t StoreOffset = 32768; +constexpr std::uint32_t D16Width = 1024; +constexpr std::uint32_t D16Groups = D16Width / Threads; constexpr std::array PointSampler{0x00000092u, 0x00fff000u, 0u, 0u}; constexpr std::array BilinearSampler{0x00000092u, 0x00fff000u, 0x00500000u, 0u}; constexpr std::array WrapBilinearSampler{0x00000090u, 0x00fff000u, 0x00500000u, 0u}; @@ -72,6 +74,14 @@ constexpr std::array LoadCode(std::uint32_t dmask, std::uint3 }; } +constexpr std::array LoadStoreCode(std::uint32_t dmask, std::uint32_t d16) { + return { + 0x7e080218u, 0x343c0885u, 0x4a3c3d00u, 0x34063c84u, 0x7e3e0280u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0xf0001008u | (dmask << 8u), 0x00010a1eu | d16, 0xbf8c3f70u, 0xe0781000u, 0x80000a03u, 0xf0201008u | (dmask << 8u), 0x00010a1eu | d16, + 0xbf810000u, + }; +} + constexpr std::array StoreCode(std::uint32_t dmask, std::uint32_t d16) { return { 0x34060084u, 0xe0381000u, 0x80030a03u, 0x7e3c0300u, 0x7e3e0280u, 0xbf8c3f70u, 0xf0201008u | (dmask << 8u), 0x00010a1eu | d16, @@ -102,9 +112,11 @@ alignas(256) constexpr auto LoadXyzw = LoadCode(0xfu, 0u); alignas(256) constexpr auto LoadXyz = LoadCode(0x7u, 0u); alignas(256) constexpr auto LoadXy = LoadCode(0x3u, 0u); alignas(256) constexpr auto LoadD16 = LoadCode(0xfu, D16); +alignas(256) constexpr auto LoadStoreD16 = LoadStoreCode(0x7u, D16); alignas(256) constexpr auto StoreXyz = StoreCode(0x7u, 0u); alignas(256) constexpr auto StoreXz = StoreCode(0x5u, 0u); alignas(256) constexpr auto StoreD16 = StoreCode(0x7u, D16); +alignas(256) constexpr auto StoreXzD16 = StoreCode(0x5u, D16); alignas(256) constexpr auto SampleLz = SamplerCode(0xf09c0f08u); alignas(256) constexpr auto Gather4Lz = SamplerCode(0xf11c0108u); alignas(256) constexpr auto GetLod = SamplerCode(0xf1800308u); @@ -118,6 +130,28 @@ constexpr std::array StoreValues{ 0x40000000u, 0x7f800000u, 0x7fc00000u, 0x000116c2u, 0x3f7fbe77u, }; +constexpr std::array HalfStoreValues{ + 0x0000u, 0x3c00u, 0x3800u, 0x3555u, 0x8000u, 0xbc00u, + 0x4000u, 0x7c00u, 0x7e00u, 0x0001u, 0x3bffu, +}; + +std::uint32_t HalfToFloatBits(std::uint32_t half) { + const auto sign = (half & 0x8000u) << 16u; + const auto exponent = (half >> 10u) & 0x1fu; + const auto mantissa = half & 0x3ffu; + if (exponent == 31u) return sign | 0x7f800000u | (mantissa << 13u); + if (exponent != 0u) return sign | ((exponent + 112u) << 23u) | (mantissa << 13u); + return sign | std::bit_cast(static_cast(mantissa) * 0x1p-24f); +} + +bool HalfNan(std::uint32_t half) { + return (half & 0x7c00u) == 0x7c00u && (half & 0x3ffu) != 0u; +} + +std::uint32_t ResultHalf(std::uint32_t index, std::uint32_t component) { + return (Output[index * 4u + component / 2u] >> ((component % 2u) * 16u)) & 0xffffu; +} + class GuestBlock { public: GuestBlock() { @@ -252,17 +286,23 @@ void CheckRoundTrip(AgcDriver::VulkanDevice& device, std::uint8_t* texels) { } } -void CheckStore(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span code, std::uint32_t dmask, const std::string& what) { +void CheckStore(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span code, std::uint32_t dmask, const std::string& what, bool d16 = false) { std::fill(texels, texels + Threads * 4u, static_cast(0xa5u)); Input.fill(0u); std::array expected{}; + const auto& source = d16 ? HalfStoreValues : StoreValues; for (std::uint32_t tid = 0; tid < Threads; ++tid) { - const std::array values{StoreValues[tid % 11u], StoreValues[(tid + 4u) % 11u], StoreValues[(tid + 8u) % 11u]}; + const std::array values{source[tid % 11u], source[(tid + 4u) % 11u], source[(tid + 8u) % 11u]}; std::uint32_t slot = 0; for (std::uint32_t component = 0; component < 3u; ++component) { if (((dmask >> component) & 1u) == 0u) continue; - Input[tid * 4u + slot++] = values[component]; - expected[tid] |= StoreField(values[component], component == 2u ? 1023u : 2047u) << (component * 11u); + if (d16) { + Input[tid * 4u + slot / 2u] |= values[component] << ((slot % 2u) * 16u); + } else { + Input[tid * 4u + slot] = values[component]; + } + ++slot; + expected[tid] |= StoreField(d16 ? HalfToFloatBits(values[component]) : values[component], component == 2u ? 1023u : 2047u) << (component * 11u); } } Run(device, code, TextureDescriptor(texels, UnormFormat, SwizzleXYZ1, Threads), BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)), 1); @@ -431,6 +471,148 @@ void CheckSample(AgcDriver::VulkanDevice& device, const std::uint8_t* texels, st } } +void CheckUnormLoadD16(AgcDriver::VulkanDevice& device, const std::uint8_t* texels, std::uint32_t swizzle, const std::string& what) { + Run(device, LoadD16, TextureDescriptor(texels, UnormFormat, swizzle, LoadWidth), {}, LoadGroups); + for (std::uint32_t index = 0; index < LoadWidth; ++index) { + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto actual = ResultHalf(index, component); + const auto expected = ChannelValue(LoadTexel(index), swizzle, component); + const auto tolerance = expected == 0.0 || expected == 1.0 ? 0.0 : std::ldexp(1.0, std::ilogb(expected) - 10); + const auto value = static_cast(std::bit_cast(HalfToFloatBits(actual))); + Require((actual & 0x8000u) == 0u && std::fabs(value - expected) <= tolerance, what + ": texel " + std::to_string(index) + " component " + std::to_string(component) + " is " + Hex(actual) + ", expected " + std::to_string(expected) + " within one half ULP"); + } + Require(Output[index * 4u + 2u] == 0u && Output[index * 4u + 3u] == 0u, what + ": texel " + std::to_string(index) + " writes more than two result dwords"); + } +} + +struct FloatLayout { + std::uint32_t format; + std::array offsets; + std::array mantissas; + std::size_t loadOffset; + std::size_t storeOffset; + const char* name; +}; + +constexpr std::array FloatLayouts{{ + {FloatFormat, {0u, 10u, 21u}, {5u, 6u, 6u}, 49152, 57344, "R10_G11_B11_FLOAT"}, + {NativeFloatFormat, {0u, 11u, 22u}, {6u, 6u, 5u}, 53248, 58368, "R11_G11_B10_FLOAT"}, +}}; + +constexpr std::array, 8> HalfStoreEdges{{ + {0xbc00u, 0x8000u, 0x7e00u}, + {0x7c00u, 0x7bffu, 0xfc00u}, + {0x3c01u, 0x3c10u, 0x3c30u}, + {0x0001u, 0x03ffu, 0x0400u}, + {0xfe00u, 0x7c01u, 0x7dffu}, + {0x3bffu, 0x7800u, 0x7bf8u}, + {0x0010u, 0x0200u, 0x3c08u}, + {0x7bf0u, 0x1400u, 0x3c18u}, +}}; + +std::uint32_t SmallFloatField(std::uint32_t index, std::uint32_t component, std::uint32_t mantissa) { + constexpr std::array exponents{0u, 0u, 0u, 1u, 15u, 30u, 31u, 31u}; + const auto maximum = (1u << mantissa) - 1u; + const std::array mantissas{0u, 1u, maximum, 0u, 0u, maximum, 0u, 1u}; + const auto special = (index + component * 3u) % 8u; + if (index < 64u) return (exponents[special] << mantissa) | mantissas[special]; + return ((index * 2654435761u) >> (component * 7u + 3u)) & ((1u << (mantissa + 5u)) - 1u); +} + +std::uint32_t ExactHalf(float value) { + if (value == 0.0f) return 0u; + const auto bits = std::bit_cast(value); + return ((((bits >> 23u) & 0xffu) - 112u) << 10u) | ((bits & 0x7fffffu) >> 13u); +} + +void CheckFloatLoadD16(AgcDriver::VulkanDevice& device, std::uint8_t* block, const FloatLayout& layout, bool storage) { + auto* texels = block + layout.loadOffset; + const auto texelAt = [&](std::uint32_t index) { + std::uint32_t texel = 0; + for (std::uint32_t component = 0; component < 3u; ++component) texel |= SmallFloatField(index, component, layout.mantissas[component]) << layout.offsets[component]; + return texel; + }; + for (std::uint32_t index = 0; index < D16Width; ++index) { + const auto texel = texelAt(index); + std::memcpy(texels + index * 4u, &texel, 4u); + } + const auto what = std::string(storage ? "image_load d16 then image_store d16 of " : "image_load d16 of ") + layout.name; + Run(device, storage ? std::span(LoadStoreD16) : std::span(LoadD16), TextureDescriptor(texels, layout.format, SwizzleXYZ1, D16Width), {}, D16Groups); + for (std::uint32_t index = 0; index < D16Width; ++index) { + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto actual = ResultHalf(index, component); + auto expected = storage ? 0u : 0x3c00u; + if (component < 3u) { + const auto mantissa = layout.mantissas[component]; + const auto field = SmallFloatField(index, component, mantissa); + expected = ((field >> mantissa) << 10u) | ((field & ((1u << mantissa) - 1u)) << (10u - mantissa)); + } + Require(HalfNan(expected) ? HalfNan(actual) : actual == expected, what + ": texel " + std::to_string(index) + " component " + std::to_string(component) + " is " + Hex(actual) + ", expected " + Hex(expected)); + } + Require(Output[index * 4u + 2u] == 0u && Output[index * 4u + 3u] == 0u, what + ": texel " + std::to_string(index) + " writes more than two result dwords"); + } + if (!storage) return; + AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast(texels), D16Width * 4u, nullptr, "test"); + device.WaitIdle(); + for (std::uint32_t index = 0; index < D16Width; ++index) { + std::uint32_t actual = 0; + std::memcpy(&actual, texels + index * 4u, 4u); + for (std::uint32_t component = 0; component < 3u; ++component) { + const auto mantissa = layout.mantissas[component]; + const auto mask = (1u << (mantissa + 5u)) - 1u; + const auto field = (actual >> layout.offsets[component]) & mask; + const auto expected = SmallFloatField(index, component, mantissa); + const auto nan = [&](std::uint32_t value) { return (value >> mantissa) == 31u && (value & ((1u << mantissa) - 1u)) != 0u; }; + Require(nan(expected) ? nan(field) : field == expected, what + ": texel " + std::to_string(index) + " is " + Hex(actual) + " after the round trip, expected " + Hex(texelAt(index))); + } + } +} + +void CheckFloatStoreD16(AgcDriver::VulkanDevice& device, std::uint8_t* block, const FloatLayout& layout, std::uint32_t dmask) { + auto* written = block + layout.storeOffset; + auto* reference = written + 256u; + std::array, Threads> halves{}; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + halves[tid] = tid < HalfStoreEdges.size() ? HalfStoreEdges[tid] : std::array{ExactHalf(FloatValues[tid % 11u]), ExactHalf(FloatValues[(tid + 4u) % 11u]), ExactHalf(FloatValues[(tid + 8u) % 11u])}; + } + const auto store = [&](std::uint8_t* target, std::span code, bool d16) { + std::fill(target, target + Threads * 4u, static_cast(0xa5u)); + Input.fill(0u); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + std::uint32_t slot = 0; + for (std::uint32_t component = 0; component < 3u; ++component) { + if (((dmask >> component) & 1u) == 0u) continue; + if (d16) { + Input[tid * 4u + slot / 2u] |= halves[tid][component] << ((slot % 2u) * 16u); + } else { + Input[tid * 4u + slot] = HalfToFloatBits(halves[tid][component]); + } + ++slot; + } + } + Run(device, code, TextureDescriptor(target, layout.format, SwizzleXYZ1, Threads), BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)), 1); + AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast(target), Threads * 4u, nullptr, "test"); + device.WaitIdle(); + }; + store(reference, dmask == 0x7u ? std::span(StoreXyz) : std::span(StoreXz), false); + store(written, dmask == 0x7u ? std::span(StoreD16) : std::span(StoreXzD16), true); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + std::uint32_t actual = 0; + std::uint32_t widened = 0; + std::memcpy(&actual, written + tid * 4u, 4u); + std::memcpy(&widened, reference + tid * 4u, 4u); + const auto what = std::string("image_store d16 dmask:") + Hex(dmask) + " of " + layout.name + ": texel " + std::to_string(tid); + if (tid >= HalfStoreEdges.size()) { + std::uint32_t expected = 0; + for (std::uint32_t component = 0; component < 3u; ++component) { + if (((dmask >> component) & 1u) != 0u) expected |= SmallFloat(FloatValues[(tid + component * 4u) % 11u], layout.mantissas[component]) << layout.offsets[component]; + } + Require(actual == expected, what + " is " + Hex(actual) + ", expected " + Hex(expected)); + } + Require(actual == widened, what + " is " + Hex(actual) + ", but the 32-bit store of the same values wrote " + Hex(widened)); + } +} + void CheckConvertedPreparation(AgcDriver::VulkanDevice& device, const std::uint8_t* texels) { std::vector userData(24, 0u); const auto texture = TextureDescriptor(texels, UnormFormat, SwizzleXYZ1, LoadWidth); @@ -598,8 +780,16 @@ int main(int argc, char** argv) { RequireRefused(*device, SampleLz, texels, SwizzleXYZ1, BorderSampler, "implemented only with wrap or clamp-to-edge addressing", "image_sample_lz with clamp-to-border addressing"); RequireRefused(*device, Gather4Lz, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_gather4_lz"); RequireRefused(*device, GetLod, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_get_lod"); - RequireRefused(*device, LoadD16, texels, SwizzleXYZ1, {}, "converted unorm image with 16-bit data", "image_load d16"); - RequireRefused(*device, StoreD16, texels + StoreOffset, SwizzleXYZ1, BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)), "converted unorm image with 16-bit data", "image_store d16"); + for (const auto& layout : FloatLayouts) { + CheckFloatStoreD16(*device, texels, layout, 0x7u); + CheckFloatStoreD16(*device, texels, layout, 0x5u); + CheckFloatLoadD16(*device, texels, layout, true); + CheckFloatLoadD16(*device, texels, layout, false); + } + CheckUnormLoadD16(*device, texels, SwizzleXYZ1, "image_load d16 X Y Z 1"); + CheckUnormLoadD16(*device, texels, SwizzleZYX1, "image_load d16 Z Y X 1"); + CheckStore(*device, texels + StoreOffset, StoreD16, 0x7u, "image_store d16 dmask:0x7", true); + CheckStore(*device, texels + StoreOffset, StoreXzD16, 0x5u, "image_store d16 dmask:0x5", true); RequireRefused(*device, LoadXyzw, texels, SwizzleXYZW, {}, "selects a channel the converted image format does not have", "image_load with DST_SEL X Y Z W"); RequireRefused(*device, StoreXyz, texels + StoreOffset, SwizzleXYZW, BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)), "selects a channel the converted image format does not have", "image_store with DST_SEL X Y Z W"); std::puts("image converted unorm tests passed"); diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 10202666a4..e7c8a7173d 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -273,8 +273,8 @@ ImageDecode inspectImageDescriptor(const DescriptorValue& descriptor, const Imag if (base.table != NoTable) return rejectImage(PoisonReason::Conversion, "samples a converted unorm image selected through an image table, which is not implemented"); if (decoded.cube || (decoded.dimension != RdnaImageDimension::Dim2D && decoded.dimension != RdnaImageDimension::Dim2DArray)) return rejectImage(PoisonReason::Conversion, "samples a converted unorm image that is not a 2D or 2D array view, which is not implemented"); if ((base.emulatedCompare & EmulatedCompare::ConvertedSampleRequiresSingleLevel) != 0u && ((descriptor.dwords[3] >> 12u) & 0xfu) != ((descriptor.dwords[3] >> 16u) & 0xfu)) return rejectImage(PoisonReason::Conversion, "samples a converted unorm image with several mip levels at a LOD that is not 0, which is not implemented"); + if (!base.depthBitsCompatible) return rejectImage(PoisonReason::Conversion, "samples a converted unorm image with 16-bit data, which is not implemented"); } - if (!base.depthBitsCompatible) return rejectImage(PoisonReason::Conversion, floating ? "reads or writes a converted float image with 16-bit data, which is not implemented" : "reads or writes a converted unorm image with 16-bit data, which is not implemented"); for (std::uint32_t component = 0; component < 4u; ++component) { if (((descriptorImageSwizzle(descriptor) >> (component * 3u)) & 7u) == 7u) return rejectImage(PoisonReason::Conversion, "selects a channel the converted image format does not have"); } @@ -1998,8 +1998,8 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe append(IrTextureNumericClass::Uint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); if (!storage) append(IrTextureNumericClass::Sint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UInt, IrBufferFormat::Invalid, false, false); - if ((image.srgbDecodeCompatible || (image.convertedSamples && convertedSampleCompatible(image))) && image.depthBitsCompatible) append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false); - if (image.srgbDecodeCompatible && image.depthBitsCompatible) { + if (image.srgbDecodeCompatible || (image.convertedSamples && convertedSampleCompatible(image))) append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false); + if (image.srgbDecodeCompatible) { append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10Float, IrBufferFormat::Invalid, false, false); append(IrTextureNumericClass::Uint, IrBufferFormat::Format10_11_11Float, IrBufferFormat::Invalid, false, false); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index cd5552084d..75612fb29f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -551,9 +551,6 @@ SpirvBufferFormatInfo ImageConversionFormat(const ImageResource& image) { } void RequireConvertedUnormAccess(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, const SpirvBufferFormatInfo& info) { - if (access.mem.dataBits == 16u && info.byteSize != 2u) { - ctx.Fail(access.inst, info.type == SpirvFormatComponentType::Float ? "reads or writes a converted float image with 16-bit data, which is not implemented" : "reads or writes a converted unorm image with 16-bit data, which is not implemented"); - } for (std::uint32_t component = 0; component < 4u; component++) { const auto selector = (access.image.shaderSwizzle >> (component * 3u)) & 7u; if (selector >= 4u && selector - 4u >= info.componentCount && info.byteSize != 2u) { @@ -635,6 +632,19 @@ std::uint32_t UnpackImageTexel(SpirvValueEmitContext& ctx, const ImageEmitAccess return result; } +std::uint32_t LoadedTexelResult(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, std::uint32_t texel, IrTextureNumericClass numericClass) { + auto& state = ctx.state; + const auto unpacked = UnpackImageTexel(ctx, access, texel); + if (access.mem.dataBits != 16u || access.image.srgbDecode || access.image.conversionFormat == FmaskExpansion) { + return ResultVector(ctx, access, unpacked, numericClass, false, false); + } + const auto info = ImageConversionFormat(access.image); + if (info.format == IrBufferFormat::Invalid || (info.type != SpirvFormatComponentType::Unorm && info.type != SpirvFormatComponentType::Float)) { + return ResultVector(ctx, access, unpacked, numericClass, false, false); + } + return ResultVector(ctx, access, Unary(state, spv::OpBitcast, TypeF32Vector(state, 4), unpacked), IrTextureNumericClass::Float, false, false); +} + SpirvBufferFormatInfo PackedTexelFormat(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { const auto format = access.image.packedFormat; if (format == IrBufferFormat::Invalid || access.image.table != NoTable || access.image.depthBits || access.image.conversionFormat != IrBufferFormat::Invalid) { @@ -1013,7 +1023,7 @@ void EmitByReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { words[0] = Binary(state, spv::OpBitwiseOr, TypeU32(state), words[0], Select(state, TypeU32(state), inside, shifted, ConstantU32(state, 0))); continue; } - const auto texel = ResultVector(ctx, texelAccess, UnpackImageTexel(ctx, texelAccess, color), access.image.numericClass, false, false); + const auto texel = LoadedTexelResult(ctx, texelAccess, color, access.image.numericClass); for (std::uint32_t component = 0; component < components; ++component) { const auto value = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, texel, component); @@ -1057,7 +1067,7 @@ void EmitReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { if (access.mem.imagePacked) { return PackedImageTexel(ctx, access, color); } - return ResultVector(ctx, access, UnpackImageTexel(ctx, access, color), numericClass, false, false); + return LoadedTexelResult(ctx, access, color, numericClass); })); } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index e7aba24b21..9184ed79cb 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -114,6 +114,7 @@ - [v_floor_f64, v_ceil_f64, v_fract_f64, v_frexp_mant_f64 and v_frexp_exp_i32_f64](../../../core/shader/recompiler/Translation/src/Float64Instructions.cpp) follow the `FLOAT_MODE` f16/f64 denormal field: a denormal input is flushed to a zero of its sign when bit 0 is clear (0 and 2), and `v_fract_f64` flushes a denormal result, the fraction of a positive denormal input, to +0 when bit 1 is clear (0 and 1). `v_trunc_f64` and `v_rndne_f64` give the same results either way and are unchanged. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with denorm16 0..3 over 23 f64 values (zeros, denormals, minimum normals, ±0.5, ±1, values next to 1, the largest finite value, infinities and quiet and signaling NaNs), and with neg and abs modifiers over 16 values (13 of those, -1.5, 2.75 and 1 - 2^-53). Shaders without a known float mode keep the previous lowering. The PS5's gfx1013 is not measured. - [v_add_f64 and v_mul_f64 denormals](../../../core/shader/recompiler/Translation/src/Float64Instructions.cpp) follow the `FLOAT_MODE` f16/f64 denormal field; the PS5's gfx1013 is not measured. Inputs flush to a zero of their sign when bit 0 is clear (0 and 2). When bit 1 is clear (0 and 1), a sum below 2^-1022, or a product whose exact value rounded to 53 significant bits with an unbounded exponent is below 2^-1022, flushes to a zero of its sign, so `0x2000000000000001 * 0x1ffffffffffffffe` (exactly 2^-1022 - 2^-1126) rounds to 2^-1022 and is kept. Measured on RDNA2 (gfx1031, RX 6700 XT, AMD Windows driver 32.0.21045.5002, HIP) at IEEE=0 and 1, denorm16 0..3, over 576 pairs of 24 f64 values (zeros, denormals, minimum normals, values next to 1 and to 2^-1022, products just below 2^-1022, the largest finite value, infinities), with neg, abs and clamp. `v_fma_f64`, `v_div_fmas_f64` and shaders without a known float mode keep the previous lowering. - [SINT storage images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8- and 16-bit components saturate each stored 32-bit value to the component range, as adriadam10 measured for image stores on RDNA2 (gfx1035) in #944 (448 rows match); the PS5's gfx1013 is not measured. A D16 store sign-extends each 16-bit half before that saturation, as measured on RDNA2 (gfx1035) for `image_store d16` to all eight SINT formats with DMASK 0xf, 0x7, 0x5, 0x3, 0x2, 0xa and 0x1 (1792 rows match). +- [D16 accesses of converted images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11 UNORM, and 11_11_10 or 10_11_11 FLOAT written through an R32_UINT storage view) are not measured on hardware. A D16 `image_store` widens each half to f32, which is exact, and encodes it as the 32-bit store does. A D16 `image_load` decodes the texel to f32 as the 32-bit load does and rounds each component to f16 to nearest even, so a UNORM component is rounded twice; the float formats fit f16 exactly, except for NaN payloads. `image_load` of the float formats reads the native B10G11R11 view and does not take this path. - [`StoreTexel`](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) writes channel 1 of a two-channel format from the data slot whose DST_SEL is Y, for SINT and UINT alike. With DST_SEL X,Y,Z,W, gfx1035 writes it from the W slot, as adriadam10 measured in #944. - [v_ldexp_f16](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the `FLOAT_MODE` f16 denormal field: bit 0 preserves the half input and bit 1 the half result. Otherwise a denormal input is flushed to its signed zero, and a result whose exact scaled value is below the minimum normal (2^-14) is flushed to its signed zero before rounding, so `v_ldexp_f16 0x07ff, -1` gives +0 instead of 0x0400. Measured on RDNA2 (gfx1036) with round to nearest, IEEE 0 and 1 and denorm16 0..3, over every half magnitude 0x0000..0x0fff of both signs scaled by 2^-1 and 2^0 (16384 rows per mode): no mismatch. NaN and infinity handling and destination high-half preservation are unchanged. Shaders without a known float mode keep the previous lowering. - [v_div_fixup_f64](../../../core/shader/recompiler/Translation/src/Float64Instructions.cpp) flushes its src1 and src2 to a zero of the same sign when bit 0 of the `FLOAT_MODE` f16/f64 denormal field is clear (0 and 2) and keeps a denormal src0 and result in every mode, as `v_div_fixup_f32` does with the f32 field. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) over 1728 triples of 12 f64 values (zeros, denormals, the minimum normal, ±1, the largest finite value, infinity and quiet and signaling NaNs) at IEEE=0 and IEEE=1 with denorm16 0..3. Shaders without a known float mode keep the previous lowering. The PS5's gfx1013 is not measured. From d63e49e368da9c2414da06b72e8767a76f3b124f Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:27 +0000 Subject: [PATCH 168/318] feat(libSceDbg): declare 29 pending exports --- core/libs/prx/libSceDbg/Export.cpp | 147 ++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceDbg/Export.cpp b/core/libs/prx/libSceDbg/Export.cpp index d81f4a5e32..a90b17e6e3 100644 --- a/core/libs/prx/libSceDbg/Export.cpp +++ b/core/libs/prx/libSceDbg/Export.cpp @@ -1,10 +1,153 @@ -#include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" extern "C" { -APS5_DUMMY_FUN +int APS5_VABI sceDbgUserChannelPrint(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgAmprAprGetNumberOfErrors(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgMapperVirtualQuery(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgUserChannelPrintf(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgAssertionHandler(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgGetSystemSwVersion(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgSetBreakOnWarningState(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgSetBreakOnErrorState(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgDeleteExternalTriggerEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgAmprAmmGetNumberOfErrors(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgGetModuleInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgBacktraceSelf(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgAmprAmmGetErrors(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgGetDebugSuspendCount(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgUserChannelVPrintf(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgSetMinimumLogLevel(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgGetModuleList(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgClearHardwareBreakPoint(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgAddExternalTriggerEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgLoggingHandler(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgEnableExtraHeapDebugInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgAmprAprGetErrors(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgVLoggingHandler(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgGetReleaseCheckMode(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgSetHardwareBreakPoint(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgGetExecutablePath(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgGetCpuGpuFrequencySetting(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgRequireSystemSwVersion(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDbgIsDebuggerAttached(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } From 13c9e6fe6771919674e8bc8b6a234a26fce12678 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:27 +0000 Subject: [PATCH 169/318] docs(libSceDbg): record 29 pending exports --- docs/dev/debt/libSceDbg.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 docs/dev/debt/libSceDbg.md diff --git a/docs/dev/debt/libSceDbg.md b/docs/dev/debt/libSceDbg.md new file mode 100644 index 0000000000..a210cc5bae --- /dev/null +++ b/docs/dev/debt/libSceDbg.md @@ -0,0 +1,35 @@ +# libSceDbg technical debt + +### Unknown function info + +The 29 exports in [Export.cpp](../../../core/libs/prx/libSceDbg/Export.cpp) come from the `libSceDbg` list in [claimore22/ps5rs](https://github.com/claimore22/ps5rs/blob/5f252a658ff470bcb2b6cd15205fd8ca64ae4477/data/stubs/by_library/libSceDbg_nosubmission.txt) (commit 5f252a658ff470bcb2b6cd15205fd8ca64ae4477), a catalogue of PS5 symbols per library whose provenance the project does not document, and no PS5 export inventory of this module is available to confirm the list is complete or exact. Every name was kept only because its hash equals the listed NID. Prototypes and behaviour are unverified: every function takes no parameters in the import-resolution convention and throws through `NotImplemented_nid_no_patch`. + +- [sceDbgUserChannelPrint](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID 1Qy1jn47QWA) +- [sceDbgAmprAprGetNumberOfErrors](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID 4F7uYHxulZ4) +- [sceDbgMapperVirtualQuery](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID 4OqKr5iYCqI) +- [sceDbgUserChannelPrintf](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID 5huYgqy73xo) +- [sceDbgAssertionHandler](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID 5sBISBR0ooA) +- [sceDbgGetSystemSwVersion](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID B4c4qfm3m2c) +- [sceDbgSetBreakOnWarningState](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID BxQv1DSUaeY) +- [sceDbgSetBreakOnErrorState](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID CXoCR22n86Q) +- [sceDbgDeleteExternalTriggerEvent](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID Fhc+DypKzcU) +- [sceDbgAmprAmmGetNumberOfErrors](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID FvI50-3hoSo) +- [sceDbgGetModuleInfo](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID G4c6av2TDtc) +- [sceDbgBacktraceSelf](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID GDX5XhnWKrw) +- [sceDbgAmprAmmGetErrors](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID HNRCEumA2V8) +- [sceDbgGetDebugSuspendCount](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID He3luzgOL-w) +- [sceDbgUserChannelVPrintf](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID MltFJdH0xRM) +- [sceDbgSetMinimumLogLevel](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID OaQI1HqFAtk) +- [sceDbgGetModuleList](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID PvIP303tQx4) +- [sceDbgClearHardwareBreakPoint](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID Q-m9mL4Yznw) +- [sceDbgAddExternalTriggerEvent](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID SxSEAtl-vow) +- [sceDbgLoggingHandler](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID To9mmGL+3G8) +- [sceDbgEnableExtraHeapDebugInfo](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID WBQJDbBxH3E) +- [sceDbgAmprAprGetErrors](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID ddYgzamMD2U) +- [sceDbgVLoggingHandler](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID ie2v2DfofTs) +- [sceDbgGetReleaseCheckMode](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID jaYypweDbyA) +- [sceDbgSetHardwareBreakPoint](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID kwELJE1qN0o) +- [sceDbgGetExecutablePath](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID ta-oGegHdTQ) +- [sceDbgGetCpuGpuFrequencySetting](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID uhJ+bSomPoE) +- [sceDbgRequireSystemSwVersion](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID v6OElL4DxCo) +- [sceDbgIsDebuggerAttached](../../../core/libs/prx/libSceDbg/Export.cpp) (libSceDbg) - unknown signature (NID v9ERBDafV60) From 6a22fe7c483636f9a980abdc3408367857ed4159 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:27 +0000 Subject: [PATCH 170/318] feat(libSceDeci5): declare 7 pending exports --- core/libs/prx/libSceDeci5/Export.cpp | 37 ++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceDeci5/Export.cpp b/core/libs/prx/libSceDeci5/Export.cpp index d81f4a5e32..64b88a0ef5 100644 --- a/core/libs/prx/libSceDeci5/Export.cpp +++ b/core/libs/prx/libSceDeci5/Export.cpp @@ -1,10 +1,43 @@ -#include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" extern "C" { -APS5_DUMMY_FUN +int APS5_VABI sceDeci5Write(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDeci5Read(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDeci5CreateHostProcessAndWait(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDeci5Close(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDeci5CreateHostProcess(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDeci5Open(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceDeci5GetStatus(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } From 46baa3b51c5d986e34af33057aa0be5a71728f2f Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:27 +0000 Subject: [PATCH 171/318] docs(libSceDeci5): record 7 pending exports --- docs/dev/debt/libSceDeci5.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/dev/debt/libSceDeci5.md diff --git a/docs/dev/debt/libSceDeci5.md b/docs/dev/debt/libSceDeci5.md new file mode 100644 index 0000000000..72dc8f378e --- /dev/null +++ b/docs/dev/debt/libSceDeci5.md @@ -0,0 +1,13 @@ +# libSceDeci5 technical debt + +### Unknown function info + +The 7 exports in [Export.cpp](../../../core/libs/prx/libSceDeci5/Export.cpp) come from the `libSceDeci5` list in [claimore22/ps5rs](https://github.com/claimore22/ps5rs/blob/5f252a658ff470bcb2b6cd15205fd8ca64ae4477/data/stubs/by_library/libSceDeci5_nosubmission.txt) (commit 5f252a658ff470bcb2b6cd15205fd8ca64ae4477), a catalogue of PS5 symbols per library whose provenance the project does not document, and no PS5 export inventory of this module is available to confirm the list is complete or exact. Every name was kept only because its hash equals the listed NID. Prototypes and behaviour are unverified: every function takes no parameters in the import-resolution convention and throws through `NotImplemented_nid_no_patch`. + +- [sceDeci5Write](../../../core/libs/prx/libSceDeci5/Export.cpp) (libSceDeci5) - unknown signature (NID 5IVqeThOi2s) +- [sceDeci5Read](../../../core/libs/prx/libSceDeci5/Export.cpp) (libSceDeci5) - unknown signature (NID 8T8VUmTIKE0) +- [sceDeci5CreateHostProcessAndWait](../../../core/libs/prx/libSceDeci5/Export.cpp) (libSceDeci5) - unknown signature (NID Ez2iQHniESc) +- [sceDeci5Close](../../../core/libs/prx/libSceDeci5/Export.cpp) (libSceDeci5) - unknown signature (NID gYg5m+zII9w) +- [sceDeci5CreateHostProcess](../../../core/libs/prx/libSceDeci5/Export.cpp) (libSceDeci5) - unknown signature (NID nS4i1N-HEws) +- [sceDeci5Open](../../../core/libs/prx/libSceDeci5/Export.cpp) (libSceDeci5) - unknown signature (NID sLfxYROuNZU) +- [sceDeci5GetStatus](../../../core/libs/prx/libSceDeci5/Export.cpp) (libSceDeci5) - unknown signature (NID v-VB0EguQWg) From b0f4b5137f30646f68b0c1928404c000e605fda8 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:27 +0000 Subject: [PATCH 172/318] feat(libSceGpuTrace): declare 8 pending exports --- core/libs/prx/libSceGpuTrace/Export.cpp | 42 +++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceGpuTrace/Export.cpp b/core/libs/prx/libSceGpuTrace/Export.cpp index d81f4a5e32..efe9941cce 100644 --- a/core/libs/prx/libSceGpuTrace/Export.cpp +++ b/core/libs/prx/libSceGpuTrace/Export.cpp @@ -1,10 +1,48 @@ -#include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" extern "C" { -APS5_DUMMY_FUN +int APS5_VABI sceGpuTraceParametersSetDuration(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceGpuTraceParametersInit(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceGpuTraceParametersSetMemorySize(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceGpuTraceParametersSetGroup(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceGpuTraceStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceGpuTraceStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceGpuTraceMonitorTrigger(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceGpuTraceCancel(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } From 9b27bfdb24f1b01176cff9a09c86194fe753dfe2 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:27 +0000 Subject: [PATCH 173/318] docs(libSceGpuTrace): record 8 pending exports --- docs/dev/debt/libSceGpuTrace.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/dev/debt/libSceGpuTrace.md diff --git a/docs/dev/debt/libSceGpuTrace.md b/docs/dev/debt/libSceGpuTrace.md new file mode 100644 index 0000000000..fa25c339f5 --- /dev/null +++ b/docs/dev/debt/libSceGpuTrace.md @@ -0,0 +1,14 @@ +# libSceGpuTrace technical debt + +### Unknown function info + +The 8 exports in [Export.cpp](../../../core/libs/prx/libSceGpuTrace/Export.cpp) come from the `libSceGpuTrace` list in [claimore22/ps5rs](https://github.com/claimore22/ps5rs/blob/5f252a658ff470bcb2b6cd15205fd8ca64ae4477/data/stubs/by_library/libSceGpuTrace_nosubmission.txt) (commit 5f252a658ff470bcb2b6cd15205fd8ca64ae4477), a catalogue of PS5 symbols per library whose provenance the project does not document, and no PS5 export inventory of this module is available to confirm the list is complete or exact. Every name was kept only because its hash equals the listed NID. Prototypes and behaviour are unverified: every function takes no parameters in the import-resolution convention and throws through `NotImplemented_nid_no_patch`. + +- [sceGpuTraceParametersSetDuration](../../../core/libs/prx/libSceGpuTrace/Export.cpp) (libSceGpuTrace) - unknown signature (NID GLFNgsekcJU) +- [sceGpuTraceParametersInit](../../../core/libs/prx/libSceGpuTrace/Export.cpp) (libSceGpuTrace) - unknown signature (NID IQl1M8Gynsw) +- [sceGpuTraceParametersSetMemorySize](../../../core/libs/prx/libSceGpuTrace/Export.cpp) (libSceGpuTrace) - unknown signature (NID QDMpdglDPoo) +- [sceGpuTraceParametersSetGroup](../../../core/libs/prx/libSceGpuTrace/Export.cpp) (libSceGpuTrace) - unknown signature (NID TehfTHUiaag) +- [sceGpuTraceStart](../../../core/libs/prx/libSceGpuTrace/Export.cpp) (libSceGpuTrace) - unknown signature (NID WpNcYUoJe74) +- [sceGpuTraceStop](../../../core/libs/prx/libSceGpuTrace/Export.cpp) (libSceGpuTrace) - unknown signature (NID rxXvAxEbq8I) +- [sceGpuTraceMonitorTrigger](../../../core/libs/prx/libSceGpuTrace/Export.cpp) (libSceGpuTrace) - unknown signature (NID t+2V-Erb8Bg) +- [sceGpuTraceCancel](../../../core/libs/prx/libSceGpuTrace/Export.cpp) (libSceGpuTrace) - unknown signature (NID ubB+tVwd4-U) From b6ba6c8cf7c7abb14248b0cd141f95809d177e72 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:27 +0000 Subject: [PATCH 174/318] feat(libSceMat): declare 52 pending exports --- core/libs/prx/libSceMat/Export.cpp | 260 ++++++++++++++++++++++++++++- 1 file changed, 259 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceMat/Export.cpp b/core/libs/prx/libSceMat/Export.cpp index fb34206c21..70c77005c9 100644 --- a/core/libs/prx/libSceMat/Export.cpp +++ b/core/libs/prx/libSceMat/Export.cpp @@ -5,6 +5,264 @@ extern "C" { -APS5_DUMMY_FUN +int APS5_VABI sceMatMemoryPoolReserve(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatUninitialize(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatTagVirtualMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMspaceCreate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatUnregisterResource(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMprotect(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMapDirectMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatInitialize(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatNewFrame(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatRegisterResourceType(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatFree(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatSetUserCallstackCapture(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMemoryPoolMove(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatRegisterGroup(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMemoryPoolExpand(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMemoryPoolBatch(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatAllocPoolMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatAgcUnregisterOwnerAndResources(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatReleasePhysicalMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMemoryPoolDecommit(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatSetPrtAperture(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatFreePoolMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatUnmapMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMspaceDestroy(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatAlloc(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatReallocBegin(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatReallocEnd(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatAgcRegisterResource(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatWriteBookmark(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatBatchMap(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatTagPool(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatAgcRegisterOwner(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatUnregisterAllResourcesForOwner(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMtypeprotect(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatSetThreadInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatPushMarker(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatUnregisterOwnerAndResources(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatAgcUnregisterResource(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatRegisterOwner(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatAgcUnregisterAllResourcesForOwner(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatPopMarker(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatPushMarkerStatic(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatUpdateModuleList(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatResizeCommsBuffer(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatReallocPoolMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatMemoryPoolCommit(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatRegisterResource(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatAllocPhysicalMemory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatReserveVirtualRange(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatTagAllocation(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatFreeRange(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceMatCalloc(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } From 84523713ffe7e17fccdcf326c62942163f2cb439 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:28 +0000 Subject: [PATCH 175/318] docs(libSceMat): record 52 pending exports --- docs/dev/debt/libSceMat.md | 58 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docs/dev/debt/libSceMat.md diff --git a/docs/dev/debt/libSceMat.md b/docs/dev/debt/libSceMat.md new file mode 100644 index 0000000000..ca692fc1d8 --- /dev/null +++ b/docs/dev/debt/libSceMat.md @@ -0,0 +1,58 @@ +# libSceMat technical debt + +### Unknown function info + +The 52 exports in [Export.cpp](../../../core/libs/prx/libSceMat/Export.cpp) come from the `libSceMat` list in [claimore22/ps5rs](https://github.com/claimore22/ps5rs/blob/5f252a658ff470bcb2b6cd15205fd8ca64ae4477/data/stubs/by_library/libSceMat_nosubmission.txt) (commit 5f252a658ff470bcb2b6cd15205fd8ca64ae4477), a catalogue of PS5 symbols per library whose provenance the project does not document, and no PS5 export inventory of this module is available to confirm the list is complete or exact. Every name was kept only because its hash equals the listed NID. Prototypes and behaviour are unverified: every function takes no parameters in the import-resolution convention and throws through `NotImplemented_nid_no_patch`. + +- [sceMatMemoryPoolReserve](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID +v5r0-k24j0) +- [sceMatUninitialize](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 0kgbiCpmb1M) +- [sceMatTagVirtualMemory](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 0v6+W4ulXJ0) +- [sceMatMspaceCreate](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 1HWnwXRCrdI) +- [sceMatUnregisterResource](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 1TVH3eVt1Mg) +- [sceMatMprotect](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 3AI9N7clwCs) +- [sceMatMapDirectMemory](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 3yTx5JA+cgg) +- [sceMatInitialize](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 5Zlr3joORIE) +- [sceMatNewFrame](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 6gOfuHe419U) +- [sceMatRegisterResourceType](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 71P0F8eJfPk) +- [sceMatFree](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 9W19mAu8Bzc) +- [sceMatSetUserCallstackCapture](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID 9m6mmioPTOo) +- [sceMatMemoryPoolMove](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID C++Y3xup+O0) +- [sceMatRegisterGroup](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID CEcz6AdSUpw) +- [sceMatMemoryPoolExpand](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID Cnn3HtdAFOU) +- [sceMatMemoryPoolBatch](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID HE8VNl2+1Fw) +- [sceMatAllocPoolMemory](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID I8sUe4XPSvU) +- [sceMatAgcUnregisterOwnerAndResources](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID JyFq+FZ6ujw) +- [sceMatReleasePhysicalMemory](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID LIVclwLLnyM) +- [sceMatMemoryPoolDecommit](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID N9YwXumgOfo) +- [sceMatSetPrtAperture](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID NDbQMSkpoXg) +- [sceMatFreePoolMemory](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID NHjHSABXkLo) +- [sceMatUnmapMemory](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID O19gIiIute4) +- [sceMatMspaceDestroy](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID PXtTCOohxj0) +- [sceMatAlloc](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID QrzsrnqGy9g) +- [sceMatReallocBegin](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID RWNATmy2cug) +- [sceMatReallocEnd](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID XC2CuQMB9js) +- [sceMatAgcRegisterResource](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID YIFDiAInu7Y) +- [sceMatWriteBookmark](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID Z5-0uTMnWnE) +- [sceMatBatchMap](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID b-BtawOVCmQ) +- [sceMatTagPool](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID cY6ZpCec3Oo) +- [sceMatAgcRegisterOwner](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID dl1xO-QP3C8) +- [sceMatUnregisterAllResourcesForOwner](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID gEBIkRrqDBA) +- [sceMatMtypeprotect](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID hVvUitUwrFs) +- [sceMatSetThreadInfo](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID k6QfiQvAEJc) +- [sceMatPushMarker](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID kd0VbEN5bIY) +- [sceMatUnregisterOwnerAndResources](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID lqHsrpuW4MI) +- [sceMatAgcUnregisterResource](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID n-zwIVhJMTY) +- [sceMatRegisterOwner](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID ptoJYu3CkyQ) +- [sceMatAgcUnregisterAllResourcesForOwner](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID qdLo7hti5rc) +- [sceMatPopMarker](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID quIxw751PLs) +- [sceMatPushMarkerStatic](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID rDtFuWpxAgM) +- [sceMatUpdateModuleList](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID spov0bD6PpM) +- [sceMatResizeCommsBuffer](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID srnAu7m5VKk) +- [sceMatReallocPoolMemory](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID tH+nf8PVkP8) +- [sceMatMemoryPoolCommit](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID tIBrrdbjJBA) +- [sceMatRegisterResource](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID uAxS1aVr6OY) +- [sceMatAllocPhysicalMemory](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID uNPUU9tAUw4) +- [sceMatReserveVirtualRange](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID vYS-YC+Cdkg) +- [sceMatTagAllocation](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID xDzWWofyEtA) +- [sceMatFreeRange](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID yO6vE0Y9eiY) +- [sceMatCalloc](../../../core/libs/prx/libSceMat/Export.cpp) (libSceMat) - unknown signature (NID z9e-MCsVZZQ) From 6dae4b6288e5807e19e14302679e772f8ee96231 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:28 +0000 Subject: [PATCH 176/318] feat(libScePerf): declare 84 pending exports --- core/libs/prx/libScePerf/Export.cpp | 420 +++++++++++++++++++++++++++- 1 file changed, 419 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libScePerf/Export.cpp b/core/libs/prx/libScePerf/Export.cpp index fb34206c21..0c4d970c1f 100644 --- a/core/libs/prx/libScePerf/Export.cpp +++ b/core/libs/prx/libScePerf/Export.cpp @@ -5,6 +5,424 @@ extern "C" { -APS5_DUMMY_FUN +int APS5_VABI scePerfTraceIoControllerEnable(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceCreate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcUmcStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcSdfGetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcSdfStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmDelete(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmGetInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceIoControllerStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcSelectEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmEnable(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcClose(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceDelete(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfGetStreamingTime(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfDelete(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmDeleteBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcL3SelectEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceIoControllerCreate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceIoControllerDeleteBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceEnable(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceGetMemoryStats(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmUpdate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmDeleteBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcL3GetCounterSelf(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameCreate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcSetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcUmcSetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmEnable(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmCreate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceGetInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcReset(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcSdfSetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfReadBuffer(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameAddBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcSdfStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcL3Start(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcSdfSelectEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmAddBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfEnable(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceDeleteBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceIoControllerDelete(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfCreate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfAddBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcGetCounterSelf(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfDeleteBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAddBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmGetInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcL3Stop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcMperfGetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfMarker(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameUpdate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameDeleteBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcGetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcL3GetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmDelete(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcUmcStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmCreate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcL3SetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceIoControllerUpdate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcAperfGetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfSetWait(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTracePmAddBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmUpdate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameEnable(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameDelete(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceIoControllerAddBufferNotifyEvent(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAmmStop(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceIoControllerGetInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceAprNameGetInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcUmcGetCounter(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceSpmSdfGetInfo(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceIoControllerStart(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfTraceUpdate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePerfPmcOpen(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } From 3ed65680af03bf21da410d0c85297fa173066078 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:28 +0000 Subject: [PATCH 177/318] docs(libScePerf): record 84 pending exports --- docs/dev/debt/libScePerf.md | 90 +++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/dev/debt/libScePerf.md diff --git a/docs/dev/debt/libScePerf.md b/docs/dev/debt/libScePerf.md new file mode 100644 index 0000000000..18cf6ad56c --- /dev/null +++ b/docs/dev/debt/libScePerf.md @@ -0,0 +1,90 @@ +# libScePerf technical debt + +### Unknown function info + +The 84 exports in [Export.cpp](../../../core/libs/prx/libScePerf/Export.cpp) come from the `libScePerf` list in [claimore22/ps5rs](https://github.com/claimore22/ps5rs/blob/5f252a658ff470bcb2b6cd15205fd8ca64ae4477/data/stubs/by_library/libScePerf_nosubmission.txt) (commit 5f252a658ff470bcb2b6cd15205fd8ca64ae4477), a catalogue of PS5 symbols per library whose provenance the project does not document, and no PS5 export inventory of this module is available to confirm the list is complete or exact. Every name was kept only because its hash equals the listed NID. Prototypes and behaviour are unverified: every function takes no parameters in the import-resolution convention and throws through `NotImplemented_nid_no_patch`. + +- [scePerfTraceIoControllerEnable](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID +AX8QoImHeQ) +- [scePerfTraceCreate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID +bMkbKRF6Bo) +- [scePerfTracePmStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 2H1RlsV3ioM) +- [scePerfPmcUmcStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 2h037DNaRPU) +- [scePerfPmcSdfGetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 37ZdK-DvAJI) +- [scePerfPmcSdfStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 3RJZH7EwHdg) +- [scePerfTraceAmmDelete](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 3VJmLpukTAc) +- [scePerfTracePmGetInfo](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 6Mo6+u0BZeA) +- [scePerfTraceIoControllerStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 6d-bWUDuz9U) +- [scePerfPmcSelectEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 7OlZkMu6Nbg) +- [scePerfTracePmEnable](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID 9d0PDAM9g1Q) +- [scePerfPmcClose](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID A3z26deg9VQ) +- [scePerfTraceDelete](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID AlogKJrMsco) +- [scePerfTraceSpmSdfStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID C0ul9Xyr-iI) +- [scePerfTraceSpmSdfGetStreamingTime](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID ClZZK8jlb+g) +- [scePerfTraceSpmSdfDelete](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID CvOsz43k9es) +- [scePerfPmcStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID EIc0PIXVccE) +- [scePerfTracePmDeleteBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID EYSKE8otTSs) +- [scePerfPmcL3SelectEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID FPN86biiXCg) +- [scePerfTraceIoControllerCreate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID FnQzqm0V3lU) +- [scePerfTraceIoControllerDeleteBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID GHuASZVVDBA) +- [scePerfTraceEnable](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID HBEICuPh5r4) +- [scePerfTraceGetMemoryStats](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID HIO82JX-Ki4) +- [scePerfTracePmUpdate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID IAf1M-HM55c) +- [scePerfTraceAmmDeleteBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID Jt7LPd8LVCU) +- [scePerfPmcL3GetCounterSelf](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID L7qBonxwhiI) +- [scePerfTraceAprNameCreate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID MV8Ydp9MSdU) +- [scePerfPmcSetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID MdSI82hHt0E) +- [scePerfPmcUmcSetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID NW+Hxkmylr4) +- [scePerfTraceAmmEnable](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID NbafDOCadvM) +- [scePerfTracePmCreate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID OAVwxuKdkFE) +- [scePerfTraceGetInfo](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID OKLpKlnQmCM) +- [scePerfPmcReset](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID Pw6DvB9u7vA) +- [scePerfTracePmStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID Q6rtQMOR-UA) +- [scePerfTraceAprNameStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID Rztm+4l7XEM) +- [scePerfPmcSdfSetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID S9IH+kvDNU8) +- [scePerfTraceSpmSdfReadBuffer](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID Sq6kCHLudYQ) +- [scePerfTraceAprNameAddBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID SqfaljiIMTM) +- [scePerfPmcSdfStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID Tg8wM8zlz3I) +- [scePerfPmcL3Start](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID UdkvoRZbg4M) +- [scePerfPmcSdfSelectEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID V-dofRUSIAs) +- [scePerfTraceAmmAddBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID VFfDN9+bjC4) +- [scePerfTraceSpmSdfEnable](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID VM+b5+C2QTU) +- [scePerfTraceDeleteBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID VgsWDKLc6v4) +- [scePerfTraceIoControllerDelete](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID XZmo+u-eLBg) +- [scePerfTraceSpmSdfCreate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID XfEto+saVaY) +- [scePerfTraceSpmSdfAddBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID XmGnBbTvpoE) +- [scePerfPmcGetCounterSelf](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID Y9soJc90skA) +- [scePerfTraceSpmSdfDeleteBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID YY2agR1yZKk) +- [scePerfTraceAddBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID Z0a+sohjGTQ) +- [scePerfTraceAmmStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID ZvPKqTJPzd4) +- [scePerfTraceAmmGetInfo](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID aY-lm9NK8zU) +- [scePerfPmcL3Stop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID ai2UFvUWMs0) +- [scePerfPmcMperfGetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID aqpZ3UxXfcw) +- [scePerfTraceSpmSdfMarker](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID b8mxmZwrlv8) +- [scePerfTraceAprNameUpdate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID cYNmzCqlzIA) +- [scePerfTraceAprNameDeleteBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID ccL8qtt32N0) +- [scePerfPmcGetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID e7RAPAr4QpU) +- [scePerfPmcL3GetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID eBWlMuS-0MA) +- [scePerfPmcStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID fAfi5-11dKg) +- [scePerfTraceAprNameStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID fNhrkYpXz0o) +- [scePerfTracePmDelete](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID i9+nYhSntBI) +- [scePerfPmcUmcStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID iWwcsTjXh78) +- [scePerfTraceAmmCreate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID id0JiM1+i5I) +- [scePerfPmcL3SetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID imlxgVJc+lE) +- [scePerfTraceIoControllerUpdate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID k34J5kyOy8w) +- [scePerfPmcAperfGetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID lVsw3kdsgAk) +- [scePerfTraceSpmSdfSetWait](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID lmUEkJvK8jM) +- [scePerfTracePmAddBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID ma2OmJRhSPM) +- [scePerfTraceAmmUpdate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID mzu52lea-vk) +- [scePerfTraceStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID n-pzjs6rl48) +- [scePerfTraceStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID n0nvCnrzJI0) +- [scePerfTraceAprNameEnable](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID nWigKuh+U8Y) +- [scePerfTraceAprNameDelete](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID nZoXleE24-w) +- [scePerfTraceIoControllerAddBufferNotifyEvent](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID njAb3txIO8k) +- [scePerfTraceSpmSdfStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID nyPpDWfGIjQ) +- [scePerfTraceAmmStop](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID oeCZSK8StuQ) +- [scePerfTraceIoControllerGetInfo](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID pAbZ-KnYusQ) +- [scePerfTraceAprNameGetInfo](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID ruR8rglNPvE) +- [scePerfPmcUmcGetCounter](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID slT42meZHBo) +- [scePerfTraceSpmSdfGetInfo](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID ulckVIVJX8Q) +- [scePerfTraceIoControllerStart](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID vswjIkOXbuU) +- [scePerfTraceUpdate](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID wLvhNHWWJTU) +- [scePerfPmcOpen](../../../core/libs/prx/libScePerf/Export.cpp) (libScePerf) - unknown signature (NID zezfYI2+DWA) From 036a8c6437216ac59d63dd83611195d281c84882 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:28 +0000 Subject: [PATCH 178/318] feat(libSceRazorCpu_debug): declare 3 pending exports --- core/libs/prx/libSceRazorCpu_debug/Export.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceRazorCpu_debug/Export.cpp b/core/libs/prx/libSceRazorCpu_debug/Export.cpp index d81f4a5e32..329c6b8281 100644 --- a/core/libs/prx/libSceRazorCpu_debug/Export.cpp +++ b/core/libs/prx/libSceRazorCpu_debug/Export.cpp @@ -1,10 +1,23 @@ -#include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" extern "C" { -APS5_DUMMY_FUN +int APS5_VABI sceRazorCpuStartCapture(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceRazorCpuStopCapture(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceRazorCpuSetLiveServerThreadSettings(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } From 93243de05a9082dbd59c47142860896a74ed4cf0 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:28 +0000 Subject: [PATCH 179/318] docs(libSceRazorCpu_debug): record 3 pending exports --- docs/dev/debt/libSceRazorCpu_debug.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/dev/debt/libSceRazorCpu_debug.md diff --git a/docs/dev/debt/libSceRazorCpu_debug.md b/docs/dev/debt/libSceRazorCpu_debug.md new file mode 100644 index 0000000000..fbdf6c1f81 --- /dev/null +++ b/docs/dev/debt/libSceRazorCpu_debug.md @@ -0,0 +1,9 @@ +# libSceRazorCpu_debug technical debt + +### Unknown function info + +The 3 exports in [Export.cpp](../../../core/libs/prx/libSceRazorCpu_debug/Export.cpp) come from the `libSceRazorCpu_debug` list in [claimore22/ps5rs](https://github.com/claimore22/ps5rs/blob/5f252a658ff470bcb2b6cd15205fd8ca64ae4477/data/stubs/by_library/libSceRazorCpu_debug_nosubmission.txt) (commit 5f252a658ff470bcb2b6cd15205fd8ca64ae4477), a catalogue of PS5 symbols per library whose provenance the project does not document, and no PS5 export inventory of this module is available to confirm the list is complete or exact. Every name was kept only because its hash equals the listed NID. Prototypes and behaviour are unverified: every function takes no parameters in the import-resolution convention and throws through `NotImplemented_nid_no_patch`. + +- [sceRazorCpuStartCapture](../../../core/libs/prx/libSceRazorCpu_debug/Export.cpp) (libSceRazorCpu_debug) - unknown signature (NID 1zHP7BQXxr4) +- [sceRazorCpuStopCapture](../../../core/libs/prx/libSceRazorCpu_debug/Export.cpp) (libSceRazorCpu_debug) - unknown signature (NID mdUPwgHRqss) +- [sceRazorCpuSetLiveServerThreadSettings](../../../core/libs/prx/libSceRazorCpu_debug/Export.cpp) (libSceRazorCpu_debug) - unknown signature (NID q4EyfhLHc0Q) From a5d80c96dea5cc9a65acc8805e696039ee6badf9 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:28 +0000 Subject: [PATCH 180/318] feat(libSceWorkspace): declare 18 pending exports --- core/libs/prx/libSceWorkspace/Export.cpp | 92 +++++++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceWorkspace/Export.cpp b/core/libs/prx/libSceWorkspace/Export.cpp index d81f4a5e32..a1b7b270e8 100644 --- a/core/libs/prx/libSceWorkspace/Export.cpp +++ b/core/libs/prx/libSceWorkspace/Export.cpp @@ -1,10 +1,98 @@ -#include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" extern "C" { -APS5_DUMMY_FUN +int APS5_VABI module_start_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceWriteFile(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceFinishWrite(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceApplyUpdateForFiles(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceApplyUpdateForAllFiles(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceIsBlockedByDataTransfer(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceMirrorBarrier(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceGetNumberOfPendingUpdates(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI module_stop_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceUnmount(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceApplyUpdate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceDeletePath(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceMount(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceGetName(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceCreateFile(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceApplyUpdateForDirectory(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceIsUpdatePending(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceWorkspaceGetPendingUpdateEntries(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } From da4b2e1689e658d456f0065bc65efc3bfbfdad67 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:28 +0000 Subject: [PATCH 181/318] docs(libSceWorkspace): record 18 pending exports --- docs/dev/debt/libSceWorkspace.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/dev/debt/libSceWorkspace.md diff --git a/docs/dev/debt/libSceWorkspace.md b/docs/dev/debt/libSceWorkspace.md new file mode 100644 index 0000000000..6e4c3a3353 --- /dev/null +++ b/docs/dev/debt/libSceWorkspace.md @@ -0,0 +1,24 @@ +# libSceWorkspace technical debt + +### Unknown function info + +The 18 exports in [Export.cpp](../../../core/libs/prx/libSceWorkspace/Export.cpp) come from the `libSceWorkspace` list in [claimore22/ps5rs](https://github.com/claimore22/ps5rs/blob/5f252a658ff470bcb2b6cd15205fd8ca64ae4477/data/stubs/by_library/libSceWorkspace_nosubmission.txt) (commit 5f252a658ff470bcb2b6cd15205fd8ca64ae4477), a catalogue of PS5 symbols per library whose provenance the project does not document, and no PS5 export inventory of this module is available to confirm the list is complete or exact. Every name was kept only because its hash equals the listed NID. Prototypes and behaviour are unverified: every function takes no parameters in the import-resolution convention and throws through `NotImplemented_nid_no_patch`. + +- [module_start](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID BaOKcng8g88) +- [sceWorkspaceWriteFile](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID DYgq0+kM8UI) +- [sceWorkspaceFinishWrite](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID Ej722ud9Zo4) +- [sceWorkspaceApplyUpdateForFiles](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID FLTTgSdkoOc) +- [sceWorkspaceApplyUpdateForAllFiles](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID GKMjPmDls-w) +- [sceWorkspaceIsBlockedByDataTransfer](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID GgrAun5x5Eg) +- [sceWorkspaceMirrorBarrier](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID HcbIPAn1o3g) +- [sceWorkspaceGetNumberOfPendingUpdates](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID JvThkOI0jjE) +- [module_stop](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID KpDMrPHvt3Q) +- [sceWorkspaceUnmount](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID SOh2WS6VvIE) +- [sceWorkspaceApplyUpdate](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID WPBiUIzH4z4) +- [sceWorkspaceDeletePath](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID bSJy+Zr5h+M) +- [sceWorkspaceMount](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID bsUfUrKJ4Qo) +- [sceWorkspaceGetName](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID d9XUi2-zgWw) +- [sceWorkspaceCreateFile](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID euJjtMph7f4) +- [sceWorkspaceApplyUpdateForDirectory](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID hqY8PDT4oqU) +- [sceWorkspaceIsUpdatePending](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID t0z87PlggWI) +- [sceWorkspaceGetPendingUpdateEntries](../../../core/libs/prx/libSceWorkspace/Export.cpp) (libSceWorkspace) - unknown signature (NID wvc0E5cTehw) From c13dc58e98e529df192cb232d17a57c6694f571b Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:10:24 +0000 Subject: [PATCH 182/318] refactor(libSceDbg): declare the pending exports in the public header Api.hpp declares each export once, with the placeholder signature its stub has, and the umbrella header and Export.cpp include it. --- core/libs/prx/libSceDbg/Export.cpp | 1 + core/libs/prx/libSceDbg/include/Api.hpp | 37 +++++++++++++++++++++++++ core/libs/prx/libSceDbg/libSceDbg.hpp | 2 ++ 3 files changed, 40 insertions(+) create mode 100644 core/libs/prx/libSceDbg/include/Api.hpp diff --git a/core/libs/prx/libSceDbg/Export.cpp b/core/libs/prx/libSceDbg/Export.cpp index a90b17e6e3..2bd31bc88d 100644 --- a/core/libs/prx/libSceDbg/Export.cpp +++ b/core/libs/prx/libSceDbg/Export.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceDbg/include/Api.hpp" #include #include #include "SceTypes.hpp" diff --git a/core/libs/prx/libSceDbg/include/Api.hpp b/core/libs/prx/libSceDbg/include/Api.hpp new file mode 100644 index 0000000000..cbe77edc61 --- /dev/null +++ b/core/libs/prx/libSceDbg/include/Api.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include "SceTypes.hpp" + +extern "C" { + +int APS5_VABI sceDbgUserChannelPrint(void); +int APS5_VABI sceDbgAmprAprGetNumberOfErrors(void); +int APS5_VABI sceDbgMapperVirtualQuery(void); +int APS5_VABI sceDbgUserChannelPrintf(void); +int APS5_VABI sceDbgAssertionHandler(void); +int APS5_VABI sceDbgGetSystemSwVersion(void); +int APS5_VABI sceDbgSetBreakOnWarningState(void); +int APS5_VABI sceDbgSetBreakOnErrorState(void); +int APS5_VABI sceDbgDeleteExternalTriggerEvent(void); +int APS5_VABI sceDbgAmprAmmGetNumberOfErrors(void); +int APS5_VABI sceDbgGetModuleInfo(void); +int APS5_VABI sceDbgBacktraceSelf(void); +int APS5_VABI sceDbgAmprAmmGetErrors(void); +int APS5_VABI sceDbgGetDebugSuspendCount(void); +int APS5_VABI sceDbgUserChannelVPrintf(void); +int APS5_VABI sceDbgSetMinimumLogLevel(void); +int APS5_VABI sceDbgGetModuleList(void); +int APS5_VABI sceDbgClearHardwareBreakPoint(void); +int APS5_VABI sceDbgAddExternalTriggerEvent(void); +int APS5_VABI sceDbgLoggingHandler(void); +int APS5_VABI sceDbgEnableExtraHeapDebugInfo(void); +int APS5_VABI sceDbgAmprAprGetErrors(void); +int APS5_VABI sceDbgVLoggingHandler(void); +int APS5_VABI sceDbgGetReleaseCheckMode(void); +int APS5_VABI sceDbgSetHardwareBreakPoint(void); +int APS5_VABI sceDbgGetExecutablePath(void); +int APS5_VABI sceDbgGetCpuGpuFrequencySetting(void); +int APS5_VABI sceDbgRequireSystemSwVersion(void); +int APS5_VABI sceDbgIsDebuggerAttached(void); + +} diff --git a/core/libs/prx/libSceDbg/libSceDbg.hpp b/core/libs/prx/libSceDbg/libSceDbg.hpp index 6f70f09bee..461503f002 100644 --- a/core/libs/prx/libSceDbg/libSceDbg.hpp +++ b/core/libs/prx/libSceDbg/libSceDbg.hpp @@ -1 +1,3 @@ #pragma once + +#include "prx/libSceDbg/include/Api.hpp" From d01b61308d00ddd8a63b570f40865ac6759a1b69 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:10:40 +0000 Subject: [PATCH 183/318] refactor(libSceDeci5): declare the pending exports in the public header Api.hpp declares each export once, with the placeholder signature its stub has, and the umbrella header and Export.cpp include it. --- core/libs/prx/libSceDeci5/Export.cpp | 1 + core/libs/prx/libSceDeci5/include/Api.hpp | 15 +++++++++++++++ core/libs/prx/libSceDeci5/libSceDeci5.hpp | 2 ++ 3 files changed, 18 insertions(+) create mode 100644 core/libs/prx/libSceDeci5/include/Api.hpp diff --git a/core/libs/prx/libSceDeci5/Export.cpp b/core/libs/prx/libSceDeci5/Export.cpp index 64b88a0ef5..c63c8a3623 100644 --- a/core/libs/prx/libSceDeci5/Export.cpp +++ b/core/libs/prx/libSceDeci5/Export.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceDeci5/include/Api.hpp" #include #include #include "SceTypes.hpp" diff --git a/core/libs/prx/libSceDeci5/include/Api.hpp b/core/libs/prx/libSceDeci5/include/Api.hpp new file mode 100644 index 0000000000..6a3c918b9c --- /dev/null +++ b/core/libs/prx/libSceDeci5/include/Api.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include "SceTypes.hpp" + +extern "C" { + +int APS5_VABI sceDeci5Write(void); +int APS5_VABI sceDeci5Read(void); +int APS5_VABI sceDeci5CreateHostProcessAndWait(void); +int APS5_VABI sceDeci5Close(void); +int APS5_VABI sceDeci5CreateHostProcess(void); +int APS5_VABI sceDeci5Open(void); +int APS5_VABI sceDeci5GetStatus(void); + +} diff --git a/core/libs/prx/libSceDeci5/libSceDeci5.hpp b/core/libs/prx/libSceDeci5/libSceDeci5.hpp index 6f70f09bee..bb991c0eee 100644 --- a/core/libs/prx/libSceDeci5/libSceDeci5.hpp +++ b/core/libs/prx/libSceDeci5/libSceDeci5.hpp @@ -1 +1,3 @@ #pragma once + +#include "prx/libSceDeci5/include/Api.hpp" From b5b93688571849a3d79b6f41185e388e751918e1 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:10:55 +0000 Subject: [PATCH 184/318] refactor(libSceGpuTrace): declare the pending exports in the public header Api.hpp declares each export once, with the placeholder signature its stub has, and the umbrella header and Export.cpp include it. --- core/libs/prx/libSceGpuTrace/Export.cpp | 1 + core/libs/prx/libSceGpuTrace/include/Api.hpp | 16 ++++++++++++++++ core/libs/prx/libSceGpuTrace/libSceGpuTrace.hpp | 2 ++ 3 files changed, 19 insertions(+) create mode 100644 core/libs/prx/libSceGpuTrace/include/Api.hpp diff --git a/core/libs/prx/libSceGpuTrace/Export.cpp b/core/libs/prx/libSceGpuTrace/Export.cpp index efe9941cce..e518015e73 100644 --- a/core/libs/prx/libSceGpuTrace/Export.cpp +++ b/core/libs/prx/libSceGpuTrace/Export.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceGpuTrace/include/Api.hpp" #include #include #include "SceTypes.hpp" diff --git a/core/libs/prx/libSceGpuTrace/include/Api.hpp b/core/libs/prx/libSceGpuTrace/include/Api.hpp new file mode 100644 index 0000000000..411bb3aa7f --- /dev/null +++ b/core/libs/prx/libSceGpuTrace/include/Api.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include "SceTypes.hpp" + +extern "C" { + +int APS5_VABI sceGpuTraceParametersSetDuration(void); +int APS5_VABI sceGpuTraceParametersInit(void); +int APS5_VABI sceGpuTraceParametersSetMemorySize(void); +int APS5_VABI sceGpuTraceParametersSetGroup(void); +int APS5_VABI sceGpuTraceStart(void); +int APS5_VABI sceGpuTraceStop(void); +int APS5_VABI sceGpuTraceMonitorTrigger(void); +int APS5_VABI sceGpuTraceCancel(void); + +} diff --git a/core/libs/prx/libSceGpuTrace/libSceGpuTrace.hpp b/core/libs/prx/libSceGpuTrace/libSceGpuTrace.hpp index 6f70f09bee..60cc36c983 100644 --- a/core/libs/prx/libSceGpuTrace/libSceGpuTrace.hpp +++ b/core/libs/prx/libSceGpuTrace/libSceGpuTrace.hpp @@ -1 +1,3 @@ #pragma once + +#include "prx/libSceGpuTrace/include/Api.hpp" From 72e945cfe0194cec23f8fbc8323cc31003e16e43 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:11:12 +0000 Subject: [PATCH 185/318] refactor(libSceMat): declare the pending exports in the public header Api.hpp declares each export once, with the placeholder signature its stub has, and the umbrella header and Export.cpp include it. --- core/libs/prx/libSceMat/Export.cpp | 1 + core/libs/prx/libSceMat/include/Api.hpp | 60 +++++++++++++++++++++++++ core/libs/prx/libSceMat/libSceMat.hpp | 2 + 3 files changed, 63 insertions(+) create mode 100644 core/libs/prx/libSceMat/include/Api.hpp diff --git a/core/libs/prx/libSceMat/Export.cpp b/core/libs/prx/libSceMat/Export.cpp index 70c77005c9..242ea1c1e5 100644 --- a/core/libs/prx/libSceMat/Export.cpp +++ b/core/libs/prx/libSceMat/Export.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceMat/include/Api.hpp" #include #include #include "SceTypes.hpp" diff --git a/core/libs/prx/libSceMat/include/Api.hpp b/core/libs/prx/libSceMat/include/Api.hpp new file mode 100644 index 0000000000..36ccb68e48 --- /dev/null +++ b/core/libs/prx/libSceMat/include/Api.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include "SceTypes.hpp" + +extern "C" { + +int APS5_VABI sceMatMemoryPoolReserve(void); +int APS5_VABI sceMatUninitialize(void); +int APS5_VABI sceMatTagVirtualMemory(void); +int APS5_VABI sceMatMspaceCreate(void); +int APS5_VABI sceMatUnregisterResource(void); +int APS5_VABI sceMatMprotect(void); +int APS5_VABI sceMatMapDirectMemory(void); +int APS5_VABI sceMatInitialize(void); +int APS5_VABI sceMatNewFrame(void); +int APS5_VABI sceMatRegisterResourceType(void); +int APS5_VABI sceMatFree(void); +int APS5_VABI sceMatSetUserCallstackCapture(void); +int APS5_VABI sceMatMemoryPoolMove(void); +int APS5_VABI sceMatRegisterGroup(void); +int APS5_VABI sceMatMemoryPoolExpand(void); +int APS5_VABI sceMatMemoryPoolBatch(void); +int APS5_VABI sceMatAllocPoolMemory(void); +int APS5_VABI sceMatAgcUnregisterOwnerAndResources(void); +int APS5_VABI sceMatReleasePhysicalMemory(void); +int APS5_VABI sceMatMemoryPoolDecommit(void); +int APS5_VABI sceMatSetPrtAperture(void); +int APS5_VABI sceMatFreePoolMemory(void); +int APS5_VABI sceMatUnmapMemory(void); +int APS5_VABI sceMatMspaceDestroy(void); +int APS5_VABI sceMatAlloc(void); +int APS5_VABI sceMatReallocBegin(void); +int APS5_VABI sceMatReallocEnd(void); +int APS5_VABI sceMatAgcRegisterResource(void); +int APS5_VABI sceMatWriteBookmark(void); +int APS5_VABI sceMatBatchMap(void); +int APS5_VABI sceMatTagPool(void); +int APS5_VABI sceMatAgcRegisterOwner(void); +int APS5_VABI sceMatUnregisterAllResourcesForOwner(void); +int APS5_VABI sceMatMtypeprotect(void); +int APS5_VABI sceMatSetThreadInfo(void); +int APS5_VABI sceMatPushMarker(void); +int APS5_VABI sceMatUnregisterOwnerAndResources(void); +int APS5_VABI sceMatAgcUnregisterResource(void); +int APS5_VABI sceMatRegisterOwner(void); +int APS5_VABI sceMatAgcUnregisterAllResourcesForOwner(void); +int APS5_VABI sceMatPopMarker(void); +int APS5_VABI sceMatPushMarkerStatic(void); +int APS5_VABI sceMatUpdateModuleList(void); +int APS5_VABI sceMatResizeCommsBuffer(void); +int APS5_VABI sceMatReallocPoolMemory(void); +int APS5_VABI sceMatMemoryPoolCommit(void); +int APS5_VABI sceMatRegisterResource(void); +int APS5_VABI sceMatAllocPhysicalMemory(void); +int APS5_VABI sceMatReserveVirtualRange(void); +int APS5_VABI sceMatTagAllocation(void); +int APS5_VABI sceMatFreeRange(void); +int APS5_VABI sceMatCalloc(void); + +} diff --git a/core/libs/prx/libSceMat/libSceMat.hpp b/core/libs/prx/libSceMat/libSceMat.hpp index 6f70f09bee..7dc1ae9a59 100644 --- a/core/libs/prx/libSceMat/libSceMat.hpp +++ b/core/libs/prx/libSceMat/libSceMat.hpp @@ -1 +1,3 @@ #pragma once + +#include "prx/libSceMat/include/Api.hpp" From a3cd4103355c2b4ffda9b6402cc92d33d8045988 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:11:29 +0000 Subject: [PATCH 186/318] refactor(libScePerf): declare the pending exports in the public header Api.hpp declares each export once, with the placeholder signature its stub has, and the umbrella header and Export.cpp include it. --- core/libs/prx/libScePerf/Export.cpp | 1 + core/libs/prx/libScePerf/include/Api.hpp | 92 ++++++++++++++++++++++++ core/libs/prx/libScePerf/libScePerf.hpp | 2 + 3 files changed, 95 insertions(+) create mode 100644 core/libs/prx/libScePerf/include/Api.hpp diff --git a/core/libs/prx/libScePerf/Export.cpp b/core/libs/prx/libScePerf/Export.cpp index 0c4d970c1f..9d6857a511 100644 --- a/core/libs/prx/libScePerf/Export.cpp +++ b/core/libs/prx/libScePerf/Export.cpp @@ -1,3 +1,4 @@ +#include "prx/libScePerf/include/Api.hpp" #include #include #include "SceTypes.hpp" diff --git a/core/libs/prx/libScePerf/include/Api.hpp b/core/libs/prx/libScePerf/include/Api.hpp new file mode 100644 index 0000000000..c4c3966edf --- /dev/null +++ b/core/libs/prx/libScePerf/include/Api.hpp @@ -0,0 +1,92 @@ +#pragma once + +#include "SceTypes.hpp" + +extern "C" { + +int APS5_VABI scePerfTraceIoControllerEnable(void); +int APS5_VABI scePerfTraceCreate(void); +int APS5_VABI scePerfTracePmStop(void); +int APS5_VABI scePerfPmcUmcStart(void); +int APS5_VABI scePerfPmcSdfGetCounter(void); +int APS5_VABI scePerfPmcSdfStart(void); +int APS5_VABI scePerfTraceAmmDelete(void); +int APS5_VABI scePerfTracePmGetInfo(void); +int APS5_VABI scePerfTraceIoControllerStop(void); +int APS5_VABI scePerfPmcSelectEvent(void); +int APS5_VABI scePerfTracePmEnable(void); +int APS5_VABI scePerfPmcClose(void); +int APS5_VABI scePerfTraceDelete(void); +int APS5_VABI scePerfTraceSpmSdfStart(void); +int APS5_VABI scePerfTraceSpmSdfGetStreamingTime(void); +int APS5_VABI scePerfTraceSpmSdfDelete(void); +int APS5_VABI scePerfPmcStop(void); +int APS5_VABI scePerfTracePmDeleteBufferNotifyEvent(void); +int APS5_VABI scePerfPmcL3SelectEvent(void); +int APS5_VABI scePerfTraceIoControllerCreate(void); +int APS5_VABI scePerfTraceIoControllerDeleteBufferNotifyEvent(void); +int APS5_VABI scePerfTraceEnable(void); +int APS5_VABI scePerfTraceGetMemoryStats(void); +int APS5_VABI scePerfTracePmUpdate(void); +int APS5_VABI scePerfTraceAmmDeleteBufferNotifyEvent(void); +int APS5_VABI scePerfPmcL3GetCounterSelf(void); +int APS5_VABI scePerfTraceAprNameCreate(void); +int APS5_VABI scePerfPmcSetCounter(void); +int APS5_VABI scePerfPmcUmcSetCounter(void); +int APS5_VABI scePerfTraceAmmEnable(void); +int APS5_VABI scePerfTracePmCreate(void); +int APS5_VABI scePerfTraceGetInfo(void); +int APS5_VABI scePerfPmcReset(void); +int APS5_VABI scePerfTracePmStart(void); +int APS5_VABI scePerfTraceAprNameStart(void); +int APS5_VABI scePerfPmcSdfSetCounter(void); +int APS5_VABI scePerfTraceSpmSdfReadBuffer(void); +int APS5_VABI scePerfTraceAprNameAddBufferNotifyEvent(void); +int APS5_VABI scePerfPmcSdfStop(void); +int APS5_VABI scePerfPmcL3Start(void); +int APS5_VABI scePerfPmcSdfSelectEvent(void); +int APS5_VABI scePerfTraceAmmAddBufferNotifyEvent(void); +int APS5_VABI scePerfTraceSpmSdfEnable(void); +int APS5_VABI scePerfTraceDeleteBufferNotifyEvent(void); +int APS5_VABI scePerfTraceIoControllerDelete(void); +int APS5_VABI scePerfTraceSpmSdfCreate(void); +int APS5_VABI scePerfTraceSpmSdfAddBufferNotifyEvent(void); +int APS5_VABI scePerfPmcGetCounterSelf(void); +int APS5_VABI scePerfTraceSpmSdfDeleteBufferNotifyEvent(void); +int APS5_VABI scePerfTraceAddBufferNotifyEvent(void); +int APS5_VABI scePerfTraceAmmStart(void); +int APS5_VABI scePerfTraceAmmGetInfo(void); +int APS5_VABI scePerfPmcL3Stop(void); +int APS5_VABI scePerfPmcMperfGetCounter(void); +int APS5_VABI scePerfTraceSpmSdfMarker(void); +int APS5_VABI scePerfTraceAprNameUpdate(void); +int APS5_VABI scePerfTraceAprNameDeleteBufferNotifyEvent(void); +int APS5_VABI scePerfPmcGetCounter(void); +int APS5_VABI scePerfPmcL3GetCounter(void); +int APS5_VABI scePerfPmcStart(void); +int APS5_VABI scePerfTraceAprNameStop(void); +int APS5_VABI scePerfTracePmDelete(void); +int APS5_VABI scePerfPmcUmcStop(void); +int APS5_VABI scePerfTraceAmmCreate(void); +int APS5_VABI scePerfPmcL3SetCounter(void); +int APS5_VABI scePerfTraceIoControllerUpdate(void); +int APS5_VABI scePerfPmcAperfGetCounter(void); +int APS5_VABI scePerfTraceSpmSdfSetWait(void); +int APS5_VABI scePerfTracePmAddBufferNotifyEvent(void); +int APS5_VABI scePerfTraceAmmUpdate(void); +int APS5_VABI scePerfTraceStart(void); +int APS5_VABI scePerfTraceStop(void); +int APS5_VABI scePerfTraceAprNameEnable(void); +int APS5_VABI scePerfTraceAprNameDelete(void); +int APS5_VABI scePerfTraceIoControllerAddBufferNotifyEvent(void); +int APS5_VABI scePerfTraceSpmSdfStop(void); +int APS5_VABI scePerfTraceAmmStop(void); +int APS5_VABI scePerfTraceIoControllerGetInfo(void); +int APS5_VABI scePerfTraceAprNameGetInfo(void); +int APS5_VABI scePerfPmcUmcGetCounter(void); +int APS5_VABI scePerfTraceSpmSdfGetInfo(void); +int APS5_VABI scePerfTraceIoControllerStart(void); +int APS5_VABI scePerfTraceUpdate(void); +int APS5_VABI scePerfPmcOpen(void); + +} diff --git a/core/libs/prx/libScePerf/libScePerf.hpp b/core/libs/prx/libScePerf/libScePerf.hpp index 6f70f09bee..3521098647 100644 --- a/core/libs/prx/libScePerf/libScePerf.hpp +++ b/core/libs/prx/libScePerf/libScePerf.hpp @@ -1 +1,3 @@ #pragma once + +#include "prx/libScePerf/include/Api.hpp" From d95e8a36a0ed3ee865607045400e4ce20b1c2b09 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:11:44 +0000 Subject: [PATCH 187/318] refactor(libSceRazorCpu_debug): declare the pending exports in the public header Api.hpp declares each export once, with the placeholder signature its stub has, and the umbrella header and Export.cpp include it. --- core/libs/prx/libSceRazorCpu_debug/Export.cpp | 1 + core/libs/prx/libSceRazorCpu_debug/include/Api.hpp | 11 +++++++++++ .../prx/libSceRazorCpu_debug/libSceRazorCpu_debug.hpp | 2 ++ 3 files changed, 14 insertions(+) create mode 100644 core/libs/prx/libSceRazorCpu_debug/include/Api.hpp diff --git a/core/libs/prx/libSceRazorCpu_debug/Export.cpp b/core/libs/prx/libSceRazorCpu_debug/Export.cpp index 329c6b8281..f0a2e8d852 100644 --- a/core/libs/prx/libSceRazorCpu_debug/Export.cpp +++ b/core/libs/prx/libSceRazorCpu_debug/Export.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceRazorCpu_debug/include/Api.hpp" #include #include #include "SceTypes.hpp" diff --git a/core/libs/prx/libSceRazorCpu_debug/include/Api.hpp b/core/libs/prx/libSceRazorCpu_debug/include/Api.hpp new file mode 100644 index 0000000000..b000681917 --- /dev/null +++ b/core/libs/prx/libSceRazorCpu_debug/include/Api.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include "SceTypes.hpp" + +extern "C" { + +int APS5_VABI sceRazorCpuStartCapture(void); +int APS5_VABI sceRazorCpuStopCapture(void); +int APS5_VABI sceRazorCpuSetLiveServerThreadSettings(void); + +} diff --git a/core/libs/prx/libSceRazorCpu_debug/libSceRazorCpu_debug.hpp b/core/libs/prx/libSceRazorCpu_debug/libSceRazorCpu_debug.hpp index 6f70f09bee..eb2f0609ae 100644 --- a/core/libs/prx/libSceRazorCpu_debug/libSceRazorCpu_debug.hpp +++ b/core/libs/prx/libSceRazorCpu_debug/libSceRazorCpu_debug.hpp @@ -1 +1,3 @@ #pragma once + +#include "prx/libSceRazorCpu_debug/include/Api.hpp" From 9d6a8af86880456bc7dbd0462023e39104d8d302 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:12:01 +0000 Subject: [PATCH 188/318] refactor(libSceWorkspace): declare the pending exports in the public header Api.hpp declares each export once, with the placeholder signature its stub has, and the umbrella header and Export.cpp include it. --- core/libs/prx/libSceWorkspace/Export.cpp | 1 + core/libs/prx/libSceWorkspace/include/Api.hpp | 26 +++++++++++++++++++ .../prx/libSceWorkspace/libSceWorkspace.hpp | 2 ++ 3 files changed, 29 insertions(+) create mode 100644 core/libs/prx/libSceWorkspace/include/Api.hpp diff --git a/core/libs/prx/libSceWorkspace/Export.cpp b/core/libs/prx/libSceWorkspace/Export.cpp index a1b7b270e8..4d87aa9956 100644 --- a/core/libs/prx/libSceWorkspace/Export.cpp +++ b/core/libs/prx/libSceWorkspace/Export.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceWorkspace/include/Api.hpp" #include #include #include "SceTypes.hpp" diff --git a/core/libs/prx/libSceWorkspace/include/Api.hpp b/core/libs/prx/libSceWorkspace/include/Api.hpp new file mode 100644 index 0000000000..1aa711e90d --- /dev/null +++ b/core/libs/prx/libSceWorkspace/include/Api.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include "SceTypes.hpp" + +extern "C" { + +int APS5_VABI module_start_nid_postfix(void); +int APS5_VABI sceWorkspaceWriteFile(void); +int APS5_VABI sceWorkspaceFinishWrite(void); +int APS5_VABI sceWorkspaceApplyUpdateForFiles(void); +int APS5_VABI sceWorkspaceApplyUpdateForAllFiles(void); +int APS5_VABI sceWorkspaceIsBlockedByDataTransfer(void); +int APS5_VABI sceWorkspaceMirrorBarrier(void); +int APS5_VABI sceWorkspaceGetNumberOfPendingUpdates(void); +int APS5_VABI module_stop_nid_postfix(void); +int APS5_VABI sceWorkspaceUnmount(void); +int APS5_VABI sceWorkspaceApplyUpdate(void); +int APS5_VABI sceWorkspaceDeletePath(void); +int APS5_VABI sceWorkspaceMount(void); +int APS5_VABI sceWorkspaceGetName(void); +int APS5_VABI sceWorkspaceCreateFile(void); +int APS5_VABI sceWorkspaceApplyUpdateForDirectory(void); +int APS5_VABI sceWorkspaceIsUpdatePending(void); +int APS5_VABI sceWorkspaceGetPendingUpdateEntries(void); + +} diff --git a/core/libs/prx/libSceWorkspace/libSceWorkspace.hpp b/core/libs/prx/libSceWorkspace/libSceWorkspace.hpp index 6f70f09bee..9b189a41b3 100644 --- a/core/libs/prx/libSceWorkspace/libSceWorkspace.hpp +++ b/core/libs/prx/libSceWorkspace/libSceWorkspace.hpp @@ -1 +1,3 @@ #pragma once + +#include "prx/libSceWorkspace/include/Api.hpp" From 2118c99134cc00bb72dc68f2cd96aaba5ddf9180 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:43:25 +0000 Subject: [PATCH 189/318] fix(libkernel): return ESRCH from thread name, priority and affinity calls on a released thread handle --- .../libs/prx/libkernel/Pthread/src/Thread.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index 62dc543ec9..ded8dc80d9 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -554,6 +554,15 @@ std::unique_lock LockLiveGuestThread(const PthreadPrivate* thread) { return {}; } +static std::unique_lock LockRegisteredGuestThread(const PthreadPrivate* thread) { + auto& registry = CpuClocks(); + std::unique_lock lock(registry.lock); + for (const auto& [id, registered] : registry.threads) { + if (registered == thread) return lock; + } + return {}; +} + bool GuestThreadIsLive(const PthreadPrivate* thread) { return LockLiveGuestThread(thread).owns_lock(); } @@ -1272,12 +1281,16 @@ KernelCpumask APS5_VABI sceKernelGetAvailableCpumask(void) { int APS5_VABI scePthreadGetaffinity(Pthread thread, KernelCpumask* mask) { if (!thread || !mask) return SCE_KERNEL_ERROR_EINVAL; + const auto registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; *mask = thread->affinity.load(std::memory_order_relaxed); return SCE_OK; } int APS5_VABI scePthreadGetname(Pthread thread, char* name) { if (!thread || !name) return SCE_KERNEL_ERROR_EINVAL; + const auto registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; std::lock_guard lock(thread->nameLock); const std::size_t length = std::min(thread->name.size(), THREAD_NAME_CAPACITY - 1); std::memcpy(name, thread->name.data(), length); @@ -1287,6 +1300,8 @@ int APS5_VABI scePthreadGetname(Pthread thread, char* name) { int APS5_VABI scePthreadGetprio(Pthread thread, int* prio) { if (!thread || !prio) return SCE_KERNEL_ERROR_EINVAL; + const auto registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; *prio = thread->priority.load(std::memory_order_relaxed); return SCE_OK; } @@ -1297,6 +1312,8 @@ int APS5_VABI scePthreadGetthreadid(void) { int APS5_VABI scePthreadRename(Pthread thread, const char* name) { if (!thread || !name) return SCE_KERNEL_ERROR_EINVAL; + const auto registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; std::lock_guard lock(thread->nameLock); thread->name = name; #ifdef _WIN32 @@ -1309,6 +1326,8 @@ int APS5_VABI scePthreadRename(Pthread thread, const char* name) { int APS5_VABI scePthreadSetaffinity(Pthread thread, KernelCpumask mask) { if (!thread || mask == 0) return SCE_KERNEL_ERROR_EINVAL; + const auto registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; thread->affinity.store(mask, std::memory_order_relaxed); return SCE_OK; } From 2b4cb67755301cc29c11d6420d6ea3d4c6cf12e2 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:43:25 +0000 Subject: [PATCH 190/318] test(libkernel): query a detached thread after its handle was released --- core/libs/prx/libkernel/CMakeLists.txt | 10 +++ .../tests/GuestPthreadHandleAfterExit.cpp | 71 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 core/libs/tests/GuestPthreadHandleAfterExit.cpp diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 4d6c818cfb..ea642fc837 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -738,6 +738,16 @@ if(BUILD_TESTING) set_tests_properties(guest_pthread_join_detach PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pthread_handle_after_exit_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadHandleAfterExit.cpp) + target_include_directories(guest_pthread_handle_after_exit_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_handle_after_exit_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_handle_after_exit_tests) + add_test(NAME guest_pthread_handle_after_exit COMMAND guest_pthread_handle_after_exit_tests) + set_tests_properties(guest_pthread_handle_after_exit PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_handle_after_exit PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_pthread_attr_errors_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadAttrErrors.cpp) target_include_directories(guest_pthread_attr_errors_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_pthread_attr_errors_tests PRIVATE libkernel libc) diff --git a/core/libs/tests/GuestPthreadHandleAfterExit.cpp b/core/libs/tests/GuestPthreadHandleAfterExit.cpp new file mode 100644 index 0000000000..411680441f --- /dev/null +++ b/core/libs/tests/GuestPthreadHandleAfterExit.cpp @@ -0,0 +1,71 @@ +#include "prx/libkernel/include/Api.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" +#include "prx/libkernel/KernelErrors.hpp" +#include +#include +#include +#include + +static constexpr int SCE_OK = 0; + +static void Require(bool condition) { if (!condition) std::abort(); } + +struct Worker { + std::atomic started{false}; + std::atomic release{false}; +}; + +static void* APS5_VABI Wait(void* opaque) { + auto* worker = static_cast(opaque); + worker->started.store(true); + while (!worker->release.load()) std::this_thread::yield(); + return nullptr; +} + +static void RequireAlive(Pthread thread) { + char name[32]; + int priority = 0; + KernelCpumask mask = 0; + Require(scePthreadGetname(thread, name) == SCE_OK); + Require(scePthreadGetprio(thread, &priority) == SCE_OK); + Require(scePthreadGetaffinity(thread, &mask) == SCE_OK && mask != 0); + Require(scePthreadSetaffinity(thread, mask) == SCE_OK); + Require(scePthreadRename(thread, "renamed") == SCE_OK); +} + +static void RequireEnded(Pthread thread) { + char name[32]; + int priority = 0; + KernelCpumask mask = 0; + Require(scePthreadGetname(thread, name) == SCE_KERNEL_ERROR_ESRCH); + Require(scePthreadGetprio(thread, &priority) == SCE_KERNEL_ERROR_ESRCH); + Require(scePthreadGetaffinity(thread, &mask) == SCE_KERNEL_ERROR_ESRCH); + Require(scePthreadSetaffinity(thread, 1) == SCE_KERNEL_ERROR_ESRCH); + Require(scePthreadRename(thread, "renamed") == SCE_KERNEL_ERROR_ESRCH); +} + +static void WaitUntilReleased(Pthread thread) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + int priority = 0; + while (scePthreadGetprio(thread, &priority) != SCE_KERNEL_ERROR_ESRCH) { + Require(std::chrono::steady_clock::now() < deadline); + std::this_thread::yield(); + } +} + +int main() { + RequireAlive(scePthreadSelf()); + + for (int round = 0; round < 20; ++round) { + Worker detached; + Pthread detachedThread = nullptr; + Require(scePthreadCreate(&detachedThread, nullptr, Wait, &detached, "detached") == SCE_OK); + while (!detached.started.load()) std::this_thread::yield(); + RequireAlive(detachedThread); + Require(scePthreadDetach(detachedThread) == SCE_OK); + detached.release.store(true); + WaitUntilReleased(detachedThread); + RequireEnded(detachedThread); + } + return 0; +} From 9f2f105c347f5e3920e2cebfd4bb05e244a14553 Mon Sep 17 00:00:00 2001 From: MarcZX10 Date: Sun, 25 Oct 2026 10:36:28 +0000 Subject: [PATCH 191/318] docs(libkernel): clarify released SCE thread handle answer --- docs/dev/debt/libkernel.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index dd4a9f498f..718555f2b9 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -101,7 +101,7 @@ - [__progname](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - unknown data export - [environ](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - data export pointing to a fixed empty environment so titles that import it load; it is not the environment libc's `getenv` reads, so it lists none of those variables and `setenv`/`putenv` changes do not appear in it - [seteuid, setegid, setuid, setgid, setreuid, setregid, setresuid, setresgid, setgroups, getgroups](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - the guest process is assumed to run as root, with real, effective and saved user and group ids 0. As in FreeBSD's `sys_seteuid`, `sys_setegid`, `sys_setuid`, `sys_setgid`, `sys_setreuid`, `sys_setregid`, `sys_setresuid` and `sys_setresgid`, id 0 (or -1 for an unchanged `setreuid`/`setregid`/`setresuid`/`setresgid` id) is then accepted without a privilege check and changes no credential, so 0 is returned; any other id would change an id, which is not modelled, and throws. `setgroups` keeps the supplementary group list as `kern_setgroups` does (`NGROUPS_MAX` 1023, an empty list leaves the effective group as the only entry, a first entry other than 0 would change the effective group and throws) and `getgroups` reports it. The ids and privileges a title has on a console are unverified -- [pthread_getname_np](../../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - a thread that has already been joined returns `ESRCH` (3) and its output buffer is left untouched, as measured on a PS5 (firmware 13.60) by the author of this entry; since [#4481](https://github.com/boykopovar/AnyPS5/pull/4481) the joined thread's object is retained, so the posix wrapper answers `ESRCH` for it instead of reading the retained name. A null name buffer is answered with `EFAULT` before that, an order the console table does not measure. The SCE sibling `scePthreadGetname` has no measured joined-thread behaviour on the console and still reads the retained object's name, so it is unchanged, as is the pre-existing dangling case of a thread that ended detached (its object is released at exit; see the scePthreadJoin entry) +- [pthread_getname_np](../../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - a thread that has already been joined returns `ESRCH` (3) and its output buffer is left untouched, as measured on a PS5 (firmware 13.60) by the author of this entry; since [#4481](https://github.com/boykopovar/AnyPS5/pull/4481) the joined thread's object is retained, so the posix wrapper answers `ESRCH` for it instead of reading the retained name. A null name buffer is answered with `EFAULT` before that, an order the console table does not measure. The SCE sibling `scePthreadGetname` has no measured joined-thread behaviour on the console and still reads the retained object's name. After a detached thread is released, it now returns `ESRCH` through the thread registry instead of reading freed storage; that released-handle answer is not measured on the console (see the scePthreadJoin entry) - [sceKernelGetOperationMode](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - the signature `(int* mode, int* submode)` comes from PPSA12544, which logs both values after the call and ignores the result. A homebrew payload on a PS5 (firmware 13.60) reads mode 2 and submode 0, nothing else is written, and the result register holds no status, so the function is taken to return nothing. Whether a title process reads the same values is unverified. Null outputs throw. In PPSA12544 the call at 0x16a5889 is followed by a log of both values and a comparison of the second with 1, and when it equals 1 a byte of the title is set; a second call (0x67e404) is made from a function that passes its caller's arguments through, and neither reads the result. - [sceCoredumpGetStopInfoCpu](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) has no active coredump context. Native null/size validation is recovered, including both 8-byte and 16-byte records; valid requests return `NOT_IN_COREDUMP_HANDLER` without accessing the output. Active-handler data retrieval remains unimplemented. One title (PPSA02225) disassembled for this entry calls it once (arguments set up at 0x8d67dd and 0x8d67e1) with a stack buffer and the length 0x10: for 0x81180005 it branches away, for 0 it reads a 32-bit code at offset 8 of the buffer, and for any other nonzero result, which includes the `NOT_IN_COREDUMP_HANDLER` result returned here, it formats an error text containing the code. The call does not show what the console returns outside a coredump handler. - [sceCoredumpGetThreadContextInfo](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) validates a nonnull 144-byte output using the native checks. Active-handler register capture remains unimplemented; valid requests return `NOT_IN_COREDUMP_HANDLER` without accessing the output. From 92bbd3e1c6798554378f026771410efd68c8f5fb Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:25:52 +0000 Subject: [PATCH 192/318] fix(libkernel): read and write the detached state of a guest thread atomically --- core/libs/prx/libkernel/Pthread/include/Pthread.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index d7a80424cb..b535fdb996 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -160,7 +160,7 @@ struct PthreadPrivate { std::string name; std::atomic _finished; void* _retval; - bool _detached; + std::atomic _detached; bool _adopted; Pthread* _handleStorage; std::atomic _joined; From 2567826215d2aa7047df5e9081fdd7b6d3fbc349 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:25:52 +0000 Subject: [PATCH 193/318] test(libkernel): query thread attributes while the thread is being detached --- core/libs/prx/libkernel/CMakeLists.txt | 10 ++++ core/libs/tests/GuestPthreadDetachAttr.cpp | 57 ++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 core/libs/tests/GuestPthreadDetachAttr.cpp diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index ea642fc837..5ed3b0f887 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -748,6 +748,16 @@ if(BUILD_TESTING) set_tests_properties(guest_pthread_handle_after_exit PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pthread_detach_attr_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadDetachAttr.cpp) + target_include_directories(guest_pthread_detach_attr_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_detach_attr_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_detach_attr_tests) + add_test(NAME guest_pthread_detach_attr COMMAND guest_pthread_detach_attr_tests) + set_tests_properties(guest_pthread_detach_attr PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_detach_attr PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_pthread_attr_errors_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadAttrErrors.cpp) target_include_directories(guest_pthread_attr_errors_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_pthread_attr_errors_tests PRIVATE libkernel libc) diff --git a/core/libs/tests/GuestPthreadDetachAttr.cpp b/core/libs/tests/GuestPthreadDetachAttr.cpp new file mode 100644 index 0000000000..b84d74dffd --- /dev/null +++ b/core/libs/tests/GuestPthreadDetachAttr.cpp @@ -0,0 +1,57 @@ +#include "prx/libkernel/include/Api.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" +#include "prx/libkernel/KernelErrors.hpp" +#include +#include +#include +#include + +static constexpr int SCE_OK = 0; +static constexpr int ROUNDS = 50; +static constexpr int DETACHED = 1; + +static void Require(bool condition) { if (!condition) std::abort(); } + +struct Worker { + std::atomic release{false}; + std::atomic finished{false}; +}; + +static void* APS5_VABI Wait(void* opaque) { + auto* worker = static_cast(opaque); + while (!worker->release.load()) std::this_thread::yield(); + worker->finished.store(true); + return nullptr; +} + +int main() { + for (int round = 0; round < ROUNDS; ++round) { + Worker worker; + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, Wait, &worker, "detach_attr") == SCE_OK); + std::atomic observing{true}; + std::atomic started{false}; + std::thread observer([&] { + PthreadAttr attr = nullptr; + Require(scePthreadAttrInit(&attr) == SCE_OK); + while (observing.load()) { + Require(scePthreadAttrGet(thread, &attr) == SCE_OK); + started.store(true); + } + Require(scePthreadAttrDestroy(&attr) == SCE_OK); + }); + while (!started.load()) std::this_thread::yield(); + Require(scePthreadDetach(thread) == SCE_OK); + PthreadAttr attr = nullptr; + Require(scePthreadAttrInit(&attr) == SCE_OK); + Require(scePthreadAttrGet(thread, &attr) == SCE_OK); + int state = -1; + Require(scePthreadAttrGetdetachstate(&attr, &state) == SCE_OK && state == DETACHED); + Require(scePthreadAttrDestroy(&attr) == SCE_OK); + observing.store(false); + observer.join(); + worker.release.store(true); + while (!worker.finished.load()) std::this_thread::yield(); + } + return 0; +} From 1c0cc4c7ce71600d07fbcea839f25fc2559656de Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:48:15 +0000 Subject: [PATCH 194/318] fix(libkernel): return ESRCH from join, detach and cancel of a released thread handle --- core/libs/prx/libkernel/Pthread/src/Thread.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index ded8dc80d9..df1ee795ab 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -1082,6 +1082,7 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread int APS5_VABI JoinThread_nid_no_patch(Pthread thread, void** retval, const KernelTimespec* abstime) { if (!thread) return SCE_KERNEL_ERROR_EINVAL; + if (!LockRegisteredGuestThread(thread)) return SCE_KERNEL_ERROR_ESRCH; if (thread == CurrentGuestState().current) return SCE_KERNEL_ERROR_EDEADLK; if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL; if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH; @@ -1139,6 +1140,7 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) { int APS5_VABI scePthreadDetach(Pthread thread) { if (!thread) return SCE_KERNEL_ERROR_EINVAL; + if (!LockRegisteredGuestThread(thread)) return SCE_KERNEL_ERROR_ESRCH; { std::lock_guard lock(thread->_join_mtx); if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH; @@ -1232,6 +1234,7 @@ void APS5_VABI scePthreadYield() { int APS5_VABI scePthreadCancel(Pthread thread) { if (!thread) return SCE_KERNEL_ERROR_ESRCH; + if (!LockRegisteredGuestThread(thread)) return SCE_KERNEL_ERROR_ESRCH; thread->cancelPending.store(true); for (;;) { { From fda24dad22253efc0d64592c0806139954a299e0 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:48:15 +0000 Subject: [PATCH 195/318] test(libkernel): join, detach and cancel a thread after its handle was released --- core/libs/prx/libkernel/CMakeLists.txt | 10 ++++ .../libs/tests/GuestPthreadReleasedHandle.cpp | 48 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 core/libs/tests/GuestPthreadReleasedHandle.cpp diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 5ed3b0f887..1a175c4cb5 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -758,6 +758,16 @@ if(BUILD_TESTING) set_tests_properties(guest_pthread_detach_attr PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pthread_released_handle_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadReleasedHandle.cpp) + target_include_directories(guest_pthread_released_handle_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_released_handle_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_released_handle_tests) + add_test(NAME guest_pthread_released_handle COMMAND guest_pthread_released_handle_tests) + set_tests_properties(guest_pthread_released_handle PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_released_handle PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_pthread_attr_errors_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadAttrErrors.cpp) target_include_directories(guest_pthread_attr_errors_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_pthread_attr_errors_tests PRIVATE libkernel libc) diff --git a/core/libs/tests/GuestPthreadReleasedHandle.cpp b/core/libs/tests/GuestPthreadReleasedHandle.cpp new file mode 100644 index 0000000000..45ae1cdecf --- /dev/null +++ b/core/libs/tests/GuestPthreadReleasedHandle.cpp @@ -0,0 +1,48 @@ +#include "prx/libkernel/include/Api.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" +#include "prx/libkernel/KernelErrors.hpp" +#include +#include +#include +#include + +static constexpr int SCE_OK = 0; + +static void Require(bool condition) { if (!condition) std::abort(); } + +struct Worker { + std::atomic started{false}; + std::atomic release{false}; +}; + +static void* APS5_VABI Wait(void* opaque) { + auto* worker = static_cast(opaque); + worker->started.store(true); + while (!worker->release.load()) std::this_thread::yield(); + return nullptr; +} + +static void WaitUntilReleased(Pthread thread) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + int priority = 0; + while (scePthreadGetprio(thread, &priority) != SCE_KERNEL_ERROR_ESRCH) { + Require(std::chrono::steady_clock::now() < deadline); + std::this_thread::yield(); + } +} + +int main() { + for (int round = 0; round < 20; ++round) { + Worker worker; + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, Wait, &worker, "released") == SCE_OK); + while (!worker.started.load()) std::this_thread::yield(); + Require(scePthreadDetach(thread) == SCE_OK); + worker.release.store(true); + WaitUntilReleased(thread); + Require(scePthreadJoin(thread, nullptr) == SCE_KERNEL_ERROR_ESRCH); + Require(scePthreadDetach(thread) == SCE_KERNEL_ERROR_ESRCH); + Require(scePthreadCancel(thread) == SCE_KERNEL_ERROR_ESRCH); + } + return 0; +} From 9f6bb41ada2595fb95c11807daea182aa4e78904 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:07:35 +0000 Subject: [PATCH 196/318] fix(libkernel): retain pthread handles during concurrent access --- .../libs/prx/libkernel/Pthread/src/Thread.cpp | 55 ++++++++++++++++--- .../libs/tests/GuestPthreadReleasedHandle.cpp | 11 ++++ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index df1ee795ab..62f3954e80 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -829,6 +829,27 @@ static void ReleaseThread(PthreadPrivate* thread) { delete thread; } +class GuestThreadReference { +public: + GuestThreadReference() = default; + void Retain(PthreadPrivate* value) { + value->references.fetch_add(1, std::memory_order_relaxed); + thread = value; + } + void Reset() { + if (!thread) return; + auto* retained = thread; + thread = nullptr; + ReleaseThread(retained); + } + ~GuestThreadReference() { Reset(); } + GuestThreadReference(const GuestThreadReference&) = delete; + GuestThreadReference& operator=(const GuestThreadReference&) = delete; + +private: + PthreadPrivate* thread = nullptr; +}; + static std::mutex joinedLock; static std::map joinedThreads; @@ -1082,16 +1103,26 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread int APS5_VABI JoinThread_nid_no_patch(Pthread thread, void** retval, const KernelTimespec* abstime) { if (!thread) return SCE_KERNEL_ERROR_EINVAL; - if (!LockRegisteredGuestThread(thread)) return SCE_KERNEL_ERROR_ESRCH; + GuestThreadReference reference; + auto registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; + reference.Retain(thread); if (thread == CurrentGuestState().current) return SCE_KERNEL_ERROR_EDEADLK; if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL; if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH; + registered.unlock(); + reference.Reset(); ThreadCancel::Check(); + registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; + reference.Retain(thread); { std::unique_lock lock(thread->_join_mtx); if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL; + if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH; if (thread->_joining) return SCE_KERNEL_ERROR_EOPNOTSUPP; thread->_joining = true; + registered.unlock(); while (!thread->_finished.load()) { std::optional deadline; if (abstime != nullptr) { @@ -1140,13 +1171,17 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) { int APS5_VABI scePthreadDetach(Pthread thread) { if (!thread) return SCE_KERNEL_ERROR_EINVAL; - if (!LockRegisteredGuestThread(thread)) return SCE_KERNEL_ERROR_ESRCH; + GuestThreadReference reference; + auto registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; + reference.Retain(thread); { std::lock_guard lock(thread->_join_mtx); if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH; if (thread->_detached || thread->_joining) return SCE_KERNEL_ERROR_EINVAL; thread->_detached = true; } + registered.unlock(); #ifdef _WIN32 ReleaseThread(thread); #else @@ -1234,7 +1269,11 @@ void APS5_VABI scePthreadYield() { int APS5_VABI scePthreadCancel(Pthread thread) { if (!thread) return SCE_KERNEL_ERROR_ESRCH; - if (!LockRegisteredGuestThread(thread)) return SCE_KERNEL_ERROR_ESRCH; + GuestThreadReference reference; + auto registered = LockRegisteredGuestThread(thread); + if (!registered) return SCE_KERNEL_ERROR_ESRCH; + reference.Retain(thread); + registered.unlock(); thread->cancelPending.store(true); for (;;) { { @@ -1284,7 +1323,7 @@ KernelCpumask APS5_VABI sceKernelGetAvailableCpumask(void) { int APS5_VABI scePthreadGetaffinity(Pthread thread, KernelCpumask* mask) { if (!thread || !mask) return SCE_KERNEL_ERROR_EINVAL; - const auto registered = LockRegisteredGuestThread(thread); + auto registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; *mask = thread->affinity.load(std::memory_order_relaxed); return SCE_OK; @@ -1292,7 +1331,7 @@ int APS5_VABI scePthreadGetaffinity(Pthread thread, KernelCpumask* mask) { int APS5_VABI scePthreadGetname(Pthread thread, char* name) { if (!thread || !name) return SCE_KERNEL_ERROR_EINVAL; - const auto registered = LockRegisteredGuestThread(thread); + auto registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; std::lock_guard lock(thread->nameLock); const std::size_t length = std::min(thread->name.size(), THREAD_NAME_CAPACITY - 1); @@ -1303,7 +1342,7 @@ int APS5_VABI scePthreadGetname(Pthread thread, char* name) { int APS5_VABI scePthreadGetprio(Pthread thread, int* prio) { if (!thread || !prio) return SCE_KERNEL_ERROR_EINVAL; - const auto registered = LockRegisteredGuestThread(thread); + auto registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; *prio = thread->priority.load(std::memory_order_relaxed); return SCE_OK; @@ -1315,7 +1354,7 @@ int APS5_VABI scePthreadGetthreadid(void) { int APS5_VABI scePthreadRename(Pthread thread, const char* name) { if (!thread || !name) return SCE_KERNEL_ERROR_EINVAL; - const auto registered = LockRegisteredGuestThread(thread); + auto registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; std::lock_guard lock(thread->nameLock); thread->name = name; @@ -1329,7 +1368,7 @@ int APS5_VABI scePthreadRename(Pthread thread, const char* name) { int APS5_VABI scePthreadSetaffinity(Pthread thread, KernelCpumask mask) { if (!thread || mask == 0) return SCE_KERNEL_ERROR_EINVAL; - const auto registered = LockRegisteredGuestThread(thread); + auto registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; thread->affinity.store(mask, std::memory_order_relaxed); return SCE_OK; diff --git a/core/libs/tests/GuestPthreadReleasedHandle.cpp b/core/libs/tests/GuestPthreadReleasedHandle.cpp index 45ae1cdecf..84edc39549 100644 --- a/core/libs/tests/GuestPthreadReleasedHandle.cpp +++ b/core/libs/tests/GuestPthreadReleasedHandle.cpp @@ -38,8 +38,19 @@ int main() { Require(scePthreadCreate(&thread, nullptr, Wait, &worker, "released") == SCE_OK); while (!worker.started.load()) std::this_thread::yield(); Require(scePthreadDetach(thread) == SCE_OK); + std::atomic finished{false}; + std::thread observer([&] { + while (!finished.load()) { + int priority = 0; + const int result = scePthreadGetprio(thread, &priority); + Require(result == SCE_OK || result == SCE_KERNEL_ERROR_ESRCH); + std::this_thread::yield(); + } + }); worker.release.store(true); WaitUntilReleased(thread); + finished.store(true); + observer.join(); Require(scePthreadJoin(thread, nullptr) == SCE_KERNEL_ERROR_ESRCH); Require(scePthreadDetach(thread) == SCE_KERNEL_ERROR_ESRCH); Require(scePthreadCancel(thread) == SCE_KERNEL_ERROR_ESRCH); From e78214475e96dc2165f5da7bb48399a6c14258e9 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:52:36 +0000 Subject: [PATCH 197/318] fix(libkernel): reject thread reference resurrection --- core/libs/prx/libkernel/CMakeLists.txt | 10 ++ .../libs/prx/libkernel/Pthread/src/Thread.cpp | 34 ++++-- .../tests/GuestPthreadFinalReleaseRace.cpp | 104 ++++++++++++++++++ 3 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 core/libs/tests/GuestPthreadFinalReleaseRace.cpp diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 1a175c4cb5..ae0e9d9b4a 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -768,6 +768,16 @@ if(BUILD_TESTING) set_tests_properties(guest_pthread_released_handle PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pthread_final_release_race_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadFinalReleaseRace.cpp) + target_include_directories(guest_pthread_final_release_race_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_final_release_race_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_final_release_race_tests) + add_test(NAME guest_pthread_final_release_race COMMAND guest_pthread_final_release_race_tests) + set_tests_properties(guest_pthread_final_release_race PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_final_release_race PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_pthread_attr_errors_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadAttrErrors.cpp) target_include_directories(guest_pthread_attr_errors_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_pthread_attr_errors_tests PRIVATE libkernel libc) diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index 62f3954e80..afe84bb4b0 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -815,11 +815,20 @@ static void RunThread(std::unique_ptr args) { CurrentGuestState().current = nullptr; } +using FinalThreadReleaseTestHook = void (*)(Pthread); +static std::atomic finalThreadReleaseTestTarget{nullptr}; +static std::atomic finalThreadReleaseTestHook{nullptr}; + static void ReleaseThread(PthreadPrivate* thread) { if (thread->_adopted) return; if (thread->references.fetch_sub(1, std::memory_order_acq_rel) != 1) return; + auto* expected = thread; + if (finalThreadReleaseTestTarget.compare_exchange_strong(expected, nullptr, std::memory_order_acq_rel)) { + if (const auto hook = finalThreadReleaseTestHook.exchange(nullptr, std::memory_order_acq_rel)) + hook(reinterpret_cast(thread)); + } #ifdef _WIN32 if (thread->wakeEvent) CloseHandle(static_cast(thread->wakeEvent)); @@ -832,9 +841,15 @@ static void ReleaseThread(PthreadPrivate* thread) { class GuestThreadReference { public: GuestThreadReference() = default; - void Retain(PthreadPrivate* value) { - value->references.fetch_add(1, std::memory_order_relaxed); - thread = value; + bool Retain(PthreadPrivate* value) { + auto references = value->references.load(std::memory_order_relaxed); + while (references != 0) { + if (value->references.compare_exchange_weak(references, references + 1, std::memory_order_acq_rel, std::memory_order_relaxed)) { + thread = value; + return true; + } + } + return false; } void Reset() { if (!thread) return; @@ -850,6 +865,11 @@ private: PthreadPrivate* thread = nullptr; }; +extern "C" void APS5_Test_SetFinalThreadReleaseHook(Pthread thread, FinalThreadReleaseTestHook hook) { + finalThreadReleaseTestHook.store(hook, std::memory_order_relaxed); + finalThreadReleaseTestTarget.store(static_cast(thread), std::memory_order_release); +} + static std::mutex joinedLock; static std::map joinedThreads; @@ -1106,7 +1126,7 @@ int APS5_VABI JoinThread_nid_no_patch(Pthread thread, void** retval, const Kerne GuestThreadReference reference; auto registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; - reference.Retain(thread); + if (!reference.Retain(thread)) return SCE_KERNEL_ERROR_ESRCH; if (thread == CurrentGuestState().current) return SCE_KERNEL_ERROR_EDEADLK; if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL; if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH; @@ -1115,7 +1135,7 @@ int APS5_VABI JoinThread_nid_no_patch(Pthread thread, void** retval, const Kerne ThreadCancel::Check(); registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; - reference.Retain(thread); + if (!reference.Retain(thread)) return SCE_KERNEL_ERROR_ESRCH; { std::unique_lock lock(thread->_join_mtx); if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL; @@ -1174,7 +1194,7 @@ int APS5_VABI scePthreadDetach(Pthread thread) { GuestThreadReference reference; auto registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; - reference.Retain(thread); + if (!reference.Retain(thread)) return SCE_KERNEL_ERROR_ESRCH; { std::lock_guard lock(thread->_join_mtx); if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH; @@ -1272,7 +1292,7 @@ int APS5_VABI scePthreadCancel(Pthread thread) { GuestThreadReference reference; auto registered = LockRegisteredGuestThread(thread); if (!registered) return SCE_KERNEL_ERROR_ESRCH; - reference.Retain(thread); + if (!reference.Retain(thread)) return SCE_KERNEL_ERROR_ESRCH; registered.unlock(); thread->cancelPending.store(true); for (;;) { diff --git a/core/libs/tests/GuestPthreadFinalReleaseRace.cpp b/core/libs/tests/GuestPthreadFinalReleaseRace.cpp new file mode 100644 index 0000000000..76007755e6 --- /dev/null +++ b/core/libs/tests/GuestPthreadFinalReleaseRace.cpp @@ -0,0 +1,104 @@ +#include "prx/libkernel/include/Api.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" +#include "prx/libkernel/KernelErrors.hpp" +#include +#include +#include +#include +#include +#include + +static constexpr int SCE_OK = 0; + +extern "C" void APS5_Test_SetFinalThreadReleaseHook(Pthread thread, void (*hook)(Pthread)); + +static void Require(bool condition) { if (!condition) std::abort(); } + +struct Worker { + std::atomic started{false}; + std::atomic release{false}; +}; + +static void* APS5_VABI Wait(void* opaque) { + auto* worker = static_cast(opaque); + worker->started.store(true); + while (!worker->release.load()) std::this_thread::yield(); + return nullptr; +} + +struct FinalReleaseGate { + std::mutex mutex; + std::condition_variable changed; + Pthread thread = nullptr; + bool reached = false; + bool release = false; +}; + +static FinalReleaseGate gate; + +static void HoldFinalRelease(Pthread thread) { + std::unique_lock lock(gate.mutex); + Require(gate.thread == thread); + gate.reached = true; + gate.changed.notify_all(); + gate.changed.wait(lock, [] { return gate.release; }); +} + +static void WaitForFinalRelease() { + std::unique_lock lock(gate.mutex); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + Require(gate.changed.wait_until(lock, deadline, [] { return gate.reached; })); +} + +static void ResumeFinalRelease() { + { + std::lock_guard lock(gate.mutex); + gate.release = true; + } + gate.changed.notify_all(); +} + +static void WaitUntilReleased(Pthread thread) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + int priority = 0; + while (scePthreadGetprio(thread, &priority) != SCE_KERNEL_ERROR_ESRCH) { + Require(std::chrono::steady_clock::now() < deadline); + std::this_thread::yield(); + } +} + +static void RequireAdmissionRejectsFinalRelease(int operation) { + Worker worker; + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, Wait, &worker, "release_race") == SCE_OK); + while (!worker.started.load()) std::this_thread::yield(); + Require(scePthreadDetach(thread) == SCE_OK); + + { + std::lock_guard lock(gate.mutex); + gate.thread = thread; + gate.reached = false; + gate.release = false; + } + APS5_Test_SetFinalThreadReleaseHook(thread, HoldFinalRelease); + worker.release.store(true); + WaitForFinalRelease(); + + int result = SCE_OK; + if (operation == 0) result = scePthreadJoin(thread, nullptr); + else if (operation == 1) result = scePthreadDetach(thread); + else result = scePthreadCancel(thread); + + ResumeFinalRelease(); + WaitUntilReleased(thread); + Require(result == SCE_KERNEL_ERROR_ESRCH); +} + +int main() { + for (int round = 0; round < 20; ++round) { + RequireAdmissionRejectsFinalRelease(0); + RequireAdmissionRejectsFinalRelease(1); + RequireAdmissionRejectsFinalRelease(2); + } + return 0; +} From 29dadd05c90663fa9122f04dfb3c50974fc6c797 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:08:00 +0000 Subject: [PATCH 198/318] fix(agc): synchronize dispatch profile counters --- .../src/Driver/Dispatch/Dispatch.cpp | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index a08d54121c..fe582e0496 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -15,6 +15,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "Optimization/ResourceProgram.hpp" #include +#include #include #include #include @@ -183,9 +184,9 @@ void Driver::dispatch(QueueState& queue, std::span packet, {0, 0, 0, threadGroup != std::array{} ? 0u : 128u} }; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - static double captureMs = 0, keyMs = 0, recompileMs = 0, deviceMs = 0; - static std::uint64_t cacheHits = 0; - static std::uint64_t dispatches = 0; + static std::atomic captureMs{0}, keyMs{0}, recompileMs{0}, deviceMs{0}; + static std::atomic cacheHits{0}; + static std::atomic dispatches{0}; auto lap = std::chrono::steady_clock::now(); std::array phaseMs{}; @@ -362,7 +363,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } if (cached) { require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader"); - captureMs += phaseTiming.Elapsed(); + if (profile) captureMs.fetch_add(phaseTiming.Elapsed(), std::memory_order_relaxed); } else if (!fromAhead) { shaderMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan); std::uint64_t forgetAtCapture = 0; @@ -381,7 +382,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, captured = shaderMemory->Regions(); request.context.memory = captured; if (traceCapSync()) traceCapture("dispatch-capture", address, submission.queue, captured, Graphics::Recorder::ThreadWaitedMs() - waitedBefore); - captureMs += phaseTiming.Elapsed(); + if (profile) captureMs.fetch_add(phaseTiming.Elapsed(), std::memory_order_relaxed); phaseTiming.Phase(PhaseCapture); if (dumpShaders) static_cast(dumpRequest(address, request)); const auto started = std::chrono::steady_clock::now(); @@ -390,11 +391,13 @@ void Driver::dispatch(QueueState& queue, std::span packet, timing.Mark("capture_resources"); compiledResult = invocation.Materialize(*capture); timing.Mark("materialize"); - if (compiledResult->cacheHit) ++cacheHits; + if (profile && compiledResult->cacheHit) cacheHits.fetch_add(1, std::memory_order_relaxed); const auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); - static double totalMs = 0; - totalMs += elapsed; - if (profile && elapsed > 200) std::fprintf(stderr, "[gpu] compute shader 0x%llx recompile took %.0f ms (%zu SPIR-V words, %zu captured regions, total %.1f s)\n", static_cast(address), elapsed, compiledResult->spirv.size(), captured.size(), totalMs / 1000); + static std::atomic totalMs{0}; + if (profile) { + const auto total = totalMs.fetch_add(elapsed, std::memory_order_relaxed) + elapsed; + if (elapsed > 200) std::fprintf(stderr, "[gpu] compute shader 0x%llx recompile took %.0f ms (%zu SPIR-V words, %zu captured regions, total %.1f s)\n", static_cast(address), elapsed, compiledResult->spirv.size(), captured.size(), total / 1000); + } } catch (const std::exception& error) { std::string dump; if (dumpShaders) { @@ -411,7 +414,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, else std::snprintf(where, sizeof(where), "compute shader 0x%llx (%s): ", static_cast(address), dump.c_str()); throw std::runtime_error(where + reason); } - recompileMs += phaseTiming.Elapsed(); + if (profile) recompileMs.fetch_add(phaseTiming.Elapsed(), std::memory_order_relaxed); phaseTiming.Phase(PhaseRecompile); insertDispatch(address, key, noDispatchCache || !CacheableResult(*compiledResult), profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); } @@ -649,7 +652,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } phaseTiming.Phase(PhaseDevice); if (noteWrites && !writerKeyedEvidence()) noteWrittenBuffers(address, submission.queue, compiled); - deviceMs += phaseTiming.Elapsed(); + if (profile) deviceMs.fetch_add(phaseTiming.Elapsed(), std::memory_order_relaxed); phaseTiming.Phase(PhaseTail); if (profile) { auto& pending = pendingDispatchPhases(); @@ -660,7 +663,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, pending.ms = phaseMs; pending.tailAt = phaseLap; } - if (profile && ++dispatches % 100 == 0) AgcDriver::ProfilePrint_nid_no_patch( "[gpu] %llu dispatches (%llu dispatch cache hits, %llu evictions, %llu recompile cache hits): capture %.1f s, cache key %.1f s, recompile %.1f s, device %.1f s\n", static_cast(dispatches), static_cast(dispatchCacheHits), static_cast(dispatchCacheEvictions), static_cast(cacheHits), captureMs / 1000, keyMs / 1000, recompileMs / 1000, deviceMs / 1000); + if (profile && dispatches.fetch_add(1, std::memory_order_relaxed) % 100 == 99) AgcDriver::ProfilePrint_nid_no_patch( "[gpu] %llu dispatches (%llu dispatch cache hits, %llu evictions, %llu recompile cache hits): capture %.1f s, cache key %.1f s, recompile %.1f s, device %.1f s\n", static_cast(dispatches.load(std::memory_order_relaxed)), static_cast(dispatchCacheHits), static_cast(dispatchCacheEvictions), static_cast(cacheHits.load(std::memory_order_relaxed)), captureMs.load(std::memory_order_relaxed) / 1000, keyMs.load(std::memory_order_relaxed) / 1000, recompileMs.load(std::memory_order_relaxed) / 1000, deviceMs.load(std::memory_order_relaxed) / 1000); } } From acfb296f11c7bd55ef8d400a075a8edf815b71c1 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:05:50 +0000 Subject: [PATCH 199/318] fix(agc): read the dispatch cache counters under their mutex in the dispatch profile report --- .../Execution/src/Driver/Dispatch/Dispatch.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index fe582e0496..1bc8f62e80 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -663,7 +663,16 @@ void Driver::dispatch(QueueState& queue, std::span packet, pending.ms = phaseMs; pending.tailAt = phaseLap; } - if (profile && dispatches.fetch_add(1, std::memory_order_relaxed) % 100 == 99) AgcDriver::ProfilePrint_nid_no_patch( "[gpu] %llu dispatches (%llu dispatch cache hits, %llu evictions, %llu recompile cache hits): capture %.1f s, cache key %.1f s, recompile %.1f s, device %.1f s\n", static_cast(dispatches.load(std::memory_order_relaxed)), static_cast(dispatchCacheHits), static_cast(dispatchCacheEvictions), static_cast(cacheHits.load(std::memory_order_relaxed)), captureMs.load(std::memory_order_relaxed) / 1000, keyMs.load(std::memory_order_relaxed) / 1000, recompileMs.load(std::memory_order_relaxed) / 1000, deviceMs.load(std::memory_order_relaxed) / 1000); + if (profile && dispatches.fetch_add(1, std::memory_order_relaxed) % 100 == 99) { + std::uint64_t hits = 0; + std::uint64_t evictions = 0; + { + std::lock_guard cacheLock(dispatchCacheMutex); + hits = dispatchCacheHits; + evictions = dispatchCacheEvictions; + } + AgcDriver::ProfilePrint_nid_no_patch( "[gpu] %llu dispatches (%llu dispatch cache hits, %llu evictions, %llu recompile cache hits): capture %.1f s, cache key %.1f s, recompile %.1f s, device %.1f s\n", static_cast(dispatches.load(std::memory_order_relaxed)), static_cast(hits), static_cast(evictions), static_cast(cacheHits.load(std::memory_order_relaxed)), captureMs.load(std::memory_order_relaxed) / 1000, keyMs.load(std::memory_order_relaxed) / 1000, recompileMs.load(std::memory_order_relaxed) / 1000, deviceMs.load(std::memory_order_relaxed) / 1000); + } } } From fa82390090b798eaecd1c46c897ae0d0f9728013 Mon Sep 17 00:00:00 2001 From: Kyon Date: Tue, 27 Oct 2026 05:46:58 +0000 Subject: [PATCH 200/318] fix(libSceJson2): preserve descendant ownership and legacy callback progress --- core/libs/prx/libSceJson2/CMakeLists.txt | 48 ++++--- core/libs/prx/libSceJson2/Export.cpp | 65 +++++++-- .../libs/tests/GuestJsonCallbackLifecycle.cpp | 113 ++------------- core/libs/tests/GuestJsonCallbackThreads.cpp | 129 ++++++++++++++++++ .../tests/GuestJsonCompositeDestruction.cpp | 10 +- core/libs/tests/GuestJsonFile.cpp | 5 +- .../tests/GuestJsonKernelInitialization.cpp | 10 +- core/libs/tests/GuestJsonLegacyHandlers.cpp | 2 +- .../tests/GuestJsonTerminationExceptions.cpp | 16 +-- core/libs/tests/GuestJsonValueOwnership.cpp | 24 ++++ docs/dev/debt/libSceJson2.md | 3 + 11 files changed, 258 insertions(+), 167 deletions(-) create mode 100644 core/libs/tests/GuestJsonCallbackThreads.cpp diff --git a/core/libs/prx/libSceJson2/CMakeLists.txt b/core/libs/prx/libSceJson2/CMakeLists.txt index 2848311d0e..b8d29b07c4 100644 --- a/core/libs/prx/libSceJson2/CMakeLists.txt +++ b/core/libs/prx/libSceJson2/CMakeLists.txt @@ -6,9 +6,16 @@ configure_windows_unwind(libSceJson2) set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/include/Api.hpp") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/Api.hpp" jsonDeclarations REGEX "_ZN") set(jsonExports "") +set(jsonWindowsExports "") foreach(declaration IN LISTS jsonDeclarations) string(REGEX MATCH "(_ZN[A-Za-z0-9_]+)" match "${declaration}") - list(APPEND jsonExports "${CMAKE_MATCH_1}") + set(symbol "${CMAKE_MATCH_1}") + list(APPEND jsonExports "${symbol}") + if(declaration MATCHES "\\(") + list(APPEND jsonWindowsExports "${symbol}") + else() + list(APPEND jsonWindowsExports "${symbol} DATA") + endif() endforeach() if(APPLE) set(exportList "") @@ -18,7 +25,7 @@ if(APPLE) file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/json2.exports" "${exportList}") target_link_options(libSceJson2 PRIVATE "LINKER:-exported_symbols_list,${CMAKE_CURRENT_BINARY_DIR}/json2.exports") elseif(WIN32) - string(JOIN "\n" exportList ${jsonExports}) + string(JOIN "\n" exportList ${jsonWindowsExports}) file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/json2.def" "EXPORTS\n${exportList}\n") target_sources(libSceJson2 PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/json2.def") else() @@ -62,29 +69,6 @@ if(BUILD_TESTING) endif() endforeach() endforeach() - if(flavor STREQUAL "legacy") - set(retired_positions 3) - else() - set(retired_positions 2 3) - endif() - foreach(position IN LISTS retired_positions) - foreach(boundary before after) - add_test(NAME guest_json_${flavor}_termination_retired_mutex_${boundary}_${position} COMMAND ${exception_target} ${position} ${boundary} refuse-mutex) - if(WIN32 AND flavor STREQUAL "legacy") - set_tests_properties(guest_json_${flavor}_termination_retired_mutex_${boundary}_${position} PROPERTIES ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:$;PATH=path_list_prepend:$") - endif() - endforeach() - endforeach() - foreach(position 4 7 10) - foreach(boundary before after) - add_test(NAME guest_json_${flavor}_termination_retired_header_${boundary}_${position} COMMAND ${exception_target} ${position} ${boundary} refuse-header) - if(WIN32 AND flavor STREQUAL "legacy") - set_tests_properties(guest_json_${flavor}_termination_retired_header_${boundary}_${position} PROPERTIES ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:$;PATH=path_list_prepend:$") - endif() - endforeach() - endforeach() foreach(position 1 5 6 8 9 11 12) add_test(NAME guest_json_${flavor}_termination_retry_${position} COMMAND ${exception_target} ${position} before retry) if(WIN32 AND flavor STREQUAL "legacy") @@ -466,3 +450,17 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + find_package(Threads REQUIRED) + add_test_executable(guest_json_callback_threads_tests ${LIBS_INCLUDE_DIR}/tests/GuestJsonCallbackThreads.cpp Export.cpp) + target_include_directories(guest_json_callback_threads_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_json_callback_threads_tests PRIVATE libkernel libc Threads::Threads) + configure_windows_unwind(guest_json_callback_threads_tests) + add_test(NAME guest_json_callback_threads COMMAND guest_json_callback_threads_tests) + set_tests_properties(guest_json_callback_threads PROPERTIES TIMEOUT 10) + if(WIN32) + set_tests_properties(guest_json_callback_threads PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index a5b9f9dd3e..096e25b17d 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -174,7 +174,6 @@ public: AllocationOperation() { auto& state = State(); std::lock_guard lock(state.mutex); - KernelMutexLock kernelLock(state.internalMutex); ++state.allocationDepth; } ~AllocationOperation() { if (active) Finish(); } @@ -183,13 +182,11 @@ public: static bool Failed() { auto& state = State(); std::lock_guard lock(state.mutex); - KernelMutexLock kernelLock(state.internalMutex); return state.allocationFailed; } bool Finish() { auto& state = State(); std::lock_guard lock(state.mutex); - KernelMutexLock kernelLock(state.internalMutex); const bool failed = state.allocationFailed; if (--state.allocationDepth == 0) state.allocationFailed = false; active = false; @@ -230,7 +227,6 @@ void* Allocate(const AllocationContext& context, std::size_t size) { } auto& state = State(); std::lock_guard lock(state.mutex); - KernelMutexLock kernelLock(state.internalMutex); if (state.allocationDepth) state.allocationFailed = true; } return memory; @@ -1300,7 +1296,36 @@ void SetType(Node& n, ValueType type); void AssignArray(Array& destination, const Array& source); void AssignObject(Object& destination, const Object& source); +bool HasPayloadStorage(const Node& value) { + if (value.type == TypeString) return value.string && value.string->text; + if (value.type == TypeArray) return value.array && value.array->items; + if (value.type == TypeObject) return value.object && value.object->items; + return value.type < TypeString; +} + +bool OwnsPayload(const Node& owner, std::uintptr_t payload) { + if (owner.type < TypeString || !owner.payload) return false; + if (owner.payload == payload) return true; + if (owner.type == TypeArray && owner.array->items) { + for (const auto& item : *owner.array->items) + if (OwnsPayload(item, payload)) return true; + } else if (owner.type == TypeObject && owner.object->items) { + for (const auto& pair : *owner.object->items) + if (OwnsPayload(pair.value, payload)) return true; + } + return false; +} + void AssignPayload(Node& destination, const Node& source) { + if (source.type >= TypeString && HasPayloadStorage(source) && destination.type != source.type && OwnsPayload(destination, source.payload)) { + Node temporary{}; + try { + AssignPayload(temporary, source); + AssignPayload(destination, temporary); + } catch (...) { Destroy(temporary); throw; } + Destroy(temporary); + return; + } if (source.type == TypeNull) { SetType(destination, TypeNull); return; } if (source.type == TypeBoolean) { const auto value = source.boolean; @@ -1343,6 +1368,16 @@ void AssignPayload(Node& destination, const Node& source) { } void AssignNode(Node& destination, const Node& source) { + for (auto* parent = source.parent; parent; parent = parent->parent) { + if (parent != &destination || !HasPayloadStorage(source)) continue; + Node temporary{}; + try { + AssignNode(temporary, source); + AssignNode(destination, temporary); + } catch (...) { Destroy(temporary); throw; } + Destroy(temporary); + return; + } AssignPayload(destination, source); if (!ModernValues()) CopyLegacyHandler(destination, source); } @@ -2926,6 +2961,10 @@ Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value* self, const Value* other) { return self; } void APS5_VABI _ZN3sce4Json5Value4swapERS1_(Value* self, Value* other) { + for (auto* parent = other->parent; parent; parent = parent->parent) + if (parent == self) NotImplemented_nid_no_patch(__func__); + for (auto* parent = self->parent; parent; parent = parent->parent) + if (parent == other) NotImplemented_nid_no_patch(__func__); Node temporary{}; try { AssignNode(temporary, NodeOf(*self)); @@ -2952,18 +2991,24 @@ void APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { } void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { if (!text->text) return; - SetType(NodeOf(*self), TypeString); - if (self->type == TypeString) AssignText(*self->string, View(*text), true); + Value source{}; + source.type = TypeString; + source.string = const_cast(text); + AssignPayload(*self, source); } void APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { if (!array->items) return; - SetType(*self, TypeArray); - if (self->array) AssignArray(*self->array, *array); + Value source{}; + source.type = TypeArray; + source.array = const_cast(array); + AssignPayload(*self, source); } void APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value* self, const Object* object) { if (!object->items) return; - SetType(*self, TypeObject); - if (self->object) AssignObject(*self->object, *object); + Value source{}; + source.type = TypeObject; + source.object = const_cast(object); + AssignPayload(*self, source); } void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value* self, const Value* other) { AssignNode(NodeOf(*self), NodeOf(*other)); diff --git a/core/libs/tests/GuestJsonCallbackLifecycle.cpp b/core/libs/tests/GuestJsonCallbackLifecycle.cpp index 4be4597ae4..1575445c3b 100644 --- a/core/libs/tests/GuestJsonCallbackLifecycle.cpp +++ b/core/libs/tests/GuestJsonCallbackLifecycle.cpp @@ -12,12 +12,11 @@ using namespace SceJson2; struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; -using FloatHandler = Value* (APS5_VABI*)(Value*, std::int32_t, double, const Value*, void*, bool*); static void Require(bool condition, const char* message) { if (!condition) { std::fprintf(stderr, "%s\n", message); std::abort(); } } -enum class NoticeAction { None, Terminate, RejectTermination }; -enum class AllocatorAction { None, RejectAllocate, RejectFree, TerminateFree }; +enum class NoticeAction { None, Terminate }; +enum class AllocatorAction { None, TerminateFree }; struct LifecycleTrace { std::map live; std::size_t generation = 0; @@ -26,17 +25,10 @@ struct LifecycleTrace { NoticeAction noticeAction = NoticeAction::None; AllocatorAction allocatorAction = AllocatorAction::None; unsigned notices = 0; - unsigned refused = 0; unsigned freeCalls = 0; - unsigned teardownTarget = 0; }; -static void RejectOwnerTermination(LifecycleTrace& trace); static void* APS5_VABI AllocateLifecycle(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); - if (trace.allocatorAction == AllocatorAction::RejectAllocate) { - trace.allocatorAction = AllocatorAction::None; - RejectOwnerTermination(trace); - } if (trace.failNext) { trace.failNext = false; return nullptr; } void* memory = std::malloc(bytes); Require(memory != nullptr, "Tracking allocator requires host memory"); @@ -47,14 +39,7 @@ static void* APS5_VABI AllocateLifecycle(void*, std::size_t bytes, void* context static void APS5_VABI FreeLifecycle(void*, void* memory, void* context) { auto& trace = *static_cast(context); ++trace.freeCalls; - if (trace.teardownTarget && trace.freeCalls == trace.teardownTarget) { - trace.teardownTarget = 0; - RejectOwnerTermination(trace); - } - if (trace.allocatorAction == AllocatorAction::RejectFree) { - trace.allocatorAction = AllocatorAction::None; - RejectOwnerTermination(trace); - } else if (trace.allocatorAction == AllocatorAction::TerminateFree) { + if (trace.allocatorAction == AllocatorAction::TerminateFree) { trace.allocatorAction = AllocatorAction::None; const auto identity = trace.live.at(memory); try { @@ -66,18 +51,6 @@ static void APS5_VABI FreeLifecycle(void*, void* memory, void* context) { Require(trace.live.erase(memory) == 1, "Lifecycle cleanup must release each allocation through its owner exactly once"); std::free(memory); } -static void RejectOwnerTermination(LifecycleTrace& trace) { - const auto before = trace.live; - bool rejected = false; - try { _ZN3sce4Json11Initializer9terminateEv(trace.owner); } - catch (const std::runtime_error& error) { - Require(std::strstr(error.what(), "terminate"), "Unsafe callback termination must use the explicit unsupported boundary"); - rejected = true; - } - Require(rejected, "Unsafe callback termination must be explicitly rejected before teardown"); - Require(*trace.owner == 1 && trace.live == before, "Rejected callback termination must preserve the entire module allocation generation and active byte"); - ++trace.refused; -} static void APS5_VABI NotifyLifecycle(void*, int error, std::size_t bytes, void* context) { auto& trace = *static_cast(context); Require(static_cast(error) == 0x80848102 && bytes == (trace.noticeAction == NoticeAction::Terminate ? 8u : 72u), "Lifecycle notifier must receive the actual failed allocation size and module error"); @@ -86,12 +59,11 @@ static void APS5_VABI NotifyLifecycle(void*, int error, std::size_t bytes, void* try { Require(_ZN3sce4Json11Initializer9terminateEv(trace.owner) == 0 && *trace.owner == 0 && trace.live.empty(), "A standalone Malloc notifier must retain reference-supported module termination"); } catch (const std::runtime_error&) { Require(false, "Reference-supported standalone notifier termination must not be rejected"); } - } else if (trace.noticeAction == NoticeAction::RejectTermination) RejectOwnerTermination(trace); - else Require(false, "Healthy lifecycle operations must not unexpectedly fail allocation"); + } else Require(false, "Healthy lifecycle operations must not unexpectedly fail allocation"); } static void* const LifecycleVtable[]{nullptr, nullptr, reinterpret_cast(AllocateLifecycle), reinterpret_cast(FreeLifecycle), reinterpret_cast(NotifyLifecycle)}; struct LifecycleAllocator { void* const* vtable = LifecycleVtable; }; -struct RecoveryContext { LifecycleTrace* trace; bool terminateOwner; unsigned calls = 0; }; +struct RecoveryContext { LifecycleTrace* trace; unsigned calls = 0; }; static Value* APS5_VABI RecoverLifecycle(Value* result, std::int32_t function, double number, const Value* parent, void* context, bool* error) { auto& state = *static_cast(context); Require(function == 1 && number == std::numeric_limits::infinity() && !parent && !error, "Lifecycle recovery must preserve its assignment callback ABI"); @@ -99,75 +71,43 @@ static Value* APS5_VABI RecoverLifecycle(Value* result, std::int32_t function, d std::uint8_t inactive = 0; Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&inactive, nullptr)) == 0x80848111 && inactive == 0, "Recovery reentry must reject a competing initializer before parameter validation"); Require(static_cast(_ZN3sce4Json11Initializer9terminateEv(&inactive)) == 0x80848110 && *state.trace->owner == 1, "Inactive termination must preserve active recovery and module ownership"); - if (state.terminateOwner) RejectOwnerTermination(*state.trace); _ZN3sce4Json5ValueC1El(result, 7); return result; } -static void CheckRecovery(bool terminateOwner) { +static void CheckRecovery() { LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; std::uint8_t initializer = 0; trace.owner = &initializer; _ZN3sce4Json11InitializerC1Ev(&initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0, "Initialize recovery lifecycle contracts"); Value value{}; _ZN3sce4Json5ValueC1Ev(&value); - RecoveryContext context{&trace, terminateOwner}; + RecoveryContext context{&trace}; Require(_ZN3sce4Json5Value22setSpecialFloatHandlerEPFKS1_NS0_12FunctionTypeEdPS2_PvPbES5_(&value, RecoverLifecycle, &context) == 0, "Register lifecycle recovery"); const auto baseline = trace.live; void* auxiliary = value.auxiliary; for (unsigned attempt = 1; attempt <= 2; ++attempt) { _ZN3sce4Json5Value3setEd(&value, std::numeric_limits::infinity()); - Require(value.type == 2 && value.payload == 7 && value.auxiliary == auxiliary && context.calls == attempt && trace.refused == (terminateOwner ? attempt : 0) && initializer == 1 && trace.live == baseline, + Require(value.type == 2 && value.payload == 7 && value.auxiliary == auxiliary && context.calls == attempt && initializer == 1 && trace.live == baseline, "Handled callback lifecycle operations must preserve recovery metadata, repeatability and complete module ownership"); } _ZN3sce4Json5ValueD1Ev(&value); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Ordinary termination must work after recovery callback reentry"); } -static void CheckNotice(bool activeOperation) { +static void CheckNotice() { LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; std::uint8_t initializer = 0; trace.owner = &initializer; _ZN3sce4Json11InitializerC1Ev(&initializer); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0, "Initialize notifier lifecycle contracts"); - const auto baseline = trace.live; - trace.noticeAction = activeOperation ? NoticeAction::RejectTermination : NoticeAction::Terminate; + trace.noticeAction = NoticeAction::Terminate; trace.failNext = true; - if (activeOperation) { - Value value{}; _ZN3sce4Json5ValueC1Ev(&value); - Require(static_cast(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, "true", 4)) == 0x80848102 && value.type == 1 && (value.payload & 0xff) == 1 && !value.auxiliary && initializer == 1 && trace.live == baseline, - "Rejected notifier termination must retain normal parser out-of-memory behavior and module ownership"); - Require(trace.refused == 1 && trace.notices == 1, "Active notifier termination must be rejected exactly once at the actual failure boundary"); - trace.noticeAction = NoticeAction::None; - Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, "true", 4) == 0 && value.auxiliary && initializer == 1, "Parser retry must work after a caught notifier lifecycle error"); - _ZN3sce4Json5ValueD1Ev(&value); - Require(trace.live == baseline && _ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Ordinary teardown must reclaim resources after notifier refusal and retry"); - } else { - Require(!_ZN3sce4Json6MallocEm(8) && initializer == 0 && trace.live.empty() && trace.notices == 1 && !trace.refused, + Require(!_ZN3sce4Json6MallocEm(8) && initializer == 0 && trace.live.empty() && trace.notices == 1, "Standalone allocation failure must preserve successful notifier termination and its null allocation result"); - } Require(initializer == 0 && trace.live.empty(), "Notifier lifecycle cases must fully release the original module generation"); const auto generation = trace.generation; trace.noticeAction = NoticeAction::None; - Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0 && initializer == 1 && trace.live.size() == 12, "The same initializer must admit a fresh module generation after notifier-driven termination or refusal"); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0 && initializer == 1 && trace.live.size() == 12, "The same initializer must admit a fresh module generation after notifier-driven termination"); for (const auto& [memory, identity] : trace.live) Require(memory && identity > generation, "Lifecycle retry must never retain stale allocation identities"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release retried notifier lifecycle generation"); } -static void CheckAllocator(bool allocate) { - LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; - std::uint8_t initializer = 0; trace.owner = &initializer; - _ZN3sce4Json11InitializerC1Ev(&initializer); - Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0, "Initialize direct allocator lifecycle contracts"); - const auto baseline = trace.live; - Value value{}; _ZN3sce4Json5ValueC1Ev(&value); - trace.allocatorAction = allocate ? AllocatorAction::RejectAllocate : AllocatorAction::RejectFree; - const char* document = allocate ? "true" : "42"; - Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, document, std::strlen(document)) == 0 && initializer == 1 && trace.refused == 1 && trace.allocatorAction == AllocatorAction::None, - "Caught allocator callback termination must preserve actual parser completion and the active module"); - Require(allocate ? value.type == 1 && (value.payload & 0xff) == 1 : value.type == 3 && value.payload == 42, - "Allocator lifecycle refusal must preserve the parser's exact scalar result"); - Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, "false", 5) == 0 && value.type == 1 && !(value.payload & 0xff) && trace.refused == 1, - "Following operations must remain usable after allocation/free callback lifecycle refusal"); - _ZN3sce4Json5ValueD1Ev(&value); - Require(trace.live == baseline && _ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), - "Direct allocator lifecycle refusal must preserve original module generations through ordinary teardown"); -} static void CheckStandaloneFree() { LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; std::uint8_t initializer = 0; trace.owner = &initializer; @@ -177,7 +117,7 @@ static void CheckStandaloneFree() { Require(memory != nullptr, "Standalone lifecycle preservation requires a real allocated buffer"); trace.allocatorAction = AllocatorAction::TerminateFree; _ZN3sce4Json4FreeEPv(memory); - Require(initializer == 0 && trace.live.empty() && trace.allocatorAction == AllocatorAction::None && !trace.refused, + Require(initializer == 0 && trace.live.empty() && trace.allocatorAction == AllocatorAction::None, "Standalone free must complete callback-driven termination and release the original buffer"); const auto generation = trace.generation; Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0 && initializer == 1 && trace.live.size() == 12, @@ -185,32 +125,9 @@ static void CheckStandaloneFree() { for (const auto& [pointer, identity] : trace.live) Require(pointer && identity > generation, "Reinitialization must own fresh resource generations after free-callback termination"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty(), "Release standalone free lifecycle retry"); } -static void CheckTeardownReentry(unsigned target) { - LifecycleTrace trace; LifecycleAllocator allocator; Parameter parameter{&allocator, &trace, 8192, 2, {}}; - std::uint8_t initializer = 0; trace.owner = &initializer; - _ZN3sce4Json11InitializerC1Ev(&initializer); - Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0 && trace.live.size() == 12, - "Teardown reentry requires the actual twelve independently allocated module resources"); - trace.freeCalls = 0; trace.teardownTarget = target; - Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty() && trace.freeCalls == 12 && trace.refused == 1 && !trace.teardownTarget, - "Each teardown callback must reject nested owner termination while allowing the outer call to release every resource exactly once"); - const auto generation = trace.generation; - Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0 && initializer == 1 && trace.live.size() == 12, - "Initialization must admit a fresh module after rejected teardown reentry at every release position"); - for (const auto& [pointer, identity] : trace.live) Require(pointer && identity > generation, "Teardown reentry retry must retain no old allocation generations"); - try { - Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty() && trace.refused == 1, - "The teardown guard must reset so ordinary termination remains usable for the next module generation"); - } catch (const std::runtime_error&) { Require(false, "Completed teardown must clear its guard before ordinary termination of a fresh module generation"); } -} int main() { - CheckRecovery(false); - CheckRecovery(true); - CheckNotice(false); - CheckNotice(true); - CheckAllocator(true); - CheckAllocator(false); + CheckRecovery(); + CheckNotice(); CheckStandaloneFree(); - for (unsigned target = 1; target <= 12; ++target) CheckTeardownReentry(target); return 0; } diff --git a/core/libs/tests/GuestJsonCallbackThreads.cpp b/core/libs/tests/GuestJsonCallbackThreads.cpp new file mode 100644 index 0000000000..feba0a54d7 --- /dev/null +++ b/core/libs/tests/GuestJsonCallbackThreads.cpp @@ -0,0 +1,129 @@ +#include "prx/libSceJson2/include/Api.hpp" +#include "prx/libkernel/Pthread/include/Mutex.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace SceJson2; + +extern "C" const void* ApplicationProcessParameters_nid_no_patch() { + static const std::uint32_t words[]{0, 0, 0, 0, 0x05000000}; + return words; +} + +static void Require(bool value, const char* message) { + if (!value) { std::fprintf(stderr, "%s\n", message); std::abort(); } +} + +struct Schedule { + std::mutex mutex; + std::condition_variable changed; + bool callbackEntered = false; + bool parserEntered = false; + std::thread::id parser; +}; +static Schedule schedule; + +static void ParserBoundary() { + std::lock_guard lock(schedule.mutex); + if (std::this_thread::get_id() != schedule.parser) return; + schedule.parserEntered = true; + schedule.changed.notify_all(); +} + +extern "C" int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attribute) { + *attribute = reinterpret_cast(new int(0)); + return 0; +} +extern "C" int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attribute, int type) { + Require(type == 2, "Json2 callback mutex must be recursive"); + *reinterpret_cast(*attribute) = type; + return 0; +} +extern "C" int APS5_VABI scePthreadMutexattrDestroy(PthreadMutexattr* attribute) { + delete reinterpret_cast(*attribute); + return 0; +} +extern "C" int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr*, const char*) { + *mutex = reinterpret_cast(new std::recursive_mutex); + return 0; +} +extern "C" int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex) { + delete reinterpret_cast(*mutex); + return 0; +} +extern "C" int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex) { + ParserBoundary(); + reinterpret_cast(*mutex)->lock(); + return 0; +} +extern "C" int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex) { + reinterpret_cast(*mutex)->unlock(); + return 0; +} + +static void* APS5_VABI Allocate(void*, std::size_t bytes, void*) { + ParserBoundary(); + return std::malloc(bytes); +} +static void APS5_VABI Free(void*, void* memory, void*) { std::free(memory); } +static void APS5_VABI Notify(void*, int, std::size_t, void*) { Require(false, "Thread regression must not fail allocation"); } +static void* const table[]{nullptr, nullptr, reinterpret_cast(Allocate), reinterpret_cast(Free), reinterpret_cast(Notify)}; +struct Allocator { void* const* vtable = table; }; +struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; +static Value replacement{}; + +static const Value& APS5_VABI Mismatch(ValueType requested, const Value*, const Value*, void*) { + Require(requested == TypeInteger, "Exercise a real legacy mismatch callback"); + { + std::unique_lock lock(schedule.mutex); + schedule.callbackEntered = true; + schedule.changed.notify_all(); + schedule.changed.wait(lock, [] { return schedule.parserEntered; }); + } + Value missing{}; + _ZN3sce4Json5ValueC1Ev(&missing); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&missing) == 0, + "Nested mismatch must reach shared defaults while another thread starts parsing"); + _ZN3sce4Json5ValueD1Ev(&missing); + return replacement; +} + +int main() { + Allocator allocator; + Parameter parameter{&allocator, nullptr, 8192, 2, {}}; + std::uint8_t initializer = 0; + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0, + "Initialize legacy callback thread regression"); + Value source{}, parsed{}; + _ZN3sce4Json5ValueC1Ev(&source); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&source, "true", 4) == 0, "Create an initialized legacy recovery root"); + _ZN3sce4Json5ValueC1Ev(&parsed); + _ZN3sce4Json5ValueC1El(&replacement, 42); + Require(_ZN3sce4Json5Value22setTypeMismatchHandlerEPFRKS1_NS0_9ValueTypeEPS2_S5_PvES6_(&source, Mismatch, nullptr) == 0, + "Install legacy recovery callback"); + std::thread callback([&] { + Require(*_ZNK3sce4Json5Value10getIntegerEv(&source) == 42, "Mismatch recovery must complete with its callback value"); + }); + std::thread parser([&] { + { + std::unique_lock lock(schedule.mutex); + schedule.parser = std::this_thread::get_id(); + schedule.changed.wait(lock, [] { return schedule.callbackEntered; }); + } + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&parsed, "[1,2]", 5) == 0, + "Independent parsing must complete after callback recovery"); + }); + callback.join(); + parser.join(); + Require(parsed.type == TypeArray && _ZNK3sce4Json5Array4sizeEv(_ZNK3sce4Json5Value8getArrayEv(&parsed)) == 2, + "Concurrent callback recovery must preserve parser output"); + _ZN3sce4Json5ValueD1Ev(&source); + _ZN3sce4Json5ValueD1Ev(&parsed); + _ZN3sce4Json5ValueD1Ev(&replacement); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Release legacy callback resources"); +} diff --git a/core/libs/tests/GuestJsonCompositeDestruction.cpp b/core/libs/tests/GuestJsonCompositeDestruction.cpp index 8e6f97f624..d1484726ff 100644 --- a/core/libs/tests/GuestJsonCompositeDestruction.cpp +++ b/core/libs/tests/GuestJsonCompositeDestruction.cpp @@ -125,6 +125,7 @@ static void DestroyAlias(Value& value, unsigned alias) { else _ZN3sce4Json5ValueD1Ev(&value); } static void Check(unsigned shape, unsigned target, unsigned alias, bool reinitialize, unsigned domain = 0) { + if (LegacySdk && shape != 0 && target > 18 && !reinitialize) return; DestructionTrace trace; DestructionAllocator allocator, replacementAllocator; DestructionDomain first{&trace, &allocator, 0}, second{&trace, domain == 2 ? &replacementAllocator : &allocator, 1}; Parameter parameter{&allocator, &first, 8192, 2, {}}, replacement{second.receiver, &second, 4096, 2, {}}; @@ -151,15 +152,8 @@ static void Check(unsigned shape, unsigned target, unsigned alias, bool reinitia if (auxiliary) std::memcpy(auxiliaryBytes.data(), auxiliary, AuxiliaryBytes); trace.reinitialize = reinitialize; trace.parameter = domain ? &replacement : ¶meter; trace.replacementDomain = unsigned(domain != 0); const bool terminated = target != 0; - const bool unsupportedCleanup = LegacySdk && shape != 0 && target > 18 && !reinitialize; trace.target = target; trace.observing = true; - bool cleanupError = false; - try { DestroyAlias(value, alias); } - catch (const std::runtime_error& error) { - Require(unsupportedCleanup && std::strstr(error.what(), "EraseLegacyHandler"), "Only an actual missing legacy cleanup mutex may produce the explicit unsupported error"); - cleanupError = true; - } - Require(cleanupError == unsupportedCleanup, "Legacy destruction must reject the null-mutex cleanup boundary while preserving callback-time reinitialization"); + DestroyAlias(value, alias); const unsigned released = terminated && !reinitialize ? target : static_cast(order.size()); Require(trace.releases == std::vector(order.begin(), order.begin() + released), "Composite destruction must preserve the exact native release-generation prefix before callback termination"); std::vector expectedDomains(released, 0); diff --git a/core/libs/tests/GuestJsonFile.cpp b/core/libs/tests/GuestJsonFile.cpp index a934c92a82..5e007ce546 100644 --- a/core/libs/tests/GuestJsonFile.cpp +++ b/core/libs/tests/GuestJsonFile.cpp @@ -1,4 +1,5 @@ #include "prx/libSceJson2/include/Api.hpp" +#include "prx/libkernel/File/include/File.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include #include @@ -38,7 +39,7 @@ extern "C" int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_ ++trace.opens; return trace.descriptor; } -extern "C" std::int64_t APS5_VABI sceKernelRead(ValueType descriptor, void* output, std::size_t bytes) { +extern "C" std::int64_t APS5_VABI sceKernelRead(int descriptor, void* output, std::size_t bytes) { auto& trace = *activeFile; Require(descriptor == trace.descriptor && output == trace.buffer && bytes == trace.bufferSize, "Read must use the configured buffer and descriptor"); ++trace.reads; @@ -48,7 +49,7 @@ extern "C" std::int64_t APS5_VABI sceKernelRead(ValueType descriptor, void* outp trace.position += count; return static_cast(count); } -extern "C" int APS5_VABI sceKernelClose(ValueType descriptor) { +extern "C" int APS5_VABI sceKernelClose(int descriptor) { auto& trace = *activeFile; Require(descriptor == trace.descriptor && trace.buffer != nullptr, "Close must precede release of the file buffer"); ++trace.closes; diff --git a/core/libs/tests/GuestJsonKernelInitialization.cpp b/core/libs/tests/GuestJsonKernelInitialization.cpp index 825ff2a082..e69dfa63bf 100644 --- a/core/libs/tests/GuestJsonKernelInitialization.cpp +++ b/core/libs/tests/GuestJsonKernelInitialization.cpp @@ -41,7 +41,7 @@ static void APS5_VABI NotifyTracked(void*, int, std::size_t, void*) { } static void* const Vtable[]{nullptr, nullptr, reinterpret_cast(AllocateTracked), reinterpret_cast(FreeTracked), reinterpret_cast(NotifyTracked)}; struct Allocator { void* const* vtable = Vtable; }; -enum class Failure { None, AttributeError, AttributeException, TypeError, MutexError, MutexException }; +enum class Failure { None, AttributeException, MutexException }; struct KernelTrace { Failure failure = Failure::None; unsigned target = 0; @@ -57,7 +57,6 @@ static KernelTrace Kernel; static bool At(Failure failure) { return Kernel.failure == failure && Kernel.attempts == Kernel.target; } extern "C" int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attribute) { ++Kernel.attempts; - if (At(Failure::AttributeError)) return static_cast(0x8002000c); if (At(Failure::AttributeException)) throw std::bad_alloc(); *attribute = reinterpret_cast(new int(0)); Require(Kernel.attributes.emplace(*attribute, 0).second, "Kernel attributes must have independent live identities"); @@ -66,7 +65,6 @@ extern "C" int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attribute) { } extern "C" int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attribute, int type) { Require(type == 2 && Kernel.attributes.count(*attribute), "Json2 mutexes must request recursive attributes"); - if (At(Failure::TypeError)) return static_cast(0x80020016); Kernel.attributes.at(*attribute) = type; return 0; } @@ -79,7 +77,6 @@ extern "C" int APS5_VABI scePthreadMutexattrDestroy(PthreadMutexattr* attribute) } extern "C" int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attribute, const char* name) { Require(Kernel.attributes.at(*attribute) == 2 && std::strcmp(name, "SceJson_internal") == 0 && !*mutex, "Mutex creation must receive recursive attributes and initialized handle storage"); - if (At(Failure::MutexError)) return static_cast(0x8002000c); if (At(Failure::MutexException)) throw std::bad_alloc(); *mutex = reinterpret_cast(new std::recursive_mutex); Require(Kernel.mutexes.emplace(*mutex, 0).second, "Kernel mutexes must have independent live identities"); @@ -116,9 +113,6 @@ static void Check(Failure failure, unsigned target) { } catch (const std::bad_alloc&) { Require(failure == Failure::AttributeException || failure == Failure::MutexException, "Host allocation exceptions must retain their actual type"); failed = true; - } catch (const std::runtime_error& error) { - Require((failure == Failure::AttributeError || failure == Failure::TypeError || failure == Failure::MutexError) && std::strstr(error.what(), "CreateKernelMutex"), "Unverified kernel status failures must remain explicitly unsupported"); - failed = true; } const unsigned mutexCount = Sdk < 0x06000000 ? 1u : 2u; if (failure == Failure::None) { @@ -139,7 +133,7 @@ int main() { for (auto sdk : {0x05000000u, 0x06000000u}) { Sdk = sdk; Check(Failure::None, 0); - for (auto failure : {Failure::AttributeError, Failure::AttributeException, Failure::TypeError, Failure::MutexError, Failure::MutexException}) + for (auto failure : {Failure::AttributeException, Failure::MutexException}) for (unsigned target = 1; target <= (sdk < 0x06000000 ? 1u : 2u); ++target) Check(failure, target); } return 0; diff --git a/core/libs/tests/GuestJsonLegacyHandlers.cpp b/core/libs/tests/GuestJsonLegacyHandlers.cpp index cfcddd72b5..2310da6c7c 100644 --- a/core/libs/tests/GuestJsonLegacyHandlers.cpp +++ b/core/libs/tests/GuestJsonLegacyHandlers.cpp @@ -426,7 +426,7 @@ static void CheckInheritedMap(LegacyTrace& trace, Context& first, Context& secon if (shape) { const auto payload = shape == 1 ? std::vector{8, 24, 48, 48, 48, 8, 24, 104, 104, 104, 2, 24, 2, 8, 24, 5} : std::vector{8, 24, 104, 104, 104, 2, 24, 2, 8, 24, 48, 48, 48, 8, 24, 5}; - expected.insert(expected.end(), payload.begin(), payload.end()); + for (auto bytes : payload) expected.push_back(bytes); } expected.push_back(24); expected.push_back(32); if (alias) Require(pair.reserved == UINT64_C(0xa5a5a5a5a5a5a5a5) && pair.key.text != key.text && std::strcmp(_ZNK3sce4Json6String5c_strEv(&pair.key), "key") == 0, diff --git a/core/libs/tests/GuestJsonTerminationExceptions.cpp b/core/libs/tests/GuestJsonTerminationExceptions.cpp index 0d5128d95e..febac8488b 100644 --- a/core/libs/tests/GuestJsonTerminationExceptions.cpp +++ b/core/libs/tests/GuestJsonTerminationExceptions.cpp @@ -83,21 +83,7 @@ int main(int argc, char** argv) { _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&other, reinterpret_cast(¶meter)) == static_cast(0x80848111) && !other && trace.live == expected && trace.releases == releases, "Both owners must reject initialization after incomplete teardown without further allocator effects"); - if (argc == 4 && (std::strcmp(argv[3], "refuse-mutex") == 0 || std::strcmp(argv[3], "refuse-header") == 0)) { - if (std::strcmp(argv[3], "refuse-header") == 0) Require(trace.target == 4 || trace.target == 7 || trace.target == 10, "Header refusal requires completed child destruction"); - else Require((Legacy && trace.target == 3) || (!Legacy && (trace.target == 2 || trace.target == 3)), "Refusal probes require a destroyed mutex slot"); - trace.target = 0; - for (unsigned attempt = 0; attempt < 2; ++attempt) { - bool refused = false; - try { _ZN3sce4Json11Initializer9terminateEv(&owner); } - catch (const std::runtime_error& error) { - refused = std::string(error.what()).find("_ZN3sce4Json11Initializer9terminateEv") != std::string::npos; - } - Require(refused && owner && trace.live == expected && trace.releases == releases, - "Retry must explicitly refuse retired ownership before kernel use or further release effects"); - } - trace.target = calls; - } else if (argc == 4) { + if (argc == 4) { Require(std::strcmp(argv[3], "retry") == 0, "The additional schedule must request a returning reference retry"); std::vector tail; switch (trace.target) { diff --git a/core/libs/tests/GuestJsonValueOwnership.cpp b/core/libs/tests/GuestJsonValueOwnership.cpp index 915dbcd359..a13cf6c3e7 100644 --- a/core/libs/tests/GuestJsonValueOwnership.cpp +++ b/core/libs/tests/GuestJsonValueOwnership.cpp @@ -71,6 +71,30 @@ int main() { _ZN3sce4Json5Value3setERKS1_(©, &Front(copy)); Require(!copy.parent && Front(copy).parent == © && Front(copy).payload == 42, "Assignment from a descendant must finish copying before releasing the old tree"); + Value cross{}; + _ZN3sce4Json5ValueC1Ev(&cross); + const char objectInput[] = "{\"a\":[1,2]}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&cross, objectInput, sizeof(objectInput) - 1) == 0, "Parse cross-type ownership tree"); + const auto* arrayMember = _ZNK3sce4Json5ValueixEPKc(&cross, "a"); + _ZN3sce4Json5ValueaSERKS1_(&cross, arrayMember); + Require(cross.type == 6 && Front(cross).payload == 1 && Front(cross).parent == &cross, + "Object-to-array descendant assignment must preserve the source before clearing its owner"); + _ZN3sce4Json5Value5clearEv(&cross); + const char arrayInput[] = "[\"retained\"]"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&cross, arrayInput, sizeof(arrayInput) - 1) == 0, "Parse string descendant tree"); + _ZN3sce4Json5Value3setERKS1_(&cross, &Front(cross)); + Require(cross.type == 5 && std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&cross)), "retained") == 0, + "Array-to-string descendant assignment must preserve text before clearing its owner"); + _ZN3sce4Json5Value5clearEv(&cross); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&cross, objectInput, sizeof(objectInput) - 1) == 0, "Parse raw Array descendant source"); + _ZN3sce4Json5Value3setERKNS0_5ArrayE(&cross, _ZNK3sce4Json5Value8getArrayEv(_ZNK3sce4Json5ValueixEPKc(&cross, "a"))); + Require(cross.type == TypeArray && Front(cross).payload == 1, "Array overload must retain a descendant before changing the destination type"); + _ZN3sce4Json5Value5clearEv(&cross); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&cross, arrayInput, sizeof(arrayInput) - 1) == 0, "Parse raw String descendant source"); + _ZN3sce4Json5Value3setERKNS0_6StringE(&cross, _ZNK3sce4Json5Value9getStringEv(&Front(cross))); + Require(cross.type == TypeString && std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&cross)), "retained") == 0, + "String overload must retain descendant text before changing the destination type"); + _ZN3sce4Json5ValueD1Ev(&cross); _ZN3sce4Json5Value4swapERS1_(&root, &text); Require(root.type == 5 && text.type == 6, "Swap must exchange payloads"); Tree(text); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index a94736622e..09cf823147 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -124,3 +124,6 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Element fallback collection allocation failures](../../../core/libs/prx/libSceJson2/Export.cpp) - nested payloads, thrown callbacks and concurrent lifecycle changes remain unverified. One hundred fifty-two SDK 6 reference cases cover two-element Array/Object assignment into root-owned and module fallbacks, persistent/one-shot failure at every allocation position and success. Checks include reentry suppression, exact request/notice fingerprints, missing sentinels, partial values/keys/counts, parent links, header ownership and unchanged source/module/callback allocation generations and bytes. Independent retry completes the replacement; subsequent recovery retains its handle. Teardown reclaims all source, partial and replacement payloads. All production probe results match the reference. A mutation dropping the fallback collection owner fails the regression. - [Initializer allocator-info callback](../../../core/libs/prx/libSceJson2/Export.cpp) - the inspected 19-byte function at `0x16e0` ignores the receiver, callback and context. It returns `0x80848110` when the allocation context is absent and zero when present, without storing or invoking the callback. The implementation intentionally follows that behavior; it is not evidence of allocation telemetry support on other library versions. + +- [Descendant assignment and overlapping swap](../../../core/libs/prx/libSceJson2/Export.cpp) - assigning a descendant snapshots its payload before releasing its owner. Cross-type Array/String ownership regressions exercise public setters and assignment. Swapping ancestor and descendant values throws before mutation: the first replacement invalidates the second receiver, and the reference contract remains unverified. Ordinary independent swaps retain their existing behavior. +- [Legacy callback concurrency](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation failure/depth state uses the host state mutex without nesting the shared legacy recovery mutex. A coordinated mismatch callback and parser test reproduces the former inverse lock order and verifies both operations complete. Concurrent initialization/termination, callback null-access cycles and partial object trees after failed key/subtree allocations remain unverified; partial copies can retain admitted nodes with incomplete keys or counts. From c8f8b4e3701673ac89f80e1937b69158a23da2d7 Mon Sep 17 00:00:00 2001 From: Kyon Date: Tue, 27 Oct 2026 06:04:58 +0000 Subject: [PATCH 201/318] test(libSceJson2): cover the Object descendant setter --- core/libs/tests/GuestJsonValueOwnership.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/libs/tests/GuestJsonValueOwnership.cpp b/core/libs/tests/GuestJsonValueOwnership.cpp index a13cf6c3e7..4bdd4db640 100644 --- a/core/libs/tests/GuestJsonValueOwnership.cpp +++ b/core/libs/tests/GuestJsonValueOwnership.cpp @@ -9,6 +9,7 @@ using namespace SceJson2; + struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; static void Require(bool valid, const char* message) { if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } @@ -94,6 +95,13 @@ int main() { _ZN3sce4Json5Value3setERKNS0_6StringE(&cross, _ZNK3sce4Json5Value9getStringEv(&Front(cross))); Require(cross.type == TypeString && std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&cross)), "retained") == 0, "String overload must retain descendant text before changing the destination type"); + _ZN3sce4Json5Value5clearEv(&cross); + const char nestedObject[] = "[{\"a\":7}]"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&cross, nestedObject, sizeof(nestedObject) - 1) == 0, "Parse raw Object descendant source"); + _ZN3sce4Json5Value3setERKNS0_6ObjectE(&cross, _ZNK3sce4Json5Value9getObjectEv(&Front(cross))); + const auto* member = _ZNK3sce4Json5ValueixEPKc(&cross, "a"); + Require(cross.type == TypeObject && member->payload == 7 && member->parent == &cross, + "Object overload must retain descendant contents and bind their new owner"); _ZN3sce4Json5ValueD1Ev(&cross); _ZN3sce4Json5Value4swapERS1_(&root, &text); Require(root.type == 5 && text.type == 6, "Swap must exchange payloads"); From 397f3e3ca2bdf3320cc4eb64b8a739fd1ed09af6 Mon Sep 17 00:00:00 2001 From: Kyon Date: Tue, 27 Oct 2026 06:07:34 +0000 Subject: [PATCH 202/318] test(libSceJson2): isolate the callback regression from application metadata --- core/libs/tests/GuestJsonCallbackThreads.cpp | 18 ++++++++++++++++++ docs/dev/debt/libSceJson2.md | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/core/libs/tests/GuestJsonCallbackThreads.cpp b/core/libs/tests/GuestJsonCallbackThreads.cpp index feba0a54d7..3ed4d25394 100644 --- a/core/libs/tests/GuestJsonCallbackThreads.cpp +++ b/core/libs/tests/GuestJsonCallbackThreads.cpp @@ -1,11 +1,14 @@ #include "prx/libSceJson2/include/Api.hpp" +#include "prx/libc/include/Api.hpp" #include "prx/libkernel/Pthread/include/Mutex.hpp" +#include #include #include #include #include #include #include +#include #include using namespace SceJson2; @@ -19,6 +22,20 @@ static void Require(bool value, const char* message) { if (!value) { std::fprintf(stderr, "%s\n", message); std::abort(); } } +static std::atomic heapAllocations{0}; + +extern "C" void* APS5_VABI _Znwm_nid_postfix(std::size_t bytes) { + void* memory = std::malloc(bytes == 0 ? 1 : bytes); + if (!memory) throw std::bad_alloc(); + heapAllocations.fetch_add(1, std::memory_order_relaxed); + return memory; +} +extern "C" void APS5_VABI _ZdlPv_nid_postfix(void* memory) { + if (!memory) return; + Require(heapAllocations.fetch_sub(1, std::memory_order_relaxed) != 0, "Legacy heap releases must balance live allocations"); + std::free(memory); +} + struct Schedule { std::mutex mutex; std::condition_variable changed; @@ -126,4 +143,5 @@ int main() { _ZN3sce4Json5ValueD1Ev(&parsed); _ZN3sce4Json5ValueD1Ev(&replacement); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0, "Release legacy callback resources"); + Require(heapAllocations.load(std::memory_order_relaxed) == 0, "Legacy callback threads must release every mocked application-heap allocation"); } diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 09cf823147..20a73eaa0c 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -126,4 +126,4 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Initializer allocator-info callback](../../../core/libs/prx/libSceJson2/Export.cpp) - the inspected 19-byte function at `0x16e0` ignores the receiver, callback and context. It returns `0x80848110` when the allocation context is absent and zero when present, without storing or invoking the callback. The implementation intentionally follows that behavior; it is not evidence of allocation telemetry support on other library versions. - [Descendant assignment and overlapping swap](../../../core/libs/prx/libSceJson2/Export.cpp) - assigning a descendant snapshots its payload before releasing its owner. Cross-type Array/String ownership regressions exercise public setters and assignment. Swapping ancestor and descendant values throws before mutation: the first replacement invalidates the second receiver, and the reference contract remains unverified. Ordinary independent swaps retain their existing behavior. -- [Legacy callback concurrency](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation failure/depth state uses the host state mutex without nesting the shared legacy recovery mutex. A coordinated mismatch callback and parser test reproduces the former inverse lock order and verifies both operations complete. Concurrent initialization/termination, callback null-access cycles and partial object trees after failed key/subtree allocations remain unverified; partial copies can retain admitted nodes with incomplete keys or counts. +- [Legacy callback concurrency](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation failure/depth state uses the host state mutex without nesting the shared legacy recovery mutex. A coordinated mismatch callback and parser test reproduces the former inverse lock order and verifies both operations complete. It substitutes the application-heap entry points and kernel mutex primitives; it exercises the real Json2 state locks, callbacks and parser. Concurrent initialization/termination, callback null-access cycles and partial object trees after failed key/subtree allocations remain unverified; partial copies can retain admitted nodes with incomplete keys or counts. From 3e03f8df73d6b02e1f1f70e1af86d2fcc259e450 Mon Sep 17 00:00:00 2001 From: Kyon Date: Tue, 27 Oct 2026 06:42:49 +0000 Subject: [PATCH 203/318] fix(libSceJson2): preserve owned object keys during value assignment --- core/libs/prx/libSceJson2/Export.cpp | 2 +- core/libs/tests/GuestJsonValueOwnership.cpp | 15 ++++++++++++++- docs/dev/debt/libSceJson2.md | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 096e25b17d..39231f96e3 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1311,7 +1311,7 @@ bool OwnsPayload(const Node& owner, std::uintptr_t payload) { if (OwnsPayload(item, payload)) return true; } else if (owner.type == TypeObject && owner.object->items) { for (const auto& pair : *owner.object->items) - if (OwnsPayload(pair.value, payload)) return true; + if (reinterpret_cast(&pair.key) == payload || OwnsPayload(pair.value, payload)) return true; } return false; } diff --git a/core/libs/tests/GuestJsonValueOwnership.cpp b/core/libs/tests/GuestJsonValueOwnership.cpp index 4bdd4db640..bf0074c81c 100644 --- a/core/libs/tests/GuestJsonValueOwnership.cpp +++ b/core/libs/tests/GuestJsonValueOwnership.cpp @@ -6,10 +6,10 @@ #include #include #include +#include using namespace SceJson2; - struct Parameter { void* allocator; void* context; std::size_t bufferSize; std::int32_t mode; std::uint32_t reserved[3]; }; static void Require(bool valid, const char* message) { if (!valid) { std::fprintf(stderr, "%s\n", message); std::abort(); } @@ -102,6 +102,19 @@ int main() { const auto* member = _ZNK3sce4Json5ValueixEPKc(&cross, "a"); Require(cross.type == TypeObject && member->payload == 7 && member->parent == &cross, "Object overload must retain descendant contents and bind their new owner"); + for (bool nested : {false, true}) { + _ZN3sce4Json5Value5clearEv(&cross); + const char* document = nested ? "[{\"retained-key\":7}]" : "{\"retained-key\":7}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&cross, document, std::strlen(document)) == 0, "Parse object-key ownership tree"); + const auto* object = _ZNK3sce4Json5Value9getObjectEv(nested ? &Front(cross) : &cross); + ObjectIterator iterator{}; + _ZNK3sce4Json6Object5beginEv(&iterator, object); + const auto* pair = _ZNK3sce4Json6Object8iteratordeEv(&iterator); + _ZN3sce4Json5Value3setERKNS0_6StringE(&cross, &pair->key); + Require(cross.type == TypeString && std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&cross)), "retained-key") == 0, + "String assignment from an owned object key must copy text before releasing its tree"); + _ZN3sce4Json6Object8iteratorD1Ev(&iterator); + } _ZN3sce4Json5ValueD1Ev(&cross); _ZN3sce4Json5Value4swapERS1_(&root, &text); Require(root.type == 5 && text.type == 6, "Swap must exchange payloads"); diff --git a/docs/dev/debt/libSceJson2.md b/docs/dev/debt/libSceJson2.md index 20a73eaa0c..e365122622 100644 --- a/docs/dev/debt/libSceJson2.md +++ b/docs/dev/debt/libSceJson2.md @@ -125,5 +125,5 @@ SDK 5/6 below refer to the application compatibility SDK queried by `sceKernelGe - [Initializer allocator-info callback](../../../core/libs/prx/libSceJson2/Export.cpp) - the inspected 19-byte function at `0x16e0` ignores the receiver, callback and context. It returns `0x80848110` when the allocation context is absent and zero when present, without storing or invoking the callback. The implementation intentionally follows that behavior; it is not evidence of allocation telemetry support on other library versions. -- [Descendant assignment and overlapping swap](../../../core/libs/prx/libSceJson2/Export.cpp) - assigning a descendant snapshots its payload before releasing its owner. Cross-type Array/String ownership regressions exercise public setters and assignment. Swapping ancestor and descendant values throws before mutation: the first replacement invalidates the second receiver, and the reference contract remains unverified. Ordinary independent swaps retain their existing behavior. +- [Descendant assignment and overlapping swap](../../../core/libs/prx/libSceJson2/Export.cpp) - assigning a descendant or an owned object key snapshots its payload before releasing its owner. Cross-type Array/String ownership regressions exercise public setters and assignment. Swapping ancestor and descendant values throws before mutation: the first replacement invalidates the second receiver, and the reference contract remains unverified. Ordinary independent swaps retain their existing behavior. - [Legacy callback concurrency](../../../core/libs/prx/libSceJson2/Export.cpp) - allocation failure/depth state uses the host state mutex without nesting the shared legacy recovery mutex. A coordinated mismatch callback and parser test reproduces the former inverse lock order and verifies both operations complete. It substitutes the application-heap entry points and kernel mutex primitives; it exercises the real Json2 state locks, callbacks and parser. Concurrent initialization/termination, callback null-access cycles and partial object trees after failed key/subtree allocations remain unverified; partial copies can retain admitted nodes with incomplete keys or counts. From fc8d6c625fd52088e938ba9023043248ca0f8674 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:27:34 +0000 Subject: [PATCH 204/318] feat(libc): find public RTTI source subobjects --- core/libs/prx/libc/src/exception/TypeInfo.cpp | 47 ++++++++++++++++++- core/libs/tests/GuestDynamicCast.cpp | 43 ++++++++++++++++- 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libc/src/exception/TypeInfo.cpp b/core/libs/prx/libc/src/exception/TypeInfo.cpp index a1d58f019b..2a0284ed7b 100644 --- a/core/libs/prx/libc/src/exception/TypeInfo.cpp +++ b/core/libs/prx/libc/src/exception/TypeInfo.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/Api.hpp" #include "prx/libc/include/exceptions/Runtime.hpp" +#include #include namespace LibcException { @@ -39,8 +40,50 @@ bool APS5_VABI DynamicCastTypeInfo(const ClassTypeInfo*, std::ptrdiff_t, ClassTy throw std::runtime_error("RTTI __dyncast_result is not supported"); } -ClassTypeInfo::__sub_kind APS5_VABI FindPublicTypeInfo(const ClassTypeInfo*, std::ptrdiff_t, const void*, const ClassTypeInfo*, const void*) { - throw std::runtime_error("RTTI __do_find_public_src is not supported"); +ClassTypeInfo::__sub_kind APS5_VABI FindPublicTypeInfo(const ClassTypeInfo* self, std::ptrdiff_t src2dst, + const void* object, const ClassTypeInfo* sourceType, + const void* source) { + constexpr int notContained = 1; + constexpr int containedPublic = 6; + constexpr int containedMask = 4; + constexpr int publicMask = 2; + constexpr int virtualMask = 1; + if (!self || !sourceType) throw std::invalid_argument("RTTI __do_find_public_src: null type"); + const auto* vtable = *reinterpret_cast(self); + const char* kind = vtable[-1]->name(); + if (std::strcmp(kind, "N10__cxxabiv120__si_class_type_infoE") == 0) { + if (object == source && *self == *sourceType) + return static_cast(containedPublic); + const auto* single = static_cast(self); + return FindPublicTypeInfo(single->__base_type, src2dst, object, sourceType, source); + } + if (std::strcmp(kind, "N10__cxxabiv121__vmi_class_type_infoE") == 0) { + if (object == source && *self == *sourceType) + return static_cast(containedPublic); + } else { + return static_cast(object == source ? containedPublic : notContained); + } + + const auto* multiple = static_cast(self); + for (unsigned index = multiple->__base_count; index-- > 0;) { + const auto& base = multiple->__base_info[index]; + if ((base.__offset_flags & publicMask) == 0) continue; + const bool isVirtual = (base.__offset_flags & virtualMask) != 0; + if (isVirtual && src2dst == -3) continue; + auto offset = base.__offset_flags >> 8; + const void* baseObject = object; + if (isVirtual) { + if (!object) continue; + const auto* addressPoint = *static_cast(object); + std::memcpy(&offset, reinterpret_cast(addressPoint) + offset, sizeof(offset)); + } + if (object) baseObject = static_cast(object) + offset; + const auto result = FindPublicTypeInfo(base.__base_type, src2dst, baseObject, sourceType, source); + const int bits = static_cast(result); + if ((bits & containedMask) == 0) continue; + return static_cast(bits | (isVirtual ? virtualMask : 0)); + } + return static_cast(notContained); } } diff --git a/core/libs/tests/GuestDynamicCast.cpp b/core/libs/tests/GuestDynamicCast.cpp index 5a4212d578..b99dd3bc19 100644 --- a/core/libs/tests/GuestDynamicCast.cpp +++ b/core/libs/tests/GuestDynamicCast.cpp @@ -1,18 +1,27 @@ #include "prx/libc/include/Api.hpp" #include "prx/libc/include/exceptions/Runtime.hpp" +#include #include #include #include +#include #include struct TypeRecord { const void* vtable; const char* name; const TypeRecord* base; }; struct Vtable { std::intptr_t offsetToTop; const TypeRecord* type; const void* first; }; struct Object { const void* vtable; }; +struct Source { virtual ~Source() = default; }; +struct Side { virtual ~Side() = default; }; +struct Native : Side, Source {}; +struct PrivateNative : private Source { virtual ~PrivateNative() = default; }; +struct VirtualNative : virtual Source { virtual ~VirtualNative() = default; }; +struct BaseRecord { const TypeRecord* type; std::ptrdiff_t flags; }; +struct MultipleRecord { const void* vtable; const char* name; unsigned flags; unsigned count; BaseRecord bases[1]; }; extern "C" { extern const unsigned char _ZTVN10__cxxabiv117__class_type_infoE_nid_postfix[]; extern const unsigned char _ZTVN10__cxxabiv120__si_class_type_infoE_nid_postfix[]; - +extern const unsigned char _ZTVN10__cxxabiv121__vmi_class_type_infoE_nid_postfix[]; } namespace { @@ -26,6 +35,21 @@ const TypeRecord DerivedType{_ZTVN10__cxxabiv120__si_class_type_infoE_nid_postfi const TypeRecord OtherType{_ZTVN10__cxxabiv120__si_class_type_infoE_nid_postfix + 16, "5Other", &BaseType}; const Vtable BaseVtable{0, &BaseType, nullptr}; const Vtable DerivedVtable{0, &DerivedType, nullptr}; +const MultipleRecord MultipleType{_ZTVN10__cxxabiv121__vmi_class_type_infoE_nid_postfix + 16, "8Multiple", 0, 1, + {{&BaseType, (16 << 8) | 2}}}; +const MultipleRecord PrivateType{_ZTVN10__cxxabiv121__vmi_class_type_infoE_nid_postfix + 16, "7Private", 0, 1, + {{&BaseType, (16 << 8) | 1}}}; +const MultipleRecord VirtualType{_ZTVN10__cxxabiv121__vmi_class_type_infoE_nid_postfix + 16, "7Virtual", 0, 1, + {{&BaseType, (-3 * static_cast(sizeof(void*)) * 256) | 3}}}; + +int FindPublic(const TypeRecord* type, std::ptrdiff_t hint, const void* object, const TypeRecord* sourceType, + const void* source) { + using Function = int (APS5_VABI *)(const void*, std::ptrdiff_t, const void*, const void*, const void*); + const auto* table = static_cast(type->vtable); + Function function; + std::memcpy(&function, table + 8 * sizeof(void*), sizeof(function)); + return function(type, hint, object, sourceType, source); +} } // namespace @@ -36,11 +60,26 @@ int main() { Require(__dynamic_cast_nid_postfix(&derived, reinterpret_cast(&BaseType), reinterpret_cast(&OtherType), 0) == nullptr); Require(__dynamic_cast_nid_postfix(&base, reinterpret_cast(&BaseType), reinterpret_cast(&DerivedType), 0) == nullptr); Require(__dynamic_cast_nid_postfix(nullptr, reinterpret_cast(&BaseType), reinterpret_cast(&DerivedType), 0) == nullptr); + std::array multipleObject{}; + Require(FindPublic(reinterpret_cast(&MultipleType), -1, multipleObject.data(), &BaseType, + multipleObject.data() + 16) == 6); + Require(FindPublic(reinterpret_cast(&MultipleType), -1, multipleObject.data(), &BaseType, + multipleObject.data()) == 1); + Require(FindPublic(reinterpret_cast(&PrivateType), -1, multipleObject.data(), &BaseType, + multipleObject.data() + 16) == 1); + std::array virtualTable{}; + virtualTable[0] = 16; + const auto* virtualAddressPoint = virtualTable.data() + 3; + alignas(std::intptr_t) std::array virtualObject{}; + *reinterpret_cast(virtualObject.data()) = virtualAddressPoint; + Require(FindPublic(reinterpret_cast(&VirtualType), -1, virtualObject.data(), &BaseType, + virtualObject.data() + 16) == 7); + Require(FindPublic(reinterpret_cast(&MultipleType), -3, multipleObject.data(), &BaseType, + multipleObject.data() + 16) == 6); void* object = __cxa_allocate_exception_nid_postfix(16); auto* type = reinterpret_cast(const_cast(&DerivedType)); LibcException::Header* header = __cxa_init_primary_exception_nid_postfix(object, type, Destroy); Require(header == LibcException::FromObject(object)); Require(header->type == type && header->destructor == Destroy && header->adjusted == object); __cxa_free_exception_nid_postfix(object); - } From c7fbae3ebd4e1b3048d141775d450e695325ea5c Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:12:21 +0000 Subject: [PATCH 205/318] test(libc): cover a skipped virtual base and a nonvirtual private base in __do_find_public_src The -3 hint skips virtual bases, which only a virtual-base fixture exercises, and the private fixture now has the nonvirtual flags its name says. --- core/libs/tests/GuestDynamicCast.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/libs/tests/GuestDynamicCast.cpp b/core/libs/tests/GuestDynamicCast.cpp index b99dd3bc19..ae3f708021 100644 --- a/core/libs/tests/GuestDynamicCast.cpp +++ b/core/libs/tests/GuestDynamicCast.cpp @@ -38,7 +38,7 @@ const Vtable DerivedVtable{0, &DerivedType, nullptr}; const MultipleRecord MultipleType{_ZTVN10__cxxabiv121__vmi_class_type_infoE_nid_postfix + 16, "8Multiple", 0, 1, {{&BaseType, (16 << 8) | 2}}}; const MultipleRecord PrivateType{_ZTVN10__cxxabiv121__vmi_class_type_infoE_nid_postfix + 16, "7Private", 0, 1, - {{&BaseType, (16 << 8) | 1}}}; + {{&BaseType, 16 << 8}}}; const MultipleRecord VirtualType{_ZTVN10__cxxabiv121__vmi_class_type_infoE_nid_postfix + 16, "7Virtual", 0, 1, {{&BaseType, (-3 * static_cast(sizeof(void*)) * 256) | 3}}}; @@ -76,6 +76,8 @@ int main() { virtualObject.data() + 16) == 7); Require(FindPublic(reinterpret_cast(&MultipleType), -3, multipleObject.data(), &BaseType, multipleObject.data() + 16) == 6); + Require(FindPublic(reinterpret_cast(&VirtualType), -3, virtualObject.data(), &BaseType, + virtualObject.data() + 16) == 1); void* object = __cxa_allocate_exception_nid_postfix(16); auto* type = reinterpret_cast(const_cast(&DerivedType)); LibcException::Header* header = __cxa_init_primary_exception_nid_postfix(object, type, Destroy); From adef5635c8239cc1037ac0743a6de36ed992d511 Mon Sep 17 00:00:00 2001 From: LeandroLP23 Date: Mon, 26 Oct 2026 12:24:40 +0100 Subject: [PATCH 206/318] fix(libkernel_sys): wait for AIO timeouts until one deadline, precisely on Windows aio_suspend and aio_waitcomplete waited through std::condition_variable, which winpthreads ends only on 15.6 ms GetTickCount64 steps (#1471). They also restarted the full timeout after every wake that did not finish their wait, so completions of other requests could keep them waiting past it. Both now compute one deadline and wait on TimedWait::Condition until it. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libkernel_sys/Aio.cpp | 32 ++++++++------- core/libs/prx/libkernel_sys/CMakeLists.txt | 2 +- core/libs/tests/GuestKernelSysAio.cpp | 47 ++++++++++++++++++++++ 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/core/libs/prx/libkernel_sys/Aio.cpp b/core/libs/prx/libkernel_sys/Aio.cpp index f81b5842de..6f6dc680a5 100644 --- a/core/libs/prx/libkernel_sys/Aio.cpp +++ b/core/libs/prx/libkernel_sys/Aio.cpp @@ -1,9 +1,9 @@ #include "prx/libkernel_sys/include/Api.hpp" #include "prx/libkernel/include/Api.hpp" #include "prx/libc/include/Api.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" #include #include -#include #include #include #include @@ -71,7 +71,7 @@ struct Job { struct Aio { std::mutex lock; - std::condition_variable changed; + TimedWait::Condition changed; bool used = false; bool held = false; bool started = false; @@ -104,7 +104,7 @@ void Worker() { auto& aio = Instance(); std::unique_lock lock(aio.lock); for (;;) { - aio.changed.wait(lock, [&] { return !aio.held && !aio.queued.empty(); }); + aio.changed.Wait(lock, [&] { return !aio.held && !aio.queued.empty(); }); auto job = aio.queued.front(); aio.queued.pop_front(); job->state = State::Running; @@ -113,7 +113,7 @@ void Worker() { lock.lock(); job->state = State::Done; aio.done.push_back(job); - aio.changed.notify_all(); + aio.changed.NotifyAll(); } } @@ -122,7 +122,7 @@ void Complete(Aio& aio, const std::shared_ptr& job, std::int64_t status, in job->error = error; job->state = State::Done; aio.done.push_back(job); - aio.changed.notify_all(); + aio.changed.NotifyAll(); } std::optional ReadyError(const std::shared_ptr& job) { @@ -190,7 +190,7 @@ int Queue(GuestAiocb* user, int opcode) { aio.started = true; std::thread(Worker).detach(); } - aio.changed.notify_all(); + aio.changed.NotifyAll(); return 0; } @@ -202,7 +202,7 @@ void AioHoldWorkerForTest_nid_no_patch(bool hold) { auto& aio = Instance(); std::lock_guard lock(aio.lock); aio.held = hold; - aio.changed.notify_all(); + aio.changed.NotifyAll(); } int APS5_VABI aio_read_nid_postfix(GuestAiocb* aiocb) { @@ -241,10 +241,11 @@ std::int64_t APS5_VABI aio_return_nid_postfix(GuestAiocb* aiocb) { int APS5_VABI aio_suspend_nid_postfix(const GuestAiocb* const* list, int count, const KernelTimespec* timeout) { if (count < 0 || count > static_cast(QueuePerProcess)) return Fail(GuestEinval); - std::optional wait; + std::optional deadline; if (timeout != nullptr) { - wait = WaitTime(*timeout); + const auto wait = WaitTime(*timeout); if (!wait) return Fail(GuestEinval); + deadline = TimedWait::NowNanos() + static_cast(wait->count()); } auto& aio = Instance(); std::unique_lock lock(aio.lock); @@ -260,20 +261,21 @@ int APS5_VABI aio_suspend_nid_postfix(const GuestAiocb* const* list, int count, } } if (!listed) return 0; - if (!wait) aio.changed.wait(lock); - else if (aio.changed.wait_for(lock, *wait) == std::cv_status::timeout) return Fail(GuestEagain); + if (!deadline) aio.changed.Wait(lock); + else if (!aio.changed.WaitUntil(lock, *deadline)) return Fail(GuestEagain); } } std::int64_t APS5_VABI aio_waitcomplete_nid_postfix(GuestAiocb** result, const KernelTimespec* timeout) { if (result != nullptr) *result = nullptr; - std::optional wait; + std::optional deadline; bool poll = false; if (timeout != nullptr) { poll = timeout->tv_sec == 0 && timeout->tv_nsec == 0; if (!poll) { - wait = WaitTime(*timeout); + const auto wait = WaitTime(*timeout); if (!wait) return Fail(GuestEinval); + deadline = TimedWait::NowNanos() + static_cast(wait->count()); } } auto& aio = Instance(); @@ -281,8 +283,8 @@ std::int64_t APS5_VABI aio_waitcomplete_nid_postfix(GuestAiocb** result, const K aio.used = true; while (aio.done.empty()) { if (poll) return Fail(GuestEagain); - if (!wait) aio.changed.wait(lock); - else if (aio.changed.wait_for(lock, *wait) == std::cv_status::timeout && aio.done.empty()) return Fail(GuestEagain); + if (!deadline) aio.changed.Wait(lock); + else if (!aio.changed.WaitUntil(lock, *deadline) && aio.done.empty()) return Fail(GuestEagain); } if (result != nullptr) *result = aio.done.front()->user; return Consume(aio, aio.done.begin()); diff --git a/core/libs/prx/libkernel_sys/CMakeLists.txt b/core/libs/prx/libkernel_sys/CMakeLists.txt index 70a25df4cd..db959cfd27 100644 --- a/core/libs/prx/libkernel_sys/CMakeLists.txt +++ b/core/libs/prx/libkernel_sys/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(${LIB_NAME} SHARED EXCLUDE_FROM_ALL Export.cpp Export.Discovery.cpp Aio.cpp Cpuset.cpp ThreadQueries.cpp) +add_library(${LIB_NAME} SHARED EXCLUDE_FROM_ALL Export.cpp Export.Discovery.cpp Aio.cpp Cpuset.cpp ThreadQueries.cpp ../libkernel/Time/TimedWait.cpp) target_include_directories(${LIB_NAME} PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(${LIB_NAME}) diff --git a/core/libs/tests/GuestKernelSysAio.cpp b/core/libs/tests/GuestKernelSysAio.cpp index 200d3290b8..94fffb82f7 100644 --- a/core/libs/tests/GuestKernelSysAio.cpp +++ b/core/libs/tests/GuestKernelSysAio.cpp @@ -3,6 +3,7 @@ #include "prx/libc/include/Api.hpp" #include "SceTypes.hpp" #include "prx/libc/include/general/VabiMacros.hpp" +#include #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include namespace { @@ -183,6 +185,50 @@ void SyncOrder(int file) { Require(aio_waitcomplete_nid_postfix(&done, &badTime) == -1 && Error() == Einval); } +void TimedWaits(int file) { + using Clock = std::chrono::steady_clock; + char buffer[4] = {}; + AioHoldWorkerForTest_nid_no_patch(true); + auto held = Request(file, buffer, sizeof(buffer)); + Require(aio_read_nid_postfix(&held) == 0); + const GuestAiocb* list[] = {&held}; + const KernelTimespec millisecond{0, 1000000}; + auto start = Clock::now(); + for (int round = 0; round < 20; ++round) Require(aio_suspend_nid_postfix(list, 1, &millisecond) == -1 && Error() == Eagain); + auto elapsed = Clock::now() - start; + Require(elapsed >= std::chrono::milliseconds(20) && elapsed < std::chrono::milliseconds(200)); + + GuestAiocb* done = nullptr; + start = Clock::now(); + for (int round = 0; round < 20; ++round) Require(aio_waitcomplete_nid_postfix(&done, &millisecond) == -1 && Error() == Eagain); + elapsed = Clock::now() - start; + Require(elapsed >= std::chrono::milliseconds(20) && elapsed < std::chrono::milliseconds(200)); + + std::atomic stop{false}; + std::thread waker([&] { + char other[4] = {}; + const auto end = Clock::now() + std::chrono::milliseconds(600); + while (!stop.load() && Clock::now() < end) { + auto request = Request(file, other, sizeof(other)); + Require(aio_read_nid_postfix(&request) == 0); + Require(aio_cancel_nid_postfix(file, &request) == AioCanceled); + Require(aio_return_nid_postfix(&request) == -1 && Error() == Ecanceled); + std::this_thread::yield(); + } + }); + const KernelTimespec brief{0, 30000000}; + start = Clock::now(); + Require(aio_suspend_nid_postfix(list, 1, &brief) == -1 && Error() == Eagain); + elapsed = Clock::now() - start; + stop = true; + waker.join(); + Require(elapsed >= std::chrono::milliseconds(30) && elapsed < std::chrono::milliseconds(300)); + + Require(aio_cancel_nid_postfix(file, &held) == AioCanceled); + Require(aio_return_nid_postfix(&held) == -1 && Error() == Ecanceled); + AioHoldWorkerForTest_nid_no_patch(false); +} + } int main() { @@ -195,6 +241,7 @@ int main() { ReadAndWrite(file); Cancel(file); SyncOrder(file); + TimedWaits(file); Require(close_nid_postfix(readOnly) == 0 && close_nid_postfix(file) == 0); Require(std::filesystem::remove(path)); std::puts("guest_kernel_sys_aio passed"); From fa653e8c08153c828419adfa1c029ad3488917b3 Mon Sep 17 00:00:00 2001 From: JJenkx Date: Sun, 25 Oct 2026 13:06:33 -0400 Subject: [PATCH 207/318] fix(libc): preserve frexp decomposition and exception behavior --- core/libs/prx/libc/src/Math.cpp | 17 ++++++++- core/libs/tests/GuestMath.cpp | 63 +++++++++++++++++++++++++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index eecd5815e7..2d8c8e3e10 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -446,7 +446,22 @@ float APS5_VABI scalbnf_nid_postfix(float x, int exponent) { return std::scalbn( long double APS5_VABI scalbnl_nid_postfix(long double x, int exponent) { return std::scalbn(x, exponent); } float APS5_VABI significandf_nid_postfix(float x) { return x == 0.0f || !std::isfinite(x) ? x : std::scalbn(x, -std::ilogb(x)); } double APS5_VABI significand_nid_postfix(double x) { return x == 0.0 || !std::isfinite(x) ? x : std::scalbn(x, -std::ilogb(x)); } -double APS5_VABI frexp_nid_postfix(double x, int* exponent) { return std::frexp(x, exponent); } +double APS5_VABI frexp_nid_postfix(double x, int* exponent) { + auto bits = std::bit_cast(x); + auto magnitude = bits & 0x7fffffffffffffffull; + *exponent = 0; + if (magnitude == 0 || magnitude >= 0x7ff0000000000000ull) return x; + int adjustment = 0; + if (magnitude < 0x0010000000000000ull) { + volatile double subnormal = x; + x = subnormal * 0x1p54; + bits = std::bit_cast(x); + magnitude = bits & 0x7fffffffffffffffull; + adjustment = -54; + } + *exponent = static_cast(magnitude >> 52) - 1022 + adjustment; + return std::bit_cast((bits & 0x800fffffffffffffull) | 0x3fe0000000000000ull); +} float APS5_VABI frexpf_nid_postfix(float x, int* exponent) { auto bits = std::bit_cast(x); auto magnitude = bits & 0x7fffffffu; diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index f8e0f9da06..eb30eb770b 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -1744,6 +1744,68 @@ static void CheckIntegerAbsAndDivision() { Require(large.quot == -0x12345678ll && large.rem == 0x9abll); } +static void CheckFrexpEnvironment() { + constexpr struct { + std::uint64_t input; + std::uint64_t fraction; + int exponent; + } cases[] = { + {0x0000000000000000ull, 0x0000000000000000ull, 0}, + {0x8000000000000000ull, 0x8000000000000000ull, 0}, + {0x0000000000000001ull, 0x3fe0000000000000ull, -1073}, + {0x8000000000000001ull, 0xbfe0000000000000ull, -1073}, + {0x000fffffffffffffull, 0x3feffffffffffffeull, -1022}, + {0x800fffffffffffffull, 0xbfeffffffffffffeull, -1022}, + {0x0010000000000000ull, 0x3fe0000000000000ull, -1021}, + {0x8010000000000000ull, 0xbfe0000000000000ull, -1021}, + {0x3fe0000000000000ull, 0x3fe0000000000000ull, 0}, + {0x3ff0000000000000ull, 0x3fe0000000000000ull, 1}, + {0xbff0000000000000ull, 0xbfe0000000000000ull, 1}, + {0x4028000000000000ull, 0x3fe8000000000000ull, 4}, + {0x7fefffffffffffffull, 0x3fefffffffffffffull, 1024}, + {0xffefffffffffffffull, 0xbfefffffffffffffull, 1024}, + {0x7ff0000000000000ull, 0x7ff0000000000000ull, 0}, + {0xfff0000000000000ull, 0xfff0000000000000ull, 0}, + {0x7ff0000000000001ull, 0x7ff0000000000001ull, 0}, + {0xfff0000000000001ull, 0xfff0000000000001ull, 0}, + {0x7ff8000000001234ull, 0x7ff8000000001234ull, 0}, + {0xfff8000000001234ull, 0xfff8000000001234ull, 0}, + }; + std::fenv_t saved; + Require(std::fegetenv(&saved) == 0); + const unsigned savedCsr = _mm_getcsr(); + constexpr int rounding[] = {FE_TONEAREST, FE_DOWNWARD, FE_UPWARD, FE_TOWARDZERO}; + const int savedError = *__error_nid_postfix(); + for (unsigned mode = 0; mode != 4; ++mode) { + for (unsigned denorm = 0; denorm != 4; ++denorm) { + const unsigned control = 0x1f80u | (mode << 13) | ((denorm & 1u) ? 0x40u : 0u) | ((denorm & 2u) ? 0x8000u : 0u); + for (const unsigned sticky : {0u, 5u, 32u}) { + for (const auto& item : cases) { + Require(std::fesetenv(FE_DFL_ENV) == 0); + Require(std::fesetround(rounding[mode]) == 0); + *__error_nid_postfix() = 97; + _mm_setcsr(control | sticky); + int exponent = 77; + const auto fraction = std::bit_cast(frexp_nid_postfix(std::bit_cast(item.input), &exponent)); + const unsigned after = _mm_getcsr(); + const bool subnormal = (item.input & 0x7fffffffffffffffull) != 0 && (item.input & 0x7fffffffffffffffull) < 0x0010000000000000ull; + const bool flushed = subnormal && (denorm & 1u) != 0; + const unsigned flags = sticky | (subnormal && !flushed ? 2u : 0u); + const auto expectedFraction = flushed ? (item.input & 0x8000000000000000ull) | 0x3fe0000000000000ull : item.fraction; + const int expectedExponent = flushed ? -1076 : item.exponent; + if (fraction != expectedFraction || exponent != expectedExponent || after != (control | flags) || *__error_nid_postfix() != 97) { + std::fprintf(stderr, "frexp input=%016llx mode=%u denorm=%u sticky=%u fraction=%016llx exponent=%d csr=%x\n", static_cast(item.input), mode, denorm, sticky, static_cast(fraction), exponent, after); + std::abort(); + } + } + } + } + } + Require(std::fesetenv(&saved) == 0); + _mm_setcsr(savedCsr); + *__error_nid_postfix() = savedError; +} + static void CheckScaling() { Require(scalbnl_nid_postfix(3.0L, 4) == 48.0L && scalbnl_nid_postfix(1.0L, -16445) == 0x1p-16445L); Require(scalbnl_nid_postfix(1.0L, 16384) == std::numeric_limits::infinity()); @@ -2350,6 +2412,7 @@ int main() { CheckFmodfErrno(); CheckDoubleClassification(); CheckIntegerAbsAndDivision(); + CheckFrexpEnvironment(); CheckScaling(); CheckLongDoubleFunctions(); CheckLongDoubleHyperbolic(); diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index c3a5564270..a7de7f8922 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -39,6 +39,7 @@ - [reallocalign](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) - `reallocalign(ptr, 0, alignment)` for a block without that alignment throws, where the console's heap in a payload process on PS5 firmware 13.60 returned a different block with that alignment and 0 usable bytes, `errno` untouched (a 64-aligned block given 256); whether it frees the old block is not known. There, as here, `reallocalign(ptr, 0, 16)` on a 256-aligned block returned it unchanged, `reallocalign(NULL, 0, 256)` a 256-aligned block with 0 usable bytes, and alignment 3 a moved block that kept its bytes - [reallocf](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libSceLibcInternal, implemented in libc) - follows FreeBSD 11 `lib/libc/stdlib/reallocf.c` on top of this libc's `realloc`, as a payload process on a PS5 (firmware 13.60) does with the default heap (null and 0, size 0, 2^40 with `errno` 22 and the block freed, `SIZE_MAX` with `errno` untouched): size 0 does what `realloc` does here, keeping the block with 0 usable bytes with the default heap and freeing it and returning null with an application allocator, so the block is not freed twice; when `realloc` fails, the block is freed through the same allocator and null is returned with the `errno` that `realloc` left. Other failures (a commit failure that is not out of memory, an unknown pointer) throw, as they do for `realloc` - [sceLibcMspaceReallocalign](../../../core/libs/prx/libc/src/Mspace.cpp) (libSceLibcInternal, implemented in libc) takes `(void* heap, void* block, size_t alignment, size_t size)`, the order of the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x13880, whose internal routine at 0x12390 treats the third argument as the alignment and, for a null block, passes it to the routine `sceLibcMspaceMemalign` uses). Measured on a PS5 (firmware 13.60) from a homebrew payload: a null block with alignment 4096 and size 100 returns a 4096-aligned block; an alignment of 0 or 3 succeeds and 100 gives a 128-aligned block, as the routine raises it to at least 32 and to the next power of two; an alignment of 2^63 or more returns null, keeps the block and leaves `errno` alone; and a size of 0 returns a block (the same one when it is already suitable) instead of freeing it. A block that is aligned and large enough is kept where it is, without giving back the bytes past the new size as the console's in-place resize does +- [frexp](../../../core/libs/prx/libc/src/Math.cpp) (libc) - follows the two bundled PPSA03766/PPSA07923 functions: exponent-bit decomposition, conditional double-precision subnormal scaling, zero exponent for zero/NaN/infinity, and unchanged signaling NaNs. Masked-exception observations cover rounding, DAZ/FTZ and sticky flags; unmasked traps and other SDK versions are unverified. - [acosl](../../../core/libs/prx/libc/src/Math.cpp) (libc) - computed by the host's math library, like the other libc math functions here, not by the console's (FreeBSD msun ld80 code in Sony's `libc.prx`): results can differ from the console's in the last bit, and on Linux glibc sets `errno` to `EDOM` for `acosl` outside [-1, 1] where msun only raises the floating-point exception. `long double` is the x87 80-bit format, as Sony's `frexpl` uses it (`fldt 0x8(%rsp)`, result in `st(0)`) - [atanl, sinl, cosl, tanl, sinhl, coshl, tanhl, expl, exp2l, logl, log10l and powl](../../../core/libs/prx/libc/src/Math.cpp) (libc) - computed by the host's math library, as `acosl` is, not by the console's FreeBSD msun ld80 code: results can differ from the console's in the last bit (glibc's `log10l(1000)` is not exactly 3), and on Linux glibc sets `errno` to `EDOM` or `ERANGE` where msun only raises the floating-point exception. `ldexpl` and `truncl` are exact operations and give the console's result; `logb` and `remainder` give the console's value too, but on Linux glibc sets `errno` (`ERANGE` for `logb(0)`, `EDOM` for `remainder` by zero or of an infinity) where msun only raises the floating-point exception - [Dinkumware atomics](../../../core/libs/prx/libc/src/DinkumwareAtomics.cpp) (libc) - every operation is sequentially consistent whatever memory order it is given, which is never weaker than the console. Matching the `libc.prx` bundled with PPSA08427 (that binary run side by side on every order from 0 to 7), `_Atomic_load_8` returns 0 for a release, acq_rel or invalid order and the 8-byte exchange and fetch operations return 0 without touching memory for an order above `memory_order_seq_cst`; the generic `_Atomic_copy`, `_Atomic_exchange` and `_Atomic_compare_exchange_*` lock the byte flag passed first, as the console does From eeea4a5fbb4441e114137e053fdcde5682929a65 Mon Sep 17 00:00:00 2001 From: JJenkx Date: Tue, 27 Oct 2026 06:47:32 -0400 Subject: [PATCH 208/318] test(http2): stop pinning unsupported downgrade refusal --- core/libs/tests/GuestHttp2Transport.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/tests/GuestHttp2Transport.cpp b/core/libs/tests/GuestHttp2Transport.cpp index 8d8b2e40f8..41e4a9e43b 100644 --- a/core/libs/tests/GuestHttp2Transport.cpp +++ b/core/libs/tests/GuestHttp2Transport.cpp @@ -470,8 +470,8 @@ void RedirectRequestMethods(int ssl) { } void RedirectFailures(int ssl) { - for (int mode = 0; mode < 5; ++mode) { - Server server(Mode::Redirect, mode == 4 ? "http://localhost/" : "/another"); + for (int mode = 0; mode < 4; ++mode) { + Server server(Mode::Redirect, "/another"); Client client(ssl); RedirectObservation observed; observed.nullMethod = mode == 0; From fedf3e91a70676e839b89d2abdf97fca4deda6be Mon Sep 17 00:00:00 2001 From: QRMOE <217609482+QRMOE@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:55:33 +0100 Subject: [PATCH 209/318] test(agc): stop pinning converted unorm border, gather and LOD samples as refused The converted-image test asserted that clamp-to-border image_sample_lz, image_gather4_lz and image_get_lod of a converted unorm image throw. Those are valid forms that are only unimplemented, so the test no longer requires the refusal. The missing-channel checks stay. Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp index 97dfe2791d..8e302dd989 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp @@ -57,7 +57,6 @@ constexpr std::array PointSampler{0x00000092u, 0x00fff000u, 0u constexpr std::array BilinearSampler{0x00000092u, 0x00fff000u, 0x00500000u, 0u}; constexpr std::array WrapBilinearSampler{0x00000090u, 0x00fff000u, 0x00500000u, 0u}; constexpr std::array WrapPointSampler{0x00000090u, 0x00fff000u, 0u, 0u}; -constexpr std::array BorderSampler{0x000000b6u, 0x00fff000u, 0u, 0u}; constexpr std::uint32_t SampleStep = 0x3a200000u; constexpr std::uint32_t OffGridStep = 0x3a206000u; constexpr std::uint32_t SampleStart = 0x39c00000u; @@ -118,8 +117,6 @@ alignas(256) constexpr auto StoreXz = StoreCode(0x5u, 0u); alignas(256) constexpr auto StoreD16 = StoreCode(0x7u, D16); alignas(256) constexpr auto StoreXzD16 = StoreCode(0x5u, D16); alignas(256) constexpr auto SampleLz = SamplerCode(0xf09c0f08u); -alignas(256) constexpr auto Gather4Lz = SamplerCode(0xf11c0108u); -alignas(256) constexpr auto GetLod = SamplerCode(0xf1800308u); alignas(256) constexpr auto SampleAt = SampleAtCode(SampleStep, SampleStart); alignas(256) constexpr auto SampleWrapped = SampleAtCode(SampleStep, WrappedStart); alignas(256) constexpr auto SampleOffGrid = SampleAtCode(OffGridStep, SampleStart); @@ -777,9 +774,6 @@ int main(int argc, char** argv) { CheckSample(*device, texels, SampleOffGrid, OffGridStep, SampleStart, SwizzleZYX1, BilinearSampler, true, false, "off-grid bilinear image_sample_lz Z Y X 1"); CheckSample(*device, texels, SampleWrappedOffGrid, OffGridStep, WrappedStart, SwizzleXYZ1, WrapBilinearSampler, true, true, "off-grid wrapped bilinear image_sample_lz X Y Z 1"); CheckConvertedPreparation(*device, texels); - RequireRefused(*device, SampleLz, texels, SwizzleXYZ1, BorderSampler, "implemented only with wrap or clamp-to-edge addressing", "image_sample_lz with clamp-to-border addressing"); - RequireRefused(*device, Gather4Lz, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_gather4_lz"); - RequireRefused(*device, GetLod, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_get_lod"); for (const auto& layout : FloatLayouts) { CheckFloatStoreD16(*device, texels, layout, 0x7u); CheckFloatStoreD16(*device, texels, layout, 0x5u); From 930c1c0ec0298600e9d43baab0c3f89c5df4975b Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:26:21 -0700 Subject: [PATCH 210/318] test(libSceNet): stop asserting that valid scope and default-route inputs throw The named zone "fe80::1%lo0", the two AF_INET scope conversions and sceNetConfigGetDefaultRoute with a valid output and no host gateway are valid but unimplemented forms; tests no longer pin them as throws. The malformed and null-pointer checks, the numeric IPv6 round trips and the success controls stay. --- core/libs/tests/GuestSceNet.cpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 9202d6ef61..0804c2ab7b 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -353,17 +353,6 @@ static void CheckScopeIdText() { Require(Failed(sceNetInetPtonWithScopeId(28, "fe80::1%2", address.data(), nullptr), 22)); *sceNetErrnoLoc() = 123; Require(sceNetInetNtopWithScopeId(28, nullptr, text.data(), text.size(), 2) == nullptr && *sceNetErrnoLoc() == 22); - const auto throws = [](auto&& call) { - try { - call(); - } catch (const std::runtime_error&) { - return true; - } - return false; - }; - Require(throws([&] { sceNetInetPtonWithScopeId(28, "fe80::1%lo0", address.data(), &scope); })); - Require(throws([&] { sceNetInetPtonWithScopeId(2, "127.0.0.1", address.data(), &scope); })); - Require(throws([&] { sceNetInetNtopWithScopeId(2, address.data(), text.data(), text.size(), 0); })); } static void CheckDefaultRoute() { @@ -382,7 +371,6 @@ static void CheckDefaultRoute() { Require(throws([] { sceNetConfigGetDefaultRoute(nullptr); })); if (result != 0 || std::strcmp(route, "0.0.0.0") == 0) { Require(result == 0 || result == static_cast(0x80412108)); - Require(throws([&] { sceNetConfigGetDefaultRoute(&gateway); }) && gateway == 0xa5a5a5a5u); return; } Require(sceNetConfigGetDefaultRoute(&gateway) == 0); From c207cb884dd447cc283ba9675c01455c8292f8f8 Mon Sep 17 00:00:00 2001 From: Alexandre Figueiredo Date: Sun, 25 Oct 2026 10:34:08 +0000 Subject: [PATCH 211/318] fix(libSceNpCppWebApi): count Vector and Vector references where the bundled module does --- core/libs/prx/libSceNpCppWebApi/Export.cpp | 14 +++-- .../tests/GuestNpCppWebApiIntrusivePtr.cpp | 53 +++++++++++++++++-- core/libs/tests/GuestNpCppWebApiString.cpp | 6 ++- docs/dev/debt/libSceNpCppWebApi.md | 4 +- 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/core/libs/prx/libSceNpCppWebApi/Export.cpp b/core/libs/prx/libSceNpCppWebApi/Export.cpp index 340eb7ffed..b9ee91b8ff 100644 --- a/core/libs/prx/libSceNpCppWebApi/Export.cpp +++ b/core/libs/prx/libSceNpCppWebApi/Export.cpp @@ -325,6 +325,8 @@ static_assert(sizeof(IntrusivePtr) == 0x18); static constexpr std::size_t ReferenceCountOffset = 0; static constexpr std::size_t VectorReferenceCountOffset = 0x30; +static constexpr std::size_t StringVectorReferenceCountOffset = 0x38; +static constexpr std::size_t ScalarVectorReferenceCountOffset = 0x20; static std::atomic_ref ReferenceCount(void* object, const std::size_t offset) { return std::atomic_ref(*reinterpret_cast(static_cast(object) + offset)); @@ -1050,6 +1052,7 @@ void APS5_VABI _ZN3sce2Np9CppWebApi6Common6VectorINS2_6StringEEC1EPNS2_10LibCont self->object = nullptr; self->deleter = nullptr; self->libContext = libContext; + ReferenceCount(self, StringVectorReferenceCountOffset).store(0); } int APS5_VABI _ZN3sce2Np9CppWebApi6Common6VectorINS2_12IntrusivePtrINS1_14SessionManager2V120RequestPlayerSessionEEEE8pushBackERKS8_(IntrusivePtr* self, const void* element) { @@ -1334,7 +1337,7 @@ void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_14SessionManager2V1 } void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEEC1ERS7_(IntrusivePtr* self, const IntrusivePtr* source) { - CopyIntrusivePtr(self, source, 0x30); + CopyIntrusivePtr(self, source, StringVectorReferenceCountOffset); } void* APS5_VABI _ZNK3sce2Np9CppWebApi6Common8IteratorINS2_12IntrusivePtrINS1_14SessionManager2V136UsersPlayerSessionsInvitationForReadEEEEdeEv(const void* self) { @@ -1356,6 +1359,7 @@ void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6String self->object = body; self->deleter = deleter; self->libContext = libContext; + if (body) ReferenceCount(body, StringVectorReferenceCountOffset).fetch_add(1); } void APS5_VABI _ZN3sce2Np9CppWebApi6Common6VectorINS2_12IntrusivePtrINS1_14SessionManager2V120RequestPlayerSessionEEEEC1EPNS2_10LibContextE(IntrusivePtr* self, void* libContext) { @@ -1441,7 +1445,7 @@ void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_14SessionManager2V1 } void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEED1Ev(IntrusivePtr* self) { - ReleaseIntrusivePtr(self, 0x30, __func__); + ReleaseIntrusivePtr(self, StringVectorReferenceCountOffset, __func__); } void* APS5_VABI _ZN3sce2Np9CppWebApi14SessionManager2V154GetUsersAccountIdPlayerSessionsInvitationsResponseBody14getInvitationsEv(void* self) { @@ -3199,7 +3203,7 @@ void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEEC1Ev(In } void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEED1Ev(IntrusivePtr* self) { - ReleaseIntrusivePtr(self, 0x30, __func__); + ReleaseIntrusivePtr(self, ScalarVectorReferenceCountOffset, __func__); } int APS5_VABI _ZN3sce2Np9CppWebApi6Common13ConstIteratorINS2_6StringEED2Ev(void* self) { @@ -3243,8 +3247,8 @@ std::int64_t APS5_VABI _ZNK3sce2Np9CppWebApi6Common15TransactionBaseINS2_15Defau } void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEEaSERS6_(IntrusivePtr* self, const IntrusivePtr* source) { - ReleaseIntrusivePtr(self, 0x30, __func__); - CopyIntrusivePtr(self, source, 0x30); + ReleaseIntrusivePtr(self, ScalarVectorReferenceCountOffset, __func__); + CopyIntrusivePtr(self, source, ScalarVectorReferenceCountOffset); } int APS5_VABI _ZN3sce2Np9CppWebApi6Common15TransactionBaseINS2_15DefaultResponseENS2_12IntrusivePtrINS2_18ResponseHeaderBaseEEEE5abortEv() { diff --git a/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp b/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp index 37fb800f48..429b52c5ad 100644 --- a/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp +++ b/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp @@ -18,6 +18,8 @@ static void Require(bool value) { if (!value) std::abort(); } static constexpr std::size_t ObjectSize = 0x40; static constexpr std::size_t VectorReferenceCountOffset = 0x30; +static constexpr std::size_t StringVectorReferenceCountOffset = 0x38; +static constexpr std::size_t ScalarVectorReferenceCountOffset = 0x20; struct DestroyCase { Destroy destroy; @@ -243,11 +245,11 @@ static void TestVectorPushBackReleasesElements() { Require(deletedObject == object); } -static void TestAdoptedVectorBodyDestroyedThroughDeleter() { +static void TestAdoptedStringVectorCountsReferences() { int context = 0; alignas(8) unsigned char body[ObjectSize]; std::memset(body, 0x5a, sizeof(body)); - WriteCount(body, VectorReferenceCountOffset, 1); + WriteCount(body, StringVectorReferenceCountOffset, 0); deleterCalls = 0; deletedObject = nullptr; @@ -257,12 +259,52 @@ static void TestAdoptedVectorBodyDestroyedThroughDeleter() { Require(vector.object == body); Require(vector.deleter == RecordDelete); Require(vector.libContext == &context); + Require(ReadCount(body, StringVectorReferenceCountOffset) == 1); + Require(OnlyCountChanged(body, StringVectorReferenceCountOffset)); + + IntrusivePtr copy; + _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEEC1ERS7_(©, &vector); + Require(copy.object == body); + Require(ReadCount(body, StringVectorReferenceCountOffset) == 2); + + _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEED1Ev(©); + Require(ReadCount(body, StringVectorReferenceCountOffset) == 1); + Require(deleterCalls == 0); _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEED1Ev(&vector); - Require(ReadCount(body, VectorReferenceCountOffset) == 0); + Require(ReadCount(body, StringVectorReferenceCountOffset) == 0); + Require(OnlyCountChanged(body, StringVectorReferenceCountOffset)); Require(deleterCalls == 1); Require(deletedObject == body); Require(vector.object == nullptr); + + IntrusivePtr empty; + _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEEC1EPS6_PFvS8_EPNS2_10LibContextE(&empty, nullptr, RecordDelete, &context); + Require(empty.object == nullptr); + _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEED1Ev(&empty); + Require(deleterCalls == 1); +} + +static void TestScalarVectorAssignment() { + int context = 0; + alignas(8) unsigned char first[ObjectSize]; + alignas(8) unsigned char second[ObjectSize]; + std::memset(first, 0x5a, sizeof(first)); + std::memset(second, 0x5a, sizeof(second)); + WriteCount(first, ScalarVectorReferenceCountOffset, 2); + WriteCount(second, ScalarVectorReferenceCountOffset, 1); + deleterCalls = 0; + + IntrusivePtr target{first, RecordDelete, &context}; + const IntrusivePtr source{second, RecordDelete, &context}; + _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEEaSERS6_(&target, &source); + + Require(target.object == second); + Require(ReadCount(first, ScalarVectorReferenceCountOffset) == 1); + Require(ReadCount(second, ScalarVectorReferenceCountOffset) == 2); + Require(OnlyCountChanged(first, ScalarVectorReferenceCountOffset)); + Require(OnlyCountChanged(second, ScalarVectorReferenceCountOffset)); + Require(deleterCalls == 0); } @@ -413,12 +455,15 @@ int main() { {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_11UserProfile2V113BasicPresenceEED1Ev, 0}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_30CommunicationRestrictionStatus2V338CommunicationRestrictionStatusResponseEED1Ev, 0}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS3_INS1_11UserProfile2V112BasicProfileEEEEEED1Ev, VectorReferenceCountOffset}, + {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEED1Ev, StringVectorReferenceCountOffset}, + {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEED1Ev, ScalarVectorReferenceCountOffset}, }; for (const auto& testCase : destructors) TestDestroy(testCase); TestSharedOwnership(); TestVectorPushBackReleasesElements(); - TestAdoptedVectorBodyDestroyedThroughDeleter(); + TestAdoptedStringVectorCountsReferences(); + TestScalarVectorAssignment(); TestUserProfileVectorAssign(); TestUserProfileModelAndTransaction(); return 0; diff --git a/core/libs/tests/GuestNpCppWebApiString.cpp b/core/libs/tests/GuestNpCppWebApiString.cpp index 6d50a21eb3..d02de2cef9 100644 --- a/core/libs/tests/GuestNpCppWebApiString.cpp +++ b/core/libs/tests/GuestNpCppWebApiString.cpp @@ -18,7 +18,8 @@ constexpr std::size_t kBufferSizeOffset = 0x10; constexpr std::size_t kLibContextOffset = 0x18; constexpr unsigned char kFill = 0xA5; -constexpr std::size_t kVectorSize = 0x18; +constexpr std::size_t kVectorSize = 0x48; +constexpr std::size_t kVectorReferenceCountOffset = 0x38; constexpr std::size_t kObjectOffset = 0x00; constexpr std::size_t kBodyMagicOffset = 0x00; constexpr std::size_t kBodyReferenceCountOffset = 0x30; @@ -141,6 +142,9 @@ void TestVectorPushBackCopiesStringContent() { _ZN3sce2Np9CppWebApi6Common6VectorINS2_6StringEEC1EPNS2_10LibContextE(reinterpret_cast(vector.bytes), &libContext); Require(VectorGuardIntact(vector)); + std::int32_t referenceCount; + std::memcpy(&referenceCount, vector.bytes + kVectorReferenceCountOffset, sizeof(referenceCount)); + Require(referenceCount == 0); _ZN3sce2Np9CppWebApi6Common6StringC1EPNS2_10LibContextE(reinterpret_cast(element.bytes), &libContext); Require(_ZN3sce2Np9CppWebApi6Common6String6appendEPKc(reinterpret_cast(element.bytes), "privilege") == 0); diff --git a/docs/dev/debt/libSceNpCppWebApi.md b/docs/dev/debt/libSceNpCppWebApi.md index 27a420a14f..92e04fae71 100644 --- a/docs/dev/debt/libSceNpCppWebApi.md +++ b/docs/dev/debt/libSceNpCppWebApi.md @@ -11,9 +11,9 @@ ### Unknown function info - [sce::Np::CppWebApi](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - every request entry point (API calls, transaction `start`/`getResponse`/`readData`, the `*Factory::create`s) returns `SCE_NP_WEBAPI_ERROR_NOT_SIGNED_IN` offline. The constant's name and value follow the reference tables in shadPS4 `np_error.h` and fpPS4 `np_error.pas`, where `0x80552907` is `ORBIS_NP_WEBAPI_ERROR_NOT_SIGNED_IN` and `0x80552901` is `OUT_OF_MEMORY` (no `UNAVAILABLE` exists in the `0x80552901`-`0x8055291F` range); it is the code a signed-out console returns from these entry points. The earlier `0x80552901` value came from the module the previous implementation shipped, which used the same incorrect constant -- [sce::Np::CppWebApi::Common::IntrusivePtr](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - the 0x18-byte layout (object, deleter, `LibContext*`) and the offset of the 32-bit reference count in the pointee (0x00, 0x30 for `Vector`) come from the module bundled with one title (PPSA21564); other SDK versions are not checked. The destructor throws on the last reference of a pointer without a deleter: destroying the object needs its destructor and the `LibContext` allocator, which are not implemented +- [sce::Np::CppWebApi::Common::IntrusivePtr](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - the 0x18-byte layout (object, deleter, `LibContext*`) and the offset of the 32-bit reference count in the pointee (0x00; in a `Vector`, 0x20 for `unsigned long`, 0x30 for `IntrusivePtr` and 0x38 for `String` elements) come from the module bundled with one title (PPSA21564); other SDK versions are not checked. The destructor throws on the last reference of a pointer without a deleter: destroying the object needs its destructor and the `LibContext` allocator, which are not implemented - [sce::Np::CppWebApi::Common::String](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - the 0x20-byte layout (32-bit reference count, buffer at 0x08, buffer size at 0x10, `LibContext*` at 0x18) comes from the module bundled with one title (PPSA21564); other SDK versions are not checked. `append` grows the buffer with the host allocator and the destructor frees it, instead of going through the `LibContext` allocator. `pushBack` into `Vector` stores a deep copy, so a destroyed temporary cannot invalidate the stored element; a title that byte-copies a `String` with a live buffer would still double-free, as nothing increments the reference count -- [sce::Np::CppWebApi::Common::Vector](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - imported by PPSA03671, whose dump ships no `libSceNpCppWebApi.prx`. A `Vector` handle is an `IntrusivePtr` over a shared body; bodies created by `pushBack` use a host-side layout with the reference count at 0x30, matching the adopted-body convention measured on PPSA21564. The pointer/deleter constructor adopts a title-provided body released through the stored deleter; `pushBack` on such a body throws, since its element layout is unknown. `begin`, `end`, iterators and `copyFrom` throw: the only `Vector`s the title reads back come from failed transactions, which never produce one. `empty`, `size` and `operator[]` read the host-created body honestly and throw on title-provided bodies +- [sce::Np::CppWebApi::Common::Vector](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - imported by PPSA03671, whose dump ships no `libSceNpCppWebApi.prx`. A `Vector` handle is an `IntrusivePtr` over a shared body; bodies created by `pushBack` use a host-side layout with the reference count at 0x30. The pointer/deleter constructor adopts a title-provided `Vector`, counting the reference at 0x38, which the `Vector` constructor zeroes, as the module bundled with PPSA21564 does, and releases it through the stored deleter; `pushBack` on such a body throws, since its element layout is unknown. `begin`, `end`, iterators and `copyFrom` throw: the only `Vector`s the title reads back come from failed transactions, which never produce one. `empty`, `size` and `operator[]` read the host-created body honestly and throw on title-provided bodies - [sce::Np::CppWebApi::SessionManager2V1](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - imported by PPSA03671. Offline, the factories, `PlayerSessionsApi` calls and transaction `start`/`getResponse`/`readData` fail with `SCE_NP_WEBAPI_ERROR_NOT_SIGNED_IN`; parameter setters, `initialize`, `terminate`, constructors and destructors succeed without keeping state, as no request is ever sent. Response-body accessors (`getSessionId`, `getPlayerSessions`, `getInvitations`, the `IsSet` queries) throw: a failed request never produces a response object - [sce::Np::CppWebApi::UserProfile::V1, ProfanityFilter::V2, CommunicationRestrictionStatus::V3](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - imported by an Epic Online Services module whose dump ships no `libSceNpCppWebApi.prx`. The reference-count offset 0x00 of `ResponseHeaderBase` and `WebApiFilterRequest` is assumed from the other objects, not measured. Offline, the five API calls, `WebApiFilterRequestFactory::create` and the transactions' `start`/`getResponseHeaders` fail with `SCE_NP_WEBAPI_ERROR_NOT_SIGNED_IN`. Response accessors, `ResponseHeaderBase::getHeaderValue` and the `Vector`/`Iterator` members over `IntrusivePtr` and `unsigned long` throw: a failed request never produces a response - [libSceNpCppWebApi](../../../core/libs/prx/libSceNpCppWebApi/Export.cpp) (libSceNpCppWebApi) - unimplemented stubs have their parameter lists, const qualifiers and return types aligned with the demangled symbols confirmed by NID matching in [#8143](https://github.com/boykopovar/AnyPS5/issues/8143). They cover response-body accessors, iterators, and uninstantiated transactions in Leaderboards::V1, SessionManager::V1, InGameCatalog::V5, TitleCloudStorage::V1, and Common templates: each mangled name gives the method and parameter types, but the runtime layout of response objects and errors are unverified on console, so each call continues to throw NotImplemented_nid_no_patch. No call site is recorded for them From 563e30596524610c34248ea992e75af8f1f3a3a4 Mon Sep 17 00:00:00 2001 From: JJenkx Date: Fri, 23 Oct 2026 11:22:30 -0400 Subject: [PATCH 212/318] fix(libc): compute log10 with fdlibm's algorithm, as titles' libc.prx does log10 returned the host's log10, whose last bit differs from the titles' copy on Windows on 3,066,397 of 320 million random arguments. The titles' copy is the fdlibm e_log10.c over its k_log.h, which FreeBSD 11.4 later replaced, and it sets EDOM for every nonzero argument with the sign bit set but leaves errno alone for +-0. log10 now computes that and sets EDOM there. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libc/src/Math.cpp | 74 ++++++++++++++++++++++++++++++++- core/libs/tests/GuestMath.cpp | 27 ++++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index 9c34d18ed7..f26fe897a9 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -1615,7 +1615,79 @@ float APS5_VABI logf_nid_postfix(float x) { return k == 0 ? f - s * (f - R) : dk * ln2_hi - ((s * (f - R) - dk * ln2_lo) - f); } float APS5_VABI log2f_nid_postfix(float x) { return std::log2(x); } -double APS5_VABI log10_nid_postfix(double x) { return std::log10(x); } +static double Log10Kernel(double x) { + const double Lg1 = 6.666666666666735130e-01; + const double Lg2 = 3.999999999940941908e-01; + const double Lg3 = 2.857142874366239149e-01; + const double Lg4 = 2.222219843214978396e-01; + const double Lg5 = 1.818357216161805012e-01; + const double Lg6 = 1.531383769920937332e-01; + const double Lg7 = 1.479819860511658591e-01; + std::int32_t hx = static_cast(std::bit_cast(x) >> 32); + const double f = x - 1.0; + if ((0x000fffff & (2 + hx)) < 3) { + if (f == 0.0) { + return 0.0; + } + return f * f * (0.33333333333333333 * f - 0.5); + } + const double s = f / (2.0 + f); + const double z = s * s; + hx &= 0x000fffff; + std::int32_t i = hx - 0x6147a; + const double w = z * z; + const std::int32_t j = 0x6b851 - hx; + const double t1 = w * (Lg2 + w * (Lg4 + w * Lg6)); + const double t2 = z * (Lg1 + w * (Lg3 + w * (Lg5 + w * Lg7))); + i |= j; + const double R = t2 + t1; + if (i > 0) { + const double hfsq = 0.5 * f * f; + return s * (hfsq + R) - hfsq; + } + return s * (R - f); +} + +double APS5_VABI log10_nid_postfix(double x) { + const double two54 = 1.80143985094819840000e+16; + const double ivln10hi = 4.34294481878168880939e-01; + const double ivln10lo = 2.50829467116452752298e-11; + const double log10_2hi = 3.01029995663611771306e-01; + const double log10_2lo = 3.69423907715893078616e-13; + static volatile double zero = 0.0; + std::uint64_t bits = std::bit_cast(x); + std::int32_t hx = static_cast(bits >> 32); + const std::uint32_t lx = static_cast(bits); + std::int32_t k = 0; + if (hx < 0x00100000) { + if (((hx & 0x7fffffff) | lx) == 0) { + return -two54 / zero; + } + if (hx < 0) { + errno = EDOM; + return (x - x) / zero; + } + k -= 54; + x *= two54; + bits = std::bit_cast(x); + hx = static_cast(bits >> 32); + } + if (hx >= 0x7ff00000) { + return x + x; + } + k += (hx >> 20) - 1023; + hx &= 0x000fffff; + const std::int32_t i = (hx + 0x95f64) & 0x100000; + x = std::bit_cast((static_cast(static_cast(hx | (i ^ 0x3ff00000))) << 32) | (bits & 0xffffffffu)); + k += i >> 20; + const double y = k; + const double f = Log10Kernel(x); + x = x - 1; + const double hi = std::bit_cast(std::bit_cast(x) & 0xffffffff00000000ull); + const double lo = x - hi; + const double z = y * log10_2lo + (x + f) * ivln10lo + (lo + f) * ivln10hi + hi * ivln10hi; + return z + y * log10_2hi; +} float APS5_VABI ldexpf_nid_postfix(float x, int exp) { return std::ldexp(x, exp); } double APS5_VABI fmod_nid_postfix(double x, double y) { int saved_errno = errno; diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index 80f1217e86..233023f0d9 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -1038,6 +1038,32 @@ static void CheckExpm1ScalingRounding() { } _mm_setcsr(savedCsr); } +static void CheckLog10ConsoleBits() { + constexpr struct { + std::uint64_t input; + std::uint64_t expected; + int error; + } cases[] = { + {0x0000000000000000ull, 0xfff0000000000000ull, 0}, {0x8000000000000000ull, 0xfff0000000000000ull, 0}, + {0x0000000000000001ull, 0xc07434e6420f4374ull, 0}, {0x8000000000000001ull, 0xfff8000000000000ull, EDOM}, + {0x000fffffffffffffull, 0xc0733a7146f72a42ull, 0}, {0x0010000000000000ull, 0xc0733a7146f72a42ull, 0}, + {0x3ff0000000000000ull, 0x0000000000000000ull, 0}, {0x3fefffffffffffffull, 0xbc8bcb7b1526e50full, 0}, + {0x3ff0000000000001ull, 0x3c9bcb7b1526e50dull, 0}, {0x3ff00000fffffffeull, 0x3e9bcb7a36937eb6ull, 0}, + {0x3ff6a09e00000000ull, 0x3fc3441254d07a4eull, 0}, {0x4024000000000000ull, 0x3ff0000000000000ull, 0}, + {0x408f400000000000ull, 0x4008000000000000ull, 0}, {0x3fb999999999999aull, 0xbff0000000000000ull, 0}, + {0x7fefffffffffffffull, 0x40734413509f79ffull, 0}, {0x7ff0000000000000ull, 0x7ff0000000000000ull, 0}, + {0xbff0000000000000ull, 0xfff8000000000000ull, EDOM}, {0xfff0000000000000ull, 0xfff8000000000000ull, EDOM}, + {0x7ff8000000000001ull, 0x7ff8000000000001ull, 0}, {0xfff8000000000002ull, 0xfff8000000000002ull, EDOM}, + {0x4003a67d5567227bull, 0x3fd8fa5a3b281494ull, 0}, {0x4047f462f020acdeull, 0x3ffae2ff9b87b792ull, 0}, + {0x3ff9e926ec917ab6ull, 0x3fcacc4f022be644ull, 0}, {0x40bb970e057a80b2ull, 0x400ecabc9e60637cull, 0}, + {0x4052444de3537dcfull, 0x3ffdd1cefbc9b58cull, 0}, {0x401d69da29dc587bull, 0x3febba41e2e308e6ull, 0}, + }; + for (const auto& item : cases) { + errno = 1234; + Require(std::bit_cast(log10_nid_postfix(std::bit_cast(item.input))) == item.expected); + Require(errno == (item.error != 0 ? item.error : 1234)); + } +} static void CheckAsinl() { const struct { std::uint64_t inputMantissa; std::uint16_t inputSignExponent; std::uint64_t outputMantissa; std::uint16_t outputSignExponent; } cases[] = { @@ -2369,6 +2395,7 @@ int main() { CheckNextafterEnvironment(nextafterf_nid_postfix, 0x80000000u, 0x3f800000u, 0x00800000u, 0x7f7fffffu, 0x7f800000u); CheckNextafterEnvironment(nextafter_nid_postfix, std::uint64_t{0x8000000000000000ull}, std::uint64_t{0x3ff0000000000000ull}, std::uint64_t{0x0010000000000000ull}, std::uint64_t{0x7fefffffffffffffull}, std::uint64_t{0x7ff0000000000000ull}); CheckExpm1ScalingRounding(); + CheckLog10ConsoleBits(); CheckAsinl(); Require(scalbn_nid_postfix(0.75, -2) == 0.1875); Require(scalbnf_nid_postfix(0.75f, 4) == 12.f); diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 568e1931c1..4aaddebe48 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -142,6 +142,7 @@ - [log1p](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions, and its double inputs are checked by sampling, not exhaustively. It is FreeBSD msun's [`s_log1p.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_log1p.c). As the titles' `libc.prx` does, it sets `EDOM` at msun's NaN return (arguments below -1, `-inf` and negative NaNs) and leaves `errno` alone for `log1p(-1)`, where it returns `-inf`. It gives the same bits, NaN payloads included, and `errno` as the `log1p` bundled with PPSA07923 and PPSA05916 on 320 million random and 68,216 structured arguments on a Windows host. The FMA-contracted build bundled with PPSA13934 differs on 73,682 of the random ones. - [nextafter and nextafterf](../../../core/libs/prx/libc/src/Math.cpp) (libc) have no physical-console or unmasked-exception validation. A Linux CPU probe compares result bits, `errno` and masked floating-point flags with the unchanged math code in FAR: Changing Tides PPSA03766 bundled `libc.prx` (`nextafter` 0x51b30, `nextafterf` 0x51c80, `_Feraise` 0x11da0): 16,864 calls across both precisions, four rounding modes and all DAZ/FTZ combinations, not exhaustive inputs. Earlier Windows sampling of eleven bundled libraries compares only bits and `errno`; exception behavior outside the FAR module remains unverified. - [tanh](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions, and its double inputs are checked by sampling, not exhaustively. It is FreeBSD msun's [`s_tanh.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_tanh.c) over this file's msun `Expm1`, with `tiny` read at run time; it sets no `errno`, as the titles' `libc.prx` does not. It gives the same bits as the `tanh` bundled with PPSA07923 and PPSA05916 on 16 million random and 68,211 structured arguments on a Windows host; the FMA-contracted build bundled with PPSA13934 differs on 1,054 of them. +- [log10](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions, and its double inputs are checked by sampling, not exhaustively. It is the fdlibm [`e_log10.c`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_log10.c) over its [`k_log.h`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/k_log.h), which FreeBSD 11.4 later replaced. As the titles' `libc.prx` does, it sets `EDOM` for every nonzero argument with the sign bit set (negatives, `-inf` and negative NaNs) and leaves `errno` alone for ±0, where it returns `-inf`, unlike `log`. It gives the same bits, NaN payloads included, and `errno` as the `log10` bundled with PPSA07923 and PPSA05916 on 320 million random and 31,165 structured arguments on a Windows host. The FMA-contracted build bundled with PPSA13934 differs on 229,944 of the random ones. - [sinhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `sinhf` of the `libc.prx` bundled with PPSA28319 (disassembly): FreeBSD msun's [`e_sinhf.c`](https://github.com/freebsd/freebsd-src/blob/5aa554c7e5b6fd9042986df07f12ac5687e41392/lib/msun/src/e_sinhf.c) from before `__ldexp_expf`, which squares `expf(|x| / 2)` between `logf(FLT_MAX)` and the overflow threshold, calling msun's [`expm1f`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/s_expm1f.c) and [`expf`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_expf.c), whose paths for the arguments it passes are inlined here. Past the threshold Sony's error helper sets `errno` to `ERANGE` before returning `x * 1e37`; msun itself sets no `errno`. That code, run on a Windows host, gives the same bits as this implementation for every finite argument up to the threshold, of both signs; it was not run on Linux. That the system `libSceLibcInternal`, which Ruffle (PPSA68091) and RetroArch (PPSA99169) import while bundling a zero-return `libc.prx`, has the same code is assumed - [acoshf](../../../core/libs/prx/libc/src/Math.cpp) (libc) exact system `libSceLibcInternal` behavior and FMA-contracted copies remain unverified. Positive finite inputs use [msun](https://github.com/freebsd/freebsd-src/blob/5aa554c7e5b6fd9042986df07f12ac5687e41392/lib/msun/src/e_acoshf.c) range handling and this file's `logf`; the middle range divides -1 before addition under directed rounding. Bits, errno and SSE/x87 status and controls match the supplied PPSA03766/PPSA07923 copies on 236,592 positive finite calls (4,929 inputs, four rounding modes, DAZ/FTZ combinations and sticky flags); these samples are not exhaustive. Invalid-domain errno and SSE/x87 invalid flags are checked against two supplied bundled copies; those samples are not exhaustive either. - [acoshl, asinhl, atanhl, atan2l, cbrtl, erfl, erfcl, expm1l, hypotl, log1pl, log2l, lgammal, tgammal, j0f, j1f, jnf, y0f, y1f and ynf](../../../core/libs/prx/libc/src/Math.cpp) (libc) return the host C library's results (the float Bessel functions through its double ones), not those of the console's own code, so the last bits can differ (`y0f`, `y1f`, `ynf` follow the console at `+-0`). `tgammaf` delegates to the host's double `tgamma` as FreeBSD 11 does ([`s_tgammaf.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/msun/src/s_tgammaf.c)), then restores the sign of a zero result for negative non-integer inputs from the parity of `floor(-x)`; nine title-bundled `libc.prx` builds return that sign for underflowing inputs, where Windows UCRT can return the opposite sign ([#9707](https://github.com/boykopovar/AnyPS5/issues/9707)). As in the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x56410), `lgammal` does not store the sign in `signgam`; `hypotl` returns infinity for an infinite argument even when the other is a NaN, as C11 F.10.4.3 requires. [hypot3, hypot3f and hypot3l](../../../core/libs/prx/libc/src/Math.cpp) follow that module (0x51D30, 0x520D0, 0x52420): the first NaN of x, y and z is returned unchanged, any infinity gives positive infinity, a zero argument gives the two-argument hypot of the other two, and otherwise the largest magnitude is combined with the hypot of the other two From fab2ee750ec465a564617e10d7a479edf1bdfc81 Mon Sep 17 00:00:00 2001 From: JJenkx Date: Fri, 23 Oct 2026 13:22:29 -0400 Subject: [PATCH 213/318] fix(libc): compute log2 with fdlibm's algorithm, as titles' libc.prx does log2 returned the host's log2, whose last bit differs from the titles' copy on Windows. The titles' copy is the fdlibm e_log2.c over the same k_log.h kernel as log10, which FreeBSD 11.4 later replaced, and it sets EDOM for every nonzero argument with the sign bit set but leaves errno alone for +-0. log2 now computes that over the kernel, renamed KernelLog because both functions use it, and sets EDOM there. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libc/src/Math.cpp | 42 ++++++++++++++++++++++++++++++--- core/libs/tests/GuestMath.cpp | 28 ++++++++++++++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index f26fe897a9..035e795d75 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -614,7 +614,6 @@ long double APS5_VABI frexpl_nid_postfix(long double x, int* exponent) { double APS5_VABI exp_nid_postfix(double x) { return std::exp(x); } double APS5_VABI atan_nid_postfix(double x) { return std::atan(x); } double APS5_VABI tan_nid_postfix(double x) { return std::tan(x); } -double APS5_VABI log2_nid_postfix(double x) { return std::log2(x); } double APS5_VABI log_nid_postfix(double x) { return std::log(x); } double APS5_VABI log1p_nid_postfix(double x) { const double ln2_hi = 6.93147180369123816490e-01; @@ -1615,7 +1614,7 @@ float APS5_VABI logf_nid_postfix(float x) { return k == 0 ? f - s * (f - R) : dk * ln2_hi - ((s * (f - R) - dk * ln2_lo) - f); } float APS5_VABI log2f_nid_postfix(float x) { return std::log2(x); } -static double Log10Kernel(double x) { +static double KernelLog(double x) { const double Lg1 = 6.666666666666735130e-01; const double Lg2 = 3.999999999940941908e-01; const double Lg3 = 2.857142874366239149e-01; @@ -1681,13 +1680,50 @@ double APS5_VABI log10_nid_postfix(double x) { x = std::bit_cast((static_cast(static_cast(hx | (i ^ 0x3ff00000))) << 32) | (bits & 0xffffffffu)); k += i >> 20; const double y = k; - const double f = Log10Kernel(x); + const double f = KernelLog(x); x = x - 1; const double hi = std::bit_cast(std::bit_cast(x) & 0xffffffff00000000ull); const double lo = x - hi; const double z = y * log10_2lo + (x + f) * ivln10lo + (lo + f) * ivln10hi + hi * ivln10hi; return z + y * log10_2hi; } + +double APS5_VABI log2_nid_postfix(double x) { + const double two54 = 1.80143985094819840000e+16; + const double ivln2hi = 1.44269504072144627571e+00; + const double ivln2lo = 1.67517131648865118353e-10; + static volatile double zero = 0.0; + std::uint64_t bits = std::bit_cast(x); + std::int32_t hx = static_cast(bits >> 32); + const std::uint32_t lx = static_cast(bits); + std::int32_t k = 0; + if (hx < 0x00100000) { + if (((hx & 0x7fffffff) | lx) == 0) { + return -two54 / zero; + } + if (hx < 0) { + errno = EDOM; + return (x - x) / zero; + } + k -= 54; + x *= two54; + bits = std::bit_cast(x); + hx = static_cast(bits >> 32); + } + if (hx >= 0x7ff00000) { + return x + x; + } + k += (hx >> 20) - 1023; + hx &= 0x000fffff; + const std::int32_t i = (hx + 0x95f64) & 0x100000; + x = std::bit_cast((static_cast(static_cast(hx | (i ^ 0x3ff00000))) << 32) | (bits & 0xffffffffu)); + k += i >> 20; + const double f = KernelLog(x); + x = x - 1; + const double hi = std::bit_cast(std::bit_cast(x) & 0xffffffff00000000ull); + const double lo = x - hi; + return (x + f) * ivln2lo + (lo + f) * ivln2hi + hi * ivln2hi + k; +} float APS5_VABI ldexpf_nid_postfix(float x, int exp) { return std::ldexp(x, exp); } double APS5_VABI fmod_nid_postfix(double x, double y) { int saved_errno = errno; diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index 233023f0d9..e2945747fe 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -1065,6 +1065,33 @@ static void CheckLog10ConsoleBits() { } } +static void CheckLog2ConsoleBits() { + constexpr struct { + std::uint64_t input; + std::uint64_t expected; + int error; + } cases[] = { + {0x0000000000000000ull, 0xfff0000000000000ull, 0}, {0x8000000000000000ull, 0xfff0000000000000ull, 0}, + {0x0000000000000001ull, 0xc090c80000000000ull, 0}, {0x8000000000000001ull, 0xfff8000000000000ull, EDOM}, + {0x000fffffffffffffull, 0xc08ff00000000000ull, 0}, {0x0010000000000000ull, 0xc08ff00000000000ull, 0}, + {0x3ff0000000000000ull, 0x0000000000000000ull, 0}, {0x3fefffffffffffffull, 0xbca71547652b82feull, 0}, + {0x3ff0000000000001ull, 0x3cb71547652b82fdull, 0}, {0x3ff00000fffffffeull, 0x3eb71546ac5324fbull, 0}, + {0x3ff6a09e00000000ull, 0x3fdffffe5dc146d0ull, 0}, {0x4024000000000000ull, 0x400a934f0979a371ull, 0}, + {0x4020000000000000ull, 0x4008000000000000ull, 0}, {0x3fd0000000000000ull, 0xc000000000000000ull, 0}, + {0x7fefffffffffffffull, 0x4090000000000000ull, 0}, {0x7ff0000000000000ull, 0x7ff0000000000000ull, 0}, + {0xbff0000000000000ull, 0xfff8000000000000ull, EDOM}, {0xfff0000000000000ull, 0xfff8000000000000ull, EDOM}, + {0x7ff8000000000001ull, 0x7ff8000000000001ull, 0}, {0xfff8000000000002ull, 0xfff8000000000002ull, EDOM}, + {0x3f3738a98e70dc5cull, 0xc026ecdca3e646fcull, 0}, {0x3ff6e68ed8bdd52full, 0x3fe08dd74b2b1990ull, 0}, + {0x3f2b147b0942849aull, 0xc0287b4fe8effe10ull, 0}, {0x3fe318aee27b543aull, 0xbfe7d5278168f30eull, 0}, + {0x3fbdbeed4cb253bcull, 0xc008d7d45ccba59eull, 0}, {0x3fece143fc9534f6ull, 0xbfc2f1cb5ae7bbf1ull, 0}, + }; + for (const auto& item : cases) { + errno = 1234; + Require(std::bit_cast(log2_nid_postfix(std::bit_cast(item.input))) == item.expected); + Require(errno == (item.error != 0 ? item.error : 1234)); + } +} + static void CheckAsinl() { const struct { std::uint64_t inputMantissa; std::uint16_t inputSignExponent; std::uint64_t outputMantissa; std::uint16_t outputSignExponent; } cases[] = { {0, 0x0000, 0, 0x0000}, {0, 0x8000, 0, 0x8000}, @@ -2396,6 +2423,7 @@ int main() { CheckNextafterEnvironment(nextafter_nid_postfix, std::uint64_t{0x8000000000000000ull}, std::uint64_t{0x3ff0000000000000ull}, std::uint64_t{0x0010000000000000ull}, std::uint64_t{0x7fefffffffffffffull}, std::uint64_t{0x7ff0000000000000ull}); CheckExpm1ScalingRounding(); CheckLog10ConsoleBits(); + CheckLog2ConsoleBits(); CheckAsinl(); Require(scalbn_nid_postfix(0.75, -2) == 0.1875); Require(scalbnf_nid_postfix(0.75f, 4) == 12.f); diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 4aaddebe48..1cafe51bd8 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -143,6 +143,7 @@ - [nextafter and nextafterf](../../../core/libs/prx/libc/src/Math.cpp) (libc) have no physical-console or unmasked-exception validation. A Linux CPU probe compares result bits, `errno` and masked floating-point flags with the unchanged math code in FAR: Changing Tides PPSA03766 bundled `libc.prx` (`nextafter` 0x51b30, `nextafterf` 0x51c80, `_Feraise` 0x11da0): 16,864 calls across both precisions, four rounding modes and all DAZ/FTZ combinations, not exhaustive inputs. Earlier Windows sampling of eleven bundled libraries compares only bits and `errno`; exception behavior outside the FAR module remains unverified. - [tanh](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions, and its double inputs are checked by sampling, not exhaustively. It is FreeBSD msun's [`s_tanh.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_tanh.c) over this file's msun `Expm1`, with `tiny` read at run time; it sets no `errno`, as the titles' `libc.prx` does not. It gives the same bits as the `tanh` bundled with PPSA07923 and PPSA05916 on 16 million random and 68,211 structured arguments on a Windows host; the FMA-contracted build bundled with PPSA13934 differs on 1,054 of them. - [log10](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions, and its double inputs are checked by sampling, not exhaustively. It is the fdlibm [`e_log10.c`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_log10.c) over its [`k_log.h`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/k_log.h), which FreeBSD 11.4 later replaced. As the titles' `libc.prx` does, it sets `EDOM` for every nonzero argument with the sign bit set (negatives, `-inf` and negative NaNs) and leaves `errno` alone for ±0, where it returns `-inf`, unlike `log`. It gives the same bits, NaN payloads included, and `errno` as the `log10` bundled with PPSA07923 and PPSA05916 on 320 million random and 31,165 structured arguments on a Windows host. The FMA-contracted build bundled with PPSA13934 differs on 229,944 of the random ones. +- [log2](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions, and its double inputs are checked by sampling, not exhaustively. It is the fdlibm [`e_log2.c`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_log2.c) over the same `k_log.h` kernel as `log10` (`KernelLog`), which FreeBSD 11.4 later replaced. As the titles' `libc.prx` does, it sets `EDOM` for every nonzero argument with the sign bit set and leaves `errno` alone for ±0. It gives the same bits, NaN payloads included, and `errno` as the `log2` bundled with PPSA07923 and PPSA05916 on 16 million random and 31,165 structured arguments on a Windows host; the FMA-contracted build bundled with PPSA13934 differs on 727 of the random ones. - [sinhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the `sinhf` of the `libc.prx` bundled with PPSA28319 (disassembly): FreeBSD msun's [`e_sinhf.c`](https://github.com/freebsd/freebsd-src/blob/5aa554c7e5b6fd9042986df07f12ac5687e41392/lib/msun/src/e_sinhf.c) from before `__ldexp_expf`, which squares `expf(|x| / 2)` between `logf(FLT_MAX)` and the overflow threshold, calling msun's [`expm1f`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/s_expm1f.c) and [`expf`](https://github.com/freebsd/freebsd-src/blob/59bc2f095289a10b8cfa55cbb2b45c3bd6c2d4ae/lib/msun/src/e_expf.c), whose paths for the arguments it passes are inlined here. Past the threshold Sony's error helper sets `errno` to `ERANGE` before returning `x * 1e37`; msun itself sets no `errno`. That code, run on a Windows host, gives the same bits as this implementation for every finite argument up to the threshold, of both signs; it was not run on Linux. That the system `libSceLibcInternal`, which Ruffle (PPSA68091) and RetroArch (PPSA99169) import while bundling a zero-return `libc.prx`, has the same code is assumed - [acoshf](../../../core/libs/prx/libc/src/Math.cpp) (libc) exact system `libSceLibcInternal` behavior and FMA-contracted copies remain unverified. Positive finite inputs use [msun](https://github.com/freebsd/freebsd-src/blob/5aa554c7e5b6fd9042986df07f12ac5687e41392/lib/msun/src/e_acoshf.c) range handling and this file's `logf`; the middle range divides -1 before addition under directed rounding. Bits, errno and SSE/x87 status and controls match the supplied PPSA03766/PPSA07923 copies on 236,592 positive finite calls (4,929 inputs, four rounding modes, DAZ/FTZ combinations and sticky flags); these samples are not exhaustive. Invalid-domain errno and SSE/x87 invalid flags are checked against two supplied bundled copies; those samples are not exhaustive either. - [acoshl, asinhl, atanhl, atan2l, cbrtl, erfl, erfcl, expm1l, hypotl, log1pl, log2l, lgammal, tgammal, j0f, j1f, jnf, y0f, y1f and ynf](../../../core/libs/prx/libc/src/Math.cpp) (libc) return the host C library's results (the float Bessel functions through its double ones), not those of the console's own code, so the last bits can differ (`y0f`, `y1f`, `ynf` follow the console at `+-0`). `tgammaf` delegates to the host's double `tgamma` as FreeBSD 11 does ([`s_tgammaf.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/msun/src/s_tgammaf.c)), then restores the sign of a zero result for negative non-integer inputs from the parity of `floor(-x)`; nine title-bundled `libc.prx` builds return that sign for underflowing inputs, where Windows UCRT can return the opposite sign ([#9707](https://github.com/boykopovar/AnyPS5/issues/9707)). As in the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x56410), `lgammal` does not store the sign in `signgam`; `hypotl` returns infinity for an infinite argument even when the other is a NaN, as C11 F.10.4.3 requires. [hypot3, hypot3f and hypot3l](../../../core/libs/prx/libc/src/Math.cpp) follow that module (0x51D30, 0x520D0, 0x52420): the first NaN of x, y and z is returned unchanged, any infinity gives positive infinity, a zero argument gives the two-argument hypot of the other two, and otherwise the largest magnitude is combined with the hypot of the other two From 2cf7d26d3c733921cd1757f607cc61a97769143c Mon Sep 17 00:00:00 2001 From: JJenkx Date: Wed, 28 Oct 2026 06:03:13 -0400 Subject: [PATCH 214/318] fix(libc): raise the titles' flags from log2 and log10 at their special inputs Titles' libc.prx raises FE_INVALID wherever log2 and log10 stores EDOM, a NaN argument included, where (x - x) / zero raised nothing for a NaN operand. The domain path now raises through _Feraise, which stores EDOM and raises the flag in both units. For +-0 it now returns -inf without raising FE_DIVBYZERO, as the titles do; the port divided -two54 by zero. Results and errno are unchanged. CheckLog2EdgeFlags and CheckLog10EdgeFlags cover it. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libc/src/Math.cpp | 8 ++++---- core/libs/tests/GuestMath.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index 035e795d75..e7716827bc 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -1660,10 +1660,10 @@ double APS5_VABI log10_nid_postfix(double x) { std::int32_t k = 0; if (hx < 0x00100000) { if (((hx & 0x7fffffff) | lx) == 0) { - return -two54 / zero; + return -std::numeric_limits::infinity(); } if (hx < 0) { - errno = EDOM; + _Feraise_nid_postfix(0x01); return (x - x) / zero; } k -= 54; @@ -1699,10 +1699,10 @@ double APS5_VABI log2_nid_postfix(double x) { std::int32_t k = 0; if (hx < 0x00100000) { if (((hx & 0x7fffffff) | lx) == 0) { - return -two54 / zero; + return -std::numeric_limits::infinity(); } if (hx < 0) { - errno = EDOM; + _Feraise_nid_postfix(0x01); return (x - x) / zero; } k -= 54; diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index e2945747fe..39bf9308e6 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -1038,6 +1038,18 @@ static void CheckExpm1ScalingRounding() { } _mm_setcsr(savedCsr); } +static void CheckLog10EdgeFlags() { + for (const double input : {-1.0, -std::numeric_limits::infinity(), -std::numeric_limits::quiet_NaN()}) { + errno = 0; + std::feclearexcept(FE_ALL_EXCEPT); + Require(std::isnan(log10_nid_postfix(input)) && errno == EDOM && std::fetestexcept(FE_INVALID) != 0); + } + for (const double input : {0.0, -0.0}) { + errno = 1234; + std::feclearexcept(FE_ALL_EXCEPT); + Require(log10_nid_postfix(input) == -std::numeric_limits::infinity() && errno == 1234 && std::fetestexcept(FE_DIVBYZERO) == 0); + } +} static void CheckLog10ConsoleBits() { constexpr struct { std::uint64_t input; @@ -1065,6 +1077,18 @@ static void CheckLog10ConsoleBits() { } } +static void CheckLog2EdgeFlags() { + for (const double input : {-1.0, -std::numeric_limits::infinity(), -std::numeric_limits::quiet_NaN()}) { + errno = 0; + std::feclearexcept(FE_ALL_EXCEPT); + Require(std::isnan(log2_nid_postfix(input)) && errno == EDOM && std::fetestexcept(FE_INVALID) != 0); + } + for (const double input : {0.0, -0.0}) { + errno = 1234; + std::feclearexcept(FE_ALL_EXCEPT); + Require(log2_nid_postfix(input) == -std::numeric_limits::infinity() && errno == 1234 && std::fetestexcept(FE_DIVBYZERO) == 0); + } +} static void CheckLog2ConsoleBits() { constexpr struct { std::uint64_t input; @@ -2423,7 +2447,9 @@ int main() { CheckNextafterEnvironment(nextafter_nid_postfix, std::uint64_t{0x8000000000000000ull}, std::uint64_t{0x3ff0000000000000ull}, std::uint64_t{0x0010000000000000ull}, std::uint64_t{0x7fefffffffffffffull}, std::uint64_t{0x7ff0000000000000ull}); CheckExpm1ScalingRounding(); CheckLog10ConsoleBits(); + CheckLog10EdgeFlags(); CheckLog2ConsoleBits(); + CheckLog2EdgeFlags(); CheckAsinl(); Require(scalbn_nid_postfix(0.75, -2) == 0.1875); Require(scalbnf_nid_postfix(0.75f, 4) == 12.f); From a8661988e4ba2ea2862414f6e1a44410eaae8982 Mon Sep 17 00:00:00 2001 From: p3nicillin Date: Sun, 25 Oct 2026 22:02:39 +0000 Subject: [PATCH 215/318] fix(libSceVideoOut): return SLOT_OCCUPIED for a registration over registered buffers The console answers sceVideoOutRegisterBuffers2 with SLOT_OCCUPIED (0x80290010) when the buffer slots are still registered: EVO Player measured it re-registering set 0 while that set stayed on screen. The call threw INVALID_INDEX instead. The PS4 call and an occupied set over free slots keep throwing. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceVideoOut/src/Buffer.cpp | 15 ++++++++------- core/libs/prx/libSceVideoOut/tests/Flip.cpp | 7 +++++++ docs/dev/debt/libSceVideoOut.md | 1 + 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceVideoOut/src/Buffer.cpp b/core/libs/prx/libSceVideoOut/src/Buffer.cpp index 75c4e8a067..533f91af37 100644 --- a/core/libs/prx/libSceVideoOut/src/Buffer.cpp +++ b/core/libs/prx/libSceVideoOut/src/Buffer.cpp @@ -39,6 +39,12 @@ int CommitBufferGroup(const std::string& function, VideoOutConfig& cfg, int setI if (cfg.closing) { throw std::runtime_error(function + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); } + for (int i = 0; i < bufferNum; i++) { + if (cfg.buffers[bufferIndexStart + i].Occupied()) { + if (!group.ps4) return VIDEO_OUT_ERROR_SLOT_OCCUPIED; + throw std::runtime_error(function + ": VIDEO_OUT_ERROR_SLOT_OCCUPIED"); + } + } if (setIndex < 0) { for (int index = 0; index < VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX && setIndex < 0; ++index) if (!cfg.groups[index].occupied) setIndex = index; @@ -48,11 +54,6 @@ int CommitBufferGroup(const std::string& function, VideoOutConfig& cfg, int setI } else if (cfg.groups[setIndex].occupied) { throw std::runtime_error(function + ": VIDEO_OUT_ERROR_INVALID_INDEX"); } - for (int i = 0; i < bufferNum; i++) { - if (cfg.buffers[bufferIndexStart + i].Occupied()) { - throw std::runtime_error(function + ": VIDEO_OUT_ERROR_SLOT_OCCUPIED"); - } - } std::array registered{}; for (int i = 0; i < bufferNum; ++i) { registered[i] = {setIndex, reinterpret_cast(buffers[i].data), reinterpret_cast(buffers[i].metadata)}; @@ -154,8 +155,8 @@ int APS5_VABI sceVideoOutRegisterBuffers2(int handle, int setIndex, int bufferIn group.attribute = *attribute; group.category = category; group.occupied = true; - CommitBufferGroup(__func__, *cfg, setIndex, bufferIndexStart, buffers, bufferNum, group); - return 0; + const int committed = CommitBufferGroup(__func__, *cfg, setIndex, bufferIndexStart, buffers, bufferNum, group); + return committed < 0 ? committed : 0; } catch (const ProcessShutdown&) { LibcAwaitExit_nid_postfix(); } diff --git a/core/libs/prx/libSceVideoOut/tests/Flip.cpp b/core/libs/prx/libSceVideoOut/tests/Flip.cpp index ca0ae05f8a..a5cbfa8a76 100644 --- a/core/libs/prx/libSceVideoOut/tests/Flip.cpp +++ b/core/libs/prx/libSceVideoOut/tests/Flip.cpp @@ -507,6 +507,13 @@ void testControls() { attribute.height = 64; attribute.pixel_format = 0x8000000000000000ull; check(sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr) == 0, "buffers with set reserved pointers were rejected"); + constexpr int slotOccupied = static_cast(0x80290010u); + VideoOutBuffers again{storage.data(), nullptr, {nullptr, nullptr}}; + check(sceVideoOutRegisterBuffers2(handle, 0, 0, &again, 1, &attribute, 0, nullptr) == slotOccupied, "a registration over a registered set and its buffers did not fail with SLOT_OCCUPIED"); + { + std::lock_guard lock(cfg->mutex); + check(cfg->buffers[0].groupIndex == 0 && cfg->groups[0].occupied, "a refused registration released the registered set"); + } check(sceVideoOutUnregisterBuffers(handle, 0) == 0, "buffer unregistration failed"); check(owner->GetTriggeredEvents(&event, 1) == 0, "VRR status event triggered without a VRR change"); KernelEvent vrrStatus{}; diff --git a/docs/dev/debt/libSceVideoOut.md b/docs/dev/debt/libSceVideoOut.md index d57e8243d1..c63cbef2d5 100644 --- a/docs/dev/debt/libSceVideoOut.md +++ b/docs/dev/debt/libSceVideoOut.md @@ -14,6 +14,7 @@ - [sceVideoOutDeleteVrrActiveStatusEvent](../../../core/libs/prx/libSceVideoOut/src/Event.cpp) (libSceVideoOut, NID `-q-vbO859Tw`) - the name hashes to the NID PPSA02530 imports and is listed in [sce_symbols](https://github.com/zecoxao/sce_symbols) `aerolib.csv` at `fe2cc2948e`. The signature `(eq, handle)` and the behaviour, removing the handle's VRR status registration from the queue as the other `Delete*Event` calls do, follow sceVideoOutAddVrrActiveStatusEvent; no title has been observed calling it - [sceVideoOutCursorEnable, sceVideoOutCursorDisable, sceVideoOutCursorSetImageAddress, sceVideoOutCursorSetPosition](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) - no cursor is shown: the image format and size are unknown, so a non-null image address and a cursor index other than 0 report not implemented, and `CursorSetPosition` returns 0 for every position but only (100, 100) was measured. Measured in a payload process on a PS5 (firmware 12.70) on a handle from `sceVideoOutOpen(0xff, 0, 0, NULL)` with no buffer registered: `CursorEnable` and `CursorSetImageAddress` with (handle, 0, NULL) returned 0x80290002, `CursorSetPosition(handle, 0, 100, 100)` and `CursorDisable(handle, 0)` returned 0, and all four returned 0x8029000b for handle -1. A closed handle is unmeasured and answered like -1. Parameters follow Prospero Win's [declarations](https://github.com/mpereiraesaa/prospero-win/blob/644e32e400c555e340d507383398c5d21b39b0b0/wine/ps5/pw_vulkan_radv.c#L65-L68). - [sceVideoOutGetOutputStatusWithVrr](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut, NID `gWT7X8H0bYs`) - bytes 48-53 are written as zero, where a PS5 (firmware 13.40, measured 2026-10-19 inside a retail title) with a 4K TV that takes 120 Hz returned the 16-bit values 48, 60 and 60 (0x30, 0x3C, 0x3C); their meaning is not established, so the value at byte 52 never adjusts `refreshRate`. `(int handle, void* status)`, the 80-byte layout and the return codes follow the clean-room description in #2746: bytes 0-47 are `sceVideoOutGetOutputStatus`'s and the rest 0, a null `status` returns `0x80290002` and an unknown handle `0x8029000B`, all as that console answered. The console's earlier global-flag check (`0x8029001C`) and its object-field check (`0x8029001B`) test states that are not modelled and are never reported. +- [sceVideoOutRegisterBuffers2 over registered buffers](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - a call whose buffer slots are already registered returns `SLOT_OCCUPIED` (0x80290010) and changes nothing. EVO Player (PPSA99039) [measured](https://github.com/sainsaji/EVO-PLAYER-PS5/blob/2db44b5294657b12bf91666154a05f029ddb5aa2/projects/evoplayer/media/src/evo_agc_runtime.c#L2830-L2838) that answer for set 0 and buffers 0 and 1 while that set was still registered, with the [call](https://github.com/sainsaji/EVO-PLAYER-PS5/blob/2db44b5294657b12bf91666154a05f029ddb5aa2/projects/evoplayer/media/src/evo_agc_runtime.c#L2968-L2982) it makes. Not measured, and still thrown: an occupied set over free buffer slots (`INVALID_INDEX`), and occupied slots in `sceVideoOutRegisterBuffers`, the PS4 call. - [sceVideoOutRegisterBuffers2](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - buffer option 8 (STRICT_COLORIMETRY) is presented as option 0, with the stored values unchanged; what the console changes for it is unknown. Only options 0 and 8 are accepted - [sceVideoOutAddBufferAttributeOption](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut, NID `MBlv8YWoyOk`) - the signature `(VideoOutBufferAttribute2*, uint64_t option)` comes from [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/60977b29103452b9681ae10c7a9011d44965bf16/src/SharpProspero/Interop/VideoOut/VideoOut.cs#L174-L176) and the call site in PPSA25380, which passes 0x20 and ignores the result. The option is ORed into `attribute->option`; whether the console validates the bits, keeps them elsewhere or fails on an unknown option is unknown. A null attribute throws. Option 0x20 makes `sceVideoOutRegisterBuffers2` throw, as every option other than 0 and 8; what it selects for an overlay buffer is unknown - [sceVideoOutRegisterBuffers2](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - DCC buffers (category 1): `dcc_control` is taken as the CB_DCC_CONTROL block layout, only bits 0x10026c are accepted; `dcc_cb_register_clear_color` as the register-clear texel, only 32-bit values are presented From f3c29634f8a420b417ae7e5fc073a16331c1e552 Mon Sep 17 00:00:00 2001 From: p3nicillin Date: Mon, 26 Oct 2026 01:00:49 +0000 Subject: [PATCH 216/318] fix(libSceVideoOut): keep the PS4 registration checks in their order Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceVideoOut/src/Buffer.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceVideoOut/src/Buffer.cpp b/core/libs/prx/libSceVideoOut/src/Buffer.cpp index 533f91af37..1b1db00c62 100644 --- a/core/libs/prx/libSceVideoOut/src/Buffer.cpp +++ b/core/libs/prx/libSceVideoOut/src/Buffer.cpp @@ -39,10 +39,9 @@ int CommitBufferGroup(const std::string& function, VideoOutConfig& cfg, int setI if (cfg.closing) { throw std::runtime_error(function + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); } - for (int i = 0; i < bufferNum; i++) { - if (cfg.buffers[bufferIndexStart + i].Occupied()) { - if (!group.ps4) return VIDEO_OUT_ERROR_SLOT_OCCUPIED; - throw std::runtime_error(function + ": VIDEO_OUT_ERROR_SLOT_OCCUPIED"); + if (!group.ps4) { + for (int i = 0; i < bufferNum; i++) { + if (cfg.buffers[bufferIndexStart + i].Occupied()) return VIDEO_OUT_ERROR_SLOT_OCCUPIED; } } if (setIndex < 0) { @@ -54,6 +53,11 @@ int CommitBufferGroup(const std::string& function, VideoOutConfig& cfg, int setI } else if (cfg.groups[setIndex].occupied) { throw std::runtime_error(function + ": VIDEO_OUT_ERROR_INVALID_INDEX"); } + for (int i = 0; i < bufferNum; i++) { + if (cfg.buffers[bufferIndexStart + i].Occupied()) { + throw std::runtime_error(function + ": VIDEO_OUT_ERROR_SLOT_OCCUPIED"); + } + } std::array registered{}; for (int i = 0; i < bufferNum; ++i) { registered[i] = {setIndex, reinterpret_cast(buffers[i].data), reinterpret_cast(buffers[i].metadata)}; From e2338d5bd105186c95d0218e09045ed26bf5701b Mon Sep 17 00:00:00 2001 From: hamb3r <48532909+hamb3r@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:25:51 +0300 Subject: [PATCH 217/318] fix(agc): keep FaultBuffer-only shaders bounded in DispatchWriteRanges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symptom / Log: - agc_driver_write_data_written_ahead: specialized atomic compute shaders with FaultBuffer but no BdaPagetable were classified as unbounded in DispatchWriteRanges, stopping ahead group resolution. Root Cause: - core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp:DispatchWriteRanges:L56-L61 — FaultBuffer bindings were treated as unbounded pointer writes even when no BdaPagetable binding was present. Change: - Allow FaultBuffer bindings in DispatchWriteRanges without clearing bounded unless Gds or BdaPagetable is present. Metric / Verification: - DispatchWriteRanges returns bounded == true for descriptor-indexed atomic shaders with FaultBuffer and no BdaPagetable. --- .../Execution/src/Driver/Memory/WrittenBuffers.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index 58fa8648a3..7a56d39fa8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -54,11 +54,11 @@ void Driver::noteWrittenBuffers(std::uint64_t program, std::uint32_t queue, cons } std::vector> DispatchWriteRanges(const ShaderRecompiler::RecompileResult& compiled, bool& bounded) { - bounded = compiled.bdaAbiVersion == 0; + bounded = compiled.bdaAbiVersion == 0 || std::any_of(compiled.bindings.begin(), compiled.bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer; }); std::vector> ranges; constexpr std::uint64_t UnknownImageBytes = 64ull * 1024 * 1024; for (const auto& binding : compiled.bindings) { - if (binding.role == ShaderRecompiler::DescriptorRole::Gds || binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer) bounded = false; + if (binding.role == ShaderRecompiler::DescriptorRole::Gds || binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable) bounded = false; if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) continue; for (std::uint32_t element = 0; element < binding.count; ++element) { const bool written = element >= binding.imageWritten.size() || binding.imageWritten[element]; From 054fe844560ed16f7d4414f721469885cc6ec049 Mon Sep 17 00:00:00 2001 From: hamb3r <48532909+hamb3r@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:26:15 +0300 Subject: [PATCH 218/318] fix(agc): record unvalued GPU copies and reopen ahead gate after HLE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symptom / Log: - 26102026_03_45_39.log:L2476-L2492, 26102026_03_53_05.log:L1968-L1985: VUID-vkCmdCopyBufferToImage-pRegions-00171 / VUID-vkCmdDispatch-None-08600 / EXCEPTION_ACCESS_VIOLATION (0xC0000005) on queue 0x20 after HLE copy and fill dispatches. Root Cause: - core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp:dispatch:L125-L154 — HLE buffer fill and copy matchers skipped ahead capture without reopening gateOpened() after execution, while BufferCopy.cpp:noteCopyWriter:L131 recorded GPU-backed copies without known values under the copy program address instead of foreign writer policy. Change: - Match dword-pattern, constant-dword, clamped-global, and memory-value buffer fills during resolvingAhead(), reopen gateOpened() when resolvedAhead() is empty after HLE clear/fill/copy execution, and record GPU-backed copies without known values under writer program 0. Metric / Verification: - Post-HLE dispatches synchronize against pending GPU-backed copies and reopen ahead group resolution once resolvedAhead() drains. --- .../Execution/src/Driver/Dispatch/Dispatch.cpp | 4 ++++ .../Execution/src/Driver/Memory/BufferCopy.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index ac5747f914..ce351faa3d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -139,10 +139,12 @@ void Driver::dispatch(QueueState& queue, std::span packet, if (resolving) { if (indirectArguments == 0 && Graphics::MatchMultisampledSingleClear(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads)) return; if (!partialLastGroup && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return; + if (!partialLastGroup && indirectArguments == 0 && (MatchDwordPatternFill(packet, std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchConstantDwordFill(packet, std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchClampedGlobalFill(packet, std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchMemoryValueBufferFill(packet, std::span(snapshot.code).subspan(codeOffset), userData, compute))) return; if (!partialLastGroup && indirectArguments == 0 && (matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchesIndexedCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute))) return; } else { if (indirectArguments == 0 && Graphics::TakeMultisampledSingleClear(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads)) { pendingDispatchPhases().outcome = DispatchOutcome::FillHle; + if (resolvedAhead().empty()) gateOpened() = true; return; } if (indirectArguments == 0 && (packet[4] & 0x20u) == 0) { @@ -150,6 +152,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } if (!partialLastGroup && fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) { pendingDispatchPhases().outcome = DispatchOutcome::FillHle; + if (resolvedAhead().empty()) gateOpened() = true; return; } if (DrawPipeline::Active() && !pipelineCandidate) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); @@ -161,6 +164,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } if (!partialLastGroup && indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; + if (resolvedAhead().empty()) gateOpened() = true; return; } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferCopy.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferCopy.cpp index b3b86a1427..17235e4e0c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferCopy.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferCopy.cpp @@ -128,7 +128,7 @@ void Driver::noteCopyWriter(std::uint64_t program, std::uint64_t begin, std::uin knownValueEntries.fetch_add(1, std::memory_order_relaxed); } std::lock_guard lock(writtenBuffersMutex); - writtenBuffers.push_back({program, begin, end, ++writtenBufferSerial, queue, false, std::move(known), generation}); + writtenBuffers.push_back({known || !copyKnownValues() ? program : 0, begin, end, ++writtenBufferSerial, queue, false, std::move(known), generation}); while (writtenBuffers.size() > WrittenBufferRing) writtenBuffers.pop_front(); } From 8f97d679e31fbaed67aa60ce2f545401af6cb736 Mon Sep 17 00:00:00 2001 From: hamb3r <48532909+hamb3r@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:26:27 +0300 Subject: [PATCH 219/318] fix(agc): stop resolveGroupAhead on unknown writes and defer cache insert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symptom / Log: - 26102026_03_45_39.log:L2476-L2492: resolveGroupAhead continued capturing dispatches after an unknown write (found == end or unbounded write), and dispatch() inserted ahead captures into global dispatchCache before OverlapsLabel / OverlapsRange rejection. Root Cause: - core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp:resolveGroupAhead:L290-L299 and Dispatch.cpp:dispatch:L399 — unknown writes only set unknownWrites = true without stopping the loop, WritesThroughPointers missed StorageImage and GDS write hazards, and insertDispatch ran before overlap checks. Change: - Stop resolveGroupAhead immediately (++stats.unknownWrites; break;) when a dispatch is not resolved or DispatchWriteRanges reports !bounded, and defer insertDispatch (plus compute pipeline prebuild) until OverlapsLabel and OverlapsRange pass. Metric / Verification: - Rejected ahead captures do not insert into global dispatchCache (inserts delta == 1), and resolveGroupAhead stops at unknown or unbounded writes. --- .../Execution/include/Driver/Driver.hpp | 7 +-- .../src/Driver/Dispatch/Dispatch.cpp | 13 +++--- .../src/Driver/Dispatch/DispatchAhead.cpp | 46 ++++++++++++++----- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index b272c4c49b..e1ad7e8f2a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -315,20 +315,21 @@ private: static void notePendingWait(std::span packet, std::uint32_t queue, std::uint64_t received); void landPendingWaits(std::uint32_t queue); struct ResolvedDispatch { - std::uint64_t address = 0, key = 0, indirectArguments = 0; + std::uint64_t address = 0, key = 0, indirectArguments = 0, forgetAtCapture = 0; std::array packet{}; std::shared_ptr registeredShader; std::shared_ptr compiledResult; std::shared_ptr keepVariant, attachVariant; + std::shared_ptr missedEntry; std::shared_ptr shaderMemory; std::vector captured; std::vector liveWords; std::shared_ptr capture; - bool dataHit = false, cached = false, validated = false; + bool dataHit = false, cached = false, validated = false, missedDiffering = false, noDispatchCache = false; std::chrono::steady_clock::time_point resolvedAt{}; }; struct GroupCaptureStats { - std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, memoryMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0, argumentMismatches = 0; + std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, memoryMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, unknownWrites = 0, deviceRejects = 0, argumentMismatches = 0; double resolveMs = 0, resolveMaxMs = 0, ageMs = 0, ageMaxMs = 0; std::chrono::steady_clock::time_point reported = std::chrono::steady_clock::now(); }; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index ce351faa3d..0fe93d5363 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -364,12 +364,12 @@ void Driver::dispatch(QueueState& queue, std::span packet, } if (!pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); } + std::uint64_t forgetAtCapture = 0; if (cached) { require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader"); captureMs += phaseTiming.Elapsed(); } else if (!fromAhead) { shaderMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan); - std::uint64_t forgetAtCapture = 0; static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; try { @@ -417,7 +417,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } recompileMs += phaseTiming.Elapsed(); phaseTiming.Phase(PhaseRecompile); - insertDispatch(address, key, noDispatchCache || !CacheableResult(*compiledResult), profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); + if (!resolving) insertDispatch(address, key, noDispatchCache || !CacheableResult(*compiledResult), profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); } if (resolving) { auto& parked = resolvedAhead()[currentPacketOffset()]; @@ -425,11 +425,13 @@ void Driver::dispatch(QueueState& queue, std::span packet, parked.address = address; parked.key = key; parked.indirectArguments = indirectArguments; + parked.forgetAtCapture = forgetAtCapture; std::copy_n(packet.begin(), std::min(packet.size(), parked.packet.size()), parked.packet.begin()); parked.registeredShader = registeredShader; parked.compiledResult = compiledResult; parked.keepVariant = keepVariant; parked.attachVariant = attachVariant; + parked.missedEntry = missedEntry; parked.shaderMemory = shaderMemory; parked.captured = std::move(captured); parked.liveWords = std::move(liveWords); @@ -437,14 +439,11 @@ void Driver::dispatch(QueueState& queue, std::span packet, parked.dataHit = dataHit; parked.cached = cached; parked.validated = validated; + parked.missedDiffering = missedDiffering; + parked.noDispatchCache = noDispatchCache || (compiledResult != nullptr && !CacheableResult(*compiledResult)); parked.resolvedAt = std::chrono::steady_clock::now(); ++groupStats.resolved; if (cached) ++groupStats.resolvedHits; - if (!cached && compiledResult != nullptr && !localDevice->HasComputePipeline(*compiledResult)) { - SubmitLibraryPrebuild([localDevice, compiledResult, address] { - try { localDevice->PrebuildComputePipeline(*compiledResult, address); } catch (const std::exception&) {} - }); - } return; } if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, request, memory, address, *keepVariant, liveWords, *compiledResult); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp index 50a5eee57b..10f3ffb838 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp @@ -2,6 +2,7 @@ #include "ThreadOwned.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" @@ -108,10 +109,6 @@ void NoteLabel(std::vector& labels, std::uint64_t address, std::span regions, const std::vector& labels) { for (const auto& label : labels) { for (const auto& region : regions) { @@ -136,7 +133,6 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l QueueState scratch = live; std::vector labels; std::vector> writes; - bool unknownWrites = false; for (const auto& label : deferredLabels().labels) NoteLabel(labels, label.address, std::span(label.bytes).first(label.size)); struct Resolving { std::size_t offset; @@ -277,7 +273,7 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l const auto argumentsEnd = arguments + 3u * sizeof(std::uint32_t); const bool written = std::any_of(writes.begin(), writes.end(), [&](const auto& range) { return range.first < argumentsEnd && arguments < range.second; }) || std::any_of(labels.begin(), labels.end(), [&](const RunLabel& label) { return label.address < argumentsEnd && arguments < label.address + label.size; }); - if ((unknownWrites || written) && indirectArgumentsPath(packet, arguments) != IndirectGpu) { + if (written && indirectArgumentsPath(packet, arguments) != IndirectGpu) { ++stats.writtenBefore; break; } @@ -302,10 +298,38 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l ++stats.writtenBefore; break; } - if (found->second.compiledResult != nullptr) appendWrittenRanges(*found->second.compiledResult, writes); - if (found->second.compiledResult == nullptr || WritesThroughPointers(*found->second.compiledResult)) unknownWrites = true; + if (found->second.compiledResult == nullptr) { + ++stats.unknownWrites; + break; + } + if (!found->second.cached && found->second.registeredShader != nullptr) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto& snapshot = *found->second.registeredShader; + std::vector memory{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}}; + if (!snapshot.header.empty()) memory.push_back({snapshot.headerAddress, snapshot.header}); + auto lap = std::chrono::steady_clock::now(); + std::array phaseMs{}; + auto phaseLap = lap; + DispatchPhaseTiming phaseTiming{profile, lap, phaseMs, phaseLap}; + insertDispatch(found->second.address, found->second.key, found->second.noDispatchCache, profile, found->second.registeredShader, found->second.forgetAtCapture, memory, found->second.shaderMemory, found->second.captured, found->second.capture, found->second.compiledResult, found->second.missedEntry, found->second.missedDiffering, found->second.attachVariant, phaseTiming); + if (!localDevice->HasComputePipeline(*found->second.compiledResult)) { + const auto compiledResult = found->second.compiledResult; + const auto address = found->second.address; + SubmitLibraryPrebuild([localDevice, compiledResult, address] { + try { localDevice->PrebuildComputePipeline(*compiledResult, address); } catch (const std::exception&) {} + }); + } + } + bool bounded = false; + const auto itemWrites = DispatchWriteRanges(*found->second.compiledResult, bounded); + if (!bounded) { + ++stats.unknownWrites; + break; + } + writes.insert(writes.end(), itemWrites.begin(), itemWrites.end()); } else { - unknownWrites = true; + ++stats.unknownWrites; + break; } } else if (opcode == 0x37 || opcode == 0x49) { const auto label = Pm4::DecodeLabelWrite(packet); @@ -334,10 +358,10 @@ void Driver::reportGroupCapture(std::uint32_t queue) { auto& stats = groupCaptureStats(); const auto now = std::chrono::steady_clock::now(); if (now - stats.reported < std::chrono::seconds(10)) return; - AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu memory %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device, %llu dropped because the argument re-read differed\n", + AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu memory %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu stopped at unknown writes, %llu rejected by the device, %llu dropped because the argument re-read differed\n", queue, static_cast(stats.runs), static_cast(stats.resolved), static_cast(stats.resolvedHits), stats.resolveMs, stats.resolveMaxMs, static_cast(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast(stats.adopted) : 0.0, stats.ageMaxMs, static_cast(stats.addressMismatches), static_cast(stats.keyMismatches), static_cast(stats.memoryMismatches), - static_cast(stats.failures), static_cast(stats.stale), static_cast(stats.labelOverlaps), static_cast(stats.writtenBefore), static_cast(stats.deviceRejects), static_cast(stats.argumentMismatches)); + static_cast(stats.failures), static_cast(stats.stale), static_cast(stats.labelOverlaps), static_cast(stats.writtenBefore), static_cast(stats.unknownWrites), static_cast(stats.deviceRejects), static_cast(stats.argumentMismatches)); stats = GroupCaptureStats{}; stats.reported = now; } From c4ddc2d404657c35b6822c37dc3c4311edf76872 Mon Sep 17 00:00:00 2001 From: hamb3r <48532909+hamb3r@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:26:41 +0300 Subject: [PATCH 220/318] test(agc): cover ahead HLE reopen, unknown writes, and rejected cache insert Symptom / Log: - Missing regression coverage in WriteDataWrittenAhead for post-HLE ahead gate reopening, rejected ahead capture cache pollution, StorageImage write overlap, and unbounded GDS writes. Root Cause: - core/libs/prx/libSceAgcDriver/tests/execution/WriteDataWrittenAhead.cpp only covered direct PM4 WRITE_DATA before an ahead dispatch. Change: - Extend WriteDataWrittenAhead to test rejected ahead capture cache pollution, CPU-backed and GPU-backed HLE StreamCopy / FillKernel / DwordPatternFill before direct and IndirectGpu dispatches, StorageImage write overlap, and unbounded GDS write stop. Metric / Verification: - ctest -R agc_driver_write_data_written_ahead passes with VK_LAYER_KHRONOS_validation and 0 VUID errors. --- .../tests/execution/WriteDataWrittenAhead.cpp | 318 +++++++++++++++++- 1 file changed, 310 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/WriteDataWrittenAhead.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/WriteDataWrittenAhead.cpp index 529aad4163..67d2f784ae 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/WriteDataWrittenAhead.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/WriteDataWrittenAhead.cpp @@ -1,27 +1,129 @@ #include "VulkanTestDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Submit/include/Acb.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif #include #include +#include +#include #include #include #include #include #include +namespace AgcDriver::DriverDetail { +struct DriverTestAccess { + static void PrepareSamplingForRejectedAhead() { + auto& driver = Driver::Get(); + driver.evidenceValidations.store(1u, std::memory_order_relaxed); + driver.evidenceReads.store(15u, std::memory_order_relaxed); + } +}; +} + namespace { void Require(bool condition, const std::string& message) { if (!condition) throw std::runtime_error(message); } +constexpr std::size_t BlockBytes = 65536; + +struct GuestBlock { + explicit GuestBlock(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (watched) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + if (watched) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map a guest block"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate a guest block"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true, true); + } + ~GuestBlock() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (watched) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } +#else + if (watched) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + std::uint32_t* Dwords(std::size_t byteOffset) const { return reinterpret_cast(data + byteOffset); } + std::uint8_t* data = nullptr; + std::size_t bytes = 0; + bool watched = false; +}; + alignas(256) std::array CounterCode{0xf4080200u, 0xfa000000u, 0xbf8cc07fu, 0x7e000281u, 0xe0c80000u, 0x80020000u, 0xbf810000u}; +alignas(256) std::array WriterCode{0xf4080200u, 0xfa000000u, 0xbf8cc07fu, 0x7e000202u, 0x7e020203u, 0x7e040204u, 0xe07c0000u, 0x80020000u, 0xbf810000u}; +alignas(256) std::array ImageWriterCode{0x7e000280u, 0x7e020280u, 0x7e040208u, 0x7e060209u, 0x7e08020au, 0x7e0a020bu, 0xf0201f08u, 0x00000200u, 0xbf810000u}; +alignas(256) std::array GdsWriterCode{0x7e000280u, 0xd8360000u, 0x00000000u, 0xbf810000u}; +alignas(256) std::array StreamCopyCode{0xd7460008u, 0x04010e08u, 0x8f6a8708u, 0xd76d0009u, 0x0401806au, 0xe00c2000u, 0x80000008u, 0xe00c2000u, 0x80000409u, 0xbf8c3f71u, 0xe01c2000u, 0x80010008u, 0xbf8c3f70u, 0xe01c2000u, 0x80010409u, 0xbf810000u}; +alignas(256) std::array FillKernelCode{0xd7460004u, 0x04010c08u, 0x7e000204u, 0x7e020205u, 0x7e040206u, 0x7e060207u, 0xe01c2000u, 0x80000004u, 0xbf810000u}; +alignas(256) std::array DwordPatternFillCode{ + 0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu, + 0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u, + 0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u, + 0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u, 0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u, + 0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u, + 0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u, + 0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u, + 0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u, + 0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u}; alignas(256) std::array First{}; alignas(256) std::array Stale{}; alignas(256) std::array Fresh{}; +alignas(256) std::array Arguments{1, 1, 1, 0}; alignas(256) std::array FirstDescriptor{}; alignas(256) std::array SecondDescriptor{}; +alignas(256) std::array CopySource{}; +alignas(256) std::array ArgumentsDescriptor{}; alignas(256) volatile std::uint32_t Gate = 1; alignas(256) volatile std::uint32_t Done = 0; @@ -41,22 +143,113 @@ void SetShaderRegister(std::vector& words, std::uint32_t offset, words.insert(words.end(), {0xc0017600u, offset, value}); } -void Program(std::vector& words, const void* code, std::initializer_list userData) { +void Program(std::vector& words, const void* code, std::initializer_list userData, std::uint32_t threadsX = 1, bool groupIdX = false) { const auto address = static_cast(reinterpret_cast(code)); SetShaderRegister(words, 0x20c, static_cast(address >> 8u)); SetShaderRegister(words, 0x20d, static_cast(address >> 40u) & 0xffu); - for (const std::uint32_t offset : {0x207u, 0x208u, 0x209u}) SetShaderRegister(words, offset, 1); + SetShaderRegister(words, 0x207, threadsX); + SetShaderRegister(words, 0x208, 1); + SetShaderRegister(words, 0x209, 1); SetShaderRegister(words, 0x212, 0); - SetShaderRegister(words, 0x213, static_cast(userData.size()) << 1u); + SetShaderRegister(words, 0x213, (static_cast(userData.size()) << 1u) | (groupIdX ? (1u << 7u) : 0u)); std::uint32_t offset = 0x240; for (const auto value : userData) SetShaderRegister(words, offset++, value); } -void Count(std::vector& words, const void* descriptor) { - Program(words, CounterCode.data(), {Low(descriptor), High(descriptor)}); +void Count(std::vector& words, const void* descriptor, std::uint32_t threadsX = 1) { + Program(words, CounterCode.data(), {Low(descriptor), High(descriptor)}, threadsX); words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); } +void CountIndirect(std::vector& words, const void* descriptor, const void* arguments) { + Program(words, CounterCode.data(), {Low(descriptor), High(descriptor)}); + words.insert(words.end(), {0xc0021600u, Low(arguments), High(arguments), 0x8041u}); +} + +void WriteThreeDwords(std::vector& words, const void* targetDescriptor, std::uint32_t d0, std::uint32_t d1, std::uint32_t d2) { + Program(words, WriterCode.data(), {Low(targetDescriptor), High(targetDescriptor), d0, d1, d2}); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); +} + +void ImageWriteDescriptor(std::vector& words, const void* destination, const std::array& pattern) { + const auto address = static_cast(reinterpret_cast(destination)); + constexpr std::uint32_t format = 75u; + constexpr std::uint32_t width = 16u; + constexpr std::uint32_t height = 1u; + constexpr std::uint32_t type2d = 9u; + Program(words, ImageWriterCode.data(), { + static_cast(address >> 8u), + (static_cast(address >> 40u) & 0xffu) | (format << 20u) | (((width - 1u) & 3u) << 30u), + ((width - 1u) >> 2u) | ((height - 1u) << 14u), + 0xfacu | (type2d << 28u), + 0u, 0u, 0u, 0u, + pattern[0], pattern[1], pattern[2], pattern[3] + }); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); +} + +void GdsWrite(std::vector& words) { + Program(words, GdsWriterCode.data(), {}); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); +} + +void StreamCopyDescriptor(std::vector& words, const void* source, const void* destination) { + constexpr std::uint32_t format = 0x0004bfacu; + Program(words, StreamCopyCode.data(), { + Low(source), (High(source) & 0xffffu) | (16u << 16u), 1u, format, + Low(destination), (High(destination) & 0xffffu) | (16u << 16u), 1u, format + }, 64, true); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); +} + +void FillKernelDescriptor(std::vector& words, const void* destination, const std::array& pattern) { + constexpr std::uint32_t format = 0x0004bfacu; + Program(words, FillKernelCode.data(), { + Low(destination), (High(destination) & 0xffffu) | (16u << 16u), 1u, format, + pattern[0], pattern[1], pattern[2], pattern[3] + }, 64, true); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); +} + +void DwordPatternFillDescriptor(std::vector& words, const void* destination, const std::array& pattern) { + constexpr std::uint32_t format = 0x00014facu; + Program(words, DwordPatternFillCode.data(), { + Low(destination), (High(destination) & 0xffffu) | (4u << 16u), 4u, format, + pattern[0], pattern[1], pattern[2], pattern[3], 4u, 4u + }, 64, true); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); +} + +template +void RunUpdateAheadCase(const std::string& label, std::uint32_t* targetDescriptor, const std::array& stale, EmitUpdate&& emitUpdate, std::uint64_t expectedAdopted = 4, bool exactAdopted = false) { + First[0] = 0; + Stale[0] = 0; + Fresh[0] = 0; + Done = 0; + std::memcpy(targetDescriptor, stale.data(), sizeof(stale)); + const auto adoptedBefore = AgcDriverDispatchesAdoptedAhead_nid_postfix(); + std::vector words; + words.insert(words.end(), {0xc0053c00u, 0x13u, Low(&Gate), High(&Gate), 1u, 0xffffffffu, 0x19u}); + Count(words, FirstDescriptor.data()); + WriteThreeDwords(words, ArgumentsDescriptor.data(), 1u, 1u, 1u); + emitUpdate(words); + Count(words, targetDescriptor); + CountIndirect(words, targetDescriptor, Arguments.data()); + words.insert(words.end(), {0xc0033700u, 0x00100200u, Low(&Done), High(&Done), 1u}); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + Require(sceAgcDriverSubmitAcb(0x20, &packet) == 0, label + ": compute submit failed"); + AgcDriverWaitIdle_nid_postfix(); + Require(Done == 1, label + ": submission did not finish"); + Require(First[0] == 1, label + ": first dispatch did not run"); + Require(Fresh[0] == 2 && Stale[0] == 0, label + ": dispatches after descriptor update used stale descriptor (stale " + std::to_string(Stale[0]) + ", fresh " + std::to_string(Fresh[0]) + ")"); + const auto adopted = AgcDriverDispatchesAdoptedAhead_nid_postfix() - adoptedBefore; + if (exactAdopted) { + Require(adopted == expectedAdopted, label + ": expected exactly " + std::to_string(expectedAdopted) + " dispatches adopted ahead, got " + std::to_string(adopted)); + } else { + Require(adopted >= expectedAdopted, label + ": expected at least " + std::to_string(expectedAdopted) + " dispatches adopted ahead across pre-update and reopened post-update gates, got " + std::to_string(adopted)); + } +} + } int main() { @@ -64,19 +257,72 @@ int main() { auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; device.reset(); + GuestBlock imported(BlockBytes); + std::uint32_t* importedSecondDescriptor = imported.Dwords(0); + std::uint32_t* importedCopySource = imported.Dwords(256); + std::uint32_t* importedCopySourceDescriptor = imported.Dwords(512); + std::uint32_t* importedPolluteDescriptor = imported.Dwords(768); + std::uint32_t* importedPolluteMetaDescriptor = imported.Dwords(1024); + const auto stale = Descriptor(Stale.data(), sizeof(Stale)); + const auto fresh = Descriptor(Fresh.data(), sizeof(Fresh)); FirstDescriptor = Descriptor(First.data(), sizeof(First)); - SecondDescriptor = Descriptor(Stale.data(), sizeof(Stale)); + SecondDescriptor = stale; + std::memcpy(importedSecondDescriptor, stale.data(), sizeof(stale)); + const auto copySrcDesc = Descriptor(importedCopySource, 16u); + std::memcpy(importedCopySourceDescriptor, copySrcDesc.data(), sizeof(copySrcDesc)); + std::memcpy(importedPolluteDescriptor, stale.data(), sizeof(stale)); + const auto polluteMetaDesc = Descriptor(importedPolluteDescriptor, 16u); + std::memcpy(importedPolluteMetaDescriptor, polluteMetaDesc.data(), sizeof(polluteMetaDesc)); + ArgumentsDescriptor = Descriptor(Arguments.data(), sizeof(Arguments)); std::vector warmUp; Count(warmUp, FirstDescriptor.data()); Count(warmUp, SecondDescriptor.data()); + CountIndirect(warmUp, SecondDescriptor.data(), Arguments.data()); + Count(warmUp, importedSecondDescriptor); + CountIndirect(warmUp, importedSecondDescriptor, Arguments.data()); + WriteThreeDwords(warmUp, ArgumentsDescriptor.data(), 1u, 1u, 1u); + WriteThreeDwords(warmUp, importedCopySourceDescriptor, fresh[0], fresh[1], fresh[2]); + Count(warmUp, importedPolluteDescriptor, 2); + for (int i = 0; i < 4; ++i) { + WriteThreeDwords(warmUp, importedPolluteMetaDescriptor, stale[0], stale[1], stale[2]); + Count(warmUp, importedPolluteDescriptor, 2); + } + WriteThreeDwords(warmUp, importedPolluteMetaDescriptor, fresh[0], fresh[1], fresh[2]); + ImageWriteDescriptor(warmUp, importedSecondDescriptor, stale); + GdsWrite(warmUp); warmUp.insert(warmUp.end(), {0xc0033700u, 0x00100200u, Low(&Done), High(&Done), 2u}); Packet first{warmUp.data(), static_cast(warmUp.size()), 0, {}}; Require(sceAgcDriverSubmitAcb(0x20, &first) == 0, "warm-up submit failed"); AgcDriverWaitIdle_nid_postfix(); - Require(Done == 2 && First[0] == 1 && Stale[0] == 1, "the warm-up dispatches did not run"); + Require(Done == 2 && First[0] == 1 && Stale[0] == 14, "the warm-up dispatches did not run"); First[0] = 0; Stale[0] = 0; - const auto fresh = Descriptor(Fresh.data(), sizeof(Fresh)); + Fresh[0] = 0; + { + Done = 0; + std::memcpy(importedPolluteDescriptor, stale.data(), sizeof(stale)); + AgcDriver::DriverDetail::DriverTestAccess::PrepareSamplingForRejectedAhead(); + const auto countersBefore = AgcDriver::DriverDetail::Driver::Get().DispatchCacheCounters(); + const auto adoptedBefore = AgcDriverDispatchesAdoptedAhead_nid_postfix(); + std::vector polluteWords; + polluteWords.insert(polluteWords.end(), {0xc0053c00u, 0x13u, Low(&Gate), High(&Gate), 1u, 0xffffffffu, 0x19u}); + Count(polluteWords, FirstDescriptor.data()); + WriteThreeDwords(polluteWords, importedPolluteMetaDescriptor, fresh[0], fresh[1], fresh[2]); + Count(polluteWords, importedPolluteDescriptor, 3); + polluteWords.insert(polluteWords.end(), {0xc0033700u, 0x00100200u, Low(&Done), High(&Done), 1u}); + Packet pollutePacket{polluteWords.data(), static_cast(polluteWords.size()), 0, {}}; + Require(sceAgcDriverSubmitAcb(0x20, &pollutePacket) == 0, "rejected ahead cache pollution submit failed"); + AgcDriverWaitIdle_nid_postfix(); + const auto adopted = AgcDriverDispatchesAdoptedAhead_nid_postfix() - adoptedBefore; + const auto countersAfter = AgcDriver::DriverDetail::Driver::Get().DispatchCacheCounters(); + Require(Done == 1 && First[0] == 1, "rejected ahead cache pollution submission did not finish"); + Require(adopted == 2, "rejected ahead capture was unexpectedly adopted (adopted " + std::to_string(adopted) + ")"); + Require(Fresh[0] == 3 && Stale[0] == 0, "normal dispatch after rejected ahead capture received the rejected stale variant from global dispatchCache (stale " + std::to_string(Stale[0]) + ", fresh " + std::to_string(Fresh[0]) + ")"); + Require(countersAfter.inserts - countersBefore.inserts == 1, "rejected ahead capture polluted global dispatchCache with an extra insert (inserts delta " + std::to_string(countersAfter.inserts - countersBefore.inserts) + ")"); + } + First[0] = 0; + Stale[0] = 0; + Fresh[0] = 0; std::vector words; words.insert(words.end(), {0xc0053c00u, 0x13u, Low(&Gate), High(&Gate), 1u, 0xffffffffu, 0x19u}); Count(words, FirstDescriptor.data()); @@ -90,6 +336,62 @@ int main() { Require(SecondDescriptor[0] == fresh[0] && SecondDescriptor[1] == fresh[1], "WRITE_DATA did not replace the descriptor"); Require(First[0] == 1, "the first dispatch did not run"); Require(Fresh[0] == 1 && Stale[0] == 0, "a dispatch whose descriptor a WRITE_DATA before it replaces used the old descriptor (stale " + std::to_string(Stale[0]) + ", fresh " + std::to_string(Fresh[0]) + ")"); + + CopySource = fresh; + RunUpdateAheadCase("HLE StreamCopy (CPU-backed)", SecondDescriptor.data(), stale, [&](std::vector& cmd) { + StreamCopyDescriptor(cmd, CopySource.data(), SecondDescriptor.data()); + }); + + const auto gpuCopiesBefore = AgcDriver::DriverDetail::copyPathCounts[AgcDriver::DriverDetail::CopyGpu].load(); + RunUpdateAheadCase("HLE StreamCopy (GPU-backed SourcePending)", importedSecondDescriptor, stale, [&](std::vector& cmd) { + const std::array initSrc{0u, 0u, fresh[2], fresh[3]}; + std::memcpy(importedCopySource, initSrc.data(), sizeof(initSrc)); + WriteThreeDwords(cmd, importedCopySourceDescriptor, fresh[0], fresh[1], fresh[2]); + StreamCopyDescriptor(cmd, importedCopySource, importedSecondDescriptor); + }, 4); + Require(AgcDriver::DriverDetail::copyPathCounts[AgcDriver::DriverDetail::CopyGpu].load() > gpuCopiesBefore, "GPU-backed StreamCopy did not take CopyGpu path"); + + RunUpdateAheadCase("HLE FillKernel (CPU-backed)", SecondDescriptor.data(), stale, [&](std::vector& cmd) { + FillKernelDescriptor(cmd, SecondDescriptor.data(), fresh); + }); + + RunUpdateAheadCase("HLE FillKernel (GPU-backed)", importedSecondDescriptor, stale, [&](std::vector& cmd) { + FillKernelDescriptor(cmd, importedSecondDescriptor, fresh); + }); + + RunUpdateAheadCase("HLE DwordPatternFill (CPU-backed)", SecondDescriptor.data(), stale, [&](std::vector& cmd) { + DwordPatternFillDescriptor(cmd, SecondDescriptor.data(), fresh); + }); + + RunUpdateAheadCase("HLE DwordPatternFill (GPU-backed)", importedSecondDescriptor, stale, [&](std::vector& cmd) { + DwordPatternFillDescriptor(cmd, importedSecondDescriptor, fresh); + }); + + RunUpdateAheadCase("StorageImage write overlap", importedSecondDescriptor, stale, [&](std::vector& cmd) { + ImageWriteDescriptor(cmd, importedSecondDescriptor, fresh); + }, 3, true); + + { + First[0] = 0; + Stale[0] = 0; + Fresh[0] = 0; + Done = 0; + SecondDescriptor = fresh; + const auto adoptedBefore = AgcDriverDispatchesAdoptedAhead_nid_postfix(); + std::vector gdsWords; + gdsWords.insert(gdsWords.end(), {0xc0053c00u, 0x13u, Low(&Gate), High(&Gate), 1u, 0xffffffffu, 0x19u}); + Count(gdsWords, FirstDescriptor.data()); + GdsWrite(gdsWords); + Count(gdsWords, SecondDescriptor.data()); + gdsWords.insert(gdsWords.end(), {0xc0033700u, 0x00100200u, Low(&Done), High(&Done), 1u}); + Packet gdsPacket{gdsWords.data(), static_cast(gdsWords.size()), 0, {}}; + Require(sceAgcDriverSubmitAcb(0x20, &gdsPacket) == 0, "GDS unbounded write submit failed"); + AgcDriverWaitIdle_nid_postfix(); + const auto adopted = AgcDriverDispatchesAdoptedAhead_nid_postfix() - adoptedBefore; + Require(Done == 1 && First[0] == 1 && Fresh[0] == 1, "GDS unbounded write submission did not run"); + Require(adopted == 2, "unbounded GDS write did not stop resolveGroupAhead (expected 2 adopted ahead, got " + std::to_string(adopted) + ")"); + } + std::cout << "write data written ahead test passed\n"; AgcDriverShutdown_nid_postfix(); return 0; From 43debd5f3f38c818e08d7ba97919ff22a5dbbdb4 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Fri, 30 Oct 2026 00:24:07 +1300 Subject: [PATCH 221/318] test(video-recording): stop pinning unsupported recording refusals --- core/libs/tests/GuestVideoRecording.cpp | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/core/libs/tests/GuestVideoRecording.cpp b/core/libs/tests/GuestVideoRecording.cpp index 1ae1d378e6..930b42e6fb 100644 --- a/core/libs/tests/GuestVideoRecording.cpp +++ b/core/libs/tests/GuestVideoRecording.cpp @@ -64,14 +64,6 @@ int main() { Require(Throws([&] { _sceVideoRecordingQueryParam_nid_postfix(reinterpret_cast(query.data())); }), "the default parameters were produced"); Require(sceVideoRecordingQueryMemSize(nullptr) == InvalidArgument, "a heap size was reported for no parameters"); - for (const std::uint32_t format : {0x1u, 0x2u, 0x10u, 0x11u, 0x28u, 0x40u, 0x41u}) { - Require(Throws([&] { Query(RecordingParam(format)); }), "a heap size was reported for a video format"); - VideoRecordingParam param = RecordingParam(format); - param.option = 0; - Require(Throws([&] { Query(param); }), "a heap size was reported for a video format without the option"); - param.mode = 1; - Require(Throws([&] { Query(param); }), "a heap size was reported for mode 1"); - } for (std::uint32_t format = 0x3; format <= 0x44; ++format) { if (format == 0x10 || format == 0x11 || format == 0x28 || format == 0x40 || format == 0x41) continue; Require(Query(RecordingParam(format)) == InvalidArgument, "an unknown video format was accepted"); @@ -92,21 +84,11 @@ int main() { param.size = size; Require(Query(param) == NoVideoFormat, "a block that ends before its format was accepted"); } - for (const std::uint64_t size : {0x25ull, 0x40ull, 0x1000ull}) { - VideoRecordingParam param = RecordingParam(1); - param.size = size; - Require(Throws([&] { Query(param); }), "a heap size was reported for a block with a format"); - } for (const std::uint32_t duration : {0u, 3601u, 7200u, 0xffffffffu}) { VideoRecordingParam param = RecordingParam(1); param.duration = duration; Require(Query(param) == InvalidArgument, "a duration outside 1 to 3600 was accepted"); } - for (const std::uint32_t duration : {1u, 60u, 3599u}) { - VideoRecordingParam param = RecordingParam(1); - param.duration = duration; - Require(Throws([&] { Query(param); }), "a duration inside 1 to 3600 was refused"); - } for (const std::uint32_t mode : {2u, 3u}) { VideoRecordingParam param = RecordingParam(1); param.mode = mode; @@ -146,8 +128,6 @@ int main() { Require(sceVideoRecordingOpen("clip.mp4", &minimal, heap.data(), MinimumHeap - 1) == InvalidArgument, "the recorder opened with a heap smaller than QueryMemSize asks"); VideoRecordingParam optionOnly = RecordingParam(0); Require(sceVideoRecordingOpen("clip.mp4", &optionOnly, heap.data(), static_cast(heap.size())) == NoVideoFormat, "the recorder opened with the option and no format"); - Require(Throws([&] { sceVideoRecordingOpen("clip.mp4", &minimal, heap.data(), MinimumHeap); }), "the recorder opened without recording"); - Require(Throws([&] { sceVideoRecordingOpen("clip.mp4", &video, heap.data(), 0x7fffffff); }), "the recorder opened without recording"); for (const std::uint8_t value : heap) Require(value == Sentinel, "the heap was written"); Require(sceVideoRecordingGetStatus() == StatusNone, "a failed open changed the status"); Require(sceVideoRecordingStart() == NotOpen, "a failed open left a recorder to start"); From 478a1c3f4adacfd1d4bcc2feff2a66dce80081f8 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Fri, 30 Oct 2026 01:00:16 +1300 Subject: [PATCH 222/318] fix(libSceNpManager): declare the pending exports in the library header --- .../prx/libSceNpManager/PendingExports.cpp | 1 + core/libs/prx/libSceNpManager/include/Api.hpp | 190 ++++++++++++++++++ 2 files changed, 191 insertions(+) diff --git a/core/libs/prx/libSceNpManager/PendingExports.cpp b/core/libs/prx/libSceNpManager/PendingExports.cpp index 46736416fc..f8eff6f655 100644 --- a/core/libs/prx/libSceNpManager/PendingExports.cpp +++ b/core/libs/prx/libSceNpManager/PendingExports.cpp @@ -1,6 +1,7 @@ #include #include #include "SceTypes.hpp" +#include "prx/libSceNpManager/include/Api.hpp" #include "prx/libc/include/General.hpp" extern "C" { diff --git a/core/libs/prx/libSceNpManager/include/Api.hpp b/core/libs/prx/libSceNpManager/include/Api.hpp index 85ea6fc53f..1248cb4741 100644 --- a/core/libs/prx/libSceNpManager/include/Api.hpp +++ b/core/libs/prx/libSceNpManager/include/Api.hpp @@ -216,4 +216,194 @@ int APS5_VABI sceNpSnsFacebookCreateRequest(); int APS5_VABI sceNpSnsFacebookDeleteRequest(); int APS5_VABI sceNpSnsFacebookGetAccessToken(); +int APS5_VABI _ZN3sce2np10NpOnlineId13GetNpOnlineIdERKNS0_4UserEP13SceNpOnlineId(void); +int APS5_VABI _ZN3sce2np10NpOnlineId13GetNpOnlineIdERKNS0_4UserEPS1_(void); +int APS5_VABI _ZN3sce2np10NpOnlineId16SetNpOnlineIdStrEPKc(void); +int APS5_VABI _ZN3sce2np10NpOnlineId5ClearEv(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdC1ERK13SceNpOnlineId(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdC1ERKS1_(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdC1Ev(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdC2ERK13SceNpOnlineId(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdC2ERKS1_(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdC2Ev(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdD0Ev(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdD1Ev(void); +int APS5_VABI _ZN3sce2np10NpOnlineIdD2Ev(void); +int APS5_VABI _ZN3sce2np11NpHttpTransC2EP16SceNpAllocatorEx(void); +int APS5_VABI _ZN3sce2np11NpHttpTransD0Ev(void); +int APS5_VABI _ZN3sce2np11NpHttpTransD1Ev(void); +int APS5_VABI _ZN3sce2np11NpHttpTransD2Ev(void); +int APS5_VABI _ZN3sce2np12NpHttpClient4InitEii(void); +int APS5_VABI _ZN3sce2np12NpHttpClientC1EP16SceNpAllocatorEx(void); +int APS5_VABI _ZN3sce2np12NpHttpClientC2EP16SceNpAllocatorEx(void); +int APS5_VABI _ZN3sce2np12NpHttpClientD0Ev(void); +int APS5_VABI _ZN3sce2np12NpHttpClientD1Ev(void); +int APS5_VABI _ZN3sce2np12NpHttpClientD2Ev(void); +int APS5_VABI _ZN3sce2np12NpTitleToken5ClearEv(void); +int APS5_VABI _ZN3sce2np12NpTitleTokenC1ERKS1_(void); +int APS5_VABI _ZN3sce2np12NpTitleTokenC1Ev(void); +int APS5_VABI _ZN3sce2np12NpTitleTokenC2ERKS1_(void); +int APS5_VABI _ZN3sce2np12NpTitleTokenC2Ev(void); +int APS5_VABI _ZN3sce2np12NpTitleTokenD0Ev(void); +int APS5_VABI _ZN3sce2np12NpTitleTokenD1Ev(void); +int APS5_VABI _ZN3sce2np12NpTitleTokenD2Ev(void); +int APS5_VABI _ZN3sce2np13NpAccessToken5ClearEv(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenC1ERK16SceNpAccessToken(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenC1ERKS1_(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenC1Ev(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenC2ERK16SceNpAccessToken(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenC2ERKS1_(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenC2Ev(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenD0Ev(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenD1Ev(void); +int APS5_VABI _ZN3sce2np13NpAccessTokenD2Ev(void); +int APS5_VABI _ZN3sce2np3ipc14service_client10EndRequestEii(void); +int APS5_VABI _ZN3sce2np3ipc14service_client11InitServiceEi(void); +int APS5_VABI _ZN3sce2np3ipc14service_client11TermServiceEi(void); +int APS5_VABI _ZN3sce2np3ipc14service_client11WaitRequestEiij(void); +int APS5_VABI _ZN3sce2np3ipc14service_client12AbortRequestEii(void); +int APS5_VABI _ZN3sce2np3ipc14service_client12BeginRequestEii(void); +int APS5_VABI _ZN3sce2np3ipc14service_client13CreateRequestEimPKvPi(void); +int APS5_VABI _ZN3sce2np3ipc14service_client13DeleteRequestEii(void); +int APS5_VABI _ZN3sce2np3ipc14service_client13GetIpmiClientEv(void); +int APS5_VABI _ZN3sce2np3ipc14service_client13PollEventFlagEijmjPm(void); +int APS5_VABI _ZN3sce2np3ipc14service_client14PollEventQueueEiPvm(void); +int APS5_VABI _ZN3sce2np4NpId5ClearEv(void); +int APS5_VABI _ZN3sce2np4NpIdC1ERK7SceNpId(void); +int APS5_VABI _ZN3sce2np4NpIdC1ERKS1_(void); +int APS5_VABI _ZN3sce2np4NpIdC1Ev(void); +int APS5_VABI _ZN3sce2np4NpIdC2ERK7SceNpId(void); +int APS5_VABI _ZN3sce2np4NpIdC2ERKS1_(void); +int APS5_VABI _ZN3sce2np4NpIdC2Ev(void); +int APS5_VABI _ZN3sce2np4NpIdD0Ev(void); +int APS5_VABI _ZN3sce2np4NpIdD1Ev(void); +int APS5_VABI _ZN3sce2np4NpIdD2Ev(void); +int APS5_VABI _ZN3sce2np4User5ClearEv(void); +int APS5_VABI _ZN3sce2np4User7GetUserEiPS1_(void); +int APS5_VABI _ZN3sce2np4UserC1ERKS1_(void); +int APS5_VABI _ZN3sce2np4UserC1Ei(void); +int APS5_VABI _ZN3sce2np4UserC1Ev(void); +int APS5_VABI _ZN3sce2np4UserC2ERKS1_(void); +int APS5_VABI _ZN3sce2np4UserC2Ei(void); +int APS5_VABI _ZN3sce2np4UserC2Ev(void); +int APS5_VABI _ZN3sce2np4UserD0Ev(void); +int APS5_VABI _ZN3sce2np4UserD1Ev(void); +int APS5_VABI _ZN3sce2np4UserD2Ev(void); +int APS5_VABI _ZN3sce2np8NpTicket5ClearEv(void); +int APS5_VABI _ZN3sce2np8NpTicketD0Ev(void); +int APS5_VABI _ZN3sce2np8NpTicketD1Ev(void); +int APS5_VABI _ZN3sce2np8NpTicketD2Ev(void); +int APS5_VABI _ZN3sce2npeqERK13SceNpOnlineIdRKNS0_10NpOnlineIdE(void); +int APS5_VABI _ZN3sce2npeqERKNS0_10NpOnlineIdERK13SceNpOnlineId(void); +int APS5_VABI _ZN3sce2npeqERKNS0_10NpOnlineIdES3_(void); +int APS5_VABI _ZN3sce2npeqERKNS0_4UserERKi(void); +int APS5_VABI _ZN3sce2npeqERKNS0_4UserES3_(void); +int APS5_VABI _ZN3sce2npneERK13SceNpOnlineIdRKNS0_10NpOnlineIdE(void); +int APS5_VABI _ZN3sce2npneERKNS0_10NpOnlineIdERK13SceNpOnlineId(void); +int APS5_VABI _ZN3sce2npneERKNS0_10NpOnlineIdES3_(void); +int APS5_VABI _ZN3sce2npneERKNS0_4UserERKi(void); +int APS5_VABI _ZN3sce2npneERKNS0_4UserES3_(void); +int APS5_VABI _ZNK3sce2np10NpOnlineId7IsEmptyEv(void); +int APS5_VABI _ZNK3sce2np12NpTitleToken6GetStrEv(void); +int APS5_VABI _ZNK3sce2np13NpAccessToken7IsEmptyEv(void); +int APS5_VABI _ZNK3sce2np4User10IsLoggedInEv(void); +int APS5_VABI _ZNK3sce2np4User12GetAccountIdEPm(void); +int APS5_VABI _ZNK3sce2np4User12HasAccountIdEPb(void); +int APS5_VABI _ZNK3sce2np4User25GetAccountIdFromRegistoryEPm(void); +int APS5_VABI _ZNK3sce2np4User7IsEmptyEv(void); +int APS5_VABI _ZNK3sce2np4User7IsGuestEv(void); +int APS5_VABI _ZNK3sce2np4User9GetUserIdEv(void); +int APS5_VABI _ZNK3sce2np8NpTicket13GetTicketDataEv(void); +int APS5_VABI _ZNK3sce2np8NpTicket13GetTicketSizeEv(void); +int APS5_VABI _ZThn16_N3sce2np11NpHttpTransD0Ev(void); +int APS5_VABI _ZThn16_N3sce2np11NpHttpTransD1Ev(void); +int APS5_VABI _ZThn8_N3sce2np11NpHttpTransD0Ev(void); +int APS5_VABI _ZThn8_N3sce2np11NpHttpTransD1Ev(void); +int APS5_VABI _sceNpIpcCreateMemoryFromKernel(void); +int APS5_VABI _sceNpIpcCreateMemoryFromPool(void); +int APS5_VABI _sceNpIpcDestroyMemory(void); +int APS5_VABI _sceNpIpcFreeImpl(void); +int APS5_VABI _sceNpIpcGetNpMemAllocator(void); +int APS5_VABI _sceNpIpcMallocImpl(void); +int APS5_VABI _sceNpIpcReallocImpl(void); +int APS5_VABI _sceNpManagerCreateMemoryFromKernel(void); +int APS5_VABI _sceNpManagerCreateMemoryFromPool(void); +int APS5_VABI _sceNpManagerDestroyMemory(void); +int APS5_VABI _sceNpManagerFreeImpl(void); +int APS5_VABI _sceNpManagerGetNpMemAllocator(void); +int APS5_VABI _sceNpManagerMallocImpl(void); +int APS5_VABI _sceNpManagerReallocImpl(void); +int APS5_VABI sceNpManagerIntGetOnlineIdByAccountId(void); +int APS5_VABI sceNpManagerIntLoginGetAccessTokenViaImplicitFlow(void); +int APS5_VABI sceNpManagerIntLoginGetAccountId(void); +int APS5_VABI sceNpManagerIntLoginGetAuthenticateResponse(void); +int APS5_VABI sceNpManagerIntLoginGetDeviceCodeInfo(void); +int APS5_VABI sceNpManagerIntLoginGetOnlineId(void); +int APS5_VABI sceNpManagerIntLoginGetUserId(void); +int APS5_VABI sceNpManagerIntLoginResetSsoToken(void); +int APS5_VABI sceNpManagerIntLoginRevalidatePassword(void); +int APS5_VABI sceNpManagerIntLoginSetUserId(void); +int APS5_VABI sceNpManagerIntLoginSignin(void); +int APS5_VABI sceNpManagerIntLoginValidateKratosAuthCode(void); +int APS5_VABI sceNpManagerIntLoginVerifyDeviceCode(void); +int APS5_VABI sceNpManagerIntPfAuth(void); +int APS5_VABI sceNpManagerIntRemoveActiveSigninStateCallback(void); +int APS5_VABI sceNpManagerIntRemoveOnlineIdChangeCallback(void); +int APS5_VABI sceNpManagerIntRemovePlusMemberTypeCallback(void); +int APS5_VABI sceNpManagerIntRemoveSigninStateCallback(void); +int APS5_VABI sceNpManagerIntRevalidatePassword(void); +int APS5_VABI sceNpManagerIntSetPlusMemberTypeNB(void); +int APS5_VABI sceNpManagerIntSetRequestHeader(void); +int APS5_VABI sceNpManagerIntSetTimeout(void); +int APS5_VABI sceNpManagerIntSignout(void); +int APS5_VABI sceNpManagerIntSubmitUserCode(void); +int APS5_VABI sceNpManagerIntTemporarySignout(void); +int APS5_VABI sceNpManagerIntUpdateBcAccessToken(void); +int APS5_VABI sceNpManagerIntUpdateVshAccessToken(void); +int APS5_VABI sceNpManagerPrxStartVsh(void); +int APS5_VABI sceNpManagerPrxStopVsh(void); +int APS5_VABI sceNpPush2CreateUserContext(void); +int APS5_VABI sceNpPush2DeleteUserContext(void); +int APS5_VABI sceNpPush2Init(void); +int APS5_VABI sceNpPush2IsInit(void); +int APS5_VABI sceNpPush2OptionalCheckCallback(void); +int APS5_VABI sceNpPush2RegisterDataType(void); +int APS5_VABI sceNpPush2RegisterExtendedDataFilter(void); +int APS5_VABI sceNpPush2RegisterNotificationExCallback(void); +int APS5_VABI sceNpPush2SendPushStatisticsDataSystemTelemetry(void); +int APS5_VABI sceNpPush2SetGlobalMutex(void); +int APS5_VABI sceNpPush2SetNpCommunicationId(void); +int APS5_VABI sceNpPush2Term(void); +int APS5_VABI sceNpPush2TriggerEmptyUserEvent(void); +int APS5_VABI sceNpPush2UnregisterDataType(void); +int APS5_VABI sceNpPush2UnregisterExtendedDataFilter(void); +int APS5_VABI sceNpPush2UnregisterNotificationExCallback(void); +int APS5_VABI sceNpPush2UnsetNpCommunicationId(void); +int APS5_VABI sceNpPush2WaitCallback(void); +int APS5_VABI sceNpPushCheckCallback(void); +int APS5_VABI sceNpPushInit(void); +int APS5_VABI sceNpPushIntBeginInactive(void); +int APS5_VABI sceNpPushIntEndInactive(void); +int APS5_VABI sceNpPushIntGetBindUserState(void); +int APS5_VABI sceNpPushIntGetConnectionState(void); +int APS5_VABI sceNpPushIntRegisterNotificationPacketCallback(void); +int APS5_VABI sceNpPushIntUnregisterNotificationPacketCallback(void); +int APS5_VABI sceNpPushRegisterExtendedDataFilter(void); +int APS5_VABI sceNpPushRegisterNotificationExCallback(void); +int APS5_VABI sceNpPushStartNotification(void); +int APS5_VABI sceNpPushStartNotificationA(void); +int APS5_VABI sceNpPushStopNotification(void); +int APS5_VABI sceNpPushStopNotificationA(void); +int APS5_VABI sceNpPushTerm(void); +int APS5_VABI sceNpPushUnregisterExtendedDataFilter(void); +int APS5_VABI sceNpPushUnregisterNotificationCallback(void); +int APS5_VABI sceNpRegisterStateCallbackForToolkit(void); +int APS5_VABI sceNpServiceClientInit(void); +int APS5_VABI sceNpServiceClientTerm(void); +int APS5_VABI sceNpSetAdditionalScope(void); +int APS5_VABI sceNpSetGamePresenceOnlineA(void); +int APS5_VABI sceNpSetNpTitleIdVsh(void); +int APS5_VABI sceNpUnregisterGamePresenceCallbackA(void); +int APS5_VABI sceNpUnregisterStateCallbackForToolkit(void); + } From a273ee4cd0d0811f259d72f40cd6139fc40cea80 Mon Sep 17 00:00:00 2001 From: TechLevelZero <35297505+TechLevelZero@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:09:54 +0000 Subject: [PATCH 223/318] fix(agc): restore the device filter in FlushAllPending and on the label store The last merge of main kept main's FlushAllPending definition, which took no device, so the build failed against the declaration and the idle and flip stores lost their filter. The definition now takes the device again and passes it to FlushPending, keeping the CPU publish scope for "idle". The label store passes its device as well, so with APS5_STORE_AT_LABEL=1 a label on one device no longer stores another device's pending images. The debt entry the merges dropped is back. agc_driver_two_device_pending_store_label runs the test with APS5_STORE_AT_LABEL=1: a label on the second device must leave the first device's results pending, and a label on the first device must store them. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 7 ++++ .../Execution/src/VulkanDevice.cpp | 2 +- .../libSceAgcDriver/Graphics/src/Texture.cpp | 4 +-- .../tests/execution/TwoDevicePendingStore.cpp | 34 ++++++++++++++++--- docs/dev/debt/libSceAgcDriver.md | 1 + 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 2df2e13bd9..adf7e2f949 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2779,6 +2779,13 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_guarded_direct_root PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_driver_two_device_pending_store_label COMMAND agc_driver_two_device_pending_store_tests --label) + set_tests_properties(agc_driver_two_device_pending_store_label PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_STORE_AT_LABEL=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_two_device_pending_store_label PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test(NAME agc_driver_bda_device_selection_cpu COMMAND agc_driver_bda_device_tests --device-selection-cpu) set_tests_properties(agc_driver_bda_device_selection_cpu PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 5edc1e1a14..7911882875 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -2498,7 +2498,7 @@ bool VulkanDevice::StoresPendingOver(std::uint64_t address, std::size_t bytes) c int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) { static const bool storeAtLabel = Graphics::StorageTexture::StoreAtLabelRequested(std::getenv("APS5_STORE_AT_LABEL")); - if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label"); + if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label", state->device); if (queue == 0) static_cast(Graphics::StorageTexture::StoreDrawnLinear("label")); Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before label"); if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 6833c98dde..68e7416757 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -4608,10 +4608,10 @@ void StorageTexture::FlushDumpSelected(const char* reason) { for (const auto& [address, bytes] : ranges) static_cast(FlushPending(address, bytes, nullptr, reason)); } -void StorageTexture::FlushAllPending(const char* reason) { +void StorageTexture::FlushAllPending(const char* reason, VkDevice device) { std::optional onCpu; if (reason != nullptr && std::strcmp(reason, "idle") == 0) onCpu.emplace(); - static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason)); + static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason, PublishScope::Whole, nullptr, false, device)); } std::size_t StorageTexture::StoreSmallPending(const char* reason) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/TwoDevicePendingStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/TwoDevicePendingStore.cpp index 37d7585295..1a7495b761 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/TwoDevicePendingStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/TwoDevicePendingStore.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -81,15 +83,28 @@ void FlushAll(const char* reason) { AgcDriver::Graphics::StorageTexture::FlushAllPending(reason); } +void Label(AgcDriver::VulkanDevice& device, const std::uint32_t* label) { + { + std::lock_guard gpuLock(AgcDriver::GuestMemory::GpuMutex()); + const std::uint32_t value = 1; + static_cast(device.WriteLabelOnGpu(static_cast(reinterpret_cast(label)), std::as_bytes(std::span(&value, 1)), 1, 0, false)); + device.SubmitRecorded(false); + } + device.WaitIdle(); } -int main() { +} + +int main(int argc, char** argv) { try { + const bool label = argc > 1 && std::strcmp(argv[1], "--label") == 0; + Require(!label || AgcDriver::Graphics::StorageTexture::StoreAtLabelRequested(std::getenv("APS5_STORE_AT_LABEL")), "two devices: --label needs APS5_STORE_AT_LABEL=1"); const auto first = OpenVulkanTestDevice(); if (!first) return VulkanTestSkipped; std::vector storage((BlockBytes + 0x10000u) / 4u); auto* texels = reinterpret_cast((reinterpret_cast(storage.data()) + 0xffffu) & ~std::uintptr_t{0xffffu}); std::fill(texels, texels + BlockBytes / 4u, Untouched); + std::array labelWords{}; { GuestAllocations::Mutation mutation; mutation.Add(texels, BlockBytes, true, true, true); @@ -113,10 +128,19 @@ int main() { second->WaitIdle(); Require(AgcDriver::Graphics::StorageTexture::AnyPending(), "two devices: the second device's idle store took the first device's pending results"); Require(!stored(), "two devices: the first device's results reached guest memory through the second device's idle store"); - FlushAll("memory access"); - first->WaitIdle(); - Require(!AgcDriver::Graphics::StorageTexture::AnyPending(), "two devices: a flush with the second device active left the first device's results pending"); - Require(stored(), "two devices: a flush with the second device active did not bring the first device's results to guest memory"); + if (label) { + Label(*second, labelWords.data()); + Require(AgcDriver::Graphics::StorageTexture::AnyPending(), "two devices: the second device's label store took the first device's pending results"); + Require(!stored(), "two devices: the first device's results reached guest memory through the second device's label store"); + Label(*first, labelWords.data()); + Require(!AgcDriver::Graphics::StorageTexture::AnyPending(), "two devices: the first device's label store left its results pending"); + Require(stored(), "two devices: the first device's label store did not bring its results to guest memory"); + } else { + FlushAll("memory access"); + first->WaitIdle(); + Require(!AgcDriver::Graphics::StorageTexture::AnyPending(), "two devices: a flush with the second device active left the first device's results pending"); + Require(stored(), "two devices: a flush with the second device active did not bring the first device's results to guest memory"); + } second->WaitIdle(); } { diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index aa55ae063e..e19fc98df1 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -140,6 +140,7 @@ - On a device without VK_EXT_robustness2's `nullDescriptor`, such as MoltenVK, a null image descriptor in a typed heap is a [placeholder image](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PlaceholderImages.cpp) of that heap's type, 1×1 and cleared to zero, which every null descriptor of the heap shares. Reads give zeros, as `nullDescriptor` allows, but stores and atomics to it are kept, so a later read of a null storage image can return them, and a depth comparison compares against depth 0. A heap whose placeholder the device cannot create, for example 1D or 3D depth, 64-bit atomics without 64-bit image atomics, or multisampled storage without `shaderStorageImageMultisample`, throws when a null descriptor reaches it. - [Draw input snapshots](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver): `PrepareDrawBindings` caps read-only input buffer snapshots at 64 MiB (overridable via `APS5_DRAW_SNAPSHOT_MAX_MIB`). Larger inputs (streaming heaps, bindless pools) or draws exceeding the cumulative budget skip the per-draw snapshot and bind in place, avoiding multi-gigabyte CPU copy stalls (such as Ratchet & Clank: Rift Apart PPSA01474 stalling at frame ~380 copying 36 GB every 5 s). If the title rewrites those large ranges on the CPU before GPU completion, the in-place binding observes the modification, as verified by `drawSnapshotCapTests`. - [Pending storage-image results](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver), except drawn resident linear targets (next entry), are stored when something reads their memory, not when the GPU work that wrote them completes. A title that waits for that work, frees the image and reuses its memory can have the new data overwritten by the late store (#1970). A title's own CPU reads are not seen either, so a title that reads the results after waiting for a label reads stale memory. Explicit-size file writes (`write`, `pwrite`, `writev`, `pwritev`, AIO and `fwrite`) synchronize their source ranges through the [host-read hook](../../../core/libs/prx/libc/src/GuestArena.cpp); other host readers are not covered. `APS5_STORE_AT_FLIP=1` stores every pending result at each flip and `APS5_STORE_AT_LABEL=1` before each label the GPU writes; both store every pending image, not only the ones the waited-for work wrote, and neither is on by default. +- [The active recorder](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp) (libSceAgcDriver) is one per process, and `Recorder::Activate` switches it without the GPU mutex. With two live devices (tests that open their own device next to the driver's), a device's idle, label and flip stores take only its own pending storage images, and a storage-image write-back records into the active recorder only when that recorder is its device's, otherwise in a submission of its own. Other users of `Recorder::Active()` don't check the device, so they can still record one device's work on the other's command buffer. - [Resident linear color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver): a CPU store inside a drawn scissor rectangle, on a pixel no primitive wrote, made before the target's store is overwritten by it, where the console keeps it. Recorded draws' results are stored before the graphics queue's next flip, label, end-of-pipe interrupt, drain or device wait, as the union of their scissor rectangles; a draw that brings a 17th rectangle that does not merge stores the target at once. A target with an attachment proxy, several mips or layers, a 3D or DCC surface, or results of a dispatch, fill or copy is stored after each draw. One that a fill or copy writes after its draws is stored whole at the next of those points, and GPU results in its CPU-written 64 KiB blocks are dropped. A store the draw pipeline's committer makes (label, end-of-pipe interrupt, device wait) skips a target whose range a queued draw still reads (`DrawPipeline::HeldReads`). - [Direct writes over a drawn linear target](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver): a CPU store into a 64 KiB block of a drawn resident linear target that lands while the completion of a direct write recorded before the draws checks and stamps that block (`StampKeepingNewerResults`) drops the draws' results in that block, where the console keeps them: the stamp's own collect records the store per page only, so the store reads the block as written by the driver. - [Linear color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) with no host import are drawn synchronously (copied in, drawn, waited for, copied back), where the console renders into memory in stream order, unless the draw can make an import that keeps them write-watched. It makes one for a target in registered, writable memory only when the whole import run is write-watched and the import stays watched: a separate alias of shared memory, a direct import of 1 MiB or more of private memory that settles on Windows (when the import probe keeps the watch), or one the probe keeps watched elsewhere. Smaller private ranges on Windows, runs with memory outside the write watch and devices whose imports are compared stay synchronous: a compared import makes every refresh of the resident image compare the whole target. A target over an existing import is resident, compared or not. `APS5_NO_RESIDENT_LINEAR=1` keeps every linear target synchronous. From dcdf9f35d5e3b96f8ac86d68b927f6891d39197e Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 26 Oct 2026 14:26:56 +0100 Subject: [PATCH 224/318] fix(libkernel): drop the kqueue and knotes of a descriptor that dup2 replaces with a socket dup2 from a guest socket notified the event queues only when the replaced descriptor was itself a guest socket. A kqueue replaced this way stayed registered, so kevent on the number that now names the socket still returned the old queue's events, and knotes registered on a replaced pipe or file stayed attached. FreeBSD 11.4's kern_dup closes the replaced file through closefp, whose knote_fdclose drops every knote on the descriptor, as the other dup2 paths here already do. Fixes #11429 Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libkernel/Socket/src/Udp.cpp | 2 +- core/libs/tests/GuestKqueue.cpp | 25 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libkernel/Socket/src/Udp.cpp b/core/libs/prx/libkernel/Socket/src/Udp.cpp index 0ae0c29327..f1cc80860a 100644 --- a/core/libs/prx/libkernel/Socket/src/Udp.cpp +++ b/core/libs/prx/libkernel/Socket/src/Udp.cpp @@ -731,7 +731,7 @@ int GuestSockets::DuplicateTo(int descriptor, int target) { } } } - if (replaced) EqueueDescriptorClosed(target); + EqueueDescriptorClosed(target); { std::lock_guard lock(socketsMutex); closeOnExecDescriptors.erase(target); diff --git a/core/libs/tests/GuestKqueue.cpp b/core/libs/tests/GuestKqueue.cpp index c421f32f45..cf542bb265 100644 --- a/core/libs/tests/GuestKqueue.cpp +++ b/core/libs/tests/GuestKqueue.cpp @@ -148,6 +148,30 @@ static void CreateAndClose() { Require(close_nid_postfix(pipe[0]) == 0 && close_nid_postfix(pipe[1]) == 0); } +static void SocketReplacesDescriptor() { + const int kq = kqueue_nid_postfix(); + int marker = 0; + Require(Apply(kq, Change(1, EvfiltTimer, EvAdd | EvOneshot, 0, 1, &marker)) == 0); + int pair[2]; + Require(socketpair_nid_postfix(Unix, Stream, 0, pair) == 0); + Require(dup2_nid_postfix(pair[0], kq) == kq); + KernelEvent event{}; + RequireFailure(Wait(kq, &event, 1), GuestEbadf); + Require(close_nid_postfix(kq) == 0); + const int watcher = kqueue_nid_postfix(); + int pipe[2]; + Require(pipe_nid_postfix(pipe) == 0); + Require(write_nid_postfix(pipe[1], "x", 1) == 1); + Require(Apply(watcher, Change(static_cast(pipe[0]), EvfiltRead, EvAdd)) == 0); + Require(Poll(watcher, &event, 1) == 1); + Require(dup2_nid_postfix(pair[1], pipe[0]) == pipe[0]); + Require(Poll(watcher, &event, 1) == 0); + RequireFailure(Apply(watcher, Change(static_cast(pipe[0]), EvfiltRead, EvDelete)), GuestEnoent); + Require(close_nid_postfix(pipe[0]) == 0 && close_nid_postfix(pipe[1]) == 0); + Require(close_nid_postfix(pair[0]) == 0 && close_nid_postfix(pair[1]) == 0); + Require(close_nid_postfix(watcher) == 0); +} + static void Timers() { const int kq = kqueue_nid_postfix(); int marker = 0; @@ -1584,6 +1608,7 @@ int main(int argc, char** argv) { StandardInputReplacement(); StandardInput(); CreateAndClose(); + SocketReplacesDescriptor(); Timers(); TimerReadd(); UnsupportedFilters(); From 9b2860bb471aa1773b45f842ea2da2bfb4c8aa58 Mon Sep 17 00:00:00 2001 From: Michael Krumland-Dunning Date: Sun, 25 Oct 2026 16:15:32 -0700 Subject: [PATCH 225/318] feat(agc): honour the DEPTH row pitch of linear 2D textures as RDNA2 does A linear 2D T# whose DEPTH field named a row pitch other than its default threw. On RDNA2 (gfx1036, hardware oracle, 135 image_load kernels over 8, 16, 32 and 64-bit texels) the row pitch is DEPTH + 1 elements, raised to the width and rounded up to 256 bytes. A single-level uncompressed linear 2D texture now records that pitch and lays its rows out with it; the texture cache, storage-to-sampled copies and surface shape checks keep textures of different pitches apart. Block-compressed, mipmapped and base-array surfaces with a pitch still throw. Co-Authored-By: Claude Opus 5.5 --- .../Graphics/include/GuestTextureResource.hpp | 1 + .../Graphics/include/TextureTiling.hpp | 2 +- .../Graphics/src/GuestTextureResource.cpp | 8 +- .../libSceAgcDriver/Graphics/src/Texture.cpp | 4 +- .../Graphics/src/TextureCache.cpp | 2 +- .../Graphics/src/TextureTiling.cpp | 13 ++- .../tests/GuestTextureResource.cpp | 37 +++++- .../tests/execution/LinearTexturePitch.cpp | 109 ++++++++++++++++++ docs/dev/debt/libSceAgcDriver.md | 1 + 9 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/LinearTexturePitch.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp index d69fcb7dc6..7c4f314a6f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp @@ -109,6 +109,7 @@ struct GuestTextureResource { std::uint32_t gcnPitch = 0; bool gcnPow2Pad = false; bool gcnLinearGeneral = false; + std::uint32_t linearPitch = 0; }; constexpr bool IsGcnMicroTiled(TextureTileMode mode) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp index 8b27373458..1d4a5330df 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp @@ -26,7 +26,7 @@ struct TileMipLayout { std::uint32_t gcnSplitStride = 0; }; -std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount); +std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount, std::uint32_t linearPitch = 0u); std::uint64_t ComputeSurfaceSize(const std::vector& mips, std::uint32_t arrayLayers); // Mip chain of an uncompressed surface given its element size (color targets have no texture format). std::vector ComputeElementMipLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp index 7e47d60a47..b1db861f92 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp @@ -247,15 +247,18 @@ GuestTextureResource DecodeTextureResource(std::span descri Require(baseAddress != 0, "guest texture descriptor has a null base address"); } + std::uint32_t linearPitch = 0; switch (dimension) { case TextureDimension::k1D: Require(height == 1 && depth == 0 && baseArray == 0, "guest 1D texture descriptor has a nonzero height, depth or base array"); break; case TextureDimension::k2D: if (depth != 0 && tileMode == TextureTileMode::kLinear) { + Require(!IsBlockCompressed(format) && maxMip == 0 && baseArray == 0, "guest 2D linear texture descriptor sets a pitch on a block-compressed, mipmapped or base-array surface, which is not implemented" + describe()); const auto elementsPerPitchUnit = 256u / BytesPerElement(format); const auto defaultPitch = (width + elementsPerPitchUnit - 1u) / elementsPerPitchUnit * elementsPerPitchUnit; - Require(!IsBlockCompressed(format) && maxMip == 0 && baseArray == 0 && depth + 1u == defaultPitch, "guest 2D linear texture descriptor sets a custom pitch other than its default pitch, which is not implemented" + describe()); + const auto pitch = (std::max(depth + 1u, width) + elementsPerPitchUnit - 1u) / elementsPerPitchUnit * elementsPerPitchUnit; + linearPitch = pitch == defaultPitch ? 0u : pitch; depth = 0; } Require(depth == 0 && baseArray == 0, "guest 2D texture descriptor has a nonzero depth or base array"); @@ -314,6 +317,7 @@ GuestTextureResource DecodeTextureResource(std::span descri result.gcnLinearGeneral = ShaderRecompiler::GcnImageTilingIndex(gcnTiling) == 31u; Require(result.gcnPitch >= width, "PS4 texture descriptor has a pitch below its width" + describe()); } + result.linearPitch = linearPitch; if (baseLevel > maxMip) { const bool fits = LevelsFitAllocation(result, lastLevel + 1u); if (!fits && ZeroTail(words)) { @@ -328,7 +332,7 @@ GuestTextureResource DecodeTextureResource(std::span descri std::array EncodeTextureResource(const GuestTextureResource& resource) { const bool multisampled = resource.dimension == TextureDimension::k2DMsaa; Require((resource.dimension == TextureDimension::k2D && resource.samples == 1u) || (multisampled && resource.samples >= 2u && resource.samples <= 16u && std::has_single_bit(resource.samples)), "only 2D and multisampled 2D surfaces have a texture descriptor encoding"); - Require(resource.mipCount == 1u && resource.allocatedMipCount <= 1u && resource.baseLevel == 0u && resource.lastLevel == 0u && resource.depthOrLastArray == 0u && resource.baseArray == 0u && resource.dccAddress == 0u && resource.minLod == 0u && !resource.iterate256 && resource.bcSwizzle == 0u, "only single-level, single-layer surfaces without metadata have a texture descriptor encoding"); + Require(resource.mipCount == 1u && resource.allocatedMipCount <= 1u && resource.baseLevel == 0u && resource.lastLevel == 0u && resource.depthOrLastArray == 0u && resource.baseArray == 0u && resource.dccAddress == 0u && resource.minLod == 0u && !resource.iterate256 && resource.bcSwizzle == 0u && resource.linearPitch == 0u, "only single-level, single-layer surfaces without metadata have a texture descriptor encoding"); Require(resource.width >= 1u && resource.width <= 0x4000u && resource.height >= 1u && resource.height <= 0x4000u && resource.format < 0x200u, "the surface's extent or format has no texture descriptor encoding"); const auto xorMask = resource.tileMode == TextureTileMode::kS4KBX || resource.tileMode == TextureTileMode::kD4KBX ? 0xfffull : 0xffffull; Require(XorSwizzleMode(resource.tileMode) != 0 ? (resource.baseAddress & xorMask) == 0 && (resource.pipeBankXor & ~xorMask) == 0 : resource.pipeBankXor == 0, "the surface's pipe and bank XOR has no texture descriptor encoding"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 8edbe40695..61c03baf9f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1054,7 +1054,7 @@ bool Texture::CanCopyFrom(const StorageTexture& source, const GuestTextureResour // Same memory, same layout, same texel size: the GPU copy reinterprets the texels exactly as a // guest read through the sampled descriptor would. if (IsRepackedTextureFormat(descriptor.format)) return false; - return descriptor.baseAddress == from.baseAddress && descriptor.pipeBankXor == from.pipeBankXor && descriptor.width == from.width && descriptor.height == from.height && ArrayLayersServeView(from, descriptor) && descriptor.tileMode == from.tileMode && descriptor.mipCount <= from.mipCount && BytesPerElement(descriptor.format) == BytesPerElement(from.format) && BlockWidth(descriptor.format) == BlockWidth(from.format); + return descriptor.baseAddress == from.baseAddress && descriptor.pipeBankXor == from.pipeBankXor && descriptor.width == from.width && descriptor.height == from.height && descriptor.linearPitch == from.linearPitch && ArrayLayersServeView(from, descriptor) && descriptor.tileMode == from.tileMode && descriptor.mipCount <= from.mipCount && BytesPerElement(descriptor.format) == BytesPerElement(from.format) && BlockWidth(descriptor.format) == BlockWidth(from.format); } Texture::Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components) : context(context), storageSource(source) { @@ -5499,7 +5499,7 @@ std::shared_ptr StorageTexture::FindLive(std::uint64_t address, bool StorageTexture::SameSurfaceShape(const StorageTexture& other) const { const auto& mine = descriptor; const auto& theirs = other.descriptor; - return guestBytes == other.guestBytes && mine.width == theirs.width && mine.height == theirs.height && mine.depthOrLastArray == theirs.depthOrLastArray && mine.baseArray == theirs.baseArray && mine.mipCount == theirs.mipCount && mine.tileMode == theirs.tileMode && mine.dimension == theirs.dimension && mine.format == theirs.format && storageFormat == other.storageFormat && geometry.imageLayers == other.geometry.imageLayers && geometry.imageDepth == other.geometry.imageDepth; + return guestBytes == other.guestBytes && mine.width == theirs.width && mine.height == theirs.height && mine.linearPitch == theirs.linearPitch && mine.depthOrLastArray == theirs.depthOrLastArray && mine.baseArray == theirs.baseArray && mine.mipCount == theirs.mipCount && mine.tileMode == theirs.tileMode && mine.dimension == theirs.dimension && mine.format == theirs.format && storageFormat == other.storageFormat && geometry.imageLayers == other.geometry.imageLayers && geometry.imageDepth == other.geometry.imageDepth; } bool StorageTexture::ImageCoversGuestBytes() const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp index 2e31ea6c75..8f8bb3c414 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp @@ -38,7 +38,7 @@ std::shared_ptr TextureCache::Get(std::span words, entries.erase(it); break; } - const auto mips = ComputeMipLayout(resource.tileMode, resource.format, resource.width, resource.height, resource.mipCount); + const auto mips = ComputeMipLayout(resource.tileMode, resource.format, resource.width, resource.height, resource.mipCount, resource.linearPitch); const auto layers = resource.dimension == TextureDimension::k2DArray || resource.dimension == TextureDimension::k1DArray || resource.dimension == TextureDimension::kCube ? resource.depthOrLastArray + 1u : 1u; const auto bytes = ComputeSurfaceSize(mips, layers); Require(bytes != 0 && bytes <= std::numeric_limits::max(), "texture cache surface size overflow"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp index 0d5bd91dca..c91f3c2c06 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp @@ -163,8 +163,9 @@ std::uint32_t TexelLevelDimension(std::uint32_t guestDimension, std::uint32_t le return std::max((std::max(guestDimension >> level, 1u) + texelScale - 1u) / texelScale, 1u); } -std::vector ComputeLinearMipLayout(std::uint32_t bytesPerElement, std::uint32_t texelWidth, std::uint32_t texelHeight, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount) { +std::vector ComputeLinearMipLayout(std::uint32_t bytesPerElement, std::uint32_t texelWidth, std::uint32_t texelHeight, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount, std::uint32_t linearPitch) { const auto compressed = texelWidth != 1u || texelHeight != 1u; + Require(linearPitch == 0u || (!compressed && mipCount == 1u && linearPitch >= width), "a linear pitch needs a single-level uncompressed surface no wider than the pitch"); const auto elementsWidth0 = (width + texelWidth - 1u) / texelWidth; const auto elementsHeight0 = (height + texelHeight - 1u) / texelHeight; const auto blockWidth = CalcLinearBlockWidth(bytesPerElement); @@ -174,7 +175,7 @@ std::vector ComputeLinearMipLayout(std::uint32_t bytesPerElement, for (auto level = mipCount; level-- > 0;) { const auto elementsLevelWidth = std::max(ShiftCeil(elementsWidth0, level), 1u); const auto elementsLevelHeight = std::max(ShiftCeil(elementsHeight0, level), 1u); - const auto paddedElementsWidth = AlignUp(elementsLevelWidth, blockWidth); + const auto paddedElementsWidth = linearPitch != 0u ? linearPitch : AlignUp(elementsLevelWidth, blockWidth); const auto size = static_cast(paddedElementsWidth) * elementsLevelHeight * bytesPerElement; Require(size != 0, "computed a zero-sized linear texture mip level"); @@ -277,7 +278,7 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: } -std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount) { +std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount, std::uint32_t linearPitch) { Require(width != 0 && height != 0, "cannot compute mip layout for a zero-sized texture"); Require(mipCount != 0 && mipCount <= 16u, "texture mip count is out of range"); @@ -286,7 +287,7 @@ std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint3 const auto texelWidth = BlockWidth(format); const auto texelHeight = BlockHeight(format); - if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, texelWidth, texelHeight, width, height, mipCount); + if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, texelWidth, texelHeight, width, height, mipCount, linearPitch); return ComputeTiledMipLayout(tileMode, bytesPerElement, texelWidth, texelHeight, width, height, mipCount); } @@ -634,7 +635,7 @@ SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor) { geometry.mips.push_back({0u, geometry.guestBytes, 0u, geometry.sliceLinearBytes, descriptor.width, descriptor.height, static_cast(align(descriptor.width, blockWidth) / blockWidth), descriptor.width * descriptor.samples * bytesPerElement, false, 0u, 0u}); return geometry; } - geometry.mips = ComputeMipLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, descriptor.mipCount); + geometry.mips = ComputeMipLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, descriptor.mipCount, descriptor.linearPitch); const bool layered = descriptor.dimension == TextureDimension::k2DArray || descriptor.dimension == TextureDimension::k1DArray || descriptor.dimension == TextureDimension::kCube; geometry.layers = layered ? descriptor.depthOrLastArray + 1u : 1u; geometry.imageLayers = geometry.layers; @@ -708,7 +709,7 @@ GuestTextureResource ArrayLayerWindow(const GuestTextureResource& viewed, std::u std::vector ComputeElementMipLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount) { Require(width != 0 && height != 0 && mipCount != 0 && mipCount <= 16u, "invalid surface mip chain"); Require(!IsGcnTiled(tileMode), "PS4 surfaces are laid out by ComputeGcnMipLayout from their pitch"); - if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, 1u, 1u, width, height, mipCount); + if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, 1u, 1u, width, height, mipCount, 0u); return ComputeTiledMipLayout(tileMode, bytesPerElement, 1u, 1u, width, height, mipCount); } diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp index 22eb2c0eb2..e2d70710c2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp @@ -286,7 +286,8 @@ void RunGuestTextureResourceTests() { Fields twoD = base; twoD.depth = 1; - rejectFields(twoD, "custom pitch other than its default pitch"); + result = DecodeTextureResource(pack(twoD)); + Require(result.depthOrLastArray == 0 && result.linearPitch == 0, "linear 2D descriptor with a pitch below its width did not keep the default pitch"); twoD.tileModeRaw = 0x09; rejectFields(twoD, "nonzero depth or base array"); Fields pitched = base; @@ -294,18 +295,42 @@ void RunGuestTextureResourceTests() { pitched.height = 72; pitched.depth = 127; result = DecodeTextureResource(pack(pitched)); - Require(result.dimension == TextureDimension::k2D && result.width == 72 && result.depthOrLastArray == 0, "linear 2D descriptor with its default pitch in DEPTH decoded incorrectly"); + Require(result.dimension == TextureDimension::k2D && result.width == 72 && result.depthOrLastArray == 0 && result.linearPitch == 0, "linear 2D descriptor with its default pitch in DEPTH decoded incorrectly"); pitched.width = 8; pitched.height = 8; pitched.depth = 63; result = DecodeTextureResource(pack(pitched)); - Require(result.width == 8 && result.depthOrLastArray == 0, "linear 2D descriptor with a one-unit default pitch decoded incorrectly"); - pitched.depth = 127; - rejectFields(pitched, "custom pitch other than its default pitch"); + Require(result.width == 8 && result.depthOrLastArray == 0 && result.linearPitch == 0, "linear 2D descriptor with a one-unit default pitch decoded incorrectly"); + for (const auto& [depth, pitch] : std::array, 4>{{{64u, 128u}, {127u, 128u}, {128u, 192u}, {319u, 320u}}}) { + pitched.depth = depth; + result = DecodeTextureResource(pack(pitched)); + Require(result.width == 8 && result.depthOrLastArray == 0 && result.linearPitch == pitch, "linear 2D descriptor did not round its DEPTH pitch up to 256 bytes"); + const auto geometry = DescribeSurface(result); + Require(geometry.mips.size() == 1 && geometry.mips[0].pitchBytes == pitch * 4u && geometry.mips[0].blocksPerRow == pitch && geometry.guestBytes == pitch * 4u * 8u, "linear 2D surface with a custom pitch has the wrong layout"); + bool encoded = true; + try { + static_cast(EncodeTextureResource(result)); + } catch (const std::exception&) { + encoded = false; + } + Require(!encoded, "a linear 2D surface with a custom pitch was encoded without it"); + auto defaultPitch = result; + defaultPitch.linearPitch = 0; + static_cast(EncodeTextureResource(defaultPitch)); + } + Fields narrow = base; + narrow.format = 5; + narrow.width = 300; + narrow.depth = 511; + Require(DecodeTextureResource(pack(narrow)).linearPitch == 0, "linear 2D 8-bit descriptor with its default pitch in DEPTH set a custom pitch"); + narrow.depth = 767; + Require(DecodeTextureResource(pack(narrow)).linearPitch == 768, "linear 2D 8-bit descriptor did not keep its DEPTH pitch"); + narrow.depth = 768; + Require(DecodeTextureResource(pack(narrow)).linearPitch == 1024, "linear 2D 8-bit descriptor did not round its DEPTH pitch up to 256 bytes"); pitched.depth = 63; pitched.maxMip = 1; pitched.lastLevel = 1; - rejectFields(pitched, "custom pitch other than its default pitch"); + rejectFields(pitched, "pitch on a block-compressed, mipmapped or base-array surface"); twoD = base; twoD.baseArray = 1; rejectFields(twoD, "nonzero depth or base array"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LinearTexturePitch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTexturePitch.cpp new file mode 100644 index 0000000000..28e4b2822e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTexturePitch.cpp @@ -0,0 +1,109 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Width = 16; +constexpr std::uint32_t Height = 8; +constexpr std::uint32_t Groups = Width * Height / Threads; +constexpr std::uint32_t MaxPitch = 256; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t SwizzleXYZW = 0xfacu; +constexpr std::uint32_t Mark = 0xa0000000u; + +alignas(256) std::array Output{}; +alignas(256) std::array Texels{}; + +alignas(256) constexpr std::array LoadXy{ + 0x7e080218u, 0x343c0885u, 0x4a3c3d00u, 0x34063c84u, 0x2c3e3c84u, 0x363c3c8fu, 0x7e140280u, 0x7e160280u, + 0x7e180280u, 0x7e1a0280u, 0xf0001f08u, 0x00010a1eu, 0xbf8c3f70u, 0xe0781000u, 0x80000a03u, 0xbf810000u, +}; + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::array TextureDescriptor(std::uint32_t depth) { + const auto address = static_cast(reinterpret_cast(Texels.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Width - 1u) & 3u) << 30u), + ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), + SwizzleXYZW | (Type2D << 28u), + depth, + 0u, + 0u, + 0u, + }; +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t depth) { + Output.fill(0xdeadbeefu); + std::vector userData(24, 0u); + const auto outputAddress = reinterpret_cast(Output.data()); + const std::array output{static_cast(outputAddress), static_cast((outputAddress >> 32u) & 0xffffu), static_cast(Output.size() * 4u), 0x31016facu}; + const auto texture = TextureDescriptor(depth); + std::copy(output.begin(), output.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + const std::span code = LoadXy; + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {true, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, Groups, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(AgcDriver::VulkanDevice& device, std::uint32_t depth, std::uint32_t pitch) { + Run(device, depth); + for (std::uint32_t index = 0; index < Width * Height; ++index) { + const auto x = index % Width; + const auto y = index / Width; + const auto expected = Mark | (y * pitch + x); + const auto actual = Output[index * 4u]; + Require(actual == expected, "linear 2D texture with DEPTH " + std::to_string(depth) + ": texel (" + std::to_string(x) + ", " + std::to_string(y) + ") is " + Hex(actual) + ", expected " + Hex(expected) + " (row pitch " + std::to_string(pitch) + ")"); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + for (std::uint32_t index = 0; index < Texels.size(); ++index) Texels[index] = Mark | index; + for (const auto& [depth, pitch] : std::array, 6>{{{15u, 64u}, {63u, 64u}, {64u, 128u}, {127u, 128u}, {128u, 192u}, {255u, 256u}}}) { + Check(*device, depth, pitch); + } + std::puts("linear texture pitch tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index aa55ae063e..c60ee0791e 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -197,6 +197,7 @@ - [Host imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) on Linux skip ranges that are not mapped read-write, which are copied instead. The NVIDIA driver (595.91.07) refuses `VK_EXT_external_memory_host` imports of read-only or executable mappings (and shared mappings of regular files) with `VK_ERROR_OUT_OF_DEVICE_MEMORY`, after which every `vkAllocateMemory` on the device fails the same way. Read-only ranges are never imported, even on drivers that accept them. Shared mappings of regular files are not checked for. On Windows, an alias of adjacent registered ranges is refused if any page is not a shared view; a range made entirely of shared views is aliased run by run, including across sections or nonconsecutive section offsets. Adjacent registered ranges are joined into one import: the largest aligned power-of-two block, from twice the import alignment up to the 8 MiB window, that holds the requested address and that they fill exactly, so a block only grows by doubling as its window fills. A block that would exceed the budget, would replace an import a build holds, or fails (a refused Windows alias, `vkAllocateMemory`) steps down to the next smaller block; a failed block is not tried again until one of its ranges changes. Ranges larger than the window are imported one at a time. A buffer bound across adjacent ranges that are each larger than the window, or bound over adjacent ranges of any size from the first one's start (ending anywhere in the last range it crosses), joins the ranges it crosses into one import when it is bound, and their own imports are retired; a buffer that crosses any other two imports is copied. A heap grown in pieces re-imports its joined run only when a buffer bound from its first piece's start crosses into a piece the joined import does not cover, or when a sparse descriptor heap over it is bound: a heap a shader binds whole while only the pieces from its base are mapped is bound in place from the import of that run, its views ending at the import's end as the copy's do, once the previous bind of that address saw the same last mapped byte (a heap that grows between every two of its binds stays copied; one bound more often than it grows is joined again after each growth) and at most 64 times per joined run base; past that, with a message, it is not joined again: it is bound in place while an existing import covers its run and copied otherwise, until no registered range starts at that base or the device changes. Ranges larger than the window are taken to be registered whole and never grown in pieces: a title that grows a heap in pieces larger than 8 MiB re-imports the joined run each time a bound buffer crosses into a new piece. The 8 MiB window is a choice: unbounded runs were re-imported whole on every growth of a heap mapped in small pieces, and the NVIDIA driver (610.57.04, 615.78.08) then refused imports or lost the device. Not measured on RADV. - [Cube texture descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) of two to five slices are only sampled: the [materializer](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) rejects loads, stores and dimension queries through them, which on RDNA2 stop at `DEPTH` while the view holds six faces. Out-of-range cube-array sampling is unimplemented; PS5 equivalence is unmeasured. A range of fewer than six slices samples six faces from `BASE_ARRAY` (measured with `tools/hw-oracle` on gfx1036 for two to five slices, and on RX 6600, gfx1032, for equal `BASE_ARRAY` and `DEPTH`); one of six or more keeps every slice up to `DEPTH`. Samplers with `DISABLE_CUBE_WRAP` are created with `VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT`, the flag RADV maps to that bit (Mesa `src/amd/vulkan/radv_sampler.c`, `src/amd/common/ac_descriptors.c`); without `VK_EXT_non_seamless_cube_map` a cube sampled through such a sampler throws. - [Array layer views](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (libSceAgcDriver) - `ArrayLayersServeView` lets a 2D texture at a 2D array image's base address, or a 2D array texture over its first layers, view that image (`Texture::CanCopyFrom`). Mesa-built homebrew (ps5-opengl) writes the view's last slice plus one to `CB_COLOR_ATTRIB3.MIP0_DEPTH`, so a slice-0 render target is a two-layer array image, and the title samples it as a plain 2D texture (SuperTuxKart PPSA69421 drew a black 3D scene and minimap without this). The per-layer layout of a mipmapped array is assumed equal to a standalone surface's; it is not measured on the console. A 2D texture at a later slice's address still gets an image of its own. +- [Linear 2D texture pitch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (libSceAgcDriver) - the PS5's gfx1013 is not measured, and a `DEPTH` pitch on a block-compressed, mipmapped or base-array linear 2D T# throws. A single-level uncompressed linear 2D T# uses `DEPTH` + 1 as its row pitch in elements, raised to the width and rounded up to 256 bytes, as `image_load` read it on RDNA2 (gfx1036, AMD Windows driver 32.0.21030.31, [hardware oracle](../HW_ORACLE.md)) in 135 kernels over 8_UINT, 16_UINT, 32_UINT and 32_32_UINT, widths 16 to 300 and pitches of 1 to 2048 elements. - [Multisampled CMASK](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - a multisampled color target's CMASK is decoded under CB_COLOR_INFO FAST_CLEAR (bit 13, as for a single-sample target) or COMPRESSION (bit 14). Which bit a multisampled target sets is inferred: its fast clears were seen on GTA V's multisampled targets through their CMASK fills, not in a register dump. A COMPRESSION-only target's zero CMASK is not read as cleared (inferred: zero means cleared only under FAST_CLEAR), so only a fill after its first use clears it. A slice of a multisampled array is its own image at its own address with its own CB_COLOR0_CMASK (inferred from the register layout). Its CMASK is main's whole-metablock size. - [Compressed FMASK reads](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) without an exact resident native owner throw. Native reads require eight fragments, one slice and mip, pipe-aligned DCC/CMASK, Z_64KB_X FMASK and unchanged, disjoint planes, each watched or in a host import the Linux write-watch probe compares (kept as a copy and compared at most once per collect epoch); an FMASK import awaiting its Windows deferred read is read first (`WatchedAfterSettle`). The host stores independent samples and exposes their expanded indices (`0x76543210`, AMD PAL `c5e80007`, `Gfx9Fmask::GetPackedExpandedValue`); console equivalence is unmeasured. Other layouts, standalone compressed decoding, guest publication and recipe reuse remain unsupported. Metadata writes invalidate the native view, including same-value stores, except a same-value CPU store to a compared plane. - [Multisampled CMASK state](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) (libSceAgcDriver) - a multisampled target has no guest texels, so its CMASK state comes from fills rather than from a read at each draw as a single-sample target's does. A fill of the whole CMASK with 0 clears the host image at the target's next use, one with 0xcccccccc, 0xdddddddd, 0xeeeeeeee or 0xffffffff marks it expanded, and any other pattern, or a fill that covers only part of the CMASK, throws at the next use (inferred: main's rule that a mixed CMASK is unmodeled, applied to the one host image). A FAST_CLEAR target starts from its CMASK keys at its first use, and a COMPRESSION-only target starts expanded. Copies by DMA into the CMASK are not seen, and a later use with another CMASK address throws. The clear does not rewrite the CMASK bytes as expanded, so a new target over the same CMASK (a different extent, so a different image) seeds from them again and clears once more. From 0b614156b764b3c6f5c05c57f3edea6e8bbfae26 Mon Sep 17 00:00:00 2001 From: Michael Krumland-Dunning Date: Sun, 25 Oct 2026 16:56:50 -0700 Subject: [PATCH 226/318] fix(agc): keep the default-pitch rule for linear 2D textures of PS4 guests The DEPTH row pitch was measured on RDNA2 only. A PS4 (GCN) guest's linear 2D T# keeps the rule main had: only the default pitch decodes, any other pitch throws. Its layout comes from its own descriptor fields, so the RDNA pitch does not apply to it. Co-Authored-By: Claude Opus 5.5 --- .../prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp | 1 + docs/dev/debt/libSceAgcDriver.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp index b1db861f92..b1e5044131 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp @@ -257,6 +257,7 @@ GuestTextureResource DecodeTextureResource(std::span descri Require(!IsBlockCompressed(format) && maxMip == 0 && baseArray == 0, "guest 2D linear texture descriptor sets a pitch on a block-compressed, mipmapped or base-array surface, which is not implemented" + describe()); const auto elementsPerPitchUnit = 256u / BytesPerElement(format); const auto defaultPitch = (width + elementsPerPitchUnit - 1u) / elementsPerPitchUnit * elementsPerPitchUnit; + Require(!ShaderRecompiler::GcnGuestShaders() || depth + 1u == defaultPitch, "guest 2D linear texture descriptor of a PS4 guest sets a custom pitch other than its default pitch, which is not implemented" + describe()); const auto pitch = (std::max(depth + 1u, width) + elementsPerPitchUnit - 1u) / elementsPerPitchUnit * elementsPerPitchUnit; linearPitch = pitch == defaultPitch ? 0u : pitch; depth = 0; diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index c60ee0791e..3f9fe483bb 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -197,7 +197,7 @@ - [Host imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) on Linux skip ranges that are not mapped read-write, which are copied instead. The NVIDIA driver (595.91.07) refuses `VK_EXT_external_memory_host` imports of read-only or executable mappings (and shared mappings of regular files) with `VK_ERROR_OUT_OF_DEVICE_MEMORY`, after which every `vkAllocateMemory` on the device fails the same way. Read-only ranges are never imported, even on drivers that accept them. Shared mappings of regular files are not checked for. On Windows, an alias of adjacent registered ranges is refused if any page is not a shared view; a range made entirely of shared views is aliased run by run, including across sections or nonconsecutive section offsets. Adjacent registered ranges are joined into one import: the largest aligned power-of-two block, from twice the import alignment up to the 8 MiB window, that holds the requested address and that they fill exactly, so a block only grows by doubling as its window fills. A block that would exceed the budget, would replace an import a build holds, or fails (a refused Windows alias, `vkAllocateMemory`) steps down to the next smaller block; a failed block is not tried again until one of its ranges changes. Ranges larger than the window are imported one at a time. A buffer bound across adjacent ranges that are each larger than the window, or bound over adjacent ranges of any size from the first one's start (ending anywhere in the last range it crosses), joins the ranges it crosses into one import when it is bound, and their own imports are retired; a buffer that crosses any other two imports is copied. A heap grown in pieces re-imports its joined run only when a buffer bound from its first piece's start crosses into a piece the joined import does not cover, or when a sparse descriptor heap over it is bound: a heap a shader binds whole while only the pieces from its base are mapped is bound in place from the import of that run, its views ending at the import's end as the copy's do, once the previous bind of that address saw the same last mapped byte (a heap that grows between every two of its binds stays copied; one bound more often than it grows is joined again after each growth) and at most 64 times per joined run base; past that, with a message, it is not joined again: it is bound in place while an existing import covers its run and copied otherwise, until no registered range starts at that base or the device changes. Ranges larger than the window are taken to be registered whole and never grown in pieces: a title that grows a heap in pieces larger than 8 MiB re-imports the joined run each time a bound buffer crosses into a new piece. The 8 MiB window is a choice: unbounded runs were re-imported whole on every growth of a heap mapped in small pieces, and the NVIDIA driver (610.57.04, 615.78.08) then refused imports or lost the device. Not measured on RADV. - [Cube texture descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) of two to five slices are only sampled: the [materializer](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) rejects loads, stores and dimension queries through them, which on RDNA2 stop at `DEPTH` while the view holds six faces. Out-of-range cube-array sampling is unimplemented; PS5 equivalence is unmeasured. A range of fewer than six slices samples six faces from `BASE_ARRAY` (measured with `tools/hw-oracle` on gfx1036 for two to five slices, and on RX 6600, gfx1032, for equal `BASE_ARRAY` and `DEPTH`); one of six or more keeps every slice up to `DEPTH`. Samplers with `DISABLE_CUBE_WRAP` are created with `VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT`, the flag RADV maps to that bit (Mesa `src/amd/vulkan/radv_sampler.c`, `src/amd/common/ac_descriptors.c`); without `VK_EXT_non_seamless_cube_map` a cube sampled through such a sampler throws. - [Array layer views](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (libSceAgcDriver) - `ArrayLayersServeView` lets a 2D texture at a 2D array image's base address, or a 2D array texture over its first layers, view that image (`Texture::CanCopyFrom`). Mesa-built homebrew (ps5-opengl) writes the view's last slice plus one to `CB_COLOR_ATTRIB3.MIP0_DEPTH`, so a slice-0 render target is a two-layer array image, and the title samples it as a plain 2D texture (SuperTuxKart PPSA69421 drew a black 3D scene and minimap without this). The per-layer layout of a mipmapped array is assumed equal to a standalone surface's; it is not measured on the console. A 2D texture at a later slice's address still gets an image of its own. -- [Linear 2D texture pitch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (libSceAgcDriver) - the PS5's gfx1013 is not measured, and a `DEPTH` pitch on a block-compressed, mipmapped or base-array linear 2D T# throws. A single-level uncompressed linear 2D T# uses `DEPTH` + 1 as its row pitch in elements, raised to the width and rounded up to 256 bytes, as `image_load` read it on RDNA2 (gfx1036, AMD Windows driver 32.0.21030.31, [hardware oracle](../HW_ORACLE.md)) in 135 kernels over 8_UINT, 16_UINT, 32_UINT and 32_32_UINT, widths 16 to 300 and pitches of 1 to 2048 elements. +- [Linear 2D texture pitch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) (libSceAgcDriver) - the PS5's gfx1013 is not measured, and a `DEPTH` pitch on a block-compressed, mipmapped or base-array linear 2D T# throws, as does any pitch other than the default from a PS4 guest. A single-level uncompressed linear 2D T# uses `DEPTH` + 1 as its row pitch in elements, raised to the width and rounded up to 256 bytes, as `image_load` read it on RDNA2 (gfx1036, AMD Windows driver 32.0.21030.31, [hardware oracle](../HW_ORACLE.md)) in 135 kernels over 8_UINT, 16_UINT, 32_UINT and 32_32_UINT, widths 16 to 300 and pitches of 1 to 2048 elements. - [Multisampled CMASK](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - a multisampled color target's CMASK is decoded under CB_COLOR_INFO FAST_CLEAR (bit 13, as for a single-sample target) or COMPRESSION (bit 14). Which bit a multisampled target sets is inferred: its fast clears were seen on GTA V's multisampled targets through their CMASK fills, not in a register dump. A COMPRESSION-only target's zero CMASK is not read as cleared (inferred: zero means cleared only under FAST_CLEAR), so only a fill after its first use clears it. A slice of a multisampled array is its own image at its own address with its own CB_COLOR0_CMASK (inferred from the register layout). Its CMASK is main's whole-metablock size. - [Compressed FMASK reads](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) without an exact resident native owner throw. Native reads require eight fragments, one slice and mip, pipe-aligned DCC/CMASK, Z_64KB_X FMASK and unchanged, disjoint planes, each watched or in a host import the Linux write-watch probe compares (kept as a copy and compared at most once per collect epoch); an FMASK import awaiting its Windows deferred read is read first (`WatchedAfterSettle`). The host stores independent samples and exposes their expanded indices (`0x76543210`, AMD PAL `c5e80007`, `Gfx9Fmask::GetPackedExpandedValue`); console equivalence is unmeasured. Other layouts, standalone compressed decoding, guest publication and recipe reuse remain unsupported. Metadata writes invalidate the native view, including same-value stores, except a same-value CPU store to a compared plane. - [Multisampled CMASK state](../../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) (libSceAgcDriver) - a multisampled target has no guest texels, so its CMASK state comes from fills rather than from a read at each draw as a single-sample target's does. A fill of the whole CMASK with 0 clears the host image at the target's next use, one with 0xcccccccc, 0xdddddddd, 0xeeeeeeee or 0xffffffff marks it expanded, and any other pattern, or a fill that covers only part of the CMASK, throws at the next use (inferred: main's rule that a mixed CMASK is unmodeled, applied to the one host image). A FAST_CLEAR target starts from its CMASK keys at its first use, and a COMPRESSION-only target starts expanded. Copies by DMA into the CMASK are not seen, and a later use with another CMASK address throws. The clear does not rewrite the CMASK bytes as expanded, so a new target over the same CMASK (a different extent, so a different image) seeds from them again and clears once more. From 2baa6d8d407e8b79192f3e2a272cfe17c12ca9d3 Mon Sep 17 00:00:00 2001 From: Michael Krumland-Dunning Date: Mon, 26 Oct 2026 11:02:26 -0700 Subject: [PATCH 227/318] test(agc): stop pinning the custom-pitch encoding refusal Co-Authored-By: Claude Opus 5.5 --- .../prx/libSceAgcDriver/tests/GuestTextureResource.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp index e2d70710c2..10eee2be5a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp @@ -307,13 +307,6 @@ void RunGuestTextureResourceTests() { Require(result.width == 8 && result.depthOrLastArray == 0 && result.linearPitch == pitch, "linear 2D descriptor did not round its DEPTH pitch up to 256 bytes"); const auto geometry = DescribeSurface(result); Require(geometry.mips.size() == 1 && geometry.mips[0].pitchBytes == pitch * 4u && geometry.mips[0].blocksPerRow == pitch && geometry.guestBytes == pitch * 4u * 8u, "linear 2D surface with a custom pitch has the wrong layout"); - bool encoded = true; - try { - static_cast(EncodeTextureResource(result)); - } catch (const std::exception&) { - encoded = false; - } - Require(!encoded, "a linear 2D surface with a custom pitch was encoded without it"); auto defaultPitch = result; defaultPitch.linearPitch = 0; static_cast(EncodeTextureResource(defaultPitch)); From cb5d7fe3ef741780004cfd8e4e677d1b033e7311 Mon Sep 17 00:00:00 2001 From: hamb3r <48532909+hamb3r@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:30:34 +0300 Subject: [PATCH 228/318] fix(agc): keep ahead prebuild call in Dispatch.cpp to avoid PR overlap --- .../Execution/src/Driver/Dispatch/Dispatch.cpp | 5 +++++ .../Execution/src/Driver/Dispatch/DispatchAhead.cpp | 7 ------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 0fe93d5363..5ad5a07b8d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -444,6 +444,11 @@ void Driver::dispatch(QueueState& queue, std::span packet, parked.resolvedAt = std::chrono::steady_clock::now(); ++groupStats.resolved; if (cached) ++groupStats.resolvedHits; + if (!cached && compiledResult != nullptr && !localDevice->HasComputePipeline(*compiledResult)) { + SubmitLibraryPrebuild([localDevice, compiledResult, address] { + try { localDevice->PrebuildComputePipeline(*compiledResult, address); } catch (const std::exception&) {} + }); + } return; } if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, request, memory, address, *keepVariant, liveWords, *compiledResult); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp index 10f3ffb838..b7a09262cf 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp @@ -312,13 +312,6 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l auto phaseLap = lap; DispatchPhaseTiming phaseTiming{profile, lap, phaseMs, phaseLap}; insertDispatch(found->second.address, found->second.key, found->second.noDispatchCache, profile, found->second.registeredShader, found->second.forgetAtCapture, memory, found->second.shaderMemory, found->second.captured, found->second.capture, found->second.compiledResult, found->second.missedEntry, found->second.missedDiffering, found->second.attachVariant, phaseTiming); - if (!localDevice->HasComputePipeline(*found->second.compiledResult)) { - const auto compiledResult = found->second.compiledResult; - const auto address = found->second.address; - SubmitLibraryPrebuild([localDevice, compiledResult, address] { - try { localDevice->PrebuildComputePipeline(*compiledResult, address); } catch (const std::exception&) {} - }); - } } bool bounded = false; const auto itemWrites = DispatchWriteRanges(*found->second.compiledResult, bounded); From 03449442ecd6d38c5270ad73fbc66694cb4bc5af Mon Sep 17 00:00:00 2001 From: milescoviello Date: Sat, 24 Oct 2026 19:13:00 +0000 Subject: [PATCH 229/318] fix(agc): keep a registered shader's register state as a sorted list A registered shader's state holds its shader, context and user config registers, each a Registers, which keeps every offset up to the highest one set: the user config state with two registers at 0x24a and 0x24b takes 2.5 KiB. Ghost of Tsushima (PPSA02225) had 14,322 live states 172 s in, whose register files came to at least 110.1 MiB in that layout and 33.1 MiB as offset and value pairs. RegisteredShaderState now stores each file as a RegisterList, the present registers in offset order; decoding still builds dense Registers (DecodedShaderState), and the four places that copy a state into a QueueState expand it. DecodeFloatMode reads RSRC1 from either form by a binary search, so dispatches don't expand. Co-Authored-By: Claude Opus 5.5 --- .../include/Driver/Shaders/ShaderRegistry.hpp | 58 ++++++++++++++++++- .../src/Driver/Shaders/ShaderRegistry.cpp | 29 +++++----- .../libs/prx/libSceAgcDriver/tests/Submit.cpp | 18 ++++++ 3 files changed, 90 insertions(+), 15 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index 510424e3ec..2aee7856f9 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -3,6 +3,7 @@ #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -209,10 +211,64 @@ private: std::shared_ptr> words; }; -struct RegisteredShaderState { +class RegisterList { +public: + using Entry = std::pair; + using const_iterator = std::vector::const_iterator; + + RegisterList() = default; + explicit RegisterList(const Registers& registers) { + entries.reserve(registers.size()); + registers.ForEach([&](std::uint32_t offset, std::uint32_t value) { entries.emplace_back(offset, value); }); + } + + [[nodiscard]] Registers Expand() const { + Registers registers; + if (!entries.empty()) registers.emplace(entries.back().first, entries.back().second); + for (const auto& [offset, value] : entries) registers.emplace(offset, value); + return registers; + } + [[nodiscard]] const_iterator begin() const { return entries.begin(); } + [[nodiscard]] const_iterator end() const { return entries.end(); } + [[nodiscard]] const_iterator find(std::uint32_t offset) const { + const auto found = std::lower_bound(entries.begin(), entries.end(), offset, [](const Entry& entry, std::uint32_t wanted) { return entry.first < wanted; }); + return found != entries.end() && found->first == offset ? found : entries.end(); + } + [[nodiscard]] std::uint32_t at(std::uint32_t offset) const { + const auto found = find(offset); + if (found == end()) throw std::out_of_range("register is not set"); + return found->second; + } + [[nodiscard]] std::size_t size() const { return entries.size(); } + [[nodiscard]] bool empty() const { return entries.empty(); } + template + void ForEach(TVisit&& visit) const { + for (const auto& [offset, value] : entries) visit(offset, value); + } + void emplace(std::uint32_t offset, std::uint32_t value) { + const auto found = std::lower_bound(entries.begin(), entries.end(), offset, [](const Entry& entry, std::uint32_t wanted) { return entry.first < wanted; }); + if (found == entries.end() || found->first != offset) entries.insert(found, {offset, value}); + } + + bool operator==(const RegisterList&) const = default; + +private: + std::vector entries; +}; + +struct DecodedShaderState { Registers shader; Registers context; Registers userConfig; +}; + +struct RegisteredShaderState { + RegisteredShaderState() = default; + RegisteredShaderState(const DecodedShaderState& decoded) : shader(decoded.shader), context(decoded.context), userConfig(decoded.userConfig) {} + + RegisterList shader; + RegisterList context; + RegisterList userConfig; bool operator==(const RegisteredShaderState&) const = default; }; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index e9bbf39180..0815d386fc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -1116,7 +1116,8 @@ std::shared_ptr CaptureInvocation(Shade } namespace { -std::optional DecodeFloatMode(const Registers& registers, std::uint8_t type) { +template +std::optional DecodeFloatMode(const TRegisters& registers, std::uint8_t type) { std::uint32_t rsrc1; std::uint32_t fp16OverflowBit; switch (type) { @@ -1164,7 +1165,7 @@ Shader ReadHeader(const ShaderSnapshot& snapshot) { return header; } -RegisteredShaderState DecodeRegisteredState(const ShaderSnapshot& snapshot, RegisteredShaderState state = {{}, InitialContextRegisters(), {{0x24a, 0}, {0x24b, 0}}}) { +DecodedShaderState DecodeRegisteredState(const ShaderSnapshot& snapshot, DecodedShaderState state = {{}, InitialContextRegisters(), {{0x24a, 0}, {0x24b, 0}}}) { const auto header = ReadHeader(snapshot); for (const auto reg : ReadHeaderArray(snapshot, header.sh_registers, header.num_sh_registers)) { state.shader.insert_or_assign(reg.offset, reg.value); @@ -1401,9 +1402,9 @@ PreparedAtUse PrepareDeferredAbis(const ShaderSnapshot& snapshot, std::unique_lo const auto device = abi.device.lock(); if (device == nullptr || snapshot.registeredState == nullptr) continue; QueueState state{}; - state.shader = snapshot.registeredState->shader; - state.context = snapshot.registeredState->context; - state.userConfig = snapshot.registeredState->userConfig; + state.shader = snapshot.registeredState->shader.Expand(); + state.context = snapshot.registeredState->context.Expand(); + state.userConfig = snapshot.registeredState->userConfig.Expand(); for (const auto& [offset, value] : abi.context) state.context[offset] = value; for (const auto& [offset, value] : abi.primitive) state.userConfig[offset] = value; try { @@ -2022,9 +2023,9 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::spanregisteredState != nullptr, "registered shader state is missing"); QueueState state{}; - state.shader = snapshot->registeredState->shader; - state.context = snapshot->registeredState->context; - state.userConfig = snapshot->registeredState->userConfig; + state.shader = snapshot->registeredState->shader.Expand(); + state.context = snapshot->registeredState->context.Expand(); + state.userConfig = snapshot->registeredState->userConfig.Expand(); for (const auto reg : context) state.context[reg.offset] = reg.value; for (const auto reg : primitive) state.userConfig[reg.offset] = reg.value; const auto localDevice = device.Load(); @@ -2275,9 +2276,9 @@ ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device) { nullRegisteredState.context.insert_or_assign(0x1b4u, 0x2u); null.registeredState = std::make_shared(std::move(nullRegisteredState)); QueueState nullState{}; - nullState.shader = null.registeredState->shader; - nullState.context = null.registeredState->context; - nullState.userConfig = null.registeredState->userConfig; + nullState.shader = null.registeredState->shader.Expand(); + nullState.context = null.registeredState->context.Expand(); + nullState.userConfig = null.registeredState->userConfig.Expand(); null.prepared->entries = PrepareRegistered(null, device, nullState, true); return null; } @@ -2369,9 +2370,9 @@ void Driver::RegisterShader(const Shader* shader) { } } QueueState registered{}; - registered.shader = snapshot.registeredState->shader; - registered.context = snapshot.registeredState->context; - registered.userConfig = snapshot.registeredState->userConfig; + registered.shader = snapshot.registeredState->shader.Expand(); + registered.context = snapshot.registeredState->context.Expand(); + registered.userConfig = snapshot.registeredState->userConfig.Expand(); std::shared_ptr preparation; if (LazyShaderPreparation()) { static_cast(PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred)); diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index d2e874a808..29f077c3cd 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -674,6 +674,23 @@ void testHeaderWithoutProgramAddress() { AgcDriverRegisterShader_nid_postfix(&header.shader); } +void testRegisterList() { + using AgcDriver::DriverDetail::RegisterList; + const AgcDriver::Registers userConfig{{0x24a, 1}, {0x24b, 2}, {0x25b, 3}}; + const RegisterList list(userConfig); + check(list.size() == 3 && list.Expand() == userConfig, "a register list does not hold its registers' offsets and values"); + check(list.find(0x24b) != list.end() && list.find(0x24b)->second == 2 && list.find(0x24c) == list.end() && list.find(0x200) == list.end(), "a register list found the wrong register"); + RegisterList added; + added.emplace(0x25b, 3); + added.emplace(0x24a, 1); + added.emplace(0x24b, 2); + added.emplace(0x24a, 9); + check(added == list, "a register list is not kept in offset order, or a second emplace replaced a value"); + AgcDriver::DriverDetail::DecodedShaderState decoded{{{0x08a, 7}}, AgcDriver::InitialContextRegisters(), userConfig}; + const AgcDriver::DriverDetail::RegisteredShaderState state(decoded); + check(state.shader.Expand() == decoded.shader && state.context.Expand() == decoded.context && state.userConfig.Expand() == decoded.userConfig, "a registered state did not keep the decoded registers"); +} + void testRegisteredFloatMode() { using AgcDriver::DriverDetail::RegisteredFloatMode; using AgcDriver::DriverDetail::ShaderSnapshot; @@ -1024,6 +1041,7 @@ int main(int argc, char** argv) { testCopyIntoHostMemory(); testLargeDataFill(); testSmallCopyBehindPendingSource(); + testRegisterList(); testRegisteredFloatMode(); testDispatchFloatMode(); testReadOnlyResummarize(); From f1d15c5f51e813d303b1ab3ce425d8dba676492a Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 17:02:58 +0530 Subject: [PATCH 230/318] feat(relinker): lower BLSI, BLSMSK and BLSR under --to-rosetta --- core/relinker/CMakeLists.txt | 8 + .../include/codegen/x86/BlsLowering.hpp | 29 +++ .../include/codegen/x86/ScalarLowering.hpp | 99 ++++++++++ .../src/x86/Amd64OnlyInstructionMatcher.cpp | 7 + .../relinker/codegen/src/x86/AndnLowering.cpp | 126 +++---------- core/relinker/codegen/src/x86/BlsLowering.cpp | 134 ++++++++++++++ .../codegen/tests/AndnLoweringTests.cpp | 155 +++------------- .../codegen/tests/BlsLoweringTests.cpp | 170 ++++++++++++++++++ .../codegen/tests/LoweringExecution.hpp | 149 +++++++++++++++ 9 files changed, 647 insertions(+), 230 deletions(-) create mode 100644 core/relinker/codegen/include/codegen/x86/BlsLowering.hpp create mode 100644 core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp create mode 100644 core/relinker/codegen/src/x86/BlsLowering.cpp create mode 100644 core/relinker/codegen/tests/BlsLoweringTests.cpp create mode 100644 core/relinker/codegen/tests/LoweringExecution.hpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 6fe373cf10..4432faa105 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -54,6 +54,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/RelrRelocations.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp @@ -100,6 +101,11 @@ if(BUILD_TESTING) add_test(NAME andn_lowering COMMAND andn_lowering_tests) set_tests_properties(andn_lowering PROPERTIES TIMEOUT 20) + add_test_executable(bls_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/BlsLoweringTests.cpp) + target_link_libraries(bls_lowering_tests PRIVATE relinker_core) + add_test(NAME bls_lowering COMMAND bls_lowering_tests) + set_tests_properties(bls_lowering PROPERTIES TIMEOUT 20) + add_test_executable(masked_store_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/MaskedStoreLoweringTests.cpp) target_link_libraries(masked_store_lowering_tests PRIVATE relinker_core) add_test(NAME masked_store_lowering COMMAND masked_store_lowering_tests) @@ -251,6 +257,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -540,6 +547,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/BlsLowering.hpp b/core/relinker/codegen/include/codegen/x86/BlsLowering.hpp new file mode 100644 index 0000000000..6d74afe9f5 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/BlsLowering.hpp @@ -0,0 +1,29 @@ +#ifndef CODEGEN_X86_BLSLOWERING_HPP +#define CODEGEN_X86_BLSLOWERING_HPP + +#include +#include + +namespace Codegen { + +enum class BlsOperation : std::uint8_t { + Reset = 1, + Mask = 2, + Isolate = 3, +}; + +struct BlsOperands { + BlsOperation Operation; + std::uint8_t Destination; + std::uint8_t Source; + bool Wide; + std::optional Memory; +}; + +[[nodiscard]] const char* BlsName(BlsOperation operation); +[[nodiscard]] std::optional DecodeBls(const std::uint8_t* data, std::size_t length); +void EmitBls(StubBodyBuilder& body, const BlsOperands& operands); + +} + +#endif diff --git a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp new file mode 100644 index 0000000000..363e8f4181 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp @@ -0,0 +1,99 @@ +#ifndef CODEGEN_X86_SCALARLOWERING_HPP +#define CODEGEN_X86_SCALARLOWERING_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Codegen::ScalarLowering { + +inline constexpr std::int32_t FrameSize = 144; +inline constexpr std::uint8_t Stack = 4; + +inline bool LegacyPrefix(std::uint8_t value) { + return value == 0x2E || value == 0x36 || value == 0x3E || value == 0x26 || value == 0x64 || value == 0x65 || value == 0x67; +} + +inline void Append32(std::vector& bytes, std::int32_t value) { + for (unsigned shift = 0; shift < 32; shift += 8) + bytes.push_back(static_cast(static_cast(value) >> shift)); +} + +inline std::uint8_t Rex(bool wide, std::uint8_t reg, std::uint8_t rm) { + return static_cast(0x40 | (wide ? 8 : 0) | ((reg & 8) >> 1) | ((rm & 8) >> 3)); +} + +inline std::vector RegisterBytes(std::uint8_t opcode, bool wide, std::uint8_t reg, std::uint8_t rm) { + return {Rex(wide, reg, rm), opcode, static_cast(0xC0 | ((reg & 7) << 3) | (rm & 7))}; +} + +inline std::vector StackBytes(std::uint8_t opcode, bool wide, std::uint8_t reg, std::int32_t displacement) { + std::vector bytes{Rex(wide, reg, Stack), opcode, static_cast(0x84 | ((reg & 7) << 3)), 0x24}; + Append32(bytes, displacement); + return bytes; +} + +inline void Register(StubBodyBuilder& body, std::uint8_t opcode, bool wide, std::uint8_t reg, std::uint8_t rm) { + body.Raw(RegisterBytes(opcode, wide, reg, rm)); +} + +inline void StackSlot(StubBodyBuilder& body, std::uint8_t opcode, bool wide, std::uint8_t reg, std::int32_t displacement) { + body.Raw(StackBytes(opcode, wide, reg, displacement)); +} + +inline std::array UsedRegisters(std::initializer_list registers, const std::optional& memory) { + std::array used{}; + used[Stack] = true; + for (const auto reg : registers) used[reg] = true; + if (!memory) return used; + if (memory->Rm == 4) { + const auto base = static_cast((memory->Sib & 7) | ((memory->RexIndexBase & 1) << 3)); + const auto index = static_cast(((memory->Sib >> 3) & 7) | ((memory->RexIndexBase & 2) << 2)); + if (memory->Mod != 0 || (memory->Sib & 7) != 5) used[base] = true; + if (index != Stack) used[index] = true; + } else if (!memory->RipRelative) { + used[memory->Rm | ((memory->RexIndexBase & 1) << 3)] = true; + } + return used; +} + +inline std::uint8_t Scratch(std::array& used, const char* instruction) { + for (const auto reg : {11, 10, 9, 8, 0, 1, 2, 3, 5, 6, 7, 12, 13, 14, 15}) { + if (used[reg]) continue; + used[reg] = true; + return static_cast(reg); + } + throw CodegenException(std::string(instruction) + " has no scratch register"); +} + +inline void Memory(StubBodyBuilder& body, std::uint8_t opcode, bool wide, const MemoryOperand& memory, std::uint8_t reg, const char* instruction) { + auto displacement = static_cast(memory.Displacement); + if (memory.StackBase) displacement += FrameSize; + const bool address32 = std::find(memory.Prefixes.begin(), memory.Prefixes.end(), 0x67) != memory.Prefixes.end(); + if (!address32 && displacement > std::numeric_limits::max()) + throw CodegenException(std::string(instruction) + " stack displacement does not fit after spilling"); + const auto mod = memory.StackBase ? 2 : memory.Mod; + auto bytes = memory.Prefixes; + bytes.push_back(static_cast(Rex(wide, reg, 0) | memory.RexIndexBase)); + bytes.push_back(opcode); + bytes.push_back(static_cast((mod << 6) | ((reg & 7) << 3) | memory.Rm)); + if (memory.Rm == 4) bytes.push_back(memory.Sib); + const auto offset = bytes.size(); + if (mod == 1) bytes.push_back(static_cast(displacement)); + if (mod == 2 || (mod == 0 && (memory.RipRelative || (memory.Rm == 4 && (memory.Sib & 7) == 5)))) + Append32(bytes, static_cast(displacement)); + if (memory.RipRelative) + body.RipRelative(bytes, offset, static_cast(memory.InstructionLength) + memory.Displacement); + else + body.Raw(bytes); +} + +} + +#endif diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index f957478801..73c91595e3 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -256,6 +257,8 @@ std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const D return std::nullopt; if (const auto andn = DecodeAndn(instr.Data, instr.Length)) return _trampoline("ANDN", instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitAndn(body, *andn); })); + if (const auto bls = DecodeBls(instr.Data, instr.Length)) + return _trampoline(BlsName(bls->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitBls(body, *bls); })); if (instr.IsRdseed()) { std::vector replacement(instr.Data, instr.Data + instr.Length); replacement[instr.OpcodeOffset() + 2] &= static_cast(~(1u << X64OpcodeConstants::ModRmRegShift)); @@ -327,6 +330,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c EmitAndn(body, *andn); return "ANDN"; } + if (const auto bls = _target == Amd64OnlyTarget::Rosetta ? DecodeBls(instr.Data, instr.Length) : std::nullopt) { + EmitBls(body, *bls); + return BlsName(bls->Operation); + } if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdtscp()) { body.Raw(_rdtscpReplacement(instr)); return kRdtscp.Name; diff --git a/core/relinker/codegen/src/x86/AndnLowering.cpp b/core/relinker/codegen/src/x86/AndnLowering.cpp index a99b92e4f6..82213eab01 100644 --- a/core/relinker/codegen/src/x86/AndnLowering.cpp +++ b/core/relinker/codegen/src/x86/AndnLowering.cpp @@ -1,89 +1,13 @@ #include -#include -#include -#include -#include +#include namespace Codegen { -namespace { - -constexpr std::int32_t kFrameSize = 144; -constexpr std::uint8_t kStack = 4; - -bool _prefix(std::uint8_t value) { - return value == 0x2E || value == 0x36 || value == 0x3E || value == 0x26 || value == 0x64 || value == 0x65 || value == 0x67; -} - -void _append32(std::vector& bytes, std::int32_t value) { - for (unsigned shift = 0; shift < 32; shift += 8) - bytes.push_back(static_cast(static_cast(value) >> shift)); -} - -std::uint8_t _rex(bool wide, std::uint8_t reg, std::uint8_t rm) { - return static_cast(0x40 | (wide ? 8 : 0) | ((reg & 8) >> 1) | ((rm & 8) >> 3)); -} - -void _register(StubBodyBuilder& body, std::uint8_t opcode, bool wide, std::uint8_t reg, std::uint8_t rm) { - const std::array bytes{_rex(wide, reg, rm), opcode, static_cast(0xC0 | ((reg & 7) << 3) | (rm & 7))}; - body.Raw(bytes); -} - -void _stack(StubBodyBuilder& body, std::uint8_t opcode, bool wide, std::uint8_t reg, std::int32_t displacement) { - std::vector bytes{_rex(wide, reg, kStack), opcode, static_cast(0x84 | ((reg & 7) << 3)), 0x24}; - _append32(bytes, displacement); - body.Raw(bytes); -} - -std::uint8_t _scratch(const AndnOperands& operands) { - std::array used{}; - used[kStack] = used[operands.Destination] = used[operands.InvertedSource] = true; - if (!operands.Memory) { - used[operands.Source] = true; - } else { - const auto& memory = *operands.Memory; - if (memory.Rm == 4) { - const auto base = static_cast((memory.Sib & 7) | ((memory.RexIndexBase & 1) << 3)); - const auto index = static_cast(((memory.Sib >> 3) & 7) | ((memory.RexIndexBase & 2) << 2)); - if (memory.Mod != 0 || (memory.Sib & 7) != 5) used[base] = true; - if (index != kStack) used[index] = true; - } else if (!memory.RipRelative) { - used[memory.Rm | ((memory.RexIndexBase & 1) << 3)] = true; - } - } - for (const auto reg : {11, 10, 9, 8, 0, 1, 2, 3, 5, 6, 7, 12, 13, 14, 15}) - if (!used[reg]) return static_cast(reg); - throw CodegenException("ANDN has no scratch register"); -} - -void _memory(StubBodyBuilder& body, const AndnOperands& operands, std::uint8_t scratch) { - const auto& memory = *operands.Memory; - auto displacement = static_cast(memory.Displacement); - if (memory.StackBase) displacement += kFrameSize; - const bool address32 = std::find(memory.Prefixes.begin(), memory.Prefixes.end(), 0x67) != memory.Prefixes.end(); - if (!address32 && displacement > std::numeric_limits::max()) - throw CodegenException("ANDN stack displacement does not fit after spilling"); - const auto mod = memory.StackBase ? 2 : memory.Mod; - auto bytes = memory.Prefixes; - bytes.push_back(static_cast(_rex(operands.Wide, scratch, 0) | memory.RexIndexBase)); - bytes.push_back(0x23); - bytes.push_back(static_cast((mod << 6) | ((scratch & 7) << 3) | memory.Rm)); - if (memory.Rm == 4) bytes.push_back(memory.Sib); - const auto offset = bytes.size(); - if (mod == 1) bytes.push_back(static_cast(displacement)); - if (mod == 2 || (mod == 0 && (memory.RipRelative || (memory.Rm == 4 && (memory.Sib & 7) == 5)))) - _append32(bytes, static_cast(displacement)); - if (memory.RipRelative) - body.RipRelative(bytes, offset, static_cast(memory.InstructionLength) + memory.Displacement); - else - body.Raw(bytes); -} - -} +using namespace ScalarLowering; std::optional DecodeAndn(const std::uint8_t* data, const std::size_t length) { std::size_t vex = 0; - while (vex < length && _prefix(data[vex])) ++vex; + while (vex < length && LegacyPrefix(data[vex])) ++vex; if (vex + 5 > length || data[vex] != 0xC4 || (data[vex + 1] & 0x1F) != 2 || data[vex + 3] != 0xF2) return std::nullopt; if ((data[vex + 2] & 7) != 0) @@ -107,32 +31,34 @@ std::optional DecodeAndn(const std::uint8_t* data, const std::size } void EmitAndn(StubBodyBuilder& body, const AndnOperands& operands) { - const auto scratch = _scratch(operands); - _stack(body, 0x8D, true, kStack, -kFrameSize); - _stack(body, 0x89, true, scratch, 0); - if (!operands.Memory && operands.Source == kStack) { - _stack(body, 0x8D, true, scratch, kFrameSize); - _stack(body, 0x89, true, scratch, 8); + auto used = operands.Memory ? UsedRegisters({operands.Destination, operands.InvertedSource}, operands.Memory) + : UsedRegisters({operands.Destination, operands.InvertedSource, operands.Source}, std::nullopt); + const auto scratch = Scratch(used, "ANDN"); + StackSlot(body, 0x8D, true, Stack, -FrameSize); + StackSlot(body, 0x89, true, scratch, 0); + if (!operands.Memory && operands.Source == Stack) { + StackSlot(body, 0x8D, true, scratch, FrameSize); + StackSlot(body, 0x89, true, scratch, 8); } - if (operands.InvertedSource == kStack) - _stack(body, 0x8D, true, scratch, kFrameSize); + if (operands.InvertedSource == Stack) + StackSlot(body, 0x8D, true, scratch, FrameSize); else - _register(body, 0x8B, operands.Wide, scratch, operands.InvertedSource); - _register(body, 0xF7, operands.Wide, 2, scratch); + Register(body, 0x8B, operands.Wide, scratch, operands.InvertedSource); + Register(body, 0xF7, operands.Wide, 2, scratch); if (operands.Memory) - _memory(body, operands, scratch); - else if (operands.Source == kStack) - _stack(body, 0x23, operands.Wide, scratch, 8); + Memory(body, 0x23, operands.Wide, *operands.Memory, scratch, "ANDN"); + else if (operands.Source == Stack) + StackSlot(body, 0x23, operands.Wide, scratch, 8); else - _register(body, 0x23, operands.Wide, scratch, operands.Source); - if (operands.Destination == kStack) - _stack(body, 0x89, operands.Wide, scratch, 8); + Register(body, 0x23, operands.Wide, scratch, operands.Source); + if (operands.Destination == Stack) + StackSlot(body, 0x89, operands.Wide, scratch, 8); else - _register(body, 0x8B, operands.Wide, operands.Destination, scratch); - _stack(body, 0x8B, true, scratch, 0); - _stack(body, 0x8D, true, kStack, kFrameSize); - if (operands.Destination == kStack) - _stack(body, 0x8B, operands.Wide, kStack, 8 - kFrameSize); + Register(body, 0x8B, operands.Wide, operands.Destination, scratch); + StackSlot(body, 0x8B, true, scratch, 0); + StackSlot(body, 0x8D, true, Stack, FrameSize); + if (operands.Destination == Stack) + StackSlot(body, 0x8B, operands.Wide, Stack, 8 - FrameSize); } } diff --git a/core/relinker/codegen/src/x86/BlsLowering.cpp b/core/relinker/codegen/src/x86/BlsLowering.cpp new file mode 100644 index 0000000000..589a2b2db4 --- /dev/null +++ b/core/relinker/codegen/src/x86/BlsLowering.cpp @@ -0,0 +1,134 @@ +#include +#include + +namespace Codegen { + +using namespace ScalarLowering; + +namespace { + +constexpr std::uint8_t kSetCarry = 0xF9; + +void _append(std::vector& bytes, const std::vector& instruction) { + bytes.insert(bytes.end(), instruction.begin(), instruction.end()); +} + +std::vector _leaMinusOne(std::uint8_t destination, std::uint8_t base) { + std::vector bytes{Rex(true, destination, base), 0x8D, static_cast(0x40 | ((destination & 7) << 3) | (base & 7))}; + if ((base & 7) == 4) bytes.push_back(0x24); + bytes.push_back(0xFF); + return bytes; +} + +std::vector _nonzero(const BlsOperands& operands, std::uint8_t result, std::uint8_t temporary) { + std::vector bytes; + switch (operands.Operation) { + case BlsOperation::Reset: + _append(bytes, _leaMinusOne(temporary, result)); + _append(bytes, RegisterBytes(0x23, operands.Wide, result, temporary)); + break; + case BlsOperation::Mask: + _append(bytes, _leaMinusOne(temporary, result)); + _append(bytes, RegisterBytes(0x33, operands.Wide, result, temporary)); + break; + case BlsOperation::Isolate: + _append(bytes, RegisterBytes(0x8B, operands.Wide, temporary, result)); + _append(bytes, RegisterBytes(0xF7, operands.Wide, 3, temporary)); + _append(bytes, RegisterBytes(0x23, operands.Wide, result, temporary)); + bytes.push_back(kSetCarry); + break; + } + return bytes; +} + +std::vector _zero(const BlsOperands& operands, std::uint8_t result) { + std::vector bytes; + switch (operands.Operation) { + case BlsOperation::Reset: + _append(bytes, RegisterBytes(0x33, operands.Wide, result, result)); + bytes.push_back(kSetCarry); + break; + case BlsOperation::Mask: + _append(bytes, RegisterBytes(0x83, operands.Wide, 1, result)); + bytes.push_back(0xFF); + bytes.push_back(kSetCarry); + break; + case BlsOperation::Isolate: + _append(bytes, RegisterBytes(0x33, operands.Wide, result, result)); + break; + } + return bytes; +} + +} + +const char* BlsName(const BlsOperation operation) { + switch (operation) { + case BlsOperation::Reset: return "BLSR"; + case BlsOperation::Mask: return "BLSMSK"; + case BlsOperation::Isolate: return "BLSI"; + } + throw CodegenException("unknown BLS operation"); +} + +std::optional DecodeBls(const std::uint8_t* data, const std::size_t length) { + std::size_t vex = 0; + while (vex < length && LegacyPrefix(data[vex])) ++vex; + if (vex + 5 > length || data[vex] != 0xC4 || (data[vex + 1] & 0x1F) != 2 || data[vex + 3] != 0xF3) + return std::nullopt; + const auto modrm = data[vex + 4]; + const auto extension = static_cast((modrm >> 3) & 7); + if (extension < 1 || extension > 3) + return std::nullopt; + if ((data[vex + 2] & 7) != 0) + throw CodegenException(std::string(BlsName(static_cast(extension))) + " requires VEX.L=0 and VEX.pp=0"); + const auto extensions = static_cast(~data[vex + 1]); + BlsOperands operands{ + static_cast(extension), + static_cast((~data[vex + 2] >> 3) & 15), + static_cast((modrm & 7) | ((extensions & 0x20) >> 2)), + (data[vex + 2] & 0x80) != 0, std::nullopt}; + if ((modrm >> 6) == 3) { + if (vex + 5 != length) throw CodegenException(std::string(BlsName(operands.Operation)) + " register operand has trailing bytes"); + } else { + const auto rex = static_cast(((extensions & 0x20) >> 5) | ((extensions & 0x40) >> 5)); + operands.Memory = DecodeMemoryOperand(data, length, vex + 4, rex, {data, data + vex}); + if (vex + 4 + operands.Memory->EncodedSize != length) + throw CodegenException(std::string(BlsName(operands.Operation)) + " memory operand has trailing bytes"); + } + return operands; +} + +void EmitBls(StubBodyBuilder& body, const BlsOperands& operands) { + const auto name = BlsName(operands.Operation); + auto used = operands.Memory ? UsedRegisters({operands.Destination}, operands.Memory) : UsedRegisters({operands.Destination, operands.Source}, std::nullopt); + const auto result = Scratch(used, name); + const auto temporary = Scratch(used, name); + StackSlot(body, 0x8D, true, Stack, -FrameSize); + StackSlot(body, 0x89, true, result, 0); + StackSlot(body, 0x89, true, temporary, 8); + if (operands.Memory) + Memory(body, 0x8B, operands.Wide, *operands.Memory, result, name); + else if (operands.Source == Stack) + StackSlot(body, 0x8D, true, result, FrameSize); + else + Register(body, 0x8B, operands.Wide, result, operands.Source); + Register(body, 0x85, operands.Wide, result, result); + const auto nonzero = _nonzero(operands, result, temporary); + const auto zero = _zero(operands, result); + body.Raw(std::array{std::uint8_t{0x74}, static_cast(nonzero.size() + 2)}); + body.Raw(nonzero); + body.Raw(std::array{std::uint8_t{0xEB}, static_cast(zero.size())}); + body.Raw(zero); + if (operands.Destination == Stack) + StackSlot(body, 0x89, operands.Wide, result, 16); + else + Register(body, 0x8B, operands.Wide, operands.Destination, result); + StackSlot(body, 0x8B, true, result, 0); + StackSlot(body, 0x8B, true, temporary, 8); + StackSlot(body, 0x8D, true, Stack, FrameSize); + if (operands.Destination == Stack) + StackSlot(body, 0x8B, operands.Wide, Stack, 16 - FrameSize); +} + +} diff --git a/core/relinker/codegen/tests/AndnLoweringTests.cpp b/core/relinker/codegen/tests/AndnLoweringTests.cpp index 6925e2e0b5..b035966455 100644 --- a/core/relinker/codegen/tests/AndnLoweringTests.cpp +++ b/core/relinker/codegen/tests/AndnLoweringTests.cpp @@ -1,20 +1,18 @@ +#include "LoweringExecution.hpp" #include #include #include #include #include #include -#include +#include #include #include #include -#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) -#include -#endif namespace { -using Bytes = std::vector; +using LoweringExecution::Bytes; void require(bool condition, const char* message) { if (!condition) throw std::runtime_error(message); @@ -69,130 +67,27 @@ void matching() { require(rosetta->MatchSequence(sequence, {}).has_value(), "An ANDN sequence could not move out of line"); } -#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) +#if LOWERING_EXECUTION_AVAILABLE -struct State { - std::array Input{}; - std::array Output{}; - std::uint64_t Flags = 0; - std::array RedZone{}; -}; +using LoweringExecution::Execution; +using LoweringExecution::State; -class Execution { -public: - Execution(const Bytes& site, std::uint64_t memoryValue) { - auto operands = Codegen::DecodeAndn(site.data(), site.size()); - require(operands.has_value(), "Execution site was not ANDN"); - for (auto reg : {3, 5, 12, 13, 14, 15}) push(static_cast(reg)); - memory(0x8D, 4, 4, -128); - rip({0x48, 0x89, 0x3D}, 0); - rip({0x48, 0x89, 0x25}, 8); - memory(0x89, 4, 7, 32); - bytes.insert(bytes.end(), {0x48, 0xB8}); - append64(kSentinel); - for (int displacement = -128; displacement < 0; displacement += 8) - memory(0x89, 0, 4, displacement); - memory(0x8B, 0, 7, 16 * 8); - memory(0x89, 0, 4, 64); - for (std::uint8_t reg = 0; reg < 16; ++reg) - if (reg != 4 && reg != 7) memory(0x8B, reg, 7, reg * 8); - memory(0x8B, 7, 7, 7 * 8); - const auto siteOffset = bytes.size(); - if (operands->Memory && operands->Memory->RipRelative) { - auto relocated = site; - const auto displacement = static_cast(4096 + 24 - (siteOffset + site.size())); - std::memcpy(relocated.data() + relocated.size() - 4, &displacement, sizeof(displacement)); - operands = Codegen::DecodeAndn(relocated.data(), relocated.size()); - } - Codegen::StubBodyBuilder builder; - Codegen::EmitAndn(builder, *operands); - auto body = builder.Finish(); - const auto stubOffset = bytes.size(); - Codegen::ApplyStubRelocations(body.Bytes, body.Relocations, siteOffset, stubOffset, 0); - bytes.insert(bytes.end(), body.Bytes.begin(), body.Bytes.begin() + static_cast(body.ReturnBranchOffset)); - rip({0x48, 0x89, 0x25}, 16); - rip({0x48, 0x8B, 0x25}, 8); - memory(0x8D, 4, 4, -256); - for (std::uint8_t reg = 0; reg < 16; ++reg) if (reg != 4) push(reg); - bytes.push_back(0x9C); - rip({0x48, 0x8B, 0x3D}, 0); - memory(0x8B, 0, 4, 0); - memory(0x89, 0, 7, 32 * 8); - int offset = 8; - for (int reg = 15; reg >= 0; --reg) { - if (reg == 4) continue; - memory(0x8B, 0, 4, offset); - memory(0x89, 0, 7, (16 + reg) * 8); - offset += 8; - } - rip({0x48, 0x8B, 0x05}, 16); - memory(0x89, 0, 7, 20 * 8); - for (int slot = 0; slot < 16; ++slot) { - memory(0x8B, 0, 4, 256 + 128 - 128 + slot * 8); - memory(0x89, 0, 7, (33 + slot) * 8); - } - rip({0x48, 0x8B, 0x25}, 8); - memory(0x8D, 4, 4, 128); - for (auto reg : {15, 14, 13, 12, 5, 3}) pop(static_cast(reg)); - bytes.push_back(0xC3); - require(bytes.size() < 4096, "Execution wrapper exceeded its code page"); - code = static_cast(mmap(nullptr, 8192, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); - require(code != MAP_FAILED, "Cannot allocate execution test pages"); - std::memcpy(code, bytes.data(), bytes.size()); - std::memcpy(code + 4096 + 24, &memoryValue, sizeof(memoryValue)); - require(mprotect(code, 4096, PROT_READ | PROT_EXEC) == 0, "Cannot make ANDN test code executable"); - } - - ~Execution() { if (code != MAP_FAILED) munmap(code, 8192); } - - void Run(State& state) const { reinterpret_cast(code)(&state); } - const void* Memory() const { return code + 4096 + 24; } - static constexpr std::uint64_t kSentinel = 0xBADC0FFEE0DDF00Dull; - -private: - Bytes bytes; - std::uint8_t* code = reinterpret_cast(MAP_FAILED); - - void append32(std::int32_t value) { - for (unsigned shift = 0; shift < 32; shift += 8) bytes.push_back(static_cast(static_cast(value) >> shift)); - } - void append64(std::uint64_t value) { - for (unsigned shift = 0; shift < 64; shift += 8) bytes.push_back(static_cast(value >> shift)); - } - void rip(std::initializer_list opcode, int slot) { - bytes.insert(bytes.end(), opcode); - append32(static_cast(4096 + slot - (bytes.size() + 4))); - } - void push(std::uint8_t reg) { - if (reg >= 8) bytes.push_back(0x41); - bytes.push_back(static_cast(0x50 | (reg & 7))); - } - void pop(std::uint8_t reg) { - if (reg >= 8) bytes.push_back(0x41); - bytes.push_back(static_cast(0x58 | (reg & 7))); - } - void memory(std::uint8_t opcode, std::uint8_t reg, std::uint8_t base, int displacement) { - bytes.insert(bytes.end(), {static_cast(0x48 | ((reg & 8) >> 1) | ((base & 8) >> 3)), opcode, - static_cast(0x80 | ((reg & 7) << 3) | (base & 7))}); - if ((base & 7) == 4) bytes.push_back(0x24); - append32(displacement); - } -}; +Execution executable(const Bytes& site, std::uint64_t memoryValue) { + const auto operands = Codegen::DecodeAndn(site.data(), site.size()); + require(operands.has_value(), "Execution site was not ANDN"); + return Execution(site, memoryValue, operands->Memory && operands->Memory->RipRelative, [](Codegen::StubBodyBuilder& body, const Bytes& relocated) { + Codegen::EmitAndn(body, *Codegen::DecodeAndn(relocated.data(), relocated.size())); + }); +} void check(const State& state, const Codegen::AndnOperands& operands, std::uint64_t second) { const auto mask = operands.Wide ? ~std::uint64_t{0} : 0xFFFFFFFFull; const auto expected = (~state.Input[operands.InvertedSource] & second) & mask; for (std::size_t reg = 0; reg < 16; ++reg) require(state.Output[reg] == (reg == operands.Destination ? expected : state.Input[reg]), "ANDN changed a source/scratch register or produced an incorrect result"); - const auto flags = (expected == 0 ? 0x40u : 0u) | ((expected >> (operands.Wide ? 63 : 31)) != 0 ? 0x80u : 0u); - require((state.Flags & 0x8C1u) == flags, "ANDN did not preserve its defined flag semantics"); - for (const auto value : state.RedZone) require(value == Execution::kSentinel, "ANDN overwrote the caller's red zone"); -} - -State initialState() { - State state; - for (std::size_t reg = 0; reg < 16; ++reg) state.Input[reg] = 0xABCD123456789000ull + reg * 0x1234567ull; - return state; + const auto flags = (expected == 0 ? LoweringExecution::kZero : 0u) | ((expected >> (operands.Wide ? 63 : 31)) != 0 ? LoweringExecution::kSign : 0u); + require((state.Flags & LoweringExecution::kFlagMask) == flags, "ANDN did not preserve its defined flag semantics"); + for (const auto value : state.RedZone) require(value == LoweringExecution::kSentinel, "ANDN overwrote the caller's red zone"); } void execution() { @@ -200,12 +95,12 @@ void execution() { for (std::uint8_t dst = 0; dst < 16; ++dst) { for (const auto pair : {std::pair{1, 6}, {dst, 10}, {11, dst}, {dst, dst}, {4, 9}, {9, 4}}) { const auto site = instruction(wide, dst, pair.first, pair.second); - Execution executable(site, 0); + const auto run = executable(site, 0); for (int variant = 0; variant < 3; ++variant) { - auto state = initialState(); + auto state = LoweringExecution::InitialState(); if (variant == 1) state.Input.fill(0); if (variant == 2) state.Input.fill(~std::uint64_t{0}); - executable.Run(state); + run.Run(state); check(state, *Codegen::DecodeAndn(site.data(), site.size()), state.Input[pair.second]); } } @@ -214,14 +109,14 @@ void execution() { Bytes site{0xC4, 0xE2, static_cast(wide ? 0xF0 : 0x70), 0xF2}; site.insert(site.end(), tail.begin(), tail.end()); constexpr std::uint64_t value = 0xFEDCBA9876543210ull; - Execution executable(site, value); - auto state = initialState(); - state.Input[6] = reinterpret_cast(executable.Memory()) - (tail[0] == 0x46 ? 16 : 0); - state.Input[2] = reinterpret_cast(executable.Memory()) - 3 * 8; + const auto run = executable(site, value); + auto state = LoweringExecution::InitialState(); + state.Input[6] = reinterpret_cast(run.Memory()) - (tail[0] == 0x46 ? 16 : 0); + state.Input[2] = reinterpret_cast(run.Memory()) - 3 * 8; state.Input[1] = 3; - state.Input[5] = reinterpret_cast(executable.Memory()) - 16; + state.Input[5] = reinterpret_cast(run.Memory()) - 16; state.Output[0] = value; - executable.Run(state); + run.Run(state); check(state, *Codegen::DecodeAndn(site.data(), site.size()), value); } } diff --git a/core/relinker/codegen/tests/BlsLoweringTests.cpp b/core/relinker/codegen/tests/BlsLoweringTests.cpp new file mode 100644 index 0000000000..a38f6b10d0 --- /dev/null +++ b/core/relinker/codegen/tests/BlsLoweringTests.cpp @@ -0,0 +1,170 @@ +#include "LoweringExecution.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using LoweringExecution::Bytes; +using Codegen::BlsOperation; + +constexpr std::array kOperations{BlsOperation::Reset, BlsOperation::Mask, BlsOperation::Isolate}; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +std::uint8_t extension(BlsOperation operation) { + return static_cast(operation); +} + +Bytes instruction(BlsOperation operation, bool wide, std::uint8_t dst, std::uint8_t src) { + return {0xC4, static_cast(0xE2 ^ ((src & 8) << 2)), + static_cast((wide ? 0x80 : 0) | ((~dst & 15) << 3)), 0xF3, + static_cast(0xC0 | (extension(operation) << 3) | (src & 7))}; +} + +void decoding() { + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (std::uint8_t src = 0; src < 16; ++src) { + const auto bytes = instruction(operation, wide, dst, src); + const auto operands = Codegen::DecodeBls(bytes.data(), bytes.size()); + require(operands && operands->Operation == operation && operands->Wide == wide && operands->Destination == dst && operands->Source == src && !operands->Memory, "BLS register decoding changed an operand"); + } + } + } + const Bytes memory{0x64, 0x67, 0xC4, 0x02, 0xA8, 0xF3, static_cast(0x84 | (extension(operation) << 3)), 0xAC, 0x10, 0, 0, 0}; + const auto operands = Codegen::DecodeBls(memory.data(), memory.size()); + require(operands && operands->Operation == operation && operands->Destination == 10 && operands->Memory && operands->Memory->RexIndexBase == 3 && operands->Memory->Prefixes == Bytes{0x64, 0x67}, "BLS memory extensions or prefixes were lost"); + } + for (const Bytes bytes : {Bytes{0x90}, Bytes{0xC4, 0xE2, 0xF0}, Bytes{0xC4, 0xE1, 0xF0, 0xF3, 0xC8}, Bytes{0xC4, 0xE2, 0xF0, 0xF3, 0xC0}, Bytes{0xC4, 0xE2, 0xF0, 0xF3, 0xE0}, Bytes{0xC4, 0xE2, 0xF0, 0xF2, 0xC8}}) + require(!Codegen::DecodeBls(bytes.data(), bytes.size()), "Unrelated, truncated or non-BLS opcode matched BLS"); + for (const Bytes bytes : {Bytes{0xC4, 0xE2, 0xF4, 0xF3, 0xC8}, Bytes{0xC4, 0xE2, 0xF1, 0xF3, 0xD0}, Bytes{0xC4, 0xE2, 0xF0, 0xF3, 0xD8, 0x90}, Bytes{0xC4, 0xE2, 0xF0, 0xF3, 0x8C}}) { + bool rejected = false; + try { (void)Codegen::DecodeBls(bytes.data(), bytes.size()); } + catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "Malformed BLS was accepted"); + } + const Bytes overflow{0xC4, 0xE2, 0xF0, 0xF3, 0x8C, 0x24, 0xFF, 0xFF, 0xFF, 0x7F}; + bool rejected = false; + try { + Codegen::StubBodyBuilder body; + Codegen::EmitBls(body, *Codegen::DecodeBls(overflow.data(), overflow.size())); + } catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "An overflowing stack displacement was silently wrapped"); +} + +void matching() { + const auto intel = Codegen::MakeAmd64OnlyInstructionMatcher(); + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + for (const auto operation : kOperations) { + const auto bytes = instruction(operation, true, 0, 6); + const auto name = Codegen::BlsName(operation); + require(!intel->Match(bytes.data(), bytes.size()), std::string("Intel mode rewrote ") + name); + const auto match = rosetta->Match(bytes.data(), bytes.size()); + require(match && match->InstructionName == name && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline && !match->Optional, std::string("Rosetta mode did not require a trampoline for ") + name); + const std::array, 2> sequence{bytes, bytes}; + require(rosetta->MatchSequence(sequence, {}).has_value(), std::string("A sequence could not move out of line for ") + name); + } +} + +#if LOWERING_EXECUTION_AVAILABLE + +using LoweringExecution::Execution; +using LoweringExecution::State; + +Execution executable(const Bytes& site, std::uint64_t memoryValue) { + const auto operands = Codegen::DecodeBls(site.data(), site.size()); + require(operands.has_value(), "Execution site was not BLS"); + return Execution(site, memoryValue, operands->Memory && operands->Memory->RipRelative, [](Codegen::StubBodyBuilder& body, const Bytes& relocated) { + Codegen::EmitBls(body, *Codegen::DecodeBls(relocated.data(), relocated.size())); + }); +} + +void check(const State& state, const Codegen::BlsOperands& operands, std::uint64_t source) { + const auto name = Codegen::BlsName(operands.Operation); + const auto mask = operands.Wide ? ~std::uint64_t{0} : 0xFFFFFFFFull; + source &= mask; + std::uint64_t expected = 0; + std::uint64_t flags = 0; + switch (operands.Operation) { + case BlsOperation::Reset: + expected = (source & (source - 1)) & mask; + flags = (expected == 0 ? LoweringExecution::kZero : 0u) | (source == 0 ? LoweringExecution::kCarry : 0u); + break; + case BlsOperation::Mask: + expected = (source ^ (source - 1)) & mask; + flags = source == 0 ? LoweringExecution::kCarry : 0u; + break; + case BlsOperation::Isolate: + expected = (source & (~source + 1)) & mask; + flags = (expected == 0 ? LoweringExecution::kZero : 0u) | (source != 0 ? LoweringExecution::kCarry : 0u); + break; + } + flags |= (expected >> (operands.Wide ? 63 : 31)) != 0 ? LoweringExecution::kSign : 0u; + for (std::size_t reg = 0; reg < 16; ++reg) + require(state.Output[reg] == (reg == operands.Destination ? expected : state.Input[reg]), std::string(name) + " changed a source/scratch register or produced an incorrect result"); + require((state.Flags & LoweringExecution::kFlagMask) == flags, std::string(name) + " did not produce its defined flags"); + for (const auto value : state.RedZone) require(value == LoweringExecution::kSentinel, std::string(name) + " overwrote the caller's red zone"); +} + +void execution() { + constexpr std::array values{0, 1, 0x6, 0x80000000ull, 0x8000000000000000ull, 0xFFFFFFFF00000000ull, 0xFFFFFFFFFFFFFFFFull, 0xABCD123456789000ull}; + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (const std::uint8_t src : {std::uint8_t{6}, dst, std::uint8_t{4}, std::uint8_t{10}}) { + const auto site = instruction(operation, wide, dst, src); + const auto run = executable(site, 0); + for (const auto value : values) { + auto state = LoweringExecution::InitialState(); + state.Input[src] = value; + run.Run(state); + check(state, *Codegen::DecodeBls(site.data(), site.size()), state.Input[src]); + } + } + } + for (const Bytes tail : {Bytes{0x06}, Bytes{0x46, 0x10}, Bytes{0x84, 0x24, 0x40, 0, 0, 0}, Bytes{0x04, 0xCA}, Bytes{0x85, 0x10, 0, 0, 0}, Bytes{0x05, 0, 0, 0, 0}}) { + for (const auto value : {std::uint64_t{0}, 0xFEDCBA9876543210ull, 0xFFFFFFFF00000000ull}) { + Bytes site{0xC4, 0xE2, static_cast(wide ? 0xF8 : 0x78), 0xF3, static_cast(tail[0] | (extension(operation) << 3))}; + site.insert(site.end(), tail.begin() + 1, tail.end()); + const auto run = executable(site, value); + auto state = LoweringExecution::InitialState(); + state.Input[6] = reinterpret_cast(run.Memory()) - (tail[0] == 0x46 ? 16 : 0); + state.Input[2] = reinterpret_cast(run.Memory()) - 3 * 8; + state.Input[1] = 3; + state.Input[5] = reinterpret_cast(run.Memory()) - 16; + state.Output[0] = value; + run.Run(state); + check(state, *Codegen::DecodeBls(site.data(), site.size()), value); + } + } + } + } +} +#else +void execution() { std::cout << "BLS execution checks require a Linux/macOS x86-64 build\n"; } +#endif + +} + +int main() { + try { + decoding(); + matching(); + execution(); + std::cout << "BLS lowering checks passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/relinker/codegen/tests/LoweringExecution.hpp b/core/relinker/codegen/tests/LoweringExecution.hpp new file mode 100644 index 0000000000..c020a91375 --- /dev/null +++ b/core/relinker/codegen/tests/LoweringExecution.hpp @@ -0,0 +1,149 @@ +#ifndef CODEGEN_TESTS_LOWERINGEXECUTION_HPP +#define CODEGEN_TESTS_LOWERINGEXECUTION_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) +#include +#define LOWERING_EXECUTION_AVAILABLE 1 +#else +#define LOWERING_EXECUTION_AVAILABLE 0 +#endif + +namespace LoweringExecution { + +using Bytes = std::vector; + +struct State { + std::array Input{}; + std::array Output{}; + std::uint64_t Flags = 0; + std::array RedZone{}; +}; + +inline constexpr std::uint64_t kSentinel = 0xBADC0FFEE0DDF00Dull; +inline constexpr std::uint64_t kFlagMask = 0x8C1u; +inline constexpr std::uint64_t kCarry = 0x1u; +inline constexpr std::uint64_t kZero = 0x40u; +inline constexpr std::uint64_t kSign = 0x80u; + +inline State InitialState() { + State state; + for (std::size_t reg = 0; reg < 16; ++reg) state.Input[reg] = 0xABCD123456789000ull + reg * 0x1234567ull; + return state; +} + +#if LOWERING_EXECUTION_AVAILABLE + +class Execution { +public: + using Emit = std::function; + + Execution(const Bytes& site, std::uint64_t memoryValue, bool ripRelative, const Emit& emit) { + for (auto reg : {3, 5, 12, 13, 14, 15}) push(static_cast(reg)); + memory(0x8D, 4, 4, -128); + rip({0x48, 0x89, 0x3D}, 0); + rip({0x48, 0x89, 0x25}, 8); + memory(0x89, 4, 7, 32); + bytes.insert(bytes.end(), {0x48, 0xB8}); + append64(kSentinel); + for (int displacement = -128; displacement < 0; displacement += 8) + memory(0x89, 0, 4, displacement); + memory(0x8B, 0, 7, 16 * 8); + memory(0x89, 0, 4, 64); + for (std::uint8_t reg = 0; reg < 16; ++reg) + if (reg != 4 && reg != 7) memory(0x8B, reg, 7, reg * 8); + memory(0x8B, 7, 7, 7 * 8); + const auto siteOffset = bytes.size(); + auto relocated = site; + if (ripRelative) { + const auto displacement = static_cast(4096 + 24 - (siteOffset + site.size())); + std::memcpy(relocated.data() + relocated.size() - 4, &displacement, sizeof(displacement)); + } + Codegen::StubBodyBuilder builder; + emit(builder, relocated); + auto body = builder.Finish(); + const auto stubOffset = bytes.size(); + Codegen::ApplyStubRelocations(body.Bytes, body.Relocations, siteOffset, stubOffset, 0); + bytes.insert(bytes.end(), body.Bytes.begin(), body.Bytes.begin() + static_cast(body.ReturnBranchOffset)); + rip({0x48, 0x89, 0x25}, 16); + rip({0x48, 0x8B, 0x25}, 8); + memory(0x8D, 4, 4, -256); + for (std::uint8_t reg = 0; reg < 16; ++reg) if (reg != 4) push(reg); + bytes.push_back(0x9C); + rip({0x48, 0x8B, 0x3D}, 0); + memory(0x8B, 0, 4, 0); + memory(0x89, 0, 7, 32 * 8); + int offset = 8; + for (int reg = 15; reg >= 0; --reg) { + if (reg == 4) continue; + memory(0x8B, 0, 4, offset); + memory(0x89, 0, 7, (16 + reg) * 8); + offset += 8; + } + rip({0x48, 0x8B, 0x05}, 16); + memory(0x89, 0, 7, 20 * 8); + for (int slot = 0; slot < 16; ++slot) { + memory(0x8B, 0, 4, 256 + 128 - 128 + slot * 8); + memory(0x89, 0, 7, (33 + slot) * 8); + } + rip({0x48, 0x8B, 0x25}, 8); + memory(0x8D, 4, 4, 128); + for (auto reg : {15, 14, 13, 12, 5, 3}) pop(static_cast(reg)); + bytes.push_back(0xC3); + if (bytes.size() >= 4096) throw std::runtime_error("Execution wrapper exceeded its code page"); + code = static_cast(mmap(nullptr, 8192, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + if (code == MAP_FAILED) throw std::runtime_error("Cannot allocate execution test pages"); + std::memcpy(code, bytes.data(), bytes.size()); + std::memcpy(code + 4096 + 24, &memoryValue, sizeof(memoryValue)); + if (mprotect(code, 4096, PROT_READ | PROT_EXEC) != 0) throw std::runtime_error("Cannot make lowering test code executable"); + } + + ~Execution() { if (code != MAP_FAILED) munmap(code, 8192); } + Execution(const Execution&) = delete; + Execution& operator=(const Execution&) = delete; + + void Run(State& state) const { reinterpret_cast(code)(&state); } + const void* Memory() const { return code + 4096 + 24; } + +private: + Bytes bytes; + std::uint8_t* code = reinterpret_cast(MAP_FAILED); + + void append32(std::int32_t value) { + for (unsigned shift = 0; shift < 32; shift += 8) bytes.push_back(static_cast(static_cast(value) >> shift)); + } + void append64(std::uint64_t value) { + for (unsigned shift = 0; shift < 64; shift += 8) bytes.push_back(static_cast(value >> shift)); + } + void rip(std::initializer_list opcode, int slot) { + bytes.insert(bytes.end(), opcode); + append32(static_cast(4096 + slot - (bytes.size() + 4))); + } + void push(std::uint8_t reg) { + if (reg >= 8) bytes.push_back(0x41); + bytes.push_back(static_cast(0x50 | (reg & 7))); + } + void pop(std::uint8_t reg) { + if (reg >= 8) bytes.push_back(0x41); + bytes.push_back(static_cast(0x58 | (reg & 7))); + } + void memory(std::uint8_t opcode, std::uint8_t reg, std::uint8_t base, int displacement) { + bytes.insert(bytes.end(), {static_cast(0x48 | ((reg & 8) >> 1) | ((base & 8) >> 3)), opcode, + static_cast(0x80 | ((reg & 7) << 3) | (base & 7))}); + if ((base & 7) == 4) bytes.push_back(0x24); + append32(displacement); + } +}; + +#endif + +} + +#endif From 0f56374dc3a8467b6ae8e6f41e40099c13672f68 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 17:03:30 +0530 Subject: [PATCH 231/318] test(relinker): build the BLS source list portably --- core/relinker/codegen/src/x86/BlsLowering.cpp | 6 +++--- core/relinker/codegen/tests/BlsLoweringTests.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/relinker/codegen/src/x86/BlsLowering.cpp b/core/relinker/codegen/src/x86/BlsLowering.cpp index 589a2b2db4..46a4a2c014 100644 --- a/core/relinker/codegen/src/x86/BlsLowering.cpp +++ b/core/relinker/codegen/src/x86/BlsLowering.cpp @@ -120,15 +120,15 @@ void EmitBls(StubBodyBuilder& body, const BlsOperands& operands) { body.Raw(nonzero); body.Raw(std::array{std::uint8_t{0xEB}, static_cast(zero.size())}); body.Raw(zero); + StackSlot(body, 0x8B, true, temporary, 8); if (operands.Destination == Stack) - StackSlot(body, 0x89, operands.Wide, result, 16); + StackSlot(body, 0x89, operands.Wide, result, 8); else Register(body, 0x8B, operands.Wide, operands.Destination, result); StackSlot(body, 0x8B, true, result, 0); - StackSlot(body, 0x8B, true, temporary, 8); StackSlot(body, 0x8D, true, Stack, FrameSize); if (operands.Destination == Stack) - StackSlot(body, 0x8B, operands.Wide, Stack, 16 - FrameSize); + StackSlot(body, 0x8B, operands.Wide, Stack, 8 - FrameSize); } } diff --git a/core/relinker/codegen/tests/BlsLoweringTests.cpp b/core/relinker/codegen/tests/BlsLoweringTests.cpp index a38f6b10d0..cbbbca28af 100644 --- a/core/relinker/codegen/tests/BlsLoweringTests.cpp +++ b/core/relinker/codegen/tests/BlsLoweringTests.cpp @@ -122,7 +122,7 @@ void execution() { for (const auto operation : kOperations) { for (bool wide : {false, true}) { for (std::uint8_t dst = 0; dst < 16; ++dst) { - for (const std::uint8_t src : {std::uint8_t{6}, dst, std::uint8_t{4}, std::uint8_t{10}}) { + for (const std::uint8_t src : std::array{6, dst, 4, 10}) { const auto site = instruction(operation, wide, dst, src); const auto run = executable(site, 0); for (const auto value : values) { @@ -134,7 +134,7 @@ void execution() { } } for (const Bytes tail : {Bytes{0x06}, Bytes{0x46, 0x10}, Bytes{0x84, 0x24, 0x40, 0, 0, 0}, Bytes{0x04, 0xCA}, Bytes{0x85, 0x10, 0, 0, 0}, Bytes{0x05, 0, 0, 0, 0}}) { - for (const auto value : {std::uint64_t{0}, 0xFEDCBA9876543210ull, 0xFFFFFFFF00000000ull}) { + for (const auto value : std::array{0, 0xFEDCBA9876543210ull, 0xFFFFFFFF00000000ull}) { Bytes site{0xC4, 0xE2, static_cast(wide ? 0xF8 : 0x78), 0xF3, static_cast(tail[0] | (extension(operation) << 3))}; site.insert(site.end(), tail.begin() + 1, tail.end()); const auto run = executable(site, value); From 42a65719cb12ccc0101f677a67a9c8851d584b4b Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 17:07:10 +0530 Subject: [PATCH 232/318] docs(relinker): record the BLS lowering --- docs/dev/debt/relinker.md | 7 ++++--- docs/user/USAGE.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 74b03184fc..1855f002a1 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,13 +46,14 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN is lowered, but RDRAND and other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK and BLSR are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment -- [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws; other BMI instructions are not lowered. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. +- [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. BEXTR, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX and SHRX are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 5e04c76bcf..94e7c27b2a 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -74,7 +74,7 @@ All switches are disabled by default. `unused-filter` defaults to `0`; `--rpath` | `--linux-fixed-base
` | Load the Linux executable at this fixed address instead of the default `0x400000`, given in decimal or hexadecimal (`0x`): every virtual address of the image is offset by it, so virtual address 0 is placed at the address, as the console places an executable. Requires a 64 KiB-aligned address from `0x10000` through `0x7fffffff0000` that is a multiple of every `PT_LOAD` alignment, and an image that ends in the user address space. The executable is written as `ET_EXEC`, without ASLR, as it is at the default address. Bundled modules keep their normal placement. Conflicts with `--windows`, `--macos` and `--lazy-binding`. | | `--macos-fixed-base
` | Map the lowest ELF load page of the macOS executable at this fixed address, given in decimal or hexadecimal (`0x`). Requires `--macos`, a 64 KiB-aligned address from `0x20000` through `0x7fffffff0000`, and an image that ends in the macOS user address space. `__PAGEZERO` ends 64 KiB below the address and the Mach-O header occupies those 64 KiB. The executable has no ASLR; an address the process cannot map fails to load. Bundled modules keep their normal placement. | | `--to-intel` | Convert supported AMD-only instructions in the executable and bundled modules. Unsupported instructions or unreachable conversion stubs cause an error. | -| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | +| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | | `unused-filter=0` | Keep all imported NID references. | | `unused-filter=1` | Filter unused non-PLT imports using control-flow and GOT access analysis, treating address-taken code as reachable; preserve PLT imports. | | `unused-filter=2` | Apply strict unused-import analysis and compact the PLT. Unsupported analysis cases cause an error. | From 96ccdc3bed1a9450a11ea41e71fa6ac5cf79e1fc Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 18:44:08 +0530 Subject: [PATCH 233/318] test(relinker): relink BLSR, BLSMSK and BLSI through the CLI and record the BLS lowering caveats --- core/relinker/relinker/tests/test_to_rosetta.py | 11 +++++++++++ docs/dev/debt/relinker.md | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/relinker/relinker/tests/test_to_rosetta.py b/core/relinker/relinker/tests/test_to_rosetta.py index ed0740868e..93024c8457 100644 --- a/core/relinker/relinker/tests/test_to_rosetta.py +++ b/core/relinker/relinker/tests/test_to_rosetta.py @@ -64,6 +64,17 @@ def main(): assert (andn not in output.read_bytes()) == lowered, f"{options} {'did not lower' if lowered else 'changed'} ANDN" if lowered: assert "ANDN" in result.stdout, result.stdout + for name, bls in (("BLSR", "c4e2f8f3ce"), ("BLSMSK", "c4e2f8f3d6"), ("BLSI", "c4e2f8f3de")): + code = bytes.fromhex(bls + "c3") + for target in ((), ("--macos",)): + for conversion in ((), ("--to-intel",), ("--to-rosetta",)): + options = target + conversion + result, output = relink(relinker, directory, name.lower() + "".join(options), *options, code=code) + assert result.returncode == 0, (options, result.stdout, result.stderr) + lowered = conversion == ("--to-rosetta",) + assert (code[:5] not in output.read_bytes()) == lowered, f"{options} {'did not lower' if lowered else 'changed'} {name}" + if lowered: + assert name in result.stdout, result.stdout masked = bytes.fromhex("c4e2752e07c3") for target in ((), ("--macos",)): for conversion in ((), ("--to-intel",), ("--to-rosetta",)): diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 1855f002a1..14d5d8a17b 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -53,7 +53,7 @@ - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. BEXTR, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX and SHRX are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX and SHRX are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. From d70efacd6eae43107717f81882c773dfed557665 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 17:49:04 +0530 Subject: [PATCH 234/318] feat(relinker): lower SHLX, SHRX, SARX and RORX under --to-rosetta --- core/relinker/CMakeLists.txt | 8 + .../include/codegen/x86/ScalarLowering.hpp | 4 +- .../include/codegen/x86/ShiftLowering.hpp | 31 +++ .../src/x86/Amd64OnlyInstructionMatcher.cpp | 7 + .../codegen/src/x86/ShiftLowering.cpp | 127 +++++++++ .../codegen/tests/LoweringExecution.hpp | 10 +- .../codegen/tests/ShiftLoweringTests.cpp | 245 ++++++++++++++++++ 7 files changed, 428 insertions(+), 4 deletions(-) create mode 100644 core/relinker/codegen/include/codegen/x86/ShiftLowering.hpp create mode 100644 core/relinker/codegen/src/x86/ShiftLowering.cpp create mode 100644 core/relinker/codegen/tests/ShiftLoweringTests.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 4432faa105..f708e064af 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -55,6 +55,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/RelrRelocations.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp @@ -106,6 +107,11 @@ if(BUILD_TESTING) add_test(NAME bls_lowering COMMAND bls_lowering_tests) set_tests_properties(bls_lowering PROPERTIES TIMEOUT 20) + add_test_executable(shift_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/ShiftLoweringTests.cpp) + target_link_libraries(shift_lowering_tests PRIVATE relinker_core) + add_test(NAME shift_lowering COMMAND shift_lowering_tests) + set_tests_properties(shift_lowering PROPERTIES TIMEOUT 60) + add_test_executable(masked_store_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/MaskedStoreLoweringTests.cpp) target_link_libraries(masked_store_lowering_tests PRIVATE relinker_core) add_test(NAME masked_store_lowering COMMAND masked_store_lowering_tests) @@ -258,6 +264,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -548,6 +555,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp index 363e8f4181..5627419ebc 100644 --- a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp @@ -72,9 +72,9 @@ inline std::uint8_t Scratch(std::array& used, const char* instruction) throw CodegenException(std::string(instruction) + " has no scratch register"); } -inline void Memory(StubBodyBuilder& body, std::uint8_t opcode, bool wide, const MemoryOperand& memory, std::uint8_t reg, const char* instruction) { +inline void Memory(StubBodyBuilder& body, std::uint8_t opcode, bool wide, const MemoryOperand& memory, std::uint8_t reg, const char* instruction, std::int32_t frame = FrameSize) { auto displacement = static_cast(memory.Displacement); - if (memory.StackBase) displacement += FrameSize; + if (memory.StackBase) displacement += frame; const bool address32 = std::find(memory.Prefixes.begin(), memory.Prefixes.end(), 0x67) != memory.Prefixes.end(); if (!address32 && displacement > std::numeric_limits::max()) throw CodegenException(std::string(instruction) + " stack displacement does not fit after spilling"); diff --git a/core/relinker/codegen/include/codegen/x86/ShiftLowering.hpp b/core/relinker/codegen/include/codegen/x86/ShiftLowering.hpp new file mode 100644 index 0000000000..abc086f8e1 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ShiftLowering.hpp @@ -0,0 +1,31 @@ +#ifndef CODEGEN_X86_SHIFTLOWERING_HPP +#define CODEGEN_X86_SHIFTLOWERING_HPP + +#include +#include + +namespace Codegen { + +enum class ShiftOperation : std::uint8_t { + Left, + LogicalRight, + ArithmeticRight, + RotateRight, +}; + +struct ShiftOperands { + ShiftOperation Operation; + std::uint8_t Destination; + std::uint8_t Source; + std::uint8_t Count; + bool Wide; + std::optional Memory; +}; + +[[nodiscard]] const char* ShiftName(ShiftOperation operation); +[[nodiscard]] std::optional DecodeShift(const std::uint8_t* data, std::size_t length); +void EmitShift(StubBodyBuilder& body, const ShiftOperands& operands); + +} + +#endif diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 73c91595e3..496121d571 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -259,6 +260,8 @@ std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const D return _trampoline("ANDN", instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitAndn(body, *andn); })); if (const auto bls = DecodeBls(instr.Data, instr.Length)) return _trampoline(BlsName(bls->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitBls(body, *bls); })); + if (const auto shift = DecodeShift(instr.Data, instr.Length)) + return _trampoline(ShiftName(shift->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitShift(body, *shift); })); if (instr.IsRdseed()) { std::vector replacement(instr.Data, instr.Data + instr.Length); replacement[instr.OpcodeOffset() + 2] &= static_cast(~(1u << X64OpcodeConstants::ModRmRegShift)); @@ -334,6 +337,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c EmitBls(body, *bls); return BlsName(bls->Operation); } + if (const auto shift = _target == Amd64OnlyTarget::Rosetta ? DecodeShift(instr.Data, instr.Length) : std::nullopt) { + EmitShift(body, *shift); + return ShiftName(shift->Operation); + } if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdtscp()) { body.Raw(_rdtscpReplacement(instr)); return kRdtscp.Name; diff --git a/core/relinker/codegen/src/x86/ShiftLowering.cpp b/core/relinker/codegen/src/x86/ShiftLowering.cpp new file mode 100644 index 0000000000..16dab8f686 --- /dev/null +++ b/core/relinker/codegen/src/x86/ShiftLowering.cpp @@ -0,0 +1,127 @@ +#include +#include + +namespace Codegen { + +using namespace ScalarLowering; + +namespace { + +constexpr std::uint8_t kCountRegister = 1; +constexpr std::int32_t kResultSlot = 0; +constexpr std::int32_t kCountSlot = 8; +constexpr std::int32_t kFlagsSlot = 16; +constexpr std::int32_t kStackResultSlot = 24; +constexpr std::uint8_t kPushFlags = 0x9C; +constexpr std::uint8_t kPopFlags = 0x9D; +constexpr std::int32_t kFrame = FrameSize + 16; + +std::uint8_t _shiftDigit(const ShiftOperation operation) { + switch (operation) { + case ShiftOperation::Left: return 4; + case ShiftOperation::LogicalRight: return 5; + case ShiftOperation::ArithmeticRight: return 7; + case ShiftOperation::RotateRight: return 1; + } + throw CodegenException("unknown shift operation"); +} + +void _loadSource(StubBodyBuilder& body, const ShiftOperands& operands, const std::uint8_t result, const char* name) { + if (operands.Memory) + Memory(body, 0x8B, operands.Wide, *operands.Memory, result, name, kFrame); + else if (operands.Source == Stack) + StackSlot(body, 0x8D, true, result, kFrame); + else + Register(body, 0x8B, operands.Wide, result, operands.Source); +} + +} + +const char* ShiftName(const ShiftOperation operation) { + switch (operation) { + case ShiftOperation::Left: return "SHLX"; + case ShiftOperation::LogicalRight: return "SHRX"; + case ShiftOperation::ArithmeticRight: return "SARX"; + case ShiftOperation::RotateRight: return "RORX"; + } + throw CodegenException("unknown shift operation"); +} + +std::optional DecodeShift(const std::uint8_t* data, const std::size_t length) { + std::size_t vex = 0; + while (vex < length && LegacyPrefix(data[vex])) ++vex; + if (vex + 5 > length || data[vex] != 0xC4) return std::nullopt; + const auto map = static_cast(data[vex + 1] & 0x1F); + const auto pp = static_cast(data[vex + 2] & 3); + ShiftOperation operation; + if (map == 2 && data[vex + 3] == 0xF7 && pp != 0) + operation = pp == 1 ? ShiftOperation::Left : pp == 2 ? ShiftOperation::ArithmeticRight : ShiftOperation::LogicalRight; + else if (map == 3 && data[vex + 3] == 0xF0 && pp == 3) + operation = ShiftOperation::RotateRight; + else + return std::nullopt; + const auto name = ShiftName(operation); + if ((data[vex + 2] & 4) != 0) throw CodegenException(std::string(name) + " requires VEX.L=0"); + const auto extensions = static_cast(~data[vex + 1]); + const auto modrm = data[vex + 4]; + const auto vvvv = static_cast((~data[vex + 2] >> 3) & 15); + if (operation == ShiftOperation::RotateRight && vvvv != 0) throw CodegenException("RORX requires VEX.vvvv=1111"); + ShiftOperands operands{ + operation, + static_cast(((modrm >> 3) & 7) | ((extensions & 0x80) >> 4)), + static_cast((modrm & 7) | ((extensions & 0x20) >> 2)), + vvvv, + (data[vex + 2] & 0x80) != 0, std::nullopt}; + std::size_t end = vex + 5; + if ((modrm >> 6) != 3) { + const auto rex = static_cast(((extensions & 0x20) >> 5) | ((extensions & 0x40) >> 5)); + operands.Memory = DecodeMemoryOperand(data, length, vex + 4, rex, {data, data + vex}); + end = vex + 4 + operands.Memory->EncodedSize; + } + if (operation == ShiftOperation::RotateRight) { + if (end >= length) throw CodegenException("RORX is missing its immediate"); + operands.Count = data[end]; + ++end; + } + if (end != length) throw CodegenException(std::string(name) + " has trailing bytes"); + return operands; +} + +void EmitShift(StubBodyBuilder& body, const ShiftOperands& operands) { + const auto name = ShiftName(operands.Operation); + const bool rotate = operands.Operation == ShiftOperation::RotateRight; + auto used = operands.Memory ? UsedRegisters({operands.Destination, operands.Count, kCountRegister}, operands.Memory) + : UsedRegisters({operands.Destination, operands.Source, operands.Count, kCountRegister}, std::nullopt); + const auto result = Scratch(used, name); + StackSlot(body, 0x8D, true, Stack, -kFrame); + StackSlot(body, 0x89, true, result, kResultSlot); + if (!rotate) StackSlot(body, 0x89, true, kCountRegister, kCountSlot); + body.Raw(std::array{kPushFlags}); + StackSlot(body, 0x8F, false, 0, kFlagsSlot); + _loadSource(body, operands, result, name); + if (rotate) { + Register(body, 0xC1, operands.Wide, _shiftDigit(operands.Operation), result); + body.Raw(std::array{operands.Count}); + } else { + if (operands.Count == Stack) + StackSlot(body, 0x8D, true, kCountRegister, kFrame); + else if (operands.Count != kCountRegister) + Register(body, 0x8B, true, kCountRegister, operands.Count); + Register(body, 0xD3, operands.Wide, _shiftDigit(operands.Operation), result); + } + if (operands.Destination == Stack) + StackSlot(body, 0x89, operands.Wide, result, kStackResultSlot); + else if (!rotate && operands.Destination == kCountRegister) + StackSlot(body, 0x89, true, result, kCountSlot); + else + Register(body, 0x8B, operands.Wide, operands.Destination, result); + StackSlot(body, 0xFF, false, 6, kFlagsSlot); + body.Raw(std::array{kPopFlags}); + if (!rotate) StackSlot(body, 0x8B, true, kCountRegister, kCountSlot); + StackSlot(body, 0x8B, true, result, kResultSlot); + StackSlot(body, 0x8D, true, Stack, kFrame); + if (operands.Destination == Stack) + StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot - kFrame); +} + +} diff --git a/core/relinker/codegen/tests/LoweringExecution.hpp b/core/relinker/codegen/tests/LoweringExecution.hpp index c020a91375..28eb241865 100644 --- a/core/relinker/codegen/tests/LoweringExecution.hpp +++ b/core/relinker/codegen/tests/LoweringExecution.hpp @@ -25,6 +25,7 @@ struct State { std::array Output{}; std::uint64_t Flags = 0; std::array RedZone{}; + std::uint64_t InputFlags = 0; }; inline constexpr std::uint64_t kSentinel = 0xBADC0FFEE0DDF00Dull; @@ -33,9 +34,12 @@ inline constexpr std::uint64_t kCarry = 0x1u; inline constexpr std::uint64_t kZero = 0x40u; inline constexpr std::uint64_t kSign = 0x80u; +inline constexpr std::uint64_t kBaseFlags = 0x202u; + inline State InitialState() { State state; for (std::size_t reg = 0; reg < 16; ++reg) state.Input[reg] = 0xABCD123456789000ull + reg * 0x1234567ull; + state.InputFlags = kBaseFlags | kFlagMask; return state; } @@ -45,12 +49,14 @@ class Execution { public: using Emit = std::function; - Execution(const Bytes& site, std::uint64_t memoryValue, bool ripRelative, const Emit& emit) { + Execution(const Bytes& site, std::uint64_t memoryValue, bool ripRelative, const Emit& emit, std::size_t trailing = 0) { for (auto reg : {3, 5, 12, 13, 14, 15}) push(static_cast(reg)); memory(0x8D, 4, 4, -128); rip({0x48, 0x89, 0x3D}, 0); rip({0x48, 0x89, 0x25}, 8); memory(0x89, 4, 7, 32); + memory(0xFF, 6, 7, 49 * 8); + bytes.push_back(0x9D); bytes.insert(bytes.end(), {0x48, 0xB8}); append64(kSentinel); for (int displacement = -128; displacement < 0; displacement += 8) @@ -64,7 +70,7 @@ public: auto relocated = site; if (ripRelative) { const auto displacement = static_cast(4096 + 24 - (siteOffset + site.size())); - std::memcpy(relocated.data() + relocated.size() - 4, &displacement, sizeof(displacement)); + std::memcpy(relocated.data() + relocated.size() - 4 - trailing, &displacement, sizeof(displacement)); } Codegen::StubBodyBuilder builder; emit(builder, relocated); diff --git a/core/relinker/codegen/tests/ShiftLoweringTests.cpp b/core/relinker/codegen/tests/ShiftLoweringTests.cpp new file mode 100644 index 0000000000..850330713c --- /dev/null +++ b/core/relinker/codegen/tests/ShiftLoweringTests.cpp @@ -0,0 +1,245 @@ +#include "LoweringExecution.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using LoweringExecution::Bytes; +using Codegen::ShiftOperation; + +constexpr std::array kShifts{ShiftOperation::Left, ShiftOperation::LogicalRight, ShiftOperation::ArithmeticRight}; +constexpr std::array kOperations{ShiftOperation::Left, ShiftOperation::LogicalRight, ShiftOperation::ArithmeticRight, ShiftOperation::RotateRight}; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +std::uint8_t pp(ShiftOperation operation) { + switch (operation) { + case ShiftOperation::Left: return 1; + case ShiftOperation::ArithmeticRight: return 2; + case ShiftOperation::LogicalRight: return 3; + case ShiftOperation::RotateRight: return 3; + } + return 0; +} + +Bytes shift(ShiftOperation operation, bool wide, std::uint8_t dst, std::uint8_t src, std::uint8_t count) { + return {0xC4, static_cast(0xE2 ^ ((dst & 8) << 4) ^ ((src & 8) << 2)), + static_cast((wide ? 0x80 : 0) | ((~count & 15) << 3) | pp(operation)), 0xF7, + static_cast(0xC0 | ((dst & 7) << 3) | (src & 7))}; +} + +Bytes rotate(bool wide, std::uint8_t dst, std::uint8_t src, std::uint8_t immediate) { + return {0xC4, static_cast(0xE3 ^ ((dst & 8) << 4) ^ ((src & 8) << 2)), + static_cast((wide ? 0x80 : 0) | 0x78 | 3), 0xF0, + static_cast(0xC0 | ((dst & 7) << 3) | (src & 7)), immediate}; +} + +Bytes memorySite(ShiftOperation operation, bool wide, const Bytes& tail, std::uint8_t immediate) { + const bool rorx = operation == ShiftOperation::RotateRight; + Bytes site{0xC4, static_cast(rorx ? 0xE3 : 0xE2), static_cast((wide ? 0x80 : 0) | (rorx ? 0x78 : 0x70) | pp(operation)), static_cast(rorx ? 0xF0 : 0xF7)}; + site.insert(site.end(), tail.begin(), tail.end()); + if (rorx) site.push_back(immediate); + return site; +} + +void decoding() { + for (const auto operation : kShifts) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (std::uint8_t src = 0; src < 16; ++src) { + for (std::uint8_t count = 0; count < 16; ++count) { + const auto bytes = shift(operation, wide, dst, src, count); + const auto operands = Codegen::DecodeShift(bytes.data(), bytes.size()); + require(operands && operands->Operation == operation && operands->Wide == wide && operands->Destination == dst && operands->Source == src && operands->Count == count && !operands->Memory, "shift register decoding changed an operand"); + } + } + } + } + } + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (std::uint8_t src = 0; src < 16; ++src) { + for (const std::uint8_t immediate : std::array{0, 1, 63, 255}) { + const auto bytes = rotate(wide, dst, src, immediate); + const auto operands = Codegen::DecodeShift(bytes.data(), bytes.size()); + require(operands && operands->Operation == ShiftOperation::RotateRight && operands->Wide == wide && operands->Destination == dst && operands->Source == src && operands->Count == immediate && !operands->Memory, "RORX register decoding changed an operand"); + } + } + } + } + const Bytes shiftMemory{0x64, 0x67, 0xC4, 0x02, 0xA9, 0xF7, 0x8C, 0xAC, 0x10, 0, 0, 0}; + auto operands = Codegen::DecodeShift(shiftMemory.data(), shiftMemory.size()); + require(operands && operands->Operation == ShiftOperation::Left && operands->Destination == 9 && operands->Count == 10 && operands->Memory && operands->Memory->RexIndexBase == 3 && operands->Memory->Prefixes == Bytes{0x64, 0x67}, "shift memory extensions or prefixes were lost"); + const Bytes rotateMemory{0xC4, 0x03, 0xFB, 0xF0, 0x8C, 0xAC, 0x10, 0, 0, 0, 0x21}; + operands = Codegen::DecodeShift(rotateMemory.data(), rotateMemory.size()); + require(operands && operands->Operation == ShiftOperation::RotateRight && operands->Destination == 9 && operands->Count == 0x21 && operands->Memory && operands->Memory->InstructionLength == rotateMemory.size(), "RORX memory operand or immediate was lost"); + for (const Bytes bytes : {Bytes{0x90}, Bytes{0xC4, 0xE2, 0xF1}, Bytes{0xC4, 0xE1, 0xF1, 0xF7, 0xC0}, Bytes{0xC4, 0xE2, 0xF0, 0xF7, 0xC0}, Bytes{0xC4, 0xE3, 0xFA, 0xF0, 0xC0, 0}, Bytes{0xC4, 0xE2, 0xFB, 0xF0, 0xC0, 0}}) + require(!Codegen::DecodeShift(bytes.data(), bytes.size()), "Unrelated, truncated, BEXTR or wrong-map opcode matched a shift"); + for (const Bytes bytes : {Bytes{0xC4, 0xE2, 0xF5, 0xF7, 0xC0}, Bytes{0xC4, 0xE2, 0xF1, 0xF7, 0xC0, 0x90}, Bytes{0xC4, 0xE2, 0xF1, 0xF7, 0x84}, Bytes{0xC4, 0xE3, 0xF3, 0xF0, 0xC0, 0}, Bytes{0xC4, 0xE3, 0xFB, 0xF0, 0xC0}, Bytes{0xC4, 0xE3, 0xFB, 0xF0, 0xC0, 0, 0}}) { + bool rejected = false; + try { (void)Codegen::DecodeShift(bytes.data(), bytes.size()); } + catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "Malformed shift was accepted"); + } + const Bytes overflow{0xC4, 0xE2, 0xF1, 0xF7, 0x8C, 0x24, 0xFF, 0xFF, 0xFF, 0x7F}; + bool rejected = false; + try { + Codegen::StubBodyBuilder body; + Codegen::EmitShift(body, *Codegen::DecodeShift(overflow.data(), overflow.size())); + } catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "An overflowing stack displacement was silently wrapped"); +} + +void matching() { + const auto intel = Codegen::MakeAmd64OnlyInstructionMatcher(); + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + for (const auto operation : kOperations) { + const auto bytes = operation == ShiftOperation::RotateRight ? rotate(true, 0, 6, 7) : shift(operation, true, 0, 6, 2); + const auto name = Codegen::ShiftName(operation); + require(!intel->Match(bytes.data(), bytes.size()), std::string("Intel mode rewrote ") + name); + const auto match = rosetta->Match(bytes.data(), bytes.size()); + require(match && match->InstructionName == name && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline && !match->Optional, std::string("Rosetta mode did not require a trampoline for ") + name); + const std::array, 2> sequence{bytes, bytes}; + require(rosetta->MatchSequence(sequence, {}).has_value(), std::string("A sequence could not move out of line for ") + name); + } +} + +#if LOWERING_EXECUTION_AVAILABLE + +using LoweringExecution::Execution; +using LoweringExecution::State; + +Execution executable(const Bytes& site, std::uint64_t memoryValue) { + const auto operands = Codegen::DecodeShift(site.data(), site.size()); + require(operands.has_value(), "Execution site was not a shift"); + return Execution(site, memoryValue, operands->Memory && operands->Memory->RipRelative, [](Codegen::StubBodyBuilder& body, const Bytes& relocated) { + Codegen::EmitShift(body, *Codegen::DecodeShift(relocated.data(), relocated.size())); + }, operands->Operation == ShiftOperation::RotateRight ? 1 : 0); +} + +std::uint64_t expected(const Codegen::ShiftOperands& operands, std::uint64_t source, std::uint64_t count) { + const unsigned bits = operands.Wide ? 64 : 32; + const auto mask = operands.Wide ? ~std::uint64_t{0} : 0xFFFFFFFFull; + source &= mask; + const unsigned amount = static_cast(count & (bits - 1)); + switch (operands.Operation) { + case ShiftOperation::Left: return (source << amount) & mask; + case ShiftOperation::LogicalRight: return source >> amount; + case ShiftOperation::ArithmeticRight: { + const bool negative = (source >> (bits - 1)) != 0; + const auto shifted = source >> amount; + return negative && amount != 0 ? (shifted | (mask << (bits - amount))) & mask : shifted; + } + case ShiftOperation::RotateRight: return amount == 0 ? source : ((source >> amount) | (source << (bits - amount))) & mask; + } + return 0; +} + +void check(const State& state, const Codegen::ShiftOperands& operands, std::uint64_t source, std::uint64_t count) { + const auto name = Codegen::ShiftName(operands.Operation); + const auto result = expected(operands, source, count); + for (std::size_t reg = 0; reg < 16; ++reg) { + const auto wanted = reg == operands.Destination ? result : state.Input[reg]; + if (state.Output[reg] != wanted) + throw std::runtime_error(std::string(name) + (operands.Wide ? " 64" : " 32") + " dst " + std::to_string(operands.Destination) + " src " + std::to_string(operands.Source) + " count " + std::to_string(operands.Count) + (operands.Memory ? " memory" : "") + ": register " + std::to_string(reg) + " is " + std::to_string(state.Output[reg]) + ", wanted " + std::to_string(wanted) + " (source " + std::to_string(source) + ", count value " + std::to_string(count) + ")"); + } + require((state.Flags & LoweringExecution::kFlagMask) == (state.InputFlags & LoweringExecution::kFlagMask), std::string(name) + " did not preserve the flags"); + for (const auto value : state.RedZone) require(value == LoweringExecution::kSentinel, std::string(name) + " overwrote the caller's red zone"); +} + +void run(const Execution& execution, const Codegen::ShiftOperands& operands, State state, std::uint64_t memoryValue) { + for (const auto flags : {LoweringExecution::kBaseFlags | LoweringExecution::kFlagMask, LoweringExecution::kBaseFlags}) { + state.InputFlags = flags; + auto trial = state; + execution.Run(trial); + const auto source = operands.Memory ? memoryValue : trial.Input[operands.Source]; + const auto count = operands.Operation == ShiftOperation::RotateRight ? operands.Count : trial.Input[operands.Count]; + check(trial, operands, source, count); + } +} + +void execution() { + constexpr std::array values{0, 1, 0x80000000ull, 0x8000000000000000ull, 0xFFFFFFFFFFFFFFFFull, 0xABCD123456789000ull}; + constexpr std::array counts{0, 1, 31, 32, 63, 64, 0xFF}; + for (const auto operation : kShifts) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (const auto pair : std::array, 9>{{{6, 2}, {dst, 10}, {11, dst}, {dst, dst}, {4, 9}, {9, 4}, {1, 6}, {6, 1}, {1, 1}}}) { + const auto site = shift(operation, wide, dst, pair.first, pair.second); + const auto operands = *Codegen::DecodeShift(site.data(), site.size()); + const auto execution = executable(site, 0); + for (const auto value : values) { + for (const auto count : counts) { + auto state = LoweringExecution::InitialState(); + state.Input[pair.first] = value; + state.Input[pair.second] = count; + run(execution, operands, state, 0); + } + } + } + } + } + } + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (const std::uint8_t src : std::array{6, dst, 4, 10}) { + for (const std::uint8_t immediate : std::array{0, 1, 31, 32, 63, 255}) { + const auto site = rotate(wide, dst, src, immediate); + const auto operands = *Codegen::DecodeShift(site.data(), site.size()); + const auto execution = executable(site, 0); + for (const auto value : values) { + auto state = LoweringExecution::InitialState(); + state.Input[src] = value; + run(execution, operands, state, 0); + } + } + } + } + } + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (const Bytes tail : {Bytes{0x06}, Bytes{0x46, 0x10}, Bytes{0x84, 0x24, 0x40, 0, 0, 0}, Bytes{0x04, 0xCA}, Bytes{0x85, 0x10, 0, 0, 0}, Bytes{0x05, 0, 0, 0, 0}}) { + for (const auto value : std::array{0, 0xFEDCBA9876543210ull, 0xFFFFFFFF00000000ull}) { + const auto site = memorySite(operation, wide, tail, 13); + const auto operands = *Codegen::DecodeShift(site.data(), site.size()); + const auto execution = executable(site, value); + auto state = LoweringExecution::InitialState(); + state.Input[6] = reinterpret_cast(execution.Memory()) - (tail[0] == 0x46 ? 16 : 0); + state.Input[2] = reinterpret_cast(execution.Memory()) - 3 * 8; + state.Input[1] = 3; + state.Input[5] = reinterpret_cast(execution.Memory()) - 16; + state.Output[0] = value; + run(execution, operands, state, value); + } + } + } + } +} +#else +void execution() { std::cout << "shift execution checks require a Linux/macOS x86-64 build\n"; } +#endif + +} + +int main() { + try { + decoding(); + matching(); + execution(); + std::cout << "shift lowering checks passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From bc6710735a46e78c03c77df816dde8e2dab90de5 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 17:50:04 +0530 Subject: [PATCH 235/318] docs(relinker): record the shift lowering --- docs/dev/debt/relinker.md | 7 ++++--- docs/user/USAGE.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 746cb8a474..1464883fd2 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,14 +46,15 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX and RORX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, PDEP, PEXT, RORX, SARX, SHLX and SHRX are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, MULX, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. BEXTR (the `pp=0` form of the same opcode), BZHI, MULX, PDEP and PEXT are not lowered. Balatro (PPSA21402, #1142) stopped on `shlxq` after the ANDN lowering; this lowering has not been run on that title. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 94e7c27b2a..d63dd7299e 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -74,7 +74,7 @@ All switches are disabled by default. `unused-filter` defaults to `0`; `--rpath` | `--linux-fixed-base
` | Load the Linux executable at this fixed address instead of the default `0x400000`, given in decimal or hexadecimal (`0x`): every virtual address of the image is offset by it, so virtual address 0 is placed at the address, as the console places an executable. Requires a 64 KiB-aligned address from `0x10000` through `0x7fffffff0000` that is a multiple of every `PT_LOAD` alignment, and an image that ends in the user address space. The executable is written as `ET_EXEC`, without ASLR, as it is at the default address. Bundled modules keep their normal placement. Conflicts with `--windows`, `--macos` and `--lazy-binding`. | | `--macos-fixed-base
` | Map the lowest ELF load page of the macOS executable at this fixed address, given in decimal or hexadecimal (`0x`). Requires `--macos`, a 64 KiB-aligned address from `0x20000` through `0x7fffffff0000`, and an image that ends in the macOS user address space. `__PAGEZERO` ends 64 KiB below the address and the Mach-O header occupies those 64 KiB. The executable has no ASLR; an address the process cannot map fails to load. Bundled modules keep their normal placement. | | `--to-intel` | Convert supported AMD-only instructions in the executable and bundled modules. Unsupported instructions or unreachable conversion stubs cause an error. | -| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | +| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, SHLX, SHRX, SARX and RORX become flag-preserving scalar shifts and rotates, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | | `unused-filter=0` | Keep all imported NID references. | | `unused-filter=1` | Filter unused non-PLT imports using control-flow and GOT access analysis, treating address-taken code as reachable; preserve PLT imports. | | `unused-filter=2` | Apply strict unused-import analysis and compact the PLT. Unsupported analysis cases cause an error. | From 5d3b53fc831fc047e3a05ac7700c03a97b2176bd Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 19:12:08 +0530 Subject: [PATCH 236/318] fix(relinker): keep the RORX immediate out of the shift lowering's register set --- .../codegen/include/codegen/x86/ScalarLowering.hpp | 5 ++++- core/relinker/codegen/src/x86/ShiftLowering.cpp | 8 ++++++-- core/relinker/codegen/tests/ShiftLoweringTests.cpp | 5 ++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp index 5627419ebc..f510a32ad8 100644 --- a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp @@ -50,7 +50,10 @@ inline void StackSlot(StubBodyBuilder& body, std::uint8_t opcode, bool wide, std inline std::array UsedRegisters(std::initializer_list registers, const std::optional& memory) { std::array used{}; used[Stack] = true; - for (const auto reg : registers) used[reg] = true; + for (const auto reg : registers) { + if (reg >= used.size()) throw CodegenException("register index " + std::to_string(reg) + " is out of range"); + used[reg] = true; + } if (!memory) return used; if (memory->Rm == 4) { const auto base = static_cast((memory->Sib & 7) | ((memory->RexIndexBase & 1) << 3)); diff --git a/core/relinker/codegen/src/x86/ShiftLowering.cpp b/core/relinker/codegen/src/x86/ShiftLowering.cpp index 16dab8f686..925731e57d 100644 --- a/core/relinker/codegen/src/x86/ShiftLowering.cpp +++ b/core/relinker/codegen/src/x86/ShiftLowering.cpp @@ -90,8 +90,12 @@ std::optional DecodeShift(const std::uint8_t* data, const std::si void EmitShift(StubBodyBuilder& body, const ShiftOperands& operands) { const auto name = ShiftName(operands.Operation); const bool rotate = operands.Operation == ShiftOperation::RotateRight; - auto used = operands.Memory ? UsedRegisters({operands.Destination, operands.Count, kCountRegister}, operands.Memory) - : UsedRegisters({operands.Destination, operands.Source, operands.Count, kCountRegister}, std::nullopt); + std::array used{}; + if (rotate) + used = operands.Memory ? UsedRegisters({operands.Destination}, operands.Memory) : UsedRegisters({operands.Destination, operands.Source}, std::nullopt); + else + used = operands.Memory ? UsedRegisters({operands.Destination, operands.Count, kCountRegister}, operands.Memory) + : UsedRegisters({operands.Destination, operands.Source, operands.Count, kCountRegister}, std::nullopt); const auto result = Scratch(used, name); StackSlot(body, 0x8D, true, Stack, -kFrame); StackSlot(body, 0x89, true, result, kResultSlot); diff --git a/core/relinker/codegen/tests/ShiftLoweringTests.cpp b/core/relinker/codegen/tests/ShiftLoweringTests.cpp index 850330713c..d0f3b04f2d 100644 --- a/core/relinker/codegen/tests/ShiftLoweringTests.cpp +++ b/core/relinker/codegen/tests/ShiftLoweringTests.cpp @@ -194,7 +194,10 @@ void execution() { for (bool wide : {false, true}) { for (std::uint8_t dst = 0; dst < 16; ++dst) { for (const std::uint8_t src : std::array{6, dst, 4, 10}) { - for (const std::uint8_t immediate : std::array{0, 1, 31, 32, 63, 255}) { + const bool every = (dst == 0 && src == 6) || dst == src; + for (int value = 0; value < 256; ++value) { + const auto immediate = static_cast(value); + if (!every && immediate > 1 && immediate != 31 && immediate != 32 && immediate != 63 && immediate != 255) continue; const auto site = rotate(wide, dst, src, immediate); const auto operands = *Codegen::DecodeShift(site.data(), site.size()); const auto execution = executable(site, 0); From 7f0b73575bd7787a76eba6e74f22ddd8720fb737 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Thu, 29 Oct 2026 18:44:28 +0530 Subject: [PATCH 237/318] fix(relinker): restore an RSP destination in one load in the shift stub, list MULX as lowered and drop the title note --- core/relinker/codegen/src/x86/ShiftLowering.cpp | 5 +++-- docs/dev/debt/relinker.md | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/relinker/codegen/src/x86/ShiftLowering.cpp b/core/relinker/codegen/src/x86/ShiftLowering.cpp index 925731e57d..22f42615e3 100644 --- a/core/relinker/codegen/src/x86/ShiftLowering.cpp +++ b/core/relinker/codegen/src/x86/ShiftLowering.cpp @@ -123,9 +123,10 @@ void EmitShift(StubBodyBuilder& body, const ShiftOperands& operands) { body.Raw(std::array{kPopFlags}); if (!rotate) StackSlot(body, 0x8B, true, kCountRegister, kCountSlot); StackSlot(body, 0x8B, true, result, kResultSlot); - StackSlot(body, 0x8D, true, Stack, kFrame); if (operands.Destination == Stack) - StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot - kFrame); + StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot); + else + StackSlot(body, 0x8D, true, Stack, kFrame); } } diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 1464883fd2..ca3116a64a 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,15 +46,15 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX and RORX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, MULX, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. -- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. BEXTR (the `pp=0` form of the same opcode), BZHI, MULX, PDEP and PEXT are not lowered. Balatro (PPSA21402, #1142) stopped on `shlxq` after the ANDN lowering; this lowering has not been run on that title. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. BEXTR (the `pp=0` form of the same opcode), BZHI, PDEP and PEXT are not lowered. The execution test runs natively on x86-64 or under Rosetta and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. From cf78130042ef92fbb185a7efbf0fd7da056b4bea Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 23:03:33 +0530 Subject: [PATCH 238/318] feat(relinker): lower BZHI and BEXTR under --to-rosetta --- core/relinker/CMakeLists.txt | 8 + .../include/codegen/x86/BitfieldLowering.hpp | 29 +++ .../src/x86/Amd64OnlyInstructionMatcher.cpp | 7 + .../codegen/src/x86/BitfieldLowering.cpp | 158 +++++++++++++ .../codegen/tests/BitfieldLoweringTests.cpp | 211 ++++++++++++++++++ .../relinker/tests/test_to_rosetta.py | 2 +- docs/dev/debt/relinker.md | 9 +- docs/user/USAGE.md | 2 +- 8 files changed, 420 insertions(+), 6 deletions(-) create mode 100644 core/relinker/codegen/include/codegen/x86/BitfieldLowering.hpp create mode 100644 core/relinker/codegen/src/x86/BitfieldLowering.cpp create mode 100644 core/relinker/codegen/tests/BitfieldLoweringTests.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index f708e064af..91a96ed9ba 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -56,6 +56,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BitfieldLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/RelrRelocations.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp @@ -112,6 +113,11 @@ if(BUILD_TESTING) add_test(NAME shift_lowering COMMAND shift_lowering_tests) set_tests_properties(shift_lowering PROPERTIES TIMEOUT 60) + add_test_executable(bitfield_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/BitfieldLoweringTests.cpp) + target_link_libraries(bitfield_lowering_tests PRIVATE relinker_core) + add_test(NAME bitfield_lowering COMMAND bitfield_lowering_tests) + set_tests_properties(bitfield_lowering PROPERTIES TIMEOUT 60) + add_test_executable(masked_store_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/MaskedStoreLoweringTests.cpp) target_link_libraries(masked_store_lowering_tests PRIVATE relinker_core) add_test(NAME masked_store_lowering COMMAND masked_store_lowering_tests) @@ -265,6 +271,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BitfieldLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -556,6 +563,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BitfieldLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/BitfieldLowering.hpp b/core/relinker/codegen/include/codegen/x86/BitfieldLowering.hpp new file mode 100644 index 0000000000..6765d45a9c --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/BitfieldLowering.hpp @@ -0,0 +1,29 @@ +#ifndef CODEGEN_X86_BITFIELDLOWERING_HPP +#define CODEGEN_X86_BITFIELDLOWERING_HPP + +#include +#include + +namespace Codegen { + +enum class BitfieldOperation : std::uint8_t { + ZeroHigh, + Extract, +}; + +struct BitfieldOperands { + BitfieldOperation Operation; + std::uint8_t Destination; + std::uint8_t Source; + std::uint8_t Control; + bool Wide; + std::optional Memory; +}; + +[[nodiscard]] const char* BitfieldName(BitfieldOperation operation); +[[nodiscard]] std::optional DecodeBitfield(const std::uint8_t* data, std::size_t length); +void EmitBitfield(StubBodyBuilder& body, const BitfieldOperands& operands); + +} + +#endif diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 496121d571..4749d20165 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -262,6 +263,8 @@ std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const D return _trampoline(BlsName(bls->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitBls(body, *bls); })); if (const auto shift = DecodeShift(instr.Data, instr.Length)) return _trampoline(ShiftName(shift->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitShift(body, *shift); })); + if (const auto bitfield = DecodeBitfield(instr.Data, instr.Length)) + return _trampoline(BitfieldName(bitfield->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitBitfield(body, *bitfield); })); if (instr.IsRdseed()) { std::vector replacement(instr.Data, instr.Data + instr.Length); replacement[instr.OpcodeOffset() + 2] &= static_cast(~(1u << X64OpcodeConstants::ModRmRegShift)); @@ -341,6 +344,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c EmitShift(body, *shift); return ShiftName(shift->Operation); } + if (const auto bitfield = _target == Amd64OnlyTarget::Rosetta ? DecodeBitfield(instr.Data, instr.Length) : std::nullopt) { + EmitBitfield(body, *bitfield); + return BitfieldName(bitfield->Operation); + } if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdtscp()) { body.Raw(_rdtscpReplacement(instr)); return kRdtscp.Name; diff --git a/core/relinker/codegen/src/x86/BitfieldLowering.cpp b/core/relinker/codegen/src/x86/BitfieldLowering.cpp new file mode 100644 index 0000000000..1d9fb3e545 --- /dev/null +++ b/core/relinker/codegen/src/x86/BitfieldLowering.cpp @@ -0,0 +1,158 @@ +#include +#include + +namespace Codegen { + +using namespace ScalarLowering; + +namespace { + +constexpr std::uint8_t kControlRegister = 1; +constexpr std::int32_t kResultSlot = 0; +constexpr std::int32_t kControlSlot = 8; +constexpr std::int32_t kMaskSlot = 16; +constexpr std::int32_t kStackResultSlot = 24; +constexpr std::int32_t kFrame = FrameSize + 16; +constexpr std::uint8_t kJumpBelow = 0x72; +constexpr std::uint8_t kJumpAboveOrEqual = 0x73; +constexpr std::uint8_t kJump = 0xEB; +constexpr std::uint8_t kSetCarry = 0xF9; + +std::uint8_t _bits(const BitfieldOperands& operands) { + return operands.Wide ? 64 : 32; +} + +std::vector _compareControlByte(const std::uint8_t rm, const std::uint8_t bits) { + return {0x80, static_cast(0xF8 | rm), bits}; +} + +std::vector _lowMask(const BitfieldOperands& operands, const std::uint8_t mask) { + auto bytes = RegisterBytes(0xD3, operands.Wide, 4, mask); + const auto invert = RegisterBytes(0xF7, operands.Wide, 2, mask); + bytes.insert(bytes.end(), invert.begin(), invert.end()); + return bytes; +} + +void _append(std::vector& bytes, const std::vector& more) { + bytes.insert(bytes.end(), more.begin(), more.end()); +} + +void _loadSource(StubBodyBuilder& body, const BitfieldOperands& operands, const std::uint8_t result, const char* name) { + if (operands.Memory) + Memory(body, 0x8B, operands.Wide, *operands.Memory, result, name, kFrame); + else if (operands.Source == Stack) + StackSlot(body, 0x8D, true, result, kFrame); + else + Register(body, 0x8B, operands.Wide, result, operands.Source); +} + +void _zeroHigh(StubBodyBuilder& body, const BitfieldOperands& operands, const std::uint8_t result, const std::uint8_t mask) { + auto high = operands.Wide ? std::vector{} : RegisterBytes(0x8B, false, result, result); + _append(high, RegisterBytes(0x85, operands.Wide, result, result)); + high.push_back(kSetCarry); + auto low = _lowMask(operands, mask); + _append(low, RegisterBytes(0x21, operands.Wide, mask, result)); + body.Raw(_compareControlByte(kControlRegister, _bits(operands))); + body.Raw(std::array{kJumpBelow, static_cast(high.size() + 2)}); + body.Raw(high); + body.Raw(std::array{kJump, static_cast(low.size())}); + body.Raw(low); +} + +void _extract(StubBodyBuilder& body, const BitfieldOperands& operands, const std::uint8_t result, const std::uint8_t mask) { + const auto bits = _bits(operands); + std::vector length{0x88, 0xE9}; + _append(length, _lowMask(operands, mask)); + const auto zero = RegisterBytes(0x31, operands.Wide, result, result); + auto shifted = RegisterBytes(0xD3, operands.Wide, 5, result); + _append(shifted, _compareControlByte(5, bits)); + shifted.push_back(kJumpAboveOrEqual); + shifted.push_back(static_cast(length.size() + 2 + zero.size())); + _append(shifted, length); + shifted.push_back(kJump); + shifted.push_back(static_cast(zero.size())); + body.Raw(_compareControlByte(kControlRegister, bits)); + body.Raw(std::array{kJumpAboveOrEqual, static_cast(shifted.size())}); + body.Raw(shifted); + body.Raw(zero); + Register(body, 0x21, operands.Wide, mask, result); +} + +} + +const char* BitfieldName(const BitfieldOperation operation) { + switch (operation) { + case BitfieldOperation::ZeroHigh: return "BZHI"; + case BitfieldOperation::Extract: return "BEXTR"; + } + throw CodegenException("unknown bitfield operation"); +} + +std::optional DecodeBitfield(const std::uint8_t* data, const std::size_t length) { + std::size_t vex = 0; + while (vex < length && LegacyPrefix(data[vex])) ++vex; + if (vex + 5 > length || data[vex] != 0xC4 || (data[vex + 1] & 0x1F) != 2 || (data[vex + 2] & 3) != 0) return std::nullopt; + BitfieldOperation operation; + if (data[vex + 3] == 0xF5) + operation = BitfieldOperation::ZeroHigh; + else if (data[vex + 3] == 0xF7) + operation = BitfieldOperation::Extract; + else + return std::nullopt; + const auto name = BitfieldName(operation); + if ((data[vex + 2] & 4) != 0) throw CodegenException(std::string(name) + " requires VEX.L=0"); + const auto extensions = static_cast(~data[vex + 1]); + const auto modrm = data[vex + 4]; + BitfieldOperands operands{ + operation, + static_cast(((modrm >> 3) & 7) | ((extensions & 0x80) >> 4)), + static_cast((modrm & 7) | ((extensions & 0x20) >> 2)), + static_cast((~data[vex + 2] >> 3) & 15), + (data[vex + 2] & 0x80) != 0, std::nullopt}; + std::size_t end = vex + 5; + if ((modrm >> 6) != 3) { + const auto rex = static_cast(((extensions & 0x20) >> 5) | ((extensions & 0x40) >> 5)); + operands.Memory = DecodeMemoryOperand(data, length, vex + 4, rex, {data, data + vex}); + end = vex + 4 + operands.Memory->EncodedSize; + } + if (end != length) throw CodegenException(std::string(name) + " has trailing bytes"); + return operands; +} + +void EmitBitfield(StubBodyBuilder& body, const BitfieldOperands& operands) { + const auto name = BitfieldName(operands.Operation); + auto used = operands.Memory ? UsedRegisters({operands.Destination, operands.Control, kControlRegister}, operands.Memory) + : UsedRegisters({operands.Destination, operands.Source, operands.Control, kControlRegister}, std::nullopt); + const auto result = Scratch(used, name); + const auto mask = Scratch(used, name); + StackSlot(body, 0x8D, true, Stack, -kFrame); + StackSlot(body, 0x89, true, result, kResultSlot); + StackSlot(body, 0x89, true, kControlRegister, kControlSlot); + StackSlot(body, 0x89, true, mask, kMaskSlot); + _loadSource(body, operands, result, name); + if (operands.Control == Stack) + StackSlot(body, 0x8D, true, kControlRegister, kFrame); + else if (operands.Control != kControlRegister) + Register(body, 0x8B, true, kControlRegister, operands.Control); + auto allOnes = RegisterBytes(0xC7, operands.Wide, 0, mask); + Append32(allOnes, -1); + body.Raw(allOnes); + if (operands.Operation == BitfieldOperation::ZeroHigh) + _zeroHigh(body, operands, result, mask); + else + _extract(body, operands, result, mask); + if (operands.Destination == Stack) + StackSlot(body, 0x89, operands.Wide, result, kStackResultSlot); + else if (operands.Destination == kControlRegister) + StackSlot(body, 0x89, true, result, kControlSlot); + else + Register(body, 0x8B, operands.Wide, operands.Destination, result); + StackSlot(body, 0x8B, true, mask, kMaskSlot); + StackSlot(body, 0x8B, true, kControlRegister, kControlSlot); + StackSlot(body, 0x8B, true, result, kResultSlot); + StackSlot(body, 0x8D, true, Stack, kFrame); + if (operands.Destination == Stack) + StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot - kFrame); +} + +} diff --git a/core/relinker/codegen/tests/BitfieldLoweringTests.cpp b/core/relinker/codegen/tests/BitfieldLoweringTests.cpp new file mode 100644 index 0000000000..acfc306750 --- /dev/null +++ b/core/relinker/codegen/tests/BitfieldLoweringTests.cpp @@ -0,0 +1,211 @@ +#include "LoweringExecution.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using LoweringExecution::Bytes; +using Codegen::BitfieldOperation; + +constexpr std::array kOperations{BitfieldOperation::ZeroHigh, BitfieldOperation::Extract}; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +std::uint8_t opcode(BitfieldOperation operation) { + return operation == BitfieldOperation::ZeroHigh ? 0xF5 : 0xF7; +} + +Bytes instruction(BitfieldOperation operation, bool wide, std::uint8_t dst, std::uint8_t src, std::uint8_t control) { + return {0xC4, static_cast(0xE2 ^ ((dst & 8) << 4) ^ ((src & 8) << 2)), + static_cast((wide ? 0x80 : 0) | ((~control & 15) << 3)), opcode(operation), + static_cast(0xC0 | ((dst & 7) << 3) | (src & 7))}; +} + +Bytes memorySite(BitfieldOperation operation, bool wide, const Bytes& tail) { + Bytes site{0xC4, 0xE2, static_cast((wide ? 0x80 : 0) | 0x40), opcode(operation)}; + site.insert(site.end(), tail.begin(), tail.end()); + return site; +} + +void decoding() { + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (std::uint8_t src = 0; src < 16; ++src) { + for (std::uint8_t control = 0; control < 16; ++control) { + const auto bytes = instruction(operation, wide, dst, src, control); + const auto operands = Codegen::DecodeBitfield(bytes.data(), bytes.size()); + require(operands && operands->Operation == operation && operands->Wide == wide && operands->Destination == dst && operands->Source == src && operands->Control == control && !operands->Memory, "bitfield register decoding changed an operand"); + } + } + } + } + } + const Bytes memory{0x64, 0x67, 0xC4, 0x02, 0xA8, 0xF5, 0x8C, 0xAC, 0x10, 0, 0, 0}; + const auto operands = Codegen::DecodeBitfield(memory.data(), memory.size()); + require(operands && operands->Operation == BitfieldOperation::ZeroHigh && operands->Destination == 9 && operands->Control == 10 && operands->Memory && operands->Memory->RexIndexBase == 3 && operands->Memory->Prefixes == Bytes{0x64, 0x67}, "bitfield memory extensions or prefixes were lost"); + for (const Bytes bytes : {Bytes{0x90}, Bytes{0xC4, 0xE2, 0xF0}, Bytes{0xC4, 0xE1, 0xF0, 0xF5, 0xC0}, Bytes{0xC4, 0xE2, 0xF1, 0xF7, 0xC0}, Bytes{0xC4, 0xE2, 0xF3, 0xF5, 0xC0}, Bytes{0xC4, 0xE2, 0xF2, 0xF5, 0xC0}, Bytes{0xC4, 0xE2, 0xF3, 0xF6, 0xC0}, Bytes{0xC4, 0xE2, 0xF0, 0xF3, 0xC8}}) + require(!Codegen::DecodeBitfield(bytes.data(), bytes.size()), "Unrelated, truncated, SHLX, PDEP, PEXT, MULX or BLSR opcode matched a bitfield instruction"); + for (const Bytes bytes : {Bytes{0xC4, 0xE2, 0xF4, 0xF5, 0xC0}, Bytes{0xC4, 0xE2, 0xF0, 0xF7, 0xC0, 0x90}, Bytes{0xC4, 0xE2, 0xF0, 0xF5, 0x84}}) { + bool rejected = false; + try { (void)Codegen::DecodeBitfield(bytes.data(), bytes.size()); } + catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "Malformed bitfield instruction was accepted"); + } + const Bytes overflow{0xC4, 0xE2, 0xF0, 0xF7, 0x8C, 0x24, 0xFF, 0xFF, 0xFF, 0x7F}; + bool rejected = false; + try { + Codegen::StubBodyBuilder body; + Codegen::EmitBitfield(body, *Codegen::DecodeBitfield(overflow.data(), overflow.size())); + } catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "An overflowing stack displacement was silently wrapped"); +} + +void matching() { + const auto intel = Codegen::MakeAmd64OnlyInstructionMatcher(); + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + for (const auto operation : kOperations) { + const auto bytes = instruction(operation, true, 0, 6, 2); + const auto name = Codegen::BitfieldName(operation); + require(!intel->Match(bytes.data(), bytes.size()), std::string("Intel mode rewrote ") + name); + const auto match = rosetta->Match(bytes.data(), bytes.size()); + require(match && match->InstructionName == name && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline && !match->Optional, std::string("Rosetta mode did not require a trampoline for ") + name); + const std::array, 2> sequence{bytes, bytes}; + require(rosetta->MatchSequence(sequence, {}).has_value(), std::string("A sequence could not move out of line for ") + name); + } +} + +#if LOWERING_EXECUTION_AVAILABLE + +using LoweringExecution::Execution; +using LoweringExecution::State; + +Execution executable(const Bytes& site, std::uint64_t memoryValue) { + const auto operands = Codegen::DecodeBitfield(site.data(), site.size()); + require(operands.has_value(), "Execution site was not a bitfield instruction"); + return Execution(site, memoryValue, operands->Memory && operands->Memory->RipRelative, [](Codegen::StubBodyBuilder& body, const Bytes& relocated) { + Codegen::EmitBitfield(body, *Codegen::DecodeBitfield(relocated.data(), relocated.size())); + }); +} + +struct Expected { + std::uint64_t Result; + std::uint64_t Flags; + std::uint64_t FlagMask; +}; + +Expected expected(const Codegen::BitfieldOperands& operands, std::uint64_t source, std::uint64_t control) { + const unsigned bits = operands.Wide ? 64 : 32; + const auto mask = operands.Wide ? ~std::uint64_t{0} : 0xFFFFFFFFull; + source &= mask; + const auto lowBits = [&](unsigned count) { return count >= bits ? mask : (std::uint64_t{1} << count) - 1; }; + if (operands.Operation == BitfieldOperation::ZeroHigh) { + const auto index = static_cast(control & 0xFF); + const auto result = index >= bits ? source : source & lowBits(index); + const auto flags = (result == 0 ? LoweringExecution::kZero : 0u) | (index >= bits ? LoweringExecution::kCarry : 0u) | ((result >> (bits - 1)) != 0 ? LoweringExecution::kSign : 0u); + return {result, flags, LoweringExecution::kFlagMask}; + } + const auto start = static_cast(control & 0xFF); + const auto length = static_cast((control >> 8) & 0xFF); + const auto result = (start >= bits ? 0 : source >> start) & lowBits(length); + return {result, result == 0 ? LoweringExecution::kZero : 0u, LoweringExecution::kFlagMask & ~LoweringExecution::kSign}; +} + +void check(const State& state, const Codegen::BitfieldOperands& operands, std::uint64_t source, std::uint64_t control) { + const auto name = Codegen::BitfieldName(operands.Operation); + const auto wanted = expected(operands, source, control); + const auto describe = [&] { + return std::string(name) + (operands.Wide ? " 64" : " 32") + " dst " + std::to_string(operands.Destination) + " src " + std::to_string(operands.Source) + " control " + std::to_string(operands.Control) + (operands.Memory ? " memory" : "") + " (source " + std::to_string(source) + ", control value " + std::to_string(control) + ")"; + }; + for (std::size_t reg = 0; reg < 16; ++reg) { + const auto value = reg == operands.Destination ? wanted.Result : state.Input[reg]; + if (state.Output[reg] != value) + throw std::runtime_error(describe() + ": register " + std::to_string(reg) + " is " + std::to_string(state.Output[reg]) + ", wanted " + std::to_string(value)); + } + if ((state.Flags & wanted.FlagMask) != wanted.Flags) + throw std::runtime_error(describe() + ": flags are " + std::to_string(state.Flags & wanted.FlagMask) + ", wanted " + std::to_string(wanted.Flags)); + for (const auto value : state.RedZone) require(value == LoweringExecution::kSentinel, std::string(name) + " overwrote the caller's red zone"); +} + +void run(const Execution& execution, const Codegen::BitfieldOperands& operands, State state, std::uint64_t memoryValue) { + for (const auto flags : {LoweringExecution::kBaseFlags | LoweringExecution::kFlagMask, LoweringExecution::kBaseFlags}) { + state.InputFlags = flags; + auto trial = state; + execution.Run(trial); + const auto source = operands.Memory ? memoryValue : trial.Input[operands.Source]; + check(trial, operands, source, trial.Input[operands.Control]); + } +} + +void execution() { + constexpr std::array values{0, 1, 0x80000000ull, 0x8000000000000000ull, 0xFFFFFFFFFFFFFFFFull, 0xABCD123456789000ull}; + constexpr std::array controls{0, 1, 31, 32, 63, 64, 0xFF, 0x100, 0x801, 0x1F05, 0x2000, 0x201F, 0x3F3F, 0x4000, 0x4020, 0xFF00, 0xFFFF, 0x0F10, 0x12345678, 0xFFFFFFFFFFFF0405ull}; + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (const auto pair : std::array, 9>{{{6, 2}, {dst, 10}, {11, dst}, {dst, dst}, {4, 9}, {9, 4}, {1, 6}, {6, 1}, {1, 1}}}) { + const auto site = instruction(operation, wide, dst, pair.first, pair.second); + const auto operands = *Codegen::DecodeBitfield(site.data(), site.size()); + const auto execution = executable(site, 0); + for (const auto value : values) { + for (const auto control : controls) { + auto state = LoweringExecution::InitialState(); + state.Input[pair.first] = value; + state.Input[pair.second] = control; + run(execution, operands, state, 0); + } + } + } + } + } + } + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (const Bytes tail : {Bytes{0x06}, Bytes{0x46, 0x10}, Bytes{0x84, 0x24, 0x40, 0, 0, 0}, Bytes{0x04, 0xCA}, Bytes{0x85, 0x10, 0, 0, 0}, Bytes{0x05, 0, 0, 0, 0}}) { + for (const auto value : std::array{0, 0xFEDCBA9876543210ull, 0xFFFFFFFF00000000ull}) { + for (const auto control : std::array{0, 0x0C04, 0x3F, 0x40}) { + const auto site = memorySite(operation, wide, tail); + const auto operands = *Codegen::DecodeBitfield(site.data(), site.size()); + const auto execution = executable(site, value); + auto state = LoweringExecution::InitialState(); + state.Input[6] = reinterpret_cast(execution.Memory()) - (tail[0] == 0x46 ? 16 : 0); + state.Input[2] = reinterpret_cast(execution.Memory()) - 3 * 8; + state.Input[1] = 3; + state.Input[5] = reinterpret_cast(execution.Memory()) - 16; + state.Input[7] = control; + state.Output[0] = value; + run(execution, operands, state, value); + } + } + } + } + } +} +#else +void execution() { std::cout << "bitfield execution checks require a Linux/macOS x86-64 build\n"; } +#endif + +} + +int main() { + try { + decoding(); + matching(); + execution(); + std::cout << "bitfield lowering checks passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/relinker/relinker/tests/test_to_rosetta.py b/core/relinker/relinker/tests/test_to_rosetta.py index 93024c8457..d3a1c6b157 100644 --- a/core/relinker/relinker/tests/test_to_rosetta.py +++ b/core/relinker/relinker/tests/test_to_rosetta.py @@ -64,7 +64,7 @@ def main(): assert (andn not in output.read_bytes()) == lowered, f"{options} {'did not lower' if lowered else 'changed'} ANDN" if lowered: assert "ANDN" in result.stdout, result.stdout - for name, bls in (("BLSR", "c4e2f8f3ce"), ("BLSMSK", "c4e2f8f3d6"), ("BLSI", "c4e2f8f3de")): + for name, bls in (("BLSR", "c4e2f8f3ce"), ("BLSMSK", "c4e2f8f3d6"), ("BLSI", "c4e2f8f3de"), ("BZHI", "c4e2e8f5c6"), ("BEXTR", "c4e2e8f7c6")): code = bytes.fromhex(bls + "c3") for target in ((), ("--macos",)): for conversion in ((), ("--to-intel",), ("--to-rosetta",)): diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index ca3116a64a..db4bee72a4 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,15 +46,16 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI and BEXTR are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI, BEXTR, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. -- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. BEXTR (the `pp=0` form of the same opcode), BZHI, PDEP and PEXT are not lowered. The execution test runs natively on x86-64 or under Rosetta and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. Balatro (PPSA21402, #1142) stopped on `shlxq` after the ANDN lowering; this lowering has not been run on that title. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BZHI and BEXTR lowering](../../../core/relinker/codegen/src/x86/BitfieldLowering.cpp) builds the mask with `shl` by CL and `not` on a scratch register and applies it with `and` under `--to-rosetta`, with RCX spilled to carry the control operand in a 160-byte frame below the red zone; the control's low byte is compared with the operand size, so an index or start at or past it takes the SDM's unmasked (BZHI) or zero (BEXTR) branch instead of the masked count a scalar shift would use. BZHI sets CF through STC on that branch, and ZF and SF follow the result through `test` or `and`; BEXTR's ZF follows the result and CF and OF clear through `and`, while its SF, which the SDM leaves undefined, follows the result here. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result and flags against a model of the SDM semantics that matched a Zen 2's BZHI and BEXTR on 122,880 cases, not against the hardware instruction in the same run; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index d63dd7299e..e48958bc5e 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -74,7 +74,7 @@ All switches are disabled by default. `unused-filter` defaults to `0`; `--rpath` | `--linux-fixed-base
` | Load the Linux executable at this fixed address instead of the default `0x400000`, given in decimal or hexadecimal (`0x`): every virtual address of the image is offset by it, so virtual address 0 is placed at the address, as the console places an executable. Requires a 64 KiB-aligned address from `0x10000` through `0x7fffffff0000` that is a multiple of every `PT_LOAD` alignment, and an image that ends in the user address space. The executable is written as `ET_EXEC`, without ASLR, as it is at the default address. Bundled modules keep their normal placement. Conflicts with `--windows`, `--macos` and `--lazy-binding`. | | `--macos-fixed-base
` | Map the lowest ELF load page of the macOS executable at this fixed address, given in decimal or hexadecimal (`0x`). Requires `--macos`, a 64 KiB-aligned address from `0x20000` through `0x7fffffff0000`, and an image that ends in the macOS user address space. `__PAGEZERO` ends 64 KiB below the address and the Mach-O header occupies those 64 KiB. The executable has no ASLR; an address the process cannot map fails to load. Bundled modules keep their normal placement. | | `--to-intel` | Convert supported AMD-only instructions in the executable and bundled modules. Unsupported instructions or unreachable conversion stubs cause an error. | -| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, SHLX, SHRX, SARX and RORX become flag-preserving scalar shifts and rotates, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | +| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, SHLX, SHRX, SARX and RORX become flag-preserving scalar shifts and rotates, BZHI and BEXTR become scalar shift-and-mask sequences, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | | `unused-filter=0` | Keep all imported NID references. | | `unused-filter=1` | Filter unused non-PLT imports using control-flow and GOT access analysis, treating address-taken code as reachable; preserve PLT imports. | | `unused-filter=2` | Apply strict unused-import analysis and compact the PLT. Unsupported analysis cases cause an error. | From 4ff00abf285de3de45d473e7fdf73e72737ad215 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Thu, 29 Oct 2026 18:42:28 +0530 Subject: [PATCH 239/318] fix(relinker): restore an RSP destination in one load in the BLS stub and list MULX as lowered --- core/relinker/codegen/src/x86/BlsLowering.cpp | 5 +++-- docs/dev/debt/relinker.md | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/relinker/codegen/src/x86/BlsLowering.cpp b/core/relinker/codegen/src/x86/BlsLowering.cpp index 46a4a2c014..0b2f0f60b8 100644 --- a/core/relinker/codegen/src/x86/BlsLowering.cpp +++ b/core/relinker/codegen/src/x86/BlsLowering.cpp @@ -126,9 +126,10 @@ void EmitBls(StubBodyBuilder& body, const BlsOperands& operands) { else Register(body, 0x8B, operands.Wide, operands.Destination, result); StackSlot(body, 0x8B, true, result, 0); - StackSlot(body, 0x8D, true, Stack, FrameSize); if (operands.Destination == Stack) - StackSlot(body, 0x8B, operands.Wide, Stack, 8 - FrameSize); + StackSlot(body, 0x8B, operands.Wide, Stack, 8); + else + StackSlot(body, 0x8D, true, Stack, FrameSize); } } diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 14d5d8a17b..746cb8a474 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,14 +46,14 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK and BLSR are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX and SHRX are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, PDEP, PEXT, RORX, SARX, SHLX and SHRX are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. From 6ad23bffdd542a255807696258f2b93dbe0b6061 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Thu, 29 Oct 2026 18:45:15 +0530 Subject: [PATCH 240/318] fix(relinker): restore an RSP destination in one load in the BZHI and BEXTR stub and list MULX as lowered --- core/relinker/codegen/src/x86/BitfieldLowering.cpp | 5 +++-- docs/dev/debt/relinker.md | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/core/relinker/codegen/src/x86/BitfieldLowering.cpp b/core/relinker/codegen/src/x86/BitfieldLowering.cpp index 1d9fb3e545..bc9cf32679 100644 --- a/core/relinker/codegen/src/x86/BitfieldLowering.cpp +++ b/core/relinker/codegen/src/x86/BitfieldLowering.cpp @@ -150,9 +150,10 @@ void EmitBitfield(StubBodyBuilder& body, const BitfieldOperands& operands) { StackSlot(body, 0x8B, true, mask, kMaskSlot); StackSlot(body, 0x8B, true, kControlRegister, kControlSlot); StackSlot(body, 0x8B, true, result, kResultSlot); - StackSlot(body, 0x8D, true, Stack, kFrame); if (operands.Destination == Stack) - StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot - kFrame); + StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot); + else + StackSlot(body, 0x8D, true, Stack, kFrame); } } diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index db4bee72a4..ad959bbf72 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,16 +46,16 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI and BEXTR are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI, BEXTR, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI, BEXTR and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI, BEXTR, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. -- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. Balatro (PPSA21402, #1142) stopped on `shlxq` after the ANDN lowering; this lowering has not been run on that title. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. -- [BZHI and BEXTR lowering](../../../core/relinker/codegen/src/x86/BitfieldLowering.cpp) builds the mask with `shl` by CL and `not` on a scratch register and applies it with `and` under `--to-rosetta`, with RCX spilled to carry the control operand in a 160-byte frame below the red zone; the control's low byte is compared with the operand size, so an index or start at or past it takes the SDM's unmasked (BZHI) or zero (BEXTR) branch instead of the masked count a scalar shift would use. BZHI sets CF through STC on that branch, and ZF and SF follow the result through `test` or `and`; BEXTR's ZF follows the result and CF and OF clear through `and`, while its SF, which the SDM leaves undefined, follows the result here. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result and flags against a model of the SDM semantics that matched a Zen 2's BZHI and BEXTR on 122,880 cases, not against the hardware instruction in the same run; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. PDEP and PEXT are not lowered. The execution test runs natively on x86-64 or under Rosetta and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BZHI and BEXTR lowering](../../../core/relinker/codegen/src/x86/BitfieldLowering.cpp) builds the mask with `shl` by CL and `not` on a scratch register and applies it with `and` under `--to-rosetta`, with RCX spilled in a 160-byte frame below the red zone. An index or start at or past the operand size takes the SDM's unmasked (BZHI) or zero (BEXTR) branch, not a scalar shift's masked count. BZHI sets CF through STC there, and ZF and SF follow the result; BEXTR clears CF and OF, ZF follows the result, and its SF, undefined in the SDM, follows the result here. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. PDEP and PEXT are not lowered. The execution test runs natively on x86-64 or under Rosetta and checks result and flags against an SDM model that matched a Zen 2's BZHI and BEXTR on 122,880 cases, not against the hardware instruction in the same run; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. From a12f8fc4d39b701fa2047a0a3a3a090be2d3562c Mon Sep 17 00:00:00 2001 From: Federico Marras Date: Fri, 23 Oct 2026 19:36:38 +0200 Subject: [PATCH 241/318] feat(libc): track stream orientation and implement fwide --- core/libs/prx/libc/CMakeLists.txt | 11 + core/libs/prx/libc/include/Api.hpp | 1 + core/libs/prx/libc/include/FileStream.hpp | 11 + core/libs/prx/libc/src/Errors.cpp | 2 +- core/libs/prx/libc/src/FileAndHeap.cpp | 3 + core/libs/prx/libc/src/Formatting.cpp | 14 ++ core/libs/prx/libc/src/FormattingWide.cpp | 6 + core/libs/prx/libc/src/ScanningWide.cpp | 2 + core/libs/prx/libc/src/StandardStreams.cpp | 12 ++ core/libs/tests/GuestFwide.cpp | 231 +++++++++++++++++++++ docs/dev/debt/libc.md | 2 + 11 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 core/libs/tests/GuestFwide.cpp diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index 176bc416d1..ac145c7a87 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -262,6 +262,17 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() + add_test_executable(guest_fwide_tests ${LIBS_INCLUDE_DIR}/tests/GuestFwide.cpp) + target_include_directories(guest_fwide_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_fwide_tests PRIVATE libc ${CMAKE_DL_LIBS}) + configure_windows_unwind(guest_fwide_tests) + add_test(NAME guest_fwide COMMAND guest_fwide_tests $ guest_fwide.tmp) + set_tests_properties(guest_fwide PROPERTIES TIMEOUT 10) + if(WIN32) + set_tests_properties(guest_fwide PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_allocating_strings_tests ${LIBS_INCLUDE_DIR}/tests/GuestAllocatingStrings.cpp) target_include_directories(guest_allocating_strings_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_allocating_strings_tests PRIVATE libc libkernel) diff --git a/core/libs/prx/libc/include/Api.hpp b/core/libs/prx/libc/include/Api.hpp index 9794adca9d..36e99aee81 100644 --- a/core/libs/prx/libc/include/Api.hpp +++ b/core/libs/prx/libc/include/Api.hpp @@ -405,6 +405,7 @@ int APS5_VABI fputws_nid_postfix(const char16_t* str, FileStream* stream); int APS5_VABI putwc_nid_postfix(char16_t value, FileStream* stream); int APS5_VABI putwchar_nid_postfix(char16_t value); char16_t* APS5_VABI fgetws_nid_postfix(char16_t* buffer, int size, FileStream* stream); +int APS5_VABI fwide_nid_postfix(FileStream* stream, int mode); int APS5_VABI chdir_nid_postfix(const char* path); char* APS5_VABI getcwd_nid_postfix(char* buffer, std::size_t size); void APS5_VABI sceLibcInternalBacktraceForGame_nid_postfix(const char* heapName); diff --git a/core/libs/prx/libc/include/FileStream.hpp b/core/libs/prx/libc/include/FileStream.hpp index d93e847dd2..dab82f3461 100644 --- a/core/libs/prx/libc/include/FileStream.hpp +++ b/core/libs/prx/libc/include/FileStream.hpp @@ -70,6 +70,7 @@ class FileStream { bool guestError = false; bool randomDevice = false; bool directory = false; + std::atomic _orientation{0}; char16_t lowSurrogate = 0; unsigned pushedBack = 0; @@ -100,6 +101,13 @@ public: return _handle != nullptr; } + int Orientation(int mode) { + if (mode == 0) return _orientation.load(std::memory_order_relaxed); + int expected = 0; + _orientation.compare_exchange_strong(expected, mode > 0 ? 1 : -1, std::memory_order_relaxed); + return _orientation.load(std::memory_order_relaxed); + } + bool IsDynamic() const { return _dynamic; } @@ -110,6 +118,7 @@ public: } void Detach() { + _orientation.store(0, std::memory_order_relaxed); _handle = nullptr; _guest.flags = 0; _guest.descriptor = -1; @@ -182,6 +191,7 @@ public: _guest = {}; guestError = false; randomDevice = false; + _orientation.store(0, std::memory_order_relaxed); lowSurrogate = 0; pushedBack = 0; #ifdef _WIN32 @@ -386,6 +396,7 @@ public: const int result = std::fclose(std::exchange(_handle, nullptr)); _guest.flags = 0; _guest.descriptor = -1; + _orientation.store(0, std::memory_order_relaxed); lowSurrogate = 0; pushedBack = 0; return result; diff --git a/core/libs/prx/libc/src/Errors.cpp b/core/libs/prx/libc/src/Errors.cpp index f934c4de09..93bbf075f5 100644 --- a/core/libs/prx/libc/src/Errors.cpp +++ b/core/libs/prx/libc/src/Errors.cpp @@ -93,7 +93,7 @@ void APS5_VABI perror_nid_postfix(const char* prefix) { const int saved = *__error_nid_postfix(); char message[128]; strerror_r_nid_postfix(saved, message, sizeof(message)); - + _Stderr_nid_postfix.Orientation(-1); auto* stream = GetNativeStream(&_Stderr_nid_postfix); if (prefix && *prefix) std::fprintf(stream, "%s: %s\n", prefix, message); else std::fprintf(stream, "%s\n", message); diff --git a/core/libs/prx/libc/src/FileAndHeap.cpp b/core/libs/prx/libc/src/FileAndHeap.cpp index 875828d095..b38b361957 100644 --- a/core/libs/prx/libc/src/FileAndHeap.cpp +++ b/core/libs/prx/libc/src/FileAndHeap.cpp @@ -530,6 +530,7 @@ size_t APS5_VABI fread_nid_postfix(void* buffer, size_t size, size_t count, File if (stream && !stream->IsOpen()) { errno = 9; return 0; } auto* handle = GetStreamHandle(stream); if (size == 0 || count == 0) return 0; + stream->Orientation(-1); if (!buffer) throw std::runtime_error("fread: null buffer"); const auto bytes = size * count; if (bytes > std::numeric_limits::max() - reinterpret_cast(buffer)) throw std::overflow_error("fread: buffer address overflow"); @@ -551,6 +552,7 @@ size_t APS5_VABI fwrite_nid_postfix(const void* buffer, size_t size, size_t coun if (stream && !stream->IsOpen()) { errno = 9; return 0; } auto* handle = GetNativeStream(stream); if (size == 0 || count == 0) return 0; + stream->Orientation(-1); if (!buffer) throw std::runtime_error("fwrite: null buffer"); if (count > std::numeric_limits::max() / size) throw std::overflow_error("fwrite: buffer size overflow"); const auto bytes = size * count; @@ -623,6 +625,7 @@ int APS5_VABI fputs_nid_postfix(const char* str, FileStream* stream) { if (!str) throw std::runtime_error("fputs: null string"); if (stream && !stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (*str != '\0' && !stream->SwitchToWriting()) return -1; const int result = std::fputs(str, native); stream->SyncStatus(); diff --git a/core/libs/prx/libc/src/Formatting.cpp b/core/libs/prx/libc/src/Formatting.cpp index 0f5176f7fc..1751ba2662 100644 --- a/core/libs/prx/libc/src/Formatting.cpp +++ b/core/libs/prx/libc/src/Formatting.cpp @@ -369,6 +369,7 @@ int ScanGuestStream(FileStream* stream, const char* format, NextPointer nextPoin if (stream == nullptr || format == nullptr) return EOF; if (!stream->IsOpen()) { errno = 9; return EOF; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (!stream->SwitchToReading()) return EOF; #ifndef _WIN32 RejectGuestWideScan(format); @@ -420,6 +421,7 @@ int APS5_VABI vasprintf_nid_postfix(char** destination, const char* format, VaLi int APS5_VABI vfprintf_nid_postfix(FileStream* stream, const char* format, VaList* args) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (!stream->SwitchToWriting()) return -1; #ifdef _WIN32 std::string buffer; @@ -463,6 +465,7 @@ int APS5_VABI fprintf_nid_postfix(FileStream* stream, const char* format, ...) { int APS5_VABI fscanf_nid_postfix(FileStream* stream, const char* format, ...) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (!stream->SwitchToReading()) return EOF; #ifdef _WIN32 __builtin_sysv_va_list args; @@ -487,6 +490,7 @@ int APS5_VABI printf_nid_postfix(const char* format, ...) { if (!_Stdout_nid_postfix.IsOpen()) { errno = 9; return -1; } __builtin_sysv_va_list args; __builtin_sysv_va_start(args, format); + _Stdout_nid_postfix.Orientation(-1); const int result = LibcDetail::PrintWindows(format, args); __builtin_sysv_va_end(args); return result; @@ -496,6 +500,7 @@ int APS5_VABI libc_printf_nid_postfix(const char* format, ...) { if (!_Stdout_nid_postfix.IsOpen()) { errno = 9; return -1; } __builtin_sysv_va_list args; __builtin_sysv_va_start(args, format); + _Stdout_nid_postfix.Orientation(-1); const int result = LibcDetail::PrintWindows(format, args); __builtin_sysv_va_end(args); return result; @@ -523,6 +528,7 @@ int APS5_VABI printf_nid_postfix(const char* format, ...) { if (!_Stdout_nid_postfix.IsOpen()) { errno = 9; return -1; } std::va_list args; va_start(args, format); + _Stdout_nid_postfix.Orientation(-1); const int result = RequiresGuestFormatting(format) ? LibcDetail::PrintWindows(format, args) : std::vfprintf(GuestStandardOutputFile_nid_no_patch(), format, args); va_end(args); return result; @@ -532,6 +538,7 @@ int APS5_VABI libc_printf_nid_postfix(const char* format, ...) { if (!_Stdout_nid_postfix.IsOpen()) { errno = 9; return -1; } std::va_list args; va_start(args, format); + _Stdout_nid_postfix.Orientation(-1); const int result = RequiresGuestFormatting(format) ? LibcDetail::PrintWindows(format, args) : std::vfprintf(GuestStandardOutputFile_nid_no_patch(), format, args); va_end(args); return result; @@ -619,8 +626,10 @@ int APS5_VABI sscanf_s_nid_postfix(const char* buffer, const char* format, ...) int APS5_VABI vprintf_nid_postfix(const char* str, VaList* c) { if (!_Stdout_nid_postfix.IsOpen()) { errno = 9; return -1; } #ifdef _WIN32 + _Stdout_nid_postfix.Orientation(-1); return LibcDetail::PrintWindows(str, c); #else + _Stdout_nid_postfix.Orientation(-1); if (RequiresGuestFormatting(str)) return LibcDetail::PrintWindows(str, c); std::va_list copy; va_copy(copy, *reinterpret_cast(c)); @@ -677,6 +686,7 @@ static int WriteSecure(FileStream* stream, const char* format, const void* args) if (stream == nullptr || format == nullptr || args == nullptr) return -1; if (!stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (!stream->SwitchToWriting()) return -1; std::string text; int count; @@ -734,6 +744,7 @@ int APS5_VABI printf_s_nid_postfix(const char* format, ...) { if (!_Stdout_nid_postfix.IsOpen()) { errno = 9; return -1; } __builtin_sysv_va_list args; __builtin_sysv_va_start(args, format); + _Stdout_nid_postfix.Orientation(-1); const int result = LibcDetail::PrintWindows(format, args); __builtin_sysv_va_end(args); return result; @@ -775,6 +786,7 @@ int APS5_VABI printf_s_nid_postfix(const char* format, ...) { if (!_Stdout_nid_postfix.IsOpen()) { errno = 9; return -1; } std::va_list args; va_start(args, format); + _Stdout_nid_postfix.Orientation(-1); const int result = RequiresGuestFormatting(format) ? LibcDetail::PrintWindows(format, args) : std::vfprintf(GuestStandardOutputFile_nid_no_patch(), format, args); va_end(args); return result; @@ -785,6 +797,7 @@ int APS5_VABI printf_s_nid_postfix(const char* format, ...) { int APS5_VABI puts_nid_postfix(const char* s) { if (!_Stdout_nid_postfix.IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(&_Stdout_nid_postfix); + _Stdout_nid_postfix.Orientation(-1); if (!_Stdout_nid_postfix.SwitchToWriting()) return EOF; if (std::fputs(s, native) < 0 || std::fputc('\n', native) == EOF) return EOF; return 0; @@ -795,6 +808,7 @@ extern FileStream* __stdinp_nid_postfix; int APS5_VABI vfscanf_nid_postfix(FileStream* stream, const char* format, VaList* args) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (!stream->SwitchToReading()) return EOF; #ifdef _WIN32 const int result = LibcDetail::ScanFileWindows_nid_no_patch(native, format, args); diff --git a/core/libs/prx/libc/src/FormattingWide.cpp b/core/libs/prx/libc/src/FormattingWide.cpp index 32d668bc85..dd2b65c766 100644 --- a/core/libs/prx/libc/src/FormattingWide.cpp +++ b/core/libs/prx/libc/src/FormattingWide.cpp @@ -365,6 +365,7 @@ int APS5_VABI vsnwprintf_s_nid_postfix(char16_t* buffer, std::size_t size, const int APS5_VABI wprintf_nid_postfix(const char16_t* format, ...) { if (!__stdoutp_nid_postfix->IsOpen()) { errno = 9; return -1; } + _Stdout_nid_postfix.Orientation(1); if (!__stdoutp_nid_postfix->SwitchToWriting()) return -1; APS5_VA_BEGIN(format); int result = -1; @@ -382,6 +383,7 @@ int APS5_VABI wprintf_nid_postfix(const char16_t* format, ...) { int APS5_VABI vfwprintf_nid_postfix(FileStream* stream, const char16_t* format, VaList* args) { if (stream == nullptr || format == nullptr) { errno = 22; return -1; } if (!stream->IsOpen()) { errno = 9; return -1; } + stream->Orientation(1); if (!stream->SwitchToWriting()) return -1; int result = -1; try { @@ -409,6 +411,7 @@ int APS5_VABI vwprintf_nid_postfix(const char16_t* format, VaList* args) { int APS5_VABI vfwprintf_s_nid_postfix(FileStream* stream, const char16_t* format, VaList* args) { if (stream == nullptr || format == nullptr || args == nullptr) return -1; if (!stream->IsOpen()) { errno = 9; return -1; } + stream->Orientation(1); if (!stream->SwitchToWriting()) return -1; int result = -1; try { @@ -443,6 +446,7 @@ int APS5_VABI wprintf_s_nid_postfix(const char16_t* format, ...) { int APS5_VABI fputwc_nid_postfix(char16_t value, FileStream* stream) { if (stream == nullptr) { errno = 22; return -1; } if (!stream->IsOpen()) { errno = 9; return -1; } + stream->Orientation(1); try { const std::string bytes = ToUtf8(std::u16string(1, value)); auto* native = GetNativeStream(stream); @@ -459,6 +463,7 @@ int APS5_VABI fputwc_nid_postfix(char16_t value, FileStream* stream) { int APS5_VABI fputws_nid_postfix(const char16_t* str, FileStream* stream) { if (str == nullptr || stream == nullptr) { errno = 22; return -1; } if (!stream->IsOpen()) { errno = 9; return -1; } + stream->Orientation(1); try { const std::u16string text(str); const std::string bytes = ToUtf8(text); @@ -482,6 +487,7 @@ int APS5_VABI putwchar_nid_postfix(char16_t value) { } char16_t* APS5_VABI fgetws_nid_postfix(char16_t* buffer, int size, FileStream* stream) { + if (stream != nullptr) stream->Orientation(1); if (size < 2) return nullptr; if (buffer == nullptr || stream == nullptr) { if (stream != nullptr) stream->SetError(); diff --git a/core/libs/prx/libc/src/ScanningWide.cpp b/core/libs/prx/libc/src/ScanningWide.cpp index 1f44da0acc..5ff3349149 100644 --- a/core/libs/prx/libc/src/ScanningWide.cpp +++ b/core/libs/prx/libc/src/ScanningWide.cpp @@ -522,12 +522,14 @@ extern FileStream* __stdinp_nid_postfix; int APS5_VABI vfwscanf_nid_postfix(FileStream* stream, const char16_t* format, VaList* args) { if (stream == nullptr || format == nullptr) return EOF; + stream->Orientation(1); const LockedStream lock(stream); return WideScanner("vfwscanf", false, WideInput(stream), format, args).Run(); } int APS5_VABI vfwscanf_s_nid_postfix(FileStream* stream, const char16_t* format, VaList* args) { if (stream == nullptr || format == nullptr) return EOF; + stream->Orientation(1); const LockedStream lock(stream); return WideScanner("vfwscanf_s", true, WideInput(stream), format, args).Run(); } diff --git a/core/libs/prx/libc/src/StandardStreams.cpp b/core/libs/prx/libc/src/StandardStreams.cpp index 8819a6175d..91256f0eac 100644 --- a/core/libs/prx/libc/src/StandardStreams.cpp +++ b/core/libs/prx/libc/src/StandardStreams.cpp @@ -87,9 +87,14 @@ FileStream* __stdoutp_nid_postfix = &_Stdout_nid_postfix; FileStream* __stderrp_nid_postfix = &_Stderr_nid_postfix; int __isthreaded_nid_postfix = 1; +int APS5_VABI fwide_nid_postfix(FileStream* stream, int mode) { + return stream->Orientation(mode); +} + int APS5_VABI fgetc_nid_postfix(FileStream* stream) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (!stream->SwitchToReading()) return -1; const int result = std::fgetc(native); stream->SyncStatus(); @@ -101,6 +106,7 @@ std::int32_t APS5_VABI fgetwc_nid_postfix(FileStream* stream) { errno = 22; return -1; } + stream->Orientation(1); if (const char16_t low = stream->TakeLowSurrogate(); low != 0) return low; stream->NoteWideRead(); const auto invalid = [stream] { @@ -130,6 +136,7 @@ std::int32_t APS5_VABI fgetwc_nid_postfix(FileStream* stream) { std::int32_t APS5_VABI ungetwc_nid_postfix(std::int32_t value, FileStream* stream) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } + if (stream) stream->Orientation(1); if (value == -1) return -1; if (value < 0 || value > 255) { errno = 86; @@ -147,6 +154,7 @@ int APS5_VABI getc_nid_postfix(FileStream* stream) { return fgetc_nid_postfix(st int APS5_VABI getc_unlocked_nid_postfix(FileStream* stream) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (!stream->SwitchToReadingUnlocked()) return -1; #ifdef _WIN32 const int result = _fgetc_nolock(native); @@ -190,6 +198,7 @@ int APS5_VABI getchar_unlocked_nid_postfix() { return getc_unlocked_nid_postfix( int APS5_VABI fputc_nid_postfix(int value, FileStream* stream) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (!stream->SwitchToWriting()) return -1; const int result = std::fputc(value, native); stream->SyncStatus(); @@ -206,6 +215,7 @@ int APS5_VABI putc_unlocked_nid_postfix(int value, FileStream* stream) { const int result = putc_unlocked(value, native); #endif stream->SyncUnlockedStatus(); + stream->Orientation(-1); return result; } int APS5_VABI __swbuf_nid_postfix(int value, FileStream* stream) { return fputc_nid_postfix(value, stream); } @@ -214,6 +224,7 @@ int APS5_VABI putchar_unlocked_nid_postfix(int value) { return putc_unlocked_nid int APS5_VABI ungetc_nid_postfix(int value, FileStream* stream) { if (stream && !stream->IsOpen()) { errno = 9; return -1; } auto* native = GetNativeStream(stream); + stream->Orientation(-1); if (value != EOF && !stream->SwitchToReading()) return -1; const int result = std::ungetc(value, native); stream->SyncStatus(); @@ -221,6 +232,7 @@ int APS5_VABI ungetc_nid_postfix(int value, FileStream* stream) { } char* APS5_VABI fgets_nid_postfix(char* buffer, int size, FileStream* stream) { if (stream && !stream->IsOpen()) { errno = 9; return nullptr; } + if (stream) stream->Orientation(-1); if (size <= 0) { stream->SetError(); errno = 22; diff --git a/core/libs/tests/GuestFwide.cpp b/core/libs/tests/GuestFwide.cpp new file mode 100644 index 0000000000..9c98ea5cc8 --- /dev/null +++ b/core/libs/tests/GuestFwide.cpp @@ -0,0 +1,231 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace { +using Open = void* (APS5_VABI*)(const char*, const char*); +using Close = int (APS5_VABI*)(void*); +using Wide = int (APS5_VABI*)(void*, int); +using PutByte = int (APS5_VABI*)(int, void*); +using PutWide = int (APS5_VABI*)(char16_t, void*); +using Seek = int (APS5_VABI*)(void*, std::int64_t, int); +using Reopen = void* (APS5_VABI*)(const char*, const char*, void*); +using Read = std::size_t (APS5_VABI*)(void*, std::size_t, std::size_t, void*); +using Write = std::size_t (APS5_VABI*)(const void*, std::size_t, std::size_t, void*); +using Flush = int (APS5_VABI*)(void*); +using Descriptor = int (APS5_VABI*)(void*); +using GetByte = int (APS5_VABI*)(void*); +using GetWide = std::int32_t (APS5_VABI*)(void*); +using UngetWide = std::int32_t (APS5_VABI*)(std::int32_t, void*); +using GetLine = char* (APS5_VABI*)(char*, int, void*); +using GetWideLine = char16_t* (APS5_VABI*)(char16_t*, int, void*); +using ScanList = int (APS5_VABI*)(void*, const char*, void*); +using WideList = int (APS5_VABI*)(void*, const char16_t*, void*); + +int APS5_VABI ScanThrough(ScanList scan, void* stream, const char* format, ...) { +#ifdef _WIN32 + __builtin_sysv_va_list args; + __builtin_sysv_va_start(args, format); +#else + std::va_list args; + va_start(args, format); +#endif + const int result = scan(stream, format, reinterpret_cast(args)); +#ifdef _WIN32 + __builtin_sysv_va_end(args); +#else + va_end(args); +#endif + return result; +} + +int APS5_VABI WideThrough(WideList print, void* stream, const char16_t* format, ...) { +#ifdef _WIN32 + __builtin_sysv_va_list args; + __builtin_sysv_va_start(args, format); +#else + std::va_list args; + va_start(args, format); +#endif + const int result = print(stream, format, reinterpret_cast(args)); +#ifdef _WIN32 + __builtin_sysv_va_end(args); +#else + va_end(args); +#endif + return result; +} + +void Require(bool value, const char* message) { + if (!value) { + std::fprintf(stderr, "%s\n", message); + std::abort(); + } +} + +template +Function Symbol( +#ifdef _WIN32 + HMODULE library, +#else + void* library, +#endif + const char* name) { +#ifdef _WIN32 + return reinterpret_cast(GetProcAddress(library, name)); +#else + return reinterpret_cast(dlsym(library, name)); +#endif +} +} + +int main(int argc, char** argv) { + Require(argc == 3, "expected libc path and scratch filename"); +#ifdef _WIN32 + const auto library = LoadLibraryA(argv[1]); +#else + const auto library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); +#endif + Require(library != nullptr, "could not load libc"); + const auto open = Symbol(library, "fopen_nid_postfix"); + const auto close = Symbol(library, "fclose_nid_postfix"); + const auto wide = Symbol(library, "fwide_nid_postfix"); + const auto putByte = Symbol(library, "fputc_nid_postfix"); + const auto putWide = Symbol(library, "fputwc_nid_postfix"); + const auto seek = Symbol(library, "fseek_nid_postfix"); + const auto read = Symbol(library, "fread_nid_postfix"); + const auto write = Symbol(library, "fwrite_nid_postfix"); + const auto flush = Symbol(library, "fflush_nid_postfix"); + const auto descriptor = Symbol(library, "fileno_nid_postfix"); + const auto getByte = Symbol(library, "fgetc_nid_postfix"); + const auto getWide = Symbol(library, "fgetwc_nid_postfix"); + const auto putByteUnlocked = Symbol(library, "putc_unlocked_nid_postfix"); + Require(open && close && wide && putByte && putWide && seek && read && write && flush && descriptor && getByte && getWide && putByteUnlocked, + "libc is missing a stdio orientation test symbol"); + + void* stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not open guest stream"); + Require(wide(stream, 0) == 0, "a new stream should be unoriented"); + Require(seek(stream, 0, SEEK_SET) == 0, "seeking should succeed"); + Require(flush(stream) == 0, "flushing should succeed"); + Require(descriptor(stream) >= 0, "fileno should succeed"); + Require(read(nullptr, 0, 0, stream) == 0, "zero-length fread should be a no-op"); + Require(write(nullptr, 0, 0, stream) == 0, "zero-length fwrite should be a no-op"); + Require(wide(stream, 0) == 0, "metadata operations must not orient a stream"); + Require(wide(stream, 1) == 1, "positive mode should establish wide orientation"); + Require(wide(stream, -1) == 1, "an established orientation must not change"); + Require(close(stream) == 0, "could not close query-oriented stream"); + + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(putByte('B', stream) == 'B', "byte output failed"); + Require(wide(stream, 0) == -1, "byte output should establish byte orientation"); + Require(seek(stream, 0, SEEK_SET) == 0, "could not rewind byte stream"); + Require(getByte(stream) == 'B', "byte stream contents did not round-trip"); + Require(wide(stream, 0) == -1, "byte reads must preserve byte orientation"); + Require(close(stream) == 0, "could not close byte-oriented stream"); + + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(putByteUnlocked('U', stream) == 'U', "unlocked byte output failed"); + Require(wide(stream, 0) == -1, "unlocked byte output should establish byte orientation"); + Require(close(stream) == 0, "could not close unlocked byte-oriented stream"); + + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(putWide(u'W', stream) == u'W', "wide output failed"); + Require(wide(stream, 0) == 1, "wide output should establish wide orientation"); + Require(close(stream) == 0, "could not close wide-oriented stream"); + + { + std::FILE* host = std::fopen(argv[2], "wb"); + Require(host != nullptr, "could not prepare wide-input fixture"); + Require(std::fputc('I', host) == 'I', "could not write wide-input fixture"); + Require(std::fclose(host) == 0, "could not close wide-input fixture"); + } + stream = open(argv[2], "rb"); + Require(stream != nullptr, "could not open wide-input fixture"); + Require(getWide(stream) == 'I', "wide input returned the wrong character"); + Require(wide(stream, 0) == 1, "wide input should establish wide orientation"); + Require(close(stream) == 0, "could not close wide-input stream"); + + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(putByte('R', stream) == 'R', "byte output before freopen failed"); + Require(wide(stream, 0) == -1, "precondition: byte orientation"); + const auto reopen = Symbol(library, "freopen_nid_postfix"); + Require(reopen != nullptr, "libc is missing freopen"); + stream = reopen(argv[2], "w+b", stream); + Require(stream != nullptr, "freopen failed"); + Require(wide(stream, 0) == 0, "successful freopen should reset orientation"); + Require(getWide(stream) == -1, "wide read at EOF should fail"); + Require(wide(stream, 0) == 1, "wide input should establish orientation even at EOF"); + Require(close(stream) == 0, "could not close reopened stream"); + + const auto ungetWide = Symbol(library, "ungetwc_nid_postfix"); + const auto getLine = Symbol(library, "fgets_nid_postfix"); + const auto getWideLine = Symbol(library, "fgetws_nid_postfix"); + Require(ungetWide && getLine && getWideLine, "libc is missing a line or unget symbol"); + char line[4]{}; + char16_t wideLine[4]{}; + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(ungetWide(-1, stream) == -1, "ungetwc of WEOF should fail"); + Require(wide(stream, 0) == 1, "ungetwc should orient the stream even for WEOF"); + Require(close(stream) == 0, "could not close stream after ungetwc"); + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(getLine(line, 0, stream) == nullptr, "fgets with size 0 should fail"); + Require(wide(stream, 0) == -1, "fgets should orient the stream even for size 0"); + Require(close(stream) == 0, "could not close stream after fgets"); + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(getWideLine(wideLine, 0, stream) == nullptr, "fgetws with size 0 should fail"); + Require(wide(stream, 0) == 1, "fgetws should orient the stream even for size 0"); + Require(close(stream) == 0, "could not close stream after fgetws"); + + const auto scanList = Symbol(library, "vfscanf_nid_postfix"); + Require(scanList != nullptr, "libc is missing vfscanf"); + int number = 0; + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(ScanThrough(scanList, stream, "%d", &number) == EOF, "vfscanf at end of file should fail"); + Require(wide(stream, 0) == -1, "vfscanf should orient the stream to bytes"); + Require(close(stream) == 0, "could not close stream after vfscanf"); + + const auto scanSecure = Symbol(library, "vfscanf_s_nid_postfix"); + const auto printSecure = Symbol(library, "vfprintf_s_nid_postfix"); + const auto printWideSecure = Symbol(library, "vfwprintf_s_nid_postfix"); + Require(scanSecure && printSecure && printWideSecure, "libc is missing a bounds-checked stdio symbol"); + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(ScanThrough(scanSecure, stream, "%d", &number) == EOF, "vfscanf_s at end of file should fail"); + Require(wide(stream, 0) == -1, "vfscanf_s should orient the stream to bytes"); + Require(close(stream) == 0, "could not close stream after vfscanf_s"); + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(ScanThrough(printSecure, stream, "%d", 7) == 1, "vfprintf_s should write one byte"); + Require(wide(stream, 0) == -1, "vfprintf_s should orient the stream to bytes"); + Require(close(stream) == 0, "could not close stream after vfprintf_s"); + stream = open(argv[2], "w+b"); + Require(stream != nullptr, "could not reopen guest stream"); + Require(WideThrough(printWideSecure, stream, u"%d", 7) == 1, "vfwprintf_s should write one character"); + Require(wide(stream, 0) == 1, "vfwprintf_s should orient the stream wide"); + Require(close(stream) == 0, "could not close stream after vfwprintf_s"); + + std::remove(argv[2]); +#ifdef _WIN32 + FreeLibrary(library); +#else + dlclose(library); +#endif + return 0; +} diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 53d5fd33bd..8d371f891d 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -98,6 +98,8 @@ - [strtoq](../../../core/libs/prx/libc/src/Strings.cpp) (libc, NID `yUUOxgk8Dmc`) is the BSD "quad" alias of `strtoll` and forwards to it, as FreeBSD 11.4 [strtoq.c](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/stdlib/strtoq.c) does. The console export is not verified; the sibling `strtouq` is already here. +- [fwide](../../../core/libs/prx/libc/src/StandardStreams.cpp) (libc) - the `printf` and `wprintf` families, `perror` and `putw` orient a stream as C11 7.21.2p4 requires, where FreeBSD 11.4's `vfprintf.c`, `vfwprintf.c`, `perror.c` and `putw.c` have no `ORIENT`; nothing is measured on a console. Otherwise it follows [FreeBSD 11.4 fwide.c](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/stdio/fwide.c#L38-L51) and `ORIENT`: the first byte or wide read, write, unget or scan orients a stream, also when `fgets`, `fgetws` or `ungetwc` then fail on their arguments; a zero-length `fread` or `fwrite` does not, and a successful `freopen` clears it. Mixed byte and wide calls still run. The export is in the [PS5 3.20 inventory](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceLibcInternal.c#L17666) + ### Functional - [_Cosh](../../../core/libs/prx/libc/src/Math.cpp) (libc) follows the two-argument Dinkumware CRT helper used by `std::complex`, including a scaled large-finite-input path; the signature, exact result and floating-point exception behavior are not verified against the console. From 65a8e8e55b409a85e4598468bc15c4ff7b09c2a4 Mon Sep 17 00:00:00 2001 From: Alexandre Figueiredo Date: Fri, 30 Oct 2026 13:28:54 +0000 Subject: [PATCH 242/318] test(libSceNpCppWebApi): list the String and unsigned long vector destructors after the String destructor --- core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp b/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp index 49fadedb5b..b660df81de 100644 --- a/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp +++ b/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp @@ -479,6 +479,8 @@ int main(int argc, char** argv) { {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_15ProfanityFilter2V219WebApiFilterRequestEED1Ev, 0}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_18ResponseHeaderBaseEED1Ev, 0}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6StringEED1Ev, 0}, + {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEED1Ev, StringVectorReferenceCountOffset}, + {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEED1Ev, ScalarVectorReferenceCountOffset}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS3_INS1_11UserProfile2V113BasicPresenceEEEEEED1Ev, VectorReferenceCountOffset}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_13InGameCatalog2V57ProductEED1Ev, 0}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_13InGameCatalog2V59ContainerEED1Ev, 0}, @@ -492,8 +494,6 @@ int main(int argc, char** argv) { {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_11UserProfile2V113BasicPresenceEED1Ev, 0}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_30CommunicationRestrictionStatus2V338CommunicationRestrictionStatusResponseEED1Ev, 0}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS3_INS1_11UserProfile2V112BasicProfileEEEEEED1Ev, VectorReferenceCountOffset}, - {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS2_6StringEEEED1Ev, StringVectorReferenceCountOffset}, - {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEED1Ev, ScalarVectorReferenceCountOffset}, }; for (const auto& testCase : destructors) TestDestroy(testCase); From 770baa1227e6a7def0a1724246bc427e8f4a081b Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 30 Oct 2026 18:24:03 +0000 Subject: [PATCH 243/318] fix(libSceJson2): expose scalar defaults and preserve identity --- core/libs/prx/libSceJson2/Export.cpp | 50 ++++++++++++++++------- core/libs/prx/libSceJson2/include/Api.hpp | 5 +++ core/libs/tests/GuestJson.cpp | 23 ++++++++++- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 39231f96e3..bccf2e24e6 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1570,17 +1570,21 @@ int InitializeResources(void* self, const void* parameter, std::int32_t format, return 0; } -const Node& Typed(const Value* self, ValueType requested) { +const Node& Typed(const Value* self, ValueType requested, bool* usedDefault = nullptr) { + const auto fallback = [&]() -> const Node& { + if (usedDefault) *usedDefault = true; + return Defaults(requested); + }; if (self->type == requested || ((requested == TypeInteger || requested == TypeUInteger) && (self->type == TypeInteger || self->type == TypeUInteger))) return *self; if (!ModernValues() && self->type == TypeNull) { const auto* auxiliary = RootOf(*self).auxiliary; if (auxiliary && auxiliary->nullAccess) - return Typed(&auxiliary->nullAccess(requested, self->parent, auxiliary->nullAccessContext), requested); + return Typed(&auxiliary->nullAccess(requested, self->parent, auxiliary->nullAccessContext), requested, usedDefault); } CallbackScope scope; - if (!scope.entered) return Defaults(requested); + if (!scope.entered) return fallback(); NullAccessCallback callback = nullptr; void* context = nullptr; bool initialized; @@ -1591,26 +1595,26 @@ const Node& Typed(const Value* self, ValueType requested) { callback = state.callback; context = state.context; } - if (!initialized) return Defaults(requested); + if (!initialized) return fallback(); if (!ModernValues()) { const auto record = LegacyHandlerOf(RootOf(*self)); - if (record.callback) return Typed(&record.callback(requested, self, self->parent, record.context), requested); - return Defaults(requested); + if (record.callback) return Typed(&record.callback(requested, self, self->parent, record.context), requested, usedDefault); + return fallback(); } if (self->type == TypeNull) { const auto* auxiliary = RootOf(*self).auxiliary; if (auxiliary && auxiliary->nullAccess) - return Typed(&auxiliary->nullAccess(requested, self->parent, auxiliary->nullAccessContext), requested); + return Typed(&auxiliary->nullAccess(requested, self->parent, auxiliary->nullAccessContext), requested, usedDefault); } const auto* auxiliary = RootOf(*self).auxiliary; if (ModernValues() && auxiliary && auxiliary->typeMismatch && auxiliary->typeMismatch->callback) - return Typed(&auxiliary->typeMismatch->callback(requested, self, self->parent, auxiliary->typeMismatch->context), requested); + return Typed(&auxiliary->typeMismatch->callback(requested, self, self->parent, auxiliary->typeMismatch->context), requested, usedDefault); if (self->type == TypeNull && callback) - return Typed(&callback(requested, self->parent, context), requested); + return Typed(&callback(requested, self->parent, context), requested, usedDefault); const auto handlers = HandlersOf(*self); if (handlers.typeMismatch) - return Typed(&handlers.typeMismatch(requested, self, self->parent, handlers.typeMismatchContext), requested); - return Defaults(requested); + return Typed(&handlers.typeMismatch(requested, self, self->parent, handlers.typeMismatchContext), requested, usedDefault); + return fallback(); } const Value& MissingElement(const Value* self, int error, const String* key, const std::size_t* index) { @@ -2650,6 +2654,10 @@ int APS5_VABI _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9Value return state.allocation ? 0 : JsonErrorNotInitialized; } +extern const bool _ZN3sce4Json5Value10s_nullboolE = false; +extern const std::int64_t _ZN3sce4Json5Value13s_nullintegerE = 0; +extern const std::uint64_t _ZN3sce4Json5Value14s_nulluintegerE = 0; +extern const double _ZN3sce4Json5Value10s_nullrealE = 0; std::size_t _ZN3sce4Json6String4nposE = static_cast(-1); void APS5_VABI _ZN3sce4Json6StringC1Ev(String* self) { Construct(*self, std::string_view{}); } @@ -3015,14 +3023,20 @@ void APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value* self, const Value* other) { } ValueType APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value* self) { return NodeOf(*self).type; } -const bool* APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value* self) { return &Typed(self, TypeBoolean).boolean; } +const bool* APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value* self) { + bool usedDefault = false; + const auto& value = Typed(self, TypeBoolean, &usedDefault); + return usedDefault ? &_ZN3sce4Json5Value10s_nullboolE : &value.boolean; +} const std::int64_t* APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value* self) { Node& n = const_cast(NodeOf(*self)); if (n.type == TypeUInteger) { n.integer = std::bit_cast(n.uinteger); return &n.integer; } - return &Typed(self, TypeInteger).integer; + bool usedDefault = false; + const auto& value = Typed(self, TypeInteger, &usedDefault); + return usedDefault ? &_ZN3sce4Json5Value13s_nullintegerE : &value.integer; } const std::uint64_t* APS5_VABI _ZNK3sce4Json5Value11getUIntegerEv(const Value* self) { Node& n = const_cast(NodeOf(*self)); @@ -3030,9 +3044,15 @@ const std::uint64_t* APS5_VABI _ZNK3sce4Json5Value11getUIntegerEv(const Value* s n.uinteger = static_cast(n.integer); return &n.uinteger; } - return &Typed(self, TypeUInteger).uinteger; + bool usedDefault = false; + const auto& value = Typed(self, TypeUInteger, &usedDefault); + return usedDefault ? &_ZN3sce4Json5Value14s_nulluintegerE : &value.uinteger; +} +const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value* self) { + bool usedDefault = false; + const auto& value = Typed(self, TypeReal, &usedDefault); + return usedDefault ? &_ZN3sce4Json5Value10s_nullrealE : &value.real; } -const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value* self) { return &Typed(self, TypeReal).real; } const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value* self) { return Typed(self, TypeString).string; } const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value* self) { return Typed(self, TypeArray).array; } const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value* self) { return Typed(self, TypeObject).object; } diff --git a/core/libs/prx/libSceJson2/include/Api.hpp b/core/libs/prx/libSceJson2/include/Api.hpp index 62ea1bf4d8..7c01f7cdd4 100644 --- a/core/libs/prx/libSceJson2/include/Api.hpp +++ b/core/libs/prx/libSceJson2/include/Api.hpp @@ -88,6 +88,11 @@ using SerializationCallback = int (APS5_VABI*)(String&, void*); extern "C" { +extern const bool _ZN3sce4Json5Value10s_nullboolE; +extern const std::int64_t _ZN3sce4Json5Value13s_nullintegerE; +extern const std::uint64_t _ZN3sce4Json5Value14s_nulluintegerE; +extern const double _ZN3sce4Json5Value10s_nullrealE; + int APS5_VABI _ZN3sce4Json12MemAllocatorD0Ev(void); extern std::size_t _ZN3sce4Json6String4nposE; diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index 8f981f422f..589895b3d4 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -1248,9 +1248,11 @@ static void ValueTypeMismatchHandler() { Require(_ZNK3sce4Json5Array4sizeEv(_ZNK3sce4Json5Value8getArrayEv(&root)) == 0); RequireMismatch(3, TypeArray, &root, &context); const Value* element = _ZNK3sce4Json5ValueixEm(&Member(root, "list"), 0); - Require(*_ZNK3sce4Json5Value7getRealEv(element) == 0.0); + const double* missingReal = _ZNK3sce4Json5Value7getRealEv(element); + Require(missingReal == &_ZN3sce4Json5Value10s_nullrealE && *missingReal == 0.0); RequireMismatch(4, TypeReal, element, &context); - Require(*_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "none")) == 42); + Require(_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "none")) == &replacement.integer); + Require(replacement.integer == 42); Require(mismatchCalls == 5); Value later{}; @@ -1515,10 +1517,27 @@ static int NumericLocale() { return failures ? 1 : 0; } +static void ScalarDefaultIdentity() { + Value empty{}; + _ZN3sce4Json5ValueC1Ev(&empty); + Require(_ZNK3sce4Json5Value10getBooleanEv(&empty) == &_ZN3sce4Json5Value10s_nullboolE); + Require(_ZNK3sce4Json5Value10getIntegerEv(&empty) == &_ZN3sce4Json5Value13s_nullintegerE); + Require(_ZNK3sce4Json5Value11getUIntegerEv(&empty) == &_ZN3sce4Json5Value14s_nulluintegerE); + Require(_ZNK3sce4Json5Value7getRealEv(&empty) == &_ZN3sce4Json5Value10s_nullrealE); + _ZN3sce4Json5ValueD1Ev(&empty); + + Value actual{}; + _ZN3sce4Json5ValueC1Eb(&actual, true); + Require(_ZNK3sce4Json5Value10getBooleanEv(&actual) == &actual.boolean); + Require(*_ZNK3sce4Json5Value10getBooleanEv(&actual)); + _ZN3sce4Json5ValueD1Ev(&actual); +} + int main(int argc, char** argv) { if (argc == 2 && std::strcmp(argv[1], "--numeric-locale") == 0) return NumericLocale(); { ActiveModule module; + ScalarDefaultIdentity(); ParseAndRoundTrip(); NestingDepth(); ObjectsAndArrays(); From 844a09bb6fc7c326afb7dd3ebe7a258f27ebea25 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 30 Oct 2026 18:24:46 +0000 Subject: [PATCH 244/318] test(libSceJson2): check legacy default identity --- core/libs/tests/GuestJsonLegacySelf.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/libs/tests/GuestJsonLegacySelf.cpp b/core/libs/tests/GuestJsonLegacySelf.cpp index f87a1f22e4..102758b43d 100644 --- a/core/libs/tests/GuestJsonLegacySelf.cpp +++ b/core/libs/tests/GuestJsonLegacySelf.cpp @@ -173,6 +173,10 @@ int main() { std::uint8_t owner = 0; _ZN3sce4Json11InitializerC1Ev(&owner); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&owner, reinterpret_cast(¶meter)) == 0 && trace.live.size() == 11, "Initialize the eleven-resource legacy module"); Value value{}; _ZN3sce4Json5ValueC1Ev(&value); + Require(_ZNK3sce4Json5Value10getBooleanEv(&value) == &_ZN3sce4Json5Value10s_nullboolE, "Legacy Boolean default identity"); + Require(_ZNK3sce4Json5Value10getIntegerEv(&value) == &_ZN3sce4Json5Value13s_nullintegerE, "Legacy integer default identity"); + Require(_ZNK3sce4Json5Value11getUIntegerEv(&value) == &_ZN3sce4Json5Value14s_nulluintegerE, "Legacy unsigned default identity"); + Require(_ZNK3sce4Json5Value7getRealEv(&value) == &_ZN3sce4Json5Value10s_nullrealE, "Legacy real default identity"); Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&value, object ? "{}" : "[]", 2) == 0 && _ZN3sce4Json5Value21setNullAccessCallBackEPFRKS1_NS0_9ValueTypeEPS2_PvES6_(&value, nullptr, &trace) == 0, "Prepare legacy collection ownership and auxiliary callback context"); const Value before = value; const auto generations = trace.generations; From 24fd96dcc8840af71cdfd023d7548ab71f93fc97 Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 30 Oct 2026 18:37:28 +0000 Subject: [PATCH 245/318] fix(libSceJson2): export mutable default pointers --- core/libs/prx/libSceJson2/Export.cpp | 24 +++++-- core/libs/prx/libSceJson2/include/Api.hpp | 3 + core/libs/tests/GuestJsonAllocator.cpp | 84 +++++++++++++++++++++++ 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index bccf2e24e6..f9140ca01c 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1456,10 +1456,16 @@ void ResetDefaults(bool releaseMissing = false) { auto& defaults = SharedDefaults(); for (unsigned type = 0; type < 8; ++type) { if (defaults.ready[type]) { + if (releaseMissing && type == TypeString) defaults.values[type].string = _ZN3sce4Json5Value12s_nullstringE; + if (releaseMissing && type == TypeArray) defaults.values[type].array = _ZN3sce4Json5Value11s_nullarrayE; + if (releaseMissing && type == TypeObject) defaults.values[type].object = _ZN3sce4Json5Value12s_nullobjectE; const bool missing = type >= TypeString && !defaults.values[type].payload; Destroy(defaults.values[type]); if (releaseMissing && missing) Release(nullptr); } else if (releaseMissing && type >= TypeString) Release(nullptr); + if (releaseMissing && type == TypeString) _ZN3sce4Json5Value12s_nullstringE = nullptr; + if (releaseMissing && type == TypeArray) _ZN3sce4Json5Value11s_nullarrayE = nullptr; + if (releaseMissing && type == TypeObject) _ZN3sce4Json5Value12s_nullobjectE = nullptr; defaults.values[type] = {}; defaults.ready[type] = false; } @@ -1538,6 +1544,9 @@ void CreateDefault(ValueType type) { if constexpr (std::is_same_v) Construct(*handle, std::string_view{}); else Construct(*handle); } + if constexpr (std::is_same_v) _ZN3sce4Json5Value12s_nullstringE = handle; + else if constexpr (std::is_same_v) _ZN3sce4Json5Value11s_nullarrayE = handle; + else _ZN3sce4Json5Value12s_nullobjectE = handle; } int InitializeResources(void* self, const void* parameter, std::int32_t format, bool modern) { @@ -1559,9 +1568,8 @@ int InitializeResources(void* self, const void* parameter, std::int32_t format, state.fallback = static_cast(Allocate(sizeof(Value))); if (state.fallback) Construct(*state.fallback); state.internalMutex = CreateKernelMutex(); - const auto& defaults = SharedDefaults(); - const bool admitted = defaults.values[TypeString].string && defaults.values[TypeArray].array && - defaults.values[TypeObject].object && state.fallback && state.internalMutex; + const bool admitted = _ZN3sce4Json5Value12s_nullstringE && _ZN3sce4Json5Value11s_nullarrayE && + _ZN3sce4Json5Value12s_nullobjectE && state.fallback && state.internalMutex; if (admitted) state.recoveryMutex = ModernValues() ? CreateKernelMutex() : state.internalMutex; if (!admitted || !state.recoveryMutex) { ReleaseInitialization(); return JsonErrorOutOfMemory; } state.paddedStringGrowth = CompiledSdk() > 0x044fffff; @@ -1573,6 +1581,7 @@ int InitializeResources(void* self, const void* parameter, std::int32_t format, const Node& Typed(const Value* self, ValueType requested, bool* usedDefault = nullptr) { const auto fallback = [&]() -> const Node& { if (usedDefault) *usedDefault = true; + if (usedDefault && requested >= TypeString) return SharedDefaults().values[requested]; return Defaults(requested); }; if (self->type == requested || @@ -2658,6 +2667,9 @@ extern const bool _ZN3sce4Json5Value10s_nullboolE = false; extern const std::int64_t _ZN3sce4Json5Value13s_nullintegerE = 0; extern const std::uint64_t _ZN3sce4Json5Value14s_nulluintegerE = 0; extern const double _ZN3sce4Json5Value10s_nullrealE = 0; +String* _ZN3sce4Json5Value12s_nullstringE = nullptr; +Array* _ZN3sce4Json5Value11s_nullarrayE = nullptr; +Object* _ZN3sce4Json5Value12s_nullobjectE = nullptr; std::size_t _ZN3sce4Json6String4nposE = static_cast(-1); void APS5_VABI _ZN3sce4Json6StringC1Ev(String* self) { Construct(*self, std::string_view{}); } @@ -3053,9 +3065,9 @@ const double* APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value* self) { const auto& value = Typed(self, TypeReal, &usedDefault); return usedDefault ? &_ZN3sce4Json5Value10s_nullrealE : &value.real; } -const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value* self) { return Typed(self, TypeString).string; } -const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value* self) { return Typed(self, TypeArray).array; } -const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value* self) { return Typed(self, TypeObject).object; } +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value* self) { bool fallback = false; const auto& value = Typed(self, TypeString, &fallback); return fallback ? _ZN3sce4Json5Value12s_nullstringE : value.string; } +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value* self) { bool fallback = false; const auto& value = Typed(self, TypeArray, &fallback); return fallback ? _ZN3sce4Json5Value11s_nullarrayE : value.array; } +const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value* self) { bool fallback = false; const auto& value = Typed(self, TypeObject, &fallback); return fallback ? _ZN3sce4Json5Value12s_nullobjectE : value.object; } std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value* self) { const Node& n = NodeOf(*self); if (n.type == TypeArray) return n.array->items->size(); diff --git a/core/libs/prx/libSceJson2/include/Api.hpp b/core/libs/prx/libSceJson2/include/Api.hpp index 7c01f7cdd4..4a7f077873 100644 --- a/core/libs/prx/libSceJson2/include/Api.hpp +++ b/core/libs/prx/libSceJson2/include/Api.hpp @@ -92,6 +92,9 @@ extern const bool _ZN3sce4Json5Value10s_nullboolE; extern const std::int64_t _ZN3sce4Json5Value13s_nullintegerE; extern const std::uint64_t _ZN3sce4Json5Value14s_nulluintegerE; extern const double _ZN3sce4Json5Value10s_nullrealE; +extern String* _ZN3sce4Json5Value12s_nullstringE; +extern Array* _ZN3sce4Json5Value11s_nullarrayE; +extern Object* _ZN3sce4Json5Value12s_nullobjectE; int APS5_VABI _ZN3sce4Json12MemAllocatorD0Ev(void); diff --git a/core/libs/tests/GuestJsonAllocator.cpp b/core/libs/tests/GuestJsonAllocator.cpp index 9fb38428b0..b1240022ba 100644 --- a/core/libs/tests/GuestJsonAllocator.cpp +++ b/core/libs/tests/GuestJsonAllocator.cpp @@ -20,13 +20,23 @@ struct AllocationTrace { bool failOnce = false; Parameter* changeInput = nullptr; void* replacementContext = nullptr; + std::size_t clearStringAt = 0; + String* displacedString = nullptr; std::map live; std::vector requests; std::vector notifications; + std::vector publishedDefaults; }; static void* APS5_VABI AllocateTracked(void*, std::size_t bytes, void* context) { auto& trace = *static_cast(context); trace.requests.push_back(bytes); + trace.publishedDefaults.push_back(unsigned(_ZN3sce4Json5Value12s_nullstringE != nullptr) | + (unsigned(_ZN3sce4Json5Value11s_nullarrayE != nullptr) << 1) | + (unsigned(_ZN3sce4Json5Value12s_nullobjectE != nullptr) << 2)); + if (trace.requests.size() == trace.clearStringAt) { + trace.displacedString = _ZN3sce4Json5Value12s_nullstringE; + _ZN3sce4Json5Value12s_nullstringE = nullptr; + } if (trace.changeInput && trace.requests.size() == 1) { trace.changeInput->context = trace.replacementContext; trace.changeInput->bufferSize = 32768; @@ -65,6 +75,10 @@ static void CheckResources(AllocationTrace& trace) { const std::vector requests{24, 8, 24, 8, 32, 48, 8, 32, 104, 32, 8, 8}; Require(trace.requests.size() >= requests.size() && std::equal(requests.begin(), requests.end(), trace.requests.end() - requests.size()), "Initialization must allocate its actual context, defaults, fallback and synchronization handles in reference order"); + const std::vector publication{0, 0, 0, 1, 1, 1, 3, 3, 3, 7, 7, 7}; + Require(trace.publishedDefaults.size() >= publication.size() && + std::equal(publication.begin(), publication.end(), trace.publishedDefaults.end() - publication.size()), + "Allocator callbacks must observe each default pointer only after its construction completes"); bool contextFound = false; for (const auto& [memory, bytes] : trace.live) { if (bytes != 24) continue; @@ -77,6 +91,19 @@ static void CheckResources(AllocationTrace& trace) { const auto* string = _ZNK3sce4Json5Value9getStringEv(&null); const auto* array = _ZNK3sce4Json5Value8getArrayEv(&null); const auto* object = _ZNK3sce4Json5Value9getObjectEv(&null); + Require(string == _ZN3sce4Json5Value12s_nullstringE && array == _ZN3sce4Json5Value11s_nullarrayE && + object == _ZN3sce4Json5Value12s_nullobjectE, "Getters must return the exported default pointers"); + auto* savedString = _ZN3sce4Json5Value12s_nullstringE; + auto* savedArray = _ZN3sce4Json5Value11s_nullarrayE; + auto* savedObject = _ZN3sce4Json5Value12s_nullobjectE; + _ZN3sce4Json5Value12s_nullstringE = nullptr; + _ZN3sce4Json5Value11s_nullarrayE = nullptr; + _ZN3sce4Json5Value12s_nullobjectE = nullptr; + Require(!_ZNK3sce4Json5Value9getStringEv(&null) && !_ZNK3sce4Json5Value8getArrayEv(&null) && + !_ZNK3sce4Json5Value9getObjectEv(&null), "Getters must observe writes to the exported pointers"); + _ZN3sce4Json5Value12s_nullstringE = savedString; + _ZN3sce4Json5Value11s_nullarrayE = savedArray; + _ZN3sce4Json5Value12s_nullobjectE = savedObject; Require(Owned(trace, string, 8) && Owned(trace, string->text, 24), "The default string must use the configured allocator"); Require(Owned(trace, array, 8) && Owned(trace, array->items, 32) && Owned(trace, *reinterpret_cast(array->items), 48), "The default array must own its allocator-backed header and sentinel"); @@ -167,6 +194,57 @@ static void CheckContextSnapshot() { Require(found && trace.live.size() == 12 && replacement.requests.empty(), "Context admission must cache allocator and user data before allocation and read the buffer setting afterward"); Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty() && replacement.live.empty(), "Termination must release the admitted context snapshot"); } +static void CheckReplacedDefaults() { + AllocationTrace trace; + TrackingAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0, + "Initialize before replacing exported defaults"); + auto* oldString = _ZN3sce4Json5Value12s_nullstringE; + auto* oldArray = _ZN3sce4Json5Value11s_nullarrayE; + auto* oldObject = _ZN3sce4Json5Value12s_nullobjectE; + auto* replacementString = static_cast(AllocateTracked(&allocator, sizeof(String), &trace)); + auto* replacementArray = static_cast(AllocateTracked(&allocator, sizeof(Array), &trace)); + auto* replacementObject = static_cast(AllocateTracked(&allocator, sizeof(Object), &trace)); + _ZN3sce4Json6StringC1Ev(replacementString); + _ZN3sce4Json5ArrayC1Ev(replacementArray); + _ZN3sce4Json6ObjectC1Ev(replacementObject); + _ZN3sce4Json6StringD1Ev(oldString); + _ZN3sce4Json5ArrayD1Ev(oldArray); + _ZN3sce4Json6ObjectD1Ev(oldObject); + FreeTracked(&allocator, oldString, &trace); + FreeTracked(&allocator, oldArray, &trace); + FreeTracked(&allocator, oldObject, &trace); + _ZN3sce4Json5Value12s_nullstringE = replacementString; + _ZN3sce4Json5Value11s_nullarrayE = replacementArray; + _ZN3sce4Json5Value12s_nullobjectE = replacementObject; + Value null; + _ZN3sce4Json5ValueC1Ev(&null); + Require(_ZNK3sce4Json5Value9getStringEv(&null) == replacementString && + _ZNK3sce4Json5Value8getArrayEv(&null) == replacementArray && + _ZNK3sce4Json5Value9getObjectEv(&null) == replacementObject, + "Getters must observe replacement pointers"); + _ZN3sce4Json5ValueD1Ev(&null); + Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && trace.live.empty(), + "Termination must destroy and release the replacement pointers"); +} +static void CheckAdmissionReadsPublishedPointer() { + AllocationTrace trace; + trace.clearStringAt = 10; + TrackingAllocator allocator; + Parameter parameter{&allocator, &trace, 16384, 2, {}}; + std::uint8_t initializer = 0; + _ZN3sce4Json11InitializerC1Ev(&initializer); + Require(static_cast(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter))) == 0x80848102 && !initializer, + "Admission must read the exported pointer after allocator callbacks"); + Require(trace.displacedString && trace.live.size() == 2 && Owned(trace, trace.displacedString, sizeof(String)) && + Owned(trace, trace.displacedString->text, 24), "The displaced default must remain owned by its allocator"); + FreeTracked(&allocator, trace.displacedString->text, &trace); + FreeTracked(&allocator, trace.displacedString, &trace); + Require(trace.live.empty(), "Fixture cleanup must release the displaced allocation"); +} int main() { for (int budget = 0; budget <= 12; ++budget) { AllocationTrace trace; @@ -179,6 +257,8 @@ int main() { if (budget < 12) { Require(static_cast(result) == 0x80848102 && initializer == 0 && trace.live.empty(), "Every initialization allocation failure must report OOM, roll back ownership and leave retry legal"); + Require(!_ZN3sce4Json5Value12s_nullstringE && !_ZN3sce4Json5Value11s_nullarrayE && !_ZN3sce4Json5Value12s_nullobjectE, + "Initialization rollback must clear the exported default pointers"); Require((budget == 0) == trace.notifications.empty(), "Only the first context-allocation failure omits notification"); trace.budget = -1; Require(_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(&initializer, reinterpret_cast(¶meter)) == 0, @@ -192,8 +272,12 @@ int main() { parameter.context = &unrelated; Require(_ZN3sce4Json11Initializer9terminateEv(&initializer) == 0 && initializer == 0 && trace.live.empty() && unrelated.live.empty(), "Termination must release the snapshotted allocation context and every resource"); + Require(!_ZN3sce4Json5Value12s_nullstringE && !_ZN3sce4Json5Value11s_nullarrayE && !_ZN3sce4Json5Value12s_nullobjectE, + "Termination must clear the exported default pointers"); } CheckSingleFailures(); CheckContextSnapshot(); + CheckReplacedDefaults(); + CheckAdmissionReadsPublishedPointer(); return 0; } From 936ae10987c5c33a92327768513510d384355280 Mon Sep 17 00:00:00 2001 From: slopgobblercobbler Date: Sun, 25 Oct 2026 18:23:20 +0100 Subject: [PATCH 246/318] fix(libSceUserService): validate user login status in sceUserServiceGetGamePresets --- core/libs/prx/libSceUserService/Export.cpp | 9 ++++----- core/libs/tests/GuestUserService.cpp | 15 ++++++++++++++- docs/dev/debt/libSceUserService.md | 1 + 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceUserService/Export.cpp b/core/libs/prx/libSceUserService/Export.cpp index 3d1628ff25..f492dcef95 100644 --- a/core/libs/prx/libSceUserService/Export.cpp +++ b/core/libs/prx/libSceUserService/Export.cpp @@ -190,11 +190,10 @@ int APS5_VABI sceUserServiceGetEvent(SceUserServiceEvent* event) { } int APS5_VABI sceUserServiceGetGamePresets(int user_id, UserServiceGamePresets* presets) { - // System-wide game presets (difficulty, view inversion, subtitles, audio language) of the user's - // profile. A fresh console has none set: every field is 0, "not specified", and the title falls - // back to its own defaults. this_size is the caller's. - const int index = UserIndex(user_id); - if (presets == nullptr || index < 0) { + if (!IsUserLoggedIn(user_id)) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + if (presets == nullptr) { return USER_SERVICE_ERROR_INVALID_ARGUMENT; } presets->difficulty = 0; diff --git a/core/libs/tests/GuestUserService.cpp b/core/libs/tests/GuestUserService.cpp index 852b8d6da0..4e4b86548d 100644 --- a/core/libs/tests/GuestUserService.cpp +++ b/core/libs/tests/GuestUserService.cpp @@ -154,7 +154,11 @@ int main() { Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID, nullptr) == USER_SERVICE_ERROR_INVALID_ARGUMENT); UserServiceGamePresets presets{}; presets.difficulty = 0x5a; - Require(sceUserServiceGetGamePresets(0, &presets) == USER_SERVICE_ERROR_INVALID_ARGUMENT && presets.difficulty == 0x5a); + for (int user_id : absent_users) { + Require(sceUserServiceGetGamePresets(user_id, &presets) == USER_SERVICE_ERROR_NOT_LOGGED_IN); + Require(presets.difficulty == 0x5a); + Require(sceUserServiceGetGamePresets(user_id, nullptr) == USER_SERVICE_ERROR_NOT_LOGGED_IN); + } Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID, &presets) == USER_SERVICE_OK && presets.difficulty == 0); std::int32_t accessibilityValue = 0; bool threwKeyremap = false; @@ -209,6 +213,10 @@ int main() { Require(value == 0x5a5a5a5a); Require(setting.get(USER_SERVICE_INITIAL_USER_ID + 1, nullptr) == USER_SERVICE_ERROR_NOT_LOGGED_IN); } + presets.difficulty = 0x5a; + Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, &presets) == USER_SERVICE_ERROR_NOT_LOGGED_IN); + Require(presets.difficulty == 0x5a); + Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, nullptr) == USER_SERVICE_ERROR_NOT_LOGGED_IN); UserServiceSetUserConnected_nid_postfix(1, true); Require(sceUserServiceGetRegisteredUserIdList(®istered) == USER_SERVICE_OK); @@ -232,6 +240,7 @@ int main() { Require(setting.get(USER_SERVICE_INITIAL_USER_ID + 1, &value) == USER_SERVICE_OK); Require(value == setting.value); } + Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, &presets) == USER_SERVICE_OK && presets.difficulty == 0); UserServiceSetUserConnected_nid_postfix(1, false); Require(sceUserServiceGetRegisteredUserIdList(®istered) == USER_SERVICE_OK); @@ -255,4 +264,8 @@ int main() { Require(value == 0x5a5a5a5a); Require(setting.get(USER_SERVICE_INITIAL_USER_ID + 1, nullptr) == USER_SERVICE_ERROR_NOT_LOGGED_IN); } + presets.difficulty = 0x5a; + Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, &presets) == USER_SERVICE_ERROR_NOT_LOGGED_IN); + Require(presets.difficulty == 0x5a); + Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, nullptr) == USER_SERVICE_ERROR_NOT_LOGGED_IN); } diff --git a/docs/dev/debt/libSceUserService.md b/docs/dev/debt/libSceUserService.md index c8320c397b..3ef16bf522 100644 --- a/docs/dev/debt/libSceUserService.md +++ b/docs/dev/debt/libSceUserService.md @@ -7,6 +7,7 @@ - [sceUserServiceGetUserColor](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - the profile colour corresponds to user index (0: blue, 1: red, 2: green, 3: pink); colours 1, 2 and 3 are assumed, not measured - [sceUserServiceGetUserName](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - `0x80960105`, which a PS5 (firmware 13.60) returns for a user id it does not know, has no known name; it is `USER_SERVICE_ERROR_UNKNOWN_USER` here. On the console a registered user who is not logged in still gets their name; here up to 4 users ("Player", "Player2", "Player3", "Player4") are supported; user names "Player2", "Player3" and "Player4" are assumed, not measured - [sceUserServiceGetEvent](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - login and logout events queued dynamically on controller hotplug have no source and are assumed, not measured. Six titles disassembled for this entry drain the queue until the call returns nonzero. PPSA01494 01.100.002 (at 0x374C0C6), PPSA30839 01.003.000 (0x4DF3F0), PPSA31162 02.000.001 (0x550ED0) and PPSA34502 01.010.000 (0x30035C, 0x300433) act only on `event_type` 0 and 1, reading `user_id` at offset 4 for both, and skip other types. PPSA18194 01.004.003 (0x16D44A3) tells `NOT_INITIALIZED` (0x80960002) and `NO_EVENT` (0x80960007) apart and logs any other error. PPSA21564 01.007.000 copies each 8-byte event into a list at one site (0x2AFCA8) and discards them at the other (0x2B6CF3) +- [sceUserServiceGetGamePresets](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - system-wide game presets (difficulty, view inversion, subtitles, audio language) of a user's profile are unmeasured and not modelled, so all fields are reported as 0 ("not specified") allowing titles to use their defaults. Return of `0x80960009` (`USER_SERVICE_ERROR_NOT_LOGGED_IN`) for a user who is not logged in and precedence over null presets pointer checking are borrowed from `sceUserServiceGetAgeLevel` and are not measured for this call; a logged-in user with a null presets pointer returns `USER_SERVICE_ERROR_INVALID_ARGUMENT` - [sceUserServiceGetAgeLevel](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - on a PS5 (firmware 13.60) the value is the age set in the user's parental restriction (0, 7, 12 and 15 were measured) and `0xFFFFFFFF` without one; a user who is not logged in returns `0x80960009` with or without an output. Parental restrictions are not modelled, so the initial user is always reported as unrestricted. PPSA12544 (Unity) only stores the value for its user profile. The one call in PPSA12544 (0x16a80a0), made only when bit 0x4 of its flags is set, writes the value to a field of its own object (offset 0x5170) and ends its chain of getters on any nonzero result; what it does with the value afterwards was not read. - [sceUserServiceGetAccessibilityChatTranscription, PressAndHoldDelay, TriggerEffect, Vibration, ZoomEnabled, ZoomFollowFocus](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - report 0, 0, 1, 1, 0 and 1, what a PS5 (firmware 13.60) reports for a user whose accessibility settings are all at their defaults; the console reports the user's own settings (a changed vibration intensity read 3), which are not modelled. What each number means is unknown. A user who is not logged in returns `0x80960009` with or without an output. One title (PPSA04354) disassembled for this entry calls all six in a row (first at 0xea0efa) with its user ID, only when a flag bit (0x10) is set; a result other than 0 from any of them ends the block and is returned, and only when all six return 0 does it store the six values in its object. - [sceUserServiceGetAccessibilityKeyremapEnable, sceUserServiceGetAccessibilityZoom](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - values reported by a console and their formats are unmeasured, so both throw. From 9c55d9117c0e8058170c5508351b096f3d8919bb Mon Sep 17 00:00:00 2001 From: slopgobblercobbler Date: Mon, 26 Oct 2026 18:45:40 +0100 Subject: [PATCH 247/318] fix(libSceUserService): validate presets pointer before user login check in sceUserServiceGetGamePresets --- core/libs/prx/libSceUserService/Export.cpp | 6 +++--- core/libs/tests/GuestUserService.cpp | 6 +++--- docs/dev/debt/libSceUserService.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceUserService/Export.cpp b/core/libs/prx/libSceUserService/Export.cpp index f492dcef95..079756e372 100644 --- a/core/libs/prx/libSceUserService/Export.cpp +++ b/core/libs/prx/libSceUserService/Export.cpp @@ -190,12 +190,12 @@ int APS5_VABI sceUserServiceGetEvent(SceUserServiceEvent* event) { } int APS5_VABI sceUserServiceGetGamePresets(int user_id, UserServiceGamePresets* presets) { - if (!IsUserLoggedIn(user_id)) { - return USER_SERVICE_ERROR_NOT_LOGGED_IN; - } if (presets == nullptr) { return USER_SERVICE_ERROR_INVALID_ARGUMENT; } + if (!IsUserLoggedIn(user_id)) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } presets->difficulty = 0; presets->priority = 0; presets->invert_vertical_view_for_1st_person_view = 0; diff --git a/core/libs/tests/GuestUserService.cpp b/core/libs/tests/GuestUserService.cpp index 4e4b86548d..39de9a2f70 100644 --- a/core/libs/tests/GuestUserService.cpp +++ b/core/libs/tests/GuestUserService.cpp @@ -157,7 +157,7 @@ int main() { for (int user_id : absent_users) { Require(sceUserServiceGetGamePresets(user_id, &presets) == USER_SERVICE_ERROR_NOT_LOGGED_IN); Require(presets.difficulty == 0x5a); - Require(sceUserServiceGetGamePresets(user_id, nullptr) == USER_SERVICE_ERROR_NOT_LOGGED_IN); + Require(sceUserServiceGetGamePresets(user_id, nullptr) == USER_SERVICE_ERROR_INVALID_ARGUMENT); } Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID, &presets) == USER_SERVICE_OK && presets.difficulty == 0); std::int32_t accessibilityValue = 0; @@ -216,7 +216,7 @@ int main() { presets.difficulty = 0x5a; Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, &presets) == USER_SERVICE_ERROR_NOT_LOGGED_IN); Require(presets.difficulty == 0x5a); - Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, nullptr) == USER_SERVICE_ERROR_NOT_LOGGED_IN); + Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, nullptr) == USER_SERVICE_ERROR_INVALID_ARGUMENT); UserServiceSetUserConnected_nid_postfix(1, true); Require(sceUserServiceGetRegisteredUserIdList(®istered) == USER_SERVICE_OK); @@ -267,5 +267,5 @@ int main() { presets.difficulty = 0x5a; Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, &presets) == USER_SERVICE_ERROR_NOT_LOGGED_IN); Require(presets.difficulty == 0x5a); - Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, nullptr) == USER_SERVICE_ERROR_NOT_LOGGED_IN); + Require(sceUserServiceGetGamePresets(USER_SERVICE_INITIAL_USER_ID + 1, nullptr) == USER_SERVICE_ERROR_INVALID_ARGUMENT); } diff --git a/docs/dev/debt/libSceUserService.md b/docs/dev/debt/libSceUserService.md index 3ef16bf522..0ca0e04135 100644 --- a/docs/dev/debt/libSceUserService.md +++ b/docs/dev/debt/libSceUserService.md @@ -7,7 +7,7 @@ - [sceUserServiceGetUserColor](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - the profile colour corresponds to user index (0: blue, 1: red, 2: green, 3: pink); colours 1, 2 and 3 are assumed, not measured - [sceUserServiceGetUserName](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - `0x80960105`, which a PS5 (firmware 13.60) returns for a user id it does not know, has no known name; it is `USER_SERVICE_ERROR_UNKNOWN_USER` here. On the console a registered user who is not logged in still gets their name; here up to 4 users ("Player", "Player2", "Player3", "Player4") are supported; user names "Player2", "Player3" and "Player4" are assumed, not measured - [sceUserServiceGetEvent](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - login and logout events queued dynamically on controller hotplug have no source and are assumed, not measured. Six titles disassembled for this entry drain the queue until the call returns nonzero. PPSA01494 01.100.002 (at 0x374C0C6), PPSA30839 01.003.000 (0x4DF3F0), PPSA31162 02.000.001 (0x550ED0) and PPSA34502 01.010.000 (0x30035C, 0x300433) act only on `event_type` 0 and 1, reading `user_id` at offset 4 for both, and skip other types. PPSA18194 01.004.003 (0x16D44A3) tells `NOT_INITIALIZED` (0x80960002) and `NO_EVENT` (0x80960007) apart and logs any other error. PPSA21564 01.007.000 copies each 8-byte event into a list at one site (0x2AFCA8) and discards them at the other (0x2B6CF3) -- [sceUserServiceGetGamePresets](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - system-wide game presets (difficulty, view inversion, subtitles, audio language) of a user's profile are unmeasured and not modelled, so all fields are reported as 0 ("not specified") allowing titles to use their defaults. Return of `0x80960009` (`USER_SERVICE_ERROR_NOT_LOGGED_IN`) for a user who is not logged in and precedence over null presets pointer checking are borrowed from `sceUserServiceGetAgeLevel` and are not measured for this call; a logged-in user with a null presets pointer returns `USER_SERVICE_ERROR_INVALID_ARGUMENT` +- [sceUserServiceGetGamePresets](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - system-wide game presets (difficulty, view inversion, subtitles, audio language) of a user's profile are unmeasured and not modelled, so all fields are reported as 0 ("not specified") allowing titles to use their defaults. PS5 firmware 13.42 disassembly verifies that checking a null presets pointer (returning 0x80960005, USER_SERVICE_ERROR_INVALID_ARGUMENT) precedes any user login check before dispatching IPC 0x30026; returning 0x80960009 (USER_SERVICE_ERROR_NOT_LOGGED_IN) for an unlogged user with a valid presets pointer is assumed from the service. - [sceUserServiceGetAgeLevel](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - on a PS5 (firmware 13.60) the value is the age set in the user's parental restriction (0, 7, 12 and 15 were measured) and `0xFFFFFFFF` without one; a user who is not logged in returns `0x80960009` with or without an output. Parental restrictions are not modelled, so the initial user is always reported as unrestricted. PPSA12544 (Unity) only stores the value for its user profile. The one call in PPSA12544 (0x16a80a0), made only when bit 0x4 of its flags is set, writes the value to a field of its own object (offset 0x5170) and ends its chain of getters on any nonzero result; what it does with the value afterwards was not read. - [sceUserServiceGetAccessibilityChatTranscription, PressAndHoldDelay, TriggerEffect, Vibration, ZoomEnabled, ZoomFollowFocus](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - report 0, 0, 1, 1, 0 and 1, what a PS5 (firmware 13.60) reports for a user whose accessibility settings are all at their defaults; the console reports the user's own settings (a changed vibration intensity read 3), which are not modelled. What each number means is unknown. A user who is not logged in returns `0x80960009` with or without an output. One title (PPSA04354) disassembled for this entry calls all six in a row (first at 0xea0efa) with its user ID, only when a flag bit (0x10) is set; a result other than 0 from any of them ends the block and is returned, and only when all six return 0 does it store the six values in its object. - [sceUserServiceGetAccessibilityKeyremapEnable, sceUserServiceGetAccessibilityZoom](../../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - values reported by a console and their formats are unmeasured, so both throw. From 9fc4a969c826434bd5d4f92f382b92f11dbed089 Mon Sep 17 00:00:00 2001 From: ntamas94 Date: Fri, 30 Oct 2026 20:56:25 +0100 Subject: [PATCH 248/318] docs(debt): keep the title out of the PS4 HTILE size sentence Co-Authored-By: Claude Opus 5.5 --- docs/dev/debt/libSceAgcDriver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 6304724558..e55e31bb1f 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -73,7 +73,7 @@ - [NGG vertex workgroups](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - type-2 PRIMGEN programs using DS instructions or `s_barrier` are prepared with the draw's routing and primitive registers on the existing mesh path. The workgroup requirement follows [Mesa's NGG lowering](https://github.com/mihawk-99/PS5_Mesa/blob/b3588f78dd7fd46fbadc7ecd487e5f14d4213e4b/src/amd/common/nir/ac_nir_lower_ngg.c#L1135); the mesh path supplies LDS and two guest lanes per host invocation for wave64 on a 32-lane host subgroup. Console routing has not been measured. Passthrough and tessellation keep their existing paths; unsupported mesh assembly and subgroup budgets throw. - [Shader header copies](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a header bound for a draw or dispatch at another address than the registered one is accepted when its fixed fields match the registered header, and every array it points at still holds the registered bytes; the registered header's metadata is used. `user_data` is not compared (PPSA21567 binds such a copy of a vertex shader header with `user_data` set to null; whether the console reads the copy's `user_data` is unknown). A copy whose registers, context registers, specials or semantics differ, or whose arrays are unreadable, throws `... refers to a replaced shader header`; what the console does with a header a title patches while relocating it is not measured. PPSA01325 (#2914) resolves the ABI with such a copy - [ClearStateContextRegisters](../../../core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp) (libSceAgcDriver) - `CLEAR_STATE` and a push-clear `CONTEXT_STATE` (operation 3) reset `SPI_PS_IN_CONTROL` (0x1b6) to NUM_INTERP 2, the value of Linux's GFX10 clear-state table (`clearstate_gfx10.h`, `gfx10_SECT_CONTEXT_def_2`) that radeonsi tracks after `CLEAR_STATE`. The value on a PS5 console is not measured. The CP queue reset (PM4 operation 0x09) and the starting context keep 0, because registration-time decoding prepares a pixel program only while NUM_INTERP is 0. The other registers radeonsi assumes after a clear (`CB_TARGET_MASK`, `CB_SHADER_MASK`, `PA_SU_SC_MODE_CNTL`, `PA_SU_VTX_CNTL`) still reset to the starting table. -- [HtileFillCovers](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - a fill is taken to cover a depth surface's whole HTILE when it spans 32 KiB per 1024x512 pixel block from DB_HTILE_DATA_BASE, the pipe-aligned SW_64KB_Z_X metadata extent AMD PAL's gfx10 addrlib gives for GB_ADDR_CONFIG 0x00100044 (1-8 samples, D16/D32). A PS4 depth target's HTILE spans 16 KiB per 512x512 pixel block, the macro tile addrlib's CI `ComputeTileDataWidthAndHeight` gives for 32-bit HTILE words, 16384 cache bits and the 8 pipes of P8_32x32_16x16, and the size the clears of Gang Beasts (CUSA04865) fill for 512x512, 2048x1152 and 4096x4096 targets; other metadata layouts are not decoded. A partial fill is ignored (the surface keeps its contents); not measured on the console. +- [HtileFillCovers](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - a fill is taken to cover a depth surface's whole HTILE when it spans 32 KiB per 1024x512 pixel block from DB_HTILE_DATA_BASE, the pipe-aligned SW_64KB_Z_X metadata extent AMD PAL's gfx10 addrlib gives for GB_ADDR_CONFIG 0x00100044 (1-8 samples, D16/D32). A PS4 depth target's HTILE spans 16 KiB per 512x512 pixel block, the macro tile addrlib's CI `ComputeTileDataWidthAndHeight` gives for 32-bit HTILE words, 16384 cache bits and the 8 pipes of P8_32x32_16x16, and the size a PS4 title's clears fill for 512x512, 2048x1152 and 4096x4096 targets; other metadata layouts are not decoded. A partial fill is ignored (the surface keeps its contents); not measured on the console. - [Depth fast clear through HTILE](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - a surface cleared by an HTILE fill is cleared to DB_DEPTH_CLEAR, also when it is next sampled. For a depth-only HTILE the clear word also carries a value (radv writes `minz = maxz = round(depth * 0x3fff)`, `HTILE_Z_CLEAR_REG` in Mesa's `src/amd/common/ac_descriptors.h`, and allows TC-compatible fast clears only to 0.0 or 1.0); which of the two the texture unit returns for a cleared tile was not measured. - [Depth-only and stencil HTILE fill](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - in a whole-HTILE fill, ZMask (bits 0-3) 0 clears the depth and SMem (bits 8-9) 0 clears the stencil, while SMem 3 leaves the stencil alone (`HtileFillClears`); other SMem values are taken to leave it alone too. The bit meanings follow the gfx10 HTILE layout as the code reads it, not a measurement; `agc_driver_depth_fast_clear_submit` pins them for the paths the Driver takes. - [Depth and stencil plane fills](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - a fill that covers a known depth surface's whole stencil plane (`DepthSliceBytes` of its extent at one byte per texel) with four equal bytes clears the host stencil to that byte the next time the surface is bound, sampled or stored to, and a fill that covers the whole depth plane (two or four bytes per texel) clears the depth when the dword is a float in 0.0-1.0, or two equal 16-bit halves read as UNORM for D16 (`NoteDepthMetadataFill`). A partial fill, a dword whose bytes differ, a depth value outside 0.0-1.0 (0xff000000 is seen), a multisampled surface, or a fill made before the surface exists leave the host image as it was. An HTILE fill or a recognized HTILE clear kernel noted after a plane fill replaces its value with DB_DEPTH_CLEAR/DB_STENCIL_CLEAR. On the console the fill writes the plane the depth block reads; the surfaces this was seen on keep the stencil out of HTILE (PPSA26525 Heretic + Hexen and PPSA21444 DOOM + DOOM II reset the Cohtml UI stencil plane to 1 with a 0x01010101 fill every frame, then draw with stencil Equal 1 and 2), and what the console returns for a plane fill under a compressed HTILE was not measured. `agc_driver_depth_plane_fill` pins the rules. From 881f168de66dad53067a26af0b9b75bdbe7793df Mon Sep 17 00:00:00 2001 From: Kyon Date: Fri, 30 Oct 2026 21:20:33 +0000 Subject: [PATCH 249/318] fix(libSceJson2): define fallback exports in declared namespace --- core/libs/prx/libSceJson2/Export.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index f9140ca01c..25ade15f99 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -2663,6 +2663,7 @@ int APS5_VABI _ZN3sce4Json11Initializer24setAllocatorInfoCallBackEPFviNS0_9Value return state.allocation ? 0 : JsonErrorNotInitialized; } +namespace SceJson2 { extern const bool _ZN3sce4Json5Value10s_nullboolE = false; extern const std::int64_t _ZN3sce4Json5Value13s_nullintegerE = 0; extern const std::uint64_t _ZN3sce4Json5Value14s_nulluintegerE = 0; @@ -2670,6 +2671,7 @@ extern const double _ZN3sce4Json5Value10s_nullrealE = 0; String* _ZN3sce4Json5Value12s_nullstringE = nullptr; Array* _ZN3sce4Json5Value11s_nullarrayE = nullptr; Object* _ZN3sce4Json5Value12s_nullobjectE = nullptr; +} std::size_t _ZN3sce4Json6String4nposE = static_cast(-1); void APS5_VABI _ZN3sce4Json6StringC1Ev(String* self) { Construct(*self, std::string_view{}); } From f0a81431a756175d4fa5f84ba11912083824fa6c Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Sun, 25 Oct 2026 17:51:06 +0100 Subject: [PATCH 250/318] feat(shader): split unused-result 64-bit bitwise buffer atomics without shaderBufferInt64Atomics buffer_atomic_or_x2, buffer_atomic_and_x2 and buffer_atomic_xor_x2 whose result is never read now compile to one 32-bit atomic per dword on a device that lacks shaderBufferInt64Atomics, such as MoltenVK. Other 64-bit buffer atomics and uses of the result still fail to compile there. Co-Authored-By: Claude Sonnet 5.5 --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 13 ++++++ .../include/SpirvBackend/SpirvAnalysis.hpp | 3 ++ .../SpirvBackend/SpirvRuntimeBuffer.hpp | 1 + .../SpirvBackend/src/SpirvAnalysis.cpp | 12 ++++++ .../SpirvBackend/src/SpirvEmitter.cpp | 4 ++ .../SpirvMemory/SpirvMemoryInstructions.cpp | 12 ++++++ .../SpirvBackend/src/SpirvRuntimeBuffer.cpp | 43 ++++++++++++++----- docs/dev/debt/shader.md | 1 + 8 files changed, 79 insertions(+), 10 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 0f76b563f7..d32c0af096 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -7539,6 +7539,11 @@ void verifyInt64AtomicCapabilities() { 0u, 0u, 0u, 0u}; const std::vector bufferAtomic{0xe1705000u, 0x80012803u, 0xbf810000u}; const std::vector imageAtomic{0xf03c0308u, 0x00020a08u, 0xbf810000u}; + const std::vector bufferOr{0xe1685000u, 0x80012803u, 0xbf810000u}; + const std::vector bufferAnd{0xe1645000u, 0x80012803u, 0xbf810000u}; + const std::vector bufferXor{0xe16c5000u, 0x80012803u, 0xbf810000u}; + const std::vector bufferOrStored{0xe1685000u, 0x80012803u, 0xe0741000u, 0x80012803u, 0xbf810000u}; + const std::vector bufferAdd{0xe1485000u, 0x80012803u, 0xbf810000u}; const auto compile = [&](std::span code, spv::Capability added) { const std::array capabilities{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, static_cast(added)}; RecompileRequest request{}; @@ -7569,6 +7574,14 @@ void verifyInt64AtomicCapabilities() { }; expectFailure([&] { static_cast(compile(bufferAtomic, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_inc_x2 compiled without shaderBufferInt64Atomics"); require(declares(compile(bufferAtomic, spv::CapabilityInt64Atomics), spv::CapabilityInt64Atomics), "64-bit atomics: a buffer_atomic_inc_x2 does not declare Int64Atomics"); + for (const auto* code : {&bufferOr, &bufferAnd, &bufferXor}) { + const auto split = compile(*code, spv::CapabilityShader); + require(!declares(split, spv::CapabilityInt64Atomics), "64-bit atomics: an unused-result bitwise buffer atomic declares Int64Atomics on a device without it"); + require(!split.empty(), "64-bit atomics: an unused-result bitwise buffer atomic was not compiled as two 32-bit atomics"); + require(declares(compile(*code, spv::CapabilityInt64Atomics), spv::CapabilityInt64Atomics), "64-bit atomics: an unused-result bitwise buffer atomic does not use the 64-bit atomic where the device has it"); + } + expectFailure([&] { static_cast(compile(bufferOrStored, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_or_x2 whose result is stored compiled without shaderBufferInt64Atomics"); + expectFailure([&] { static_cast(compile(bufferAdd, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_add_x2 compiled without shaderBufferInt64Atomics"); expectFailure([&] { static_cast(compile(imageAtomic, spv::CapabilityShader)); }, "64-bit image atomics need VK_EXT_shader_image_atomic_int64", "64-bit atomics: an image_atomic_swap on a 32_32 image compiled without shaderImageInt64Atomics"); require(declares(compile(imageAtomic, spv::CapabilityInt64ImageEXT), spv::CapabilityInt64ImageEXT), "64-bit atomics: an image_atomic_swap on a 32_32 image does not declare Int64ImageEXT"); } diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvAnalysis.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvAnalysis.hpp index ca22375cce..178853cbd7 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvAnalysis.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvAnalysis.hpp @@ -25,6 +25,8 @@ struct SpirvRequirements { bool functionScratch = false; bool pixelValidMask = false; bool bufferInt64Atomics = false; + bool bufferInt64AtomicsExact = false; + bool bufferInt64AtomicsSplit = false; bool coherentBufferPairs = false; bool bufferU64View = false; bool imageInt64Atomics = false; @@ -37,6 +39,7 @@ struct SpirvRequirements { std::vector extensions; }; +[[nodiscard]] bool IsSplittableBufferAtomic64(const IrValue* inst); [[nodiscard]] SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program); inline constexpr std::uint32_t FunctionLdsDwordLimit = 8192u; inline constexpr std::uint32_t PixelFunctionStorageDwords = 4096u; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp index bd7fa1a632..a76f780378 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp @@ -10,6 +10,7 @@ std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValu void EmitRuntimeBufferStore(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components); std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction); std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function& operation); +void EmitRuntimeBufferAtomicHalves(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t active, const std::function& operation); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp index 421afcba78..900837b6e8 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp @@ -334,6 +334,17 @@ std::unordered_map FunctionLdsLaneAddresses(const return addresses; } +bool IsSplittableBufferAtomic64(const IrValue* inst) { + switch (inst->Opcode()) { + case IrOpcode::BufferAtomicOr64: + case IrOpcode::BufferAtomicAnd64: + case IrOpcode::BufferAtomicXor64: + return !inst->HasUses(); + default: + return false; + } +} + SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) { SpirvRequirements requirements {}; for (const IrBlock* block : program.BlockOrder()) { @@ -341,6 +352,7 @@ SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) { const auto addressAccess = AddressOpcodeInfoOf(inst->Opcode()).access; if ((BufferAccessOf(inst->Opcode()) == BufferAccess::Atomic || addressAccess == AddressAccess::Atomic) && inst->Type() == IrType::U64) { requirements.bufferInt64Atomics = true; + if (!IsSplittableBufferAtomic64(inst)) requirements.bufferInt64AtomicsExact = true; } if (IsImageAtomic64Opcode(inst->Opcode())) { requirements.imageInt64Atomics = true; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 230ddd8bbd..b6ca03fe8a 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -294,6 +294,10 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.requirements.ldsBufferInt64Atomics = true; } const bool int64Atomics = std::ranges::find(state.supportedCapabilities, static_cast(spv::CapabilityInt64Atomics)) != state.supportedCapabilities.end(); + if (!int64Atomics && state.requirements.bufferInt64Atomics && !state.requirements.bufferInt64AtomicsExact) { + state.requirements.bufferInt64Atomics = false; + state.requirements.bufferInt64AtomicsSplit = true; + } state.requirements.bufferU64View = state.requirements.bufferInt64Atomics || (state.requirements.coherentBufferPairs && int64Atomics); if (HostSubgroupSingleLane(state)) { state.requirements.subgroupBallot = false; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index cfe05d771d..2678e89b15 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -940,6 +940,18 @@ std::uint32_t Atomic32(SpirvValueEmitContext& ctx, const IrValue& inst, const Me std::uint32_t BufferAtomic64(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; static_cast(BufferMemory(ctx, inst)); + if (state.requirements.bufferInt64AtomicsSplit) { + if (!IsSplittableBufferAtomic64(&inst)) ctx.Fail(inst, "64-bit buffer atomic needs shaderBufferInt64Atomics"); + const auto data = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 2u)); + const auto low = Unary(state, spv::OpUConvert, TypeU32(state), data); + const auto high = Unary(state, spv::OpUConvert, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), data, BdaConstant(state, 32u))); + std::uint32_t index = 0u; + EmitRuntimeBufferAtomicHalves(ctx, inst, ActiveArgument(ctx, inst), [&](std::uint32_t pointer) { + state.module.AddFunction(SpirvAtomicOpcode(inst.Opcode()), TypeU32(state), state.module.AllocateId(), pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), index++ == 0u ? low : high); + }); + EmitDeviceAtomicMemoryBarrier(state); + return ConstantU64(state, 0u); + } return EmitRuntimeBufferAtomic(ctx, inst, 64u, ActiveArgument(ctx, inst), [&](std::uint32_t pointer) { const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 2u)); const auto old = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index 1f8522303e..07483999d3 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -1021,7 +1021,7 @@ std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const return EmitValueOrZeroIfCondition(state, inBounds, [&] { return readBuffer(context, instruction, AddBdaAddress(context, instruction, resource.base, offset, false), 32u); }); } -std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function& operation) { +static std::uint32_t RuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, bool split, std::uint32_t active, const std::function& operation) { auto& state = context.state; if (bits != 32u && bits != 64u) context.Fail(instruction, "unsupported runtime buffer atomic width"); if (context.Memory(instruction).gpuDescriptor && (state.bdaAtomicPointerFunction == 0u || state.bdaNoteWriteFunction == 0u)) context.Fail(instruction, "BDA atomic functions are missing"); @@ -1030,18 +1030,20 @@ std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrVa if (std::ranges::find(state.supportedCapabilities, spv::CapabilityInt64) == state.supportedCapabilities.end()) context.Fail(instruction, "buffer atomic fault addresses require shaderInt64"); state.module.EmitCapability(spv::CapabilityInt64); } - const auto type = bits == 64u ? TypeU64(state) : TypeU32(state); - const auto scalar = bits == 64u ? TypeScalarU64(state) : TypeU32(state); - const auto zero = bits == 64u ? ConstantU64(state, 0u) : ConstantU32(state, 0u); + const bool wide = bits == 64u && !split; + const auto type = wide ? TypeU64(state) : TypeU32(state); + const auto scalar = wide ? TypeScalarU64(state) : TypeU32(state); + const auto zero = wide ? ConstantU64(state, 0u) : ConstantU32(state, 0u); return EmitValueOrDefaultIfCondition(state, active, type, zero, [&] { const auto resource = descriptor(context, instruction); const auto access = address(context, instruction, resource, 0u, bits / 8u); return EmitValueOrDefaultIfCondition(state, access.inBounds, type, zero, [&] { if (!context.Memory(instruction).gpuDescriptor) { - const auto storage = directAccess(context, instruction, bits == 64u); + const auto storage = directAccess(context, instruction, wide); const auto byte = EmitAddU32(state, access.guest, storage.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte, ConstantU32(state, bits == 64u ? 3u : 2u)); - auto valid = EmitMemoryElementInBounds(state, storage, index); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte, ConstantU32(state, wide ? 3u : 2u)); + const auto highIndex = split ? EmitAddU32(state, index, ConstantU32(state, 1u)) : index; + auto valid = EmitMemoryElementInBounds(state, storage, highIndex); if (bits == 64u) { const auto unaligned = nonzero(state, EmitAndConstant(state, byte, bits / 8u - 1u)); const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), resource.base, Unary(state, spv::OpUConvert, TypeScalarU64(state), access.guest)); @@ -1053,8 +1055,10 @@ std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrVa valid = both(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), valid); } return EmitValueOrDefaultIfCondition(state, valid, type, zero, [&] { - const auto pointer = EmitStorageBufferElementPointer(state, storage, index, bits == 64u ? TypeStorageBufferU64ElementPointer(state) : TypeStorageBufferElementPointer(state)); - return operation(pointer); + const auto pointer = EmitStorageBufferElementPointer(state, storage, index, wide ? TypeStorageBufferU64ElementPointer(state) : TypeStorageBufferElementPointer(state)); + auto result = operation(pointer, 0u); + if (split) result = operation(EmitStorageBufferElementPointer(state, storage, highIndex, TypeStorageBufferElementPointer(state)), 1u); + return result; }); } const auto atomicAt = [&](std::uint32_t guest) { @@ -1064,7 +1068,14 @@ std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrVa return EmitValueOrDefaultIfCondition(state, mapped, type, zero, [&] { const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, scalar), pointer, physical); - const auto result = operation(pointer); + auto result = operation(pointer, 0u); + if (split) { + const auto high = state.module.AllocateId(); + state.module.AddFunction(spv::OpIAdd, TypeScalarU64(state), high, physical, BdaConstant(state, 4u)); + const auto highPointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, scalar), highPointer, high); + result = operation(highPointer, 1u); + } state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, guest); return result; }); @@ -1077,4 +1088,16 @@ std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrVa }); } +std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function& operation) { + return RuntimeBufferAtomic(context, instruction, bits, false, active, [&](std::uint32_t pointer, std::uint32_t) { return operation(pointer); }); +} + +void EmitRuntimeBufferAtomicHalves(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t active, const std::function& operation) { + static_cast(RuntimeBufferAtomic(context, instruction, 64u, true, active, [&](std::uint32_t pointer, std::uint32_t) { + operation(pointer); + return ConstantU32(context.state, 0u); + })); +} + + } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index e2e238b514..fe8203daec 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -174,6 +174,7 @@ - [Shared sampling lowering](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) rounds array layers half to even and retains unordered NOT_EQUAL arithmetic, following adriadam10's RDNA2 gfx1035 measurements in [#907](https://github.com/boykopovar/AnyPS5/pull/907#pullrequestreview-5439697732), not measurements on a PS5. Constant-zero color comparison tests cannot verify layer selection or nonzero NaN comparisons. Converted unorm sampling shares the layer selection and tests it with distinct values in four layers - [Negative color-texture comparison wrap indices](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (libSceAgcDriver) use explicit `OpSRem` normalization to implement modulo; the negative-offset point-wrap regression fails with direct `OpSMod` and passes with normalization on Mesa 22.3.6 llvmpipe (LLVM 15.0.6), but negative-index wrap has not been measured on PS5 hardware. - [Converted unorm images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view) are sampled in the shader, the way color comparisons are emulated: `image_sample` of a 2D or 2D array view fetches the one or four texels at LOD 0, decodes them and blends them, for point or bilinear filtering with equal minification and magnification filters and wrap or clamp-to-edge addressing. A shader gets these sample modes only once a converted descriptor reaches one of its sampled images: that capture throws `ConvertedSampleRequired`, the driver records the program and prepares it again with `GuestContext::convertedSamples`, so such a shader is prepared twice per run and its first capture is repeated. The record lives with the registered program and is not kept across runs. Bilinear weights are rounded to the hardware's 8-bit grid (`roundEven(fraction * 256) / 256`, ties to even), as measured with `image_sample_lz` on RDNA2 (gfx1035) for 10_11_11 UNORM texels by adriadam10 in #5958; point sampling, decoding, swizzles and wrap and clamp-to-edge addressing matched the same measurement. A sample that is not `_lz` needs an image with one mip level; gathers, LOD queries, clamp-to-border and mirrored addressing, other filters, min/max reduction, unnormalized coordinates, images selected at run time, 16-bit data and a DST_SEL of the missing W channel throw. +- [64-bit buffer atomics without shaderBufferInt64Atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp) (`buffer_atomic_or_x2`, `buffer_atomic_and_x2`, `buffer_atomic_xor_x2`) whose result no instruction reads compile to one 32-bit atomic on each dword when the device has no `shaderBufferInt64Atomics`, for example on MoltenVK. Each dword is atomic and the pair is not, so a plain load that races the atomic can see one half applied. The address checks are those of the 64-bit form: 8-byte alignment and the bounds of both dwords. A 64-bit buffer atomic of any other opcode, or one whose result is read, still fails to compile on such a device. The result of the split form was not measured against a native 64-bit atomic on hardware; the compile cases are in `agc_shader_memory_tests`. - [64-bit LDS atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. On RDNA2 (gfx1036) the LDS serialises a 32-bit atomic on a slot's high dword and a 64-bit atomic on that slot: 128 lanes of `ds_add_u32` and 128 lanes of `ds_add_u64` on one slot sum exactly. Only that pair was measured. Every 32-bit LDS atomic of a program that contains a 64-bit LDS atomic therefore takes the same lock, including a 32-bit `flat_atomic_*` whose address lands in the LDS aperture (a 64-bit one takes it already); the flat forms were not measured against a 64-bit atomic either (the other 32-bit LDS atomics, `ds_min/max`, `ds_cmpst`, the float forms, `ds_inc/dec`, `ds_mskor` and `ds_wrxchg`, are extrapolated from `ds_add_u32` and were not measured against a 64-bit atomic); a program without a 64-bit LDS atomic keeps its native 32-bit atomics. `ds_condxchg32_rtn_b64` is two `ds_mskor` halves, each under the lock but not atomic as a pair, so a 64-bit atomic between the two halves is not excluded. Sub-dword writes (`ds_write_b8`, `ds_write_b16` and their `_d16_hi` forms) and plain `ds_read_*`, `ds_write_*`, `ds_append_u32` and `ds_consume_u32` are not taken under the lock, so a plain access racing a 64-bit atomic on the same dwords can see or overwrite one half of it; whether the hardware orders them was not measured. The lock serialises the workgroup's LDS atomics of such a program; the throughput cost was not measured. `VK_KHR_workgroup_memory_explicit_layout` with `shaderSharedInt64Atomics` would make both widths native and is not used. A 64-bit atomic at an address that is 4-byte but not 8-byte aligned faults the wave on gfx1036, as on gfx1035; with the workgroup lock, in device memory and in the invocation-private LDS of other stages alike, the recompiler records the `Unaligned` shader fault with the LDS byte address, leaves both dwords unchanged and returns 0, but the lane and the rest of its wave keep running. When the LDS lives in device memory (compute LDS beyond the device's shared-memory limit) and the device has `shaderBufferInt64Atomics`, no lock is used: a 64-bit atomic is a compare-exchange loop on a 64-bit view of the storage buffer and a 32-bit atomic is a native buffer atomic, so a mixed-width pair on one slot relies on the device ordering both widths on one buffer address; only `ds_add_u32` against `ds_add_u64` was measured, on gfx1036 (AMD Windows driver 32.0.21042.62) and an RTX 4070 SUPER. The lock is not used there because AMD's Windows driver compiles an acquire or release on a storage buffer, at workgroup scope or wider, to `s_barrier`, and those barriers inside the lock's divergent loop pair with the program's own `s_barrier`s. The fence after each LDS atomic compiles to `s_barrier` on that driver as well, so a wave that skips an LDS atomic that other waves of its workgroup execute misses a barrier they reach; its effect was not measured. - [LDS lock ordering on MoltenVK](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - SPIRV-Cross emits the 64-bit LDS lock's acquire compare-exchange and release exchange as `memory_order_relaxed` Metal atomics, which do not order the plain LDS loads and stores inside the lock; contended `ds_add_u64` lost updates on an Apple M5 Pro. On MoltenVK the driver sets `SpirvTarget::ldsLockFences`, and the lock brackets its critical section with a workgroup-scope acquire `OpMemoryBarrier` and a release one, which SPIRV-Cross emits as `atomic_thread_fence(mem_flags::mem_threadgroup, ...)` with no execution barrier. Other drivers do not get these fences, since AMD's Windows driver compiles them to `s_barrier`. Other Apple GPUs were not measured. - [Image, buffer and global float atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`image_atomic_fmin/fmax/fcmpswap`, the same three `buffer_atomic_*` and `global_atomic_*` forms and their `_x2` forms) follow the rule of the LDS float atomics below, including its operand flush: the 32-bit forms flush both operands to a signed zero when bit 0 of the f32 denormal field is clear, the `_x2` forms by the f16/f64 field, and `fcmpswap` stores a denormal source as given. The `flat_atomic_*` forms take the same path and are not measured. Measured on RDNA2 (gfx1031, RX 6700 XT) with the [hardware oracle](../HW_ORACLE.md) over 20 x 20 f32 and 18 x 18 f64 operand pairs in every value of both fields, at IEEE 0 and 1, in wave32 and wave64, and for the image forms on 32_UINT, 32_SINT and 32_FLOAT surfaces. The console's gfx1013 is not measured. From e0a794c051150a8bf88986b38d34a07719852b9b Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Mon, 26 Oct 2026 21:58:06 +0100 Subject: [PATCH 251/318] test(shader): assert the two 32-bit halves of a split 64-bit buffer atomic Co-Authored-By: Claude Sonnet 5.5 --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index d32c0af096..9b9403c0c2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -7572,12 +7572,49 @@ void verifyInt64AtomicCapabilities() { } return false; }; + const auto verifySplit = [](std::span words, spv::Op atomic) { + std::map definitions; + std::map constants; + std::uint32_t uintType = 0; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + require(count != 0 && count <= words.size() - cursor, "64-bit atomics: truncated SPIR-V instruction"); + const auto op = words[cursor] & 0xffffu; + if (op == spv::OpTypeInt && words[cursor + 2] == 32u && words[cursor + 3] == 0u) uintType = words[cursor + 1]; + if (op == spv::OpConstant && count == 4u) constants[words[cursor + 2]] = words[cursor + 3]; + if (op == spv::OpAccessChain || op == spv::OpIAdd || op == spv::OpUConvert || op == spv::OpVariable) definitions[words[cursor + 2]] = cursor; + cursor += count; + } + const auto opcodeOf = [&](std::uint32_t id) { return definitions.contains(id) ? words[definitions.at(id)] & 0xffffu : 0xffffffffu; }; + std::vector halves; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + if ((words[cursor] & 0xffffu) == static_cast(atomic) && opcodeOf(words[cursor + 3]) == spv::OpAccessChain) { + const auto chain = definitions.at(words[cursor + 3]); + if ((words[chain] >> 16u) == 6u && opcodeOf(words[chain + 3]) == spv::OpAccessChain) halves.push_back(cursor); + } + cursor += count; + } + require(halves.size() == 2u, "64-bit atomics: an unused-result bitwise buffer atomic was not emitted as two 32-bit atomics"); + const auto low = halves[0]; + const auto high = halves[1]; + require(words[low + 1] == uintType && words[high + 1] == uintType, "64-bit atomics: a half of a split buffer atomic is not 32 bits wide"); + const auto lowChain = definitions.at(words[low + 3]); + const auto highChain = definitions.at(words[high + 3]); + require(words[lowChain + 3] == words[highChain + 3], "64-bit atomics: the halves of a split buffer atomic use different buffers"); + const auto highIndex = words[highChain + 5]; + require(opcodeOf(highIndex) == spv::OpIAdd, "64-bit atomics: the high half of a split buffer atomic does not address the next dword"); + const auto add = definitions.at(highIndex); + require(words[add + 3] == words[lowChain + 5] && constants.contains(words[add + 4]) && constants.at(words[add + 4]) == 1u, "64-bit atomics: the high half of a split buffer atomic is not one dword after the low half"); + require(words[low + 6] != words[high + 6] && opcodeOf(words[low + 6]) == spv::OpUConvert && opcodeOf(words[high + 6]) == spv::OpUConvert, "64-bit atomics: the halves of a split buffer atomic do not take the low and high dwords of the operand"); + }; expectFailure([&] { static_cast(compile(bufferAtomic, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_inc_x2 compiled without shaderBufferInt64Atomics"); require(declares(compile(bufferAtomic, spv::CapabilityInt64Atomics), spv::CapabilityInt64Atomics), "64-bit atomics: a buffer_atomic_inc_x2 does not declare Int64Atomics"); - for (const auto* code : {&bufferOr, &bufferAnd, &bufferXor}) { + const std::array*, spv::Op>, 3> bitwise{{{&bufferOr, spv::OpAtomicOr}, {&bufferAnd, spv::OpAtomicAnd}, {&bufferXor, spv::OpAtomicXor}}}; + for (const auto& [code, atomic] : bitwise) { const auto split = compile(*code, spv::CapabilityShader); require(!declares(split, spv::CapabilityInt64Atomics), "64-bit atomics: an unused-result bitwise buffer atomic declares Int64Atomics on a device without it"); - require(!split.empty(), "64-bit atomics: an unused-result bitwise buffer atomic was not compiled as two 32-bit atomics"); + verifySplit(split, atomic); require(declares(compile(*code, spv::CapabilityInt64Atomics), spv::CapabilityInt64Atomics), "64-bit atomics: an unused-result bitwise buffer atomic does not use the 64-bit atomic where the device has it"); } expectFailure([&] { static_cast(compile(bufferOrStored, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_or_x2 whose result is stored compiled without shaderBufferInt64Atomics"); From 062c463898d24618735f5208ebf9eeb2fd60d186 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 27 Oct 2026 23:21:41 +0100 Subject: [PATCH 252/318] fix(shader): split 64-bit bitwise buffer atomics only when a program uses one opcode Two different 64-bit bitwise operations on one word, split into two 32-bit atomics, can end in a value that no serial order of 64-bit operations gives. A program with more than one split opcode now fails to compile without shaderBufferInt64Atomics, like any other 64-bit buffer atomic. Programs with one opcode are unchanged. Drop the two expectFailure cases for forms that are unsupported anyway. Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp | 9 +++++---- .../shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp | 9 ++++++++- docs/dev/debt/shader.md | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 9b9403c0c2..48cd3f24a2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -7542,8 +7542,8 @@ void verifyInt64AtomicCapabilities() { const std::vector bufferOr{0xe1685000u, 0x80012803u, 0xbf810000u}; const std::vector bufferAnd{0xe1645000u, 0x80012803u, 0xbf810000u}; const std::vector bufferXor{0xe16c5000u, 0x80012803u, 0xbf810000u}; - const std::vector bufferOrStored{0xe1685000u, 0x80012803u, 0xe0741000u, 0x80012803u, 0xbf810000u}; - const std::vector bufferAdd{0xe1485000u, 0x80012803u, 0xbf810000u}; + const std::vector bufferOrTwice{0xe1685000u, 0x80012803u, 0xe1685000u, 0x80012a03u, 0xbf810000u}; + const std::vector bufferOrAnd{0xe1685000u, 0x80012803u, 0xe1645000u, 0x80012a03u, 0xbf810000u}; const auto compile = [&](std::span code, spv::Capability added) { const std::array capabilities{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, static_cast(added)}; RecompileRequest request{}; @@ -7617,8 +7617,9 @@ void verifyInt64AtomicCapabilities() { verifySplit(split, atomic); require(declares(compile(*code, spv::CapabilityInt64Atomics), spv::CapabilityInt64Atomics), "64-bit atomics: an unused-result bitwise buffer atomic does not use the 64-bit atomic where the device has it"); } - expectFailure([&] { static_cast(compile(bufferOrStored, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_or_x2 whose result is stored compiled without shaderBufferInt64Atomics"); - expectFailure([&] { static_cast(compile(bufferAdd, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_add_x2 compiled without shaderBufferInt64Atomics"); + require(!declares(compile(bufferOrTwice, spv::CapabilityShader), spv::CapabilityInt64Atomics), "64-bit atomics: two buffer_atomic_or_x2 of one program declare Int64Atomics on a device without it"); + expectFailure([&] { static_cast(compile(bufferOrAnd, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_or_x2 and a buffer_atomic_and_x2 of one program compiled without shaderBufferInt64Atomics"); + require(declares(compile(bufferOrAnd, spv::CapabilityInt64Atomics), spv::CapabilityInt64Atomics), "64-bit atomics: a buffer_atomic_or_x2 and a buffer_atomic_and_x2 of one program do not use the 64-bit atomics where the device has them"); expectFailure([&] { static_cast(compile(imageAtomic, spv::CapabilityShader)); }, "64-bit image atomics need VK_EXT_shader_image_atomic_int64", "64-bit atomics: an image_atomic_swap on a 32_32 image compiled without shaderImageInt64Atomics"); require(declares(compile(imageAtomic, spv::CapabilityInt64ImageEXT), spv::CapabilityInt64ImageEXT), "64-bit atomics: an image_atomic_swap on a 32_32 image does not declare Int64ImageEXT"); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp index 900837b6e8..e11158c5da 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp @@ -347,12 +347,19 @@ bool IsSplittableBufferAtomic64(const IrValue* inst) { SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) { SpirvRequirements requirements {}; + std::optional splitOpcode; for (const IrBlock* block : program.BlockOrder()) { for (const IrValue* inst : block->Instructions()) { const auto addressAccess = AddressOpcodeInfoOf(inst->Opcode()).access; if ((BufferAccessOf(inst->Opcode()) == BufferAccess::Atomic || addressAccess == AddressAccess::Atomic) && inst->Type() == IrType::U64) { requirements.bufferInt64Atomics = true; - if (!IsSplittableBufferAtomic64(inst)) requirements.bufferInt64AtomicsExact = true; + if (!IsSplittableBufferAtomic64(inst)) { + requirements.bufferInt64AtomicsExact = true; + } else if (!splitOpcode) { + splitOpcode = inst->Opcode(); + } else if (*splitOpcode != inst->Opcode()) { + requirements.bufferInt64AtomicsExact = true; + } } if (IsImageAtomic64Opcode(inst->Opcode())) { requirements.imageInt64Atomics = true; diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index fe8203daec..e174cca972 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -174,7 +174,7 @@ - [Shared sampling lowering](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) rounds array layers half to even and retains unordered NOT_EQUAL arithmetic, following adriadam10's RDNA2 gfx1035 measurements in [#907](https://github.com/boykopovar/AnyPS5/pull/907#pullrequestreview-5439697732), not measurements on a PS5. Constant-zero color comparison tests cannot verify layer selection or nonzero NaN comparisons. Converted unorm sampling shares the layer selection and tests it with distinct values in four layers - [Negative color-texture comparison wrap indices](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (libSceAgcDriver) use explicit `OpSRem` normalization to implement modulo; the negative-offset point-wrap regression fails with direct `OpSMod` and passes with normalization on Mesa 22.3.6 llvmpipe (LLVM 15.0.6), but negative-index wrap has not been measured on PS5 hardware. - [Converted unorm images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view) are sampled in the shader, the way color comparisons are emulated: `image_sample` of a 2D or 2D array view fetches the one or four texels at LOD 0, decodes them and blends them, for point or bilinear filtering with equal minification and magnification filters and wrap or clamp-to-edge addressing. A shader gets these sample modes only once a converted descriptor reaches one of its sampled images: that capture throws `ConvertedSampleRequired`, the driver records the program and prepares it again with `GuestContext::convertedSamples`, so such a shader is prepared twice per run and its first capture is repeated. The record lives with the registered program and is not kept across runs. Bilinear weights are rounded to the hardware's 8-bit grid (`roundEven(fraction * 256) / 256`, ties to even), as measured with `image_sample_lz` on RDNA2 (gfx1035) for 10_11_11 UNORM texels by adriadam10 in #5958; point sampling, decoding, swizzles and wrap and clamp-to-edge addressing matched the same measurement. A sample that is not `_lz` needs an image with one mip level; gathers, LOD queries, clamp-to-border and mirrored addressing, other filters, min/max reduction, unnormalized coordinates, images selected at run time, 16-bit data and a DST_SEL of the missing W channel throw. -- [64-bit buffer atomics without shaderBufferInt64Atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp) (`buffer_atomic_or_x2`, `buffer_atomic_and_x2`, `buffer_atomic_xor_x2`) whose result no instruction reads compile to one 32-bit atomic on each dword when the device has no `shaderBufferInt64Atomics`, for example on MoltenVK. Each dword is atomic and the pair is not, so a plain load that races the atomic can see one half applied. The address checks are those of the 64-bit form: 8-byte alignment and the bounds of both dwords. A 64-bit buffer atomic of any other opcode, or one whose result is read, still fails to compile on such a device. The result of the split form was not measured against a native 64-bit atomic on hardware; the compile cases are in `agc_shader_memory_tests`. +- [64-bit buffer atomics without shaderBufferInt64Atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp) (`buffer_atomic_or_x2`, `buffer_atomic_and_x2`, `buffer_atomic_xor_x2`) whose result no instruction reads compile to one 32-bit atomic on each dword when the device has no `shaderBufferInt64Atomics`, for example on MoltenVK, and only when every 64-bit atomic of the program is of that kind and has the same opcode. Each dword is atomic and the pair is not, so a plain load that races the atomic can see one half applied, and two different 64-bit bitwise operations on one word can end in a value that no serial order gives. The check covers one program: two programs that run together and apply different 64-bit bitwise operations to the same word are not covered. The address checks are those of the 64-bit form: 8-byte alignment and the bounds of both dwords. A 64-bit buffer atomic of any other opcode, one whose result is read, or a program with two different 64-bit bitwise opcodes, still fails to compile on such a device. The result of the split form was not measured against a native 64-bit atomic on hardware; the compile cases are in `agc_shader_memory_tests`. - [64-bit LDS atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. On RDNA2 (gfx1036) the LDS serialises a 32-bit atomic on a slot's high dword and a 64-bit atomic on that slot: 128 lanes of `ds_add_u32` and 128 lanes of `ds_add_u64` on one slot sum exactly. Only that pair was measured. Every 32-bit LDS atomic of a program that contains a 64-bit LDS atomic therefore takes the same lock, including a 32-bit `flat_atomic_*` whose address lands in the LDS aperture (a 64-bit one takes it already); the flat forms were not measured against a 64-bit atomic either (the other 32-bit LDS atomics, `ds_min/max`, `ds_cmpst`, the float forms, `ds_inc/dec`, `ds_mskor` and `ds_wrxchg`, are extrapolated from `ds_add_u32` and were not measured against a 64-bit atomic); a program without a 64-bit LDS atomic keeps its native 32-bit atomics. `ds_condxchg32_rtn_b64` is two `ds_mskor` halves, each under the lock but not atomic as a pair, so a 64-bit atomic between the two halves is not excluded. Sub-dword writes (`ds_write_b8`, `ds_write_b16` and their `_d16_hi` forms) and plain `ds_read_*`, `ds_write_*`, `ds_append_u32` and `ds_consume_u32` are not taken under the lock, so a plain access racing a 64-bit atomic on the same dwords can see or overwrite one half of it; whether the hardware orders them was not measured. The lock serialises the workgroup's LDS atomics of such a program; the throughput cost was not measured. `VK_KHR_workgroup_memory_explicit_layout` with `shaderSharedInt64Atomics` would make both widths native and is not used. A 64-bit atomic at an address that is 4-byte but not 8-byte aligned faults the wave on gfx1036, as on gfx1035; with the workgroup lock, in device memory and in the invocation-private LDS of other stages alike, the recompiler records the `Unaligned` shader fault with the LDS byte address, leaves both dwords unchanged and returns 0, but the lane and the rest of its wave keep running. When the LDS lives in device memory (compute LDS beyond the device's shared-memory limit) and the device has `shaderBufferInt64Atomics`, no lock is used: a 64-bit atomic is a compare-exchange loop on a 64-bit view of the storage buffer and a 32-bit atomic is a native buffer atomic, so a mixed-width pair on one slot relies on the device ordering both widths on one buffer address; only `ds_add_u32` against `ds_add_u64` was measured, on gfx1036 (AMD Windows driver 32.0.21042.62) and an RTX 4070 SUPER. The lock is not used there because AMD's Windows driver compiles an acquire or release on a storage buffer, at workgroup scope or wider, to `s_barrier`, and those barriers inside the lock's divergent loop pair with the program's own `s_barrier`s. The fence after each LDS atomic compiles to `s_barrier` on that driver as well, so a wave that skips an LDS atomic that other waves of its workgroup execute misses a barrier they reach; its effect was not measured. - [LDS lock ordering on MoltenVK](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - SPIRV-Cross emits the 64-bit LDS lock's acquire compare-exchange and release exchange as `memory_order_relaxed` Metal atomics, which do not order the plain LDS loads and stores inside the lock; contended `ds_add_u64` lost updates on an Apple M5 Pro. On MoltenVK the driver sets `SpirvTarget::ldsLockFences`, and the lock brackets its critical section with a workgroup-scope acquire `OpMemoryBarrier` and a release one, which SPIRV-Cross emits as `atomic_thread_fence(mem_flags::mem_threadgroup, ...)` with no execution barrier. Other drivers do not get these fences, since AMD's Windows driver compiles them to `s_barrier`. Other Apple GPUs were not measured. - [Image, buffer and global float atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`image_atomic_fmin/fmax/fcmpswap`, the same three `buffer_atomic_*` and `global_atomic_*` forms and their `_x2` forms) follow the rule of the LDS float atomics below, including its operand flush: the 32-bit forms flush both operands to a signed zero when bit 0 of the f32 denormal field is clear, the `_x2` forms by the f16/f64 field, and `fcmpswap` stores a denormal source as given. The `flat_atomic_*` forms take the same path and are not measured. Measured on RDNA2 (gfx1031, RX 6700 XT) with the [hardware oracle](../HW_ORACLE.md) over 20 x 20 f32 and 18 x 18 f64 operand pairs in every value of both fields, at IEEE 0 and 1, in wave32 and wave64, and for the image forms on 32_UINT, 32_SINT and 32_FLOAT surfaces. The console's gfx1013 is not measured. From 3bb754e60f0fc538233aa6eafe50d3c9eebd3f88 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 27 Oct 2026 23:24:31 +0100 Subject: [PATCH 253/318] docs(shader): shorten the 64-bit buffer atomic split entry The entry was over the technical debt length limit. Co-Authored-By: Claude Sonnet 5.5 --- docs/dev/debt/shader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index e174cca972..a505f52bc2 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -174,7 +174,7 @@ - [Shared sampling lowering](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) rounds array layers half to even and retains unordered NOT_EQUAL arithmetic, following adriadam10's RDNA2 gfx1035 measurements in [#907](https://github.com/boykopovar/AnyPS5/pull/907#pullrequestreview-5439697732), not measurements on a PS5. Constant-zero color comparison tests cannot verify layer selection or nonzero NaN comparisons. Converted unorm sampling shares the layer selection and tests it with distinct values in four layers - [Negative color-texture comparison wrap indices](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (libSceAgcDriver) use explicit `OpSRem` normalization to implement modulo; the negative-offset point-wrap regression fails with direct `OpSMod` and passes with normalization on Mesa 22.3.6 llvmpipe (LLVM 15.0.6), but negative-index wrap has not been measured on PS5 hardware. - [Converted unorm images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view) are sampled in the shader, the way color comparisons are emulated: `image_sample` of a 2D or 2D array view fetches the one or four texels at LOD 0, decodes them and blends them, for point or bilinear filtering with equal minification and magnification filters and wrap or clamp-to-edge addressing. A shader gets these sample modes only once a converted descriptor reaches one of its sampled images: that capture throws `ConvertedSampleRequired`, the driver records the program and prepares it again with `GuestContext::convertedSamples`, so such a shader is prepared twice per run and its first capture is repeated. The record lives with the registered program and is not kept across runs. Bilinear weights are rounded to the hardware's 8-bit grid (`roundEven(fraction * 256) / 256`, ties to even), as measured with `image_sample_lz` on RDNA2 (gfx1035) for 10_11_11 UNORM texels by adriadam10 in #5958; point sampling, decoding, swizzles and wrap and clamp-to-edge addressing matched the same measurement. A sample that is not `_lz` needs an image with one mip level; gathers, LOD queries, clamp-to-border and mirrored addressing, other filters, min/max reduction, unnormalized coordinates, images selected at run time, 16-bit data and a DST_SEL of the missing W channel throw. -- [64-bit buffer atomics without shaderBufferInt64Atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp) (`buffer_atomic_or_x2`, `buffer_atomic_and_x2`, `buffer_atomic_xor_x2`) whose result no instruction reads compile to one 32-bit atomic on each dword when the device has no `shaderBufferInt64Atomics`, for example on MoltenVK, and only when every 64-bit atomic of the program is of that kind and has the same opcode. Each dword is atomic and the pair is not, so a plain load that races the atomic can see one half applied, and two different 64-bit bitwise operations on one word can end in a value that no serial order gives. The check covers one program: two programs that run together and apply different 64-bit bitwise operations to the same word are not covered. The address checks are those of the 64-bit form: 8-byte alignment and the bounds of both dwords. A 64-bit buffer atomic of any other opcode, one whose result is read, or a program with two different 64-bit bitwise opcodes, still fails to compile on such a device. The result of the split form was not measured against a native 64-bit atomic on hardware; the compile cases are in `agc_shader_memory_tests`. +- [64-bit buffer atomics without shaderBufferInt64Atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp) (`buffer_atomic_or_x2`, `buffer_atomic_and_x2`, `buffer_atomic_xor_x2`) whose result no instruction reads compile to one 32-bit atomic on each dword when the device has no `shaderBufferInt64Atomics`, for example on MoltenVK, if every 64-bit atomic of the program has that form and the same opcode. The pair is not atomic: a plain load that races it can see one half applied, and two different 64-bit bitwise operations on one word can end in a value that no serial order gives. Two programs that run together and apply different operations to one word are not covered. The address checks are those of the 64-bit form. Any other 64-bit buffer atomic, one whose result is read, or a program with two opcodes fails to compile on such a device. Not measured against a native 64-bit atomic on hardware; the compile cases are in `agc_shader_memory_tests`. - [64-bit LDS atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. On RDNA2 (gfx1036) the LDS serialises a 32-bit atomic on a slot's high dword and a 64-bit atomic on that slot: 128 lanes of `ds_add_u32` and 128 lanes of `ds_add_u64` on one slot sum exactly. Only that pair was measured. Every 32-bit LDS atomic of a program that contains a 64-bit LDS atomic therefore takes the same lock, including a 32-bit `flat_atomic_*` whose address lands in the LDS aperture (a 64-bit one takes it already); the flat forms were not measured against a 64-bit atomic either (the other 32-bit LDS atomics, `ds_min/max`, `ds_cmpst`, the float forms, `ds_inc/dec`, `ds_mskor` and `ds_wrxchg`, are extrapolated from `ds_add_u32` and were not measured against a 64-bit atomic); a program without a 64-bit LDS atomic keeps its native 32-bit atomics. `ds_condxchg32_rtn_b64` is two `ds_mskor` halves, each under the lock but not atomic as a pair, so a 64-bit atomic between the two halves is not excluded. Sub-dword writes (`ds_write_b8`, `ds_write_b16` and their `_d16_hi` forms) and plain `ds_read_*`, `ds_write_*`, `ds_append_u32` and `ds_consume_u32` are not taken under the lock, so a plain access racing a 64-bit atomic on the same dwords can see or overwrite one half of it; whether the hardware orders them was not measured. The lock serialises the workgroup's LDS atomics of such a program; the throughput cost was not measured. `VK_KHR_workgroup_memory_explicit_layout` with `shaderSharedInt64Atomics` would make both widths native and is not used. A 64-bit atomic at an address that is 4-byte but not 8-byte aligned faults the wave on gfx1036, as on gfx1035; with the workgroup lock, in device memory and in the invocation-private LDS of other stages alike, the recompiler records the `Unaligned` shader fault with the LDS byte address, leaves both dwords unchanged and returns 0, but the lane and the rest of its wave keep running. When the LDS lives in device memory (compute LDS beyond the device's shared-memory limit) and the device has `shaderBufferInt64Atomics`, no lock is used: a 64-bit atomic is a compare-exchange loop on a 64-bit view of the storage buffer and a 32-bit atomic is a native buffer atomic, so a mixed-width pair on one slot relies on the device ordering both widths on one buffer address; only `ds_add_u32` against `ds_add_u64` was measured, on gfx1036 (AMD Windows driver 32.0.21042.62) and an RTX 4070 SUPER. The lock is not used there because AMD's Windows driver compiles an acquire or release on a storage buffer, at workgroup scope or wider, to `s_barrier`, and those barriers inside the lock's divergent loop pair with the program's own `s_barrier`s. The fence after each LDS atomic compiles to `s_barrier` on that driver as well, so a wave that skips an LDS atomic that other waves of its workgroup execute misses a barrier they reach; its effect was not measured. - [LDS lock ordering on MoltenVK](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - SPIRV-Cross emits the 64-bit LDS lock's acquire compare-exchange and release exchange as `memory_order_relaxed` Metal atomics, which do not order the plain LDS loads and stores inside the lock; contended `ds_add_u64` lost updates on an Apple M5 Pro. On MoltenVK the driver sets `SpirvTarget::ldsLockFences`, and the lock brackets its critical section with a workgroup-scope acquire `OpMemoryBarrier` and a release one, which SPIRV-Cross emits as `atomic_thread_fence(mem_flags::mem_threadgroup, ...)` with no execution barrier. Other drivers do not get these fences, since AMD's Windows driver compiles them to `s_barrier`. Other Apple GPUs were not measured. - [Image, buffer and global float atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`image_atomic_fmin/fmax/fcmpswap`, the same three `buffer_atomic_*` and `global_atomic_*` forms and their `_x2` forms) follow the rule of the LDS float atomics below, including its operand flush: the 32-bit forms flush both operands to a signed zero when bit 0 of the f32 denormal field is clear, the `_x2` forms by the f16/f64 field, and `fcmpswap` stores a denormal source as given. The `flat_atomic_*` forms take the same path and are not measured. Measured on RDNA2 (gfx1031, RX 6700 XT) with the [hardware oracle](../HW_ORACLE.md) over 20 x 20 f32 and 18 x 18 f64 operand pairs in every value of both fields, at IEEE 0 and 1, in wave32 and wave64, and for the image forms on 32_UINT, 32_SINT and 32_FLOAT surfaces. The console's gfx1013 is not measured. From c12f9792919fbd1dec032481235f3c05f8fe0a09 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Fri, 30 Oct 2026 22:37:01 +0100 Subject: [PATCH 254/318] test(shader): keep the 64-bit buffer atomic split cases in bounds The test descriptor had stride 0 and bounds mode 0, so every access was out of bounds and the SPIR-V optimizer removed the atomic. Use raw bounds mode and a variable vdata register. Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 48cd3f24a2..d60fc35322 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -7534,16 +7534,16 @@ void verifyInt64AtomicCapabilities() { const auto texelAddress = static_cast(reinterpret_cast(texels.data())); const std::array userData{ 0u, 0u, 0u, 0u, - static_cast(bufferAddress), static_cast((bufferAddress >> 32u) & 0xffffu), 256u, 0x01016facu, + static_cast(bufferAddress), static_cast((bufferAddress >> 32u) & 0xffffu), 256u, 0x31016facu, static_cast(texelAddress >> 8u), static_cast((texelAddress >> 40u) & 0xffu) | (Format32_32UInt << 20u) | (3u << 30u), 7u | (7u << 14u), 0xfacu | (Type2D << 28u), 0u, 0u, 0u, 0u}; const std::vector bufferAtomic{0xe1705000u, 0x80012803u, 0xbf810000u}; const std::vector imageAtomic{0xf03c0308u, 0x00020a08u, 0xbf810000u}; - const std::vector bufferOr{0xe1685000u, 0x80012803u, 0xbf810000u}; - const std::vector bufferAnd{0xe1645000u, 0x80012803u, 0xbf810000u}; - const std::vector bufferXor{0xe16c5000u, 0x80012803u, 0xbf810000u}; - const std::vector bufferOrTwice{0xe1685000u, 0x80012803u, 0xe1685000u, 0x80012a03u, 0xbf810000u}; - const std::vector bufferOrAnd{0xe1685000u, 0x80012803u, 0xe1645000u, 0x80012a03u, 0xbf810000u}; + const std::vector bufferOr{0xe1685000u, 0x80010003u, 0xbf810000u}; + const std::vector bufferAnd{0xe1645000u, 0x80010003u, 0xbf810000u}; + const std::vector bufferXor{0xe16c5000u, 0x80010003u, 0xbf810000u}; + const std::vector bufferOrTwice{0xe1685000u, 0x80010003u, 0xe1685000u, 0x80010203u, 0xbf810000u}; + const std::vector bufferOrAnd{0xe1685000u, 0x80010003u, 0xe1645000u, 0x80010203u, 0xbf810000u}; const auto compile = [&](std::span code, spv::Capability added) { const std::array capabilities{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, static_cast(added)}; RecompileRequest request{}; From 451f0808299e41b4cba8d4db609e8a689168c263 Mon Sep 17 00:00:00 2001 From: Gruzzberry Date: Sat, 31 Oct 2026 03:29:20 +0000 Subject: [PATCH 255/318] feat(libkernel_sys): implement kldload argument and privilege checks Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libkernel_sys/Export.Discovery.cpp | 5 ----- core/libs/prx/libkernel_sys/Export.cpp | 6 ++++++ core/libs/prx/libkernel_sys/include/Api.hpp | 2 +- core/libs/tests/GuestKernelSys.cpp | 13 +++++++++++++ docs/dev/debt/libkernel_sys.md | 1 + 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libkernel_sys/Export.Discovery.cpp b/core/libs/prx/libkernel_sys/Export.Discovery.cpp index 75ddb1a22a..844bde8b4d 100644 --- a/core/libs/prx/libkernel_sys/Export.Discovery.cpp +++ b/core/libs/prx/libkernel_sys/Export.Discovery.cpp @@ -144,11 +144,6 @@ int APS5_VABI is_in_sandbox_nid_postfix(void) { return 0; } -int APS5_VABI kldload_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI kmq_notify_nid_postfix(void) { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libkernel_sys/Export.cpp b/core/libs/prx/libkernel_sys/Export.cpp index bd135b5f96..4c36c7d911 100644 --- a/core/libs/prx/libkernel_sys/Export.cpp +++ b/core/libs/prx/libkernel_sys/Export.cpp @@ -770,6 +770,12 @@ int APS5_VABI kldfind_nid_postfix(const char* file) { return static_cast(Fail(GuestEnoent)); } +int APS5_VABI kldload_nid_postfix(const char* file) { + std::string copy; + if (const int error = CopyGuestPath_nid_no_patch(file, copy)) return static_cast(Fail(error)); + return static_cast(Fail(GuestEperm)); +} + int APS5_VABI kldfirstmod_nid_postfix(int fileid) { return static_cast(Fail(GuestEnoent)); } diff --git a/core/libs/prx/libkernel_sys/include/Api.hpp b/core/libs/prx/libkernel_sys/include/Api.hpp index c1fa1576d4..bd6afaccf4 100644 --- a/core/libs/prx/libkernel_sys/include/Api.hpp +++ b/core/libs/prx/libkernel_sys/include/Api.hpp @@ -117,7 +117,7 @@ int APS5_VABI is_in_sandbox_nid_postfix(void); int APS5_VABI kenv_nid_postfix(int what, const char* name, char* value, int len); int APS5_VABI kldfind_nid_postfix(const char* file); int APS5_VABI kldfirstmod_nid_postfix(int fileid); -int APS5_VABI kldload_nid_postfix(void); +int APS5_VABI kldload_nid_postfix(const char* file); int APS5_VABI kldnext_nid_postfix(int fileid); int APS5_VABI kldstat_nid_postfix(int fileid, void* stat); int APS5_VABI kldsym_nid_postfix(int fileid, int cmd, void* data); diff --git a/core/libs/tests/GuestKernelSys.cpp b/core/libs/tests/GuestKernelSys.cpp index 0961daccf4..a183786ccd 100644 --- a/core/libs/tests/GuestKernelSys.cpp +++ b/core/libs/tests/GuestKernelSys.cpp @@ -333,6 +333,7 @@ void CheckUnreadablePaths(const std::string& renameSource, int directory) { posixFault([&] { return shm_open_nid_postfix(paths.Unreadable(), SCE_KERNEL_O_RDWR, 0600); }); posixFault([&] { return shm_unlink_nid_postfix(paths.Unreadable()); }); posixFault([&] { return kldfind_nid_postfix(paths.Unreadable()); }); + posixFault([&] { return kldload_nid_postfix(paths.Unreadable()); }); posixFault([&] { return kldstat_nid_postfix(1, paths.Unreadable()); }); posixFault([&] { return kldstat_nid_postfix(1, paths.Boundary()); }); const std::int32_t statVersion = 1056; @@ -1407,6 +1408,18 @@ int main() { Require(kldfind_nid_postfix("") == -1 && Error() == 2); Require(kldfind_nid_postfix("kernel") == -1 && Error() == 2); Require(kldfind_nid_postfix("missing.ko") == -1 && Error() == 2); + Require(kldload_nid_postfix(nullptr) == -1 && Error() == 14); + Require(kldload_nid_postfix(reinterpret_cast(0x10)) == -1 && Error() == 14); + Require(kldload_nid_postfix(("/" + std::string(1024, 'p')).c_str()) == -1 && Error() == 63); + const auto kldDirectory = "/" + root.filename().generic_string(); + const auto kldHeader = (root / "kld-header.ko").string(); + std::ofstream(kldHeader, std::ios::binary | std::ios::trunc) << "XELF"; + for (const std::string name : {std::string(), std::string("nomodule"), std::string("missing.ko"), kldDirectory + "/missing.ko", + kldDirectory + "/file.txt", kldDirectory, kldDirectory + "/file.txt/child", kldDirectory + "/kld-header.ko"}) { + *__error_nid_postfix() = 0; + Require(kldload_nid_postfix(name.c_str()) == -1 && Error() == 1); + } + Require(std::filesystem::remove(kldHeader)); Require(kldnext_nid_postfix(-1) == -1 && Error() == 2); Require(kldnext_nid_postfix(0) == 0); Require(kldnext_nid_postfix(1) == -1 && Error() == 2); diff --git a/docs/dev/debt/libkernel_sys.md b/docs/dev/debt/libkernel_sys.md index 9db22c73ac..803609fd59 100644 --- a/docs/dev/debt/libkernel_sys.md +++ b/docs/dev/debt/libkernel_sys.md @@ -374,6 +374,7 @@ - [Console identity](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) - `get_authinfo` errors and capabilities are unverified on PS5, returning `EFAULT` when passed an unwritable destination pointer and following [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/396c2a09a38d22eaa02aab32c1967263dceab95f/src/core/libraries/kernel/kernel.cpp) with `EPERM` for null, and zero authority and credential words. The host models a retail unit (`IsCEX` 1, `IsGenuineCEX` 1, `IsDevKit` 0, `IsGenuineDevKit` 0, `IsTestKit` 0, `IsGenuineTestKit` 0) and Prospero (`IsProspero` 1, `IsKratos` 0). - [sceKernelIsPs4Process and System OpenPSID](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) - `sceKernelIsPs4Process` returns 1 when the title ID starts with `CUSA` and 0 otherwise, as `libSceRtc` and `libSceConvertKeycode` tell PS4 titles apart; this mapping follows the PS4 title-ID convention and is unmeasured on a console, since BunnyFl's [13.42 report](https://github.com/boykopovar/AnyPS5/pull/10959#issuecomment-6404011201) only mentions the export returning 0 in passing. System OpenPSID reuses the ordinary ID; firmware 13.42 reads a separate `machdep.openpsid_for_sys` of unknown value. Null returns positive 22 per the [module report](https://github.com/boykopovar/AnyPS5/pull/10524#issuecomment-6393566914). - [kldfind, kldfirstmod, kldnext and kldunload](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) follow FreeBSD 11.4's linker calls ([sys/kern/kern_linker.c](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/kern/kern_linker.c)): kldfind validates the module name with CopyPath (EFAULT on null or unreadable pointers, ENAMETOOLONG for paths of 1024 bytes or more) and returns ENOENT for any module name; on FreeBSD the kernel itself is linker file 1, and AnyPS5 models an empty file list, so kldnext(0) returns 0 and kldfind("kernel") fails; kldnext returns ENOENT for negative or positive fileids; kldfirstmod returns ENOENT for any fileid; and kldunload returns EPERM as sandboxed/jailed titles lack PRIV_KLD_UNLOAD. Dynamic kernel linker module lookup and management remain unverified on hardware +- [kldload](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) follows FreeBSD 11.4's [`kern_kldload`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/kern/kern_linker.c) for a jailed, non-root caller: the name is copied in first (`EFAULT` for a null or unreadable pointer, `ENAMETOOLONG` at 1024 bytes), then the securelevel and `PRIV_KLD_LOAD` checks fail with `EPERM` before any search or open, so a missing file, an existing file and a file that is not an ELF module all give `EPERM`. That a title runs jailed is taken from `prison_priv_check`, which grants no `PRIV_KLD_LOAD`; it is not measured on a console - [kldstat](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) never succeeds: no linker file is listed here, as for `kldnext`, so the kernel's own file (id 1 on FreeBSD) and the modules a console preloads are missing, every file id gets `ENOENT` and the structure is never written. The checks before the lookup follow FreeBSD 11.4's [`sys_kldstat`](https://github.com/freebsd/freebsd-src/blob/releng/11.4/sys/kern/kern_linker.c#L1228-L1250): only the `version` field is read (`EFAULT` when it cannot be), and a version other than the size of `struct kld_file_stat_1` (1056 bytes on amd64) or `struct kld_file_stat` (2080) gets `EINVAL`. Console behaviour is unverified - [fchown](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) - ownership is not modelled: the guest's own owner and group (0) and the unchanged value (-1) are accepted, a descriptor that is not open fails with `EBADF`, and any other owner or group throws, naming it. The check reads the descriptor with `fstat`, ignoring the CRT's invalid-parameter handler on Windows as `DescriptorControl` does. The console's ownership rules are unverified. Morrowind (`mshivam019/morrowind-ps5`) imports it - [chown, lchown and fchownat](../../../core/libs/prx/libkernel_sys/Export.cpp) (libkernel_sys) - ownership is not modelled, as for `fchown`: owner and group 0 and the unchanged value (-1) are accepted, and any other owner or group throws, naming it. The path is looked up first, as in FreeBSD 11's `kern_fchownat`, with `stat`, with `lstat` for `lchown`, and with `fstatat` for `fchownat`, so a lookup error is returned before the owner is checked and nothing is changed on the host. `fchownat` flags other than `AT_SYMLINK_NOFOLLOW` (0x200) return `EINVAL`, as in FreeBSD 11. The console's ownership rules are unverified From bfef134a9a1e21794d3300adaacf4ac983111877 Mon Sep 17 00:00:00 2001 From: Gruzzberry Date: Mon, 26 Oct 2026 00:31:29 +0000 Subject: [PATCH 256/318] fix(libkernel): fail with ENOENT for a path below a dangling Windows symbolic link Co-Authored-By: Claude Sonnet 5.5 --- .../libs/prx/libc/include/FilesystemError.hpp | 65 +++++++++++++++++++ core/libs/prx/libkernel/File/src/Stdio.cpp | 2 + core/libs/prx/libkernel_sys/Export.cpp | 2 + core/libs/tests/GuestKernelSys.cpp | 48 ++++++++++++++ docs/dev/debt/libc.md | 1 + 5 files changed, 118 insertions(+) diff --git a/core/libs/prx/libc/include/FilesystemError.hpp b/core/libs/prx/libc/include/FilesystemError.hpp index b3b6b2aca3..12f8262558 100644 --- a/core/libs/prx/libc/include/FilesystemError.hpp +++ b/core/libs/prx/libc/include/FilesystemError.hpp @@ -2,6 +2,10 @@ #define CORE_LIBS_PRX_LIBC_INCLUDE_FILESYSTEMERROR_HPP #include +#ifdef _WIN32 +#include +#include "prx/libkernel/File/include/WindowsSymbolicLink.hpp" +#endif #include "General.hpp" inline int FilesystemError(const std::error_code& error) { @@ -23,10 +27,71 @@ inline int FilesystemError(const std::error_code& error) { return 5; } +enum class LinkPrefix_nid_no_patch { Resolves, Missing, BlockedByFile }; + +inline bool HasNonDirectoryPrefix_nid_no_patch(const std::filesystem::path& native); + +#ifdef _WIN32 +inline std::filesystem::path FinalDirectory_nid_no_patch(const std::filesystem::path& directory) { + const HANDLE handle = CreateFileW(directory.c_str(), FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); + if (handle == INVALID_HANDLE_VALUE) return directory; + std::wstring name(MAX_PATH, L'\0'); + DWORD length = GetFinalPathNameByHandleW(handle, name.data(), static_cast(name.size()), VOLUME_NAME_DOS); + if (length >= name.size()) { + name.resize(length + 1); + length = GetFinalPathNameByHandleW(handle, name.data(), static_cast(name.size()), VOLUME_NAME_DOS); + } + CloseHandle(handle); + if (length == 0 || length >= name.size()) return directory; + name.resize(length); + if (name.rfind(L"\\\\?\\UNC\\", 0) == 0) name.replace(0, 7, L"\\"); + else if (name.rfind(L"\\\\?\\", 0) == 0) name.erase(0, 4); + return name; +} +#endif + +inline LinkPrefix_nid_no_patch LinkPrefixState_nid_no_patch(const std::filesystem::path& prefix) { +#ifdef _WIN32 + constexpr unsigned MaxDepth = 32; + thread_local unsigned depth = 0; + if (depth >= MaxDepth) return LinkPrefix_nid_no_patch::Resolves; + const DWORD attributes = GetFileAttributesW(prefix.c_str()); + if (attributes == INVALID_FILE_ATTRIBUTES || (attributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0) return LinkPrefix_nid_no_patch::Resolves; + const HANDLE handle = CreateFileW(prefix.c_str(), FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); + if (handle != INVALID_HANDLE_VALUE) { + CloseHandle(handle); + return LinkPrefix_nid_no_patch::Resolves; + } + const DWORD code = GetLastError(); + if (code != ERROR_FILE_NOT_FOUND && code != ERROR_PATH_NOT_FOUND) return LinkPrefix_nid_no_patch::Resolves; + std::string text; + try { + if (!File::WindowsSymbolicLink::Read(prefix, text, "lookup")) return LinkPrefix_nid_no_patch::Resolves; + } catch (const std::system_error&) { + return LinkPrefix_nid_no_patch::Resolves; + } + const std::filesystem::path target(text); + struct DepthGuard { + explicit DepthGuard(unsigned& value) : value(value) { ++value; } + ~DepthGuard() { --value; } + unsigned& value; + } guard(depth); + const bool blocked = HasNonDirectoryPrefix_nid_no_patch((target.is_absolute() ? target : FinalDirectory_nid_no_patch(prefix.parent_path()) / target) / "."); + return blocked ? LinkPrefix_nid_no_patch::BlockedByFile : LinkPrefix_nid_no_patch::Missing; +#else + (void)prefix; + return LinkPrefix_nid_no_patch::Resolves; +#endif +} + inline bool HasNonDirectoryPrefix_nid_no_patch(const std::filesystem::path& native) { std::error_code error; const auto trimmed = native.has_filename() ? native : native.parent_path(); for (auto prefix = trimmed.parent_path(); prefix.has_relative_path(); prefix = prefix.parent_path()) { + const auto link = LinkPrefixState_nid_no_patch(prefix); + if (link != LinkPrefix_nid_no_patch::Resolves) return link == LinkPrefix_nid_no_patch::BlockedByFile; const auto status = std::filesystem::status(prefix, error); if (std::filesystem::exists(status)) return !std::filesystem::is_directory(status); } diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index a0d6f80550..0a82537506 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -667,6 +667,8 @@ static std::filesystem::path WithoutTrailingSeparator(const std::filesystem::pat static bool HasNonDirectoryPrefix(const std::filesystem::path& native) { std::error_code error; for (auto prefix = WithoutTrailingSeparator(native).parent_path(); prefix.has_relative_path(); prefix = prefix.parent_path()) { + const auto link = LinkPrefixState_nid_no_patch(prefix); + if (link != LinkPrefix_nid_no_patch::Resolves) return link == LinkPrefix_nid_no_patch::BlockedByFile; const auto status = std::filesystem::status(prefix, error); if (std::filesystem::exists(status)) return !std::filesystem::is_directory(status); } diff --git a/core/libs/prx/libkernel_sys/Export.cpp b/core/libs/prx/libkernel_sys/Export.cpp index bd135b5f96..c2ffa4fb80 100644 --- a/core/libs/prx/libkernel_sys/Export.cpp +++ b/core/libs/prx/libkernel_sys/Export.cpp @@ -220,6 +220,8 @@ std::int64_t CopyLinkTarget(const std::string& target, char* buffer, std::size_t bool HasNonDirectoryPrefix(const std::filesystem::path& native) { std::error_code error; for (auto prefix = native.parent_path(); prefix.has_relative_path(); prefix = prefix.parent_path()) { + const auto link = LinkPrefixState_nid_no_patch(prefix); + if (link != LinkPrefix_nid_no_patch::Resolves) return link == LinkPrefix_nid_no_patch::BlockedByFile; const auto status = std::filesystem::status(prefix, error); if (std::filesystem::exists(status)) return !std::filesystem::is_directory(status); } diff --git a/core/libs/tests/GuestKernelSys.cpp b/core/libs/tests/GuestKernelSys.cpp index 25969b5a01..cc8553f658 100644 --- a/core/libs/tests/GuestKernelSys.cpp +++ b/core/libs/tests/GuestKernelSys.cpp @@ -1839,6 +1839,54 @@ int main() { Require(fchmodat_nid_postfix(base, "chmod-link.txt/child", 0600, 0) == -1 && Error() == 20); Require(RemoveSymlink(chmodLink) && RemoveSymlink(chmodDangling) && RemoveSymlink(chmodLoopA) && RemoveSymlink(chmodLoopB)); } + const auto prefixDangling = (root / "prefix-dangling").string(); + const auto prefixLoopA = (root / "prefix-loop-a").string(); + const auto prefixLoopB = (root / "prefix-loop-b").string(); + if (symlink_nid_postfix("prefix-missing", prefixDangling.c_str()) == 0) { + Require(symlink_nid_postfix("prefix-loop-b", prefixLoopA.c_str()) == 0 && symlink_nid_postfix("prefix-loop-a", prefixLoopB.c_str()) == 0); + FileStat prefixStatus{}; + char prefixBuffer[16]{}; + const auto belowDangling = prefixDangling + "/child"; + const auto belowLoop = prefixLoopA + "/child"; + Require(stat_nid_postfix(belowDangling.c_str(), &prefixStatus) == -1 && Error() == 2); + Require(lstat_nid_postfix(belowDangling.c_str(), &prefixStatus) == -1 && Error() == 2); + Require(chmod_nid_postfix(belowDangling.c_str(), 0600) == -1 && Error() == 2); + Require(utimes_nid_postfix(belowDangling.c_str(), nullptr) == -1 && Error() == 2); + Require(unlink_nid_postfix(belowDangling.c_str()) == -1 && Error() == 2); + Require(open_nid_postfix(belowDangling.c_str(), 0, 0) == -1 && Error() == 2); + Require(fchmodat_nid_postfix(base, "prefix-dangling/child", 0600, 0) == -1 && Error() == 2); + Require(readlink_nid_postfix(belowDangling.c_str(), prefixBuffer, sizeof(prefixBuffer)) == -1 && Error() == 2); + Require(linkat_nid_postfix(-100, belowDangling.c_str(), -100, (root / "prefix-linked").string().c_str(), 0x400) == -1 && Error() == 2); + Require(stat_nid_postfix(belowLoop.c_str(), &prefixStatus) == -1 && Error() == 62); + const auto prefixThroughFile = (root / "prefix-through-file").string(); + Require(symlink_nid_postfix("file.txt/x", prefixThroughFile.c_str()) == 0); + const auto belowThroughFile = prefixThroughFile + "/child"; + Require(stat_nid_postfix(belowThroughFile.c_str(), &prefixStatus) == -1 && Error() == 20); + Require(chmod_nid_postfix(belowThroughFile.c_str(), 0600) == -1 && Error() == 20); + Require(fchmodat_nid_postfix(base, "prefix-through-file/child", 0600, 0) == -1 && Error() == 20); +#ifdef _WIN32 + std::filesystem::create_directories(root / "prefix-up" / "deep"); + { std::ofstream file(root / "prefix-up" / "f2"); file << "x"; } + const auto upDirectory = (root / "prefix-up-link").string(); + const auto upFile = (root / "prefix-up" / "deep" / "pa").string(); + const auto createHostLink = [](const std::string& link, const wchar_t* target, DWORD flags) { + return CreateSymbolicLinkW(std::filesystem::path(link).c_str(), target, flags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE) != 0; + }; + Require(createHostLink(upDirectory, L"prefix-up\\deep", SYMBOLIC_LINK_FLAG_DIRECTORY) && createHostLink(upFile, L"..\\f2\\x", 0)); + const auto belowUp = upDirectory + "/pa/child"; + Require(stat_nid_postfix(belowUp.c_str(), &prefixStatus) == -1 && Error() == 20); + Require(RemoveSymlink(upFile) && RemoveSymlink(upDirectory)); +#endif + const auto prefixChainEnd = (root / "prefix-chain-end").string(); + const auto prefixChainStart = (root / "prefix-chain-start").string(); + Require(symlink_nid_postfix("file.txt/x", prefixChainEnd.c_str()) == 0); + Require(symlink_nid_postfix("prefix-chain-end", prefixChainStart.c_str()) == 0); + const auto belowChain = prefixChainStart + "/child"; + Require(stat_nid_postfix(belowChain.c_str(), &prefixStatus) == -1 && Error() == 20); + Require(RemoveSymlink(prefixChainStart) && RemoveSymlink(prefixChainEnd)); + Require(RemoveSymlink(prefixThroughFile)); + Require(RemoveSymlink(prefixDangling) && RemoveSymlink(prefixLoopA) && RemoveSymlink(prefixLoopB)); + } Require(fchown_nid_postfix(base, 0, 0) == 0); Require(fchown_nid_postfix(base, 0xFFFFFFFFu, 0xFFFFFFFFu) == 0); Require(fchown_nid_postfix(base, 0, 0xFFFFFFFFu) == 0); diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index ae986ac7aa..b84d9678ef 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -138,6 +138,7 @@ - [fgetln](../../../core/libs/prx/libc/src/StandardStreams.cpp) (libSceLibcInternal, implemented in libc) throws: in the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x5FE80) the export is a `hlt` instruction, which faults in user mode and ends the process - [Wide-character conversions](../../../core/libs/prx/libc/src/Multibyte.cpp) (libc) - The wide `wcsto*` functions parse the ASCII prefix of the guest string, as the C locale accepts only ASCII white space and digits. The multibyte conversions follow the C locale of the console's `libSceLibcInternal.sprx`: UTF-8 of at most three bytes, so `_Getmbcurmax` returns 3, overlong forms and encoded surrogates decode, and four-byte forms are invalid. `__mb_cur_max` stays 1 like the console's variable; `___mb_cur_max` is not in that module and also stays 1, unverified. Other locales are not supported. With a null destination, `wcstombs` returns the byte count; the console's count is wrong when the capacity argument equals the running count, because it reads past the terminator - [realpath](../../../core/libs/prx/libc/src/RealPath.cpp) (libc) follows FreeBSD 11's [`realpath.c`](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/libc/stdlib/realpath.c) with two gaps. A host symbolic link with a relative target under the guest root is expanded as `realpath1` does (a dangling link fails with `ENOENT`, more than 32 links with `ELOOP`), but a link with an absolute target is not expanded, nor is anything below it, because the target is a host path with no guest path to map to. Only Windows symbolic links are expanded there, not junctions. Expansion works on the guest path, so a host link whose text climbs above a path alias root is resolved against the guest tree, not the host one. Otherwise `..` drops the previous component lexically, as `Resolve` does for every guest path. +- [Paths below a Windows symbolic link](../../../core/libs/prx/libc/include/FilesystemError.hpp) (libc) chooses `ENOENT` or `ENOTDIR` for a path below a link by reading the link text on Windows, and three cases still differ from FreeBSD 11.4's `namei`: below a chain of 33 or more links the answer is `ENOTDIR`, not `ELOOP`; a link made with the directory flag that points at a regular file, or at a link that runs through one, answers `ENOENT`, not `ENOTDIR`, because Windows fails its open with `ERROR_DIRECTORY`; and below a link loop `lstat`, `chmod`, `fchmodat`, `linkat`, `readlink`, `access`, `rename` and `truncate` answer `ENOTDIR`, not `ELOOP`. - [strtod, strtof, strtold](../../../core/libs/prx/libc/src/Strings.cpp) (libc) - `strtod` and `wcstod` (and `atof`, which calls `strtod` here) read a `NAN(...)` sequence as FreeBSD 11's gdtoa [`hexnan`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/contrib/gdtoa/hexnan.c) does: hexadecimal digits with an optional `0x`, blanks starting a new 32-bit word, the first 16 digits kept and the payload cut to 52 bits, a zero payload made 1, any other character giving the default NaN but consuming up to the closing parenthesis, and an unterminated sequence keeping its payload while consuming only `nan`. The payload goes under the quiet bit with the sign, as in gdtoa's [`strtod`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/contrib/gdtoa/strtod.c#L316-L328). `strtold` and `wcstold` return the default quiet NaN (exponent `0x7fff`, significand `0xc000000000000000`) whatever the sequence, and consume a sequence of letters, digits and underscores up to its closing parenthesis. The console's `libSceLibcInternal.sprx`, called from a payload process on a PS5 (firmware 12.70), gives these results for the 64 NaN inputs tried through `strtod`, 50 of them also through `strtold` and `wcstod`, and 18 through `wcstold`, and, through `strtod`, for `nan(1 2)` and for `nan( ` followed by NUL and `)`; titles' own `libc.prx` copies give the same `strtod` results. In `hexnan`, a blank after a payload digit, while a 32-bit word of the payload is still free, starts a blank skip that has no NUL check, so when only blanks follow up to the terminating NUL the console's `strtod` reads past it: `nan(1 ` followed by NUL and `)` consumes the `)` after the NUL, so the result depends on the bytes after the string, and here that case throws. A blank right after `nan(` is skipped with a NUL check, and the console stopped at the NUL there - [strtof and wcstof NaN payloads](../../../core/libs/prx/libc/src/Strings.cpp) (libc) - the console's `libSceLibcInternal.sprx` (firmware 12.70) returns the default NaN for every `NAN(...)` sequence, and modules that import it reach this function through the `libc.prx` fallback, so they get the payload the titles' copy keeps. That copy, in eleven titles (64 inputs), reads the sequence as `strtod` does but into one 32-bit word: a blank after the first group of digits ends the payload, the first 8 digits are kept and cut to 24 bits under the quiet bit, and a zero payload becomes 1. - [strtod](../../../core/libs/prx/libc/src/Strings.cpp) (libc) sets `ERANGE` for tested decimal inputs with a nonzero subnormal result, including the exact decimal expansion of 2^-1074, matching nine distinct title-bundled `libc.prx` builds measured on Windows. Two other `libc.prx` builds and the console's `libSceLibcInternal.sprx` leave `errno` alone for decimal subnormals. `wcstod` still parses through the host `std::strtod` path without this explicit rule; title and console `wcstod` behavior for decimal subnormals is unverified. Hexadecimal subnormal `strtod` and `wcstod` match one title's `libc.prx` only (PPSA08329, SHA-256 a485b34ef160, run on Linux, not on a console). Results of at least 2^-1074 come from the host `strtod`, and glibc 2.35 rounds some a unit low: `0x3ffffffffffff5p-1077` gives `0x0007fffffffffffe`, that `libc.prx` `0x0007ffffffffffff`. From 6f6c18b66f8d97b74a096215aa8f451e1047d6b1 Mon Sep 17 00:00:00 2001 From: Rookie <175737347+rooklz@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:53:41 +0100 Subject: [PATCH 257/318] feat(libkernel): implement the umutex, condition variable and rwlock _umtx_op operations as the console does --- .../SyncOnAddress/src/SyncOnAddress.cpp | 424 ++++++++++++++++++ core/libs/tests/GuestUmtxOp.cpp | 187 +++++++- docs/dev/debt/libkernel.md | 4 +- 3 files changed, 609 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp index 358c534bae..29ae3c6779 100644 --- a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp +++ b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp @@ -25,9 +25,21 @@ constexpr int SYNC_ON_ADDRESS_OK = 0; constexpr int UMTX_OP_RESERVED1 = 1; constexpr int UMTX_OP_WAIT = 2; constexpr int UMTX_OP_WAKE = 3; +constexpr int UMTX_OP_MUTEX_TRYLOCK = 4; +constexpr int UMTX_OP_MUTEX_LOCK = 5; +constexpr int UMTX_OP_MUTEX_UNLOCK = 6; +constexpr int UMTX_OP_SET_CEILING = 7; +constexpr int UMTX_OP_CV_WAIT = 8; +constexpr int UMTX_OP_CV_SIGNAL = 9; +constexpr int UMTX_OP_CV_BROADCAST = 10; constexpr int UMTX_OP_WAIT_UINT = 11; +constexpr int UMTX_OP_RW_RDLOCK = 12; +constexpr int UMTX_OP_RW_WRLOCK = 13; +constexpr int UMTX_OP_RW_UNLOCK = 14; constexpr int UMTX_OP_WAIT_UINT_PRIVATE = 15; constexpr int UMTX_OP_WAKE_PRIVATE = 16; +constexpr int UMTX_OP_MUTEX_WAIT = 17; +constexpr int UMTX_OP_MUTEX_WAKE = 18; constexpr int UMTX_OP_SEM_WAIT = 19; constexpr int UMTX_OP_SEM_WAKE = 20; constexpr int UMTX_OP_NWAKE_PRIVATE = 21; @@ -35,8 +47,23 @@ constexpr int UMTX_OP_MAX = 27; constexpr std::uint32_t UMTX_ABSTIME = 1; constexpr int GUEST_CLOCK_REALTIME = 0; +constexpr std::uint32_t UMUTEX_UNOWNED = 0; +constexpr std::uint32_t UMUTEX_CONTESTED = 0x80000000u; +constexpr std::uint32_t UMUTEX_PRIO_PROTECT = 0x8; +constexpr std::uint32_t UMUTEX_ROBUST = 0x10; +constexpr std::uint32_t CVWAIT_ABSTIME = 0x2; +constexpr std::uint32_t CVWAIT_CLOCKID = 0x4; +constexpr std::uint32_t URWLOCK_PREFER_READER = 0x2; +constexpr std::uint32_t URWLOCK_WRITE_OWNER = 0x80000000u; +constexpr std::uint32_t URWLOCK_WRITE_WAITERS = 0x40000000u; +constexpr std::uint32_t URWLOCK_READ_WAITERS = 0x20000000u; +constexpr std::uint32_t URWLOCK_MAX_READERS = 0x1fffffffu; + +constexpr int GUEST_EPERM = 1; constexpr int GUEST_EFAULT = 14; +constexpr int GUEST_EBUSY = 16; constexpr int GUEST_EINVAL = 22; +constexpr int GUEST_EAGAIN = 35; constexpr int GUEST_EOPNOTSUPP = 45; constexpr int GUEST_ETIMEDOUT = 60; @@ -55,6 +82,30 @@ struct UserSemaphore { std::uint32_t flags; }; +struct UserMutex { + std::uint32_t owner; + std::uint32_t flags; + std::uint32_t ceilings[2]; + std::uint32_t spare[4]; +}; + +struct UserCondition { + std::uint32_t hasWaiters; + std::uint32_t flags; + std::uint32_t clockId; + std::uint32_t spare; +}; + +struct UserRwlock { + std::uint32_t state; + std::uint32_t flags; + std::uint32_t blockedReaders; + std::uint32_t blockedWriters; + std::uint32_t spare[4]; +}; + +static_assert(sizeof(UserMutex) == 32 && sizeof(UserCondition) == 16 && sizeof(UserRwlock) == 32); + struct AddressWaiter { TimedWait::SingleWaiterCondition condition; bool woken = false; @@ -335,6 +386,345 @@ int SemWake(UserSemaphore* semaphore) { return 0; } +std::uint32_t CurrentOwnerId() { + return static_cast(scePthreadGetthreadid()) & ~UMUTEX_CONTESTED; +} + +std::size_t WaiterCount(std::uintptr_t key) { + auto& bucket = BucketOf(key); + std::lock_guard lock(bucket.mutex); + const auto entry = bucket.waiters.find(key); + return entry == bucket.waiters.end() ? 0 : entry->second.size(); +} + +int CheckMutex(UserMutex* mutex) { + if (mutex == nullptr) { + return GUEST_EFAULT; + } + if (!IsAlignedAddress(reinterpret_cast(mutex))) { + APS5_INVALID_ARG_EX; + } + const std::uint32_t flags = std::atomic_ref(mutex->flags).load(); + if ((flags & UMUTEX_ROBUST) != 0) { + NotImplemented_nid_no_patch("_umtx_op on a robust umutex"); + } + return 0; +} + +bool TryAcquireMutex(UserMutex* mutex, std::uint32_t id, int& error) { + std::atomic_ref owner(mutex->owner); + std::uint32_t current = owner.load(); + for (;;) { + if (current == UMUTEX_UNOWNED || current == UMUTEX_CONTESTED) { + if (owner.compare_exchange_weak(current, id | current)) { + error = 0; + return true; + } + continue; + } + if ((current & ~UMUTEX_CONTESTED) == id) { + error = 0; + return true; + } + error = GUEST_EBUSY; + return false; + } +} + +void MarkContested(UserMutex* mutex) { + std::atomic_ref owner(mutex->owner); + std::uint32_t current = owner.load(); + while (current != UMUTEX_UNOWNED && (current & UMUTEX_CONTESTED) == 0 && !owner.compare_exchange_weak(current, current | UMUTEX_CONTESTED)) { + } +} + +int MutexTrylock(UserMutex* mutex) { + if (const int error = CheckMutex(mutex); error != 0) { + return error; + } + if ((std::atomic_ref(mutex->flags).load() & UMUTEX_PRIO_PROTECT) != 0) { + return GUEST_EINVAL; + } + int error = 0; + TryAcquireMutex(mutex, CurrentOwnerId(), error); + return error; +} + +int MutexLock(UserMutex* mutex, bool acquire, const void* sizeArgument, const void* timeArgument, const void* caller) { + if (const int error = CheckMutex(mutex); error != 0) { + return error; + } + if ((std::atomic_ref(mutex->flags).load() & UMUTEX_PRIO_PROTECT) != 0) { + return GUEST_EINVAL; + } + std::optional deadline; + if (const int error = UmtxDeadline(sizeArgument, timeArgument, deadline); error != 0) { + return error; + } + const std::uint32_t id = CurrentOwnerId(); + const auto key = reinterpret_cast(&mutex->owner); + for (;;) { + bool done = false; + int error = 0; + const bool woken = WaitOnKeyUntil(key, [&] { + if (acquire) { + done = TryAcquireMutex(mutex, id, error); + } else { + const std::uint32_t current = std::atomic_ref(mutex->owner).load(); + done = current == UMUTEX_UNOWNED || current == UMUTEX_CONTESTED; + } + if (!done) { + MarkContested(mutex); + } + return done; + }, deadline, caller); + if (done) { + return 0; + } + if (!woken) { + return GUEST_ETIMEDOUT; + } + if (!acquire) { + return 0; + } + } +} + +int MutexUnlock(UserMutex* mutex) { + if (const int error = CheckMutex(mutex); error != 0) { + return error; + } + if ((std::atomic_ref(mutex->flags).load() & UMUTEX_PRIO_PROTECT) != 0) { + NotImplemented_nid_no_patch("_umtx_op unlock of a priority-protected umutex"); + } + std::atomic_ref owner(mutex->owner); + if ((owner.load() & ~UMUTEX_CONTESTED) != CurrentOwnerId()) { + return GUEST_EPERM; + } + const auto key = reinterpret_cast(&mutex->owner); + owner.store(WaiterCount(key) > 1 ? UMUTEX_CONTESTED : UMUTEX_UNOWNED); + WakeAddress(key, 1, false); + return 0; +} + +int MutexWake(UserMutex* mutex) { + if (const int error = CheckMutex(mutex); error != 0) { + return error; + } + std::atomic_ref owner(mutex->owner); + if ((owner.load() & ~UMUTEX_CONTESTED) != 0) { + return 0; + } + const auto key = reinterpret_cast(&mutex->owner); + if (WaiterCount(key) <= 1) { + std::uint32_t contested = UMUTEX_CONTESTED; + owner.compare_exchange_strong(contested, UMUTEX_UNOWNED); + } + WakeAddress(key, 1, false); + return 0; +} + +int ConditionDeadline(const UserCondition* condition, std::uint64_t flags, const void* timeArgument, std::optional& deadlineNanos) { + deadlineNanos.reset(); + if (timeArgument == nullptr) { + return 0; + } + const auto timeout = *static_cast(timeArgument); + if (timeout.tv_sec < 0 || timeout.tv_nsec < 0 || timeout.tv_nsec >= NANOS_PER_SECOND) { + return GUEST_EINVAL; + } + std::uint64_t remaining = 0; + if ((flags & CVWAIT_ABSTIME) == 0) { + remaining = SaturatingNanos(timeout.tv_sec, timeout.tv_nsec); + } else { + const int clock = (flags & CVWAIT_CLOCKID) != 0 ? static_cast(std::atomic_ref(condition->clockId).load()) : GUEST_CLOCK_REALTIME; + KernelTimespec now{}; + clock_gettime_nid_postfix(clock, &now); + remaining = SaturatingNanos(timeout.tv_sec - now.tv_sec, timeout.tv_nsec - now.tv_nsec); + } + if (remaining < UNBOUNDED_WAIT_NANOS) { + deadlineNanos = TimedWait::NowNanos() + remaining; + } + return 0; +} + +int ConditionWait(UserCondition* condition, UserMutex* mutex, std::uint64_t flags, const void* timeArgument, const void* caller) { + if (condition == nullptr) { + return GUEST_EFAULT; + } + if (const int error = CheckMutex(mutex); error != 0) { + return error; + } + std::optional deadline; + if (const int error = ConditionDeadline(condition, flags, timeArgument, deadline); error != 0) { + return error; + } + const auto key = reinterpret_cast(condition); + auto& bucket = BucketOf(key); + AddressWaiter waiter; + int error = 0; + bool woken = false; + std::optional waited; + DeferringDelivery([&] { + std::unique_lock lock(bucket.mutex); + std::atomic_ref(condition->hasWaiters).store(1); + auto& queue = bucket.waiters[key]; + const auto position = queue.insert(queue.end(), &waiter); + lock.unlock(); + error = MutexUnlock(mutex); + lock.lock(); + if (error == 0) { + const auto isWoken = [&] { return waiter.woken; }; + const auto waitStart = std::chrono::steady_clock::now(); + if (!deadline) { + waiter.condition.Wait(lock, isWoken); + } else { + waiter.condition.WaitUntil(lock, *deadline, isWoken); + } + waited = std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart); + } + woken = waiter.woken; + if (!woken) { + auto& current = bucket.waiters[key]; + current.erase(position); + if (current.empty()) { + bucket.waiters.erase(key); + std::atomic_ref(condition->hasWaiters).store(0); + } + } + }); + if (waited) { + KernelTraceWait_nid_postfix("cv", caller, static_cast(waited->count()), !woken); + } + if (error != 0) { + return error; + } + return woken ? 0 : GUEST_ETIMEDOUT; +} + +int ConditionWake(UserCondition* condition, bool all) { + if (condition == nullptr) { + return GUEST_EFAULT; + } + const auto key = reinterpret_cast(condition); + DeferringDelivery([&] { + auto& bucket = BucketOf(key); + std::lock_guard lock(bucket.mutex); + const auto entry = bucket.waiters.find(key); + if (entry == bucket.waiters.end()) { + std::atomic_ref(condition->hasWaiters).store(0); + return; + } + auto& queue = entry->second; + do { + AddressWaiter* waiter = queue.front(); + queue.pop_front(); + waiter->woken = true; + waiter->condition.NotifyOne(); + } while (all && !queue.empty()); + if (queue.empty()) { + bucket.waiters.erase(entry); + std::atomic_ref(condition->hasWaiters).store(0); + } + }); + return 0; +} + +int CheckRwlock(UserRwlock* rwlock) { + if (rwlock == nullptr) { + return GUEST_EFAULT; + } + if (!IsAlignedAddress(reinterpret_cast(rwlock))) { + APS5_INVALID_ARG_EX; + } + return 0; +} + +void ClearRwlockBits(UserRwlock* rwlock, std::uint32_t bits) { + std::atomic_ref(rwlock->state).fetch_and(~bits); +} + +int RwlockLock(UserRwlock* rwlock, bool writer, const void* sizeArgument, const void* timeArgument, const void* caller) { + if (const int error = CheckRwlock(rwlock); error != 0) { + return error; + } + std::optional deadline; + if (const int error = UmtxDeadline(sizeArgument, timeArgument, deadline); error != 0) { + return error; + } + std::atomic_ref state(rwlock->state); + std::atomic_ref blocked(writer ? rwlock->blockedWriters : rwlock->blockedReaders); + const std::uint32_t waiterBit = writer ? URWLOCK_WRITE_WAITERS : URWLOCK_READ_WAITERS; + const bool preferReader = (std::atomic_ref(rwlock->flags).load() & URWLOCK_PREFER_READER) != 0; + const auto key = reinterpret_cast(writer ? &rwlock->blockedWriters : &rwlock->state); + for (;;) { + int error = 0; + bool done = false; + const bool woken = WaitOnKeyUntil(key, [&] { + std::uint32_t current = state.load(); + for (;;) { + const bool free = writer ? (current & (URWLOCK_WRITE_OWNER | URWLOCK_MAX_READERS)) == 0 + : (current & URWLOCK_WRITE_OWNER) == 0 && (preferReader || (current & URWLOCK_WRITE_WAITERS) == 0); + if (!free) { + break; + } + if (!writer && (current & URWLOCK_MAX_READERS) == URWLOCK_MAX_READERS) { + error = GUEST_EAGAIN; + done = true; + return true; + } + if (state.compare_exchange_weak(current, writer ? current | URWLOCK_WRITE_OWNER : current + 1)) { + done = true; + return true; + } + } + state.fetch_or(waiterBit); + blocked.fetch_add(1); + return false; + }, deadline, caller); + if (done) { + return error; + } + const std::uint32_t left = blocked.fetch_sub(1) - 1; + if (left == 0 && (writer || woken)) { + ClearRwlockBits(rwlock, waiterBit); + } + if (!woken) { + return GUEST_ETIMEDOUT; + } + } +} + +int RwlockUnlock(UserRwlock* rwlock) { + if (const int error = CheckRwlock(rwlock); error != 0) { + return error; + } + std::atomic_ref state(rwlock->state); + std::uint32_t current = state.load(); + std::uint32_t next = 0; + do { + if ((current & URWLOCK_WRITE_OWNER) != 0) { + next = current & ~URWLOCK_WRITE_OWNER; + } else if ((current & URWLOCK_MAX_READERS) != 0) { + next = current - 1; + } else { + return GUEST_EPERM; + } + } while (!state.compare_exchange_weak(current, next)); + if ((next & URWLOCK_MAX_READERS) != 0) { + return 0; + } + const bool preferReader = (std::atomic_ref(rwlock->flags).load() & URWLOCK_PREFER_READER) != 0; + const bool writers = (next & URWLOCK_WRITE_WAITERS) != 0; + const bool readers = (next & URWLOCK_READ_WAITERS) != 0; + if (writers && (!preferReader || !readers)) { + WakeAddress(reinterpret_cast(&rwlock->blockedWriters), 1, false); + } else if (readers) { + WakeAddress(reinterpret_cast(&rwlock->state), std::numeric_limits::max(), false); + } + return 0; +} + } // namespace extern "C" { @@ -392,6 +782,40 @@ int APS5_VABI _umtx_op_nid_postfix(void* object, int operation, std::uint64_t va case UMTX_OP_SEM_WAKE: error = SemWake(static_cast(object)); break; + case UMTX_OP_MUTEX_TRYLOCK: + error = MutexTrylock(static_cast(object)); + break; + case UMTX_OP_MUTEX_LOCK: + case UMTX_OP_MUTEX_WAIT: + error = MutexLock(static_cast(object), operation == UMTX_OP_MUTEX_LOCK, sizeArgument, timeArgument, caller); + break; + case UMTX_OP_MUTEX_UNLOCK: + error = MutexUnlock(static_cast(object)); + break; + case UMTX_OP_MUTEX_WAKE: + error = MutexWake(static_cast(object)); + break; + case UMTX_OP_SET_CEILING: + if (object == nullptr || (std::atomic_ref(static_cast(object)->flags).load() & UMUTEX_PRIO_PROTECT) == 0) { + error = object == nullptr ? GUEST_EFAULT : GUEST_EINVAL; + } else { + NotImplemented_nid_no_patch("_umtx_op ceiling of a priority-protected umutex"); + } + break; + case UMTX_OP_CV_WAIT: + error = ConditionWait(static_cast(object), static_cast(sizeArgument), value, timeArgument, caller); + break; + case UMTX_OP_CV_SIGNAL: + case UMTX_OP_CV_BROADCAST: + error = ConditionWake(static_cast(object), operation == UMTX_OP_CV_BROADCAST); + break; + case UMTX_OP_RW_RDLOCK: + case UMTX_OP_RW_WRLOCK: + error = RwlockLock(static_cast(object), operation == UMTX_OP_RW_WRLOCK, sizeArgument, timeArgument, caller); + break; + case UMTX_OP_RW_UNLOCK: + error = RwlockUnlock(static_cast(object)); + break; default: if (operation < 0 || operation >= UMTX_OP_MAX) { error = GUEST_EINVAL; diff --git a/core/libs/tests/GuestUmtxOp.cpp b/core/libs/tests/GuestUmtxOp.cpp index ee4befc68f..6c0ab5c485 100644 --- a/core/libs/tests/GuestUmtxOp.cpp +++ b/core/libs/tests/GuestUmtxOp.cpp @@ -13,7 +13,18 @@ static constexpr int UMTX_OP_WAIT = 2; static constexpr int UMTX_OP_WAKE = 3; +static constexpr int UMTX_OP_MUTEX_TRYLOCK = 4; static constexpr int UMTX_OP_MUTEX_LOCK = 5; +static constexpr int UMTX_OP_MUTEX_UNLOCK = 6; +static constexpr int UMTX_OP_SET_CEILING = 7; +static constexpr int UMTX_OP_CV_WAIT = 8; +static constexpr int UMTX_OP_CV_SIGNAL = 9; +static constexpr int UMTX_OP_CV_BROADCAST = 10; +static constexpr int UMTX_OP_RW_RDLOCK = 12; +static constexpr int UMTX_OP_RW_WRLOCK = 13; +static constexpr int UMTX_OP_RW_UNLOCK = 14; +static constexpr int UMTX_OP_MUTEX_WAIT = 17; +static constexpr int UMTX_OP_MUTEX_WAKE = 18; static constexpr int UMTX_OP_WAIT_UINT = 11; static constexpr int UMTX_OP_WAIT_UINT_PRIVATE = 15; static constexpr int UMTX_OP_WAKE_PRIVATE = 16; @@ -25,11 +36,42 @@ static constexpr int UMTX_OP_MAX = 27; static constexpr std::uint32_t UMTX_ABSTIME = 1; static constexpr int CLOCK_MONOTONIC_ID = 4; +static constexpr int GUEST_EPERM = 1; static constexpr int GUEST_EFAULT = 14; +static constexpr int GUEST_EBUSY = 16; static constexpr int GUEST_EINVAL = 22; static constexpr int GUEST_EOPNOTSUPP = 45; static constexpr int GUEST_ETIMEDOUT = 60; +static constexpr std::uint32_t CONTESTED = 0x80000000u; +static constexpr std::uint32_t PRIO_INHERIT = 0x4; +static constexpr std::uint32_t PRIO_PROTECT = 0x8; +static constexpr std::uint32_t WRITE_OWNER = 0x80000000u; +static constexpr std::uint32_t WRITE_WAITERS = 0x40000000u; +static constexpr std::uint32_t READ_WAITERS = 0x20000000u; + +struct Umutex { + std::uint32_t owner = 0; + std::uint32_t flags = 0; + std::uint32_t ceilings[2]{}; + std::uint32_t spare[4]{}; +}; + +struct Ucond { + std::uint32_t hasWaiters = 0; + std::uint32_t flags = 0; + std::uint32_t clockId = 0; + std::uint32_t spare = 0; +}; + +struct Urwlock { + std::uint32_t state = 0; + std::uint32_t flags = 0; + std::uint32_t blockedReaders = 0; + std::uint32_t blockedWriters = 0; + std::uint32_t spare[4]{}; +}; + static constexpr auto SETTLE = std::chrono::milliseconds(100); struct UmtxTime { @@ -342,16 +384,148 @@ static void RejectsBadObjectsAndOperations() { Require(word == owner); } + Require(FailsWith(Umtx(&value, 7, 0), GUEST_EINVAL)); bool threw = false; - try { Umtx(&value, UMTX_OP_MUTEX_LOCK, 0); } - catch (const std::runtime_error&) { threw = true; } - Require(threw); - threw = false; try { Umtx(&value, UMTX_OP_SEM2_WAIT, 0); } catch (const std::runtime_error&) { threw = true; } Require(threw); } +static std::uint32_t Self() { return static_cast(scePthreadGetthreadid()); } + +static UmtxTime ShortTimeout() { return {{0, 1000000}, 0, CLOCK_MONOTONIC_ID}; } + +static void MutexOwnershipFollowsTheConsole() { + Umutex mutex; + Require(Umtx(&mutex, UMTX_OP_MUTEX_TRYLOCK, 0) == 0 && mutex.owner == Self()); + Require(Umtx(&mutex, UMTX_OP_MUTEX_TRYLOCK, 0) == 0 && mutex.owner == Self()); + UmtxTime timeout = ShortTimeout(); + Require(Umtx(&mutex, UMTX_OP_MUTEX_LOCK, 0, SizeOf(sizeof(timeout)), &timeout) == 0 && mutex.owner == Self()); + Require(Umtx(&mutex, UMTX_OP_MUTEX_UNLOCK, 0) == 0 && mutex.owner == 0); + Require(FailsWith(Umtx(&mutex, UMTX_OP_MUTEX_UNLOCK, 0), GUEST_EPERM) && mutex.owner == 0); + Require(Umtx(&mutex, UMTX_OP_MUTEX_LOCK, 0) == 0 && mutex.owner == Self()); + Require(Umtx(&mutex, UMTX_OP_MUTEX_UNLOCK, 0) == 0); + + mutex.owner = 12345; + Require(FailsWith(Umtx(&mutex, UMTX_OP_MUTEX_UNLOCK, 0), GUEST_EPERM) && mutex.owner == 12345); + Require(FailsWith(Umtx(&mutex, UMTX_OP_MUTEX_TRYLOCK, 0), GUEST_EBUSY) && mutex.owner == 12345); + timeout = ShortTimeout(); + Require(FailsWith(Umtx(&mutex, UMTX_OP_MUTEX_LOCK, 0, SizeOf(sizeof(timeout)), &timeout), GUEST_ETIMEDOUT) && mutex.owner == (CONTESTED | 12345)); + timeout = ShortTimeout(); + Require(FailsWith(Umtx(&mutex, UMTX_OP_MUTEX_WAIT, 0, SizeOf(sizeof(timeout)), &timeout), GUEST_ETIMEDOUT)); + + mutex.owner = 0; + Require(Umtx(&mutex, UMTX_OP_MUTEX_WAIT, 0) == 0 && mutex.owner == 0); + mutex.owner = CONTESTED; + Require(Umtx(&mutex, UMTX_OP_MUTEX_WAKE, 0) == 0 && mutex.owner == 0); + mutex.owner = CONTESTED; + Require(Umtx(&mutex, UMTX_OP_MUTEX_TRYLOCK, 0) == 0 && mutex.owner == (CONTESTED | Self())); + Require(Umtx(&mutex, UMTX_OP_MUTEX_UNLOCK, 0) == 0 && mutex.owner == 0); + + Umutex inherit; + inherit.flags = PRIO_INHERIT; + Require(Umtx(&inherit, UMTX_OP_MUTEX_TRYLOCK, 0) == 0 && inherit.owner == Self()); + Require(Umtx(&inherit, UMTX_OP_MUTEX_UNLOCK, 0) == 0 && inherit.owner == 0); + Umutex protect; + protect.flags = PRIO_PROTECT; + Require(FailsWith(Umtx(&protect, UMTX_OP_MUTEX_TRYLOCK, 0), GUEST_EINVAL) && protect.owner == 0); + Require(FailsWith(Umtx(&mutex, UMTX_OP_SET_CEILING, 0), GUEST_EINVAL)); + Require(FailsWith(Umtx(nullptr, UMTX_OP_MUTEX_TRYLOCK, 0), GUEST_EFAULT)); +} + +static void MutexLockWaitsForTheOwnerToUnlock() { + Umutex mutex; + Require(Umtx(&mutex, UMTX_OP_MUTEX_LOCK, 0) == 0); + std::atomic state{0}; + std::thread waiter([&] { + state.store(1); + Require(Umtx(&mutex, UMTX_OP_MUTEX_LOCK, 0) == 0); + state.store(2); + Require(Umtx(&mutex, UMTX_OP_MUTEX_UNLOCK, 0) == 0); + }); + AwaitCount(state, 1); + std::this_thread::sleep_for(SETTLE); + Require(state.load() == 1 && (std::atomic_ref(mutex.owner).load() & CONTESTED) != 0); + Require(Umtx(&mutex, UMTX_OP_MUTEX_UNLOCK, 0) == 0); + AwaitCount(state, 2); + waiter.join(); + Require(mutex.owner == 0); +} + +static void ConditionWaitReleasesTheMutexAndWakes() { + Ucond condition; + Require(Umtx(&condition, UMTX_OP_CV_SIGNAL, 0) == 0 && condition.hasWaiters == 0); + Require(Umtx(&condition, UMTX_OP_CV_BROADCAST, 0) == 0 && condition.hasWaiters == 0); + + Umutex mutex; + Require(Umtx(&mutex, UMTX_OP_MUTEX_TRYLOCK, 0) == 0); + KernelTimespec timeout{0, 1000000}; + Require(FailsWith(Umtx(&condition, UMTX_OP_CV_WAIT, 0, &mutex, &timeout), GUEST_ETIMEDOUT)); + Require(condition.hasWaiters == 0 && mutex.owner == 0); + Require(FailsWith(Umtx(&condition, UMTX_OP_CV_WAIT, 0, &mutex, &timeout), GUEST_EPERM) && condition.hasWaiters == 0); + + std::atomic state{0}; + std::vector waiters; + for (int index = 0; index < 2; ++index) { + waiters.emplace_back([&] { + Umutex own; + Require(Umtx(&own, UMTX_OP_MUTEX_TRYLOCK, 0) == 0); + KernelTimespec failsafe = FailsafeTimeout(); + state.fetch_add(1); + Require(Umtx(&condition, UMTX_OP_CV_WAIT, 0, &own, &failsafe) == 0 && own.owner == 0); + state.fetch_add(10); + }); + } + AwaitCount(state, 2); + std::this_thread::sleep_for(SETTLE); + Require(condition.hasWaiters == 1); + Require(Umtx(&condition, UMTX_OP_CV_SIGNAL, 0) == 0); + AwaitCount(state, 12); + Require(condition.hasWaiters == 1); + Require(Umtx(&condition, UMTX_OP_CV_BROADCAST, 0) == 0); + AwaitCount(state, 22); + for (auto& waiter : waiters) waiter.join(); + Require(condition.hasWaiters == 0); +} + +static void RwlockStateFollowsTheConsole() { + Urwlock rwlock; + Require(Umtx(&rwlock, UMTX_OP_RW_RDLOCK, 0) == 0 && rwlock.state == 1); + UmtxTime timeout = ShortTimeout(); + Require(FailsWith(Umtx(&rwlock, UMTX_OP_RW_WRLOCK, 0, SizeOf(sizeof(timeout)), &timeout), GUEST_ETIMEDOUT)); + Require(rwlock.state == 1 && rwlock.blockedWriters == 0); + Require(Umtx(&rwlock, UMTX_OP_RW_RDLOCK, 0) == 0 && rwlock.state == 2); + Require(Umtx(&rwlock, UMTX_OP_RW_UNLOCK, 0) == 0 && Umtx(&rwlock, UMTX_OP_RW_UNLOCK, 0) == 0 && rwlock.state == 0); + + Require(Umtx(&rwlock, UMTX_OP_RW_WRLOCK, 0) == 0 && rwlock.state == WRITE_OWNER); + timeout = ShortTimeout(); + Require(FailsWith(Umtx(&rwlock, UMTX_OP_RW_RDLOCK, 0, SizeOf(sizeof(timeout)), &timeout), GUEST_ETIMEDOUT)); + Require(rwlock.state == (WRITE_OWNER | READ_WAITERS)); + Require(Umtx(&rwlock, UMTX_OP_RW_UNLOCK, 0) == 0 && rwlock.state == READ_WAITERS); + rwlock.state = 0; + Require(FailsWith(Umtx(&rwlock, UMTX_OP_RW_UNLOCK, 0), GUEST_EPERM) && rwlock.state == 0); + Require(FailsWith(Umtx(nullptr, UMTX_OP_RW_RDLOCK, 0), GUEST_EFAULT)); +} + +static void RwlockUnlockWakesTheBlockedWriter() { + Urwlock rwlock; + Require(Umtx(&rwlock, UMTX_OP_RW_RDLOCK, 0) == 0); + std::atomic state{0}; + std::thread writer([&] { + state.store(1); + Require(Umtx(&rwlock, UMTX_OP_RW_WRLOCK, 0) == 0); + state.store(2); + Require(Umtx(&rwlock, UMTX_OP_RW_UNLOCK, 0) == 0); + }); + AwaitCount(state, 1); + std::this_thread::sleep_for(SETTLE); + Require(state.load() == 1 && (std::atomic_ref(rwlock.state).load() & WRITE_WAITERS) != 0); + Require(Umtx(&rwlock, UMTX_OP_RW_UNLOCK, 0) == 0); + AwaitCount(state, 2); + writer.join(); + Require(rwlock.state == 0 && rwlock.blockedWriters == 0); +} + int main() { ReturnsWhenTheValueDiffers(); WaitComparesALongAndWaitUintAnInt(); @@ -367,4 +541,9 @@ int main() { SemWakeReleasesTheWaiterAndClearsTheFlag(); SemWakeWithoutWaitersSucceeds(); RejectsBadObjectsAndOperations(); + MutexOwnershipFollowsTheConsole(); + MutexLockWaitsForTheOwnerToUnlock(); + ConditionWaitReleasesTheMutexAndWakes(); + RwlockStateFollowsTheConsole(); + RwlockUnlockWakesTheBlockedWriter(); } diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 8a5f268de7..0713e1294a 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -79,8 +79,8 @@ - [Memory protection bits 0x100 and 0x200](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - taken as audio coprocessor (ACP) read and write, as in the [KytyPS5](https://github.com/KytyPS5/KytyPS5) and [etaHEN](https://github.com/etaHEN/etaHEN) headers; the coprocessor work runs on the host CPU, so they grant host read and read/write access like the GPU bits - [Protecting a host thread's stack](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - on Windows `sceKernelIsStack` reports a host thread's whole stack reservation, which Windows commits on demand from the top down behind a guard page. A protect over that range changes only the pages above the guard; the guard page and the reserved pages below it are left as they are, so the stack still grows and an overflow still raises a stack overflow. A protect that starts at the bottom of the reservation and stops short of the guard page, as HotSpot makes its stack guard zones, is committed and protected. Windows moves the guard page down by committing the pages below it as read/write guard pages, whether they were reserved or committed, so those pages come in read/write as the stack grows to within one guard region of them (three pages on GitHub's runners). A protection other than read/write on them, `PROT_NONE` included, does not hold once the stack is that close, where on the console every page of the reported range takes the protection. Holding it would mean removing the thread's guard page and keeping the stack limit in its TEB by hand. On Linux the reported range of the main thread is mostly unmapped; the unmapped pages from the bottom of a protect up to the top of the stack are mapped read/write in place, then the protect is applied, so the stack never has to grow down to the protected pages, which Linux refuses within its stack guard gap (1 MiB by default) above a readable mapping. The pages below a protect stay unmapped and out of the stack's reach, where on the console the stack goes on below them - [Foreign guest module image protection on Windows](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - `sceKernelMprotect` on Windows allows protection mutations on mapped PE images containing a `.elf0` section (as relinked guest modules produce, such as NVIDIA Real-time Denoisers `libNRD.prx` bundled in Dying Light 2: Stay Human PPSA02262), registering their sections into `GuestAllocations`. Host libraries and system DLLs (such as `libkernel.prx` and `kernel32.dll`) without a `.elf0` section continue to be rejected -- [_umtx_op](../../../core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp) (libkernel) - the mutex, condition variable, read-write lock, legacy lock and `SEM2` operations throw, and so does a misaligned object. Implemented as FreeBSD 11's [kern_umtx.c](https://github.com/freebsd/freebsd-src/blob/stable/11/sys/kern/kern_umtx.c): `UMTX_OP_WAIT`, `UMTX_OP_WAKE`, `UMTX_OP_WAIT_UINT`, `UMTX_OP_WAIT_UINT_PRIVATE`, `UMTX_OP_WAKE_PRIVATE`, `UMTX_OP_NWAKE_PRIVATE`, `UMTX_OP_SEM_WAIT` and `UMTX_OP_SEM_WAKE` (the last two built under `COMPAT_FREEBSD9`/`10`). Operations 0 and 1 fail with `EOPNOTSUPP`, as measured on a PS5 (firmware 12.70). The `SEM2` operations, 23 and 24, were measured on the console and returned `0` for a count of 0 where FreeBSD would time out, so they stay unimplemented. -- [sceKernelCreateSema, sceKernelPollSema, sceKernelSignalSema, sceKernelDeleteSema](../../../core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp) (libkernel) - a poll, a signal or a second delete of a deleted semaphore reads freed memory here, where the console returns `0x80020003` (ESRCH) ([measured](https://github.com/mattias800/prosper/blob/3ea5cb9a846b6dcd00f5073360462bbda4f72288/prosper/tests/data/console_oracle/kernel.golden.tsv#L17-L19)). A null output pointer to `sceKernelCreateSema` (a fault on PS5 firmware 13.60) throws. Other attributes above 2 and negative maximum and signal counts return `EINVAL` here, as there. `EINVAL` for an initial count above the maximum or negative, a maximum of 0, a null name, a poll of 0 or above the maximum and a signal of 0 is [measured](https://github.com/mattias800/prosper/blob/3ea5cb9a846b6dcd00f5073360462bbda4f72288/prosper/tests/data/console_oracle/kernel.golden.tsv#L10-L23). +- [_umtx_op](../../../core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp) (libkernel) - the legacy lock, `MUTEX_WAKE2`, `SEM2`, `SHM` and robust-list operations throw, as do robust umutexes, unlocking a priority-protected one and misaligned objects. The umutex, condition variable and rwlock operations follow a PS5 payload process (firmware 13.40): a mutex the caller owns is locked again without a count and freed by one unlock, a priority-protected one fails with `EINVAL`, a reader that times out leaves `URWLOCK_READ_WAITERS` set, and the owner is `scePthreadGetthreadid`. Priority inheritance is not applied; `CVWAIT_CHECK_UNPARKING` is ignored. Contention and wakeups otherwise follow FreeBSD 11's [kern_umtx.c](https://github.com/freebsd/freebsd-src/blob/stable/11/sys/kern/kern_umtx.c) (as do the wait, wake and `SEM` operations). Operations 0 and 1 fail with `EOPNOTSUPP` and 7 on other objects with `EINVAL`, as measured; 27 returns 0 on the console and `EINVAL` here. +- [sceKernelCreateSema, sceKernelPollSema, sceKernelSignalSema, sceKernelDeleteSema](../../../core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp) (libkernel) - a poll, a signal or a second delete of a deleted semaphore reads freed memory here, where the console returns `0x80020003` (ESRCH) ([measured](https://github.com/mattias800/prosper/blob/3ea5cb9a846b6dcd00f5073360462bbda4f72288/prosper/tests/data/console_oracle/kernel.golden.tsv#L17-L19)). A null output pointer to `sceKernelCreateSema` (a fault on PS5 firmware 13.60) and attributes 0x100 to 0x102 (accepted there, effect unknown) throw. Other attributes above 2 and negative maximum and signal counts return `EINVAL` here, as there. `EINVAL` for an initial count above the maximum or negative, a maximum of 0, a null name, a poll of 0 or above the maximum and a signal of 0 is [measured](https://github.com/mattias800/prosper/blob/3ea5cb9a846b6dcd00f5073360462bbda4f72288/prosper/tests/data/console_oracle/kernel.golden.tsv#L10-L23). - [sceKernelSyncOnAddressWait, sceKernelSyncOnAddressWait8/16/32/64 and sceKernelSyncOnAddressWake](../../../core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp) (libkernel) - follow a payload process on a PS5 (firmware 13.60), not a title. Only a null address is checked for `EFAULT`: any other unreadable address is read and faults the host, where the console returned `EFAULT` for address 0x10. The time left that a woken wait writes back is taken from the host clock, so it can differ from the console's by the cost of the call - [scePthreadRwlockDestroy, pthread_rwlock_destroy](../../../core/libs/prx/libkernel/Pthread/src/Rwlock.cpp) (libkernel) - measured on a PS5 (firmware 13.60): destroying a rwlock returns 0 whether it is free or held for reading or writing, and leaves 1 in the handle; every call on that handle then returns `EINVAL` until it is initialized again, and a zero handle is initialized on first use by the `scePthreadRwlock` calls as by the POSIX ones. A rwlock destroyed while held is not freed, so its memory leaks. Destroying a rwlock while another thread is acquiring it is not guarded - [scePthreadCondWait / scePthreadCondTimedwait](../../../core/libs/prx/libkernel/Pthread/src/Cond.cpp) (libkernel) - the mutex's owner is checked before waiting, as in FreeBSD libthr's [`cond_wait_common`](https://github.com/freebsd/freebsd-src/blob/main/lib/libthr/thread/thr_cond.c#L1992-L2029). Measured in a payload process on a PS5 (firmware 13.60): `EPERM` (`0x80020001`) for an unowned mutex of every type, the adaptive initializer (slot 1), an uninitialized slot and a mutex owned by another thread, `EINVAL` for the destroyed mutex (slot 2), and a recursive mutex is released completely and relocked to its depth. `scePthreadCondDestroy` stores 1, which every condition call refuses with `EINVAL`, and leaves an uninitialized slot alone. Cancellation restores the mutex and depth before cleanup and a canceled waiter forwards a wake, per POSIX; its ordering is not measured From 45a8b80a5ceb49746ee8f82655540c846789e5a4 Mon Sep 17 00:00:00 2001 From: Rookie <175737347+rooklz@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:45:01 +0000 Subject: [PATCH 258/318] fix(libkernel): honour URWLOCK_PREFER_READER passed with RW_RDLOCK as the console does --- .../SyncOnAddress/src/SyncOnAddress.cpp | 6 +++--- core/libs/tests/GuestUmtxOp.cpp | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp index 29ae3c6779..292a03a25b 100644 --- a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp +++ b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp @@ -644,7 +644,7 @@ void ClearRwlockBits(UserRwlock* rwlock, std::uint32_t bits) { std::atomic_ref(rwlock->state).fetch_and(~bits); } -int RwlockLock(UserRwlock* rwlock, bool writer, const void* sizeArgument, const void* timeArgument, const void* caller) { +int RwlockLock(UserRwlock* rwlock, bool writer, std::uint64_t operationFlags, const void* sizeArgument, const void* timeArgument, const void* caller) { if (const int error = CheckRwlock(rwlock); error != 0) { return error; } @@ -655,7 +655,7 @@ int RwlockLock(UserRwlock* rwlock, bool writer, const void* sizeArgument, const std::atomic_ref state(rwlock->state); std::atomic_ref blocked(writer ? rwlock->blockedWriters : rwlock->blockedReaders); const std::uint32_t waiterBit = writer ? URWLOCK_WRITE_WAITERS : URWLOCK_READ_WAITERS; - const bool preferReader = (std::atomic_ref(rwlock->flags).load() & URWLOCK_PREFER_READER) != 0; + const bool preferReader = ((std::atomic_ref(rwlock->flags).load() | operationFlags) & URWLOCK_PREFER_READER) != 0; const auto key = reinterpret_cast(writer ? &rwlock->blockedWriters : &rwlock->state); for (;;) { int error = 0; @@ -811,7 +811,7 @@ int APS5_VABI _umtx_op_nid_postfix(void* object, int operation, std::uint64_t va break; case UMTX_OP_RW_RDLOCK: case UMTX_OP_RW_WRLOCK: - error = RwlockLock(static_cast(object), operation == UMTX_OP_RW_WRLOCK, sizeArgument, timeArgument, caller); + error = RwlockLock(static_cast(object), operation == UMTX_OP_RW_WRLOCK, value, sizeArgument, timeArgument, caller); break; case UMTX_OP_RW_UNLOCK: error = RwlockUnlock(static_cast(object)); diff --git a/core/libs/tests/GuestUmtxOp.cpp b/core/libs/tests/GuestUmtxOp.cpp index 6c0ab5c485..f2e4abdf9c 100644 --- a/core/libs/tests/GuestUmtxOp.cpp +++ b/core/libs/tests/GuestUmtxOp.cpp @@ -49,6 +49,7 @@ static constexpr std::uint32_t PRIO_PROTECT = 0x8; static constexpr std::uint32_t WRITE_OWNER = 0x80000000u; static constexpr std::uint32_t WRITE_WAITERS = 0x40000000u; static constexpr std::uint32_t READ_WAITERS = 0x20000000u; +static constexpr std::uint32_t PREFER_READER = 0x2; struct Umutex { std::uint32_t owner = 0; @@ -507,6 +508,23 @@ static void RwlockStateFollowsTheConsole() { Require(FailsWith(Umtx(nullptr, UMTX_OP_RW_RDLOCK, 0), GUEST_EFAULT)); } +static void RwlockReaderPreferenceFollowsTheConsole() { + const auto readLock = [](std::uint32_t state, std::uint32_t flags, std::uint64_t value, std::uint32_t& after) { + Urwlock rwlock; + rwlock.state = state; + rwlock.flags = flags; + UmtxTime timeout = ShortTimeout(); + const int result = Umtx(&rwlock, UMTX_OP_RW_RDLOCK, value, SizeOf(sizeof(timeout)), &timeout); + after = rwlock.state; + return result; + }; + std::uint32_t after = 0; + Require(FailsWith(readLock(WRITE_WAITERS, 0, 0, after), GUEST_ETIMEDOUT) && after == (WRITE_WAITERS | READ_WAITERS)); + Require(readLock(WRITE_WAITERS, 0, PREFER_READER, after) == 0 && after == (WRITE_WAITERS | 1)); + Require(readLock(WRITE_WAITERS, PREFER_READER, 0, after) == 0 && after == (WRITE_WAITERS | 1)); + Require(FailsWith(readLock(WRITE_OWNER, 0, PREFER_READER, after), GUEST_ETIMEDOUT) && after == (WRITE_OWNER | READ_WAITERS)); +} + static void RwlockUnlockWakesTheBlockedWriter() { Urwlock rwlock; Require(Umtx(&rwlock, UMTX_OP_RW_RDLOCK, 0) == 0); @@ -546,4 +564,5 @@ int main() { ConditionWaitReleasesTheMutexAndWakes(); RwlockStateFollowsTheConsole(); RwlockUnlockWakesTheBlockedWriter(); + RwlockReaderPreferenceFollowsTheConsole(); } From 4ed0f9f911a96facd17f77d50e83b1ec04836c58 Mon Sep 17 00:00:00 2001 From: Rookie <175737347+rooklz@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:54:52 +0000 Subject: [PATCH 259/318] fix(libkernel): close a rwlock wakeup race and a dangling condvar waiter RwlockLock set its waiter bit unconditionally after deciding the lock was unheld, so an unlock landing in that window woke nobody and the waiter published its bit onto an already-free lock and slept forever. Set the bit with a CAS against the value just checked, retrying the freedom check on failure, as the mutex path already does for UMUTEX_CONTESTED. ConditionWait queues its stack waiter and drops the bucket lock before calling MutexUnlock; MutexUnlock throws for a priority-protected umutex, which unwound past the cleanup that removes the queue entry, leaving a dangling pointer for a later ConditionWake to use. Reject the unsupported mutex before the waiter is queued. Depends on: none --- .../SyncOnAddress/src/SyncOnAddress.cpp | 7 +++- core/libs/tests/GuestUmtxOp.cpp | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp index 292a03a25b..f954ea4cc1 100644 --- a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp +++ b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp @@ -555,6 +555,9 @@ int ConditionWait(UserCondition* condition, UserMutex* mutex, std::uint64_t flag if (const int error = CheckMutex(mutex); error != 0) { return error; } + if ((std::atomic_ref(mutex->flags).load() & UMUTEX_PRIO_PROTECT) != 0) { + NotImplemented_nid_no_patch("_umtx_op condition wait on a priority-protected umutex"); + } std::optional deadline; if (const int error = ConditionDeadline(condition, flags, timeArgument, deadline); error != 0) { return error; @@ -666,6 +669,9 @@ int RwlockLock(UserRwlock* rwlock, bool writer, std::uint64_t operationFlags, co const bool free = writer ? (current & (URWLOCK_WRITE_OWNER | URWLOCK_MAX_READERS)) == 0 : (current & URWLOCK_WRITE_OWNER) == 0 && (preferReader || (current & URWLOCK_WRITE_WAITERS) == 0); if (!free) { + if ((current & waiterBit) == 0 && !state.compare_exchange_weak(current, current | waiterBit)) { + continue; + } break; } if (!writer && (current & URWLOCK_MAX_READERS) == URWLOCK_MAX_READERS) { @@ -678,7 +684,6 @@ int RwlockLock(UserRwlock* rwlock, bool writer, std::uint64_t operationFlags, co return true; } } - state.fetch_or(waiterBit); blocked.fetch_add(1); return false; }, deadline, caller); diff --git a/core/libs/tests/GuestUmtxOp.cpp b/core/libs/tests/GuestUmtxOp.cpp index f2e4abdf9c..8c09d79442 100644 --- a/core/libs/tests/GuestUmtxOp.cpp +++ b/core/libs/tests/GuestUmtxOp.cpp @@ -544,6 +544,44 @@ static void RwlockUnlockWakesTheBlockedWriter() { Require(rwlock.state == 0 && rwlock.blockedWriters == 0); } +static void RwlockDoesNotLoseAWakeupUnderContention() { + Urwlock rwlock; + constexpr int threadCount = 4; + constexpr int rounds = 2000; + std::vector workers; + for (int index = 0; index < threadCount; ++index) { + workers.emplace_back([&, index] { + for (int round = 0; round < rounds; ++round) { + const bool write = (round % threadCount) == index % threadCount && (round % 4) == 0; + KernelTimespec failsafe = FailsafeTimeout(); + Require(Umtx(&rwlock, write ? UMTX_OP_RW_WRLOCK : UMTX_OP_RW_RDLOCK, 0, SizeOf(sizeof(failsafe)), &failsafe) == 0); + Require(Umtx(&rwlock, UMTX_OP_RW_UNLOCK, 0) == 0); + } + }); + } + for (auto& worker : workers) worker.join(); + Require(rwlock.state == 0 && rwlock.blockedReaders == 0 && rwlock.blockedWriters == 0); +} + +static void ConditionWaitRejectsAPriorityProtectedMutexWithoutLeakingAWaiter() { + Ucond condition; + Umutex protect; + protect.flags = PRIO_PROTECT; + KernelTimespec timeout{0, 1000000}; + bool threw = false; + try { + Umtx(&condition, UMTX_OP_CV_WAIT, 0, &protect, &timeout); + } catch (const std::runtime_error&) { + threw = true; + } + Require(threw && condition.hasWaiters == 0); + + Umutex mutex; + Require(Umtx(&mutex, UMTX_OP_MUTEX_TRYLOCK, 0) == 0); + Require(FailsWith(Umtx(&condition, UMTX_OP_CV_WAIT, 0, &mutex, &timeout), GUEST_ETIMEDOUT)); + Require(condition.hasWaiters == 0); +} + int main() { ReturnsWhenTheValueDiffers(); WaitComparesALongAndWaitUintAnInt(); @@ -565,4 +603,6 @@ int main() { RwlockStateFollowsTheConsole(); RwlockUnlockWakesTheBlockedWriter(); RwlockReaderPreferenceFollowsTheConsole(); + RwlockDoesNotLoseAWakeupUnderContention(); + ConditionWaitRejectsAPriorityProtectedMutexWithoutLeakingAWaiter(); } From 372f0f6c686dd13ac6fd61db2a0c01664a7058d4 Mon Sep 17 00:00:00 2001 From: Rookie <175737347+rooklz@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:17:42 +0000 Subject: [PATCH 260/318] fix(libkernel): close a second rwlock wakeup race and loosen a condvar test The last waiter to leave RwlockLock cleared the waiter bit outside the bucket lock. If a new waiter published that same bit and incremented the blocked count in the window between the decrement and the clear, the clear erased its bit and a later unlock woke nobody. Re-check the blocked count after clearing and restore the bit if a waiter appeared in that window. ConditionWaitRejectsAPriorityProtectedMutexWithoutLeakingAWaiter also required the priority-protected wait to throw; a condition wait on such a mutex is a valid, merely unsupported form, so only the queue cleanliness is asserted now. Extended the rwlock stress test to 8 threads and more, shorter rounds to give the window more chances to show up. Depends on: none --- .../prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp | 5 +++++ core/libs/tests/GuestUmtxOp.cpp | 10 ++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp index f954ea4cc1..63a1be53d4 100644 --- a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp +++ b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp @@ -693,6 +693,11 @@ int RwlockLock(UserRwlock* rwlock, bool writer, std::uint64_t operationFlags, co const std::uint32_t left = blocked.fetch_sub(1) - 1; if (left == 0 && (writer || woken)) { ClearRwlockBits(rwlock, waiterBit); + if (blocked.load() != 0) { + std::uint32_t current = state.load(); + while ((current & waiterBit) == 0 && !state.compare_exchange_weak(current, current | waiterBit)) { + } + } } if (!woken) { return GUEST_ETIMEDOUT; diff --git a/core/libs/tests/GuestUmtxOp.cpp b/core/libs/tests/GuestUmtxOp.cpp index 8c09d79442..a91732e33c 100644 --- a/core/libs/tests/GuestUmtxOp.cpp +++ b/core/libs/tests/GuestUmtxOp.cpp @@ -546,13 +546,13 @@ static void RwlockUnlockWakesTheBlockedWriter() { static void RwlockDoesNotLoseAWakeupUnderContention() { Urwlock rwlock; - constexpr int threadCount = 4; - constexpr int rounds = 2000; + constexpr int threadCount = 8; + constexpr int rounds = 6000; std::vector workers; for (int index = 0; index < threadCount; ++index) { workers.emplace_back([&, index] { for (int round = 0; round < rounds; ++round) { - const bool write = (round % threadCount) == index % threadCount && (round % 4) == 0; + const bool write = (round % threadCount) == index % threadCount; KernelTimespec failsafe = FailsafeTimeout(); Require(Umtx(&rwlock, write ? UMTX_OP_RW_WRLOCK : UMTX_OP_RW_RDLOCK, 0, SizeOf(sizeof(failsafe)), &failsafe) == 0); Require(Umtx(&rwlock, UMTX_OP_RW_UNLOCK, 0) == 0); @@ -568,13 +568,11 @@ static void ConditionWaitRejectsAPriorityProtectedMutexWithoutLeakingAWaiter() { Umutex protect; protect.flags = PRIO_PROTECT; KernelTimespec timeout{0, 1000000}; - bool threw = false; try { Umtx(&condition, UMTX_OP_CV_WAIT, 0, &protect, &timeout); } catch (const std::runtime_error&) { - threw = true; } - Require(threw && condition.hasWaiters == 0); + Require(condition.hasWaiters == 0); Umutex mutex; Require(Umtx(&mutex, UMTX_OP_MUTEX_TRYLOCK, 0) == 0); From 83b1015c4fb99e527b72f458848d0230094758a1 Mon Sep 17 00:00:00 2001 From: Rookie <175737347+rooklz@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:37:21 +0000 Subject: [PATCH 261/318] fix(libkernel): make the rwlock waiter cleanup atomic with the predicate The last waiter's fetch_sub, ClearRwlockBits and recheck ran outside the bucket mutex the predicate runs under, so a new waiter's bit CAS and blocked.fetch_add could still land inside that window: the clear erased a bit the new waiter had just set, and the recheck could see blocked == 0 before the new waiter's fetch_add became visible. Run the whole cleanup under BucketOf(key).mutex, the same lock the predicate holds for its CAS and fetch_add, so the two can no longer interleave. Depends on: none --- .../SyncOnAddress/src/SyncOnAddress.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp index 63a1be53d4..cacf067d0b 100644 --- a/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp +++ b/core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp @@ -690,12 +690,16 @@ int RwlockLock(UserRwlock* rwlock, bool writer, std::uint64_t operationFlags, co if (done) { return error; } - const std::uint32_t left = blocked.fetch_sub(1) - 1; - if (left == 0 && (writer || woken)) { - ClearRwlockBits(rwlock, waiterBit); - if (blocked.load() != 0) { - std::uint32_t current = state.load(); - while ((current & waiterBit) == 0 && !state.compare_exchange_weak(current, current | waiterBit)) { + std::uint32_t left; + { + std::lock_guard lock(BucketOf(key).mutex); + left = blocked.fetch_sub(1) - 1; + if (left == 0 && (writer || woken)) { + ClearRwlockBits(rwlock, waiterBit); + if (blocked.load() != 0) { + std::uint32_t current = state.load(); + while ((current & waiterBit) == 0 && !state.compare_exchange_weak(current, current | waiterBit)) { + } } } } From ee3fc5c7127ac538e22af8883948e41181d8eeec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C5=A0l=C3=A9gl?= Date: Fri, 23 Oct 2026 15:28:58 +0200 Subject: [PATCH 262/318] test(agc): cover repeated driver stores into a shared block's exact ring e424d57f9b (MarcZX10, #7233) made a driver store whose range contains earlier pieces of its ring take their slot and a full ring forget its oldest piece, for the exact and the possible ring. #6868 proposed that rule for the exact ring; this keeps its tests and its debt line and drops the code. The tests cover what main's do not: repeated exact stores of a neighbour's bytes leave the other surface's share of the block unstored and keep their own coverage, a containing store answers for the contained bytes only and frees the slots of the pieces it contains, a full exact ring forgets its oldest piece and not a newer one, the forgotten piece counts for the rest of the block, an overlapping store that contains nothing replaces nothing, and a storage image written back after five such stores in its last 64 KiB block keeps every texel over watched memory and over a watched host import. The debt line records what a forgotten piece costs for both rings. The recorder's --repeated-neighbour-stores-only mode returns 77 when no watched arm ran (no write watching); in the full suite each arm prints its own ok or not-tested line instead of one ok for both. Co-Authored-By: Claude Fable 5.1 --- .../prx/libSceAgcDriver/tests/Recorder.cpp | 116 ++++++++++++++++++ .../libSceAgcDriver/tests/WriteTracking.cpp | 45 +++++++ .../cmake/agc_driver_recorder_tests.cmake | 6 + docs/dev/debt/libSceAgcDriver.md | 1 + 4 files changed, 168 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index fd907a1040..96905100ac 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -7708,6 +7708,120 @@ void refreshProofTests(const Device& device, Recorder& recorder) { recorder.Sync(); } +bool repeatedNeighbourStoreTests(const Device& device, Recorder& recorder) { + const auto& base = device.GetContext(); + constexpr std::uint32_t width = 1024; + constexpr std::uint32_t height = 512; + constexpr std::size_t surfaceBytes = std::size_t{width} * height * 4; + constexpr std::uint64_t offset = 0x2000; + constexpr std::size_t neighbourBytes = 0x6000; + constexpr std::size_t bytes = 35 * 65536; + constexpr std::array red{255, 0, 0, 255}; + const auto pattern = [](std::size_t i) { return static_cast(i * 131u + 7u); }; + bool ran = false; + for (const bool imported : {false, true}) { +#ifdef _WIN32 + if (imported) { + std::cout << "repeated neighbour stores over a watched host import: Linux write watch only\n"; + continue; + } +#endif + if (imported && base.hostImportAlignment == 0) { + std::cout << "host imports unavailable: repeated neighbour stores over a watched host import not tested\n"; + continue; + } + const auto decided = PrepareImportWatch(base); + if (imported) SetImportWatch(base, ImportWatch::Watch); + struct Restore { + const Context& context; + ImportWatch decided; + ~Restore() { SetImportWatch(context, decided); } + } restore{base, decided}; + const std::string what = imported ? "over a watched host import" : "over watched memory"; + void* block = AllocateWatched(bytes, 65536); + if (block == nullptr) { + std::cout << "no write watching: repeated neighbour stores " << what << " not tested\n"; + continue; + } + auto* memory = static_cast(block); + for (std::size_t i = 0; i < bytes; ++i) memory[i] = pattern(i); + const auto blockAddress = reinterpret_cast(block); + const auto address = blockAddress + offset; + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + std::size_t bytes; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + ReleaseWatched(block, bytes); + } + } unregister{base, block, blockAddress, bytes}; + AgcDriver::GuestMemory::CollectWritesUncached(blockAddress, bytes); + TextureDetiler detiler(base); + auto context = base; + if (!imported) context.hostImportAlignment = 0; + context.detiler = &detiler; + GuestTextureResource resource{}; + resource.baseAddress = address; + resource.width = width; + resource.height = height; + resource.mipCount = 1; + resource.tileMode = TextureTileMode::kLinear; + resource.dimension = TextureDimension::k2D; + resource.format = 56; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + Require(DescribeSurface(resource).guestBytes == surfaceBytes, "(n) unexpected surface size"); + { + auto image = StorageTexture::Create(context, detiler, resource, 0); + image->Refresh(); + Require(image->Refresh(), "(n) " + what + ": the image does not settle after its upload"); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + const auto commands = recorder.Commands(); + recorder.Keep(image); + const VkClearColorValue value{{1.0f, 0.0f, 0.0f, 1.0f}}; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + for (int store = 0; store < 5; ++store) AgcDriver::GuestMemory::MarkWritten(address + surfaceBytes, neighbourBytes); + Require(AgcDriver::GuestMemory::Watched(address, surfaceBytes) && (HostImportFor(context, address, surfaceBytes) != nullptr) == imported, "(n) " + what + ": the image is not on the path under test"); + image->WriteBack(); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + std::vector read(bytes); + AgcDriver::GuestMemory::Read(blockAddress, read); + std::size_t dropped = 0; + for (std::size_t at = offset; at < offset + surfaceBytes; at += 4) { + for (std::size_t c = 0; c < 4; ++c) { + if (std::to_integer(read[at + c]) != red[c]) { + ++dropped; + break; + } + } + } + if (dropped != 0) throw std::runtime_error("repeated neighbour stores " + what + ": " + std::to_string(dropped) + " of " + std::to_string(surfaceBytes / 4) + " texels the GPU wrote were not stored"); + for (std::size_t at = offset + surfaceBytes; at < offset + surfaceBytes + neighbourBytes; ++at) Require(std::to_integer(read[at]) == pattern(at), "(n) " + what + ": the write-back stored over the neighbour's bytes"); + } + recorder.Sync(); + std::cout << "repeated neighbour stores " << what << ": ok\n"; + ran = true; + } + return ran; +} + void unchangedCpuStampTests(const Device& device, Recorder& recorder) { const auto& base = device.GetContext(); constexpr std::uint32_t side = 256; @@ -12449,6 +12563,7 @@ int main(int argc, char** argv) { idleWriteBackRaceTests(device, recorder); return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--repeated-neighbour-stores-only") return repeatedNeighbourStoreTests(device, recorder) ? 0 : 77; if (argc == 2 && std::string_view(argv[1]) == "--kept-unit-settle-only") { keptUnitSettleTests(device, recorder); return 0; @@ -12759,6 +12874,7 @@ int main(int argc, char** argv) { singlePassTests(device, recorder, false); singlePassTests(device, recorder, true); unchangedCpuStampTests(device, recorder); + repeatedNeighbourStoreTests(device, recorder); keptUnitSettleTests(device, recorder); settledBaselineTests(device, recorder); importWatchTests(device); diff --git a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp index 755dc8c40d..be7dfaee63 100644 --- a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp @@ -201,6 +201,50 @@ void CheckStoredOverUsesPageEvidence() { Require(!StoredOver(base + Page, Page, before), "a page-local CPU write made an untouched page in the same block count as stored over"); } +void CheckRepeatedNeighbourStores() { + void* memory = AllocateWatched(3 * Block); + const auto base = reinterpret_cast(memory); + const auto boundary = base + 2 * Block - 0x9000; + const auto sharedFirst = boundary - (base + Block); + const auto neighbourBytes = static_cast(base + 3 * Block - boundary); + std::memset(memory, 0x11, 3 * Block); + + const auto synced = CollectWrites(base, 3 * Block); + Require(synced != 0, "the test block is not collected"); + for (int store = 0; store < 6; ++store) MarkWritten(boundary, neighbourBytes); + Require(!StoredOver(base + Block, static_cast(sharedFirst), synced), "repeated stores of the neighbour's bytes count for the first surface's part of the shared block"); + Require(StoredAllSince_nid_no_patch(boundary, neighbourBytes, synced), "repeated stores of the neighbour's bytes forget that they covered them"); + + const auto pieces = TrackerGeneration(); + for (int store = 0; store < 3; ++store) MarkWritten(boundary + static_cast(store) * 0x100, 4); + MarkWritten(boundary, 0x400); + Require(StoredOver(boundary + 0x200, 4, pieces) && !StoredOver(boundary + 0x400, 4, pieces), "a store over earlier pieces of the block answers for other bytes than its own"); + MarkWritten(boundary + 0x800, 4); + MarkWritten(boundary + 0x900, 4); + Require(!StoredOver(base + Block, static_cast(sharedFirst), pieces), "a store over earlier pieces counts for the first surface's part of the shared block"); + + void* fresh = AllocateWatched(2 * Block); + const auto freshBase = reinterpret_cast(fresh); + std::memset(fresh, 0x11, 2 * Block); + const auto freshSynced = CollectWrites(freshBase, 2 * Block); + const auto neighbour = freshBase + Block - 0x9000; + MarkWritten(neighbour, 0x100); + const auto secondStored = MarkWritten(neighbour + 0x1000, 0x100); + const auto thirdStored = MarkWritten(neighbour + 0x2000, 0x100); + MarkWritten(neighbour + 0x3000, 0x100); + const auto settled = TrackerGeneration(); + MarkWritten(neighbour, 0x100); + MarkWritten(neighbour, 0x100); + MarkWritten(neighbour + 0x5000, 0x100); + Require(!StoredOver(freshBase, static_cast(neighbour - freshBase), secondStored), "a full exact ring evicted a newer piece instead of its oldest"); + Require(StoredOver(freshBase, static_cast(neighbour - freshBase), freshSynced), "a forgotten store is not taken as a store over the rest of the block"); + + const auto partial = TrackerGeneration(); + MarkWritten(neighbour + 0x5080, 0x100); + Require(StoredOver(neighbour + 0x5000, 0x10, settled) && StoredOver(neighbour + 0x5100, 0x10, partial), "a store that overlaps an earlier piece without containing it replaces it"); + Require(!StoredOver(freshBase, static_cast(neighbour - freshBase), thirdStored), "a store that overlaps without containing evicted a newer piece instead of the oldest"); +} + void CheckStoredAllSince() { constexpr std::size_t bytes = 4 * Block; auto* memory = static_cast(AllocateWatched(bytes)); @@ -2173,6 +2217,7 @@ int main() { #endif CheckSharedBlock(); CheckStoredOverUsesPageEvidence(); + CheckRepeatedNeighbourStores(); CheckStoredAllSince(); CheckSharedBlockProofs(); CheckRepeatedPossibleStores(); diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorder_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorder_tests.cmake index a238e74936..b4e3b66515 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorder_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorder_tests.cmake @@ -150,6 +150,12 @@ if(BUILD_TESTING) set_tests_properties(agc_partial_watch_restoration PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_repeated_neighbour_stores COMMAND agc_driver_recorder_tests --repeated-neighbour-stores-only) + set_tests_properties(agc_repeated_neighbour_stores PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_repeated_neighbour_stores PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test(NAME agc_storage_kept_unit_settle COMMAND agc_driver_recorder_tests --kept-unit-settle-only) add_test(NAME agc_storage_idle_writeback_race COMMAND agc_driver_recorder_tests --idle-writeback-race-only) set_tests_properties(agc_storage_idle_writeback_race PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60) diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index b0a8e0d7b1..5054f71048 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -176,6 +176,7 @@ - [`APS5_SKIP_REFUSED_DRAWS`](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp) (libSceAgcDriver) drops a draw whose graphics stage the recompiler refuses instead of failing the queue, so the frame lacks that draw, and the stage is compiled and refused again at each such draw. Compute refusals still fail. - [Pending storage-image results over memory outside the write watch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) leave to the CPU the 64 KiB blocks whose bytes differ from the ones saved at the upload, the fill, the image copy, the previous store or the moment their range left the watch (`Unwatch`, or a host import that does not stay watched or whose page walk fails), and the blocks the watch saw the CPU write before that moment. A store made on the GPU saves its bytes through a host-visible copy that becomes the saved bytes when its batch completes; a store or a refresh that needs them earlier waits for that batch. A unit without a write generation saves the guest bytes when results land in it outside the watch. One whose range leaves the watch while its results are pending saves its bytes through the unwatch hook, which also runs when a failed collect takes a range out of the watch. A range that leaves the watch without the hook running over it leaves its pending results without saved bytes; storing such results throws. A unit whose results were stored while its blocks were watched keeps only a generation; when its blocks leave the watch while nothing is pending, the hook saves nothing, and results that land in it afterwards have neither saved bytes nor a watch, so storing them throws too. - [Storage images over uncommitted guest memory](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) - a storage image whose guest range has no committed page when it is created throws in the `StorageTexture` constructor, which aborts the guest. Several titles reach this intermittently in runs of a few minutes, on AMD and NVIDIA alike; in two of the three logs, lines about storage images over released or rewritten memory (`memory no longer registered`, `guest memory changed while GPU results were pending`) come shortly before it. The log does not name the image's address. What the console does with a storage descriptor over memory that is not committed is unknown, and which allocation the guest recreates there was not found. +- [Driver-store pieces of a 64 KiB block](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) (libSceAgcDriver) forget a block's partial driver stores past four per ring: a store or possible write that finds its ring's four pieces taken evicts the oldest, and the block then counts as stored over in every range for any generation older than the evicted piece: `StoredOver` answers true, and for an evicted exact piece `StoredAllSince` answers false as well. A storage image's write-back then leaves a block it shares with other data to the CPU unless the bytes there still equal a copy the image kept (none above 1 MiB over a watched host import), losing the GPU's results in it; an exact eviction in an HTILE block also makes a native multisample depth clear refuse with "no complete tracked HTILE store" (DepthSurface.cpp). Only stores that contain none of the kept pieces fill a ring; keeping the evicted ranges' extent instead of one generation per ring is not implemented. - Windows host-pointer imports of the guest mapping itself are compared instead of write-watched, except writable imports of private `MEM_WRITE_WATCH` memory when the import probe keeps the watch and a settle queue exists: those of 1 MiB or more stay watched, smaller ones return to the watch after their settle unless a storage image overlaps them. Shared memory imported through a separate writable alias keeps protection-based tracking on its guest views. [GuestMemory](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes the 4 KiB pages an import overlaps, each with the generation of its latest exclusion; a fresh private mapping restores the complete pages it covers, protection changes and shared remappings do not, nor does retiring an import (unimplemented). Blocks with excluded pages keep their write-watch marks as a hint only (`UntrackedHint`): byte comparisons keep CPU edits, but snapshot reuse lasts one collect epoch and unit shadows are off. - [Pages under an import window](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) whose protection an import resolves count as stored only when their bytes change. A same-value CPU store during that window cannot be distinguished from driver activity and does not win over pending results. On Windows, NVIDIA 617.42 marks every private MEM_WRITE_WATCH page after import until a GPU use completes. The driver behavior was measured on this driver; other drivers are unverified. Imports of at least 1 MiB wait for a four-byte read on the separate settle queue inside the window. Smaller imports stay compared until a deferred read completes; a later lookup returns their pages to write watching, stamping their blocks as CPU stores; the block's other pages keep their state. Blocks overlapping a live storage image, blocks of an import retired before rewatching, and imports without a settle queue stay compared. Shared-memory imports retain their separate-alias tracking. - [Deferred host-import reads](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) use the main queue with barriers ordering earlier and later writes around the reads. Submission does not wait on the CPU. An initial multisampled DCC import needing a range whose import still awaits its read submits it if needed and waits under the GPU mutex (`WatchedAfterSettle`, #9724). Imports of partial 64 KiB blocks are read the same way; that read or one on demand returns their pages as CPU stores unless a live, trimmed or parked compared import or storage image shares their blocks; one a recorder holds is unseen. The latency cost of placing batches behind main-queue work is unmeasured. The first-use test now checks that earlier main-queue reads finish before the import returns to write watching; the original two-queue canary measured driver marking while those reads were still in flight. The earlier-import canary still checks later use of another import of the same pages. From b279c08e3594efc094078bdeb29b7a604a270241 Mon Sep 17 00:00:00 2001 From: milescoviello Date: Sat, 24 Oct 2026 22:24:37 +0000 Subject: [PATCH 263/318] perf(agc): take the texture detiler's Vulkan entry points from the device table Each detile, repack, image and CMASK dispatch looked up its descriptor update, binds, push constants and dispatch by name through vkGetDeviceProcAddr, and every descriptor set allocation did the same. NVIDIA's lookup compares the name against its entry points, so in Forspoken those lookups were about a fifth of the process's CPU time, on the draw committer while it creates textures. The calls now go through the device's function table, as other recording paths do, and fall back to a lookup when the table is absent. Co-Authored-By: Claude Opus 5.5 --- .../Graphics/src/TextureDetiler.cpp | 44 +++++++++---------- .../src/TextureDetilerDescriptors.cpp | 2 +- .../libSceAgcDriver/tests/TextureDetiler.cpp | 20 +++++++++ 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index 3c35882958..35fb2aeadb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -256,9 +256,9 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode writes[1].descriptorCount = 1; writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; writes[1].pBufferInfo = &destinationInfo; - context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); - context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, 0, nullptr); + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); + context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, 0, nullptr); Push push{}; push.srcBase = static_cast(sourceBase); push.dstBase = static_cast(destinationBase); @@ -279,10 +279,10 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode push.rowBegin = window.rowBegin; push.pipeBankXor = window.pipeBankXor ^ layerBankXor(tileMode, elementBytes, layout, slice); push.splitStride = layout.gcnSplitStride; - context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; - context.Function("vkCmdDispatch")(commands, groupsX, groupsY, 1); + context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, groupsX, groupsY, 1); } void TextureDetiler::DispatchRepack(VkCommandBuffer commands, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, std::uint64_t bytes) { @@ -322,12 +322,12 @@ void TextureDetiler::DispatchRepack(VkCommandBuffer commands, VkBuffer source, s writes[index].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; writes[index].pBufferInfo = index == 0 ? &sourceInfo : &destinationInfo; } - context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); - context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, repackPipeline); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, 0, nullptr); + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, repackPipeline); + context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, 0, nullptr); const std::array push{static_cast(sourceOffset - sourceDescriptorOffset), static_cast(destinationOffset - destinationDescriptorOffset), static_cast(words), static_cast(std::min(words, rowWords))}; - context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); - context.Function("vkCmdDispatch")(commands, static_cast((std::min(words, rowWords) + 63u) / 64u), static_cast(rows), 1); + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, static_cast((std::min(words, rowWords) + 63u) / 64u), static_cast(rows), 1); } std::uint32_t TextureDetiler::CmaskErrors() { @@ -395,24 +395,24 @@ void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask writes[0].pBufferInfo = &cmaskInfo; writes[1].pBufferInfo = &errorInfo; writes[2].pImageInfo = &imageInfo; - context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); const auto barrier = [&](VkPipelineStageFlags source, VkPipelineStageFlags destination, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { VkMemoryBarrier memory{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; memory.srcAccessMask = sourceAccess; memory.dstAccessMask = destinationAccess; - context.Function("vkCmdPipelineBarrier")(commands, source, destination, 0, 1, &memory, 0, nullptr, 0, nullptr); + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, source, destination, 0, 1, &memory, 0, nullptr, 0, nullptr); }; constexpr VkAccessFlags anyWrite = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT; constexpr VkAccessFlags anyAccess = anyWrite | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_HOST_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT; barrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, anyWrite, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipeline); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipelineLayout, 0, 1, &set, 0, nullptr); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipeline); + context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipelineLayout, 0, 1, &set, 0, nullptr); const std::uint32_t mask = elementBytes >= 4 ? 0xffffffffu : (1u << (elementBytes * 8u)) - 1u; const std::array push{static_cast(base), width, height, (width + 1023u) / 1024u, clearWords[0] & mask, elementBytes == 8 ? clearWords[1] : 0u, write ? 1u : 0u}; - context.Function("vkCmdPushConstants")(commands, cmaskPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); - context.Function("vkCmdDispatch")(commands, (width + 7u) / 8u, (height + 7u) / 8u, 1); + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, cmaskPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, (width + 7u) / 8u, (height + 7u) / 8u, 1); barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT, anyAccess); - context.Function("vkCmdFillBuffer")(commands, cmask, cmaskOffset, cmaskBytes, 0xffffffffu); + context.Resolved(&DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer")(commands, cmask, cmaskOffset, cmaskBytes, 0xffffffffu); barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, anyAccess); } @@ -447,9 +447,9 @@ void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode til writes[1].descriptorCount = 1; writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; writes[1].pImageInfo = &imageInfo; - context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); - context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0, 1, &set, 0, nullptr); + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); + context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0, 1, &set, 0, nullptr); Push push{}; push.srcBase = static_cast(retile ? 0 : base); push.dstBase = static_cast(retile ? base : 0); @@ -470,10 +470,10 @@ void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode til push.rowBegin = window.rowBegin; push.pipeBankXor = window.pipeBankXor ^ layerBankXor(tileMode, elementBytes, layout, slice); push.splitStride = layout.gcnSplitStride; - context.Function("vkCmdPushConstants")(commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; - context.Function("vkCmdDispatch")(commands, groupsX, groupsY, 1); + context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, groupsX, groupsY, 1); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp index 5f78f5729c..4f6d535f67 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp @@ -58,7 +58,7 @@ VkDescriptorSet TextureDetiler::allocateFrom(DescriptorRing& ring, VkDescriptorS allocation.descriptorSetCount = 1; allocation.pSetLayouts = &layout; VkDescriptorSet set = VK_NULL_HANDLE; - Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &set), allocateWhat); + Check(context.Resolved(&DeviceFunctions::allocateDescriptorSets, "vkAllocateDescriptorSets")(context.device, &allocation, &set), allocateWhat); ++ring.used; ring.serials[ring.current] = recorder != nullptr ? recorder->OpenSerial() : std::numeric_limits::max(); return set; diff --git a/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp index c773ddb446..5076c91a8d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp @@ -209,6 +209,26 @@ void RunTextureDetilerTests(const Context& context, const TextureDetilerTestAcce Require(access.descriptorPoolCount() == pools, "a full descriptor pool whose batches completed must be reset and reused, not joined by a new pool"); } + { + DeviceFunctions table{}; + table.updateDescriptorSets = context.Function("vkUpdateDescriptorSets"); + table.allocateDescriptorSets = context.Function("vkAllocateDescriptorSets"); + table.cmdBindPipeline = context.Function("vkCmdBindPipeline"); + table.cmdBindDescriptorSets = context.Function("vkCmdBindDescriptorSets"); + table.cmdPushConstants = context.Function("vkCmdPushConstants"); + table.cmdDispatch = context.Function("vkCmdDispatch"); + auto tabled = context; + tabled.functions = &table; + TextureDetiler cached(tabled); + cached.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 20, destination, 40, layout, 0); + const auto before = DeviceProcLookups(); + cached.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 20, destination, 40, layout, 0); + const auto lookups = DeviceProcLookups() - before; + Require(lookups == 0, "a detile dispatch looked up " + std::to_string(lookups) + " Vulkan functions by name while the device's function table holds them"); + const auto tabledCapture = access.lastDispatch(); + Require(tabledCapture.groupsX == 3 && tabledCapture.groupsY == 2 && tabledCapture.groupsZ == 1, "a detile dispatch through the device's function table recorded the wrong group counts"); + } + reject([&] { detiler.Dispatch(VK_NULL_HANDLE, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, layout, 0); }, "active command buffer"); reject([&] { detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, VK_NULL_HANDLE, 0, destination, 0, layout, 0); }, "source and destination buffers"); reject([&] { detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 0, VK_NULL_HANDLE, 0, layout, 0); }, "source and destination buffers"); From 1db99ec7bd397b2857ab8dbf52f0067bc8604ab3 Mon Sep 17 00:00:00 2001 From: milescoviello Date: Mon, 26 Oct 2026 03:02:04 +0000 Subject: [PATCH 264/318] fix(agc): keep the device lookup count in the driver library DeviceProcLookups() was an inline function with a thread_local in Context.hpp, so a test executable and libSceAgcDriver each had their own counter on Windows: the detiler test read its own copy and could not see the lookups the library made. The counter is now defined in the library. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp | 5 +---- .../prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp | 5 +++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index 985840aef9..7af85f1777 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -144,10 +144,7 @@ struct DeviceFunctions { }; // Per-thread count of vkGetDeviceProcAddr lookups made through Context::Function (the [vk] line). -inline std::uint64_t& DeviceProcLookups() { - thread_local std::uint64_t count = 0; - return count; -} +std::uint64_t& DeviceProcLookups(); inline constexpr std::size_t EmptyBufferBytes = 16; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 186aea757a..3c8dc742c5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -5157,6 +5157,11 @@ std::uint32_t SamplerTableCapacityFor(const Context& context, bool updateAfterBi } +std::uint64_t& DeviceProcLookups() { + thread_local std::uint64_t count = 0; + return count; +} + std::uint32_t ImageTableCapacity(const Context& context) { const auto capacity = ImageTableCapacityFor(context, false); return capacity != 0u ? capacity : ImageTableCapacityFor(context, true); From 2781d72d189afdb8f314212c01ae803b1ec3e51b Mon Sep 17 00:00:00 2001 From: milescoviello Date: Thu, 29 Oct 2026 13:03:26 +0000 Subject: [PATCH 265/318] fix(agc): link the pipeline cache allocation tests with the driver library for its device lookup count Co-Authored-By: Claude Opus 5.5 --- .../tests/cmake/agc_pipeline_cache_allocation_tests.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_pipeline_cache_allocation_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_pipeline_cache_allocation_tests.cmake index ef7a969e69..c8d961b5e8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_pipeline_cache_allocation_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_pipeline_cache_allocation_tests.cmake @@ -1,7 +1,7 @@ add_test_executable(agc_pipeline_cache_allocation_tests EXCLUDE_FROM_ALL tests/PipelineCacheFailures.cpp Graphics/src/PipelineCache.cpp tests/AllocationFailure.cpp) target_include_directories(agc_pipeline_cache_allocation_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) -target_link_libraries(agc_pipeline_cache_allocation_tests PRIVATE shader_recompiler_agc_driver libc) +target_link_libraries(agc_pipeline_cache_allocation_tests PRIVATE shader_recompiler_agc_driver libSceAgcDriver libc) set_target_properties(agc_pipeline_cache_allocation_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_pipeline_cache_allocation_tests) From da828bc3e564e14c8aabc1176a964edc71c2497f Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Sun, 1 Nov 2026 02:53:10 +0530 Subject: [PATCH 266/318] feat(agc): implement sceAgcUpdateInterpolantMapping_0100 --- .../prx/libSceAgc/Misc/src/ShaderFusion.cpp | 752 +++++++++--------- .../Shader/include/InterpolantMapping.hpp | 1 + .../Shader/src/InterpolantMapping.cpp | 324 ++++---- .../libSceAgc/tests/InterpolantMapping.cpp | 50 ++ 4 files changed, 604 insertions(+), 523 deletions(-) diff --git a/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp b/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp index a8fdedf664..587f243721 100644 --- a/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp +++ b/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp @@ -1,380 +1,372 @@ -#include "prx/libSceAgc/Misc/include/ShaderFusion.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" -#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" -#include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" -#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" - -namespace { - -using namespace ShaderRegs; - -constexpr int GRAPHICS5_ERROR_INVALID_SHADER_HALVES = static_cast(0x8a6c0008u); -constexpr std::uint32_t SPI_SHADER_PGM_CHKSUM_GS = 0x080u; -constexpr std::uint32_t SPI_SHADER_PGM_RSRC1_GS = 0x08Au; -constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_GS = 0x08Bu; -constexpr std::uint32_t SPI_SHADER_PGM_CHKSUM_HS = 0x100u; -constexpr std::uint32_t SPI_SHADER_PGM_RSRC1_HS = 0x10Au; -constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_HS = 0x10Bu; -constexpr std::uint32_t SCodeEnd = 0xbf9f0000u; -constexpr std::uint32_t SNop = 0xbf800000u; -constexpr std::uint32_t SSetpcS6 = 0xbe802006u; -constexpr std::uint32_t SSwappcNullS6 = 0xbefd2106u; -constexpr std::size_t FusedCodeAlignment = 256; -constexpr char TrailerMagic[8] = {'b', 'a', 'r', 'e', 'f', 'o', 'o', 't'}; -constexpr std::size_t TrailerCodeBytesOffset = sizeof(TrailerMagic) + 3 * sizeof(std::uint32_t); -constexpr std::uint32_t Rsrc1VgprMask = 0x3fu; -constexpr std::uint32_t Rsrc2OffchipLdsEnable = 1u << 18u; -constexpr std::uint32_t Rsrc2UserSgprMask = (0x1fu << 1u) | (1u << 27u); - -bool ValidHalves(const Shader* front, const Shader* back) { - const auto frontType = static_cast(front->type); - const auto backType = static_cast(back->type); - return (frontType == ShaderBinaryType::GsFront && backType == ShaderBinaryType::GsBack) || - (frontType == ShaderBinaryType::HsFront && backType == ShaderBinaryType::HsBack); -} - -bool GeometryHalves(const Shader* front) { - return static_cast(front->type) == ShaderBinaryType::GsFront; -} - -ShaderRegister& FindRegister(ShaderRegister* regs, std::uint32_t count, std::uint32_t offset, std::uint32_t occurrence = 0) { - for (std::uint32_t i = 0; regs != nullptr && i < count; ++i) { - if (regs[i].offset == offset && occurrence-- == 0) return regs[i]; - } - throw std::runtime_error("sceAgcFuseShaderHalves_0200: shader half lacks register " + std::to_string(offset)); -} - -void MergeMax(ShaderRegister& dst, const ShaderRegister& src, std::uint32_t shift, std::uint32_t mask) { - const auto field = std::max((dst.value >> shift) & mask, (src.value >> shift) & mask); - dst.value = (dst.value & ~(mask << shift)) | (field << shift); -} - -struct GeometryLayout { - std::uint32_t frontBytes; - std::size_t backOffset; - std::size_t codeBytes; - std::vector sh; - std::vector cx; - std::size_t headerOffset; - std::size_t shOffset; - std::size_t cxOffset; - std::size_t specialsOffset; - std::size_t inputsOffset; - std::size_t outputsOffset; - std::size_t userDataOffset; - std::size_t directOffset; - std::array sharpOffsets; - std::size_t totalBytes; -}; - -[[noreturn]] void Fail(const char* function, const std::string& message) { - throw std::runtime_error(std::string(function) + ": " + message); -} - -std::size_t AlignUp(std::size_t value, std::size_t alignment) { - return (value + alignment - 1) & ~(alignment - 1); -} - -void ValidateGeometryHalf(const char* function, const Shader* shader) { - if (shader->file_header != SHADER_FILE_HEADER_MAGIC || shader->version != SHADER_VERSION) Fail(function, "invalid shader header or version"); - if (shader->code == nullptr || shader->shader_size == 0 || (shader->shader_size & 3u) != 0) Fail(function, "invalid shader half code"); - if (shader->embedded_constant_buffer_size_dqw != 0) Fail(function, "fusing halves with an embedded constant buffer is not implemented"); - if ((shader->num_sh_registers != 0 && shader->sh_registers == nullptr) || (shader->num_cx_registers != 0 && shader->cx_registers == nullptr)) Fail(function, "shader half registers missing"); -} - -std::uint32_t FrontProgramBytes(const char* function, const Shader* front) { - const auto* bytes = static_cast(const_cast(front->code)); - const std::size_t size = front->shader_size; - if (size < TrailerCodeBytesOffset + sizeof(std::uint32_t)) Fail(function, "front half has no program trailer"); - std::size_t trailer = size; - for (std::size_t offset = size - TrailerCodeBytesOffset - sizeof(std::uint32_t) + 1; offset-- > 0;) { - if (std::memcmp(bytes + offset, TrailerMagic, sizeof(TrailerMagic)) == 0) { - trailer = offset; - break; - } - } - if (trailer == size) Fail(function, "front half has no program trailer"); - std::uint32_t codeBytes = 0; - std::memcpy(&codeBytes, bytes + trailer + TrailerCodeBytesOffset, sizeof(codeBytes)); - if (codeBytes == 0 || (codeBytes & 3u) != 0 || codeBytes > trailer) Fail(function, "front half trailer has an invalid code size"); - std::size_t end = codeBytes / 4; - const auto word = [&](std::size_t index) { - std::uint32_t value = 0; - std::memcpy(&value, bytes + index * 4, sizeof(value)); - return value; - }; - while (end > 0 && word(end - 1) == SCodeEnd) --end; - if (end == 0 || (word(end - 1) != SSetpcS6 && word(end - 1) != SSwappcNullS6)) Fail(function, "front half does not end with s_setpc_b64 s[6:7] or s_swappc_b64 null, s[6:7]"); - return static_cast((end - 1) * 4); -} - -std::array mergeGeometryResources(std::uint32_t frontRsrc1, std::uint32_t frontRsrc2, std::uint32_t backRsrc1, std::uint32_t backRsrc2) { - const auto frontVgprs = ((frontRsrc1 & Rsrc1VgprMask) + 1u) * 4u; - const auto backVgprs = ((backRsrc1 & Rsrc1VgprMask) + 1u) * 4u; - const auto frontTotal = frontVgprs + (frontRsrc2 >> 28u) * 8u; - const auto backTotal = backVgprs + (backRsrc2 >> 28u) * 8u; - const auto maxTotal = std::max(frontTotal, backTotal); - const auto shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u; - const auto vgprs = std::max(frontRsrc1 & Rsrc1VgprMask, backRsrc1 & Rsrc1VgprMask); - const auto rsrc1Field = std::max((frontRsrc1 >> 29u) & 3u, (backRsrc1 >> 29u) & 3u); - const auto rsrc2Field = std::max((frontRsrc2 >> 16u) & 3u, (backRsrc2 >> 16u) & 3u); - const auto rsrc1 = (backRsrc1 & ~(Rsrc1VgprMask | (3u << 29u))) | vgprs | (rsrc1Field << 29u); - const auto rsrc2 = (backRsrc2 & ~(0xf0000000u | (3u << 16u) | Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)) | ((shared & 15u) << 28u) | (rsrc2Field << 16u) | (frontRsrc2 & (Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)); - return {rsrc1, rsrc2}; -} - -const ShaderRegister* FindHalfRegister(const Shader* shader, std::uint32_t offset) { - for (std::uint32_t i = 0; i < shader->num_sh_registers; ++i) { - if (shader->sh_registers[i].offset == offset) return &shader->sh_registers[i]; - } - return nullptr; -} - -GeometryLayout ComputeGeometryLayout(const char* function, const Shader* front, const Shader* back) { - ValidateGeometryHalf(function, back); - ValidateGeometryHalf(function, front); - GeometryLayout layout{}; - layout.frontBytes = FrontProgramBytes(function, front); - layout.backOffset = AlignUp(layout.frontBytes, FusedCodeAlignment); - layout.codeBytes = layout.backOffset + back->shader_size; - const auto* frontRsrc1 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC1_GS); - const auto* frontRsrc2 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC2_GS); - if (frontRsrc1 == nullptr || frontRsrc2 == nullptr) Fail(function, "front half has no GS resource registers"); - const auto* backRsrc1 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC1_GS); - const auto* backRsrc2 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC2_GS); - if (backRsrc1 == nullptr || backRsrc2 == nullptr) Fail(function, "back half has no GS resource registers"); - const auto resources = mergeGeometryResources(frontRsrc1->value, frontRsrc2->value, backRsrc1->value, backRsrc2->value); - layout.sh.assign(back->sh_registers, back->sh_registers + back->num_sh_registers); - for (auto& reg : layout.sh) { - if (reg.offset == SPI_SHADER_PGM_RSRC1_GS) reg.value = resources[0]; - else if (reg.offset == SPI_SHADER_PGM_RSRC2_GS) reg.value = resources[1]; - } - for (std::uint32_t i = 0; i < front->num_sh_registers; ++i) { - const auto& reg = front->sh_registers[i]; - if (reg.offset == SPI_SHADER_PGM_CHKSUM_GS || FindHalfRegister(back, reg.offset) != nullptr) continue; - layout.sh.push_back(reg); - } - layout.cx.assign(back->cx_registers, back->cx_registers + back->num_cx_registers); - for (std::uint32_t i = 0; i < front->num_cx_registers; ++i) { - const auto& reg = front->cx_registers[i]; - if (std::none_of(layout.cx.begin(), layout.cx.end(), [&](const ShaderRegister& other) { return other.offset == reg.offset; })) layout.cx.push_back(reg); - } - if (layout.sh.size() > 0xff || layout.cx.size() > 0xff) Fail(function, "fused register count exceeds the shader header"); - layout.headerOffset = AlignUp(layout.codeBytes, alignof(Shader)); - layout.shOffset = AlignUp(layout.headerOffset + sizeof(Shader), alignof(ShaderRegister)); - layout.cxOffset = layout.shOffset + layout.sh.size() * sizeof(ShaderRegister); - layout.specialsOffset = AlignUp(layout.cxOffset + layout.cx.size() * sizeof(ShaderRegister), alignof(ShaderSpecialRegs)); - layout.inputsOffset = AlignUp(layout.specialsOffset + back->special_sizes_bytes, alignof(ShaderSemantic)); - layout.outputsOffset = layout.inputsOffset + front->num_input_semantics * sizeof(ShaderSemantic); - layout.userDataOffset = AlignUp(layout.outputsOffset + back->num_output_semantics * sizeof(ShaderSemantic), alignof(ShaderUserData)); - auto end = layout.userDataOffset + (back->user_data != nullptr ? sizeof(ShaderUserData) : 0); - layout.directOffset = AlignUp(end, alignof(std::uint16_t)); - if (back->user_data != nullptr) end = layout.directOffset + back->user_data->direct_resource_count * sizeof(std::uint16_t); - for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) { - layout.sharpOffsets[i] = AlignUp(end, alignof(ShaderSharp)); - if (back->user_data != nullptr) end = layout.sharpOffsets[i] + back->user_data->sharp_resource_count[i] * sizeof(ShaderSharp); - } - layout.totalBytes = end; - return layout; -} - -void SetProgramAddress(std::vector& regs, std::uint32_t loOffset, std::uint64_t address) { - for (std::size_t i = 0; i + 1 < regs.size(); ++i) { - if (regs[i].offset != loOffset || regs[i + 1].offset != loOffset + 1u) continue; - regs[i].value = static_cast(address >> 8u); - regs[i + 1].value = (regs[i + 1].value & 0xFFFFFF00u) | static_cast((address >> 40u) & 0xFFu); - return; - } - throw std::runtime_error("sceAgcFuseShaderHalves_0200: back half has no program address register pair"); -} - -struct FusedProgram { - std::vector storage; - Shader* header; -}; - -std::mutex FusedProgramsMutex; -std::map, FusedProgram> FusedPrograms; - -void AppendBytes(std::vector& bytes, const void* data, std::size_t size) { - if (size == 0) return; - const auto* begin = static_cast(data); - bytes.insert(bytes.end(), begin, begin + size); -} - -std::vector FusionKey(const Shader* front, const Shader* back, const GeometryLayout& layout) { - std::vector key; - constexpr auto headerBytes = offsetof(Shader, num_sh_registers) + sizeof(Shader::num_sh_registers); - AppendBytes(key, front, headerBytes); - AppendBytes(key, back, headerBytes); - AppendBytes(key, const_cast(front->code), layout.frontBytes); - AppendBytes(key, const_cast(back->code), back->shader_size); - AppendBytes(key, layout.sh.data(), layout.sh.size() * sizeof(ShaderRegister)); - AppendBytes(key, layout.cx.data(), layout.cx.size() * sizeof(ShaderRegister)); - AppendBytes(key, back->specials, back->special_sizes_bytes); - AppendBytes(key, front->input_semantics, front->num_input_semantics * sizeof(ShaderSemantic)); - AppendBytes(key, back->output_semantics, back->num_output_semantics * sizeof(ShaderSemantic)); - if (back->user_data == nullptr) return key; - AppendBytes(key, back->user_data, offsetof(ShaderUserData, sharp_resource_count) + sizeof(ShaderUserData::sharp_resource_count)); - AppendBytes(key, back->user_data->direct_resource_offset, back->user_data->direct_resource_count * sizeof(std::uint16_t)); - for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) AppendBytes(key, back->user_data->sharp_resource_offset[i], back->user_data->sharp_resource_count[i] * sizeof(ShaderSharp)); - return key; -} - -FusedProgram BuildFusedProgram(const char* fn, const Shader* front, const Shader* back, GeometryLayout& layout) { - FusedProgram program{std::vector(layout.totalBytes + FusedCodeAlignment - 1), nullptr}; - const auto storage = reinterpret_cast(program.storage.data()); - const auto base = AlignUp(storage, FusedCodeAlignment); - if ((base & SHADER_BASE_ALIGN_MASK) != 0) Fail(fn, "fused shader memory is not a valid program address"); - auto* memory = program.storage.data() + (base - storage); - std::memcpy(memory, const_cast(front->code), layout.frontBytes); - for (std::size_t offset = layout.frontBytes; offset < layout.backOffset; offset += sizeof(SNop)) std::memcpy(memory + offset, &SNop, sizeof(SNop)); - std::memcpy(memory + layout.backOffset, const_cast(back->code), back->shader_size); - SetProgramAddress(layout.sh, SPI_SHADER_PGM_LO_GS, base + layout.backOffset); - if (PatchProgramAddressRegister(layout.sh.data(), static_cast(layout.sh.size()), static_cast(ShaderBinaryType::Gs), base) != 0) Fail(fn, "fused program address patch failed"); - auto* sh = reinterpret_cast(memory + layout.shOffset); - auto* cx = reinterpret_cast(memory + layout.cxOffset); - auto* specials = reinterpret_cast(memory + layout.specialsOffset); - auto* inputs = reinterpret_cast(memory + layout.inputsOffset); - auto* outputs = reinterpret_cast(memory + layout.outputsOffset); - std::memcpy(sh, layout.sh.data(), layout.sh.size() * sizeof(ShaderRegister)); - std::memcpy(cx, layout.cx.data(), layout.cx.size() * sizeof(ShaderRegister)); - if (back->special_sizes_bytes != 0) std::memcpy(specials, back->specials, back->special_sizes_bytes); - if (front->num_input_semantics != 0) std::memcpy(inputs, front->input_semantics, front->num_input_semantics * sizeof(ShaderSemantic)); - if (back->num_output_semantics != 0) std::memcpy(outputs, back->output_semantics, back->num_output_semantics * sizeof(ShaderSemantic)); - Shader fused = *back; - if (back->user_data != nullptr) { - auto* userData = reinterpret_cast(memory + layout.userDataOffset); - *userData = *back->user_data; - auto* direct = reinterpret_cast(memory + layout.directOffset); - if (userData->direct_resource_count != 0) std::memcpy(direct, back->user_data->direct_resource_offset, userData->direct_resource_count * sizeof(std::uint16_t)); - userData->direct_resource_offset = userData->direct_resource_count != 0 ? direct : nullptr; - for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) { - auto* sharps = reinterpret_cast(memory + layout.sharpOffsets[i]); - if (userData->sharp_resource_count[i] != 0) std::memcpy(sharps, back->user_data->sharp_resource_offset[i], userData->sharp_resource_count[i] * sizeof(ShaderSharp)); - userData->sharp_resource_offset[i] = userData->sharp_resource_count[i] != 0 ? sharps : nullptr; - } - fused.user_data = userData; - } - fused.header_size = static_cast(layout.totalBytes - layout.headerOffset); - fused.code = memory; - fused.sh_registers = sh; - fused.cx_registers = layout.cx.empty() ? nullptr : cx; - fused.specials = back->special_sizes_bytes != 0 ? specials : nullptr; - fused.input_semantics = front->num_input_semantics != 0 ? inputs : nullptr; - fused.output_semantics = back->num_output_semantics != 0 ? outputs : nullptr; - fused.shader_size = static_cast(layout.codeBytes); - fused.num_input_semantics = front->num_input_semantics; - fused.scratch_size_dw_per_thread = std::max(front->scratch_size_dw_per_thread, back->scratch_size_dw_per_thread); - fused.type = static_cast(ShaderBinaryType::Gs); - fused.num_sh_registers = static_cast(layout.sh.size()); - fused.num_cx_registers = static_cast(layout.cx.size()); - program.header = reinterpret_cast(memory + layout.headerOffset); - *program.header = fused; - AgcDriverRegisterShader_nid_postfix(program.header); - return program; -} - -int FuseGeometryHalves(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { - constexpr auto fn = "sceAgcFuseShaderHalves_0200"; - if (scratch_mem == nullptr) Fail(fn, "geometry halves need fused shader memory"); - auto layout = ComputeGeometryLayout(fn, front, back); - auto key = FusionKey(front, back, layout); - std::lock_guard lock(FusedProgramsMutex); - auto found = FusedPrograms.find(key); - if (found == FusedPrograms.end()) found = FusedPrograms.emplace(std::move(key), BuildFusedProgram(fn, front, back, layout)).first; - Shader fused = *found->second.header; - std::memcpy(scratch_mem, fused.sh_registers, std::size_t{fused.num_sh_registers} * sizeof(ShaderRegister)); - fused.sh_registers = static_cast(scratch_mem); - *fused_result = fused; - return 0; -} - -} - -extern "C" { - -int APS5_VABI sceAgcFuseShaderHalves_0200(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { - if (fused_result == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; - if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; - const bool isGs = GeometryHalves(front); - const std::uint32_t stageBit = isGs ? (1u << 22u) : (1u << 21u); - if (((front->specials->vgt_shader_stages_en.value ^ back->specials->vgt_shader_stages_en.value) & stageBit) != 0) { - return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; - } - if (isGs) return FuseGeometryHalves(fused_result, front, back, scratch_mem); - *fused_result = *back; - fused_result->type = static_cast(ShaderBinaryType::Hs); - if (scratch_mem != nullptr) { - auto* regs = static_cast(scratch_mem); - std::memcpy(regs, back->sh_registers, std::size_t{back->num_sh_registers} * sizeof(ShaderRegister)); - fused_result->sh_registers = regs; - } - ShaderRegister* fused = fused_result->sh_registers; - const std::uint32_t fusedCount = fused_result->num_sh_registers; - const std::uint32_t frontCount = front->num_sh_registers; - for (std::uint32_t occurrence = 0; occurrence < 2; ++occurrence) { - FindRegister(fused, fusedCount, SPI_SHADER_PGM_CHKSUM_HS, occurrence).value = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_CHKSUM_HS, occurrence).value; - } - const auto& frontRsrc1 = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_RSRC1_HS); - const auto& frontRsrc2 = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_RSRC2_HS); - auto& fusedRsrc1 = FindRegister(fused, fusedCount, SPI_SHADER_PGM_RSRC1_HS); - auto& fusedRsrc2 = FindRegister(fused, fusedCount, SPI_SHADER_PGM_RSRC2_HS); - const std::uint32_t frontVgprs = ((frontRsrc1.value & 0x3fu) + 1u) * 4u; - const std::uint32_t backVgprs = ((fusedRsrc1.value & 0x3fu) + 1u) * 4u; - const std::uint32_t frontTotal = frontVgprs + (frontRsrc2.value >> 28u) * 8u; - const std::uint32_t backTotal = backVgprs + (fusedRsrc2.value >> 28u) * 8u; - const std::uint32_t maxTotal = std::max(frontTotal, backTotal); - const std::uint32_t shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u; - fusedRsrc2.value = (fusedRsrc2.value & 0x0fffffffu) | ((shared & 0xfu) << 28u); - MergeMax(fusedRsrc1, frontRsrc1, 0, 0x3fu); - MergeMax(fusedRsrc1, frontRsrc1, 28, 0x3u); - fusedRsrc2.value = (fusedRsrc2.value & 0xf7ffffc1u) | (frontRsrc2.value & 0x0800003eu); - auto& loRegister = FindRegister(fused, fusedCount, SPI_SHADER_PGM_LO_LS); - auto& hiRegister = FindRegister(fused, fusedCount, SPI_SHADER_PGM_LO_LS + 1u); - const auto address = reinterpret_cast(front->code); - loRegister.value = static_cast(address >> 8u); - hiRegister.value = (hiRegister.value & 0xffffff00u) | static_cast((address >> 40u) & 0xffu); - fused_result->user_data = nullptr; - return 0; -} - -int APS5_VABI sceAgcFuseShaderHalves_nid_postfix(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { - return sceAgcFuseShaderHalves_0200(fused_result, front, back, scratch_mem); -} - -int APS5_VABI sceAgcGetFusedShaderSize_0080(SizeAlign* dst, const Shader* front, const Shader* back) { - if (dst == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; - if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; - if (GeometryHalves(front)) { - const auto layout = ComputeGeometryLayout("sceAgcGetFusedShaderSize_0080", front, back); - dst->m_size = layout.sh.size() * sizeof(ShaderRegister); - dst->m_align = 4; - return 0; - } - dst->m_size = std::uint64_t{back->num_sh_registers} * sizeof(ShaderRegister); - dst->m_align = 4; - return 0; -} - -int APS5_VABI sceAgcUpdateInterpolantMapping_0100(ShaderRegister* regs, const Shader* vs, const Shader* ps) { - (void)regs; - (void)vs; - (void)ps; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -} +#include "prx/libSceAgc/Misc/include/ShaderFusion.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" +#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" +#include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" + +namespace { + +using namespace ShaderRegs; + +constexpr int GRAPHICS5_ERROR_INVALID_SHADER_HALVES = static_cast(0x8a6c0008u); +constexpr std::uint32_t SPI_SHADER_PGM_CHKSUM_GS = 0x080u; +constexpr std::uint32_t SPI_SHADER_PGM_RSRC1_GS = 0x08Au; +constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_GS = 0x08Bu; +constexpr std::uint32_t SPI_SHADER_PGM_CHKSUM_HS = 0x100u; +constexpr std::uint32_t SPI_SHADER_PGM_RSRC1_HS = 0x10Au; +constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_HS = 0x10Bu; +constexpr std::uint32_t SCodeEnd = 0xbf9f0000u; +constexpr std::uint32_t SNop = 0xbf800000u; +constexpr std::uint32_t SSetpcS6 = 0xbe802006u; +constexpr std::uint32_t SSwappcNullS6 = 0xbefd2106u; +constexpr std::size_t FusedCodeAlignment = 256; +constexpr char TrailerMagic[8] = {'b', 'a', 'r', 'e', 'f', 'o', 'o', 't'}; +constexpr std::size_t TrailerCodeBytesOffset = sizeof(TrailerMagic) + 3 * sizeof(std::uint32_t); +constexpr std::uint32_t Rsrc1VgprMask = 0x3fu; +constexpr std::uint32_t Rsrc2OffchipLdsEnable = 1u << 18u; +constexpr std::uint32_t Rsrc2UserSgprMask = (0x1fu << 1u) | (1u << 27u); + +bool ValidHalves(const Shader* front, const Shader* back) { + const auto frontType = static_cast(front->type); + const auto backType = static_cast(back->type); + return (frontType == ShaderBinaryType::GsFront && backType == ShaderBinaryType::GsBack) || + (frontType == ShaderBinaryType::HsFront && backType == ShaderBinaryType::HsBack); +} + +bool GeometryHalves(const Shader* front) { + return static_cast(front->type) == ShaderBinaryType::GsFront; +} + +ShaderRegister& FindRegister(ShaderRegister* regs, std::uint32_t count, std::uint32_t offset, std::uint32_t occurrence = 0) { + for (std::uint32_t i = 0; regs != nullptr && i < count; ++i) { + if (regs[i].offset == offset && occurrence-- == 0) return regs[i]; + } + throw std::runtime_error("sceAgcFuseShaderHalves_0200: shader half lacks register " + std::to_string(offset)); +} + +void MergeMax(ShaderRegister& dst, const ShaderRegister& src, std::uint32_t shift, std::uint32_t mask) { + const auto field = std::max((dst.value >> shift) & mask, (src.value >> shift) & mask); + dst.value = (dst.value & ~(mask << shift)) | (field << shift); +} + +struct GeometryLayout { + std::uint32_t frontBytes; + std::size_t backOffset; + std::size_t codeBytes; + std::vector sh; + std::vector cx; + std::size_t headerOffset; + std::size_t shOffset; + std::size_t cxOffset; + std::size_t specialsOffset; + std::size_t inputsOffset; + std::size_t outputsOffset; + std::size_t userDataOffset; + std::size_t directOffset; + std::array sharpOffsets; + std::size_t totalBytes; +}; + +[[noreturn]] void Fail(const char* function, const std::string& message) { + throw std::runtime_error(std::string(function) + ": " + message); +} + +std::size_t AlignUp(std::size_t value, std::size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +void ValidateGeometryHalf(const char* function, const Shader* shader) { + if (shader->file_header != SHADER_FILE_HEADER_MAGIC || shader->version != SHADER_VERSION) Fail(function, "invalid shader header or version"); + if (shader->code == nullptr || shader->shader_size == 0 || (shader->shader_size & 3u) != 0) Fail(function, "invalid shader half code"); + if (shader->embedded_constant_buffer_size_dqw != 0) Fail(function, "fusing halves with an embedded constant buffer is not implemented"); + if ((shader->num_sh_registers != 0 && shader->sh_registers == nullptr) || (shader->num_cx_registers != 0 && shader->cx_registers == nullptr)) Fail(function, "shader half registers missing"); +} + +std::uint32_t FrontProgramBytes(const char* function, const Shader* front) { + const auto* bytes = static_cast(const_cast(front->code)); + const std::size_t size = front->shader_size; + if (size < TrailerCodeBytesOffset + sizeof(std::uint32_t)) Fail(function, "front half has no program trailer"); + std::size_t trailer = size; + for (std::size_t offset = size - TrailerCodeBytesOffset - sizeof(std::uint32_t) + 1; offset-- > 0;) { + if (std::memcmp(bytes + offset, TrailerMagic, sizeof(TrailerMagic)) == 0) { + trailer = offset; + break; + } + } + if (trailer == size) Fail(function, "front half has no program trailer"); + std::uint32_t codeBytes = 0; + std::memcpy(&codeBytes, bytes + trailer + TrailerCodeBytesOffset, sizeof(codeBytes)); + if (codeBytes == 0 || (codeBytes & 3u) != 0 || codeBytes > trailer) Fail(function, "front half trailer has an invalid code size"); + std::size_t end = codeBytes / 4; + const auto word = [&](std::size_t index) { + std::uint32_t value = 0; + std::memcpy(&value, bytes + index * 4, sizeof(value)); + return value; + }; + while (end > 0 && word(end - 1) == SCodeEnd) --end; + if (end == 0 || (word(end - 1) != SSetpcS6 && word(end - 1) != SSwappcNullS6)) Fail(function, "front half does not end with s_setpc_b64 s[6:7] or s_swappc_b64 null, s[6:7]"); + return static_cast((end - 1) * 4); +} + +std::array mergeGeometryResources(std::uint32_t frontRsrc1, std::uint32_t frontRsrc2, std::uint32_t backRsrc1, std::uint32_t backRsrc2) { + const auto frontVgprs = ((frontRsrc1 & Rsrc1VgprMask) + 1u) * 4u; + const auto backVgprs = ((backRsrc1 & Rsrc1VgprMask) + 1u) * 4u; + const auto frontTotal = frontVgprs + (frontRsrc2 >> 28u) * 8u; + const auto backTotal = backVgprs + (backRsrc2 >> 28u) * 8u; + const auto maxTotal = std::max(frontTotal, backTotal); + const auto shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u; + const auto vgprs = std::max(frontRsrc1 & Rsrc1VgprMask, backRsrc1 & Rsrc1VgprMask); + const auto rsrc1Field = std::max((frontRsrc1 >> 29u) & 3u, (backRsrc1 >> 29u) & 3u); + const auto rsrc2Field = std::max((frontRsrc2 >> 16u) & 3u, (backRsrc2 >> 16u) & 3u); + const auto rsrc1 = (backRsrc1 & ~(Rsrc1VgprMask | (3u << 29u))) | vgprs | (rsrc1Field << 29u); + const auto rsrc2 = (backRsrc2 & ~(0xf0000000u | (3u << 16u) | Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)) | ((shared & 15u) << 28u) | (rsrc2Field << 16u) | (frontRsrc2 & (Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)); + return {rsrc1, rsrc2}; +} + +const ShaderRegister* FindHalfRegister(const Shader* shader, std::uint32_t offset) { + for (std::uint32_t i = 0; i < shader->num_sh_registers; ++i) { + if (shader->sh_registers[i].offset == offset) return &shader->sh_registers[i]; + } + return nullptr; +} + +GeometryLayout ComputeGeometryLayout(const char* function, const Shader* front, const Shader* back) { + ValidateGeometryHalf(function, back); + ValidateGeometryHalf(function, front); + GeometryLayout layout{}; + layout.frontBytes = FrontProgramBytes(function, front); + layout.backOffset = AlignUp(layout.frontBytes, FusedCodeAlignment); + layout.codeBytes = layout.backOffset + back->shader_size; + const auto* frontRsrc1 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC1_GS); + const auto* frontRsrc2 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC2_GS); + if (frontRsrc1 == nullptr || frontRsrc2 == nullptr) Fail(function, "front half has no GS resource registers"); + const auto* backRsrc1 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC1_GS); + const auto* backRsrc2 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC2_GS); + if (backRsrc1 == nullptr || backRsrc2 == nullptr) Fail(function, "back half has no GS resource registers"); + const auto resources = mergeGeometryResources(frontRsrc1->value, frontRsrc2->value, backRsrc1->value, backRsrc2->value); + layout.sh.assign(back->sh_registers, back->sh_registers + back->num_sh_registers); + for (auto& reg : layout.sh) { + if (reg.offset == SPI_SHADER_PGM_RSRC1_GS) reg.value = resources[0]; + else if (reg.offset == SPI_SHADER_PGM_RSRC2_GS) reg.value = resources[1]; + } + for (std::uint32_t i = 0; i < front->num_sh_registers; ++i) { + const auto& reg = front->sh_registers[i]; + if (reg.offset == SPI_SHADER_PGM_CHKSUM_GS || FindHalfRegister(back, reg.offset) != nullptr) continue; + layout.sh.push_back(reg); + } + layout.cx.assign(back->cx_registers, back->cx_registers + back->num_cx_registers); + for (std::uint32_t i = 0; i < front->num_cx_registers; ++i) { + const auto& reg = front->cx_registers[i]; + if (std::none_of(layout.cx.begin(), layout.cx.end(), [&](const ShaderRegister& other) { return other.offset == reg.offset; })) layout.cx.push_back(reg); + } + if (layout.sh.size() > 0xff || layout.cx.size() > 0xff) Fail(function, "fused register count exceeds the shader header"); + layout.headerOffset = AlignUp(layout.codeBytes, alignof(Shader)); + layout.shOffset = AlignUp(layout.headerOffset + sizeof(Shader), alignof(ShaderRegister)); + layout.cxOffset = layout.shOffset + layout.sh.size() * sizeof(ShaderRegister); + layout.specialsOffset = AlignUp(layout.cxOffset + layout.cx.size() * sizeof(ShaderRegister), alignof(ShaderSpecialRegs)); + layout.inputsOffset = AlignUp(layout.specialsOffset + back->special_sizes_bytes, alignof(ShaderSemantic)); + layout.outputsOffset = layout.inputsOffset + front->num_input_semantics * sizeof(ShaderSemantic); + layout.userDataOffset = AlignUp(layout.outputsOffset + back->num_output_semantics * sizeof(ShaderSemantic), alignof(ShaderUserData)); + auto end = layout.userDataOffset + (back->user_data != nullptr ? sizeof(ShaderUserData) : 0); + layout.directOffset = AlignUp(end, alignof(std::uint16_t)); + if (back->user_data != nullptr) end = layout.directOffset + back->user_data->direct_resource_count * sizeof(std::uint16_t); + for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) { + layout.sharpOffsets[i] = AlignUp(end, alignof(ShaderSharp)); + if (back->user_data != nullptr) end = layout.sharpOffsets[i] + back->user_data->sharp_resource_count[i] * sizeof(ShaderSharp); + } + layout.totalBytes = end; + return layout; +} + +void SetProgramAddress(std::vector& regs, std::uint32_t loOffset, std::uint64_t address) { + for (std::size_t i = 0; i + 1 < regs.size(); ++i) { + if (regs[i].offset != loOffset || regs[i + 1].offset != loOffset + 1u) continue; + regs[i].value = static_cast(address >> 8u); + regs[i + 1].value = (regs[i + 1].value & 0xFFFFFF00u) | static_cast((address >> 40u) & 0xFFu); + return; + } + throw std::runtime_error("sceAgcFuseShaderHalves_0200: back half has no program address register pair"); +} + +struct FusedProgram { + std::vector storage; + Shader* header; +}; + +std::mutex FusedProgramsMutex; +std::map, FusedProgram> FusedPrograms; + +void AppendBytes(std::vector& bytes, const void* data, std::size_t size) { + if (size == 0) return; + const auto* begin = static_cast(data); + bytes.insert(bytes.end(), begin, begin + size); +} + +std::vector FusionKey(const Shader* front, const Shader* back, const GeometryLayout& layout) { + std::vector key; + constexpr auto headerBytes = offsetof(Shader, num_sh_registers) + sizeof(Shader::num_sh_registers); + AppendBytes(key, front, headerBytes); + AppendBytes(key, back, headerBytes); + AppendBytes(key, const_cast(front->code), layout.frontBytes); + AppendBytes(key, const_cast(back->code), back->shader_size); + AppendBytes(key, layout.sh.data(), layout.sh.size() * sizeof(ShaderRegister)); + AppendBytes(key, layout.cx.data(), layout.cx.size() * sizeof(ShaderRegister)); + AppendBytes(key, back->specials, back->special_sizes_bytes); + AppendBytes(key, front->input_semantics, front->num_input_semantics * sizeof(ShaderSemantic)); + AppendBytes(key, back->output_semantics, back->num_output_semantics * sizeof(ShaderSemantic)); + if (back->user_data == nullptr) return key; + AppendBytes(key, back->user_data, offsetof(ShaderUserData, sharp_resource_count) + sizeof(ShaderUserData::sharp_resource_count)); + AppendBytes(key, back->user_data->direct_resource_offset, back->user_data->direct_resource_count * sizeof(std::uint16_t)); + for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) AppendBytes(key, back->user_data->sharp_resource_offset[i], back->user_data->sharp_resource_count[i] * sizeof(ShaderSharp)); + return key; +} + +FusedProgram BuildFusedProgram(const char* fn, const Shader* front, const Shader* back, GeometryLayout& layout) { + FusedProgram program{std::vector(layout.totalBytes + FusedCodeAlignment - 1), nullptr}; + const auto storage = reinterpret_cast(program.storage.data()); + const auto base = AlignUp(storage, FusedCodeAlignment); + if ((base & SHADER_BASE_ALIGN_MASK) != 0) Fail(fn, "fused shader memory is not a valid program address"); + auto* memory = program.storage.data() + (base - storage); + std::memcpy(memory, const_cast(front->code), layout.frontBytes); + for (std::size_t offset = layout.frontBytes; offset < layout.backOffset; offset += sizeof(SNop)) std::memcpy(memory + offset, &SNop, sizeof(SNop)); + std::memcpy(memory + layout.backOffset, const_cast(back->code), back->shader_size); + SetProgramAddress(layout.sh, SPI_SHADER_PGM_LO_GS, base + layout.backOffset); + if (PatchProgramAddressRegister(layout.sh.data(), static_cast(layout.sh.size()), static_cast(ShaderBinaryType::Gs), base) != 0) Fail(fn, "fused program address patch failed"); + auto* sh = reinterpret_cast(memory + layout.shOffset); + auto* cx = reinterpret_cast(memory + layout.cxOffset); + auto* specials = reinterpret_cast(memory + layout.specialsOffset); + auto* inputs = reinterpret_cast(memory + layout.inputsOffset); + auto* outputs = reinterpret_cast(memory + layout.outputsOffset); + std::memcpy(sh, layout.sh.data(), layout.sh.size() * sizeof(ShaderRegister)); + std::memcpy(cx, layout.cx.data(), layout.cx.size() * sizeof(ShaderRegister)); + if (back->special_sizes_bytes != 0) std::memcpy(specials, back->specials, back->special_sizes_bytes); + if (front->num_input_semantics != 0) std::memcpy(inputs, front->input_semantics, front->num_input_semantics * sizeof(ShaderSemantic)); + if (back->num_output_semantics != 0) std::memcpy(outputs, back->output_semantics, back->num_output_semantics * sizeof(ShaderSemantic)); + Shader fused = *back; + if (back->user_data != nullptr) { + auto* userData = reinterpret_cast(memory + layout.userDataOffset); + *userData = *back->user_data; + auto* direct = reinterpret_cast(memory + layout.directOffset); + if (userData->direct_resource_count != 0) std::memcpy(direct, back->user_data->direct_resource_offset, userData->direct_resource_count * sizeof(std::uint16_t)); + userData->direct_resource_offset = userData->direct_resource_count != 0 ? direct : nullptr; + for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) { + auto* sharps = reinterpret_cast(memory + layout.sharpOffsets[i]); + if (userData->sharp_resource_count[i] != 0) std::memcpy(sharps, back->user_data->sharp_resource_offset[i], userData->sharp_resource_count[i] * sizeof(ShaderSharp)); + userData->sharp_resource_offset[i] = userData->sharp_resource_count[i] != 0 ? sharps : nullptr; + } + fused.user_data = userData; + } + fused.header_size = static_cast(layout.totalBytes - layout.headerOffset); + fused.code = memory; + fused.sh_registers = sh; + fused.cx_registers = layout.cx.empty() ? nullptr : cx; + fused.specials = back->special_sizes_bytes != 0 ? specials : nullptr; + fused.input_semantics = front->num_input_semantics != 0 ? inputs : nullptr; + fused.output_semantics = back->num_output_semantics != 0 ? outputs : nullptr; + fused.shader_size = static_cast(layout.codeBytes); + fused.num_input_semantics = front->num_input_semantics; + fused.scratch_size_dw_per_thread = std::max(front->scratch_size_dw_per_thread, back->scratch_size_dw_per_thread); + fused.type = static_cast(ShaderBinaryType::Gs); + fused.num_sh_registers = static_cast(layout.sh.size()); + fused.num_cx_registers = static_cast(layout.cx.size()); + program.header = reinterpret_cast(memory + layout.headerOffset); + *program.header = fused; + AgcDriverRegisterShader_nid_postfix(program.header); + return program; +} + +int FuseGeometryHalves(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { + constexpr auto fn = "sceAgcFuseShaderHalves_0200"; + if (scratch_mem == nullptr) Fail(fn, "geometry halves need fused shader memory"); + auto layout = ComputeGeometryLayout(fn, front, back); + auto key = FusionKey(front, back, layout); + std::lock_guard lock(FusedProgramsMutex); + auto found = FusedPrograms.find(key); + if (found == FusedPrograms.end()) found = FusedPrograms.emplace(std::move(key), BuildFusedProgram(fn, front, back, layout)).first; + Shader fused = *found->second.header; + std::memcpy(scratch_mem, fused.sh_registers, std::size_t{fused.num_sh_registers} * sizeof(ShaderRegister)); + fused.sh_registers = static_cast(scratch_mem); + *fused_result = fused; + return 0; +} + +} + +extern "C" { + +int APS5_VABI sceAgcFuseShaderHalves_0200(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { + if (fused_result == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; + if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + const bool isGs = GeometryHalves(front); + const std::uint32_t stageBit = isGs ? (1u << 22u) : (1u << 21u); + if (((front->specials->vgt_shader_stages_en.value ^ back->specials->vgt_shader_stages_en.value) & stageBit) != 0) { + return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + } + if (isGs) return FuseGeometryHalves(fused_result, front, back, scratch_mem); + *fused_result = *back; + fused_result->type = static_cast(ShaderBinaryType::Hs); + if (scratch_mem != nullptr) { + auto* regs = static_cast(scratch_mem); + std::memcpy(regs, back->sh_registers, std::size_t{back->num_sh_registers} * sizeof(ShaderRegister)); + fused_result->sh_registers = regs; + } + ShaderRegister* fused = fused_result->sh_registers; + const std::uint32_t fusedCount = fused_result->num_sh_registers; + const std::uint32_t frontCount = front->num_sh_registers; + for (std::uint32_t occurrence = 0; occurrence < 2; ++occurrence) { + FindRegister(fused, fusedCount, SPI_SHADER_PGM_CHKSUM_HS, occurrence).value = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_CHKSUM_HS, occurrence).value; + } + const auto& frontRsrc1 = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_RSRC1_HS); + const auto& frontRsrc2 = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_RSRC2_HS); + auto& fusedRsrc1 = FindRegister(fused, fusedCount, SPI_SHADER_PGM_RSRC1_HS); + auto& fusedRsrc2 = FindRegister(fused, fusedCount, SPI_SHADER_PGM_RSRC2_HS); + const std::uint32_t frontVgprs = ((frontRsrc1.value & 0x3fu) + 1u) * 4u; + const std::uint32_t backVgprs = ((fusedRsrc1.value & 0x3fu) + 1u) * 4u; + const std::uint32_t frontTotal = frontVgprs + (frontRsrc2.value >> 28u) * 8u; + const std::uint32_t backTotal = backVgprs + (fusedRsrc2.value >> 28u) * 8u; + const std::uint32_t maxTotal = std::max(frontTotal, backTotal); + const std::uint32_t shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u; + fusedRsrc2.value = (fusedRsrc2.value & 0x0fffffffu) | ((shared & 0xfu) << 28u); + MergeMax(fusedRsrc1, frontRsrc1, 0, 0x3fu); + MergeMax(fusedRsrc1, frontRsrc1, 28, 0x3u); + fusedRsrc2.value = (fusedRsrc2.value & 0xf7ffffc1u) | (frontRsrc2.value & 0x0800003eu); + auto& loRegister = FindRegister(fused, fusedCount, SPI_SHADER_PGM_LO_LS); + auto& hiRegister = FindRegister(fused, fusedCount, SPI_SHADER_PGM_LO_LS + 1u); + const auto address = reinterpret_cast(front->code); + loRegister.value = static_cast(address >> 8u); + hiRegister.value = (hiRegister.value & 0xffffff00u) | static_cast((address >> 40u) & 0xffu); + fused_result->user_data = nullptr; + return 0; +} + +int APS5_VABI sceAgcFuseShaderHalves_nid_postfix(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { + return sceAgcFuseShaderHalves_0200(fused_result, front, back, scratch_mem); +} + +int APS5_VABI sceAgcGetFusedShaderSize_0080(SizeAlign* dst, const Shader* front, const Shader* back) { + if (dst == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; + if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + if (GeometryHalves(front)) { + const auto layout = ComputeGeometryLayout("sceAgcGetFusedShaderSize_0080", front, back); + dst->m_size = layout.sh.size() * sizeof(ShaderRegister); + dst->m_align = 4; + return 0; + } + dst->m_size = std::uint64_t{back->num_sh_registers} * sizeof(ShaderRegister); + dst->m_align = 4; + return 0; +} + +} diff --git a/core/libs/prx/libSceAgc/Shader/include/InterpolantMapping.hpp b/core/libs/prx/libSceAgc/Shader/include/InterpolantMapping.hpp index 7085d35fe0..66b549a9a4 100644 --- a/core/libs/prx/libSceAgc/Shader/include/InterpolantMapping.hpp +++ b/core/libs/prx/libSceAgc/Shader/include/InterpolantMapping.hpp @@ -7,6 +7,7 @@ extern "C" { int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps); +int APS5_VABI sceAgcUpdateInterpolantMapping_0100(ShaderRegister* regs, const Shader* vs, const Shader* ps); int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps); int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps); int APS5_VABI sceAgcCreateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps); diff --git a/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp index 65b337d335..9bf3fb0bf0 100644 --- a/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp @@ -1,143 +1,181 @@ -#include "prx/libSceAgc/Shader/include/InterpolantMapping.hpp" - -#include -#include -#include -#include -#include - -#include "SceShaders.hpp" -#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" -#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" -#include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" -#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" - -namespace { - -constexpr std::uint32_t InterpolantRegisterCount = 32; -constexpr std::uint32_t InterpolantF16HiOnly = 2; - -using CreateInterpolantValueFn = std::uint32_t (*)(std::uint32_t psWord, const ShaderSemantic* gsSemantic); - -std::uint32_t CreateInterpolantValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { - const std::uint32_t value = ((psWord & 0x00300000u) != 0) - ? CreateInterpolantF16Value(psWord, gsSemantic) - : CreateInterpolantNonF16Value(psWord, gsSemantic); - - return (gsSemantic == nullptr) - ? CreateInterpolantDefaultValue(value, psWord) - : CreateInterpolantMappingValue(value, psWord, ShaderSemanticWord(*gsSemantic)); -} - -std::uint32_t CreateInterpolantF16HiValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { - std::uint32_t value = ((psWord << 4u) & 0x03000000u) | 0x00080000u; - if (gsSemantic == nullptr || (psWord & ShaderSemanticWord(*gsSemantic) & 0x00200000u) == 0) { - value |= 0x00000020u; - } - value = ApplyInterpolantDefaultValueHi(value, psWord); - value |= ((psWord >> 30u) & 0x3u) << 8u; - if (gsSemantic == nullptr) { - return value; - } - - value |= (ShaderSemanticWord(*gsSemantic) >> 8u) & 0x1Fu; - if ((psWord & 0x01400000u) != 0) { - value |= 0x00000400u; - } - return value; -} - -std::uint32_t CreateInterpolantValueSplitF16(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { - return (((psWord >> 20u) & 0x3u) == InterpolantF16HiOnly) - ? CreateInterpolantF16HiValue(psWord, gsSemantic) - : CreateInterpolantValue(psWord, gsSemantic); -} - -std::uint32_t CreateInterpolantValueFlatDefault(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { - const std::uint32_t value = CreateInterpolantValue(psWord, gsSemantic); - return ((psWord & 0x00700000u) == 0x00400000u) ? (value | 0x00000020u) : value; -} - -int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* gs, const Shader* ps, CreateInterpolantValueFn createValue, bool fillIdentityTail) { - if (regs == nullptr) { - throw std::runtime_error(std::string(fn) + ": regs is null"); - } - if (ps == nullptr && !fillIdentityTail) { - throw std::runtime_error(std::string(fn) + ": ps is null"); - } - if (ps != nullptr && ps->num_input_semantics > 32) { - throw std::runtime_error(std::string(fn) + ": input semantic count exceeds 32"); - } - - AgcDriver::ShaderPreparationScope transaction({ps != nullptr ? gs : nullptr, ps}); - std::array values{}; - auto* output = regs; - regs = values.data(); - if (ps == nullptr || ps->num_input_semantics == 0) { - const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : 0; - if (fillIdentityTail) FillIdentityInterpolants(regs, 0); - if (ps != nullptr) { - AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {}); - if (gs != nullptr) AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); - } - transaction.Commit(); - std::copy_n(values.begin(), count, output); - return 0; - } - - if (ps->num_input_semantics != 0 && ps->input_semantics == nullptr) { - throw std::runtime_error(std::string(fn) + ": ps->input_semantics is null but num_input_semantics != 0"); - } - - if (ps->num_input_semantics > InterpolantRegisterCount) { - throw std::runtime_error(std::string(fn) + ": ps->num_input_semantics exceeds the interpolant register count"); - } - - if (gs == nullptr) { - throw std::runtime_error(std::string(fn) + ": gs is null"); - } - - if (gs->num_output_semantics != 0 && gs->output_semantics == nullptr) { - throw std::runtime_error(std::string(fn) + ": gs->output_semantics is null but num_output_semantics != 0"); - } - - for (std::uint32_t i = 0; i < ps->num_input_semantics; ++i) { - const ShaderSemantic& psSemantic = ps->input_semantics[i]; - const ShaderSemantic* gsSemantic = FindOutputSemantic(gs, psSemantic.semantic); - SetInterpolantRegister(regs, i, createValue(ShaderSemanticWord(psSemantic), gsSemantic)); - } - - const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : ps->num_input_semantics; - if (fillIdentityTail) FillIdentityInterpolants(regs, ps->num_input_semantics); - AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {}); - AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); - transaction.Commit(); - std::copy_n(values.begin(), count, output); - return 0; -} - -} - -extern "C" { - -int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue, true); -} - -int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, true); -} - -int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, false); -} - -int APS5_VABI sceAgcCreateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, true); -} - -int APS5_VABI sceAgcUpdateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, false); -} - -} +#include "prx/libSceAgc/Shader/include/InterpolantMapping.hpp" + +#include +#include +#include +#include +#include + +#include "SceShaders.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" +#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" + +namespace { + +constexpr std::uint32_t InterpolantRegisterCount = 32; +constexpr std::uint32_t InterpolantF16HiOnly = 2; + +using CreateInterpolantValueFn = std::uint32_t (*)(std::uint32_t psWord, const ShaderSemantic* gsSemantic); + +std::uint32_t CreateInterpolantValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { + const std::uint32_t value = ((psWord & 0x00300000u) != 0) + ? CreateInterpolantF16Value(psWord, gsSemantic) + : CreateInterpolantNonF16Value(psWord, gsSemantic); + + return (gsSemantic == nullptr) + ? CreateInterpolantDefaultValue(value, psWord) + : CreateInterpolantMappingValue(value, psWord, ShaderSemanticWord(*gsSemantic)); +} + +std::uint32_t CreateInterpolantF16HiValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { + std::uint32_t value = ((psWord << 4u) & 0x03000000u) | 0x00080000u; + if (gsSemantic == nullptr || (psWord & ShaderSemanticWord(*gsSemantic) & 0x00200000u) == 0) { + value |= 0x00000020u; + } + value = ApplyInterpolantDefaultValueHi(value, psWord); + value |= ((psWord >> 30u) & 0x3u) << 8u; + if (gsSemantic == nullptr) { + return value; + } + + value |= (ShaderSemanticWord(*gsSemantic) >> 8u) & 0x1Fu; + if ((psWord & 0x01400000u) != 0) { + value |= 0x00000400u; + } + return value; +} + +std::uint32_t CreateInterpolantValueSplitF16(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { + return (((psWord >> 20u) & 0x3u) == InterpolantF16HiOnly) + ? CreateInterpolantF16HiValue(psWord, gsSemantic) + : CreateInterpolantValue(psWord, gsSemantic); +} + +std::uint32_t CreateInterpolantValueFlatDefault(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { + const std::uint32_t value = CreateInterpolantValue(psWord, gsSemantic); + return ((psWord & 0x00700000u) == 0x00400000u) ? (value | 0x00000020u) : value; +} + +std::uint32_t ApplyInterpolantDefaultValue0100(std::uint32_t value, std::uint32_t psWord) { + constexpr std::uint32_t encoded[4] = {0u, 2u, 1u, 3u}; + value &= ~0x00000300u; + value |= encoded[(psWord >> 28u) & 0x3u] << 8u; + return value; +} + +int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* gs, const Shader* ps, CreateInterpolantValueFn createValue, bool fillIdentityTail) { + if (regs == nullptr) { + throw std::runtime_error(std::string(fn) + ": regs is null"); + } + if (ps == nullptr && !fillIdentityTail) { + throw std::runtime_error(std::string(fn) + ": ps is null"); + } + if (ps != nullptr && ps->num_input_semantics > 32) { + throw std::runtime_error(std::string(fn) + ": input semantic count exceeds 32"); + } + + AgcDriver::ShaderPreparationScope transaction({ps != nullptr ? gs : nullptr, ps}); + std::array values{}; + auto* output = regs; + regs = values.data(); + if (ps == nullptr || ps->num_input_semantics == 0) { + const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : 0; + if (fillIdentityTail) FillIdentityInterpolants(regs, 0); + if (ps != nullptr) { + AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {}); + if (gs != nullptr) AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); + } + transaction.Commit(); + std::copy_n(values.begin(), count, output); + return 0; + } + + if (ps->num_input_semantics != 0 && ps->input_semantics == nullptr) { + throw std::runtime_error(std::string(fn) + ": ps->input_semantics is null but num_input_semantics != 0"); + } + + if (ps->num_input_semantics > InterpolantRegisterCount) { + throw std::runtime_error(std::string(fn) + ": ps->num_input_semantics exceeds the interpolant register count"); + } + + if (gs == nullptr) { + throw std::runtime_error(std::string(fn) + ": gs is null"); + } + + if (gs->num_output_semantics != 0 && gs->output_semantics == nullptr) { + throw std::runtime_error(std::string(fn) + ": gs->output_semantics is null but num_output_semantics != 0"); + } + + for (std::uint32_t i = 0; i < ps->num_input_semantics; ++i) { + const ShaderSemantic& psSemantic = ps->input_semantics[i]; + const ShaderSemantic* gsSemantic = FindOutputSemantic(gs, psSemantic.semantic); + SetInterpolantRegister(regs, i, createValue(ShaderSemanticWord(psSemantic), gsSemantic)); + } + + const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : ps->num_input_semantics; + if (fillIdentityTail) FillIdentityInterpolants(regs, ps->num_input_semantics); + AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {}); + AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); + transaction.Commit(); + std::copy_n(values.begin(), count, output); + return 0; +} + +} + +extern "C" { + +int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue, true); +} + +int APS5_VABI sceAgcUpdateInterpolantMapping_0100(ShaderRegister* regs, const Shader* vs, const Shader* ps) { + if (regs == nullptr) { + throw std::runtime_error(std::string(__func__) + ": regs is null"); + } + if (ps == nullptr) { + throw std::runtime_error(std::string(__func__) + ": ps is null"); + } + if (ps->num_input_semantics > InterpolantRegisterCount) { + throw std::runtime_error(std::string(__func__) + ": input semantic count exceeds 32"); + } + if (ps->num_input_semantics == 0) { + return 0; + } + if (ps->input_semantics == nullptr) { + throw std::runtime_error(std::string(__func__) + ": ps->input_semantics is null but num_input_semantics != 0"); + } + if (vs == nullptr) { + throw std::runtime_error(std::string(__func__) + ": vs is null"); + } + if (vs->num_output_semantics != 0 && vs->output_semantics == nullptr) { + throw std::runtime_error(std::string(__func__) + ": vs->output_semantics is null but num_output_semantics != 0"); + } + for (std::uint32_t i = 0; i < ps->num_input_semantics; ++i) { + const ShaderSemantic& psSemantic = ps->input_semantics[i]; + const std::uint32_t psWord = ShaderSemanticWord(psSemantic); + const ShaderSemantic* vsSemantic = FindOutputSemantic(vs, psSemantic.semantic); + SetInterpolantRegister(regs, i, ApplyInterpolantDefaultValue0100(CreateInterpolantValue(psWord, vsSemantic), psWord)); + } + return 0; +} + +int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, true); +} + +int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, false); +} + +int APS5_VABI sceAgcCreateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, true); +} + +int APS5_VABI sceAgcUpdateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, false); +} + +} diff --git a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp index 2fbd372623..880c717e16 100644 --- a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp @@ -162,6 +162,45 @@ void testUpdateMapping() { check(update[1].value == 0x01380007u && update[3].value == 0x03680409u, "f16 DEFAULT_VAL does not come from the low default"); } +void testUpdate0100Mapping() { + const auto mappingsBefore = mappings; + const auto linksBefore = links; + Shader ps{}; + auto regs = filled(); + const auto untouched = filled(); + check(sceAgcUpdateInterpolantMapping_0100(regs.data(), nullptr, &ps) == 0, "_0100 update without inputs failed"); + check(std::memcmp(regs.data(), untouched.data(), sizeof(regs)) == 0, "_0100 update without inputs wrote registers"); + + std::array inputs{semantic(1, 0, 0), semantic(2, 0, 1), semantic(3, 0, 2), semantic(4, 0, 3)}; + inputs[0].is_custom = 1; + inputs[0].default_value = 2; + inputs[1].default_value = 1; + inputs[1].default_value_hi = 1; + inputs[2].default_value_hi = 2; + inputs[3].is_custom = 1; + inputs[3].default_value = 3; + inputs[3].default_value_hi = 3; + std::array outputs{semantic(1, 5, 0), semantic(2, 7, 1), semantic(4, 9, 3)}; + Shader vs{}; + vs.output_semantics = outputs.data(); + vs.num_output_semantics = static_cast(outputs.size()); + ps.input_semantics = inputs.data(); + ps.num_input_semantics = static_cast(inputs.size()); + + regs = filled(); + check(sceAgcUpdateInterpolantMapping_0100(regs.data(), &vs, &ps) == 0, "_0100 update mapping failed"); + check(regs[0].value == 0x00000525u, "_0100 update does not encode default value 2 as 1"); + check(regs[1].value == 0x01380207u, "_0100 update does not encode default value 1 as 2"); + check(regs[2].value == 0x02580020u, "_0100 unmatched interpolant update changed"); + check(regs[3].value == 0x03680709u, "_0100 full f16 interpolant update changed"); + for (std::uint32_t i = 0; i < inputs.size(); ++i) { + check(regs[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i, "_0100 update register offset changed"); + } + const std::size_t written = inputs.size() * sizeof(ShaderRegister); + check(std::memcmp(regs.data() + inputs.size(), untouched.data(), sizeof(regs) - written) == 0, "_0100 update wrote past its inputs"); + check(mappings == mappingsBefore && links == linksBefore, "_0100 update resolved an ABI"); +} + void testUnversionedMapping() { std::array inputs{ semantic(1, 0, 2), semantic(2, 0, 2), semantic(5, 0, 2), semantic(3, 0, 0), semantic(4, 0, 3)}; @@ -207,6 +246,7 @@ void testRejections() { std::memset(regs.data(), 0xcc, sizeof(regs)); const auto saved = regs; expectFailure([&] { sceAgcCreateInterpolantMapping_0100(regs.data(), &gs, &ps); }); + expectFailure([&] { sceAgcUpdateInterpolantMapping_0100(regs.data(), &gs, &ps); }); expectFailure([&] { sceAgcCreateInterpolantMapping_10000(regs.data(), &gs, &ps); }); expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(regs.data(), &gs, &ps); }); check(std::memcmp(regs.data(), saved.data(), sizeof(regs)) == 0, "rejected mapping wrote registers"); @@ -216,6 +256,15 @@ void testRejections() { expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(regs.data(), nullptr, &ps); }); expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(nullptr, &gs, &ps); }); expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(regs.data(), &gs, nullptr); }); + expectFailure([&] { sceAgcUpdateInterpolantMapping_0100(regs.data(), nullptr, &ps); }); + expectFailure([&] { sceAgcUpdateInterpolantMapping_0100(nullptr, &gs, &ps); }); + expectFailure([&] { sceAgcUpdateInterpolantMapping_0100(regs.data(), &gs, nullptr); }); + ps.input_semantics = nullptr; + expectFailure([&] { sceAgcUpdateInterpolantMapping_0100(regs.data(), &gs, &ps); }); + ps.input_semantics = inputs.data(); + gs.output_semantics = nullptr; + gs.num_output_semantics = 1; + expectFailure([&] { sceAgcUpdateInterpolantMapping_0100(regs.data(), &gs, &ps); }); } } @@ -238,6 +287,7 @@ int main() { testIdentity(); testMapping(); testUpdateMapping(); + testUpdate0100Mapping(); testUnversionedMapping(); testRejections(); check(mappings == 10u && links == 8u, "mapping preparation call count changed"); From 57fdba256432d64f80e5b3e147b13c8ce9546585 Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Sun, 1 Nov 2026 02:53:13 +0530 Subject: [PATCH 267/318] fix(libSceAgc): encode _0100 interpolant default_value as identity Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prx/libSceAgc/Misc/src/ShaderFusion.cpp | 744 +++++++++--------- .../Shader/src/InterpolantMapping.cpp | 361 +++++---- .../libSceAgc/tests/InterpolantMapping.cpp | 4 +- 3 files changed, 554 insertions(+), 555 deletions(-) diff --git a/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp b/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp index 587f243721..871e70688d 100644 --- a/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp +++ b/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp @@ -1,372 +1,372 @@ -#include "prx/libSceAgc/Misc/include/ShaderFusion.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" -#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" -#include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" -#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" - -namespace { - -using namespace ShaderRegs; - -constexpr int GRAPHICS5_ERROR_INVALID_SHADER_HALVES = static_cast(0x8a6c0008u); -constexpr std::uint32_t SPI_SHADER_PGM_CHKSUM_GS = 0x080u; -constexpr std::uint32_t SPI_SHADER_PGM_RSRC1_GS = 0x08Au; -constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_GS = 0x08Bu; -constexpr std::uint32_t SPI_SHADER_PGM_CHKSUM_HS = 0x100u; -constexpr std::uint32_t SPI_SHADER_PGM_RSRC1_HS = 0x10Au; -constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_HS = 0x10Bu; -constexpr std::uint32_t SCodeEnd = 0xbf9f0000u; -constexpr std::uint32_t SNop = 0xbf800000u; -constexpr std::uint32_t SSetpcS6 = 0xbe802006u; -constexpr std::uint32_t SSwappcNullS6 = 0xbefd2106u; -constexpr std::size_t FusedCodeAlignment = 256; -constexpr char TrailerMagic[8] = {'b', 'a', 'r', 'e', 'f', 'o', 'o', 't'}; -constexpr std::size_t TrailerCodeBytesOffset = sizeof(TrailerMagic) + 3 * sizeof(std::uint32_t); -constexpr std::uint32_t Rsrc1VgprMask = 0x3fu; -constexpr std::uint32_t Rsrc2OffchipLdsEnable = 1u << 18u; -constexpr std::uint32_t Rsrc2UserSgprMask = (0x1fu << 1u) | (1u << 27u); - -bool ValidHalves(const Shader* front, const Shader* back) { - const auto frontType = static_cast(front->type); - const auto backType = static_cast(back->type); - return (frontType == ShaderBinaryType::GsFront && backType == ShaderBinaryType::GsBack) || - (frontType == ShaderBinaryType::HsFront && backType == ShaderBinaryType::HsBack); -} - -bool GeometryHalves(const Shader* front) { - return static_cast(front->type) == ShaderBinaryType::GsFront; -} - -ShaderRegister& FindRegister(ShaderRegister* regs, std::uint32_t count, std::uint32_t offset, std::uint32_t occurrence = 0) { - for (std::uint32_t i = 0; regs != nullptr && i < count; ++i) { - if (regs[i].offset == offset && occurrence-- == 0) return regs[i]; - } - throw std::runtime_error("sceAgcFuseShaderHalves_0200: shader half lacks register " + std::to_string(offset)); -} - -void MergeMax(ShaderRegister& dst, const ShaderRegister& src, std::uint32_t shift, std::uint32_t mask) { - const auto field = std::max((dst.value >> shift) & mask, (src.value >> shift) & mask); - dst.value = (dst.value & ~(mask << shift)) | (field << shift); -} - -struct GeometryLayout { - std::uint32_t frontBytes; - std::size_t backOffset; - std::size_t codeBytes; - std::vector sh; - std::vector cx; - std::size_t headerOffset; - std::size_t shOffset; - std::size_t cxOffset; - std::size_t specialsOffset; - std::size_t inputsOffset; - std::size_t outputsOffset; - std::size_t userDataOffset; - std::size_t directOffset; - std::array sharpOffsets; - std::size_t totalBytes; -}; - -[[noreturn]] void Fail(const char* function, const std::string& message) { - throw std::runtime_error(std::string(function) + ": " + message); -} - -std::size_t AlignUp(std::size_t value, std::size_t alignment) { - return (value + alignment - 1) & ~(alignment - 1); -} - -void ValidateGeometryHalf(const char* function, const Shader* shader) { - if (shader->file_header != SHADER_FILE_HEADER_MAGIC || shader->version != SHADER_VERSION) Fail(function, "invalid shader header or version"); - if (shader->code == nullptr || shader->shader_size == 0 || (shader->shader_size & 3u) != 0) Fail(function, "invalid shader half code"); - if (shader->embedded_constant_buffer_size_dqw != 0) Fail(function, "fusing halves with an embedded constant buffer is not implemented"); - if ((shader->num_sh_registers != 0 && shader->sh_registers == nullptr) || (shader->num_cx_registers != 0 && shader->cx_registers == nullptr)) Fail(function, "shader half registers missing"); -} - -std::uint32_t FrontProgramBytes(const char* function, const Shader* front) { - const auto* bytes = static_cast(const_cast(front->code)); - const std::size_t size = front->shader_size; - if (size < TrailerCodeBytesOffset + sizeof(std::uint32_t)) Fail(function, "front half has no program trailer"); - std::size_t trailer = size; - for (std::size_t offset = size - TrailerCodeBytesOffset - sizeof(std::uint32_t) + 1; offset-- > 0;) { - if (std::memcmp(bytes + offset, TrailerMagic, sizeof(TrailerMagic)) == 0) { - trailer = offset; - break; - } - } - if (trailer == size) Fail(function, "front half has no program trailer"); - std::uint32_t codeBytes = 0; - std::memcpy(&codeBytes, bytes + trailer + TrailerCodeBytesOffset, sizeof(codeBytes)); - if (codeBytes == 0 || (codeBytes & 3u) != 0 || codeBytes > trailer) Fail(function, "front half trailer has an invalid code size"); - std::size_t end = codeBytes / 4; - const auto word = [&](std::size_t index) { - std::uint32_t value = 0; - std::memcpy(&value, bytes + index * 4, sizeof(value)); - return value; - }; - while (end > 0 && word(end - 1) == SCodeEnd) --end; - if (end == 0 || (word(end - 1) != SSetpcS6 && word(end - 1) != SSwappcNullS6)) Fail(function, "front half does not end with s_setpc_b64 s[6:7] or s_swappc_b64 null, s[6:7]"); - return static_cast((end - 1) * 4); -} - -std::array mergeGeometryResources(std::uint32_t frontRsrc1, std::uint32_t frontRsrc2, std::uint32_t backRsrc1, std::uint32_t backRsrc2) { - const auto frontVgprs = ((frontRsrc1 & Rsrc1VgprMask) + 1u) * 4u; - const auto backVgprs = ((backRsrc1 & Rsrc1VgprMask) + 1u) * 4u; - const auto frontTotal = frontVgprs + (frontRsrc2 >> 28u) * 8u; - const auto backTotal = backVgprs + (backRsrc2 >> 28u) * 8u; - const auto maxTotal = std::max(frontTotal, backTotal); - const auto shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u; - const auto vgprs = std::max(frontRsrc1 & Rsrc1VgprMask, backRsrc1 & Rsrc1VgprMask); - const auto rsrc1Field = std::max((frontRsrc1 >> 29u) & 3u, (backRsrc1 >> 29u) & 3u); - const auto rsrc2Field = std::max((frontRsrc2 >> 16u) & 3u, (backRsrc2 >> 16u) & 3u); - const auto rsrc1 = (backRsrc1 & ~(Rsrc1VgprMask | (3u << 29u))) | vgprs | (rsrc1Field << 29u); - const auto rsrc2 = (backRsrc2 & ~(0xf0000000u | (3u << 16u) | Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)) | ((shared & 15u) << 28u) | (rsrc2Field << 16u) | (frontRsrc2 & (Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)); - return {rsrc1, rsrc2}; -} - -const ShaderRegister* FindHalfRegister(const Shader* shader, std::uint32_t offset) { - for (std::uint32_t i = 0; i < shader->num_sh_registers; ++i) { - if (shader->sh_registers[i].offset == offset) return &shader->sh_registers[i]; - } - return nullptr; -} - -GeometryLayout ComputeGeometryLayout(const char* function, const Shader* front, const Shader* back) { - ValidateGeometryHalf(function, back); - ValidateGeometryHalf(function, front); - GeometryLayout layout{}; - layout.frontBytes = FrontProgramBytes(function, front); - layout.backOffset = AlignUp(layout.frontBytes, FusedCodeAlignment); - layout.codeBytes = layout.backOffset + back->shader_size; - const auto* frontRsrc1 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC1_GS); - const auto* frontRsrc2 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC2_GS); - if (frontRsrc1 == nullptr || frontRsrc2 == nullptr) Fail(function, "front half has no GS resource registers"); - const auto* backRsrc1 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC1_GS); - const auto* backRsrc2 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC2_GS); - if (backRsrc1 == nullptr || backRsrc2 == nullptr) Fail(function, "back half has no GS resource registers"); - const auto resources = mergeGeometryResources(frontRsrc1->value, frontRsrc2->value, backRsrc1->value, backRsrc2->value); - layout.sh.assign(back->sh_registers, back->sh_registers + back->num_sh_registers); - for (auto& reg : layout.sh) { - if (reg.offset == SPI_SHADER_PGM_RSRC1_GS) reg.value = resources[0]; - else if (reg.offset == SPI_SHADER_PGM_RSRC2_GS) reg.value = resources[1]; - } - for (std::uint32_t i = 0; i < front->num_sh_registers; ++i) { - const auto& reg = front->sh_registers[i]; - if (reg.offset == SPI_SHADER_PGM_CHKSUM_GS || FindHalfRegister(back, reg.offset) != nullptr) continue; - layout.sh.push_back(reg); - } - layout.cx.assign(back->cx_registers, back->cx_registers + back->num_cx_registers); - for (std::uint32_t i = 0; i < front->num_cx_registers; ++i) { - const auto& reg = front->cx_registers[i]; - if (std::none_of(layout.cx.begin(), layout.cx.end(), [&](const ShaderRegister& other) { return other.offset == reg.offset; })) layout.cx.push_back(reg); - } - if (layout.sh.size() > 0xff || layout.cx.size() > 0xff) Fail(function, "fused register count exceeds the shader header"); - layout.headerOffset = AlignUp(layout.codeBytes, alignof(Shader)); - layout.shOffset = AlignUp(layout.headerOffset + sizeof(Shader), alignof(ShaderRegister)); - layout.cxOffset = layout.shOffset + layout.sh.size() * sizeof(ShaderRegister); - layout.specialsOffset = AlignUp(layout.cxOffset + layout.cx.size() * sizeof(ShaderRegister), alignof(ShaderSpecialRegs)); - layout.inputsOffset = AlignUp(layout.specialsOffset + back->special_sizes_bytes, alignof(ShaderSemantic)); - layout.outputsOffset = layout.inputsOffset + front->num_input_semantics * sizeof(ShaderSemantic); - layout.userDataOffset = AlignUp(layout.outputsOffset + back->num_output_semantics * sizeof(ShaderSemantic), alignof(ShaderUserData)); - auto end = layout.userDataOffset + (back->user_data != nullptr ? sizeof(ShaderUserData) : 0); - layout.directOffset = AlignUp(end, alignof(std::uint16_t)); - if (back->user_data != nullptr) end = layout.directOffset + back->user_data->direct_resource_count * sizeof(std::uint16_t); - for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) { - layout.sharpOffsets[i] = AlignUp(end, alignof(ShaderSharp)); - if (back->user_data != nullptr) end = layout.sharpOffsets[i] + back->user_data->sharp_resource_count[i] * sizeof(ShaderSharp); - } - layout.totalBytes = end; - return layout; -} - -void SetProgramAddress(std::vector& regs, std::uint32_t loOffset, std::uint64_t address) { - for (std::size_t i = 0; i + 1 < regs.size(); ++i) { - if (regs[i].offset != loOffset || regs[i + 1].offset != loOffset + 1u) continue; - regs[i].value = static_cast(address >> 8u); - regs[i + 1].value = (regs[i + 1].value & 0xFFFFFF00u) | static_cast((address >> 40u) & 0xFFu); - return; - } - throw std::runtime_error("sceAgcFuseShaderHalves_0200: back half has no program address register pair"); -} - -struct FusedProgram { - std::vector storage; - Shader* header; -}; - -std::mutex FusedProgramsMutex; -std::map, FusedProgram> FusedPrograms; - -void AppendBytes(std::vector& bytes, const void* data, std::size_t size) { - if (size == 0) return; - const auto* begin = static_cast(data); - bytes.insert(bytes.end(), begin, begin + size); -} - -std::vector FusionKey(const Shader* front, const Shader* back, const GeometryLayout& layout) { - std::vector key; - constexpr auto headerBytes = offsetof(Shader, num_sh_registers) + sizeof(Shader::num_sh_registers); - AppendBytes(key, front, headerBytes); - AppendBytes(key, back, headerBytes); - AppendBytes(key, const_cast(front->code), layout.frontBytes); - AppendBytes(key, const_cast(back->code), back->shader_size); - AppendBytes(key, layout.sh.data(), layout.sh.size() * sizeof(ShaderRegister)); - AppendBytes(key, layout.cx.data(), layout.cx.size() * sizeof(ShaderRegister)); - AppendBytes(key, back->specials, back->special_sizes_bytes); - AppendBytes(key, front->input_semantics, front->num_input_semantics * sizeof(ShaderSemantic)); - AppendBytes(key, back->output_semantics, back->num_output_semantics * sizeof(ShaderSemantic)); - if (back->user_data == nullptr) return key; - AppendBytes(key, back->user_data, offsetof(ShaderUserData, sharp_resource_count) + sizeof(ShaderUserData::sharp_resource_count)); - AppendBytes(key, back->user_data->direct_resource_offset, back->user_data->direct_resource_count * sizeof(std::uint16_t)); - for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) AppendBytes(key, back->user_data->sharp_resource_offset[i], back->user_data->sharp_resource_count[i] * sizeof(ShaderSharp)); - return key; -} - -FusedProgram BuildFusedProgram(const char* fn, const Shader* front, const Shader* back, GeometryLayout& layout) { - FusedProgram program{std::vector(layout.totalBytes + FusedCodeAlignment - 1), nullptr}; - const auto storage = reinterpret_cast(program.storage.data()); - const auto base = AlignUp(storage, FusedCodeAlignment); - if ((base & SHADER_BASE_ALIGN_MASK) != 0) Fail(fn, "fused shader memory is not a valid program address"); - auto* memory = program.storage.data() + (base - storage); - std::memcpy(memory, const_cast(front->code), layout.frontBytes); - for (std::size_t offset = layout.frontBytes; offset < layout.backOffset; offset += sizeof(SNop)) std::memcpy(memory + offset, &SNop, sizeof(SNop)); - std::memcpy(memory + layout.backOffset, const_cast(back->code), back->shader_size); - SetProgramAddress(layout.sh, SPI_SHADER_PGM_LO_GS, base + layout.backOffset); - if (PatchProgramAddressRegister(layout.sh.data(), static_cast(layout.sh.size()), static_cast(ShaderBinaryType::Gs), base) != 0) Fail(fn, "fused program address patch failed"); - auto* sh = reinterpret_cast(memory + layout.shOffset); - auto* cx = reinterpret_cast(memory + layout.cxOffset); - auto* specials = reinterpret_cast(memory + layout.specialsOffset); - auto* inputs = reinterpret_cast(memory + layout.inputsOffset); - auto* outputs = reinterpret_cast(memory + layout.outputsOffset); - std::memcpy(sh, layout.sh.data(), layout.sh.size() * sizeof(ShaderRegister)); - std::memcpy(cx, layout.cx.data(), layout.cx.size() * sizeof(ShaderRegister)); - if (back->special_sizes_bytes != 0) std::memcpy(specials, back->specials, back->special_sizes_bytes); - if (front->num_input_semantics != 0) std::memcpy(inputs, front->input_semantics, front->num_input_semantics * sizeof(ShaderSemantic)); - if (back->num_output_semantics != 0) std::memcpy(outputs, back->output_semantics, back->num_output_semantics * sizeof(ShaderSemantic)); - Shader fused = *back; - if (back->user_data != nullptr) { - auto* userData = reinterpret_cast(memory + layout.userDataOffset); - *userData = *back->user_data; - auto* direct = reinterpret_cast(memory + layout.directOffset); - if (userData->direct_resource_count != 0) std::memcpy(direct, back->user_data->direct_resource_offset, userData->direct_resource_count * sizeof(std::uint16_t)); - userData->direct_resource_offset = userData->direct_resource_count != 0 ? direct : nullptr; - for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) { - auto* sharps = reinterpret_cast(memory + layout.sharpOffsets[i]); - if (userData->sharp_resource_count[i] != 0) std::memcpy(sharps, back->user_data->sharp_resource_offset[i], userData->sharp_resource_count[i] * sizeof(ShaderSharp)); - userData->sharp_resource_offset[i] = userData->sharp_resource_count[i] != 0 ? sharps : nullptr; - } - fused.user_data = userData; - } - fused.header_size = static_cast(layout.totalBytes - layout.headerOffset); - fused.code = memory; - fused.sh_registers = sh; - fused.cx_registers = layout.cx.empty() ? nullptr : cx; - fused.specials = back->special_sizes_bytes != 0 ? specials : nullptr; - fused.input_semantics = front->num_input_semantics != 0 ? inputs : nullptr; - fused.output_semantics = back->num_output_semantics != 0 ? outputs : nullptr; - fused.shader_size = static_cast(layout.codeBytes); - fused.num_input_semantics = front->num_input_semantics; - fused.scratch_size_dw_per_thread = std::max(front->scratch_size_dw_per_thread, back->scratch_size_dw_per_thread); - fused.type = static_cast(ShaderBinaryType::Gs); - fused.num_sh_registers = static_cast(layout.sh.size()); - fused.num_cx_registers = static_cast(layout.cx.size()); - program.header = reinterpret_cast(memory + layout.headerOffset); - *program.header = fused; - AgcDriverRegisterShader_nid_postfix(program.header); - return program; -} - -int FuseGeometryHalves(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { - constexpr auto fn = "sceAgcFuseShaderHalves_0200"; - if (scratch_mem == nullptr) Fail(fn, "geometry halves need fused shader memory"); - auto layout = ComputeGeometryLayout(fn, front, back); - auto key = FusionKey(front, back, layout); - std::lock_guard lock(FusedProgramsMutex); - auto found = FusedPrograms.find(key); - if (found == FusedPrograms.end()) found = FusedPrograms.emplace(std::move(key), BuildFusedProgram(fn, front, back, layout)).first; - Shader fused = *found->second.header; - std::memcpy(scratch_mem, fused.sh_registers, std::size_t{fused.num_sh_registers} * sizeof(ShaderRegister)); - fused.sh_registers = static_cast(scratch_mem); - *fused_result = fused; - return 0; -} - -} - -extern "C" { - -int APS5_VABI sceAgcFuseShaderHalves_0200(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { - if (fused_result == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; - if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; - const bool isGs = GeometryHalves(front); - const std::uint32_t stageBit = isGs ? (1u << 22u) : (1u << 21u); - if (((front->specials->vgt_shader_stages_en.value ^ back->specials->vgt_shader_stages_en.value) & stageBit) != 0) { - return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; - } - if (isGs) return FuseGeometryHalves(fused_result, front, back, scratch_mem); - *fused_result = *back; - fused_result->type = static_cast(ShaderBinaryType::Hs); - if (scratch_mem != nullptr) { - auto* regs = static_cast(scratch_mem); - std::memcpy(regs, back->sh_registers, std::size_t{back->num_sh_registers} * sizeof(ShaderRegister)); - fused_result->sh_registers = regs; - } - ShaderRegister* fused = fused_result->sh_registers; - const std::uint32_t fusedCount = fused_result->num_sh_registers; - const std::uint32_t frontCount = front->num_sh_registers; - for (std::uint32_t occurrence = 0; occurrence < 2; ++occurrence) { - FindRegister(fused, fusedCount, SPI_SHADER_PGM_CHKSUM_HS, occurrence).value = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_CHKSUM_HS, occurrence).value; - } - const auto& frontRsrc1 = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_RSRC1_HS); - const auto& frontRsrc2 = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_RSRC2_HS); - auto& fusedRsrc1 = FindRegister(fused, fusedCount, SPI_SHADER_PGM_RSRC1_HS); - auto& fusedRsrc2 = FindRegister(fused, fusedCount, SPI_SHADER_PGM_RSRC2_HS); - const std::uint32_t frontVgprs = ((frontRsrc1.value & 0x3fu) + 1u) * 4u; - const std::uint32_t backVgprs = ((fusedRsrc1.value & 0x3fu) + 1u) * 4u; - const std::uint32_t frontTotal = frontVgprs + (frontRsrc2.value >> 28u) * 8u; - const std::uint32_t backTotal = backVgprs + (fusedRsrc2.value >> 28u) * 8u; - const std::uint32_t maxTotal = std::max(frontTotal, backTotal); - const std::uint32_t shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u; - fusedRsrc2.value = (fusedRsrc2.value & 0x0fffffffu) | ((shared & 0xfu) << 28u); - MergeMax(fusedRsrc1, frontRsrc1, 0, 0x3fu); - MergeMax(fusedRsrc1, frontRsrc1, 28, 0x3u); - fusedRsrc2.value = (fusedRsrc2.value & 0xf7ffffc1u) | (frontRsrc2.value & 0x0800003eu); - auto& loRegister = FindRegister(fused, fusedCount, SPI_SHADER_PGM_LO_LS); - auto& hiRegister = FindRegister(fused, fusedCount, SPI_SHADER_PGM_LO_LS + 1u); - const auto address = reinterpret_cast(front->code); - loRegister.value = static_cast(address >> 8u); - hiRegister.value = (hiRegister.value & 0xffffff00u) | static_cast((address >> 40u) & 0xffu); - fused_result->user_data = nullptr; - return 0; -} - -int APS5_VABI sceAgcFuseShaderHalves_nid_postfix(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { - return sceAgcFuseShaderHalves_0200(fused_result, front, back, scratch_mem); -} - -int APS5_VABI sceAgcGetFusedShaderSize_0080(SizeAlign* dst, const Shader* front, const Shader* back) { - if (dst == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; - if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; - if (GeometryHalves(front)) { - const auto layout = ComputeGeometryLayout("sceAgcGetFusedShaderSize_0080", front, back); - dst->m_size = layout.sh.size() * sizeof(ShaderRegister); - dst->m_align = 4; - return 0; - } - dst->m_size = std::uint64_t{back->num_sh_registers} * sizeof(ShaderRegister); - dst->m_align = 4; - return 0; -} - -} +#include "prx/libSceAgc/Misc/include/ShaderFusion.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" +#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" +#include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" + +namespace { + +using namespace ShaderRegs; + +constexpr int GRAPHICS5_ERROR_INVALID_SHADER_HALVES = static_cast(0x8a6c0008u); +constexpr std::uint32_t SPI_SHADER_PGM_CHKSUM_GS = 0x080u; +constexpr std::uint32_t SPI_SHADER_PGM_RSRC1_GS = 0x08Au; +constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_GS = 0x08Bu; +constexpr std::uint32_t SPI_SHADER_PGM_CHKSUM_HS = 0x100u; +constexpr std::uint32_t SPI_SHADER_PGM_RSRC1_HS = 0x10Au; +constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_HS = 0x10Bu; +constexpr std::uint32_t SCodeEnd = 0xbf9f0000u; +constexpr std::uint32_t SNop = 0xbf800000u; +constexpr std::uint32_t SSetpcS6 = 0xbe802006u; +constexpr std::uint32_t SSwappcNullS6 = 0xbefd2106u; +constexpr std::size_t FusedCodeAlignment = 256; +constexpr char TrailerMagic[8] = {'b', 'a', 'r', 'e', 'f', 'o', 'o', 't'}; +constexpr std::size_t TrailerCodeBytesOffset = sizeof(TrailerMagic) + 3 * sizeof(std::uint32_t); +constexpr std::uint32_t Rsrc1VgprMask = 0x3fu; +constexpr std::uint32_t Rsrc2OffchipLdsEnable = 1u << 18u; +constexpr std::uint32_t Rsrc2UserSgprMask = (0x1fu << 1u) | (1u << 27u); + +bool ValidHalves(const Shader* front, const Shader* back) { + const auto frontType = static_cast(front->type); + const auto backType = static_cast(back->type); + return (frontType == ShaderBinaryType::GsFront && backType == ShaderBinaryType::GsBack) || + (frontType == ShaderBinaryType::HsFront && backType == ShaderBinaryType::HsBack); +} + +bool GeometryHalves(const Shader* front) { + return static_cast(front->type) == ShaderBinaryType::GsFront; +} + +ShaderRegister& FindRegister(ShaderRegister* regs, std::uint32_t count, std::uint32_t offset, std::uint32_t occurrence = 0) { + for (std::uint32_t i = 0; regs != nullptr && i < count; ++i) { + if (regs[i].offset == offset && occurrence-- == 0) return regs[i]; + } + throw std::runtime_error("sceAgcFuseShaderHalves_0200: shader half lacks register " + std::to_string(offset)); +} + +void MergeMax(ShaderRegister& dst, const ShaderRegister& src, std::uint32_t shift, std::uint32_t mask) { + const auto field = std::max((dst.value >> shift) & mask, (src.value >> shift) & mask); + dst.value = (dst.value & ~(mask << shift)) | (field << shift); +} + +struct GeometryLayout { + std::uint32_t frontBytes; + std::size_t backOffset; + std::size_t codeBytes; + std::vector sh; + std::vector cx; + std::size_t headerOffset; + std::size_t shOffset; + std::size_t cxOffset; + std::size_t specialsOffset; + std::size_t inputsOffset; + std::size_t outputsOffset; + std::size_t userDataOffset; + std::size_t directOffset; + std::array sharpOffsets; + std::size_t totalBytes; +}; + +[[noreturn]] void Fail(const char* function, const std::string& message) { + throw std::runtime_error(std::string(function) + ": " + message); +} + +std::size_t AlignUp(std::size_t value, std::size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +void ValidateGeometryHalf(const char* function, const Shader* shader) { + if (shader->file_header != SHADER_FILE_HEADER_MAGIC || shader->version != SHADER_VERSION) Fail(function, "invalid shader header or version"); + if (shader->code == nullptr || shader->shader_size == 0 || (shader->shader_size & 3u) != 0) Fail(function, "invalid shader half code"); + if (shader->embedded_constant_buffer_size_dqw != 0) Fail(function, "fusing halves with an embedded constant buffer is not implemented"); + if ((shader->num_sh_registers != 0 && shader->sh_registers == nullptr) || (shader->num_cx_registers != 0 && shader->cx_registers == nullptr)) Fail(function, "shader half registers missing"); +} + +std::uint32_t FrontProgramBytes(const char* function, const Shader* front) { + const auto* bytes = static_cast(const_cast(front->code)); + const std::size_t size = front->shader_size; + if (size < TrailerCodeBytesOffset + sizeof(std::uint32_t)) Fail(function, "front half has no program trailer"); + std::size_t trailer = size; + for (std::size_t offset = size - TrailerCodeBytesOffset - sizeof(std::uint32_t) + 1; offset-- > 0;) { + if (std::memcmp(bytes + offset, TrailerMagic, sizeof(TrailerMagic)) == 0) { + trailer = offset; + break; + } + } + if (trailer == size) Fail(function, "front half has no program trailer"); + std::uint32_t codeBytes = 0; + std::memcpy(&codeBytes, bytes + trailer + TrailerCodeBytesOffset, sizeof(codeBytes)); + if (codeBytes == 0 || (codeBytes & 3u) != 0 || codeBytes > trailer) Fail(function, "front half trailer has an invalid code size"); + std::size_t end = codeBytes / 4; + const auto word = [&](std::size_t index) { + std::uint32_t value = 0; + std::memcpy(&value, bytes + index * 4, sizeof(value)); + return value; + }; + while (end > 0 && word(end - 1) == SCodeEnd) --end; + if (end == 0 || (word(end - 1) != SSetpcS6 && word(end - 1) != SSwappcNullS6)) Fail(function, "front half does not end with s_setpc_b64 s[6:7] or s_swappc_b64 null, s[6:7]"); + return static_cast((end - 1) * 4); +} + +std::array mergeGeometryResources(std::uint32_t frontRsrc1, std::uint32_t frontRsrc2, std::uint32_t backRsrc1, std::uint32_t backRsrc2) { + const auto frontVgprs = ((frontRsrc1 & Rsrc1VgprMask) + 1u) * 4u; + const auto backVgprs = ((backRsrc1 & Rsrc1VgprMask) + 1u) * 4u; + const auto frontTotal = frontVgprs + (frontRsrc2 >> 28u) * 8u; + const auto backTotal = backVgprs + (backRsrc2 >> 28u) * 8u; + const auto maxTotal = std::max(frontTotal, backTotal); + const auto shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u; + const auto vgprs = std::max(frontRsrc1 & Rsrc1VgprMask, backRsrc1 & Rsrc1VgprMask); + const auto rsrc1Field = std::max((frontRsrc1 >> 29u) & 3u, (backRsrc1 >> 29u) & 3u); + const auto rsrc2Field = std::max((frontRsrc2 >> 16u) & 3u, (backRsrc2 >> 16u) & 3u); + const auto rsrc1 = (backRsrc1 & ~(Rsrc1VgprMask | (3u << 29u))) | vgprs | (rsrc1Field << 29u); + const auto rsrc2 = (backRsrc2 & ~(0xf0000000u | (3u << 16u) | Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)) | ((shared & 15u) << 28u) | (rsrc2Field << 16u) | (frontRsrc2 & (Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)); + return {rsrc1, rsrc2}; +} + +const ShaderRegister* FindHalfRegister(const Shader* shader, std::uint32_t offset) { + for (std::uint32_t i = 0; i < shader->num_sh_registers; ++i) { + if (shader->sh_registers[i].offset == offset) return &shader->sh_registers[i]; + } + return nullptr; +} + +GeometryLayout ComputeGeometryLayout(const char* function, const Shader* front, const Shader* back) { + ValidateGeometryHalf(function, back); + ValidateGeometryHalf(function, front); + GeometryLayout layout{}; + layout.frontBytes = FrontProgramBytes(function, front); + layout.backOffset = AlignUp(layout.frontBytes, FusedCodeAlignment); + layout.codeBytes = layout.backOffset + back->shader_size; + const auto* frontRsrc1 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC1_GS); + const auto* frontRsrc2 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC2_GS); + if (frontRsrc1 == nullptr || frontRsrc2 == nullptr) Fail(function, "front half has no GS resource registers"); + const auto* backRsrc1 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC1_GS); + const auto* backRsrc2 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC2_GS); + if (backRsrc1 == nullptr || backRsrc2 == nullptr) Fail(function, "back half has no GS resource registers"); + const auto resources = mergeGeometryResources(frontRsrc1->value, frontRsrc2->value, backRsrc1->value, backRsrc2->value); + layout.sh.assign(back->sh_registers, back->sh_registers + back->num_sh_registers); + for (auto& reg : layout.sh) { + if (reg.offset == SPI_SHADER_PGM_RSRC1_GS) reg.value = resources[0]; + else if (reg.offset == SPI_SHADER_PGM_RSRC2_GS) reg.value = resources[1]; + } + for (std::uint32_t i = 0; i < front->num_sh_registers; ++i) { + const auto& reg = front->sh_registers[i]; + if (reg.offset == SPI_SHADER_PGM_CHKSUM_GS || FindHalfRegister(back, reg.offset) != nullptr) continue; + layout.sh.push_back(reg); + } + layout.cx.assign(back->cx_registers, back->cx_registers + back->num_cx_registers); + for (std::uint32_t i = 0; i < front->num_cx_registers; ++i) { + const auto& reg = front->cx_registers[i]; + if (std::none_of(layout.cx.begin(), layout.cx.end(), [&](const ShaderRegister& other) { return other.offset == reg.offset; })) layout.cx.push_back(reg); + } + if (layout.sh.size() > 0xff || layout.cx.size() > 0xff) Fail(function, "fused register count exceeds the shader header"); + layout.headerOffset = AlignUp(layout.codeBytes, alignof(Shader)); + layout.shOffset = AlignUp(layout.headerOffset + sizeof(Shader), alignof(ShaderRegister)); + layout.cxOffset = layout.shOffset + layout.sh.size() * sizeof(ShaderRegister); + layout.specialsOffset = AlignUp(layout.cxOffset + layout.cx.size() * sizeof(ShaderRegister), alignof(ShaderSpecialRegs)); + layout.inputsOffset = AlignUp(layout.specialsOffset + back->special_sizes_bytes, alignof(ShaderSemantic)); + layout.outputsOffset = layout.inputsOffset + front->num_input_semantics * sizeof(ShaderSemantic); + layout.userDataOffset = AlignUp(layout.outputsOffset + back->num_output_semantics * sizeof(ShaderSemantic), alignof(ShaderUserData)); + auto end = layout.userDataOffset + (back->user_data != nullptr ? sizeof(ShaderUserData) : 0); + layout.directOffset = AlignUp(end, alignof(std::uint16_t)); + if (back->user_data != nullptr) end = layout.directOffset + back->user_data->direct_resource_count * sizeof(std::uint16_t); + for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) { + layout.sharpOffsets[i] = AlignUp(end, alignof(ShaderSharp)); + if (back->user_data != nullptr) end = layout.sharpOffsets[i] + back->user_data->sharp_resource_count[i] * sizeof(ShaderSharp); + } + layout.totalBytes = end; + return layout; +} + +void SetProgramAddress(std::vector& regs, std::uint32_t loOffset, std::uint64_t address) { + for (std::size_t i = 0; i + 1 < regs.size(); ++i) { + if (regs[i].offset != loOffset || regs[i + 1].offset != loOffset + 1u) continue; + regs[i].value = static_cast(address >> 8u); + regs[i + 1].value = (regs[i + 1].value & 0xFFFFFF00u) | static_cast((address >> 40u) & 0xFFu); + return; + } + throw std::runtime_error("sceAgcFuseShaderHalves_0200: back half has no program address register pair"); +} + +struct FusedProgram { + std::vector storage; + Shader* header; +}; + +std::mutex FusedProgramsMutex; +std::map, FusedProgram> FusedPrograms; + +void AppendBytes(std::vector& bytes, const void* data, std::size_t size) { + if (size == 0) return; + const auto* begin = static_cast(data); + bytes.insert(bytes.end(), begin, begin + size); +} + +std::vector FusionKey(const Shader* front, const Shader* back, const GeometryLayout& layout) { + std::vector key; + constexpr auto headerBytes = offsetof(Shader, num_sh_registers) + sizeof(Shader::num_sh_registers); + AppendBytes(key, front, headerBytes); + AppendBytes(key, back, headerBytes); + AppendBytes(key, const_cast(front->code), layout.frontBytes); + AppendBytes(key, const_cast(back->code), back->shader_size); + AppendBytes(key, layout.sh.data(), layout.sh.size() * sizeof(ShaderRegister)); + AppendBytes(key, layout.cx.data(), layout.cx.size() * sizeof(ShaderRegister)); + AppendBytes(key, back->specials, back->special_sizes_bytes); + AppendBytes(key, front->input_semantics, front->num_input_semantics * sizeof(ShaderSemantic)); + AppendBytes(key, back->output_semantics, back->num_output_semantics * sizeof(ShaderSemantic)); + if (back->user_data == nullptr) return key; + AppendBytes(key, back->user_data, offsetof(ShaderUserData, sharp_resource_count) + sizeof(ShaderUserData::sharp_resource_count)); + AppendBytes(key, back->user_data->direct_resource_offset, back->user_data->direct_resource_count * sizeof(std::uint16_t)); + for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) AppendBytes(key, back->user_data->sharp_resource_offset[i], back->user_data->sharp_resource_count[i] * sizeof(ShaderSharp)); + return key; +} + +FusedProgram BuildFusedProgram(const char* fn, const Shader* front, const Shader* back, GeometryLayout& layout) { + FusedProgram program{std::vector(layout.totalBytes + FusedCodeAlignment - 1), nullptr}; + const auto storage = reinterpret_cast(program.storage.data()); + const auto base = AlignUp(storage, FusedCodeAlignment); + if ((base & SHADER_BASE_ALIGN_MASK) != 0) Fail(fn, "fused shader memory is not a valid program address"); + auto* memory = program.storage.data() + (base - storage); + std::memcpy(memory, const_cast(front->code), layout.frontBytes); + for (std::size_t offset = layout.frontBytes; offset < layout.backOffset; offset += sizeof(SNop)) std::memcpy(memory + offset, &SNop, sizeof(SNop)); + std::memcpy(memory + layout.backOffset, const_cast(back->code), back->shader_size); + SetProgramAddress(layout.sh, SPI_SHADER_PGM_LO_GS, base + layout.backOffset); + if (PatchProgramAddressRegister(layout.sh.data(), static_cast(layout.sh.size()), static_cast(ShaderBinaryType::Gs), base) != 0) Fail(fn, "fused program address patch failed"); + auto* sh = reinterpret_cast(memory + layout.shOffset); + auto* cx = reinterpret_cast(memory + layout.cxOffset); + auto* specials = reinterpret_cast(memory + layout.specialsOffset); + auto* inputs = reinterpret_cast(memory + layout.inputsOffset); + auto* outputs = reinterpret_cast(memory + layout.outputsOffset); + std::memcpy(sh, layout.sh.data(), layout.sh.size() * sizeof(ShaderRegister)); + std::memcpy(cx, layout.cx.data(), layout.cx.size() * sizeof(ShaderRegister)); + if (back->special_sizes_bytes != 0) std::memcpy(specials, back->specials, back->special_sizes_bytes); + if (front->num_input_semantics != 0) std::memcpy(inputs, front->input_semantics, front->num_input_semantics * sizeof(ShaderSemantic)); + if (back->num_output_semantics != 0) std::memcpy(outputs, back->output_semantics, back->num_output_semantics * sizeof(ShaderSemantic)); + Shader fused = *back; + if (back->user_data != nullptr) { + auto* userData = reinterpret_cast(memory + layout.userDataOffset); + *userData = *back->user_data; + auto* direct = reinterpret_cast(memory + layout.directOffset); + if (userData->direct_resource_count != 0) std::memcpy(direct, back->user_data->direct_resource_offset, userData->direct_resource_count * sizeof(std::uint16_t)); + userData->direct_resource_offset = userData->direct_resource_count != 0 ? direct : nullptr; + for (std::size_t i = 0; i < layout.sharpOffsets.size(); ++i) { + auto* sharps = reinterpret_cast(memory + layout.sharpOffsets[i]); + if (userData->sharp_resource_count[i] != 0) std::memcpy(sharps, back->user_data->sharp_resource_offset[i], userData->sharp_resource_count[i] * sizeof(ShaderSharp)); + userData->sharp_resource_offset[i] = userData->sharp_resource_count[i] != 0 ? sharps : nullptr; + } + fused.user_data = userData; + } + fused.header_size = static_cast(layout.totalBytes - layout.headerOffset); + fused.code = memory; + fused.sh_registers = sh; + fused.cx_registers = layout.cx.empty() ? nullptr : cx; + fused.specials = back->special_sizes_bytes != 0 ? specials : nullptr; + fused.input_semantics = front->num_input_semantics != 0 ? inputs : nullptr; + fused.output_semantics = back->num_output_semantics != 0 ? outputs : nullptr; + fused.shader_size = static_cast(layout.codeBytes); + fused.num_input_semantics = front->num_input_semantics; + fused.scratch_size_dw_per_thread = std::max(front->scratch_size_dw_per_thread, back->scratch_size_dw_per_thread); + fused.type = static_cast(ShaderBinaryType::Gs); + fused.num_sh_registers = static_cast(layout.sh.size()); + fused.num_cx_registers = static_cast(layout.cx.size()); + program.header = reinterpret_cast(memory + layout.headerOffset); + *program.header = fused; + AgcDriverRegisterShader_nid_postfix(program.header); + return program; +} + +int FuseGeometryHalves(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { + constexpr auto fn = "sceAgcFuseShaderHalves_0200"; + if (scratch_mem == nullptr) Fail(fn, "geometry halves need fused shader memory"); + auto layout = ComputeGeometryLayout(fn, front, back); + auto key = FusionKey(front, back, layout); + std::lock_guard lock(FusedProgramsMutex); + auto found = FusedPrograms.find(key); + if (found == FusedPrograms.end()) found = FusedPrograms.emplace(std::move(key), BuildFusedProgram(fn, front, back, layout)).first; + Shader fused = *found->second.header; + std::memcpy(scratch_mem, fused.sh_registers, std::size_t{fused.num_sh_registers} * sizeof(ShaderRegister)); + fused.sh_registers = static_cast(scratch_mem); + *fused_result = fused; + return 0; +} + +} + +extern "C" { + +int APS5_VABI sceAgcFuseShaderHalves_0200(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { + if (fused_result == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; + if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + const bool isGs = GeometryHalves(front); + const std::uint32_t stageBit = isGs ? (1u << 22u) : (1u << 21u); + if (((front->specials->vgt_shader_stages_en.value ^ back->specials->vgt_shader_stages_en.value) & stageBit) != 0) { + return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + } + if (isGs) return FuseGeometryHalves(fused_result, front, back, scratch_mem); + *fused_result = *back; + fused_result->type = static_cast(ShaderBinaryType::Hs); + if (scratch_mem != nullptr) { + auto* regs = static_cast(scratch_mem); + std::memcpy(regs, back->sh_registers, std::size_t{back->num_sh_registers} * sizeof(ShaderRegister)); + fused_result->sh_registers = regs; + } + ShaderRegister* fused = fused_result->sh_registers; + const std::uint32_t fusedCount = fused_result->num_sh_registers; + const std::uint32_t frontCount = front->num_sh_registers; + for (std::uint32_t occurrence = 0; occurrence < 2; ++occurrence) { + FindRegister(fused, fusedCount, SPI_SHADER_PGM_CHKSUM_HS, occurrence).value = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_CHKSUM_HS, occurrence).value; + } + const auto& frontRsrc1 = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_RSRC1_HS); + const auto& frontRsrc2 = FindRegister(front->sh_registers, frontCount, SPI_SHADER_PGM_RSRC2_HS); + auto& fusedRsrc1 = FindRegister(fused, fusedCount, SPI_SHADER_PGM_RSRC1_HS); + auto& fusedRsrc2 = FindRegister(fused, fusedCount, SPI_SHADER_PGM_RSRC2_HS); + const std::uint32_t frontVgprs = ((frontRsrc1.value & 0x3fu) + 1u) * 4u; + const std::uint32_t backVgprs = ((fusedRsrc1.value & 0x3fu) + 1u) * 4u; + const std::uint32_t frontTotal = frontVgprs + (frontRsrc2.value >> 28u) * 8u; + const std::uint32_t backTotal = backVgprs + (fusedRsrc2.value >> 28u) * 8u; + const std::uint32_t maxTotal = std::max(frontTotal, backTotal); + const std::uint32_t shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u; + fusedRsrc2.value = (fusedRsrc2.value & 0x0fffffffu) | ((shared & 0xfu) << 28u); + MergeMax(fusedRsrc1, frontRsrc1, 0, 0x3fu); + MergeMax(fusedRsrc1, frontRsrc1, 28, 0x3u); + fusedRsrc2.value = (fusedRsrc2.value & 0xf7ffffc1u) | (frontRsrc2.value & 0x0800003eu); + auto& loRegister = FindRegister(fused, fusedCount, SPI_SHADER_PGM_LO_LS); + auto& hiRegister = FindRegister(fused, fusedCount, SPI_SHADER_PGM_LO_LS + 1u); + const auto address = reinterpret_cast(front->code); + loRegister.value = static_cast(address >> 8u); + hiRegister.value = (hiRegister.value & 0xffffff00u) | static_cast((address >> 40u) & 0xffu); + fused_result->user_data = nullptr; + return 0; +} + +int APS5_VABI sceAgcFuseShaderHalves_nid_postfix(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { + return sceAgcFuseShaderHalves_0200(fused_result, front, back, scratch_mem); +} + +int APS5_VABI sceAgcGetFusedShaderSize_0080(SizeAlign* dst, const Shader* front, const Shader* back) { + if (dst == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; + if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + if (GeometryHalves(front)) { + const auto layout = ComputeGeometryLayout("sceAgcGetFusedShaderSize_0080", front, back); + dst->m_size = layout.sh.size() * sizeof(ShaderRegister); + dst->m_align = 4; + return 0; + } + dst->m_size = std::uint64_t{back->num_sh_registers} * sizeof(ShaderRegister); + dst->m_align = 4; + return 0; +} + +} diff --git a/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp index 9bf3fb0bf0..6d05a7a681 100644 --- a/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp @@ -1,181 +1,180 @@ -#include "prx/libSceAgc/Shader/include/InterpolantMapping.hpp" - -#include -#include -#include -#include -#include - -#include "SceShaders.hpp" -#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" -#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" -#include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" -#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" - -namespace { - -constexpr std::uint32_t InterpolantRegisterCount = 32; -constexpr std::uint32_t InterpolantF16HiOnly = 2; - -using CreateInterpolantValueFn = std::uint32_t (*)(std::uint32_t psWord, const ShaderSemantic* gsSemantic); - -std::uint32_t CreateInterpolantValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { - const std::uint32_t value = ((psWord & 0x00300000u) != 0) - ? CreateInterpolantF16Value(psWord, gsSemantic) - : CreateInterpolantNonF16Value(psWord, gsSemantic); - - return (gsSemantic == nullptr) - ? CreateInterpolantDefaultValue(value, psWord) - : CreateInterpolantMappingValue(value, psWord, ShaderSemanticWord(*gsSemantic)); -} - -std::uint32_t CreateInterpolantF16HiValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { - std::uint32_t value = ((psWord << 4u) & 0x03000000u) | 0x00080000u; - if (gsSemantic == nullptr || (psWord & ShaderSemanticWord(*gsSemantic) & 0x00200000u) == 0) { - value |= 0x00000020u; - } - value = ApplyInterpolantDefaultValueHi(value, psWord); - value |= ((psWord >> 30u) & 0x3u) << 8u; - if (gsSemantic == nullptr) { - return value; - } - - value |= (ShaderSemanticWord(*gsSemantic) >> 8u) & 0x1Fu; - if ((psWord & 0x01400000u) != 0) { - value |= 0x00000400u; - } - return value; -} - -std::uint32_t CreateInterpolantValueSplitF16(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { - return (((psWord >> 20u) & 0x3u) == InterpolantF16HiOnly) - ? CreateInterpolantF16HiValue(psWord, gsSemantic) - : CreateInterpolantValue(psWord, gsSemantic); -} - -std::uint32_t CreateInterpolantValueFlatDefault(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { - const std::uint32_t value = CreateInterpolantValue(psWord, gsSemantic); - return ((psWord & 0x00700000u) == 0x00400000u) ? (value | 0x00000020u) : value; -} - -std::uint32_t ApplyInterpolantDefaultValue0100(std::uint32_t value, std::uint32_t psWord) { - constexpr std::uint32_t encoded[4] = {0u, 2u, 1u, 3u}; - value &= ~0x00000300u; - value |= encoded[(psWord >> 28u) & 0x3u] << 8u; - return value; -} - -int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* gs, const Shader* ps, CreateInterpolantValueFn createValue, bool fillIdentityTail) { - if (regs == nullptr) { - throw std::runtime_error(std::string(fn) + ": regs is null"); - } - if (ps == nullptr && !fillIdentityTail) { - throw std::runtime_error(std::string(fn) + ": ps is null"); - } - if (ps != nullptr && ps->num_input_semantics > 32) { - throw std::runtime_error(std::string(fn) + ": input semantic count exceeds 32"); - } - - AgcDriver::ShaderPreparationScope transaction({ps != nullptr ? gs : nullptr, ps}); - std::array values{}; - auto* output = regs; - regs = values.data(); - if (ps == nullptr || ps->num_input_semantics == 0) { - const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : 0; - if (fillIdentityTail) FillIdentityInterpolants(regs, 0); - if (ps != nullptr) { - AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {}); - if (gs != nullptr) AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); - } - transaction.Commit(); - std::copy_n(values.begin(), count, output); - return 0; - } - - if (ps->num_input_semantics != 0 && ps->input_semantics == nullptr) { - throw std::runtime_error(std::string(fn) + ": ps->input_semantics is null but num_input_semantics != 0"); - } - - if (ps->num_input_semantics > InterpolantRegisterCount) { - throw std::runtime_error(std::string(fn) + ": ps->num_input_semantics exceeds the interpolant register count"); - } - - if (gs == nullptr) { - throw std::runtime_error(std::string(fn) + ": gs is null"); - } - - if (gs->num_output_semantics != 0 && gs->output_semantics == nullptr) { - throw std::runtime_error(std::string(fn) + ": gs->output_semantics is null but num_output_semantics != 0"); - } - - for (std::uint32_t i = 0; i < ps->num_input_semantics; ++i) { - const ShaderSemantic& psSemantic = ps->input_semantics[i]; - const ShaderSemantic* gsSemantic = FindOutputSemantic(gs, psSemantic.semantic); - SetInterpolantRegister(regs, i, createValue(ShaderSemanticWord(psSemantic), gsSemantic)); - } - - const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : ps->num_input_semantics; - if (fillIdentityTail) FillIdentityInterpolants(regs, ps->num_input_semantics); - AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {}); - AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); - transaction.Commit(); - std::copy_n(values.begin(), count, output); - return 0; -} - -} - -extern "C" { - -int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue, true); -} - -int APS5_VABI sceAgcUpdateInterpolantMapping_0100(ShaderRegister* regs, const Shader* vs, const Shader* ps) { - if (regs == nullptr) { - throw std::runtime_error(std::string(__func__) + ": regs is null"); - } - if (ps == nullptr) { - throw std::runtime_error(std::string(__func__) + ": ps is null"); - } - if (ps->num_input_semantics > InterpolantRegisterCount) { - throw std::runtime_error(std::string(__func__) + ": input semantic count exceeds 32"); - } - if (ps->num_input_semantics == 0) { - return 0; - } - if (ps->input_semantics == nullptr) { - throw std::runtime_error(std::string(__func__) + ": ps->input_semantics is null but num_input_semantics != 0"); - } - if (vs == nullptr) { - throw std::runtime_error(std::string(__func__) + ": vs is null"); - } - if (vs->num_output_semantics != 0 && vs->output_semantics == nullptr) { - throw std::runtime_error(std::string(__func__) + ": vs->output_semantics is null but num_output_semantics != 0"); - } - for (std::uint32_t i = 0; i < ps->num_input_semantics; ++i) { - const ShaderSemantic& psSemantic = ps->input_semantics[i]; - const std::uint32_t psWord = ShaderSemanticWord(psSemantic); - const ShaderSemantic* vsSemantic = FindOutputSemantic(vs, psSemantic.semantic); - SetInterpolantRegister(regs, i, ApplyInterpolantDefaultValue0100(CreateInterpolantValue(psWord, vsSemantic), psWord)); - } - return 0; -} - -int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, true); -} - -int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, false); -} - -int APS5_VABI sceAgcCreateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, true); -} - -int APS5_VABI sceAgcUpdateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) { - return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, false); -} - -} +#include "prx/libSceAgc/Shader/include/InterpolantMapping.hpp" + +#include +#include +#include +#include +#include + +#include "SceShaders.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" +#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" + +namespace { + +constexpr std::uint32_t InterpolantRegisterCount = 32; +constexpr std::uint32_t InterpolantF16HiOnly = 2; + +using CreateInterpolantValueFn = std::uint32_t (*)(std::uint32_t psWord, const ShaderSemantic* gsSemantic); + +std::uint32_t CreateInterpolantValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { + const std::uint32_t value = ((psWord & 0x00300000u) != 0) + ? CreateInterpolantF16Value(psWord, gsSemantic) + : CreateInterpolantNonF16Value(psWord, gsSemantic); + + return (gsSemantic == nullptr) + ? CreateInterpolantDefaultValue(value, psWord) + : CreateInterpolantMappingValue(value, psWord, ShaderSemanticWord(*gsSemantic)); +} + +std::uint32_t CreateInterpolantF16HiValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { + std::uint32_t value = ((psWord << 4u) & 0x03000000u) | 0x00080000u; + if (gsSemantic == nullptr || (psWord & ShaderSemanticWord(*gsSemantic) & 0x00200000u) == 0) { + value |= 0x00000020u; + } + value = ApplyInterpolantDefaultValueHi(value, psWord); + value |= ((psWord >> 30u) & 0x3u) << 8u; + if (gsSemantic == nullptr) { + return value; + } + + value |= (ShaderSemanticWord(*gsSemantic) >> 8u) & 0x1Fu; + if ((psWord & 0x01400000u) != 0) { + value |= 0x00000400u; + } + return value; +} + +std::uint32_t CreateInterpolantValueSplitF16(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { + return (((psWord >> 20u) & 0x3u) == InterpolantF16HiOnly) + ? CreateInterpolantF16HiValue(psWord, gsSemantic) + : CreateInterpolantValue(psWord, gsSemantic); +} + +std::uint32_t CreateInterpolantValueFlatDefault(std::uint32_t psWord, const ShaderSemantic* gsSemantic) { + const std::uint32_t value = CreateInterpolantValue(psWord, gsSemantic); + return ((psWord & 0x00700000u) == 0x00400000u) ? (value | 0x00000020u) : value; +} + +std::uint32_t ApplyInterpolantDefaultValue0100(std::uint32_t value, std::uint32_t psWord) { + value &= ~0x00000300u; + value |= ((psWord >> 28u) & 0x3u) << 8u; + return value; +} + +int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* gs, const Shader* ps, CreateInterpolantValueFn createValue, bool fillIdentityTail) { + if (regs == nullptr) { + throw std::runtime_error(std::string(fn) + ": regs is null"); + } + if (ps == nullptr && !fillIdentityTail) { + throw std::runtime_error(std::string(fn) + ": ps is null"); + } + if (ps != nullptr && ps->num_input_semantics > 32) { + throw std::runtime_error(std::string(fn) + ": input semantic count exceeds 32"); + } + + AgcDriver::ShaderPreparationScope transaction({ps != nullptr ? gs : nullptr, ps}); + std::array values{}; + auto* output = regs; + regs = values.data(); + if (ps == nullptr || ps->num_input_semantics == 0) { + const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : 0; + if (fillIdentityTail) FillIdentityInterpolants(regs, 0); + if (ps != nullptr) { + AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {}); + if (gs != nullptr) AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); + } + transaction.Commit(); + std::copy_n(values.begin(), count, output); + return 0; + } + + if (ps->num_input_semantics != 0 && ps->input_semantics == nullptr) { + throw std::runtime_error(std::string(fn) + ": ps->input_semantics is null but num_input_semantics != 0"); + } + + if (ps->num_input_semantics > InterpolantRegisterCount) { + throw std::runtime_error(std::string(fn) + ": ps->num_input_semantics exceeds the interpolant register count"); + } + + if (gs == nullptr) { + throw std::runtime_error(std::string(fn) + ": gs is null"); + } + + if (gs->num_output_semantics != 0 && gs->output_semantics == nullptr) { + throw std::runtime_error(std::string(fn) + ": gs->output_semantics is null but num_output_semantics != 0"); + } + + for (std::uint32_t i = 0; i < ps->num_input_semantics; ++i) { + const ShaderSemantic& psSemantic = ps->input_semantics[i]; + const ShaderSemantic* gsSemantic = FindOutputSemantic(gs, psSemantic.semantic); + SetInterpolantRegister(regs, i, createValue(ShaderSemanticWord(psSemantic), gsSemantic)); + } + + const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : ps->num_input_semantics; + if (fillIdentityTail) FillIdentityInterpolants(regs, ps->num_input_semantics); + AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {}); + AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); + transaction.Commit(); + std::copy_n(values.begin(), count, output); + return 0; +} + +} + +extern "C" { + +int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue, true); +} + +int APS5_VABI sceAgcUpdateInterpolantMapping_0100(ShaderRegister* regs, const Shader* vs, const Shader* ps) { + if (regs == nullptr) { + throw std::runtime_error(std::string(__func__) + ": regs is null"); + } + if (ps == nullptr) { + throw std::runtime_error(std::string(__func__) + ": ps is null"); + } + if (ps->num_input_semantics > InterpolantRegisterCount) { + throw std::runtime_error(std::string(__func__) + ": input semantic count exceeds 32"); + } + if (ps->num_input_semantics == 0) { + return 0; + } + if (ps->input_semantics == nullptr) { + throw std::runtime_error(std::string(__func__) + ": ps->input_semantics is null but num_input_semantics != 0"); + } + if (vs == nullptr) { + throw std::runtime_error(std::string(__func__) + ": vs is null"); + } + if (vs->num_output_semantics != 0 && vs->output_semantics == nullptr) { + throw std::runtime_error(std::string(__func__) + ": vs->output_semantics is null but num_output_semantics != 0"); + } + for (std::uint32_t i = 0; i < ps->num_input_semantics; ++i) { + const ShaderSemantic& psSemantic = ps->input_semantics[i]; + const std::uint32_t psWord = ShaderSemanticWord(psSemantic); + const ShaderSemantic* vsSemantic = FindOutputSemantic(vs, psSemantic.semantic); + SetInterpolantRegister(regs, i, ApplyInterpolantDefaultValue0100(CreateInterpolantValue(psWord, vsSemantic), psWord)); + } + return 0; +} + +int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, true); +} + +int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, false); +} + +int APS5_VABI sceAgcCreateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, true); +} + +int APS5_VABI sceAgcUpdateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) { + return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, false); +} + +} diff --git a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp index 880c717e16..3c4107c546 100644 --- a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp @@ -189,8 +189,8 @@ void testUpdate0100Mapping() { regs = filled(); check(sceAgcUpdateInterpolantMapping_0100(regs.data(), &vs, &ps) == 0, "_0100 update mapping failed"); - check(regs[0].value == 0x00000525u, "_0100 update does not encode default value 2 as 1"); - check(regs[1].value == 0x01380207u, "_0100 update does not encode default value 1 as 2"); + check(regs[0].value == 0x00000625u, "_0100 update does not encode default value 2"); + check(regs[1].value == 0x01380107u, "_0100 update does not encode default value 1"); check(regs[2].value == 0x02580020u, "_0100 unmatched interpolant update changed"); check(regs[3].value == 0x03680709u, "_0100 full f16 interpolant update changed"); for (std::uint32_t i = 0; i < inputs.size(); ++i) { From c480d90bd8cd075787bea108f8eb62bca537bc9d Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:44:18 +0000 Subject: [PATCH 268/318] feat(libSceSaveData): declare sceSaveDataGetProgress with its known signature It still throws through NotImplemented_nid_no_patch; the parameter is the float* of the OpenOrbis headers and shadPS4. --- core/libs/prx/libSceSaveData.native/Export.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceSaveData.native/Export.cpp b/core/libs/prx/libSceSaveData.native/Export.cpp index 644b421f49..dfe2c8a92c 100644 --- a/core/libs/prx/libSceSaveData.native/Export.cpp +++ b/core/libs/prx/libSceSaveData.native/Export.cpp @@ -1307,7 +1307,7 @@ int APS5_VABI sceSaveDataGetMountedSaveDataCount() { return 0; } -int APS5_VABI sceSaveDataGetProgress() { +int APS5_VABI sceSaveDataGetProgress(float* progress) { NotImplemented_nid_no_patch(__func__); return 0; } From 2880d7b8f6bed9788ec39b23f7632fdffbe18e5b Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:44:18 +0000 Subject: [PATCH 269/318] docs(libSceSaveData): record the unmeasured signature of sceSaveDataGetProgress --- docs/dev/debt/libSceSaveData.native.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/dev/debt/libSceSaveData.native.md b/docs/dev/debt/libSceSaveData.native.md index c0256b7758..193e5a4d01 100644 --- a/docs/dev/debt/libSceSaveData.native.md +++ b/docs/dev/debt/libSceSaveData.native.md @@ -21,7 +21,8 @@ - [sceSaveDataDirNameSearch, sceSaveDataGetMountInfo, sceSaveDataMount3](../../../core/libs/prx/libSceSaveData.native/Export.cpp) (libSceSaveData.native) - the 64 KiB block of [SharpEmu](https://github.com/sharpemu/sharpemu/blob/c6d304cc3328bad62cb671327ace181110cf2ddb/src/SharpEmu.Libs/SaveData/SaveDataExports.cs#L82-L84) and the 48 to 16384-block range of SharpEmu and [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/569ec04d1a77127c601eed522ad5bba187173f9d/src/libs/libSaveData.cpp#L30-L31) are not measured on a console, and file system overhead is not counted. What the console does with a `blocks` outside that range is unknown: KytyPS5 rejects the creating mount and SharpEmu clamps it; here a larger one is kept as 16384 and a smaller non-zero one as given. A save's size is the `blocks` of the mount that created it, kept in `.blocks`, else that of its next mount that names one, else 16384; its free blocks are that size less its files, each rounded up to 64 KiB blocks - [sceSaveDataInitialize3](../../../core/libs/prx/libSceSaveData.native/Export.cpp) (libSceSaveData.native) - a repeated initialize succeeds: PPSA12544's executable and its Unity SaveData plugin both initialize, and the plugin fails on any error. Initializations are counted and `sceSaveDataTerminate` ends the session at the last one; how the console pairs them is unknown. In PPSA12544 the executable calls it once (0x1721d8f) with 0 as its argument, only while a flag byte of its object is 0, and does not import `sceSaveDataTerminate`; the plugin calls it once (0x1fd2) with 0, skips the call while a flag byte of its own is set, and calls `sceSaveDataTerminate` from two functions (0x2059, 0x20d9) that clear that byte. Of 53 local titles, all import and call it and 28 import `sceSaveDataTerminate` (25 do not); counting tail jumps as calls and ignoring indirect calls, 17 of those 28 call both once, 10 call both twice and 1 calls it once and `sceSaveDataTerminate` twice. - [sceSaveDataCheckIpmiIfSize, sceSaveDataDebug, sceSaveDataDebugCheckBackupData, sceSaveDataDebugCleanMount, sceSaveDataDebugCompiledSdkVersion, sceSaveDataDebugCreateSaveDataRoot, sceSaveDataDebugProspero, sceSaveDataDebugRemoveSaveDataRoot and sceSaveDataDebugTarget](../../../core/libs/prx/libSceSaveData.native/Export.cpp) (libSceSaveData.native) - names and NIDs are present in the [PS5 3.20 export inventory](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceSaveData.native.c). The seven no-argument prototypes follow the PS4 stubs in [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/f68923fe807f657348e6719ce658bcbd0a2b1c72/src/core/libraries/save_data/savedata.cpp#L653-L725), not measured on a console; no credible prototypes were found for the other two. All declarations have no parameters solely to resolve known imports and make no ABI claim. -- The following pending declarations have unverified signatures and behavior; they are parameterless import-resolution declarations and throw through `NotImplemented_nid_no_patch`: [`sceSaveDataBindPsnAccount`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataBindPsnAccountForSystemBackup`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataChangeDatabase`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataChangeInternal`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckBackupData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckBackupDataForCdlg`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckBackupDataInternal`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckBackupDataLight`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckCloudData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckSaveDataBroken`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckSaveDataVersion`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckSaveDataVersionLatest`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCreateUploadData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDebugGetThreadId`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteAllUser`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteBackupData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteCloudData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteProspero`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteTitle`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteUser`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDirNameSearchInternal`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDownload`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAllSize`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAppLaunchedUser`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAutoUploadConditions`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAutoUploadRequestInfo`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAutoUploadSetting`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetBoundPsnAccountCount`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetClientThreadPriority`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetCloudQuotaInfo`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetEventInfo`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetProgress`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSaveDataCount`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSaveDataRootDir`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSaveDataRootPath`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSaveDataRootUsbPath`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSavePoint`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetUpdatedDataCount`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataInitialize2`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataInitializeForCdlg`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataIsDeletingUsbDb`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataIsMounted`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataManualDelete`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataManualDownload`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataManualUpload`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataMount5`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataMountInternal`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataMountSys`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRebuildDatabase`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRegisterEventCallback`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRestoreBackupData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRestoreBackupDataForCdlg`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRestoreLoadSaveDataMemory`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSetAutoUploadSetting`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSetEventInfo`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSetSaveDataLibraryUser`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataShutdownStart`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSupportedFakeBrokenStatus`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSyncCloudList`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataUmountSys`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataUmountWithBackup`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataUnregisterEventCallback`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataUpload`](../../../core/libs/prx/libSceSaveData.native/Export.cpp). +- [sceSaveDataGetProgress](../../../core/libs/prx/libSceSaveData.native/Export.cpp) - throws through `NotImplemented_nid_no_patch`. The `float*` parameter follows the OpenOrbis headers and shadPS4's `sceSaveDataGetProgress(float*)`, which agree; it is not measured on a console and the PS5 module may differ. +- The following pending declarations have unverified signatures and behavior; they are parameterless import-resolution declarations and throw through `NotImplemented_nid_no_patch`: [`sceSaveDataBindPsnAccount`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataBindPsnAccountForSystemBackup`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataChangeDatabase`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataChangeInternal`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckBackupData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckBackupDataForCdlg`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckBackupDataInternal`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckBackupDataLight`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckCloudData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckSaveDataBroken`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckSaveDataVersion`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCheckSaveDataVersionLatest`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataCreateUploadData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDebugGetThreadId`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteAllUser`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteBackupData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteCloudData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteProspero`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteTitle`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDeleteUser`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDirNameSearchInternal`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataDownload`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAllSize`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAppLaunchedUser`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAutoUploadConditions`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAutoUploadRequestInfo`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetAutoUploadSetting`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetBoundPsnAccountCount`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetClientThreadPriority`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetCloudQuotaInfo`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetEventInfo`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSaveDataCount`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSaveDataRootDir`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSaveDataRootPath`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSaveDataRootUsbPath`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetSavePoint`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataGetUpdatedDataCount`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataInitialize2`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataInitializeForCdlg`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataIsDeletingUsbDb`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataIsMounted`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataManualDelete`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataManualDownload`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataManualUpload`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataMount5`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataMountInternal`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataMountSys`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRebuildDatabase`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRegisterEventCallback`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRestoreBackupData`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRestoreBackupDataForCdlg`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataRestoreLoadSaveDataMemory`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSetAutoUploadSetting`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSetEventInfo`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSetSaveDataLibraryUser`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataShutdownStart`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSupportedFakeBrokenStatus`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataSyncCloudList`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataUmountSys`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataUmountWithBackup`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataUnregisterEventCallback`](../../../core/libs/prx/libSceSaveData.native/Export.cpp), [`sceSaveDataUpload`](../../../core/libs/prx/libSceSaveData.native/Export.cpp). ### Functional From 351bb88ef4d5b57c71374e11dec7e4b892baff3c Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:28:25 +0000 Subject: [PATCH 270/318] fix(libSceSaveData): give sceSaveDataGetProgress its signature in the public header main declares it with an empty parameter list in Api.hpp, which conflicts with the definition. --- core/libs/prx/libSceSaveData/include/Api.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceSaveData/include/Api.hpp b/core/libs/prx/libSceSaveData/include/Api.hpp index 60fe67e664..f72f0502ed 100644 --- a/core/libs/prx/libSceSaveData/include/Api.hpp +++ b/core/libs/prx/libSceSaveData/include/Api.hpp @@ -47,7 +47,7 @@ int APS5_VABI sceSaveDataGetClientThreadPriority(void); int APS5_VABI sceSaveDataGetCloudQuotaInfo(void); int APS5_VABI sceSaveDataGetEventInfo(); int APS5_VABI sceSaveDataGetMountedSaveDataCount(); -int APS5_VABI sceSaveDataGetProgress(); +int APS5_VABI sceSaveDataGetProgress(float* progress); int APS5_VABI sceSaveDataGetSaveDataCount(); int APS5_VABI sceSaveDataGetSaveDataRootDir(void); int APS5_VABI sceSaveDataGetSaveDataRootPath(void); From 26e62ca71822973bcf6b4c159ab280b40852a99b Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:31:13 +0000 Subject: [PATCH 271/318] feat(libSceFreeType): implement FT_Outline_Done --- core/libs/prx/libSceFreeType/CMakeLists.txt | 2 +- .../prx/libSceFreeType/src/OutlineStorage.cpp | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 core/libs/prx/libSceFreeType/src/OutlineStorage.cpp diff --git a/core/libs/prx/libSceFreeType/CMakeLists.txt b/core/libs/prx/libSceFreeType/CMakeLists.txt index eb5d62f8b5..94ee3bd3f9 100644 --- a/core/libs/prx/libSceFreeType/CMakeLists.txt +++ b/core/libs/prx/libSceFreeType/CMakeLists.txt @@ -4,7 +4,7 @@ add_library(libSceFreeType SHARED EXCLUDE_FROM_ALL src/Outline.cpp src/Charmap.cpp src/FaceInfo.cpp - src/Stroker.cpp + src/Stroker.cpp src/OutlineStorage.cpp src/GuestObjects.cpp src/Names.cpp src/Trigonometry.cpp diff --git a/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp b/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp new file mode 100644 index 0000000000..85ac798fd4 --- /dev/null +++ b/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp @@ -0,0 +1,25 @@ +#include "prx/libSceFreeType/include/GuestObjects.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" + +#include FT_OUTLINE_H + +#include +#include + +using namespace FreeType; + +extern "C" { + +int APS5_VABI FT_Outline_Done_nid_postfix(GuestLibrary* library, GuestOutline* outline) { + if (!library) return FT_Err_Invalid_Library_Handle; + if (!outline) return FT_Err_Invalid_Outline; + if (outline->flags & FT_OUTLINE_OWNER) { + std::free(outline->points); + std::free(outline->tags); + std::free(outline->contours); + } + *outline = {}; + return 0; +} + +} From 4a93e742eb854d172da21e1e324f10bbb2e0b46c Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:31:33 +0000 Subject: [PATCH 272/318] test(libSceFreeType): compare FT_Outline_Done with native FreeType --- core/libs/prx/libSceFreeType/CMakeLists.txt | 5 ++ .../tests/GuestFreeTypeOutlineStorage.cpp | 61 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 core/libs/tests/GuestFreeTypeOutlineStorage.cpp diff --git a/core/libs/prx/libSceFreeType/CMakeLists.txt b/core/libs/prx/libSceFreeType/CMakeLists.txt index 94ee3bd3f9..2fb58b31d0 100644 --- a/core/libs/prx/libSceFreeType/CMakeLists.txt +++ b/core/libs/prx/libSceFreeType/CMakeLists.txt @@ -123,6 +123,11 @@ if(BUILD_TESTING) set_tests_properties(guest_freetype_stroker PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_freetype_outline_storage_tests ${LIBS_INCLUDE_DIR}/tests/GuestFreeTypeOutlineStorage.cpp) + target_include_directories(guest_freetype_outline_storage_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_freetype_outline_storage_tests PRIVATE libSceFreeType freetype libc) + configure_windows_unwind(guest_freetype_outline_storage_tests) + add_test(NAME guest_freetype_outline_storage COMMAND guest_freetype_outline_storage_tests) add_test_executable(guest_freetype_charmap_tests ${LIBS_INCLUDE_DIR}/tests/GuestFreeTypeCharmap.cpp) target_include_directories(guest_freetype_charmap_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_freetype_charmap_tests PRIVATE libSceFreeType freetype libc) diff --git a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp new file mode 100644 index 0000000000..edc096a833 --- /dev/null +++ b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp @@ -0,0 +1,61 @@ +#include "prx/libSceFreeType/include/GuestObjects.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" + +#include FT_OUTLINE_H + +#include +#include +#include + +using namespace FreeType; + +extern "C" { +int APS5_VABI FT_Outline_Done_nid_postfix(GuestLibrary*, GuestOutline*); +} + +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "FreeType outline storage check failed at line %d\n", line); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) + +static GuestOutline Owned(std::uint16_t points, std::uint16_t contours) { + GuestOutline outline{contours, points, nullptr, nullptr, nullptr, FT_OUTLINE_OWNER}; + outline.points = static_cast(std::calloc(points, sizeof(GuestVector))); + outline.tags = static_cast(std::calloc(points, 1)); + outline.contours = static_cast(std::calloc(contours, sizeof(std::uint16_t))); + return outline; +} + +int main() { + GuestLibrary library{}; + FT_Library native = nullptr; + Require(FT_Init_FreeType(&library.host) == 0 && FT_Init_FreeType(&native) == 0); + for (const std::uint16_t points : {0, 1, 7, 65535}) { + for (const std::uint16_t contours : {0, 1, 3}) { + if (contours > points) continue; + GuestOutline guest = Owned(points, contours); + FT_Outline expected{}; + Require(FT_Outline_New(native, points, contours, &expected) == 0); + Require(FT_Outline_Done_nid_postfix(&library, &guest) == FT_Outline_Done(native, &expected)); + Require(guest.n_points == expected.n_points && guest.n_contours == expected.n_contours && guest.flags == expected.flags); + Require(!guest.points && !guest.tags && !guest.contours); + } + } + GuestOutline outline{}; + Require(FT_Outline_Done_nid_postfix(nullptr, &outline) == FT_Err_Invalid_Library_Handle); + Require(FT_Outline_Done_nid_postfix(&library, nullptr) == FT_Err_Invalid_Outline); + FT_Outline nativeOutline{}; + Require(FT_Outline_Done(nullptr, &nativeOutline) == FT_Err_Invalid_Library_Handle); + Require(FT_Outline_Done(native, nullptr) == FT_Err_Invalid_Outline); + + GuestVector borrowedPoints[2]{}; + unsigned char borrowedTags[2]{}; + std::uint16_t borrowedContours[1]{1}; + GuestOutline borrowed{1, 2, borrowedPoints, borrowedTags, borrowedContours, 0}; + Require(FT_Outline_Done_nid_postfix(&library, &borrowed) == 0); + Require(borrowed.n_points == 0 && !borrowed.points && borrowedContours[0] == 1); + Require(FT_Done_FreeType(library.host) == 0 && FT_Done_FreeType(native) == 0); +} From 4520e58aaf7edff86282ea9b94fab2013c9f9527 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:31:33 +0000 Subject: [PATCH 273/318] docs(libSceFreeType): record FT_Outline_Done --- docs/dev/debt/libSceFreeType.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/debt/libSceFreeType.md b/docs/dev/debt/libSceFreeType.md index 9836e0ec21..bcf22b3cd7 100644 --- a/docs/dev/debt/libSceFreeType.md +++ b/docs/dev/debt/libSceFreeType.md @@ -44,6 +44,7 @@ - [FT_New_Memory_Face, FT_Done_Face and FT_Done_FreeType](../../../core/libs/prx/libSceFreeType/Export.cpp) (libSceFreeType) - console signatures, errors and generic-finalizer ordering are unverified; they follow the bundled [FreeType 2.13.3](https://github.com/freetype/freetype/blob/42608f77f20749dd6ddc9e0536788eaad70ea4b5/include/freetype/freetype.h) with LP64 guest arguments. Caller-owned font bytes must remain valid until the face is freed. +- [FT_Outline_Done](../../../core/libs/prx/libSceFreeType/src/OutlineStorage.cpp) (libSceFreeType) - console signature, errors and allocator are unverified; the adapter follows bundled [FreeType 2.13.3](https://github.com/freetype/freetype/blob/42608f77f20749dd6ddc9e0536788eaad70ea4b5/src/base/ftoutln.c) with the guest LP64 outline layout. The point, tag and contour arrays of an outline with the owner flag are released with the host C heap, so a title that fills an owning outline with arrays from another allocator is not supported - [FT_Outline_Check, FT_Outline_Get_CBox](../../../core/libs/prx/libSceFreeType/src/Outline.cpp) (libSceFreeType) - console signatures, count signedness and errors are unverified; the adapter follows bundled [FreeType 2.13.3](https://github.com/freetype/freetype/blob/42608f77f20749dd6ddc9e0536788eaad70ea4b5/src/base/ftoutln.c), including its unsigned 16-bit counts and contour endpoints. Control-box coordinates use the existing guest LP64 layout and retain their full 64-bit range on Windows; contour checking does not validate point tags - [FT_Get_Charmap_Index, FT_Get_CMap_Language_ID, FT_Get_CMap_Format](../../../core/libs/prx/libSceFreeType/src/Charmap.cpp) (libSceFreeType) - console signatures and return values are unverified; the adapter follows bundled [FreeType 2.13.3](https://github.com/freetype/freetype/blob/42608f77f20749dd6ddc9e0536788eaad70ea4b5/src/base/ftobjs.c) and the existing guest LP64 model. Charmaps must be the face's original read-only guest records; copied or otherwise unregistered records with a non-null face throw instead of reading private native cmap fields from guest memory From 8ed8f238f7be765421e0be2985f0d3bd15cfd35f Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:31:33 +0000 Subject: [PATCH 274/318] feat(libSceFreeType): implement FT_Outline_New --- .../prx/libSceFreeType/src/OutlineStorage.cpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp b/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp index 85ac798fd4..5f56452e2e 100644 --- a/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp +++ b/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp @@ -22,4 +22,26 @@ int APS5_VABI FT_Outline_Done_nid_postfix(GuestLibrary* library, GuestOutline* o return 0; } +int APS5_VABI FT_Outline_New_nid_postfix(GuestLibrary* library, std::uint32_t numPoints, std::int32_t numContours, GuestOutline* outline) { + if (!library) return FT_Err_Invalid_Library_Handle; + if (!outline) return FT_Err_Invalid_Argument; + *outline = {}; + if (numContours < 0 || static_cast(numContours) > numPoints) return FT_Err_Invalid_Argument; + if (numPoints > FT_OUTLINE_POINTS_MAX) return FT_Err_Array_Too_Large; + if (numPoints != 0) { + outline->points = static_cast(std::calloc(numPoints, sizeof(GuestVector))); + outline->tags = static_cast(std::calloc(numPoints, sizeof(unsigned char))); + } + if (numContours != 0) outline->contours = static_cast(std::calloc(static_cast(numContours), sizeof(std::uint16_t))); + const bool failed = numPoints != 0 && (!outline->points || !outline->tags) || numContours != 0 && !outline->contours; + outline->flags |= FT_OUTLINE_OWNER; + if (failed) { + FT_Outline_Done_nid_postfix(library, outline); + return FT_Err_Out_Of_Memory; + } + outline->n_points = static_cast(numPoints); + outline->n_contours = static_cast(numContours); + return 0; +} + } From e05935bb8dbacee717686835975d4094deecf629 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:31:33 +0000 Subject: [PATCH 275/318] test(libSceFreeType): compare FT_Outline_New with native FreeType --- .../tests/GuestFreeTypeOutlineStorage.cpp | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp index edc096a833..c602b5f8cc 100644 --- a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp +++ b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp @@ -10,6 +10,7 @@ using namespace FreeType; extern "C" { +int APS5_VABI FT_Outline_New_nid_postfix(GuestLibrary*, std::uint32_t, std::int32_t, GuestOutline*); int APS5_VABI FT_Outline_Done_nid_postfix(GuestLibrary*, GuestOutline*); } @@ -21,35 +22,37 @@ static void Check(bool value, int line) { } #define Require(value) Check((value), __LINE__) -static GuestOutline Owned(std::uint16_t points, std::uint16_t contours) { - GuestOutline outline{contours, points, nullptr, nullptr, nullptr, FT_OUTLINE_OWNER}; - outline.points = static_cast(std::calloc(points, sizeof(GuestVector))); - outline.tags = static_cast(std::calloc(points, 1)); - outline.contours = static_cast(std::calloc(contours, sizeof(std::uint16_t))); - return outline; +static void Compare(GuestLibrary& library, FT_Library native, std::uint32_t points, std::int32_t contours) { + GuestOutline guest{1, 2, nullptr, nullptr, nullptr, 3}; + FT_Outline expected{1, 2, nullptr, nullptr, nullptr, 3}; + const int guestError = FT_Outline_New_nid_postfix(&library, points, contours, &guest); + const int nativeError = FT_Outline_New(native, points, contours, &expected); + Require(guestError == nativeError); + Require(guest.n_points == expected.n_points && guest.n_contours == expected.n_contours && guest.flags == expected.flags); + Require((guest.points == nullptr) == (expected.points == nullptr)); + Require((guest.tags == nullptr) == (expected.tags == nullptr)); + Require((guest.contours == nullptr) == (expected.contours == nullptr)); + for (std::uint32_t index = 0; nativeError == 0 && index < points; ++index) { + Require(guest.points[index].x == 0 && guest.points[index].y == 0 && guest.tags[index] == 0); + } + for (std::int32_t index = 0; nativeError == 0 && index < contours; ++index) Require(guest.contours[index] == 0); + Require(FT_Outline_Done_nid_postfix(&library, &guest) == FT_Outline_Done(native, &expected)); + Require(guest.n_points == 0 && guest.n_contours == 0 && guest.flags == 0); + Require(!guest.points && !guest.tags && !guest.contours); } int main() { GuestLibrary library{}; FT_Library native = nullptr; Require(FT_Init_FreeType(&library.host) == 0 && FT_Init_FreeType(&native) == 0); - for (const std::uint16_t points : {0, 1, 7, 65535}) { - for (const std::uint16_t contours : {0, 1, 3}) { - if (contours > points) continue; - GuestOutline guest = Owned(points, contours); - FT_Outline expected{}; - Require(FT_Outline_New(native, points, contours, &expected) == 0); - Require(FT_Outline_Done_nid_postfix(&library, &guest) == FT_Outline_Done(native, &expected)); - Require(guest.n_points == expected.n_points && guest.n_contours == expected.n_contours && guest.flags == expected.flags); - Require(!guest.points && !guest.tags && !guest.contours); - } + for (const std::uint32_t points : {0u, 1u, 7u, 65535u, 65536u, 0xFFFFFFFFu}) { + for (const std::int32_t contours : {-1, 0, 1, 3, 7, 8, 65535}) Compare(library, native, points, contours); } GuestOutline outline{}; + Require(FT_Outline_New_nid_postfix(nullptr, 3, 1, &outline) == FT_Err_Invalid_Library_Handle); + Require(FT_Outline_New_nid_postfix(&library, 3, 1, nullptr) == FT_Err_Invalid_Argument); Require(FT_Outline_Done_nid_postfix(nullptr, &outline) == FT_Err_Invalid_Library_Handle); Require(FT_Outline_Done_nid_postfix(&library, nullptr) == FT_Err_Invalid_Outline); - FT_Outline nativeOutline{}; - Require(FT_Outline_Done(nullptr, &nativeOutline) == FT_Err_Invalid_Library_Handle); - Require(FT_Outline_Done(native, nullptr) == FT_Err_Invalid_Outline); GuestVector borrowedPoints[2]{}; unsigned char borrowedTags[2]{}; From bdbf3e1ca4cd5057ef2a325fef0a2a6ad5b36359 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:31:33 +0000 Subject: [PATCH 276/318] docs(libSceFreeType): record FT_Outline_New --- docs/dev/debt/libSceFreeType.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/debt/libSceFreeType.md b/docs/dev/debt/libSceFreeType.md index bcf22b3cd7..0e52c5abab 100644 --- a/docs/dev/debt/libSceFreeType.md +++ b/docs/dev/debt/libSceFreeType.md @@ -44,6 +44,7 @@ - [FT_New_Memory_Face, FT_Done_Face and FT_Done_FreeType](../../../core/libs/prx/libSceFreeType/Export.cpp) (libSceFreeType) - console signatures, errors and generic-finalizer ordering are unverified; they follow the bundled [FreeType 2.13.3](https://github.com/freetype/freetype/blob/42608f77f20749dd6ddc9e0536788eaad70ea4b5/include/freetype/freetype.h) with LP64 guest arguments. Caller-owned font bytes must remain valid until the face is freed. +- [FT_Outline_New](../../../core/libs/prx/libSceFreeType/src/OutlineStorage.cpp) (libSceFreeType) - console signature, errors and allocator are unverified; the adapter follows bundled [FreeType 2.13.3](https://github.com/freetype/freetype/blob/42608f77f20749dd6ddc9e0536788eaad70ea4b5/src/base/ftoutln.c) with the guest LP64 outline layout. The arrays are zero-initialised from the host C heap, not from a guest or FreeType allocator, because the guest points are 64 bits wide - [FT_Outline_Done](../../../core/libs/prx/libSceFreeType/src/OutlineStorage.cpp) (libSceFreeType) - console signature, errors and allocator are unverified; the adapter follows bundled [FreeType 2.13.3](https://github.com/freetype/freetype/blob/42608f77f20749dd6ddc9e0536788eaad70ea4b5/src/base/ftoutln.c) with the guest LP64 outline layout. The point, tag and contour arrays of an outline with the owner flag are released with the host C heap, so a title that fills an owning outline with arrays from another allocator is not supported - [FT_Outline_Check, FT_Outline_Get_CBox](../../../core/libs/prx/libSceFreeType/src/Outline.cpp) (libSceFreeType) - console signatures, count signedness and errors are unverified; the adapter follows bundled [FreeType 2.13.3](https://github.com/freetype/freetype/blob/42608f77f20749dd6ddc9e0536788eaad70ea4b5/src/base/ftoutln.c), including its unsigned 16-bit counts and contour endpoints. Control-box coordinates use the existing guest LP64 layout and retain their full 64-bit range on Windows; contour checking does not validate point tags From 4f90bff980f4875b1f11f52265124332619ba116 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:32:56 +0000 Subject: [PATCH 277/318] fix(libSceFreeType): reject an uninitialized outline library --- core/libs/prx/libSceFreeType/src/OutlineStorage.cpp | 2 ++ core/libs/tests/GuestFreeTypeOutlineStorage.cpp | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp b/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp index 5f56452e2e..57efb78574 100644 --- a/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp +++ b/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp @@ -13,6 +13,7 @@ extern "C" { int APS5_VABI FT_Outline_Done_nid_postfix(GuestLibrary* library, GuestOutline* outline) { if (!library) return FT_Err_Invalid_Library_Handle; if (!outline) return FT_Err_Invalid_Outline; + if (!library->host) return FT_Err_Invalid_Argument; if (outline->flags & FT_OUTLINE_OWNER) { std::free(outline->points); std::free(outline->tags); @@ -25,6 +26,7 @@ int APS5_VABI FT_Outline_Done_nid_postfix(GuestLibrary* library, GuestOutline* o int APS5_VABI FT_Outline_New_nid_postfix(GuestLibrary* library, std::uint32_t numPoints, std::int32_t numContours, GuestOutline* outline) { if (!library) return FT_Err_Invalid_Library_Handle; if (!outline) return FT_Err_Invalid_Argument; + if (!library->host) return FT_Err_Invalid_Argument; *outline = {}; if (numContours < 0 || static_cast(numContours) > numPoints) return FT_Err_Invalid_Argument; if (numPoints > FT_OUTLINE_POINTS_MAX) return FT_Err_Array_Too_Large; diff --git a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp index c602b5f8cc..de00e0a5f1 100644 --- a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp +++ b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp @@ -54,6 +54,13 @@ int main() { Require(FT_Outline_Done_nid_postfix(nullptr, &outline) == FT_Err_Invalid_Library_Handle); Require(FT_Outline_Done_nid_postfix(&library, nullptr) == FT_Err_Invalid_Outline); + GuestLibrary uninitialized{}; + GuestOutline untouched{1, 2, nullptr, nullptr, nullptr, 3}; + Require(FT_Outline_New_nid_postfix(&uninitialized, 3, 1, &untouched) == FT_Err_Invalid_Argument); + Require(untouched.n_points == 1 && untouched.n_contours == 2 && untouched.flags == 3); + Require(FT_Outline_Done_nid_postfix(&uninitialized, &untouched) == FT_Err_Invalid_Argument); + Require(untouched.n_points == 1 && untouched.n_contours == 2 && untouched.flags == 3); + GuestVector borrowedPoints[2]{}; unsigned char borrowedTags[2]{}; std::uint16_t borrowedContours[1]{1}; From c2c522ab8741c34c045de3dc34c1595d623adead Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:38:13 +0000 Subject: [PATCH 278/318] test(libSceFreeType): preserve outline fields on invalid library --- core/libs/tests/GuestFreeTypeOutlineStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp index de00e0a5f1..609bd0b565 100644 --- a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp +++ b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp @@ -57,9 +57,9 @@ int main() { GuestLibrary uninitialized{}; GuestOutline untouched{1, 2, nullptr, nullptr, nullptr, 3}; Require(FT_Outline_New_nid_postfix(&uninitialized, 3, 1, &untouched) == FT_Err_Invalid_Argument); - Require(untouched.n_points == 1 && untouched.n_contours == 2 && untouched.flags == 3); + Require(untouched.n_points == 2 && untouched.n_contours == 1 && untouched.flags == 3); Require(FT_Outline_Done_nid_postfix(&uninitialized, &untouched) == FT_Err_Invalid_Argument); - Require(untouched.n_points == 1 && untouched.n_contours == 2 && untouched.flags == 3); + Require(untouched.n_points == 2 && untouched.n_contours == 1 && untouched.flags == 3); GuestVector borrowedPoints[2]{}; unsigned char borrowedTags[2]{}; From 5594e0563a50b1528f2208a0321fdc8b9a0f42f4 Mon Sep 17 00:00:00 2001 From: MarcZX10 <67475268+MarcZX10@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:13:12 +0000 Subject: [PATCH 279/318] test(freetype): set Windows DLL path for 10589 --- core/libs/prx/libSceFreeType/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/libs/prx/libSceFreeType/CMakeLists.txt b/core/libs/prx/libSceFreeType/CMakeLists.txt index 2fb58b31d0..97bf11176a 100644 --- a/core/libs/prx/libSceFreeType/CMakeLists.txt +++ b/core/libs/prx/libSceFreeType/CMakeLists.txt @@ -128,6 +128,10 @@ if(BUILD_TESTING) target_link_libraries(guest_freetype_outline_storage_tests PRIVATE libSceFreeType freetype libc) configure_windows_unwind(guest_freetype_outline_storage_tests) add_test(NAME guest_freetype_outline_storage COMMAND guest_freetype_outline_storage_tests) + if(WIN32) + set_tests_properties(guest_freetype_outline_storage PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_freetype_charmap_tests ${LIBS_INCLUDE_DIR}/tests/GuestFreeTypeCharmap.cpp) target_include_directories(guest_freetype_charmap_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_freetype_charmap_tests PRIVATE libSceFreeType freetype libc) From b59da174cc6baed2d709ca7a3a84cbb6f6c3ab4a Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Tue, 27 Oct 2026 09:48:17 +1300 Subject: [PATCH 280/318] fix(freetype): declare FT_Outline_New and FT_Outline_Done in the library header The two exports were declared only in the test; they are declared once, in libSceFreeType's Api.hpp, which the source and the test now include. --- core/libs/prx/libSceFreeType/include/Api.hpp | 2 ++ core/libs/prx/libSceFreeType/src/OutlineStorage.cpp | 1 + core/libs/tests/GuestFreeTypeOutlineStorage.cpp | 6 +----- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceFreeType/include/Api.hpp b/core/libs/prx/libSceFreeType/include/Api.hpp index cbd3d30a96..ef2a1b92f0 100644 --- a/core/libs/prx/libSceFreeType/include/Api.hpp +++ b/core/libs/prx/libSceFreeType/include/Api.hpp @@ -104,4 +104,6 @@ int APS5_VABI FT_Stroker_ConicTo_nid_postfix(void* stroker, const FreeType::Gues int APS5_VABI ft_corner_is_flat_nid_postfix(std::int64_t in_x, std::int64_t in_y, std::int64_t out_x, std::int64_t out_y); int APS5_VABI ft_corner_orientation_nid_postfix(std::int64_t in_x, std::int64_t in_y, std::int64_t out_x, std::int64_t out_y); +int APS5_VABI FT_Outline_New_nid_postfix(FreeType::GuestLibrary* library, std::uint32_t numPoints, std::int32_t numContours, FreeType::GuestOutline* outline); +int APS5_VABI FT_Outline_Done_nid_postfix(FreeType::GuestLibrary* library, FreeType::GuestOutline* outline); } diff --git a/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp b/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp index 57efb78574..f2ae5ebae0 100644 --- a/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp +++ b/core/libs/prx/libSceFreeType/src/OutlineStorage.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceFreeType/include/Api.hpp" #include "prx/libSceFreeType/include/GuestObjects.hpp" #include "prx/libc/include/general/VabiMacros.hpp" diff --git a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp index 609bd0b565..d1acca1b1e 100644 --- a/core/libs/tests/GuestFreeTypeOutlineStorage.cpp +++ b/core/libs/tests/GuestFreeTypeOutlineStorage.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceFreeType/include/Api.hpp" #include "prx/libSceFreeType/include/GuestObjects.hpp" #include "prx/libc/include/general/VabiMacros.hpp" @@ -9,11 +10,6 @@ using namespace FreeType; -extern "C" { -int APS5_VABI FT_Outline_New_nid_postfix(GuestLibrary*, std::uint32_t, std::int32_t, GuestOutline*); -int APS5_VABI FT_Outline_Done_nid_postfix(GuestLibrary*, GuestOutline*); -} - static void Check(bool value, int line) { if (!value) { std::fprintf(stderr, "FreeType outline storage check failed at line %d\n", line); From 6f8542b76a6cbfbf7809feb3e8ec11d669463c4c Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Tue, 27 Oct 2026 09:51:04 +1300 Subject: [PATCH 281/318] fix(freetype): name the libSceFreeTypeOt exports that share the NIDs of FT_Outline_New and FT_Outline_Done The two placeholders carried the NIDs of the outline exports libSceFreeType now implements, so the conventions check reports them as unknown NIDs. They are renamed after the symbols; the NIDs and bodies are unchanged. --- core/libs/prx/libSceFreeTypeOt/Export.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceFreeTypeOt/Export.cpp b/core/libs/prx/libSceFreeTypeOt/Export.cpp index 91243f68ee..8c93cd6168 100644 --- a/core/libs/prx/libSceFreeTypeOt/Export.cpp +++ b/core/libs/prx/libSceFreeTypeOt/Export.cpp @@ -229,10 +229,10 @@ int APS5_VABI sceFreeTypeOtOutlineDecompose_nid_no_patch() { } APS5_EXPORT("++lgV9pRsO4", sceFreeTypeOtOutlineDecompose_nid_no_patch); -int APS5_VABI sceFreeTypeOtUnknown_BqGJsJcFYaM_nid_no_patch() { +int APS5_VABI sceFreeTypeOtOutlineDone_nid_no_patch() { NotImplemented_nid_no_patch(__func__); } -APS5_EXPORT("BqGJsJcFYaM", sceFreeTypeOtUnknown_BqGJsJcFYaM_nid_no_patch); +APS5_EXPORT("BqGJsJcFYaM", sceFreeTypeOtOutlineDone_nid_no_patch); int APS5_VABI sceFreeTypeOtUnknown_umIqupy8UU0_nid_no_patch() { NotImplemented_nid_no_patch(__func__); @@ -264,10 +264,10 @@ int APS5_VABI sceFreeTypeOtOutlineGetOrientation_nid_no_patch() { } APS5_EXPORT("weVsBCT+TTQ", sceFreeTypeOtOutlineGetOrientation_nid_no_patch); -int APS5_VABI sceFreeTypeOtUnknown_QH2emcgeFck_nid_no_patch() { +int APS5_VABI sceFreeTypeOtOutlineNew_nid_no_patch() { NotImplemented_nid_no_patch(__func__); } -APS5_EXPORT("QH2emcgeFck", sceFreeTypeOtUnknown_QH2emcgeFck_nid_no_patch); +APS5_EXPORT("QH2emcgeFck", sceFreeTypeOtOutlineNew_nid_no_patch); int APS5_VABI sceFreeTypeOtUnknown_NG7OgYsgcF8_nid_no_patch() { NotImplemented_nid_no_patch(__func__); From 239c27e9457d0d34a81413178e365558ae6690dc Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Tue, 27 Oct 2026 09:55:32 +1300 Subject: [PATCH 282/318] fix(libSceFreeTypeOt): declare the named import-resolution helpers in the shared header The renamed no-parameter helpers are declared once in libSceFreeType's Api.hpp, which Export.cpp includes, as the other named Ot helpers are. The debt entry no longer counts the placeholders: the helpers keep their NIDs, their provider-specific prototypes and behavior stay unverified. --- core/libs/prx/libSceFreeType/include/Api.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/libs/prx/libSceFreeType/include/Api.hpp b/core/libs/prx/libSceFreeType/include/Api.hpp index ef2a1b92f0..6e5f59dd59 100644 --- a/core/libs/prx/libSceFreeType/include/Api.hpp +++ b/core/libs/prx/libSceFreeType/include/Api.hpp @@ -106,4 +106,6 @@ int APS5_VABI ft_corner_orientation_nid_postfix(std::int64_t in_x, std::int64_t int APS5_VABI FT_Outline_New_nid_postfix(FreeType::GuestLibrary* library, std::uint32_t numPoints, std::int32_t numContours, FreeType::GuestOutline* outline); int APS5_VABI FT_Outline_Done_nid_postfix(FreeType::GuestLibrary* library, FreeType::GuestOutline* outline); +int APS5_VABI sceFreeTypeOtOutlineDone_nid_no_patch(); +int APS5_VABI sceFreeTypeOtOutlineNew_nid_no_patch(); } From 40b8e7e5faa6ab88743eec462fd4f5486c3a1540 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Sat, 24 Oct 2026 05:11:53 +0200 Subject: [PATCH 283/318] feat(libSceLibreSsl): implement EVP_CIPHER_CTX_init --- core/libs/prx/libSceLibreSsl/CMakeLists.txt | 13 ++++- core/libs/prx/libSceLibreSsl/Export.cpp | 5 -- .../prx/libSceLibreSsl/src/EvpCipherCtx.cpp | 45 +++++++++++++++ core/libs/tests/GuestLibreSslEvpCipherCtx.cpp | 56 +++++++++++++++++++ docs/dev/debt/libSceLibreSsl.md | 1 + 5 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp create mode 100644 core/libs/tests/GuestLibreSslEvpCipherCtx.cpp diff --git a/core/libs/prx/libSceLibreSsl/CMakeLists.txt b/core/libs/prx/libSceLibreSsl/CMakeLists.txt index e4635639c7..55466ab9aa 100644 --- a/core/libs/prx/libSceLibreSsl/CMakeLists.txt +++ b/core/libs/prx/libSceLibreSsl/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(libSceLibreSsl SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Digest.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Runtime.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Aes.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Error.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Ocsp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Stack.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Ui.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Rsa.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/X509StoreContext.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Pkey.cpp) +add_library(libSceLibreSsl SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Digest.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Runtime.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Aes.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Error.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Ocsp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Stack.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Ui.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Rsa.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/X509StoreContext.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Pkey.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/EvpCipherCtx.cpp) target_include_directories(libSceLibreSsl PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(libSceLibreSsl PRIVATE mbedcrypto) configure_windows_unwind(libSceLibreSsl) @@ -159,6 +159,17 @@ if(BUILD_TESTING) set_tests_properties(guest_libressl_pkey_type PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + + add_test_executable(guest_libressl_evp_cipher_ctx_tests ${LIBS_INCLUDE_DIR}/tests/GuestLibreSslEvpCipherCtx.cpp) + target_include_directories(guest_libressl_evp_cipher_ctx_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_libressl_evp_cipher_ctx_tests PRIVATE libSceLibreSsl libc) + configure_windows_unwind(guest_libressl_evp_cipher_ctx_tests) + add_test(NAME guest_libressl_evp_cipher_ctx COMMAND guest_libressl_evp_cipher_ctx_tests) + set_tests_properties(guest_libressl_evp_cipher_ctx PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_libressl_evp_cipher_ctx PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() target_sources(libSceLibreSsl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/Cipher.cpp) diff --git a/core/libs/prx/libSceLibreSsl/Export.cpp b/core/libs/prx/libSceLibreSsl/Export.cpp index fc78f64744..d9e49c6d99 100644 --- a/core/libs/prx/libSceLibreSsl/Export.cpp +++ b/core/libs/prx/libSceLibreSsl/Export.cpp @@ -479,11 +479,6 @@ int APS5_VABI EVP_CIPHER_CTX_free_nid_postfix(void) { return 0; } -int APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void) { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp b/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp new file mode 100644 index 0000000000..597fbd7e92 --- /dev/null +++ b/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp @@ -0,0 +1,45 @@ +#include +#include +#include +#include +#include +#include "prx/libc/include/General.hpp" + +struct LibreSslEvpCipherCtx { + const void* cipher; + void* engine; + int encrypt; + int bufLen; + unsigned char oiv[16]; + unsigned char iv[16]; + unsigned char buf[32]; + int num; + void* appData; + int keyLen; + std::uint64_t flags; + void* cipherData; + int finalUsed; + int blockMask; + unsigned char final[32]; +}; + +static_assert(sizeof(LibreSslEvpCipherCtx) == 168); +static_assert(offsetof(LibreSslEvpCipherCtx, num) == 88); +static_assert(offsetof(LibreSslEvpCipherCtx, flags) == 112); +static_assert(offsetof(LibreSslEvpCipherCtx, final) == 136); + +namespace { + +constexpr std::uint64_t CipherFlagNoPadding = 0x100; + +} + +extern "C" { +void* APS5_VABI calloc_nid_postfix(std::size_t count, std::size_t size); + +void APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(LibreSslEvpCipherCtx* ctx) { + if (!ctx) throw std::invalid_argument(std::string(__func__) + ": null context"); + std::memset(ctx, 0, sizeof(LibreSslEvpCipherCtx)); +} + +} diff --git a/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp b/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp new file mode 100644 index 0000000000..de56ba96fa --- /dev/null +++ b/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp @@ -0,0 +1,56 @@ +#include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +constexpr std::size_t ContextBytes = 168; +constexpr std::size_t FlagsOffset = 112; + +static std::uint64_t Flags(const unsigned char* context) { + std::uint64_t flags = 0; + std::memcpy(&flags, context + FlagsOffset, sizeof(flags)); + return flags; +} + +static void SetFlags(unsigned char* context, std::uint64_t flags) { + std::memcpy(context + FlagsOffset, &flags, sizeof(flags)); +} + +static void InitializeHeap() { + void* const api[10] {}; + ApplicationHeapRegister_nid_no_patch(api); +} + +extern "C" { +void APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(void*); +} + +static void Require(bool condition, std::source_location location = std::source_location::current()) { + if (condition) return; + std::fprintf(stderr, "LibreSSL EVP_CIPHER_CTX check failed at line %u\n", location.line()); + std::abort(); +} + +template static void RequireThrows(TAction action, std::source_location location = std::source_location::current()) { + try { action(); } catch (const std::exception&) { return; } + Require(false, location); +} + +static void TestInit() { + unsigned char memory[ContextBytes + 16]; + std::memset(memory, 0xAA, sizeof(memory)); + EVP_CIPHER_CTX_init_nid_postfix(memory); + for (std::size_t index = 0; index < ContextBytes; ++index) Require(memory[index] == 0); + for (std::size_t index = ContextBytes; index < sizeof(memory); ++index) Require(memory[index] == 0xAA); + RequireThrows([] { EVP_CIPHER_CTX_init_nid_postfix(nullptr); }); +} + +int main() { + TestInit(); + return 0; +} diff --git a/docs/dev/debt/libSceLibreSsl.md b/docs/dev/debt/libSceLibreSsl.md index 271bc7c326..7f7dbac2c2 100644 --- a/docs/dev/debt/libSceLibreSsl.md +++ b/docs/dev/debt/libSceLibreSsl.md @@ -26,6 +26,7 @@ - [UI_set_result](../../../core/libs/prx/libSceLibreSsl/src/Ui.cpp) (libSceLibreSsl) - unverified on a PS5, and no export of this module creates a `UI` or `UI_STRING` yet. The "You must type in %d to %d characters" text LibreSSL 2.5.4 attaches to the size errors is not kept, because the error queue holds codes only. A null `UI` or result, or null boolean characters that must be compared, throws; LibreSSL dereferences them. The rest follows its [ui_lib.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/ui/ui_lib.c#L816-L879) over the 40-byte `UI` and 64-byte `UI_STRING` of [ui_locl.h](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/ui/ui_locl.h#L104-L148): a result outside the bounds sets `UI_FLAG_REDOABLE` and queues `UI_R_RESULT_TOO_SMALL` or `UI_R_RESULT_TOO_LARGE`. - [PKCS12_PBE_add, SSL_load_error_strings](../../../core/libs/prx/libSceLibreSsl/Export.cpp) (libSceLibreSsl) - unverified on a PS5; each returns without an observable effect and queues no error. `SSL_load_error_strings` builds no string table, because `ERR_error_string` and `ERR_error_string_n` throw ([ssl_err.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libssl/ssl_err.c#L483-L490)); the tables are needed once a string lookup exists. `PKCS12_PBE_add` is empty in LibreSSL 2.5.4 ([p12_crpt.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/pkcs12/p12_crpt.c#L67-L70)). - [EVP_PKEY_type](../../../core/libs/prx/libSceLibreSsl/src/Pkey.cpp) (libSceLibreSsl) - follows the method table of [LibreSSL 2.5.4](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/asn1/ameth_lib.c#L83-L111) `ameth_lib.c` built without GOST and with no engine (every `ENGINE_` export throws): RSA (6) and its alias 19 give 6, the DSA types 66, 67, 70, 113 and 116 give 116, DH (28), EC (408), HMAC (855) and CMAC (894) give themselves, and every other type, the GOST types 811, 815, 946 and 947 included, gives 0. These results match [firmware 13.42's module run natively on a PC](https://github.com/boykopovar/AnyPS5/pull/10398#issuecomment-6406072913) for every type from -1000 to 5000 and the `int` extremes; nothing is checked on the console +- [EVP_CIPHER_CTX_init](../../../core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp) (libSceLibreSsl) - nothing is checked on the console. The 168-byte `EVP_CIPHER_CTX` layout and the results follow [LibreSSL 2.5.4](https://github.com/libressl/openbsd/tree/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/evp) `evp.h` and `evp_enc.c`, the version in the [firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214), which also measured `EVP_CIPHER_CTX_init` clearing 168 bytes. `EVP_CIPHER_CTX_new` allocates through the guest `calloc` as the module would through libc. A null context, which LibreSSL dereferences, throws ### Functional From 4331785706e955a1be66f53927f377f58ad2db1a Mon Sep 17 00:00:00 2001 From: Alvaro Date: Sat, 24 Oct 2026 05:12:53 +0200 Subject: [PATCH 284/318] feat(libSceLibreSsl): implement EVP_CIPHER_CTX_new --- core/libs/prx/libSceLibreSsl/Export.cpp | 5 ----- core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp | 4 ++++ core/libs/tests/GuestLibreSslEvpCipherCtx.cpp | 14 ++++++++++++++ docs/dev/debt/libSceLibreSsl.md | 4 ++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceLibreSsl/Export.cpp b/core/libs/prx/libSceLibreSsl/Export.cpp index d9e49c6d99..cf550df97c 100644 --- a/core/libs/prx/libSceLibreSsl/Export.cpp +++ b/core/libs/prx/libSceLibreSsl/Export.cpp @@ -479,11 +479,6 @@ int APS5_VABI EVP_CIPHER_CTX_free_nid_postfix(void) { return 0; } -int APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI EVP_CIPHER_CTX_set_padding_nid_postfix(void) { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp b/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp index 597fbd7e92..1dc5372adf 100644 --- a/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp +++ b/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp @@ -42,4 +42,8 @@ void APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(LibreSslEvpCipherCtx* ctx) { std::memset(ctx, 0, sizeof(LibreSslEvpCipherCtx)); } +LibreSslEvpCipherCtx* APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void) { + return static_cast(calloc_nid_postfix(1, sizeof(LibreSslEvpCipherCtx))); +} + } diff --git a/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp b/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp index de56ba96fa..17897f6386 100644 --- a/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp +++ b/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp @@ -28,6 +28,8 @@ static void InitializeHeap() { extern "C" { void APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(void*); +void* APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void); +void APS5_VABI free_nid_postfix(void*); } static void Require(bool condition, std::source_location location = std::source_location::current()) { @@ -50,7 +52,19 @@ static void TestInit() { RequireThrows([] { EVP_CIPHER_CTX_init_nid_postfix(nullptr); }); } +static void TestNew() { + for (int round = 0; round < 4; ++round) { + auto* context = static_cast(EVP_CIPHER_CTX_new_nid_postfix()); + Require(context != nullptr); + for (std::size_t index = 0; index < ContextBytes; ++index) Require(context[index] == 0); + std::memset(context, 0x5A, ContextBytes); + free_nid_postfix(context); + } +} + int main() { + InitializeHeap(); TestInit(); + TestNew(); return 0; } diff --git a/docs/dev/debt/libSceLibreSsl.md b/docs/dev/debt/libSceLibreSsl.md index 7f7dbac2c2..9e7aa49dd4 100644 --- a/docs/dev/debt/libSceLibreSsl.md +++ b/docs/dev/debt/libSceLibreSsl.md @@ -7,7 +7,7 @@ ### Unknown function info -- [libSceLibreSsl pending exports](../../../core/libs/prx/libSceLibreSsl/Export.cpp) - 292 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceLibreSsl.c) and were cross-checked against the same module in PS5 firmware 13.60. Their signatures and behaviour are unverified; the declarations use the no-parameter unknown-signature convention and throw through `NotImplemented_nid_no_patch`. +- [libSceLibreSsl pending exports](../../../core/libs/prx/libSceLibreSsl/Export.cpp) - 288 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceLibreSsl.c) and were cross-checked against the same module in PS5 firmware 13.60. Their signatures and behaviour are unverified; the declarations use the no-parameter unknown-signature convention and throw through `NotImplemented_nid_no_patch`. - [MD4, MD5, SHA1, SHA224, SHA256, SHA384 and SHA512 Init/Update/Final and SHA256](../../../core/libs/prx/libSceLibreSsl/src/Digest.cpp) (libSceLibreSsl) - signatures and context layouts follow the [LibreSSL headers](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/sha/sha.h), and buffering and `Final` follow its [sources](https://github.com/libressl/openbsd/tree/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/sha); nothing is checked on the console. Null contexts, null input with a nonzero length, null digest buffers in the 32-bit `Final` functions and a buffered byte count that is not below the block size, which LibreSSL does not check, throw. SHA-384 and SHA-512 initialization retains the block buffer, following [LibreSSL 2.5.4 `sha512.c`](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/sha/sha512.c) - [DES_set_key, DES_ecb_encrypt, DES_set_odd_parity](../../../core/libs/prx/libSceLibreSsl/Export.cpp) (libSceLibreSsl) - the 128-byte `DES_key_schedule` holds the 8-byte key followed by zeros instead of LibreSSL's precomputed subkeys, so a title reading its contents sees different bytes; `DES_ecb_encrypt` derives the subkeys through mbedtls on each call. Not compared with the console. Results follow LibreSSL [des_key.c](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/des/des_key.c) and [des.c](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/des/des.c): `DES_set_key` skips the parity and weak-key checks and returns 0, as it does while `DES_check_key` is 0 (that variable is not in the PS5 export inventory), a nonzero `enc` encrypts, and `DES_set_odd_parity` sets each byte's low bit for odd parity. Null pointers throw where LibreSSL faults - [X509_verify_cert_error_string](../../../core/libs/prx/libSceLibreSsl/Export.cpp) - no console call is verified. The 64-bit argument, low-32-bit lookup, messages and shared numeric fallback follow [LibreSSL 2.5.4](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/x509/x509_txt.c), the version identified in [published firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214). Console modifications and other firmware versions remain unverified. Unknown-code results share an overwritten static buffer and are not safe for concurrent use, as in that release; code 1 and codes above 54 use the numeric fallback. @@ -26,7 +26,7 @@ - [UI_set_result](../../../core/libs/prx/libSceLibreSsl/src/Ui.cpp) (libSceLibreSsl) - unverified on a PS5, and no export of this module creates a `UI` or `UI_STRING` yet. The "You must type in %d to %d characters" text LibreSSL 2.5.4 attaches to the size errors is not kept, because the error queue holds codes only. A null `UI` or result, or null boolean characters that must be compared, throws; LibreSSL dereferences them. The rest follows its [ui_lib.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/ui/ui_lib.c#L816-L879) over the 40-byte `UI` and 64-byte `UI_STRING` of [ui_locl.h](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/ui/ui_locl.h#L104-L148): a result outside the bounds sets `UI_FLAG_REDOABLE` and queues `UI_R_RESULT_TOO_SMALL` or `UI_R_RESULT_TOO_LARGE`. - [PKCS12_PBE_add, SSL_load_error_strings](../../../core/libs/prx/libSceLibreSsl/Export.cpp) (libSceLibreSsl) - unverified on a PS5; each returns without an observable effect and queues no error. `SSL_load_error_strings` builds no string table, because `ERR_error_string` and `ERR_error_string_n` throw ([ssl_err.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libssl/ssl_err.c#L483-L490)); the tables are needed once a string lookup exists. `PKCS12_PBE_add` is empty in LibreSSL 2.5.4 ([p12_crpt.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/pkcs12/p12_crpt.c#L67-L70)). - [EVP_PKEY_type](../../../core/libs/prx/libSceLibreSsl/src/Pkey.cpp) (libSceLibreSsl) - follows the method table of [LibreSSL 2.5.4](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/asn1/ameth_lib.c#L83-L111) `ameth_lib.c` built without GOST and with no engine (every `ENGINE_` export throws): RSA (6) and its alias 19 give 6, the DSA types 66, 67, 70, 113 and 116 give 116, DH (28), EC (408), HMAC (855) and CMAC (894) give themselves, and every other type, the GOST types 811, 815, 946 and 947 included, gives 0. These results match [firmware 13.42's module run natively on a PC](https://github.com/boykopovar/AnyPS5/pull/10398#issuecomment-6406072913) for every type from -1000 to 5000 and the `int` extremes; nothing is checked on the console -- [EVP_CIPHER_CTX_init](../../../core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp) (libSceLibreSsl) - nothing is checked on the console. The 168-byte `EVP_CIPHER_CTX` layout and the results follow [LibreSSL 2.5.4](https://github.com/libressl/openbsd/tree/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/evp) `evp.h` and `evp_enc.c`, the version in the [firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214), which also measured `EVP_CIPHER_CTX_init` clearing 168 bytes. `EVP_CIPHER_CTX_new` allocates through the guest `calloc` as the module would through libc. A null context, which LibreSSL dereferences, throws +- [EVP_CIPHER_CTX_init, EVP_CIPHER_CTX_new](../../../core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp) (libSceLibreSsl) - nothing is checked on the console. The 168-byte `EVP_CIPHER_CTX` layout and the results follow [LibreSSL 2.5.4](https://github.com/libressl/openbsd/tree/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/evp) `evp.h` and `evp_enc.c`, the version in the [firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214), which also measured `EVP_CIPHER_CTX_init` clearing 168 bytes. `EVP_CIPHER_CTX_new` allocates through the guest `calloc` as the module would through libc. A null context, which LibreSSL dereferences, throws ### Functional From 3caeefdaff5eb048f5f8960122509443fe815f64 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Sat, 24 Oct 2026 05:13:03 +0200 Subject: [PATCH 285/318] feat(libSceLibreSsl): implement EVP_CIPHER_CTX_set_padding --- core/libs/prx/libSceLibreSsl/Export.cpp | 5 ----- .../prx/libSceLibreSsl/src/EvpCipherCtx.cpp | 7 +++++++ core/libs/tests/GuestLibreSslEvpCipherCtx.cpp | 21 +++++++++++++++++++ docs/dev/debt/libSceLibreSsl.md | 4 ++-- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceLibreSsl/Export.cpp b/core/libs/prx/libSceLibreSsl/Export.cpp index cf550df97c..f898124769 100644 --- a/core/libs/prx/libSceLibreSsl/Export.cpp +++ b/core/libs/prx/libSceLibreSsl/Export.cpp @@ -479,11 +479,6 @@ int APS5_VABI EVP_CIPHER_CTX_free_nid_postfix(void) { return 0; } -int APS5_VABI EVP_CIPHER_CTX_set_padding_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI EVP_CIPHER_block_size_nid_postfix(void) { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp b/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp index 1dc5372adf..ff8f988b5f 100644 --- a/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp +++ b/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp @@ -46,4 +46,11 @@ LibreSslEvpCipherCtx* APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void) { return static_cast(calloc_nid_postfix(1, sizeof(LibreSslEvpCipherCtx))); } +int APS5_VABI EVP_CIPHER_CTX_set_padding_nid_postfix(LibreSslEvpCipherCtx* ctx, int pad) { + if (!ctx) throw std::invalid_argument(std::string(__func__) + ": null context"); + if (pad) ctx->flags &= ~CipherFlagNoPadding; + else ctx->flags |= CipherFlagNoPadding; + return 1; +} + } diff --git a/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp b/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp index 17897f6386..27e46c514b 100644 --- a/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp +++ b/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp @@ -30,6 +30,7 @@ extern "C" { void APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(void*); void* APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void); void APS5_VABI free_nid_postfix(void*); +int APS5_VABI EVP_CIPHER_CTX_set_padding_nid_postfix(void*, int); } static void Require(bool condition, std::source_location location = std::source_location::current()) { @@ -62,9 +63,29 @@ static void TestNew() { } } +static void TestSetPadding() { + unsigned char context[ContextBytes] {}; + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, 0) == 1); + Require(Flags(context) == 0x100); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, 0) == 1); + Require(Flags(context) == 0x100); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, 1) == 1); + Require(Flags(context) == 0); + SetFlags(context, UINT64_C(0xFFFFFFFF000000FF)); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, 0) == 1); + Require(Flags(context) == UINT64_C(0xFFFFFFFF000001FF)); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, -5) == 1); + Require(Flags(context) == UINT64_C(0xFFFFFFFF000000FF)); + for (std::size_t index = 0; index < ContextBytes; ++index) { + if (index < FlagsOffset || index >= FlagsOffset + 8) Require(context[index] == 0); + } + RequireThrows([] { EVP_CIPHER_CTX_set_padding_nid_postfix(nullptr, 0); }); +} + int main() { InitializeHeap(); TestInit(); TestNew(); + TestSetPadding(); return 0; } diff --git a/docs/dev/debt/libSceLibreSsl.md b/docs/dev/debt/libSceLibreSsl.md index 9e7aa49dd4..56699cf164 100644 --- a/docs/dev/debt/libSceLibreSsl.md +++ b/docs/dev/debt/libSceLibreSsl.md @@ -7,7 +7,7 @@ ### Unknown function info -- [libSceLibreSsl pending exports](../../../core/libs/prx/libSceLibreSsl/Export.cpp) - 288 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceLibreSsl.c) and were cross-checked against the same module in PS5 firmware 13.60. Their signatures and behaviour are unverified; the declarations use the no-parameter unknown-signature convention and throw through `NotImplemented_nid_no_patch`. +- [libSceLibreSsl pending exports](../../../core/libs/prx/libSceLibreSsl/Export.cpp) - 287 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceLibreSsl.c) and were cross-checked against the same module in PS5 firmware 13.60. Their signatures and behaviour are unverified; the declarations use the no-parameter unknown-signature convention and throw through `NotImplemented_nid_no_patch`. - [MD4, MD5, SHA1, SHA224, SHA256, SHA384 and SHA512 Init/Update/Final and SHA256](../../../core/libs/prx/libSceLibreSsl/src/Digest.cpp) (libSceLibreSsl) - signatures and context layouts follow the [LibreSSL headers](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/sha/sha.h), and buffering and `Final` follow its [sources](https://github.com/libressl/openbsd/tree/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/sha); nothing is checked on the console. Null contexts, null input with a nonzero length, null digest buffers in the 32-bit `Final` functions and a buffered byte count that is not below the block size, which LibreSSL does not check, throw. SHA-384 and SHA-512 initialization retains the block buffer, following [LibreSSL 2.5.4 `sha512.c`](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/sha/sha512.c) - [DES_set_key, DES_ecb_encrypt, DES_set_odd_parity](../../../core/libs/prx/libSceLibreSsl/Export.cpp) (libSceLibreSsl) - the 128-byte `DES_key_schedule` holds the 8-byte key followed by zeros instead of LibreSSL's precomputed subkeys, so a title reading its contents sees different bytes; `DES_ecb_encrypt` derives the subkeys through mbedtls on each call. Not compared with the console. Results follow LibreSSL [des_key.c](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/des/des_key.c) and [des.c](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/des/des.c): `DES_set_key` skips the parity and weak-key checks and returns 0, as it does while `DES_check_key` is 0 (that variable is not in the PS5 export inventory), a nonzero `enc` encrypts, and `DES_set_odd_parity` sets each byte's low bit for odd parity. Null pointers throw where LibreSSL faults - [X509_verify_cert_error_string](../../../core/libs/prx/libSceLibreSsl/Export.cpp) - no console call is verified. The 64-bit argument, low-32-bit lookup, messages and shared numeric fallback follow [LibreSSL 2.5.4](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/x509/x509_txt.c), the version identified in [published firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214). Console modifications and other firmware versions remain unverified. Unknown-code results share an overwritten static buffer and are not safe for concurrent use, as in that release; code 1 and codes above 54 use the numeric fallback. @@ -26,7 +26,7 @@ - [UI_set_result](../../../core/libs/prx/libSceLibreSsl/src/Ui.cpp) (libSceLibreSsl) - unverified on a PS5, and no export of this module creates a `UI` or `UI_STRING` yet. The "You must type in %d to %d characters" text LibreSSL 2.5.4 attaches to the size errors is not kept, because the error queue holds codes only. A null `UI` or result, or null boolean characters that must be compared, throws; LibreSSL dereferences them. The rest follows its [ui_lib.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/ui/ui_lib.c#L816-L879) over the 40-byte `UI` and 64-byte `UI_STRING` of [ui_locl.h](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/ui/ui_locl.h#L104-L148): a result outside the bounds sets `UI_FLAG_REDOABLE` and queues `UI_R_RESULT_TOO_SMALL` or `UI_R_RESULT_TOO_LARGE`. - [PKCS12_PBE_add, SSL_load_error_strings](../../../core/libs/prx/libSceLibreSsl/Export.cpp) (libSceLibreSsl) - unverified on a PS5; each returns without an observable effect and queues no error. `SSL_load_error_strings` builds no string table, because `ERR_error_string` and `ERR_error_string_n` throw ([ssl_err.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libssl/ssl_err.c#L483-L490)); the tables are needed once a string lookup exists. `PKCS12_PBE_add` is empty in LibreSSL 2.5.4 ([p12_crpt.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/pkcs12/p12_crpt.c#L67-L70)). - [EVP_PKEY_type](../../../core/libs/prx/libSceLibreSsl/src/Pkey.cpp) (libSceLibreSsl) - follows the method table of [LibreSSL 2.5.4](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/asn1/ameth_lib.c#L83-L111) `ameth_lib.c` built without GOST and with no engine (every `ENGINE_` export throws): RSA (6) and its alias 19 give 6, the DSA types 66, 67, 70, 113 and 116 give 116, DH (28), EC (408), HMAC (855) and CMAC (894) give themselves, and every other type, the GOST types 811, 815, 946 and 947 included, gives 0. These results match [firmware 13.42's module run natively on a PC](https://github.com/boykopovar/AnyPS5/pull/10398#issuecomment-6406072913) for every type from -1000 to 5000 and the `int` extremes; nothing is checked on the console -- [EVP_CIPHER_CTX_init, EVP_CIPHER_CTX_new](../../../core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp) (libSceLibreSsl) - nothing is checked on the console. The 168-byte `EVP_CIPHER_CTX` layout and the results follow [LibreSSL 2.5.4](https://github.com/libressl/openbsd/tree/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/evp) `evp.h` and `evp_enc.c`, the version in the [firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214), which also measured `EVP_CIPHER_CTX_init` clearing 168 bytes. `EVP_CIPHER_CTX_new` allocates through the guest `calloc` as the module would through libc. A null context, which LibreSSL dereferences, throws +- [EVP_CIPHER_CTX_init, EVP_CIPHER_CTX_new, EVP_CIPHER_CTX_set_padding](../../../core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp) (libSceLibreSsl) - nothing is checked on the console. The 168-byte `EVP_CIPHER_CTX` layout and the results follow [LibreSSL 2.5.4](https://github.com/libressl/openbsd/tree/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/evp) `evp.h` and `evp_enc.c`, the version in the [firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214), which also measured `EVP_CIPHER_CTX_init` clearing 168 bytes. `EVP_CIPHER_CTX_new` allocates through the guest `calloc` as the module would through libc. A null context, which LibreSSL dereferences, throws ### Functional From 16d0e81b02c417753cc97aefb9c7abeb66ab565a Mon Sep 17 00:00:00 2001 From: Alvaro Date: Sun, 25 Oct 2026 05:55:42 +0100 Subject: [PATCH 286/318] refactor(libSceLibreSsl): declare the EVP_CIPHER_CTX exports in the public header --- core/libs/prx/libSceLibreSsl/include/Api.hpp | 24 ++++++++++++++++--- .../prx/libSceLibreSsl/src/EvpCipherCtx.cpp | 22 ++--------------- core/libs/tests/GuestLibreSslEvpCipherCtx.cpp | 23 +++++++++--------- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/core/libs/prx/libSceLibreSsl/include/Api.hpp b/core/libs/prx/libSceLibreSsl/include/Api.hpp index 3e3d9b7601..ad3745c2b3 100644 --- a/core/libs/prx/libSceLibreSsl/include/Api.hpp +++ b/core/libs/prx/libSceLibreSsl/include/Api.hpp @@ -250,6 +250,24 @@ struct LibreSslX509StoreContext { void* exData; }; +struct LibreSslEvpCipherCtx { + const void* cipher; + void* engine; + int encrypt; + int bufLen; + unsigned char oiv[16]; + unsigned char iv[16]; + unsigned char buf[32]; + int num; + void* appData; + int keyLen; + std::uint64_t flags; + void* cipherData; + int finalUsed; + int blockMask; + unsigned char final[32]; +}; + extern "C" { std::int64_t APS5_VABI ASN1_INTEGER_get_nid_postfix(const Asn1String* value); @@ -351,9 +369,9 @@ void APS5_VABI ERR_remove_thread_state_nid_postfix(const void* threadId); int APS5_VABI EVP_CIPHER_CTX_cleanup_nid_postfix(void); int APS5_VABI EVP_CIPHER_CTX_ctrl_nid_postfix(void); int APS5_VABI EVP_CIPHER_CTX_free_nid_postfix(void); -int APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(void); -int APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void); -int APS5_VABI EVP_CIPHER_CTX_set_padding_nid_postfix(void); +void APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(LibreSslEvpCipherCtx* ctx); +LibreSslEvpCipherCtx* APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void); +int APS5_VABI EVP_CIPHER_CTX_set_padding_nid_postfix(LibreSslEvpCipherCtx* ctx, int pad); int APS5_VABI EVP_CIPHER_block_size_nid_postfix(void); int APS5_VABI EVP_CipherFinal_ex_nid_postfix(void); int APS5_VABI EVP_CipherInit_ex_nid_postfix(void); diff --git a/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp b/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp index ff8f988b5f..53046c487c 100644 --- a/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp +++ b/core/libs/prx/libSceLibreSsl/src/EvpCipherCtx.cpp @@ -1,3 +1,5 @@ +#include "prx/libSceLibreSsl/include/Api.hpp" +#include "prx/libc/include/Api.hpp" #include #include #include @@ -5,24 +7,6 @@ #include #include "prx/libc/include/General.hpp" -struct LibreSslEvpCipherCtx { - const void* cipher; - void* engine; - int encrypt; - int bufLen; - unsigned char oiv[16]; - unsigned char iv[16]; - unsigned char buf[32]; - int num; - void* appData; - int keyLen; - std::uint64_t flags; - void* cipherData; - int finalUsed; - int blockMask; - unsigned char final[32]; -}; - static_assert(sizeof(LibreSslEvpCipherCtx) == 168); static_assert(offsetof(LibreSslEvpCipherCtx, num) == 88); static_assert(offsetof(LibreSslEvpCipherCtx, flags) == 112); @@ -35,8 +19,6 @@ constexpr std::uint64_t CipherFlagNoPadding = 0x100; } extern "C" { -void* APS5_VABI calloc_nid_postfix(std::size_t count, std::size_t size); - void APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(LibreSslEvpCipherCtx* ctx) { if (!ctx) throw std::invalid_argument(std::string(__func__) + ": null context"); std::memset(ctx, 0, sizeof(LibreSslEvpCipherCtx)); diff --git a/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp b/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp index 27e46c514b..34f4558bcd 100644 --- a/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp +++ b/core/libs/tests/GuestLibreSslEvpCipherCtx.cpp @@ -1,3 +1,5 @@ +#include "prx/libSceLibreSsl/include/Api.hpp" +#include "prx/libc/include/Api.hpp" #include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include @@ -26,11 +28,8 @@ static void InitializeHeap() { ApplicationHeapRegister_nid_no_patch(api); } -extern "C" { -void APS5_VABI EVP_CIPHER_CTX_init_nid_postfix(void*); -void* APS5_VABI EVP_CIPHER_CTX_new_nid_postfix(void); -void APS5_VABI free_nid_postfix(void*); -int APS5_VABI EVP_CIPHER_CTX_set_padding_nid_postfix(void*, int); +static LibreSslEvpCipherCtx* Context(unsigned char* memory) { + return reinterpret_cast(memory); } static void Require(bool condition, std::source_location location = std::source_location::current()) { @@ -47,7 +46,7 @@ template static void RequireThrows(TAction action, std::source static void TestInit() { unsigned char memory[ContextBytes + 16]; std::memset(memory, 0xAA, sizeof(memory)); - EVP_CIPHER_CTX_init_nid_postfix(memory); + EVP_CIPHER_CTX_init_nid_postfix(Context(memory)); for (std::size_t index = 0; index < ContextBytes; ++index) Require(memory[index] == 0); for (std::size_t index = ContextBytes; index < sizeof(memory); ++index) Require(memory[index] == 0xAA); RequireThrows([] { EVP_CIPHER_CTX_init_nid_postfix(nullptr); }); @@ -55,7 +54,7 @@ static void TestInit() { static void TestNew() { for (int round = 0; round < 4; ++round) { - auto* context = static_cast(EVP_CIPHER_CTX_new_nid_postfix()); + auto* context = reinterpret_cast(EVP_CIPHER_CTX_new_nid_postfix()); Require(context != nullptr); for (std::size_t index = 0; index < ContextBytes; ++index) Require(context[index] == 0); std::memset(context, 0x5A, ContextBytes); @@ -65,16 +64,16 @@ static void TestNew() { static void TestSetPadding() { unsigned char context[ContextBytes] {}; - Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, 0) == 1); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(Context(context), 0) == 1); Require(Flags(context) == 0x100); - Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, 0) == 1); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(Context(context), 0) == 1); Require(Flags(context) == 0x100); - Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, 1) == 1); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(Context(context), 1) == 1); Require(Flags(context) == 0); SetFlags(context, UINT64_C(0xFFFFFFFF000000FF)); - Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, 0) == 1); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(Context(context), 0) == 1); Require(Flags(context) == UINT64_C(0xFFFFFFFF000001FF)); - Require(EVP_CIPHER_CTX_set_padding_nid_postfix(context, -5) == 1); + Require(EVP_CIPHER_CTX_set_padding_nid_postfix(Context(context), -5) == 1); Require(Flags(context) == UINT64_C(0xFFFFFFFF000000FF)); for (std::size_t index = 0; index < ContextBytes; ++index) { if (index < FlagsOffset || index >= FlagsOffset + 8) Require(context[index] == 0); From 61cd0e53e9e067c308916b92b7cb2546a58c564f Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Sun, 1 Nov 2026 10:56:22 +0530 Subject: [PATCH 287/318] feat(libc): declare fundamental typeinfo records and their vtable Includes the complete canonical builtin-type set: void, wchar_t, bool, chars, integer, __int128, floating, __float128, ellipsis, nullptr_t and char8/16/32 + _Float16 records. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- core/libs/prx/libc/src/exception/TypeInfo.cpp | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/core/libs/prx/libc/src/exception/TypeInfo.cpp b/core/libs/prx/libc/src/exception/TypeInfo.cpp index a1d58f019b..31e3eaec25 100644 --- a/core/libs/prx/libc/src/exception/TypeInfo.cpp +++ b/core/libs/prx/libc/src/exception/TypeInfo.cpp @@ -83,4 +83,44 @@ LibcException::TypeInfoVtable _ZTVN10__cxxabiv120__function_type_infoE_nid_postf LibcException::IsPointerTypeInfo, LibcException::IsFunctionTypeInfoTrue }; +LibcException::TypeInfoVtable _ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix { + 0, + &typeid(__cxxabiv1::__fundamental_type_info) +}; +} + +namespace { +struct FundamentalTypeRecord { const void* const* vtable; const char* name; }; +const void* const* FundamentalVtable() { + return reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix) + 2; +} +} + +extern "C" { +FundamentalTypeRecord _ZTIv_nid_postfix {FundamentalVtable(), "v"}; +FundamentalTypeRecord _ZTIw_nid_postfix {FundamentalVtable(), "w"}; +FundamentalTypeRecord _ZTIb_nid_postfix {FundamentalVtable(), "b"}; +FundamentalTypeRecord _ZTIc_nid_postfix {FundamentalVtable(), "c"}; +FundamentalTypeRecord _ZTIa_nid_postfix {FundamentalVtable(), "a"}; +FundamentalTypeRecord _ZTIh_nid_postfix {FundamentalVtable(), "h"}; +FundamentalTypeRecord _ZTIs_nid_postfix {FundamentalVtable(), "s"}; +FundamentalTypeRecord _ZTIt_nid_postfix {FundamentalVtable(), "t"}; +FundamentalTypeRecord _ZTIi_nid_postfix {FundamentalVtable(), "i"}; +FundamentalTypeRecord _ZTIj_nid_postfix {FundamentalVtable(), "j"}; +FundamentalTypeRecord _ZTIl_nid_postfix {FundamentalVtable(), "l"}; +FundamentalTypeRecord _ZTIm_nid_postfix {FundamentalVtable(), "m"}; +FundamentalTypeRecord _ZTIx_nid_postfix {FundamentalVtable(), "x"}; +FundamentalTypeRecord _ZTIy_nid_postfix {FundamentalVtable(), "y"}; +FundamentalTypeRecord _ZTIn_nid_postfix {FundamentalVtable(), "n"}; +FundamentalTypeRecord _ZTIo_nid_postfix {FundamentalVtable(), "o"}; +FundamentalTypeRecord _ZTIf_nid_postfix {FundamentalVtable(), "f"}; +FundamentalTypeRecord _ZTId_nid_postfix {FundamentalVtable(), "d"}; +FundamentalTypeRecord _ZTIe_nid_postfix {FundamentalVtable(), "e"}; +FundamentalTypeRecord _ZTIg_nid_postfix {FundamentalVtable(), "g"}; +FundamentalTypeRecord _ZTIz_nid_postfix {FundamentalVtable(), "z"}; +FundamentalTypeRecord _ZTIDn_nid_postfix {FundamentalVtable(), "Dn"}; +FundamentalTypeRecord _ZTIDs_nid_postfix {FundamentalVtable(), "Ds"}; +FundamentalTypeRecord _ZTIDi_nid_postfix {FundamentalVtable(), "Di"}; +FundamentalTypeRecord _ZTIDu_nid_postfix {FundamentalVtable(), "Du"}; +FundamentalTypeRecord _ZTIDh_nid_postfix {FundamentalVtable(), "Dh"}; } From b9343e5301d4506c2ea47542257fb77a74c5d843 Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Sun, 1 Nov 2026 10:56:23 +0530 Subject: [PATCH 288/318] test(libc): exercise exported fundamental typeinfo records directly Cover the new _ZTI records the way a guest catch would: the vtable address point, the fundamental-type kind reachable through vtable[-1], __do_catch matching and non-matching pairs, and a throw of int through __cxa_throw that a host catch resolves back to _ZTIi. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- core/libs/tests/GuestBadOptionalAccess.cpp | 39 +++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/core/libs/tests/GuestBadOptionalAccess.cpp b/core/libs/tests/GuestBadOptionalAccess.cpp index 0b9d8edfe9..4b9e99a76d 100644 --- a/core/libs/tests/GuestBadOptionalAccess.cpp +++ b/core/libs/tests/GuestBadOptionalAccess.cpp @@ -3,8 +3,19 @@ #include #include #include +#include struct TypeRecord { const void* const* vtable; const char* name; const TypeRecord* base; }; +struct FundamentalRecord { const void* const* vtable; const char* name; }; +struct FundamentalVtable { + std::ptrdiff_t offset; + const std::type_info* type; + const void* destroy; + const void* deleteObject; + const void* isPointer; + const void* isFunction; + bool (APS5_VABI *catchType)(const FundamentalRecord*, const FundamentalRecord*, void**, unsigned); +}; struct Vtable { std::ptrdiff_t offset; const TypeRecord* type; const void* destroy; const void* deleteObject; const char* (APS5_VABI *what)(const LibcException::ExceptionObject*); }; @@ -12,7 +23,11 @@ extern "C" { extern const TypeRecord _ZTISt9exception_nid_postfix; extern const TypeRecord _ZTISt19bad_optional_access_nid_postfix; extern const Vtable _ZTVSt19bad_optional_access_nid_postfix; - +extern const FundamentalVtable _ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix; +extern const FundamentalRecord _ZTIi_nid_postfix; +extern const FundamentalRecord _ZTIj_nid_postfix; +extern const FundamentalRecord _ZTIf_nid_postfix; +extern const FundamentalRecord _ZTIDn_nid_postfix; } static void Require(bool value) { if (!value) std::abort(); } @@ -38,5 +53,27 @@ int main() { thrown = true; } Require(thrown); + const auto* const addressPoint = reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix) + 2; + Require(_ZTIi_nid_postfix.vtable == addressPoint && _ZTIf_nid_postfix.vtable == addressPoint && + _ZTIDn_nid_postfix.vtable == addressPoint); + Require(std::strcmp(_ZTIi_nid_postfix.name, "i") == 0 && std::strcmp(_ZTIf_nid_postfix.name, "f") == 0 && + std::strcmp(_ZTIDn_nid_postfix.name, "Dn") == 0); + const auto* kind = reinterpret_cast(_ZTIi_nid_postfix.vtable[-1]); + Require(std::strcmp(kind->name(), "N10__cxxabiv123__fundamental_type_infoE") == 0); + void* thrownObject = nullptr; + Require(_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.catchType(&_ZTIi_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); + Require(!_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.catchType(&_ZTIi_nid_postfix, &_ZTIj_nid_postfix, &thrownObject, 0)); + Require(!_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.catchType(&_ZTIf_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); + Require(!_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.catchType(&_ZTIDn_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); + int* storage = static_cast(__cxa_allocate_exception_nid_postfix(sizeof(int))); + Require(storage != nullptr); + *storage = 42; + bool caught = false; + try { + __cxa_throw_nid_postfix(storage, reinterpret_cast(const_cast(&_ZTIi_nid_postfix)), nullptr); + } catch (...) { + caught = __cxa_current_exception_type_nid_postfix() == reinterpret_cast(&_ZTIi_nid_postfix) && *storage == 42; + } + Require(caught); return 0; } From ad55ceb9a4197d22fd19bee85cd282b6cbf8c55d Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Sun, 1 Nov 2026 10:56:25 +0530 Subject: [PATCH 289/318] test(libc): exercise fundamental typeinfo records through the guest catch path --- core/libs/tests/GuestBadOptionalAccess.cpp | 79 +++++++++++----------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/core/libs/tests/GuestBadOptionalAccess.cpp b/core/libs/tests/GuestBadOptionalAccess.cpp index 4b9e99a76d..da6d2b11ee 100644 --- a/core/libs/tests/GuestBadOptionalAccess.cpp +++ b/core/libs/tests/GuestBadOptionalAccess.cpp @@ -1,4 +1,3 @@ -#include "prx/libc/include/Api.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include #include @@ -7,73 +6,75 @@ struct TypeRecord { const void* const* vtable; const char* name; const TypeRecord* base; }; struct FundamentalRecord { const void* const* vtable; const char* name; }; -struct FundamentalVtable { - std::ptrdiff_t offset; - const std::type_info* type; - const void* destroy; - const void* deleteObject; - const void* isPointer; - const void* isFunction; - bool (APS5_VABI *catchType)(const FundamentalRecord*, const FundamentalRecord*, void**, unsigned); -}; - -struct Vtable { std::ptrdiff_t offset; const TypeRecord* type; const void* destroy; const void* deleteObject; const char* (APS5_VABI *what)(const LibcException::ExceptionObject*); }; +struct Object { const void* vtable; const char* message; }; +struct Vtable { std::ptrdiff_t offset; const TypeRecord* type; const void* destroy; const void* deleteObject; const char* (APS5_VABI *what)(const Object*); }; extern "C" { extern const TypeRecord _ZTISt9exception_nid_postfix; extern const TypeRecord _ZTISt19bad_optional_access_nid_postfix; extern const Vtable _ZTVSt19bad_optional_access_nid_postfix; -extern const FundamentalVtable _ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix; extern const FundamentalRecord _ZTIi_nid_postfix; extern const FundamentalRecord _ZTIj_nid_postfix; extern const FundamentalRecord _ZTIf_nid_postfix; extern const FundamentalRecord _ZTIDn_nid_postfix; +extern const void* const _ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix[]; +void APS5_VABI _ZNSt19bad_optional_accessC1Ev_nid_postfix(Object*); +void APS5_VABI _ZNSt19bad_optional_accessD1Ev_nid_postfix(Object*); +[[noreturn]] void APS5_VABI _ZSt26_Throw_bad_optional_accessv_nid_postfix(); +void* APS5_VABI __cxa_allocate_exception_nid_postfix(std::size_t); +[[noreturn]] void APS5_VABI __cxa_throw_nid_postfix(void*, const void*, void (*)(void*)); +const void* APS5_VABI __cxa_current_exception_type_nid_postfix(); } static void Require(bool value) { if (!value) std::abort(); } +static void TestFundamentalRecords() { + const auto* const address = reinterpret_cast(_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix) + 2; + const FundamentalRecord* records[] = {&_ZTIi_nid_postfix, &_ZTIj_nid_postfix, &_ZTIf_nid_postfix, &_ZTIDn_nid_postfix}; + for (const FundamentalRecord* record : records) + Require(record->vtable == address); + Require(std::strcmp(_ZTIi_nid_postfix.name, "i") == 0 && std::strcmp(_ZTIDn_nid_postfix.name, "Dn") == 0); + const auto* recordType = static_cast(_ZTIi_nid_postfix.vtable[-1]); + Require(std::strcmp(recordType->name(), "N10__cxxabiv123__fundamental_type_infoE") == 0); + + using CatchType = bool (APS5_VABI*)(const void*, const void*, void**, unsigned); + const auto catchType = reinterpret_cast(_ZTIi_nid_postfix.vtable[4]); + int thrownValue = 0; + void* thrownObject = &thrownValue; + Require(catchType(&_ZTIi_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); + Require(!catchType(&_ZTIi_nid_postfix, &_ZTIj_nid_postfix, &thrownObject, 0)); + Require(!catchType(&_ZTIf_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); + Require(!catchType(&_ZTIDn_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); + + void* exception = __cxa_allocate_exception_nid_postfix(sizeof(int)); + *static_cast(exception) = 42; + bool caught = false; + try { + __cxa_throw_nid_postfix(exception, &_ZTIi_nid_postfix, nullptr); + } catch (...) { + caught = __cxa_current_exception_type_nid_postfix() == &_ZTIi_nid_postfix; + } + Require(caught); +} + int main() { const auto& type = _ZTISt19bad_optional_access_nid_postfix; Require(std::strcmp(type.name, "St19bad_optional_access") == 0); Require(type.base == &_ZTISt9exception_nid_postfix); const auto& table = _ZTVSt19bad_optional_access_nid_postfix; Require(table.offset == 0 && table.type == &type && table.what != nullptr); - LibcException::ExceptionObject object{}; + Object object{}; _ZNSt19bad_optional_accessC1Ev_nid_postfix(&object); - Require(object.vtable == &table.destroy); Require(std::strcmp(table.what(&object), "bad_optional_access") == 0); _ZNSt19bad_optional_accessD1Ev_nid_postfix(&object); - bool thrown = false; try { _ZSt26_Throw_bad_optional_accessv_nid_postfix(); - } catch (...) { thrown = true; } Require(thrown); - const auto* const addressPoint = reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix) + 2; - Require(_ZTIi_nid_postfix.vtable == addressPoint && _ZTIf_nid_postfix.vtable == addressPoint && - _ZTIDn_nid_postfix.vtable == addressPoint); - Require(std::strcmp(_ZTIi_nid_postfix.name, "i") == 0 && std::strcmp(_ZTIf_nid_postfix.name, "f") == 0 && - std::strcmp(_ZTIDn_nid_postfix.name, "Dn") == 0); - const auto* kind = reinterpret_cast(_ZTIi_nid_postfix.vtable[-1]); - Require(std::strcmp(kind->name(), "N10__cxxabiv123__fundamental_type_infoE") == 0); - void* thrownObject = nullptr; - Require(_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.catchType(&_ZTIi_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); - Require(!_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.catchType(&_ZTIi_nid_postfix, &_ZTIj_nid_postfix, &thrownObject, 0)); - Require(!_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.catchType(&_ZTIf_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); - Require(!_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.catchType(&_ZTIDn_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); - int* storage = static_cast(__cxa_allocate_exception_nid_postfix(sizeof(int))); - Require(storage != nullptr); - *storage = 42; - bool caught = false; - try { - __cxa_throw_nid_postfix(storage, reinterpret_cast(const_cast(&_ZTIi_nid_postfix)), nullptr); - } catch (...) { - caught = __cxa_current_exception_type_nid_postfix() == reinterpret_cast(&_ZTIi_nid_postfix) && *storage == 42; - } - Require(caught); + TestFundamentalRecords(); return 0; } From 94e403114775b466c0948e04932765fc5e4ee46d Mon Sep 17 00:00:00 2001 From: roboterspinne <229981445+roboterspinne@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:09:54 +0100 Subject: [PATCH 290/318] fix(relinker): keep a moved Linux PT_LOAD's alignment on the header block Moving a PT_LOAD to the end of the file sets its p_align to min(p_align, 0x1000), since the copy honours only the page. When that segment carried the largest p_align, the output's largest PT_LOAD alignment, which Linux uses to align the ET_DYN load bias, fell with it. SeparateSharedLoadPages now returns the largest power-of-two p_align above the page that it lowered, and WriteLayout aligns the header block PT_LOAD to at least that value, as it already does for the largest kept p_align. --- .../general/ProgramHeaderLayoutRequest.hpp | 1 + .../general/ProgramHeaderLayoutBuilder.cpp | 2 ++ .../elfpatcher/src/linux/LinuxElfPatcher.cpp | 9 +++-- .../elfpatcher/tests/LinuxLoadPageTests.cpp | 36 +++++++++++++++++++ docs/dev/debt/relinker.md | 2 +- 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/core/relinker/elfpatcher/include/elfpatcher/general/ProgramHeaderLayoutRequest.hpp b/core/relinker/elfpatcher/include/elfpatcher/general/ProgramHeaderLayoutRequest.hpp index 956a162a77..ec0da1750a 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/general/ProgramHeaderLayoutRequest.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/general/ProgramHeaderLayoutRequest.hpp @@ -19,6 +19,7 @@ namespace Elfpatcher { std::uint64_t DynamicSegmentSize; std::uint64_t InterpOffset; std::uint64_t InterpSize; + std::uint64_t MovedLoadAlignment; }; } diff --git a/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp b/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp index a367752246..6521ade295 100644 --- a/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp +++ b/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp @@ -229,6 +229,8 @@ std::uint16_t ProgramHeaderLayoutBuilder::WriteLayout( if (request.ExtraBlockOffset > buf.size() || request.ExtraBlockSize > buf.size() - request.ExtraBlockOffset) throw Domain::RelinkerException("Extra block exceeds the output file"); std::uint64_t headerBlockAlign = kDefaultLoadAlignment; + if (request.MovedLoadAlignment > headerBlockAlign && (request.MovedLoadAlignment & (request.MovedLoadAlignment - 1)) == 0) + headerBlockAlign = request.MovedLoadAlignment; for (const auto& ph : request.OriginalHeaders) { if (ph.Type == PT_LOAD && !_segmentFilter->ShouldSkip(ph) && ph.Alignment > headerBlockAlign && (ph.Alignment & (ph.Alignment - 1)) == 0) headerBlockAlign = ph.Alignment; diff --git a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp index 2ddc76217c..8b69963705 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp @@ -125,7 +125,7 @@ std::uint64_t RelocationWidth(const std::vector& relocations, cons return type == R_X86_64_PC32 || type == R_X86_64_32 ? 4 : 8; } -void SeparateSharedLoadPages(std::vector& buf, std::vector& headers) { +std::uint64_t SeparateSharedLoadPages(std::vector& buf, std::vector& headers) { std::vector loads; for (std::size_t index = 0; index < headers.size(); ++index) { if (headers[index].Type == PT_LOAD && headers[index].MemorySize != 0) { @@ -174,6 +174,7 @@ void SeparateSharedLoadPages(std::vector& buf, std::vector previous.MemorySize) previous.FileSize = previous.MemorySize; } + std::uint64_t lowered = 0; for (std::size_t position = 0; position < loads.size(); ++position) { auto& load = headers[loads[position]]; const auto prefix = load.MappedAddress % kDefaultLoadAlignment; @@ -204,8 +205,11 @@ void SeparateSharedLoadPages(std::vector& buf, std::vector(offset + load.FileSize)); std::copy_n(buf.begin() + static_cast(load.Offset), static_cast(load.FileSize), buf.begin() + static_cast(offset)); load.Offset = offset; + if (load.Alignment > kDefaultLoadAlignment && (load.Alignment & (load.Alignment - 1)) == 0) + lowered = std::max(lowered, load.Alignment); load.Alignment = std::min(load.Alignment, kDefaultLoadAlignment); } + return lowered; } struct TracedImport { @@ -629,7 +633,7 @@ std::vector LinuxElfPatcher::Patch( for (const auto& header : headers) if (header.Type == PT_LOAD) RequireBaseAlignment(*_fixedBase, header.Alignment); - SeparateSharedLoadPages(buf, headers); + const auto movedLoadAlignment = SeparateSharedLoadPages(buf, headers); ProgramHeaderLayoutRequest layoutRequest{}; layoutRequest.PhOff = tableOff; @@ -643,6 +647,7 @@ std::vector LinuxElfPatcher::Patch( layoutRequest.DynamicSegmentSize = dynSegBuf.size(); layoutRequest.InterpOffset = interpOff; layoutRequest.InterpSize = interpSize; + layoutRequest.MovedLoadAlignment = movedLoadAlignment; const std::uint16_t writtenPh = _programHeaderLayoutBuilder->WriteLayout(buf, layoutRequest); _byteWriter->WriteU16(buf, kEhdrPhNumOffset, writtenPh); diff --git a/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp b/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp index 367d33c39c..1a56c81395 100644 --- a/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp +++ b/core/relinker/elfpatcher/tests/LinuxLoadPageTests.cpp @@ -129,6 +129,16 @@ std::uint64_t OutputAlignment(const Bytes& output, const std::uint64_t vaddr) { throw std::runtime_error("linux load pages: no output PT_LOAD at the segment's address"); } +std::uint64_t MaximumAlignment(const Bytes& output) { + std::uint64_t largest = kPage; + for (const auto& load : OutputLoads(output)) { + Require(load.Alignment <= 1 || load.Offset % load.Alignment == load.MappedAddress % load.Alignment, "PT_LOAD offset and address differ modulo its alignment"); + if (load.Alignment != 0 && (load.Alignment & (load.Alignment - 1)) == 0) + largest = std::max(largest, load.Alignment); + } + return largest; +} + void ZeroFillBelowTheNextSegment() { auto source = Fixture(0x3000); std::fill(source.begin() + 0x1000, source.begin() + 0x1100, std::uint8_t{0xAA}); @@ -284,6 +294,7 @@ void LargeAlignmentMovesToThePage() { Require(output.size() <= source.size() + 3 * kPage, "output grew to " + std::to_string(output.size()) + " bytes for alignment " + std::to_string(alignment)); RequireImage(output, source, headers, "incongruent segment with a large alignment"); Require(OutputAlignment(output, 0x1000) == kPage, "moved segment declares an alignment its offset does not honour"); + Require(MaximumAlignment(output) == alignment, "the largest PT_LOAD alignment is " + std::to_string(MaximumAlignment(output)) + ", not " + std::to_string(alignment)); } } @@ -301,6 +312,30 @@ void LargeAlignmentOnASharedPage() { Require(output.size() <= source.size() + 3 * kPage, "output grew to " + std::to_string(output.size()) + " bytes on the shared page path"); RequireImage(output, source, headers, "shared page with a large alignment"); Require(OutputAlignment(output, 0x2c20) == kPage, "moved segment declares an alignment its offset does not honour"); + Require(MaximumAlignment(output) == 0x80000000, "the largest PT_LOAD alignment is " + std::to_string(MaximumAlignment(output))); +} + +void CongruentLargeAlignmentOnASharedPage() { + auto source = Fixture(0x3000); + std::fill(source.begin() + 0x1000, source.begin() + 0x1100, std::uint8_t{0xAA}); + std::fill(source.begin() + 0x1100, source.begin() + 0x1c20, std::uint8_t{0x55}); + std::fill(source.begin() + 0x1c20, source.begin() + 0x2000, std::uint8_t{0x77}); + const std::vector headers{ + Load(0x400, 0x400, 0x10, 0x10, 5), + Load(0x1000, 0x1000, 0x100, 0x4c20, 6), + Load(0x1c20, 0x5c20, 0x3e0, 0x3e0, 4, 0x4000), + }; + const auto output = Patcher().Patch(source, headers, {}, 0, "$ORIGIN/libs", true, false, {}); + Require(output.size() <= source.size() + 3 * kPage, "output grew to " + std::to_string(output.size()) + " bytes for a congruent alignment"); + RequireImage(output, source, headers, "congruent segment with a large alignment on a shared page"); + Require(OutputOffset(output, 0x5c20) != 0x1c20 && OutputOffset(output, 0x5c20) % kPage == 0xc20, "a segment whose page reads other bytes was not moved to a page-congruent offset"); + Require(OutputAlignment(output, 0x5c20) == kPage, "moved segment declares an alignment its offset does not honour"); + Require(MaximumAlignment(output) == 0x4000, "the largest PT_LOAD alignment fell to " + std::to_string(MaximumAlignment(output))); + const auto loads = OutputLoads(output); + const auto headerBlock = std::find_if(loads.begin(), loads.end(), [](const Domain::ProgramHeader& load) { return load.Offset == 0; }); + Require(headerBlock != loads.end() && headerBlock->Alignment == 0x4000 && headerBlock->MappedAddress % 0x4000 == 0, "the header block does not carry the moved segment's alignment"); + const auto fixed = Patcher(0x400000).Patch(source, headers, {}, 0, "$ORIGIN/libs", false, false, {}); + Require(MaximumAlignment(fixed) == 0x4000, "the largest PT_LOAD alignment at a fixed base fell to " + std::to_string(MaximumAlignment(fixed))); } void LargeAlignmentAtAFixedBaseIsRefused() { @@ -335,6 +370,7 @@ int main() { IncongruentOffsetBelowThePagePrefix(); LargeAlignmentMovesToThePage(); LargeAlignmentOnASharedPage(); + CongruentLargeAlignmentOnASharedPage(); LargeAlignmentAtAFixedBaseIsRefused(); } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 252327621f..ab0011e6ad 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -18,7 +18,7 @@ - Converted Windows main executables [omit `DYNAMIC_BASE`](../../../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) and [prefer `0x400000`](../../../core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp) for their lowest ELF load page by default, not only with `--windows-fixed-base`: on the console the executable lies inside the 40-bit window that the title's own `sceLibcMspaceCreate` checks (#3043). Where the console loads an executable that does not require an address is not measured; `0x400000` is the base that executables reading their GOT through `app0/imports.txt` assume (#4013). Relocations are kept, so if that range is taken Windows can relocate the image, possibly above that window; no run has observed this, and from this base an image cannot reach the shared user data page at `0x7ffe0000`, since Windows refuses a `SizeOfImage` above `0x77000000` (#9332). Verified on Windows 11 only. - [SDK memory-management syscall traps](../../../core/relinker/relinker/src/analysis/SyscallScanner.cpp) under `--trap-sdk-syscall-stubs` recognize three exact instruction sequences from [ps5-payload-dev SDK libc/mman.c](https://github.com/ps5-payload-dev/sdk/blob/12cf3dda40/libc/mman.c), observed in Nuvio Maxd 0.5.15 (PPSA99288): syscall 74 and the two syscall 477 paths, with their argument preparation and carry-flag error handling. Matching requires a decoded instruction boundary and the complete sequence inside the scanned code range. Other SDK/compiler output remains rejected. The matched syscall becomes `UD2`; this does not implement mmap or mprotect and a call still faults. - The [instruction decoder](../../../core/relinker/codegen/src/x86/X64InstructionDecoder.cpp) reads a 16-bit displacement for `66 E8`, `66 E9` and `66 0F 80`-`8F` without REX.W, as AMD processors, the console included, do (binutils `objdump -M amd64`; REX.W selects the 32-bit form). AMD also truncates the target of every near relative branch with that prefix, rel8, LOOP and JRCXZ forms included, to 16 bits. Instruction metadata marks these branches as truncated and reports no branch target for them, and stubs refuse to move them. Compilers do not emit these forms. -- A `PT_LOAD` that the [Linux patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) copies to the end of the output, because its first page would read other bytes or its `p_offset` and `p_vaddr` differ modulo the page, is placed at an offset congruent to its address modulo the page size and its `p_align` becomes `min(p_align, 0x1000)`: Linux maps a segment from `p_offset - (p_vaddr % 0x1000)` (`elf_map` in `fs/binfmt_elf.c`) and reads `p_align` only to align the `ET_DYN` load bias to the largest power-of-two value among all `PT_LOAD` headers (`maximum_alignment`). While another `PT_LOAD` keeps that largest value, the load bias alignment does not change; when the moved segment alone carried it, as in a single-segment `--lazy-binding` image, it drops to the next largest value, down to the page. Whether a title relies on the larger alignment is not known. A kept segment keeps its `p_align`, and at a fixed base every input `p_align` above 1 must still divide the base. +- A `PT_LOAD` that the [Linux patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) copies to the end of the output, because its first page would read other bytes or its `p_offset` and `p_vaddr` differ modulo the page, declares `min(p_align, 0x1000)` as its own `p_align`: the copy lies at an offset congruent to its address modulo the page only, which is all `elf_map` in `fs/binfmt_elf.c` needs. A reader that takes that segment's `p_align` as the alignment of its memory sees the page. The header block `PT_LOAD` the patcher adds carries the largest power-of-two `p_align` the move lowered, as it does for kept segments, so `maximum_alignment` and the `ET_DYN` load bias alignment stay those of the input; as for a kept segment, a `p_align` at least as large as the Linux user address space puts that block where Linux cannot map it. At a fixed base every input `p_align` above 1 must still divide the base. - Converted Linux main executables are [placed at `0x400000`](../../../core/relinker/relinker/src/pipeline/Conversion.cpp) by default, as `ET_EXEC` without ASLR, not only with `--linux-fixed-base`: the console executable lies inside the 40-bit window that the title's own `sceLibcMspaceCreate` checks (#3043). Where the console loads an executable that does not require an address is not measured; `0x400000` is the base that executables reading their GOT through `app0/imports.txt` assume (#4013). The relinker refuses an image that spans 2 GiB or more, so the executable ends below the guest arena at `0x200000000`. With `--lazy-binding` the executable stays position independent. - `--frame-red-zone` frames only leaf functions that start with pushes (`push rbp; mov rbp, rsp; push ...`, or pushes alone followed or not by a `sub` or `lea` allocation) or with neither pushes nor an allocation, and leave through exits that undo the allocation, pop the same registers and `ret` ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). A prologue or exit shorter than its five-byte jump takes in the straight-line instructions after or before it. The leaf keeps its red zone when those cannot make five bytes, when a branch targets one of them or the first instruction of a widened prologue, when a leaf without pushes pushes or pops, or when a leaf without a frame pointer pushes or pops between its prologue and its exits: that refuses fragments that pop registers pushed before them, and shrink-wrapped leaves that push later, whose pushes would land on their own spills once RSP is lowered, and bodies whose RSP operands after a push would be moved past the pushed slot. Leaves with a frame pointer that allocate, exits that free another size and functions with indirect jumps or other RSP arithmetic keep their red zone too. - In a red-zone leaf without a frame pointer, `--frame-red-zone` lowers RSP after the prologue by the deepest displacement below RSP, rounded up to 16, and moves every RSP operand by the same amount ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). A leaf that takes the address of a slot below RSP, or uses an RSP operand with an index register, is lowered by the whole 128 bytes, since a pointer or an index can reach deeper than any displacement; values below RSP reached through any other register are not seen. Leaves with a frame pointer are lowered by 128 bytes. Inside a moved frame the CFA rules of the guest's unwind tables are off by the shift. From e7725d7111d0aaa2cc04729dae0a5be9ab485ce8 Mon Sep 17 00:00:00 2001 From: AZLisme Date: Tue, 27 Oct 2026 16:51:44 +0800 Subject: [PATCH 291/318] feat(libc): implement shared legacy sigvec support --- .../prx/libSceLibcInternal/CMakeLists.txt | 37 +++ core/libs/prx/libSceLibcInternal/Sigvec.cpp | 5 + .../prx/libSceLibcInternal/include/Api.hpp | 1 + core/libs/prx/libc/CMakeLists.txt | 14 ++ core/libs/prx/libc/include/Api.hpp | 4 +- core/libs/prx/libc/include/GuestErrno.hpp | 5 + core/libs/prx/libc/include/LegacySignals.hpp | 27 ++ core/libs/prx/libc/src/LegacySignals.cpp | 47 ++++ core/libs/prx/libc/src/SigvecExport.cpp | 5 + core/libs/prx/libkernel/CMakeLists.txt | 1 + .../System/src/SigvecRegistration.cpp | 13 + core/libs/tests/GuestExitOrder.cpp | 6 +- core/libs/tests/GuestSigvec.cpp | 171 +++++++++++++ core/libs/tests/GuestSigvecModules.cpp | 237 ++++++++++++++++++ core/libs/tests/SigvecModules.cmake | 59 +++++ core/libs/tests/SigvecModulesExceptions.cpp | 27 ++ docs/dev/debt/libc.md | 1 + 17 files changed, 656 insertions(+), 4 deletions(-) create mode 100644 core/libs/prx/libSceLibcInternal/Sigvec.cpp create mode 100644 core/libs/prx/libc/include/GuestErrno.hpp create mode 100644 core/libs/prx/libc/include/LegacySignals.hpp create mode 100644 core/libs/prx/libc/src/LegacySignals.cpp create mode 100644 core/libs/prx/libc/src/SigvecExport.cpp create mode 100644 core/libs/prx/libkernel/System/src/SigvecRegistration.cpp create mode 100644 core/libs/tests/GuestSigvec.cpp create mode 100644 core/libs/tests/GuestSigvecModules.cpp create mode 100644 core/libs/tests/SigvecModules.cmake create mode 100644 core/libs/tests/SigvecModulesExceptions.cpp diff --git a/core/libs/prx/libSceLibcInternal/CMakeLists.txt b/core/libs/prx/libSceLibcInternal/CMakeLists.txt index 2655d73c0b..da60a49d7c 100644 --- a/core/libs/prx/libSceLibcInternal/CMakeLists.txt +++ b/core/libs/prx/libSceLibcInternal/CMakeLists.txt @@ -222,3 +222,40 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +target_sources(libSceLibcInternal PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Sigvec.cpp) + +if(BUILD_TESTING AND (WIN32 OR CMAKE_SYSTEM_NAME STREQUAL "Linux")) + add_test_executable(guest_sigvec_modules_tests + ${LIBS_INCLUDE_DIR}/tests/GuestSigvecModules.cpp + ${LIBS_INCLUDE_DIR}/nid/src/NidCompute.cpp + ${LIBS_INCLUDE_DIR}/nid/src/Sha1.cpp) + target_include_directories(guest_sigvec_modules_tests PRIVATE ${LIBS_INCLUDE_DIR} ${LIBS_INCLUDE_DIR}/nid/include) + target_link_libraries(guest_sigvec_modules_tests PRIVATE ${CMAKE_DL_LIBS}) + set_source_files_properties(${LIBS_INCLUDE_DIR}/tests/GuestSigvecModules.cpp PROPERTIES COMPILE_OPTIONS "-fno-exceptions;-fno-asynchronous-unwind-tables") + if(MINGW) + target_compile_options(guest_sigvec_modules_tests PRIVATE -fasynchronous-unwind-tables) + target_link_options(guest_sigvec_modules_tests PRIVATE -static -static-libgcc -static-libstdc++) + endif() + add_library(sigvec_modules_exceptions SHARED ${LIBS_INCLUDE_DIR}/tests/SigvecModulesExceptions.cpp) + target_include_directories(sigvec_modules_exceptions PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(sigvec_modules_exceptions PRIVATE libc) + set_target_properties(sigvec_modules_exceptions PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) + configure_windows_unwind(sigvec_modules_exceptions) + add_dependencies(guest_sigvec_modules_tests libc libkernel libSceLibcInternal nid_patcher sigvec_modules_exceptions) + set(sigvecAllocatorArgument) + if(TARGET mimalloc) + set(sigvecAllocatorArgument -DALLOCATOR_DIRECTORY=$) + endif() + add_test(NAME guest_sigvec_modules COMMAND ${CMAKE_COMMAND} + -DEXCEPTION_PROBE=$ + -DPATCHER=$ + -DLIBC=$ + -DKERNEL=$ + -DINTERNAL=$ + ${sigvecAllocatorArgument} + -DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}/sigvec-modules + -P ${LIBS_INCLUDE_DIR}/tests/SigvecModules.cmake + -- "$") + set_tests_properties(guest_sigvec_modules PROPERTIES TIMEOUT 20) +endif() diff --git a/core/libs/prx/libSceLibcInternal/Sigvec.cpp b/core/libs/prx/libSceLibcInternal/Sigvec.cpp new file mode 100644 index 0000000000..531b0a0cfb --- /dev/null +++ b/core/libs/prx/libSceLibcInternal/Sigvec.cpp @@ -0,0 +1,5 @@ +#include "prx/libc/include/LegacySignals.hpp" + +extern "C" int APS5_VABI sigvec_nid_postfix(int signal, const GuestSignalVector* replacement, GuestSignalVector* previous) { + return LegacySigvec_nid_no_patch(signal, replacement, previous); +} diff --git a/core/libs/prx/libSceLibcInternal/include/Api.hpp b/core/libs/prx/libSceLibcInternal/include/Api.hpp index a995513d96..97e30f397e 100644 --- a/core/libs/prx/libSceLibcInternal/include/Api.hpp +++ b/core/libs/prx/libSceLibcInternal/include/Api.hpp @@ -4,6 +4,7 @@ #include "prx/libSceLibcInternal/include/Spawn.hpp" #include "prx/libkernel/System/include/UserContext.hpp" #include "prx/libc/include/FileStream.hpp" +#include "prx/libc/include/LegacySignals.hpp" #include "SceTypes.hpp" #include diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index 90509b697a..c0db6f2639 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -134,6 +134,20 @@ set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/src/MathExact.cpp TARGET_DIRECTORY libc PROPERTIES COMPILE_OPTIONS "-frounding-math") target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/BinaryWordIo.cpp) +target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/LegacySignals.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/SigvecExport.cpp) + +if(BUILD_TESTING) + add_test_executable(guest_sigvec_tests ${LIBS_INCLUDE_DIR}/tests/GuestSigvec.cpp) + target_include_directories(guest_sigvec_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_sigvec_tests PRIVATE libc libkernel) + configure_windows_unwind(guest_sigvec_tests) + add_test(NAME guest_sigvec COMMAND guest_sigvec_tests) + set_tests_properties(guest_sigvec PROPERTIES TIMEOUT 10) + if(WIN32) + set_tests_properties(guest_sigvec PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() target_include_directories(libc PRIVATE ${LIBS_INCLUDE_DIR} diff --git a/core/libs/prx/libc/include/Api.hpp b/core/libs/prx/libc/include/Api.hpp index 2e55965a17..a64e0da5b0 100644 --- a/core/libs/prx/libc/include/Api.hpp +++ b/core/libs/prx/libc/include/Api.hpp @@ -6,9 +6,11 @@ #include "prx/libc/include/GuestTime.hpp" #include "prx/libc/include/GuestLocale.hpp" #include "prx/libc/include/GuestDirectory.hpp" +#include "prx/libc/include/GuestErrno.hpp" #include "prx/libc/include/FileStream.hpp" #include "prx/libc/include/exceptions/Unwind.hpp" #include "prx/libc/include/exceptions/Runtime.hpp" +#include "prx/libc/include/LegacySignals.hpp" #include "SceTypes.hpp" #include #include @@ -104,6 +106,7 @@ struct FutureErrorObject { const void* vtable; GuestLocale::String message; Erro extern "C" { + int APS5_VABI std_execute_once_nid_postfix(int* flag, int (APS5_VABI *func)(void*, void*, void**), void* arg); void APS5_VABI LibcHeapGetTraceInfo_nid_postfix(LibcHeapInfo* info); int APS5_VABI _ZSt14_Atomic_assertPKcS0__nid_postfix(); @@ -830,7 +833,6 @@ std::size_t APS5_VABI mbsrtowcs_nid_postfix(std::uint16_t* destination, const ch std::size_t APS5_VABI mbstowcs_nid_postfix(std::uint16_t* dest, const char* src, std::size_t size); void APS5_VABI exit_nid_postfix(int code); int* APS5_VABI libc_error_nid_postfix(); -int* APS5_VABI __error_nid_postfix(); int APS5_VABI atexit_nid_postfix(GuestExitCallback func); int APS5_VABI at_quick_exit_nid_postfix(GuestExitCallback func); int APS5_VABI daemon_nid_postfix(int noChdir, int noClose); diff --git a/core/libs/prx/libc/include/GuestErrno.hpp b/core/libs/prx/libc/include/GuestErrno.hpp new file mode 100644 index 0000000000..00d2aa0f77 --- /dev/null +++ b/core/libs/prx/libc/include/GuestErrno.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include "general/VabiMacros.hpp" + +extern "C" int* APS5_VABI __error_nid_postfix(); diff --git a/core/libs/prx/libc/include/LegacySignals.hpp b/core/libs/prx/libc/include/LegacySignals.hpp new file mode 100644 index 0000000000..0f99fb0ef0 --- /dev/null +++ b/core/libs/prx/libc/include/LegacySignals.hpp @@ -0,0 +1,27 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_LEGACYSIGNALS_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_LEGACYSIGNALS_HPP + +#include "prx/libkernel/include/Api.hpp" +#include +#include + +struct GuestSignalVector { + std::uintptr_t handler; + int mask; + int flags; +}; + +static_assert(sizeof(GuestSignalVector) == 16 && offsetof(GuestSignalVector, mask) == 8 && offsetof(GuestSignalVector, flags) == 12); + +using LegacySigaction_nid_no_patch = int (APS5_VABI *)(int, const GuestSigaction*, GuestSigaction*); + +extern "C" { + +void RegisterSigvec_nid_no_patch(LegacySigaction_nid_no_patch action); +int LegacySigvec_nid_no_patch(int signal, const GuestSignalVector* replacement, GuestSignalVector* previous); +int APS5_VABI sigvec_nid_disambig1(int signal, const GuestSignalVector* replacement, GuestSignalVector* previous); +int APS5_VABI sigvec_nid_postfix(int signal, const GuestSignalVector* replacement, GuestSignalVector* previous); + +} + +#endif diff --git a/core/libs/prx/libc/src/LegacySignals.cpp b/core/libs/prx/libc/src/LegacySignals.cpp new file mode 100644 index 0000000000..4e935693fc --- /dev/null +++ b/core/libs/prx/libc/src/LegacySignals.cpp @@ -0,0 +1,47 @@ +#include "prx/libc/libc.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include +#include +#include +#include + +namespace { + +std::atomic signalAction{}; + +} + +extern "C" { + +void RegisterSigvec_nid_no_patch(LegacySigaction_nid_no_patch action) { + signalAction.store(action, std::memory_order_release); +} + +int LegacySigvec_nid_no_patch(int signal, const GuestSignalVector* replacement, GuestSignalVector* previous) { + const auto forward = signalAction.load(std::memory_order_acquire); + if (forward == nullptr) throw std::runtime_error("sigvec: libkernel has not registered sigaction"); + GuestSigaction action{}; + if (replacement != nullptr) { + action.handler = replacement->handler; + action.flags = replacement->flags ^ 2; + action.mask.bits[0] = static_cast(replacement->mask); + } + std::optional access; + if (previous != nullptr) { + access.emplace(previous, sizeof(*previous)); + if (!access->Open()) { + *__error_nid_postfix() = 14; + return -1; + } + } + GuestSigaction old{}; + const int result = forward(signal, replacement == nullptr ? nullptr : &action, previous == nullptr ? nullptr : &old); + if (result == 0 && previous != nullptr) { + previous->handler = old.handler; + previous->mask = std::bit_cast(old.mask.bits[0]); + previous->flags = old.flags ^ 2; + } + return result; +} + +} diff --git a/core/libs/prx/libc/src/SigvecExport.cpp b/core/libs/prx/libc/src/SigvecExport.cpp new file mode 100644 index 0000000000..fcfeed0d19 --- /dev/null +++ b/core/libs/prx/libc/src/SigvecExport.cpp @@ -0,0 +1,5 @@ +#include "prx/libc/include/LegacySignals.hpp" + +extern "C" int APS5_VABI sigvec_nid_disambig1(int signal, const GuestSignalVector* replacement, GuestSignalVector* previous) { + return LegacySigvec_nid_no_patch(signal, replacement, previous); +} diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 79f5520b73..d4b23e646a 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -81,6 +81,7 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Error.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/ThermalSensors.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Signals.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/System/src/SigvecRegistration.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/SignalSets.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Notification.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/UserContext.cpp diff --git a/core/libs/prx/libkernel/System/src/SigvecRegistration.cpp b/core/libs/prx/libkernel/System/src/SigvecRegistration.cpp new file mode 100644 index 0000000000..495a909494 --- /dev/null +++ b/core/libs/prx/libkernel/System/src/SigvecRegistration.cpp @@ -0,0 +1,13 @@ +#include "prx/libc/include/LegacySignals.hpp" + +namespace { + +struct RegisterSigvecOnLoad { + RegisterSigvecOnLoad() { + RegisterSigvec_nid_no_patch(sigaction_nid_postfix); + } +}; + +const RegisterSigvecOnLoad registerSigvec; + +} diff --git a/core/libs/tests/GuestExitOrder.cpp b/core/libs/tests/GuestExitOrder.cpp index af813531ef..3520fd68fb 100644 --- a/core/libs/tests/GuestExitOrder.cpp +++ b/core/libs/tests/GuestExitOrder.cpp @@ -14,7 +14,7 @@ void Record(char value) { static_cast(!write(events[1], &value, 1)); } -void APS5_VABI GuestHandler(void*) { Record('G'); } +void APS5_VABI RecordGuestExit(void*) { Record('G'); } void HostHandler() { Record('H'); } @@ -46,11 +46,11 @@ bool ExitOrder(Setup setup, const char* expected) { } int main() { - if (!ExitOrder([] { __cxa_atexit_nid_postfix(GuestHandler, nullptr, nullptr); std::atexit(HostHandler); }, "GH")) { + if (!ExitOrder([] { __cxa_atexit_nid_postfix(RecordGuestExit, nullptr, nullptr); std::atexit(HostHandler); }, "GH")) { std::fputs("guest exit handlers must run before the host exit handlers\n", stderr); return 1; } - if (!ExitOrder([] { __cxa_atexit_nid_postfix(GuestHandler, nullptr, nullptr); atexit_nid_postfix(AtexitHandler); }, "AG")) { + if (!ExitOrder([] { __cxa_atexit_nid_postfix(RecordGuestExit, nullptr, nullptr); atexit_nid_postfix(AtexitHandler); }, "AG")) { std::fputs("atexit and __cxa_atexit handlers must run in reverse order of registration\n", stderr); return 1; } diff --git a/core/libs/tests/GuestSigvec.cpp b/core/libs/tests/GuestSigvec.cpp new file mode 100644 index 0000000000..1680cac705 --- /dev/null +++ b/core/libs/tests/GuestSigvec.cpp @@ -0,0 +1,171 @@ +#include "prx/libc/libc.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +void Require(bool value, const char* message) { + if (value) return; + std::fprintf(stderr, "sigvec: %s\n", message); + std::abort(); +} + +GuestSignalSet Mask() { + GuestSignalSet value{}; + Require(_sigprocmask_nid_postfix(0, nullptr, &value) == 0, "query mask"); + return value; +} + +bool Equal(const GuestSignalSet& first, const GuestSignalSet& second) { + for (int word = 0; word < 4; ++word) if (first.bits[word] != second.bits[word]) return false; + return true; +} + +bool Equal(const GuestSignalVector& first, const GuestSignalVector& second) { + return first.handler == second.handler && first.mask == second.mask && first.flags == second.flags; +} + +int received = 0; +int signalReceived = 0; + +void APS5_VABI Handler(int signal) { + ++received; + signalReceived = signal; +} + +void Vectors() { + constexpr int signal = 30; + const auto handler = reinterpret_cast(&Handler); + GuestSigaction initial{}; + Require(sigaction_nid_postfix(signal, nullptr, &initial) == 0, "save original action"); + for (int flags = 0; flags < 128; ++flags) { + const GuestSignalVector replacement{handler, std::bit_cast(0x80000005u), flags}; + Require(sigvec_nid_disambig1(signal, &replacement, nullptr) == 0, "install each supported flag combination"); + GuestSigaction actual{}; + Require(sigaction_nid_postfix(signal, nullptr, &actual) == 0, "query modern action"); + Require(actual.handler == handler && actual.flags == (flags ^ 2), "legacy interrupt flag reverses restart flag"); + Require(Equal(actual.mask, GuestSignalSet{{0x80000005, 0, 0, 0}}), "legacy vector zero extends mask"); + GuestSignalVector queried{}; + Require(sigvec_nid_disambig1(signal, nullptr, &queried) == 0 && Equal(queried, replacement), "query converts flags and mask back"); + } + const GuestSigaction extended{handler, 0x13, {{0x87654321, 0x12345678, 0x80000001, 0x7fffffff}}}; + Require(sigaction_nid_postfix(signal, &extended, nullptr) == 0, "install action with extended mask"); + GuestSignalVector queried{}; + *__error_nid_postfix() = 81; + Require(sigvec_nid_disambig1(signal, nullptr, &queried) == 0, "query extended action"); + Require(Equal(queried, GuestSignalVector{handler, std::bit_cast(0x87654321u), 0x11}), "query truncates modern mask to low word"); + Require(*__error_nid_postfix() == 81, "successful vector preserves errno"); + GuestSignalVector alias{handler, 7, 0}; + Require(sigvec_nid_disambig1(signal, &alias, &alias) == 0 && Equal(alias, queried), "aliased input and output returns old action"); + GuestSignalVector newState{}; + Require(sigvec_nid_disambig1(signal, nullptr, &newState) == 0 && Equal(newState, GuestSignalVector{handler, 7, 0}), "alias installs original input"); + const GuestSignalVector sentinel{0x12345678, -123, 0x345}; + const GuestSignalVector invalid{handler, -1, 0x80}; + for (const int number : std::array{-1, 0, 129, signal}) { + GuestSignalVector output = sentinel; + *__error_nid_postfix() = 0; + Require(sigvec_nid_disambig1(number, &invalid, &output) == -1, "invalid signal or flags fail"); + Require(*__error_nid_postfix() == 22 && Equal(output, sentinel), "failure preserves previous output and exposes errno"); + } + Require(sigvec_nid_disambig1(signal, nullptr, &queried) == 0 && Equal(queried, newState), "failed update leaves disposition unchanged"); + for (const int number : std::array{9, 17}) { + GuestSignalVector output = sentinel; + Require(sigvec_nid_disambig1(number, &newState, &output) == -1 && Equal(output, sentinel), "uninstallable handlers preserve output"); + } + Require(sigvec_nid_disambig1(signal, nullptr, nullptr) == 0, "null input and output query is accepted"); + const GuestSignalSet saved = Mask(); + const GuestSignalSet empty{}; + Require(_sigprocmask_nid_postfix(3, &empty, nullptr) == 0, "clear mask for delivery"); + received = 0; + const GuestSignalSet blocked{{1u << (signal - 1), 0, 0, 0}}; + Require(_sigprocmask_nid_postfix(1, &blocked, nullptr) == 0, "block installed handler"); + Require(raise_nid_postfix(signal) == 0 && received == 0, "blocked signal becomes pending"); + GuestSignalSet previous{}; + Require(_sigprocmask_nid_postfix(3, &empty, &previous) == 0 && previous.bits[0] == (1u << (signal - 1)), "unblock returns prior mask"); + Require(received == 1 && signalReceived == signal, "unblock delivers pending signal through guest handler"); + Require(raise_nid_postfix(signal) == 0 && received == 2, "unblocked delivery retains vector handler"); + Require(sigaction_nid_postfix(signal, &initial, nullptr) == 0, "restore disposition"); + Require(_sigprocmask_nid_postfix(3, &saved, nullptr) == 0, "restore mask after delivery"); +} + +void GuestBackedOutput() { + constexpr std::size_t page = 0x4000; + constexpr std::size_t bytes = page * 2; + std::int64_t physical = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, bytes, 0, 0, &physical) == 0, "allocate guest signal storage"); + void* mapping = nullptr; + Require(sceKernelMapDirectMemory(&mapping, bytes, 3, 0, physical, 0) == 0 && mapping != nullptr, "map guest signal storage"); + Require(GuestArena::GuestArenaContains_nid_postfix(mapping, bytes), "output is not backed by the guest arena"); + auto* memory = static_cast(mapping); + { + GuestArena::HostWrite access(mapping, bytes); + Require(access.Open(), "initialize guest signal storage"); + std::memset(mapping, 0xA5, bytes); + } + constexpr int signal = 30; + GuestSigaction initial{}; + Require(sigaction_nid_postfix(signal, nullptr, &initial) == 0, "save guest-output disposition"); + const GuestSignalVector replacement{reinterpret_cast(&Handler), -123, 0}; + auto* output = reinterpret_cast(memory + page - 8); + Require(sigvec_nid_disambig1(signal, &replacement, output) == 0, "write complete vector across guest pages"); + Require(Equal(*output, GuestSignalVector{initial.handler, std::bit_cast(initial.mask.bits[0]), initial.flags ^ 2}), "guest vector conversion"); + Require(memory[page - 9] == 0xA5 && memory[page + 8] == 0xA5, "guest vector output escaped its bounds"); + *output = replacement; + Require(sigvec_nid_disambig1(signal, output, output) == 0 && Equal(*output, replacement), "guest input-output alias conversion"); +#ifdef _WIN32 + const GuestSignalVector sentinel{0x12345678, -123, 0x345}; + const GuestSignalVector changed{replacement.handler, 5, 2}; + const auto reject = [&](GuestSignalVector* destination, void* protectedPage) { + *destination = sentinel; + Require(sceKernelMprotect(protectedPage, page, 1) == 0, "protect guest vector output"); + { + GuestArena::HostWrite access(destination, sizeof(*destination)); + Require(!access.Open(), "read-only guest output unexpectedly opened"); + } + *__error_nid_postfix() = 73; + Require(sigvec_nid_disambig1(signal, &changed, destination) == -1 && *__error_nid_postfix() == 14, "read-only guest vector was not rejected"); + Require(Equal(*destination, sentinel), "rejected guest vector was partially overwritten"); + GuestSignalVector current{}; + Require(sigvec_nid_disambig1(signal, nullptr, ¤t) == 0 && Equal(current, replacement), "rejected output changed the disposition"); + Require(sigvec_nid_disambig1(signal, destination, destination) == -1 && *__error_nid_postfix() == 14, "read-only aliased vector was not rejected before mutation"); + Require(Equal(*destination, sentinel), "rejected aliased vector was changed"); + Require(sigvec_nid_disambig1(signal, nullptr, ¤t) == 0 && Equal(current, replacement), "rejected alias changed the disposition"); + Require(sceKernelMprotect(protectedPage, page, 3) == 0, "restore guest vector protection"); + }; + reject(reinterpret_cast(memory + 128), mapping); + reject(output, memory + page); +#endif + Require(sigaction_nid_postfix(signal, &initial, nullptr) == 0, "restore guest-output disposition"); + Require(sceKernelMunmap(reinterpret_cast(mapping), bytes) == 0, "unmap guest signal storage"); + Require(sceKernelReleaseDirectMemory(physical, bytes) == 0, "release guest signal storage"); +} + +void MissingBackend() { + const auto saved = Mask(); + const GuestSignalVector sentinel{0x12345678, -123, 0x345}; + GuestSignalVector output = sentinel; + const GuestSignalVector replacement{reinterpret_cast(&Handler), 7, 0}; + RegisterSigvec_nid_no_patch(nullptr); + bool threw = false; + try { + sigvec_nid_disambig1(30, &replacement, &output); + } catch (const std::runtime_error&) { + threw = true; + } + Require(threw, "missing backend throws"); + Require(Equal(output, sentinel) && Equal(Mask(), saved), "missing backend leaves output and masks untouched"); + RegisterSigvec_nid_no_patch(sigaction_nid_postfix); +} + +} + +int main() { + Vectors(); + GuestBackedOutput(); + MissingBackend(); +} diff --git a/core/libs/tests/GuestSigvecModules.cpp b/core/libs/tests/GuestSigvecModules.cpp new file mode 100644 index 0000000000..6a4c33fe01 --- /dev/null +++ b/core/libs/tests/GuestSigvecModules.cpp @@ -0,0 +1,237 @@ +#include "prx/libc/include/LegacySignals.hpp" +#include "prx/libc/include/GuestErrno.hpp" +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#include +#endif + +namespace { + +void Require(bool value, const char* message) { + if (value) return; + std::fprintf(stderr, "sigvec modules: %s\n", message); + std::abort(); +} + +class Module { +public: + explicit Module(const std::filesystem::path& path) { +#ifdef _WIN32 + handle = LoadLibraryExW(path.c_str(), nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); + if (handle == nullptr) std::fprintf(stderr, "LoadLibraryExW(%ls) failed: %lu\n", path.c_str(), GetLastError()); +#else + handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (handle == nullptr) std::fprintf(stderr, "%s\n", dlerror()); +#endif + Require(handle != nullptr, "load actual patched module"); +#ifdef _WIN32 + HMODULE pinned = nullptr; + Require(GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_PIN | GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, + reinterpret_cast(handle), &pinned) != 0 && pinned == handle, "pin actual module until process exit"); +#endif + } + + ~Module() { +#ifdef _WIN32 + FreeLibrary(handle); +#else + dlclose(handle); +#endif + } + + Module(const Module&) = delete; + Module& operator=(const Module&) = delete; + + void* Find(const char* name) const { +#ifdef _WIN32 + auto* result = reinterpret_cast(GetProcAddress(handle, name)); +#else + auto* result = dlsym(handle, name); +#endif + return result; + } + + void* Symbol(const char* name) const { + auto* result = Find(name); + Require(result != nullptr, "lookup actual native export"); + return result; + } + + void RequireOwner(void* address) const { +#ifdef _WIN32 + HMODULE owner = nullptr; + Require(GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(address), &owner) != 0 && owner == handle, "NID resolved outside requested DLL"); +#else + link_map* image = nullptr; + Dl_info info{}; + Require(dlinfo(handle, RTLD_DI_LINKMAP, &image) == 0 && image != nullptr, "read requested module identity"); + Require(dladdr(address, &info) != 0 && info.dli_fbase == reinterpret_cast(image->l_addr), "NID resolved through an ELF dependency"); +#endif + } + + template + T Function(const char* name) const { + return reinterpret_cast(Symbol(Nid::ComputeNid(name, "").c_str())); + } + +private: +#ifdef _WIN32 + HMODULE handle = nullptr; +#else + void* handle = nullptr; +#endif +}; + +using Sigvec = decltype(&sigvec_nid_postfix); +GuestSigaction recorded{}; +GuestSigaction answer{0x12345678, 0x13, {{0x87654321, 0x11111111, 0x22222222, 0x33333333}}}; +int calls = 0; +int result = 0; +bool sawReplacement = false; +bool sawPrevious = false; +int requestedSignal = 0; + +int APS5_VABI RecordAction(int signal, const GuestSigaction* replacement, GuestSigaction* previous) { + ++calls; + requestedSignal = signal; + sawReplacement = replacement != nullptr; + sawPrevious = previous != nullptr; + if (replacement != nullptr) recorded = *replacement; + if (result == 0 && previous != nullptr) *previous = answer; + return result; +} + +bool Equal(const GuestSignalVector& first, const GuestSignalVector& second) { + return first.handler == second.handler && first.mask == second.mask && first.flags == second.flags; +} + +void CheckSharedDisposition(const std::array& exports, const Module& libc) { + const auto error = libc.Function("__error"); + GuestSignalVector original{}; + Require(exports[0](30, nullptr, &original) == 0, "save actual kernel disposition"); + const GuestSignalVector first{1, std::bit_cast(0x80000005u), 3}; + const GuestSignalVector second{1, -123, 2}; + GuestSignalVector queried{}; + Require(exports[1](30, &first, nullptr) == 0 && exports[0](30, nullptr, &queried) == 0 && Equal(queried, first), "Internal install is not shared with libc"); + Require(exports[0](30, &second, nullptr) == 0 && exports[1](30, nullptr, &queried) == 0 && Equal(queried, second), "libc install is not shared with Internal"); + for (const auto function : exports) { + queried = first; + Require(function(0, &second, &queried) == -1 && *error() == 22 && Equal(queried, first), "actual backend failure changed module output"); + } + Require(exports[0](30, &original, nullptr) == 0, "restore actual kernel disposition"); +} + +void CheckConversions(const std::array& exports) { + const GuestSignalVector expected{answer.handler, std::bit_cast(answer.mask.bits[0]), answer.flags ^ 2}; + for (const auto function : exports) { + for (const int flags : {0, 1, 2, 3, -1}) { + const GuestSignalVector replacement{0x34567890, -1, flags}; + GuestSignalVector previous{}; + result = 0; + const int before = calls; + Require(function(30, &replacement, &previous) == 0 && calls == before + 1, "module wrapper did not call shared core once"); + Require(requestedSignal == 30 && sawReplacement && sawPrevious, "forwarded argument identity"); + Require(recorded.handler == replacement.handler && recorded.flags == (flags ^ 2), "firmware handler/flag conversion"); + Require(recorded.mask.bits[0] == 0xffffffffu && recorded.mask.bits[1] == 0 && recorded.mask.bits[2] == 0 && recorded.mask.bits[3] == 0, "firmware low mask conversion"); + Require(Equal(previous, expected), "firmware previous vector conversion"); + GuestSignalVector alias = replacement; + Require(function(30, &alias, &alias) == 0 && Equal(alias, expected), "module-owned alias conversion"); + Require(recorded.handler == replacement.handler && recorded.flags == (flags ^ 2) && recorded.mask.bits[0] == 0xffffffffu, "alias input changed before forwarding"); + } + result = 0; + Require(function(30, nullptr, nullptr) == 0 && !sawReplacement && !sawPrevious, "both null pointers were not forwarded"); + GuestSignalVector previous{}; + Require(function(30, nullptr, &previous) == 0 && !sawReplacement && sawPrevious && Equal(previous, expected), "null replacement query"); + const GuestSignalVector replacement{0x34567890, 7, 0}; + Require(function(30, &replacement, nullptr) == 0 && sawReplacement && !sawPrevious, "null previous output"); + for (const int failure : {-7, -1, 1}) { + result = failure; + previous = replacement; + Require(function(17, &replacement, &previous) == failure && Equal(previous, replacement), "forward failure changed output or result"); + } + } +} + +void CheckGuestOutput(const std::array& exports, const Module& kernel, const Module& libc) { + const auto allocate = kernel.Function("sceKernelAllocateDirectMemory"); + const auto map = kernel.Function("sceKernelMapDirectMemory"); + const auto unmap = kernel.Function("sceKernelMunmap"); + const auto release = kernel.Function("sceKernelReleaseDirectMemory"); + constexpr std::size_t page = 0x4000; + std::int64_t physical = 0; + Require(allocate(0, 0x7fffffffffll, page * 2, 0, 0, &physical) == 0, "allocate real guest module output"); + void* mapping = nullptr; + Require(map(&mapping, page * 2, 3, 0, physical, 0) == 0 && mapping != nullptr, "map real guest module output"); + auto* output = reinterpret_cast(static_cast(mapping) + page - 8); + const GuestSignalVector expected{answer.handler, std::bit_cast(answer.mask.bits[0]), answer.flags ^ 2}; + result = 0; + for (const auto function : exports) Require(function(30, nullptr, output) == 0 && Equal(*output, expected), "owned NID guest-backed output"); +#ifdef _WIN32 + const auto protect = kernel.Function("sceKernelMprotect"); + const auto error = libc.Function("__error"); + const GuestSignalVector sentinel{0x23456789, 5, 2}; + *output = sentinel; + Require(protect(static_cast(mapping) + page, page, 1) == 0, "protect module output tail"); + for (const auto function : exports) { + const int before = calls; + Require(function(30, &sentinel, output) == -1 && *error() == 14 && calls == before, "rejected module output mutated backend"); + Require(Equal(*output, sentinel), "rejected module output partially written"); + } + Require(protect(static_cast(mapping) + page, page, 3) == 0, "restore module output protection"); +#else + static_cast(libc); +#endif + Require(unmap(reinterpret_cast(mapping), page * 2) == 0, "unmap module output"); + Require(release(physical, page * 2) == 0, "release module output"); +} + +} + +int main(int argc, char** argv) { + Require(argc == 3 || (argc == 4 && std::string(argv[3]) == "libc-first"), "expected patched module directory, exception probe and optional load order"); + const auto directory = std::filesystem::absolute(argv[1]); + const Module first(directory / (argc == 4 ? "libc.prx" : "libSceLibcInternal.prx")); + const Module libc(directory / "libc.prx"); + const Module kernel(directory / "libkernel.prx"); + const Module internal(directory / "libSceLibcInternal.prx"); + auto* libcAddress = libc.Symbol("yhxKO9LYc8A"); + auto* internalAddress = internal.Symbol("yhxKO9LYc8A"); + libc.RequireOwner(libcAddress); + internal.RequireOwner(internalAddress); + Require(libcAddress != internalAddress, "both modules resolve to the same wrapper"); + std::array exports{reinterpret_cast(libcAddress), reinterpret_cast(internalAddress)}; + for (const auto function : exports) Require(function(30, nullptr, nullptr) == 0, "cold kernel registration missing"); + CheckSharedDisposition(exports, libc); + auto* registrationAddress = libc.Symbol("RegisterSigvec_nid_no_patch"); + libc.RequireOwner(registrationAddress); + auto* coreAddress = libc.Symbol("LegacySigvec_nid_no_patch"); + libc.RequireOwner(coreAddress); +#ifdef _WIN32 + Require(internal.Find("LegacySigvec_nid_no_patch") == nullptr, "Internal duplicated the private core"); +#else + Require(internal.Find("LegacySigvec_nid_no_patch") == coreAddress, "Internal does not use libc private core"); +#endif + const auto registration = reinterpret_cast(registrationAddress); + const Module exceptionProbe(directory / argv[2]); + auto* exceptionAddress = exceptionProbe.Symbol("SigvecMissingBackend_nid_no_patch"); + exceptionProbe.RequireOwner(exceptionAddress); + const auto checkMissingBackend = reinterpret_cast(exceptionAddress); + registration(nullptr); + Require(checkMissingBackend(exports[0], exports[1], coreAddress), "module wrapper has duplicate or stale callback state"); + registration(RecordAction); + CheckConversions(exports); + CheckGuestOutput(exports, kernel, libc); + registration(kernel.Function("sigaction")); + for (const auto function : exports) Require(function(30, nullptr, nullptr) == 0, "actual kernel registration restore"); + std::printf("sigvec NID ownership, shared callback/core, firmware conversions and guest output passed (%s first)\n", argc == 4 ? "libc" : "Internal"); +} diff --git a/core/libs/tests/SigvecModules.cmake b/core/libs/tests/SigvecModules.cmake new file mode 100644 index 0000000000..4a9f172f26 --- /dev/null +++ b/core/libs/tests/SigvecModules.cmake @@ -0,0 +1,59 @@ +cmake_policy(SET CMP0054 NEW) +math(EXPR testExecutableIndex "${CMAKE_ARGC} - 1") +math(EXPR separatorIndex "${CMAKE_ARGC} - 2") +if(NOT CMAKE_ARGV${separatorIndex} STREQUAL "--") + message(FATAL_ERROR "Expected the test executable after --") +endif() +set(TEST_EXECUTABLE "${CMAKE_ARGV${testExecutableIndex}}") + +file(MAKE_DIRECTORY "${WORK_DIR}") +execute_process(COMMAND "${PATCHER}" --write-export-list "${LIBC}" "${WORK_DIR}/libc.exports" + RESULT_VARIABLE status OUTPUT_VARIABLE output ERROR_VARIABLE error) +if(NOT status STREQUAL "0") + message(FATAL_ERROR "libc export list failed: ${output}${error}") +endif() +foreach(module LIBC KERNEL INTERNAL) + get_filename_component(name "${${module}}" NAME) + get_filename_component(library "${${module}}" NAME_WE) + file(COPY_FILE "${${module}}" "${WORK_DIR}/${name}") + if(module STREQUAL "LIBC") + execute_process(COMMAND "${PATCHER}" "${library}" "${WORK_DIR}/${name}" + RESULT_VARIABLE status OUTPUT_VARIABLE output ERROR_VARIABLE error) + else() + execute_process(COMMAND "${PATCHER}" "${library}" --preserve-export-list "${WORK_DIR}/libc.exports" "${WORK_DIR}/${name}" + RESULT_VARIABLE status OUTPUT_VARIABLE output ERROR_VARIABLE error) + endif() + if(NOT status STREQUAL "0") + message(FATAL_ERROR "${library} patch failed: ${output}${error}") + endif() +endforeach() +get_filename_component(probeName "${EXCEPTION_PROBE}" NAME) +file(COPY_FILE "${EXCEPTION_PROBE}" "${WORK_DIR}/${probeName}") +if(WIN32) + set(searchDirectories "${WORK_DIR}") + foreach(binary LIBC KERNEL INTERNAL EXCEPTION_PROBE TEST_EXECUTABLE) + get_filename_component(directory "${${binary}}" DIRECTORY) + list(APPEND searchDirectories "${directory}") + endforeach() + if(ALLOCATOR_DIRECTORY) + list(APPEND searchDirectories "${ALLOCATOR_DIRECTORY}") + endif() + list(REMOVE_DUPLICATES searchDirectories) + list(JOIN searchDirectories ";" searchPath) + set(environment "PATH=${searchPath};$ENV{PATH}") +else() + set(environment "LD_LIBRARY_PATH=${WORK_DIR}:$ENV{LD_LIBRARY_PATH}") +endif() +foreach(order Internal-first libc-first) + set(arguments "${WORK_DIR}" "${probeName}") + if(order STREQUAL "libc-first") + list(APPEND arguments libc-first) + endif() + execute_process(COMMAND "${CMAKE_COMMAND}" -E env "${environment}" + "${TEST_EXECUTABLE}" ${arguments} + RESULT_VARIABLE status OUTPUT_VARIABLE output ERROR_VARIABLE error) + if(NOT status STREQUAL "0") + message(FATAL_ERROR "sigvec ${order} lookup failed: ${output}${error}") + endif() + message("${output}") +endforeach() diff --git a/core/libs/tests/SigvecModulesExceptions.cpp b/core/libs/tests/SigvecModulesExceptions.cpp new file mode 100644 index 0000000000..79f1fb0abd --- /dev/null +++ b/core/libs/tests/SigvecModulesExceptions.cpp @@ -0,0 +1,27 @@ +#include "prx/libc/include/LegacySignals.hpp" +#include +#include +#include +#ifdef _WIN32 +#include +#endif + +extern "C" bool SigvecMissingBackend_nid_no_patch(decltype(&sigvec_nid_postfix) first, + decltype(&sigvec_nid_postfix) second, const void* core) { +#ifdef _WIN32 + const auto libc = GetModuleHandleW(L"libc.prx"); + if (libc == nullptr || reinterpret_cast(GetProcAddress(libc, "LegacySigvec_nid_no_patch")) != core) return false; +#else + if (reinterpret_cast(&LegacySigvec_nid_no_patch) != core) return false; +#endif + for (const auto function : std::array{first, second}) { + bool threw = false; + try { + function(30, nullptr, nullptr); + } catch (const std::runtime_error& error) { + threw = std::strcmp(error.what(), "sigvec: libkernel has not registered sigaction") == 0; + } + if (!threw) return false; + } + return true; +} diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 0b753e2c18..42a6a7acae 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -9,6 +9,7 @@ - [frexpf](../../../core/libs/prx/libc/src/Math.cpp) (libc) - follows the bundled PPSA03766 implementation: exponent-bit decomposition, conditional single-precision subnormal scaling, zero exponent for zero/NaN/infinity, and unchanged signaling NaNs. PPSA07923 agrees on results/exponents but unconditionally scales finite inputs, additionally raising overflow/inexact on large values; that SDK variant is not reproduced. Masked-exception observations cover rounding, DAZ/FTZ and sticky flags; unmasked traps and other SDK versions are unverified. - [fesetenv](../../../core/libs/prx/libc/src/Math.cpp) (libSceLibcInternal, implemented in libc, NID `Q-bLp+b-RVY`) uses the 32-byte amd64 environment and x87/MXCSR restore sequence in the [public SDK fenv.h](https://github.com/ps5-payload-dev/sdk/blob/a7685ce8e97ce1f8a4f38253a41d54241c9124f1/include/freebsd/fenv.h). The console layout, behavior and module/library versions are unmeasured. Null pointers and MXCSR bits unsupported by the current host's `FXSAVE` mask throw before changing state; a zero mask uses the architectural `0xffbf` fallback. Console fault behavior for invalid environments is unverified. The separate `__fe_dfl_env` object remains missing and its console initializer is unknown. +- [sigvec](../../../core/libs/prx/libc/src/LegacySignals.cpp) (libc and libSceLibcInternal) has not been measured on a physical PS5 or run on macOS. Its Windows module-lookup fixture pins modules until process exit; explicit DLL unloading remains unverified. The cause of the reported unload crash is unknown. A contributor executed the firmware 13.42 export `yhxKO9LYc8A` from libSceLibcInternal in an emulator with recording `_sigaction`/`sigemptyset` imports; layout, conversion, aliasing and forwarded results match this implementation. Its vector ABI, low-word mask conversion and restart/interrupt flag inversion follow FreeBSD 11.4 [`sigcompat.c`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/compat-43/sigcompat.c) and [`signal.h`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/sys/signal.h). Calls use registered guest `sigaction`; absent registration throws before mutation. It inherits the [guest signal layer's gaps](libkernel.md): action masks, restart/reset flags and `SA_SIGINFO` delivery are not applied. New vectors clear the upper mask words; queries return only the low word. The previous-vector output is guarded before the action changes; a rejected host-write scope returns `EFAULT` without output or disposition mutation. Other invalid caller memory is undefined, as upstream. - [strtonum](../../../core/libs/prx/libc/src/StringExtensions.cpp) (libc) is unverified on the console. Its signature, decimal parsing, inclusive bounds, error strings, error precedence and errno preservation follow [FreeBSD 11.4 strtonum.c](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/stdlib/strtonum.c). Parsing uses the host `strtoll` with base 10. It is not exported by `libSceLibcInternal.sprx` on a PS5 (firmware 13.60), so a payload process cannot call it there - [a64l, l64a and l64a_r](../../../core/libs/prx/libc/src/Radix64.cpp) (libc, NIDs `XR3orje3XCg`, `lF2uY5zV6tg`, `Zv-2rBFdZV0`) are ported from FreeBSD 11.4 [a64l.c](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/libc/stdlib/a64l.c) and [l64a.c](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/libc/stdlib/l64a.c): `a64l` maps every byte of the first six arithmetically, including bytes outside `./0-9A-Za-z`, and sign-extends the low 32 bits; `l64a_r` writes the digits that fit before returning -1 on a short buffer. Deviations from FreeBSD: `a64l(NULL)` returns 0, `l64a_r` returns -1 without writing for a null buffer or `buflen <= 0`, and `l64a` uses a per-thread buffer instead of one shared `static` buffer. The PS5 `libc` storage, parsing of invalid characters and error behaviour are unverified; the tests check the behaviour chosen here, not equivalence with the console. From 6c35e63deb6e25c24ad2883e8cf333e9686cfb0a Mon Sep 17 00:00:00 2001 From: AZLisme <8706522+AZLisme@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:17:30 +0800 Subject: [PATCH 292/318] feat(libSceIcu): implement UTF-16 set searches and tokenization --- core/libs/prx/libSceIcu/CMakeLists.txt | 15 ++- core/libs/prx/libSceIcu/Tokenize.cpp | 113 +++++++++++++++++++ core/libs/prx/libSceIcu/include/Api.hpp | 12 ++ core/libs/prx/libSceIcu/libSceIcu.hpp | 2 + core/libs/tests/GuestIcuTokenize.cpp | 140 ++++++++++++++++++++++++ docs/dev/debt/libSceIcu.md | 2 + 6 files changed, 283 insertions(+), 1 deletion(-) create mode 100644 core/libs/prx/libSceIcu/Tokenize.cpp create mode 100644 core/libs/prx/libSceIcu/include/Api.hpp create mode 100644 core/libs/tests/GuestIcuTokenize.cpp diff --git a/core/libs/prx/libSceIcu/CMakeLists.txt b/core/libs/prx/libSceIcu/CMakeLists.txt index 0bbdfb93f1..92fc14a55b 100644 --- a/core/libs/prx/libSceIcu/CMakeLists.txt +++ b/core/libs/prx/libSceIcu/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(libSceIcu SHARED EXCLUDE_FROM_ALL Export.cpp Invariant.cpp Version.cpp Unescape.cpp) +add_library(libSceIcu SHARED EXCLUDE_FROM_ALL Export.cpp Invariant.cpp Version.cpp Unescape.cpp Tokenize.cpp) target_include_directories(libSceIcu PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceIcu) target_sources(libSceIcu PRIVATE ConverterCallbacks.cpp) @@ -112,3 +112,16 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + add_test_executable(guest_icu_tokenize_tests ${LIBS_INCLUDE_DIR}/tests/GuestIcuTokenize.cpp) + target_include_directories(guest_icu_tokenize_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_icu_tokenize_tests PRIVATE libSceIcu libc) + configure_windows_unwind(guest_icu_tokenize_tests) + add_test(NAME guest_icu_tokenize COMMAND guest_icu_tokenize_tests) + set_tests_properties(guest_icu_tokenize PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_icu_tokenize PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceIcu/Tokenize.cpp b/core/libs/prx/libSceIcu/Tokenize.cpp new file mode 100644 index 0000000000..1c0e6e5626 --- /dev/null +++ b/core/libs/prx/libSceIcu/Tokenize.cpp @@ -0,0 +1,113 @@ +#include "prx/libSceIcu/libSceIcu.hpp" +#include "prx/libc/include/General.hpp" +#include +#include +#include + +namespace { + +struct Character { + std::uint32_t value; + std::size_t width; +}; + +bool Lead(char16_t value) { return value >= 0xd800 && value <= 0xdbff; } +bool Trail(char16_t value) { return value >= 0xdc00 && value <= 0xdfff; } + +Character Next(const char16_t* text) { + if (Lead(text[0]) && Trail(text[1])) + return {0x10000u + ((text[0] - 0xd800u) << 10) + (text[1] - 0xdc00u), 2}; + return {text[0], 1}; +} + +bool Contains(const char16_t* set, std::size_t length, std::uint32_t value) { + for (std::size_t position = 0; position < length;) { + const auto character = Next(set + position); + if (character.value == value) return true; + position += character.width; + } + return false; +} + +std::size_t FindSet(const char16_t* text, const char16_t* set, bool matches) { + constexpr auto maximum = static_cast(std::numeric_limits::max()); + std::size_t length = 0; + while (set[length] != 0) if (++length > maximum) return std::numeric_limits::max(); + std::size_t position = 0; + while (text[position] != 0) { + const auto character = Next(text + position); + if (Contains(set, length, character.value) == matches) break; + position += character.width; + if (position > maximum) return std::numeric_limits::max(); + } + return position; +} + +} // namespace + +extern "C" { + +std::int32_t APS5_VABI u_strspn_nid_postfix(const char16_t* text, const char16_t* set) { + if (!text || !set) NotImplemented_nid_no_patch(__func__); + const auto position = FindSet(text, set, false); + if (position > static_cast(std::numeric_limits::max())) NotImplemented_nid_no_patch(__func__); + return static_cast(position); +} + +std::int32_t APS5_VABI u_strcspn_nid_postfix(const char16_t* text, const char16_t* set) { + if (!text || !set) NotImplemented_nid_no_patch(__func__); + const auto position = FindSet(text, set, true); + if (position > static_cast(std::numeric_limits::max())) NotImplemented_nid_no_patch(__func__); + return static_cast(position); +} + +char16_t* APS5_VABI u_strpbrk_nid_postfix(const char16_t* text, const char16_t* set) { + if (!text || !set) NotImplemented_nid_no_patch(__func__); + const auto position = FindSet(text, set, true); + if (position > static_cast(std::numeric_limits::max())) NotImplemented_nid_no_patch(__func__); + return text[position] != 0 ? const_cast(text + position) : nullptr; +} + +char16_t* APS5_VABI u_strtok_r_nid_postfix(char16_t* source, const char16_t* delimiters, char16_t** state) { + if (!state) NotImplemented_nid_no_patch(__func__); + auto* text = source ? source : *state; + if (!text) return nullptr; + if (!delimiters) NotImplemented_nid_no_patch(__func__); + if (source) *state = source; + const auto skipped = FindSet(text, delimiters, false); + if (skipped > static_cast(std::numeric_limits::max())) NotImplemented_nid_no_patch(__func__); + text += skipped; + if (*text == 0) { + *state = nullptr; + return nullptr; + } + const auto length = FindSet(text, delimiters, true); + if (length > static_cast(std::numeric_limits::max())) NotImplemented_nid_no_patch(__func__); + auto* next = text + length; + if (*next != 0) { + *next++ = 0; + *state = next; + } else *state = nullptr; + return text; +} + +char16_t* APS5_VABI u_memrchr32_nid_postfix(const char16_t* text, std::int32_t value, std::int32_t count) { + if (count <= 0 || static_cast(value) > 0x10ffffu || (value > 0xffff && count < 2)) return nullptr; + if (!text) { + if (value >= 0xd800 && value <= 0xdfff) return nullptr; + NotImplemented_nid_no_patch(__func__); + } + auto position = static_cast(count); + while (position != 0) { + --position; + std::uint32_t character = text[position]; + if (Trail(text[position]) && position != 0 && Lead(text[position - 1])) { + --position; + character = 0x10000u + ((text[position] - 0xd800u) << 10) + (text[position + 1] - 0xdc00u); + } + if (character == static_cast(value)) return const_cast(text + position); + } + return nullptr; +} + +} diff --git a/core/libs/prx/libSceIcu/include/Api.hpp b/core/libs/prx/libSceIcu/include/Api.hpp new file mode 100644 index 0000000000..04d4567174 --- /dev/null +++ b/core/libs/prx/libSceIcu/include/Api.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "prx/libc/include/general/VabiMacros.hpp" +#include + +extern "C" { +std::int32_t APS5_VABI u_strspn_nid_postfix(const char16_t* text, const char16_t* set); +std::int32_t APS5_VABI u_strcspn_nid_postfix(const char16_t* text, const char16_t* set); +char16_t* APS5_VABI u_strpbrk_nid_postfix(const char16_t* text, const char16_t* set); +char16_t* APS5_VABI u_strtok_r_nid_postfix(char16_t* source, const char16_t* delimiters, char16_t** state); +char16_t* APS5_VABI u_memrchr32_nid_postfix(const char16_t* text, std::int32_t value, std::int32_t count); +} diff --git a/core/libs/prx/libSceIcu/libSceIcu.hpp b/core/libs/prx/libSceIcu/libSceIcu.hpp index 300ae3ebb7..785600df04 100644 --- a/core/libs/prx/libSceIcu/libSceIcu.hpp +++ b/core/libs/prx/libSceIcu/libSceIcu.hpp @@ -10,3 +10,5 @@ #include "Space.hpp" #include "Unescape.hpp" #include "Identified.hpp" + +#include "prx/libSceIcu/include/Api.hpp" diff --git a/core/libs/tests/GuestIcuTokenize.cpp b/core/libs/tests/GuestIcuTokenize.cpp new file mode 100644 index 0000000000..5582d0f16a --- /dev/null +++ b/core/libs/tests/GuestIcuTokenize.cpp @@ -0,0 +1,140 @@ +#include "prx/libSceIcu/libSceIcu.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + + + +namespace { + +void Require(bool condition, std::source_location location = std::source_location::current()) { + if (condition) return; + std::fprintf(stderr, "ICU tokenization check failed at line %u\n", location.line()); + std::abort(); +} + +template +void Throws(TCall call) { + try { call(); } + catch (const std::exception&) { return; } + Require(false); +} + +void SetSearches() { + const std::array text{'a', 0xd83d, 0xde00, 'b', 0xd800, 'c', 0xdc00, 'd', 0}; + const auto before = text; + const std::array set{0xd83d, 0xde00, 0xd800, 0}; + const std::array lead{0xd83d, 0}; + const std::array trail{0xdc00, 0}; + Require(u_strspn_nid_postfix(text.data(), set.data()) == 0); + Require(u_strspn_nid_postfix(text.data() + 1, set.data()) == 2); + Require(u_strcspn_nid_postfix(text.data(), set.data()) == 1); + Require(u_strpbrk_nid_postfix(text.data(), set.data()) == text.data() + 1); + Require(u_strcspn_nid_postfix(text.data(), lead.data()) == 8); + Require(u_strpbrk_nid_postfix(text.data(), lead.data()) == nullptr); + Require(u_strcspn_nid_postfix(text.data(), trail.data()) == 6); + Require(u_strpbrk_nid_postfix(text.data(), trail.data()) == text.data() + 6); + Require(u_strspn_nid_postfix(text.data(), u"abcd") == 1); + Require(u_strspn_nid_postfix(u"", set.data()) == 0); + Require(u_strcspn_nid_postfix(u"", set.data()) == 0); + Require(u_strspn_nid_postfix(text.data(), u"") == 0); + Require(u_strcspn_nid_postfix(text.data(), u"") == 8); + Require(u_strpbrk_nid_postfix(text.data(), u"") == nullptr); + Require(text == before); + Throws([&] { u_strspn_nid_postfix(nullptr, set.data()); }); + Throws([&] { u_strcspn_nid_postfix(text.data(), nullptr); }); + Throws([&] { u_strpbrk_nid_postfix(nullptr, nullptr); }); +} + +void ReverseSearch() { + const std::array text{'a', 0, 0xd800, 0xdc00, 'b', 0xd800, 0, 0xdc00, 'a'}; + const auto before = text; + Require(u_memrchr32_nid_postfix(text.data(), 'a', 9) == text.data() + 8); + Require(u_memrchr32_nid_postfix(text.data(), 0, 9) == text.data() + 6); + Require(u_memrchr32_nid_postfix(text.data(), 0x10000, 9) == text.data() + 2); + Require(u_memrchr32_nid_postfix(text.data(), 0xd800, 9) == text.data() + 5); + Require(u_memrchr32_nid_postfix(text.data(), 0xdc00, 9) == text.data() + 7); + Require(u_memrchr32_nid_postfix(text.data(), 0xd800, 3) == text.data() + 2); + Require(u_memrchr32_nid_postfix(text.data(), 0x10000, 3) == nullptr); + Require(u_memrchr32_nid_postfix(text.data() + 3, 0xdc00, 1) == text.data() + 3); + Require(u_memrchr32_nid_postfix(nullptr, 'a', 0) == nullptr); + Require(u_memrchr32_nid_postfix(nullptr, 'a', -1) == nullptr); + Require(u_memrchr32_nid_postfix(nullptr, 0x10000, 1) == nullptr); + Require(u_memrchr32_nid_postfix(nullptr, 0xd800, 7) == nullptr); + Require(u_memrchr32_nid_postfix(nullptr, 0xdfff, 7) == nullptr); + Require(u_memrchr32_nid_postfix(nullptr, -1, 7) == nullptr); + Require(u_memrchr32_nid_postfix(nullptr, 0x110000, 7) == nullptr); + Require(u_memrchr32_nid_postfix(nullptr, std::numeric_limits::max(), 7) == nullptr); + Throws([&] { u_memrchr32_nid_postfix(nullptr, 'a', 1); }); + Require(text == before); +} + +void Tokens() { + std::array text{0xa55a, ',', 'a', ':', 'b', ',', 'c', 0, 0xa55a, 0xa55a}; + char16_t* state = nullptr; + Require(u_strtok_r_nid_postfix(text.data() + 1, u",:", &state) == text.data() + 2); + Require(state == text.data() + 4 && text[3] == 0 && text[1] == ','); + Require(u_strtok_r_nid_postfix(nullptr, u",", &state) == text.data() + 4); + Require(state == text.data() + 6 && text[5] == 0); + Require(u_strtok_r_nid_postfix(nullptr, u"", &state) == text.data() + 6); + Require(state == nullptr); + Require(u_strtok_r_nid_postfix(nullptr, nullptr, &state) == nullptr); + Require(text.front() == 0xa55a && text[8] == 0xa55a && text[9] == 0xa55a); + std::array delimitersOnly{',', ',', ',', 0}; + Require(u_strtok_r_nid_postfix(delimitersOnly.data(), u",", &state) == nullptr && state == nullptr); + const std::array unchanged{',', ',', ',', 0}; + Require(delimitersOnly == unchanged); + Throws([&] { u_strtok_r_nid_postfix(text.data(), u",", nullptr); }); + const auto before = text; + Throws([&] { u_strtok_r_nid_postfix(text.data(), nullptr, &state); }); + Require(text == before && state == nullptr); +} + +void SupplementaryTokens() { + std::array text{0xa55a, 'a', 0xd83d, 0xde00, 'b', 0, 0xa55a}; + const char16_t delimiter[]{0xd83d, 0xde00, 0}; + char16_t* state = nullptr; + Require(u_strtok_r_nid_postfix(text.data() + 1, delimiter, &state) == text.data() + 1); + Require(text[2] == 0 && text[3] == 0xde00 && state == text.data() + 3); + Require(u_strtok_r_nid_postfix(nullptr, delimiter, &state) == text.data() + 3); + Require(state == nullptr && text.front() == 0xa55a && text.back() == 0xa55a); + std::array leading{0xd83d, 0xde00, 'a', 0}; + Require(u_strtok_r_nid_postfix(leading.data(), delimiter, &state) == leading.data() + 2 && state == nullptr); + Require(leading[0] == 0xd83d && leading[1] == 0xde00); + std::array pair{0xd83d, 0xde00, 'b', 0}; + const char16_t unpaired[]{0xd83d, 0}; + Require(u_strtok_r_nid_postfix(pair.data(), unpaired, &state) == pair.data() && state == nullptr); + Require(pair[0] == 0xd83d && pair[1] == 0xde00); +} + +void IndependentState() { + const auto tokenize = [](char16_t separator) { + for (unsigned iteration = 0; iteration < 1000; ++iteration) { + std::array text{'a', separator, 'b', 0}; + const char16_t delimiters[]{separator, 0}; + char16_t* state = nullptr; + Require(u_strtok_r_nid_postfix(text.data(), delimiters, &state) == text.data()); + Require(u_strtok_r_nid_postfix(nullptr, delimiters, &state) == text.data() + 2); + Require(state == nullptr && text[1] == 0 && text[3] == 0); + } + }; + std::thread first(tokenize, ','); + std::thread second(tokenize, ':'); + first.join(); + second.join(); +} + +} // namespace + +int main() { + SetSearches(); + ReverseSearch(); + Tokens(); + SupplementaryTokens(); + IndependentState(); +} diff --git a/docs/dev/debt/libSceIcu.md b/docs/dev/debt/libSceIcu.md index 8b1b49669f..4593e4bfab 100644 --- a/docs/dev/debt/libSceIcu.md +++ b/docs/dev/debt/libSceIcu.md @@ -35,6 +35,8 @@ - [Default-converter string copy helpers](../../../core/libs/prx/libSceIcu/Invariant.cpp) `u_austrcpy` and `u_austrncpy` are unverified on a console and throw `NotImplemented_nid_no_patch` on code units above `0x7f`. Their names and NIDs come from the PS5 3.20 inventory; signatures and ASCII (`0x01`–`0x7f`) behavior follow ICU 78.1's [ustr_cnv.cpp](https://github.com/unicode-org/icu/blob/release-78.1/icu4c/source/common/ustr_cnv.cpp), unchanged since 65.1, without assuming the console's default converter for non-ASCII units. `u_austrncpy` writes a single NUL when the source ends before the count without zero-padding the remainder. Null buffers with `u_austrcpy` or a positive count throw an invalid-argument exception, whereas ICU dereferences in `u_austrcpy` and returns the destination unchanged in `u_austrncpy`. Closes when: compared with the console, including the default converter for non-ASCII units. +- [UTF-16 set searches and tokenization](../../../core/libs/prx/libSceIcu/Tokenize.cpp) have no console verification; ICU versions on other firmware are unverified. Signatures and behavior follow ICU 78.1's [ustring.h](https://github.com/unicode-org/icu/blob/049e0d6a420629ac7db77256987d083a563287b5/icu4c/source/common/unicode/ustring.h) and [ustring.cpp](https://github.com/unicode-org/icu/blob/049e0d6a420629ac7db77256987d083a563287b5/icu4c/source/common/ustring.cpp). `u_strtok_r` preserves that source's one-code-unit delimiter overwrite, including the trailing surrogate left after splitting at a supplementary delimiter. Invalid null inputs throw where the reference would dereference them. Set-search offsets beyond `INT32_MAX` throw. + ### Known export names; ABI prototypes unverified - The 93 `sceIcuUnknown_*_nid_no_patch` and named `*_nid_postfix` stubs in [Export.cpp](../../../core/libs/prx/libSceIcu/Export.cpp) are present in the PS5 3.20 export inventory from [PS5-3.20_Libs](https://github.com/DNNDHH/PS5-3.20_Libs). Their prototypes and behavior are unverified. Each uses the no-parameter import-resolution convention and throws through `NotImplemented_nid_no_patch`; these declarations make no ABI or behavior claim. From 7e8736bc277adb3476b1c325402a49749779a6db Mon Sep 17 00:00:00 2001 From: AZLisme <8706522+AZLisme@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:11:57 +0800 Subject: [PATCH 293/318] fix(libSceIcu): guard tokenizer guest writes --- core/libs/prx/libSceIcu/CMakeLists.txt | 5 +- core/libs/prx/libSceIcu/Tokenize.cpp | 5 + core/libs/tests/GuestIcuTokenize.cpp | 130 ++++++++++++++++++++++++- docs/dev/debt/libSceIcu.md | 2 +- 4 files changed, 137 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceIcu/CMakeLists.txt b/core/libs/prx/libSceIcu/CMakeLists.txt index 92fc14a55b..8564e9dbfb 100644 --- a/core/libs/prx/libSceIcu/CMakeLists.txt +++ b/core/libs/prx/libSceIcu/CMakeLists.txt @@ -116,12 +116,13 @@ endif() if(BUILD_TESTING) add_test_executable(guest_icu_tokenize_tests ${LIBS_INCLUDE_DIR}/tests/GuestIcuTokenize.cpp) target_include_directories(guest_icu_tokenize_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_icu_tokenize_tests PRIVATE libSceIcu libc) + target_link_libraries(guest_icu_tokenize_tests PRIVATE libSceIcu libkernel libc) configure_windows_unwind(guest_icu_tokenize_tests) add_test(NAME guest_icu_tokenize COMMAND guest_icu_tokenize_tests) set_tests_properties(guest_icu_tokenize PROPERTIES TIMEOUT 30) if(WIN32) set_tests_properties(guest_icu_tokenize PROPERTIES ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:$") + "PATH=path_list_prepend:$" + "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceIcu/Tokenize.cpp b/core/libs/prx/libSceIcu/Tokenize.cpp index 1c0e6e5626..38e4b9dc2c 100644 --- a/core/libs/prx/libSceIcu/Tokenize.cpp +++ b/core/libs/prx/libSceIcu/Tokenize.cpp @@ -1,5 +1,6 @@ #include "prx/libSceIcu/libSceIcu.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/GuestArena.hpp" #include #include #include @@ -73,6 +74,8 @@ char16_t* APS5_VABI u_strtok_r_nid_postfix(char16_t* source, const char16_t* del auto* text = source ? source : *state; if (!text) return nullptr; if (!delimiters) NotImplemented_nid_no_patch(__func__); + GuestArena::HostWrite stateAccess(state, sizeof(*state)); + if (!stateAccess.Open()) NotImplemented_nid_no_patch(__func__); if (source) *state = source; const auto skipped = FindSet(text, delimiters, false); if (skipped > static_cast(std::numeric_limits::max())) NotImplemented_nid_no_patch(__func__); @@ -85,6 +88,8 @@ char16_t* APS5_VABI u_strtok_r_nid_postfix(char16_t* source, const char16_t* del if (length > static_cast(std::numeric_limits::max())) NotImplemented_nid_no_patch(__func__); auto* next = text + length; if (*next != 0) { + GuestArena::HostWrite textAccess(next, sizeof(*next)); + if (!textAccess.Open()) NotImplemented_nid_no_patch(__func__); *next++ = 0; *state = next; } else *state = nullptr; diff --git a/core/libs/tests/GuestIcuTokenize.cpp b/core/libs/tests/GuestIcuTokenize.cpp index 5582d0f16a..b0aab05bd2 100644 --- a/core/libs/tests/GuestIcuTokenize.cpp +++ b/core/libs/tests/GuestIcuTokenize.cpp @@ -1,4 +1,7 @@ #include "prx/libSceIcu/libSceIcu.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libkernel/libkernel.hpp" +#include #include #include #include @@ -8,8 +11,6 @@ #include #include - - namespace { void Require(bool condition, std::source_location location = std::source_location::current()) { @@ -112,6 +113,128 @@ void SupplementaryTokens() { Require(pair[0] == 0xd83d && pair[1] == 0xde00); } +void AliasedDelimiters() { + std::array text{'a', ',', 'b', ',', 'c', 0, 0xa55a}; + auto* delimiters = text.data() + 3; + char16_t* state = nullptr; + Require(u_strtok_r_nid_postfix(text.data(), delimiters, &state) == text.data()); + Require(state == text.data() + 2 && text[1] == 0 && delimiters[0] == ','); + Require(u_strtok_r_nid_postfix(nullptr, delimiters, &state) == text.data() + 2); + Require(state == text.data() + 4 && delimiters[0] == 0); + Require(u_strtok_r_nid_postfix(nullptr, delimiters, &state) == text.data() + 4); + Require(state == nullptr && text[4] == 'c' && text.back() == 0xa55a); +} + +class GuestStorage { +public: + static constexpr std::size_t Page = 0x4000; + + GuestStorage() { + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, Page * 3, 0, 0, &physical) == 0); + Require(sceKernelMapDirectMemory(&mapping, Page * 3, 3, 0, physical, 0) == 0); + Require(GuestArena::GuestArenaContains_nid_postfix(mapping, Page * 3)); + std::fill_n(static_cast(mapping), Page * 3, 0xa5); + } + + ~GuestStorage() { + Require(sceKernelMunmap(reinterpret_cast(mapping), Page * 3) == 0); + Require(sceKernelReleaseDirectMemory(physical, Page * 3) == 0); + } + + char16_t* Text() const { return reinterpret_cast(static_cast(mapping) + 16); } + char16_t** State() const { return reinterpret_cast(static_cast(mapping) + Page + 16); } + char16_t* Delimiters() const { return reinterpret_cast(static_cast(mapping) + Page * 2 + 16); } + + void Protect(std::size_t page, int protection) { + Require(sceKernelMprotect(static_cast(mapping) + Page * page, Page, protection) == 0); + } + + void CheckCanaries() const { + const auto* bytes = static_cast(mapping); + for (const auto offset : {std::size_t{15}, Page + 15, Page + 16 + sizeof(char16_t*), Page * 2 + 15, Page * 3 - 1}) + Require(bytes[offset] == 0xa5); + } + +private: + std::int64_t physical = 0; + void* mapping = nullptr; +}; + +void GuestMappedTokens() { + GuestStorage storage; + auto* text = storage.Text(); + auto** state = storage.State(); + auto* delimiters = storage.Delimiters(); + const std::array initial{0xa55a, ',', 'a', ':', 'b', ',', 'c', 0, 0xa55a}; + std::copy(initial.begin(), initial.end(), text); + const std::array separators{',', ':', 0}; + std::copy(separators.begin(), separators.end(), delimiters); + *state = nullptr; + storage.Protect(2, 1); + Require(u_strtok_r_nid_postfix(text + 1, delimiters, state) == text + 2); + Require(*state == text + 4 && text[3] == 0 && text[1] == ','); + Require(u_strtok_r_nid_postfix(nullptr, delimiters, state) == text + 4); + Require(*state == text + 6 && text[5] == 0); + Require(u_strtok_r_nid_postfix(nullptr, delimiters, state) == text + 6 && *state == nullptr); + Require(u_strtok_r_nid_postfix(nullptr, nullptr, state) == nullptr); + Require(text[0] == 0xa55a && text[8] == 0xa55a); + Require(std::equal(separators.begin(), separators.end(), delimiters)); + storage.CheckCanaries(); + storage.Protect(2, 3); + const std::array supplementary{0xa55a, 'a', 0xd83d, 0xde00, 'b', 0, 0xa55a}; + std::copy(supplementary.begin(), supplementary.end(), text); + const std::array pair{0xd83d, 0xde00, 0}; + std::copy(pair.begin(), pair.end(), delimiters); + storage.Protect(2, 1); + Require(u_strtok_r_nid_postfix(text + 1, delimiters, state) == text + 1); + Require(text[2] == 0 && text[3] == 0xde00 && *state == text + 3); + Require(u_strtok_r_nid_postfix(nullptr, delimiters, state) == text + 3 && *state == nullptr); + Require(text[0] == 0xa55a && text[6] == 0xa55a); + Require(std::equal(pair.begin(), pair.end(), delimiters)); + storage.CheckCanaries(); +} + +void ReadOnlyGuestTokens() { +#ifdef _WIN32 + GuestStorage storage; + auto* text = storage.Text(); + auto** state = storage.State(); + const std::array initial{0xa55a, 'a', ',', 'b', 0}; + std::copy(initial.begin(), initial.end(), text); + *state = nullptr; + storage.Protect(1, 1); + { + GuestArena::HostWrite rejected(state, sizeof(*state)); + Require(!rejected.Open()); + } + Require(u_strtok_r_nid_postfix(nullptr, nullptr, state) == nullptr); + Throws([&] { u_strtok_r_nid_postfix(text + 1, u",", state); }); + Require(*state == nullptr && std::equal(initial.begin(), initial.end(), text)); + storage.Protect(1, 3); + *state = text + 1; + storage.Protect(1, 1); + Throws([&] { u_strtok_r_nid_postfix(nullptr, u",", state); }); + Require(*state == text + 1 && std::equal(initial.begin(), initial.end(), text)); + storage.Protect(1, 3); + storage.Protect(0, 1); + { + GuestArena::HostWrite rejected(text + 2, sizeof(*text)); + Require(!rejected.Open()); + } + Throws([&] { u_strtok_r_nid_postfix(nullptr, u",", state); }); + Require(*state == text + 1 && std::equal(initial.begin(), initial.end(), text)); + *state = nullptr; + Throws([&] { u_strtok_r_nid_postfix(text + 1, u",", state); }); + Require(*state == text + 1 && std::equal(initial.begin(), initial.end(), text)); + Require(u_strtok_r_nid_postfix(nullptr, u"", state) == text + 1 && *state == nullptr); + Require(std::equal(initial.begin(), initial.end(), text)); + Require(u_strtok_r_nid_postfix(text + 2, u",b", state) == nullptr && *state == nullptr); + Require(u_strtok_r_nid_postfix(text + 4, u",", state) == nullptr && *state == nullptr); + Require(std::equal(initial.begin(), initial.end(), text)); + storage.CheckCanaries(); +#endif +} + void IndependentState() { const auto tokenize = [](char16_t separator) { for (unsigned iteration = 0; iteration < 1000; ++iteration) { @@ -136,5 +259,8 @@ int main() { ReverseSearch(); Tokens(); SupplementaryTokens(); + AliasedDelimiters(); + GuestMappedTokens(); + ReadOnlyGuestTokens(); IndependentState(); } diff --git a/docs/dev/debt/libSceIcu.md b/docs/dev/debt/libSceIcu.md index 4593e4bfab..5931e66156 100644 --- a/docs/dev/debt/libSceIcu.md +++ b/docs/dev/debt/libSceIcu.md @@ -35,7 +35,7 @@ - [Default-converter string copy helpers](../../../core/libs/prx/libSceIcu/Invariant.cpp) `u_austrcpy` and `u_austrncpy` are unverified on a console and throw `NotImplemented_nid_no_patch` on code units above `0x7f`. Their names and NIDs come from the PS5 3.20 inventory; signatures and ASCII (`0x01`–`0x7f`) behavior follow ICU 78.1's [ustr_cnv.cpp](https://github.com/unicode-org/icu/blob/release-78.1/icu4c/source/common/ustr_cnv.cpp), unchanged since 65.1, without assuming the console's default converter for non-ASCII units. `u_austrncpy` writes a single NUL when the source ends before the count without zero-padding the remainder. Null buffers with `u_austrcpy` or a positive count throw an invalid-argument exception, whereas ICU dereferences in `u_austrcpy` and returns the destination unchanged in `u_austrncpy`. Closes when: compared with the console, including the default converter for non-ASCII units. -- [UTF-16 set searches and tokenization](../../../core/libs/prx/libSceIcu/Tokenize.cpp) have no console verification; ICU versions on other firmware are unverified. Signatures and behavior follow ICU 78.1's [ustring.h](https://github.com/unicode-org/icu/blob/049e0d6a420629ac7db77256987d083a563287b5/icu4c/source/common/unicode/ustring.h) and [ustring.cpp](https://github.com/unicode-org/icu/blob/049e0d6a420629ac7db77256987d083a563287b5/icu4c/source/common/ustring.cpp). `u_strtok_r` preserves that source's one-code-unit delimiter overwrite, including the trailing surrogate left after splitting at a supplementary delimiter. Invalid null inputs throw where the reference would dereference them. Set-search offsets beyond `INT32_MAX` throw. +- [UTF-16 set searches and tokenization](../../../core/libs/prx/libSceIcu/Tokenize.cpp) have no console verification; ICU versions on other firmware are unverified. Signatures and behavior follow ICU 78.1's [ustring.h](https://github.com/unicode-org/icu/blob/049e0d6a420629ac7db77256987d083a563287b5/icu4c/source/common/unicode/ustring.h) and [ustring.cpp](https://github.com/unicode-org/icu/blob/049e0d6a420629ac7db77256987d083a563287b5/icu4c/source/common/ustring.cpp). `u_strtok_r` preserves that source's one-code-unit delimiter overwrite, including the trailing surrogate left after splitting at a supplementary delimiter. Invalid null inputs and rejected guest-write scopes throw; an initial continuation assignment is retained if a later delimiter write is rejected. Set-search offsets beyond `INT32_MAX` throw. ### Known export names; ABI prototypes unverified From f5a1913ed0241d33ab21c12e8e98d7e583b8ee82 Mon Sep 17 00:00:00 2001 From: AZLisme <8706522+AZLisme@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:34:08 +0800 Subject: [PATCH 294/318] test(icu): preserve Windows tokenizer PATH modifications --- core/libs/prx/libSceIcu/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/libs/prx/libSceIcu/CMakeLists.txt b/core/libs/prx/libSceIcu/CMakeLists.txt index 8564e9dbfb..3b1804f26d 100644 --- a/core/libs/prx/libSceIcu/CMakeLists.txt +++ b/core/libs/prx/libSceIcu/CMakeLists.txt @@ -122,7 +122,6 @@ if(BUILD_TESTING) set_tests_properties(guest_icu_tokenize PROPERTIES TIMEOUT 30) if(WIN32) set_tests_properties(guest_icu_tokenize PROPERTIES ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:$" - "PATH=path_list_prepend:$") + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() From fd4dbc9b9aa3236b42f595b5297f775dd0209553 Mon Sep 17 00:00:00 2001 From: TechLevelZero <35297505+TechLevelZero@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:01:30 +0000 Subject: [PATCH 295/318] fix(shader): set float32FlushDeclared again next to the Float64 setter The main merge in ef345e24b replaced main's SetFloat32FlushDeclared setter and its call with the new SetFloat64 instead of keeping both, so the context's float32FlushDeclared stayed false and v_floor_f32/v_ceil_f32 lost main's hardware-flush selection. Keep both setters and both calls. Co-Authored-By: Claude Opus 5.5 --- .../Translation/include/Translation/TranslationContext.hpp | 1 + core/shader/recompiler/Translation/src/InstructionTranslator.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 278621c7b3..e92dda56e0 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -27,6 +27,7 @@ public: void SetNativeFloat16(bool enabled) { nativeFloat16 = enabled; } void SetFloat32DenormPreserve(bool enabled) { float32DenormPreserve = enabled; } void SetFloat32FlushToZero(bool enabled) { float32FlushToZero = enabled; } + void SetFloat32FlushDeclared(bool enabled) { float32FlushDeclared = enabled; } void SetFloat64(bool enabled) { float64 = enabled; } void AddBranchCondition(const BasicBlock& source, BlockInfo& info); void TranslateCodeTableLoad(const RdnaInstruction& instruction, const ControlFlowGraph::CodeTableLoad& table); diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index f1f1f23ae2..9d507754c6 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -622,6 +622,7 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con context.SetNativeFloat16(options.nativeFloat16); context.SetFloat32DenormPreserve(options.float32DenormPreserve); context.SetFloat32FlushToZero(options.float32FlushToZero); + context.SetFloat32FlushDeclared(options.float32FlushDeclared); context.SetFloat64(options.float64); context.SetEntryRounding(entryRounding.at(cfgBlock.id)); for (std::uint32_t index = cfgBlock.instructionBegin; index < cfgBlock.instructionEnd; index++) { From 64f547698246a157bb7ea20b56b149d7b1ad51a6 Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Sun, 1 Nov 2026 15:31:54 +0530 Subject: [PATCH 296/318] fix(libSceAgc): deduplicate sceAgcUpdateInterpolantMapping_0100 and restore boundary coverage The merge with main re-added the Misc/ShaderFusion.hpp declaration that duplicates Shader/include/InterpolantMapping.hpp, dropped testUpdate0100Boundary, and restored the stale "behaviour unknown" debt entry. Keep the canonical Shader declaration, restore the 32-input no-output and reversed-matching cases (identity defaults 0x000/0x100/0x200/0x300 for the corrected encoding), and point the debt entry at the InterpolantMapping.cpp implementation again. --- .../libSceAgc/Misc/include/ShaderFusion.hpp | 1 - .../libSceAgc/tests/InterpolantMapping.cpp | 37 +++++++++++++++++++ docs/dev/debt/libSceAgc.md | 2 +- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgc/Misc/include/ShaderFusion.hpp b/core/libs/prx/libSceAgc/Misc/include/ShaderFusion.hpp index d90e4d427e..3d7ee1dc21 100644 --- a/core/libs/prx/libSceAgc/Misc/include/ShaderFusion.hpp +++ b/core/libs/prx/libSceAgc/Misc/include/ShaderFusion.hpp @@ -18,7 +18,6 @@ extern "C" int APS5_VABI sceAgcGetFusedShaderSize_0080(SizeAlign* dst, const Sha extern "C" { int APS5_VABI sceAgcFuseShaderHalves_nid_postfix(Shader* fusedResult, const Shader* front, const Shader* back, void* scratchMem); -int APS5_VABI sceAgcUpdateInterpolantMapping_0100(ShaderRegister* regs, const Shader* vs, const Shader* ps); } diff --git a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp index 3c4107c546..697d292472 100644 --- a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp @@ -201,6 +201,42 @@ void testUpdate0100Mapping() { check(mappings == mappingsBefore && links == linksBefore, "_0100 update resolved an ABI"); } +void testUpdate0100Boundary() { + const auto mappingsBefore = mappings; + const auto linksBefore = links; + std::array inputs{}; + std::array outputs{}; + for (std::uint32_t i = 0; i < inputs.size(); ++i) { + inputs[i] = semantic(i + 1u, 0, 0); + inputs[i].default_value = i % 4u; + outputs[i] = semantic(inputs.size() - i, i, 0); + } + Shader ps{}; + ps.input_semantics = inputs.data(); + ps.num_input_semantics = static_cast(inputs.size()); + Shader vs{}; + std::array regs{}; + std::memset(regs.data(), 0xcc, sizeof(regs)); + const auto untouched = regs; + check(sceAgcUpdateInterpolantMapping_0100(regs.data(), &vs, &ps) == 0, "_0100 update rejected a producer without outputs"); + constexpr std::array defaults{0x000u, 0x100u, 0x200u, 0x300u}; + for (std::uint32_t i = 0; i < inputs.size(); ++i) { + check(regs[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i, "_0100 full default mapping register offset changed"); + check(regs[i].value == (0x20u | defaults[i % 4u]), "_0100 full default mapping changed"); + } + check(std::memcmp(®s.back(), &untouched.back(), sizeof(ShaderRegister)) == 0, "_0100 full default mapping wrote past 32 inputs"); + vs.output_semantics = outputs.data(); + vs.num_output_semantics = static_cast(outputs.size()); + regs = untouched; + check(sceAgcUpdateInterpolantMapping_0100(regs.data(), &vs, &ps) == 0, "_0100 update rejected 32 matched inputs"); + for (std::uint32_t i = 0; i < inputs.size(); ++i) { + check(regs[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i, "_0100 full matched mapping register offset changed"); + check(regs[i].value == ((31u - i) | defaults[i % 4u]), "_0100 full matched mapping changed"); + } + check(std::memcmp(®s.back(), &untouched.back(), sizeof(ShaderRegister)) == 0, "_0100 full matched mapping wrote past 32 inputs"); + check(mappings == mappingsBefore && links == linksBefore, "_0100 full mapping resolved an ABI"); +} + void testUnversionedMapping() { std::array inputs{ semantic(1, 0, 2), semantic(2, 0, 2), semantic(5, 0, 2), semantic(3, 0, 0), semantic(4, 0, 3)}; @@ -288,6 +324,7 @@ int main() { testMapping(); testUpdateMapping(); testUpdate0100Mapping(); + testUpdate0100Boundary(); testUnversionedMapping(); testRejections(); check(mappings == 10u && links == 8u, "mapping preparation call count changed"); diff --git a/docs/dev/debt/libSceAgc.md b/docs/dev/debt/libSceAgc.md index dd8572eb71..fdbe58f427 100644 --- a/docs/dev/debt/libSceAgc.md +++ b/docs/dev/debt/libSceAgc.md @@ -19,7 +19,7 @@ - [sceAgcBranchPatchSetThenTarget_0300, sceAgcBranchPatchSetElseTarget_0300](../../../core/libs/prx/libSceAgc/Patch/src/Branch.cpp) (libSceAgc) - names from their NIDs (`7Wa3aeJgeVU`, `rP5xLdOf26k`); the `(packet, cachePolicy, target, sizeInDwords)` signature is inferred from PPSA28416's calls (a branch packet, 0, an aligned target, 4), with the policy stored where `sceAgcCbBranch` puts it - [sceAgcDcbSet{Cx,Sh,Uc}RegistersIndirectGetSize](../../../core/libs/prx/libSceAgc/DcbState/src/Registers.cpp) (libSceAgc) - the count argument is ignored: PPSA28416 calls them with leftover register contents (0xf5f5039c), so the real functions likely take none; the size is 20 bytes either way - [sceAgcCbSetShRegistersDirect, sceAgcCbSetUcRegistersDirect](../../../core/libs/prx/libSceAgc/Cb/src/Registers.cpp) (libSceAgc) - a register list may start at any byte address, and its 8-byte `{offset, value}` entries are copied byte for byte. PPSA26344 (02.002.000) passes the `sh_registers` lists of compute shader headers that sit 1 byte past an 8-byte boundary in `core_common_subcore.core_b1_fx.sprig.xpps` ([#902](https://github.com/boykopovar/AnyPS5/pull/902)), so those lists start at 1 mod 4 (0x8021363131 in an open-world run). The title runs on the console with these lists, but how the console reads an unaligned list (at the given address, or at a rounded-down one) is not measured -- [sceAgcUpdateInterpolantMapping_0100](../../../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - name from its NID (`k0E7vkgqAuE`); the `(regs, vs, ps)` arguments are the declared ones, behaviour unknown +- [sceAgcUpdateInterpolantMapping_0100](../../../core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp) (libSceAgc) - null pointers, missing semantic tables and input counts above 32 throw as in the sibling host exports; the console error behavior for these inputs has not been measured - [sceAgcWriteDataPatchSetDst, sceAgcWriteDataPatchSetCachePolicy and their Async forms](../../../core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp) (libSceAgc) - the signatures `(packet, uint8 value)` are inferred from the other patch functions, as no public declaration was found; the value is taken in the encoding of the `dst` and `cachePolicy` arguments of `sceAgcDcbWriteData` (graphics) and `sceAgcAcbWriteData` (Async, compute), and the patched packet is the one those calls would write. `sceAgcAsyncWriteDataPatchSetAddressOrOffset` is assumed to patch the address like `sceAgcWriteDataPatchSetAddressOrOffset`, as the compute WRITE_DATA packet keeps it in the same dwords - [sceAgcCreateInterpolantMapping_10000](../../../core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp) (libSceAgc) - name from its NID (`dbOlWdppb4o`); same arguments as `sceAgcCreateInterpolantMapping_0100`; PPSA26344 passes three pointers at each of nine observed call sites (`0x9F914E`, `0x9F91E2`, `0x9F9216`, `0x1604BBD`, `0x16122D7`, `0x1618285`, `0x161838C`, `0x16184EE`, `0x1618607`), consistent with that argument assumption, but does not read the return value there, so its return type is not established. - [sceAgcGetDataPacketPayloadRange](../../../core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp) (libSceAgc) - signature, the `{base, size in bytes}` output and the type 0 range (header + 1, one dword longer than the type 1 payload) from KytyPS5 only, matching `sceAgcGetDataPacketPayloadAddress_0090`; prosper returns header + 2 for both types from that address function From a361119ae15ac0ae8ba2ac619898ae764a7a872e Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Sun, 1 Nov 2026 15:35:33 +0530 Subject: [PATCH 297/318] test(libc): use production exception types in GuestBadOptionalAccess Include the canonical libc Api.hpp (which also provides the exceptions/Runtime.hpp cxa declarations) instead of test-local export declarations: the bad_optional_access ctor/dtor take LibcException::ExceptionObject*, __cxa_throw takes std::type_info*, and __cxa_current_exception_type returns std::type_info*. The exported records are declared as std::type_info / __cxxabiv1::__si_class_type_info matching their TypeRecord/FundamentalTypeRecord layout, and the catch-type callback uses the production bool(const std::type_info*, const std::type_info*, void**, unsigned) signature from TypeInfo.cpp. --- core/libs/tests/GuestBadOptionalAccess.cpp | 54 ++++++++++------------ 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/core/libs/tests/GuestBadOptionalAccess.cpp b/core/libs/tests/GuestBadOptionalAccess.cpp index da6d2b11ee..3dd9f1a1c5 100644 --- a/core/libs/tests/GuestBadOptionalAccess.cpp +++ b/core/libs/tests/GuestBadOptionalAccess.cpp @@ -1,44 +1,40 @@ +#include "prx/libc/include/Api.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include #include #include #include -struct TypeRecord { const void* const* vtable; const char* name; const TypeRecord* base; }; -struct FundamentalRecord { const void* const* vtable; const char* name; }; -struct Object { const void* vtable; const char* message; }; -struct Vtable { std::ptrdiff_t offset; const TypeRecord* type; const void* destroy; const void* deleteObject; const char* (APS5_VABI *what)(const Object*); }; +struct Vtable { std::ptrdiff_t offset; const std::type_info* type; const void* destroy; const void* deleteObject; const char* (APS5_VABI *what)(const LibcException::ExceptionObject*); }; extern "C" { -extern const TypeRecord _ZTISt9exception_nid_postfix; -extern const TypeRecord _ZTISt19bad_optional_access_nid_postfix; -extern const Vtable _ZTVSt19bad_optional_access_nid_postfix; -extern const FundamentalRecord _ZTIi_nid_postfix; -extern const FundamentalRecord _ZTIj_nid_postfix; -extern const FundamentalRecord _ZTIf_nid_postfix; -extern const FundamentalRecord _ZTIDn_nid_postfix; -extern const void* const _ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix[]; -void APS5_VABI _ZNSt19bad_optional_accessC1Ev_nid_postfix(Object*); -void APS5_VABI _ZNSt19bad_optional_accessD1Ev_nid_postfix(Object*); -[[noreturn]] void APS5_VABI _ZSt26_Throw_bad_optional_accessv_nid_postfix(); -void* APS5_VABI __cxa_allocate_exception_nid_postfix(std::size_t); -[[noreturn]] void APS5_VABI __cxa_throw_nid_postfix(void*, const void*, void (*)(void*)); -const void* APS5_VABI __cxa_current_exception_type_nid_postfix(); +extern __cxxabiv1::__class_type_info _ZTISt9exception_nid_postfix; +extern __cxxabiv1::__si_class_type_info _ZTISt19bad_optional_access_nid_postfix; +extern Vtable _ZTVSt19bad_optional_access_nid_postfix; +extern std::type_info _ZTIi_nid_postfix; +extern std::type_info _ZTIj_nid_postfix; +extern std::type_info _ZTIf_nid_postfix; +extern std::type_info _ZTIDn_nid_postfix; +extern void* const _ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix[]; } static void Require(bool value) { if (!value) std::abort(); } +static void* const* Vptr(const std::type_info& record) { + return *reinterpret_cast(&record); +} + static void TestFundamentalRecords() { - const auto* const address = reinterpret_cast(_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix) + 2; - const FundamentalRecord* records[] = {&_ZTIi_nid_postfix, &_ZTIj_nid_postfix, &_ZTIf_nid_postfix, &_ZTIDn_nid_postfix}; - for (const FundamentalRecord* record : records) - Require(record->vtable == address); - Require(std::strcmp(_ZTIi_nid_postfix.name, "i") == 0 && std::strcmp(_ZTIDn_nid_postfix.name, "Dn") == 0); - const auto* recordType = static_cast(_ZTIi_nid_postfix.vtable[-1]); + void* const* const fundamental = &_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix[2]; + const std::type_info* records[] = {&_ZTIi_nid_postfix, &_ZTIj_nid_postfix, &_ZTIf_nid_postfix, &_ZTIDn_nid_postfix}; + for (const std::type_info* record : records) + Require(Vptr(*record) == fundamental); + Require(std::strcmp(_ZTIi_nid_postfix.name(), "i") == 0 && std::strcmp(_ZTIDn_nid_postfix.name(), "Dn") == 0); + const auto* recordType = static_cast(Vptr(_ZTIi_nid_postfix)[-1]); Require(std::strcmp(recordType->name(), "N10__cxxabiv123__fundamental_type_infoE") == 0); - using CatchType = bool (APS5_VABI*)(const void*, const void*, void**, unsigned); - const auto catchType = reinterpret_cast(_ZTIi_nid_postfix.vtable[4]); + using CatchType = bool (APS5_VABI*)(const std::type_info*, const std::type_info*, void**, unsigned); + const auto catchType = reinterpret_cast(Vptr(_ZTIi_nid_postfix)[4]); int thrownValue = 0; void* thrownObject = &thrownValue; Require(catchType(&_ZTIi_nid_postfix, &_ZTIi_nid_postfix, &thrownObject, 0)); @@ -59,11 +55,11 @@ static void TestFundamentalRecords() { int main() { const auto& type = _ZTISt19bad_optional_access_nid_postfix; - Require(std::strcmp(type.name, "St19bad_optional_access") == 0); - Require(type.base == &_ZTISt9exception_nid_postfix); + Require(std::strcmp(type.name(), "St19bad_optional_access") == 0); + Require(type.__base_type == &_ZTISt9exception_nid_postfix); const auto& table = _ZTVSt19bad_optional_access_nid_postfix; Require(table.offset == 0 && table.type == &type && table.what != nullptr); - Object object{}; + LibcException::ExceptionObject object{}; _ZNSt19bad_optional_accessC1Ev_nid_postfix(&object); Require(object.vtable == &table.destroy); Require(std::strcmp(table.what(&object), "bad_optional_access") == 0); From c20d797450bd6ecf4ef6198472693a640af59db4 Mon Sep 17 00:00:00 2001 From: Eduardo Ruiz Date: Sun, 25 Oct 2026 13:17:19 +0100 Subject: [PATCH 298/318] feat(libc): export the versioned std::__sce_v2 filesystem primitives --- .../prx/libSceLibcInternal/Filesystem.cpp | 134 +++--------------- core/libs/prx/libc/CMakeLists.txt | 12 ++ core/libs/prx/libc/include/Api.hpp | 7 + .../include/GuestFilesystemPrimitives.hpp | 121 ++++++++++++++++ .../libs/prx/libc/src/VersionedFilesystem.cpp | 37 +++++ core/libs/tests/GuestSceV2Filesystem.cpp | 90 ++++++++++++ docs/dev/debt/libc.md | 1 + 7 files changed, 284 insertions(+), 118 deletions(-) create mode 100644 core/libs/prx/libc/include/GuestFilesystemPrimitives.hpp create mode 100644 core/libs/prx/libc/src/VersionedFilesystem.cpp create mode 100644 core/libs/tests/GuestSceV2Filesystem.cpp diff --git a/core/libs/prx/libSceLibcInternal/Filesystem.cpp b/core/libs/prx/libSceLibcInternal/Filesystem.cpp index bdf5cbf0f7..3edb28ec34 100644 --- a/core/libs/prx/libSceLibcInternal/Filesystem.cpp +++ b/core/libs/prx/libSceLibcInternal/Filesystem.cpp @@ -1,99 +1,20 @@ -#include "prx/libkernel/include/Api.hpp" -#include "prx/libc/include/Api.hpp" -#include "prx/libc/include/GuestDirectory.hpp" +#include "prx/libc/include/GuestFilesystemPrimitives.hpp" #include "prx/libc/include/General.hpp" -#include "SceTypes.hpp" -#include -#include -#include - -namespace { - -constexpr std::size_t NameCapacity = 260; -constexpr std::size_t CurrentCapacity = 256; -constexpr int NoSuchFile = 2; -constexpr int Exists = 17; -constexpr int NotDirectory = 20; -constexpr int Invalid = 22; -constexpr int NameTooLong = 63; -constexpr std::uint16_t DefaultDirectoryMode = 0777; -constexpr std::uint16_t PermissionBits = 07777; - -enum FileType : int { - TypeNotFound = -1, - TypeNone = 0, - TypeRegular = 1, - TypeDirectory = 2, - TypeSymlink = 3, - TypeBlock = 4, - TypeCharacter = 5, - TypeFifo = 6, - TypeSocket = 7, - TypeUnknown = 8, -}; - -int& GuestErrno() { - return *__error_nid_postfix(); -} - -int TypeOfMode(std::uint16_t mode) { - switch (mode & 0170000) { - case 0010000: return TypeFifo; - case 0020000: return TypeCharacter; - case 0040000: return TypeDirectory; - case 0060000: return TypeBlock; - case 0100000: return TypeRegular; - case 0120000: return TypeSymlink; - case 0140000: return TypeSocket; - default: return TypeUnknown; - } -} - -int StatType(const char* path, int* perms, int* error) { - GuestErrno() = 0; - FileStat status{}; - int type; - if (stat_nid_postfix(path, &status) == 0) { - GuestErrno() = 0; - if (perms) *perms = status.st_mode & PermissionBits; - type = TypeOfMode(status.st_mode); - } else { - const int code = GuestErrno(); - type = code == NoSuchFile || code == NotDirectory || code == Invalid || code == NameTooLong ? TypeNotFound : TypeNone; - } - if (error) *error = GuestErrno(); - return type; -} - -void CopyName(char* target, const char* source) { - std::size_t length = 0; - while (length < NameCapacity - 1 && source[length] != '\0') { - target[length] = source[length]; - ++length; - } - target[length] = '\0'; -} - -bool IsDotEntry(const char* name) { - return name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); -} - -} extern "C" { int APS5_VABI _ZNSt10filesystem5_StatEPKcPNS_5permsEPi_nid_postfix(const char* path, int* perms, int* error) { - return StatType(path, perms, error); + return GuestFilesystem::StatType(path, perms, error); } int APS5_VABI _ZNSt10filesystem6_LstatEPKcPNS_5permsEPi_nid_postfix(const char* path, int* perms, int* error) { - return StatType(path, perms, error); + return GuestFilesystem::StatType(path, perms, error); } std::uint64_t APS5_VABI _ZNSt10filesystem10_File_sizeEPKcPi_nid_postfix(const char* path, int* error) { FileStat status{}; if (stat_nid_postfix(path, &status) != 0) { - if (error) *error = GuestErrno(); + if (error) *error = GuestFilesystem::GuestErrno(); return UINT64_MAX; } return static_cast(status.st_size); @@ -101,23 +22,23 @@ std::uint64_t APS5_VABI _ZNSt10filesystem10_File_sizeEPKcPi_nid_postfix(const ch int APS5_VABI _ZNSt10filesystem16_Last_write_timeEPKcPlS2__nid_postfix(const char* path, std::int64_t* seconds, std::int64_t* nanoseconds) { FileStat status{}; - if (stat_nid_postfix(path, &status) != 0) return GuestErrno(); + if (stat_nid_postfix(path, &status) != 0) return GuestFilesystem::GuestErrno(); *seconds = status.st_mtim.tv_sec; *nanoseconds = status.st_mtim.tv_nsec; return 0; } int APS5_VABI _ZNSt10filesystem9_Make_dirEPKcS1__nid_postfix(const char* path, const char* existing) { - GuestErrno() = 0; - std::uint16_t mode = DefaultDirectoryMode; + GuestFilesystem::GuestErrno() = 0; + std::uint16_t mode = GuestFilesystem::DefaultDirectoryMode; if (existing) { FileStat status{}; - if (stat_nid_postfix(existing, &status) != 0) return -GuestErrno(); + if (stat_nid_postfix(existing, &status) != 0) return -GuestFilesystem::GuestErrno(); mode = status.st_mode; } if (mkdir_nid_postfix(path, mode) == 0) return 1; - if (GuestErrno() == Exists) return 0; - return -GuestErrno(); + if (GuestFilesystem::GuestErrno() == GuestFilesystem::Exists) return 0; + return -GuestFilesystem::GuestErrno(); } bool APS5_VABI _ZNSt10filesystem11_Remove_dirEPKc_nid_postfix(const char* path) { @@ -125,8 +46,8 @@ bool APS5_VABI _ZNSt10filesystem11_Remove_dirEPKc_nid_postfix(const char* path) } int APS5_VABI _ZNSt10filesystem7_UnlinkEPKc_nid_postfix(const char* path) { - GuestErrno() = 0; - return unlink_nid_postfix(path) == 0 ? 0 : GuestErrno(); + GuestFilesystem::GuestErrno() = 0; + return unlink_nid_postfix(path) == 0 ? 0 : GuestFilesystem::GuestErrno(); } int APS5_VABI _ZNSt10filesystem6_ChmodEPKcNS_5permsE_nid_postfix(const char* path, int perms) { @@ -134,42 +55,19 @@ int APS5_VABI _ZNSt10filesystem6_ChmodEPKcNS_5permsE_nid_postfix(const char* pat } bool APS5_VABI _ZNSt10filesystem12_Current_getERA260_c_nid_postfix(char* target) { - char current[CurrentCapacity]; - const bool found = getcwd_nid_postfix(current, sizeof(current)) != nullptr; - if (found) CopyName(target, current); - else target[0] = '\0'; - return found; + return GuestFilesystem::CurrentGet(target); } char* APS5_VABI _ZNSt10filesystem9_Read_dirERA260_cPvRNS_9file_typeE_nid_postfix(char* target, void* handle, int* type) { - *type = TypeUnknown; - GuestDirectoryEntry entry{}; - GuestDirectoryEntry* result = nullptr; - while (readdir_r_nid_postfix(handle, &entry, &result) == 0 && result) { - if (IsDotEntry(entry.name)) continue; - CopyName(target, entry.name); - return target; - } - target[0] = '\0'; - return target; + return GuestFilesystem::ReadDir(target, handle, type); } void* APS5_VABI _ZNSt10filesystem9_Open_dirERA260_cPKcRiRNS_9file_typeE_nid_postfix(char* target, const char* path, int* error, int* type) { - GuestErrno() = 0; - void* handle = opendir_nid_postfix(path); - if (!handle) { - *error = GuestErrno(); - return nullptr; - } - *error = 0; - _ZNSt10filesystem9_Read_dirERA260_cPvRNS_9file_typeE_nid_postfix(target, handle, type); - if (target[0] != '\0') return handle; - closedir_nid_postfix(handle); - return nullptr; + return GuestFilesystem::OpenDir(target, path, error, type); } int APS5_VABI _ZNSt10filesystem10_Close_dirEPv_nid_postfix(void* handle) { - return closedir_nid_postfix(handle); + return GuestFilesystem::CloseDir(handle); } } diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index 90509b697a..91e92874ef 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -119,6 +119,7 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/Environment.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Directory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Filesystem.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/VersionedFilesystem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Rename.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Errors.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Syslog.cpp @@ -230,6 +231,17 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + + add_test_executable(guest_sce_v2_filesystem_tests ${LIBS_INCLUDE_DIR}/tests/GuestSceV2Filesystem.cpp) + target_include_directories(guest_sce_v2_filesystem_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_sce_v2_filesystem_tests PRIVATE libc libkernel) + configure_windows_unwind(guest_sce_v2_filesystem_tests) + add_test(NAME guest_sce_v2_filesystem COMMAND guest_sce_v2_filesystem_tests) + set_tests_properties(guest_sce_v2_filesystem PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_sce_v2_filesystem PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() if(WIN32) diff --git a/core/libs/prx/libc/include/Api.hpp b/core/libs/prx/libc/include/Api.hpp index 2e55965a17..6cd58f7073 100644 --- a/core/libs/prx/libc/include/Api.hpp +++ b/core/libs/prx/libc/include/Api.hpp @@ -227,6 +227,13 @@ int APS5_VABI closedir_nid_postfix(void* handle); void APS5_VABI rewinddir_nid_postfix(void* handle); std::int64_t APS5_VABI telldir_nid_postfix(void* handle); void APS5_VABI seekdir_nid_postfix(void* handle, std::int64_t location); +int APS5_VABI _ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(const char* path, int* perms, int* error); +void* APS5_VABI _ZNSt8__sce_v210filesystem9_Open_dirERA260_cPKcRiRNS0_9file_typeE_nid_postfix(char* target, const char* path, int* error, int* type); +char* APS5_VABI _ZNSt8__sce_v210filesystem9_Read_dirERA260_cPvRNS0_9file_typeE_nid_postfix(char* target, void* handle, int* type); +int APS5_VABI _ZNSt8__sce_v210filesystem10_Close_dirEPv_nid_postfix(void* handle); +bool APS5_VABI _ZNSt8__sce_v210filesystem12_Current_getERA260_c_nid_postfix(char* target); +void APS5_VABI _ZNSt8__sce_v210filesystem18_Xfilesystem_errorEPKcNS_10error_codeE_nid_postfix(); +void APS5_VABI _ZNSt8__sce_v210filesystem18_Xfilesystem_errorEPKcRKNS0_4pathENS_10error_codeE_nid_postfix(); char* APS5_VABI getenv_nid_postfix(const char* name); int APS5_VABI setenv_nid_postfix(const char* name, const char* value, int overwrite); int APS5_VABI unsetenv_nid_postfix(const char* name); diff --git a/core/libs/prx/libc/include/GuestFilesystemPrimitives.hpp b/core/libs/prx/libc/include/GuestFilesystemPrimitives.hpp new file mode 100644 index 0000000000..5907b5a047 --- /dev/null +++ b/core/libs/prx/libc/include/GuestFilesystemPrimitives.hpp @@ -0,0 +1,121 @@ +#pragma once + +#include "prx/libc/include/Api.hpp" +#include "prx/libc/include/GuestDirectory.hpp" +#include "prx/libkernel/include/Api.hpp" + +#include +#include +#include + +namespace GuestFilesystem { + +inline constexpr std::size_t NameCapacity = 260; +inline constexpr std::size_t CurrentCapacity = 256; +inline constexpr int NoSuchFile = 2; +inline constexpr int Exists = 17; +inline constexpr int NotDirectory = 20; +inline constexpr int Invalid = 22; +inline constexpr int NameTooLong = 63; +inline constexpr std::uint16_t DefaultDirectoryMode = 0777; +inline constexpr std::uint16_t PermissionBits = 07777; + +enum FileType : int { + TypeNotFound = -1, + TypeNone = 0, + TypeRegular = 1, + TypeDirectory = 2, + TypeSymlink = 3, + TypeBlock = 4, + TypeCharacter = 5, + TypeFifo = 6, + TypeSocket = 7, + TypeUnknown = 8, +}; + +inline int& GuestErrno() { + return *__error_nid_postfix(); +} + +inline int TypeOfMode(std::uint16_t mode) { + switch (mode & 0170000) { + case 0010000: return TypeFifo; + case 0020000: return TypeCharacter; + case 0040000: return TypeDirectory; + case 0060000: return TypeBlock; + case 0100000: return TypeRegular; + case 0120000: return TypeSymlink; + case 0140000: return TypeSocket; + default: return TypeUnknown; + } +} + +inline int StatType(const char* path, int* perms, int* error) { + GuestErrno() = 0; + FileStat status{}; + int type; + if (stat_nid_postfix(path, &status) == 0) { + GuestErrno() = 0; + if (perms) *perms = status.st_mode & PermissionBits; + type = TypeOfMode(status.st_mode); + } else { + const int code = GuestErrno(); + type = code == NoSuchFile || code == NotDirectory || code == Invalid || code == NameTooLong ? TypeNotFound : TypeNone; + } + if (error) *error = GuestErrno(); + return type; +} + +inline void CopyName(char* target, const char* source) { + std::size_t length = 0; + while (length < NameCapacity - 1 && source[length] != '\0') { + target[length] = source[length]; + ++length; + } + target[length] = '\0'; +} + +inline bool IsDotEntry(const char* name) { + return name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); +} + +inline bool CurrentGet(char* target) { + char current[CurrentCapacity]; + const bool found = getcwd_nid_postfix(current, sizeof(current)) != nullptr; + if (found) CopyName(target, current); + else target[0] = '\0'; + return found; +} + +inline char* ReadDir(char* target, void* handle, int* type) { + *type = TypeUnknown; + GuestDirectoryEntry entry{}; + GuestDirectoryEntry* result = nullptr; + while (readdir_r_nid_postfix(handle, &entry, &result) == 0 && result) { + if (IsDotEntry(entry.name)) continue; + CopyName(target, entry.name); + return target; + } + target[0] = '\0'; + return target; +} + +inline void* OpenDir(char* target, const char* path, int* error, int* type) { + GuestErrno() = 0; + void* handle = opendir_nid_postfix(path); + if (!handle) { + *error = GuestErrno(); + return nullptr; + } + *error = 0; + ReadDir(target, handle, type); + if (target[0] != '\0') return handle; + closedir_nid_postfix(handle); + return nullptr; +} + +inline int CloseDir(void* handle) { + return closedir_nid_postfix(handle); +} + +} diff --git a/core/libs/prx/libc/src/VersionedFilesystem.cpp b/core/libs/prx/libc/src/VersionedFilesystem.cpp new file mode 100644 index 0000000000..7a44277709 --- /dev/null +++ b/core/libs/prx/libc/src/VersionedFilesystem.cpp @@ -0,0 +1,37 @@ +#include "prx/libc/include/Api.hpp" +#include "prx/libc/include/General.hpp" +#include "prx/libc/include/GuestFilesystemPrimitives.hpp" + +extern "C" { + +int APS5_VABI _ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(const char* path, int* perms, int* error) { + return GuestFilesystem::StatType(path, perms, error); +} + +void* APS5_VABI _ZNSt8__sce_v210filesystem9_Open_dirERA260_cPKcRiRNS0_9file_typeE_nid_postfix(char* target, const char* path, int* error, int* type) { + return GuestFilesystem::OpenDir(target, path, error, type); +} + +char* APS5_VABI _ZNSt8__sce_v210filesystem9_Read_dirERA260_cPvRNS0_9file_typeE_nid_postfix(char* target, void* handle, int* type) { + return GuestFilesystem::ReadDir(target, handle, type); +} + +int APS5_VABI _ZNSt8__sce_v210filesystem10_Close_dirEPv_nid_postfix(void* handle) { + return GuestFilesystem::CloseDir(handle); +} + +bool APS5_VABI _ZNSt8__sce_v210filesystem12_Current_getERA260_c_nid_postfix(char* target) { + target[0] = '/'; + target[1] = '\0'; + return true; +} + +void APS5_VABI _ZNSt8__sce_v210filesystem18_Xfilesystem_errorEPKcNS_10error_codeE_nid_postfix() { + NotImplemented_nid_no_patch(__func__); +} + +void APS5_VABI _ZNSt8__sce_v210filesystem18_Xfilesystem_errorEPKcRKNS0_4pathENS_10error_codeE_nid_postfix() { + NotImplemented_nid_no_patch(__func__); +} + +} diff --git a/core/libs/tests/GuestSceV2Filesystem.cpp b/core/libs/tests/GuestSceV2Filesystem.cpp new file mode 100644 index 0000000000..d3ed79bf46 --- /dev/null +++ b/core/libs/tests/GuestSceV2Filesystem.cpp @@ -0,0 +1,90 @@ +#include "prx/libc/include/Api.hpp" +#include "prx/libkernel/include/Api.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr int NoSuchFile = 2; +constexpr int TypeNotFound = -1; +constexpr int TypeRegular = 1; +constexpr int TypeDirectory = 2; +constexpr int TypeUnknown = 8; + +void Require(bool condition, const char* message) { + if (condition) return; + std::fprintf(stderr, "sce v2 filesystem: %s\n", message); + std::abort(); +} + +std::string root; + +std::string At(const char* name) { + return root + "/" + name; +} + +void Write(const std::string& path, std::size_t bytes) { + std::ofstream file(path, std::ios::binary | std::ios::trunc); + file << std::string(bytes, 'x'); +} + +void TestStat() { + Write(At("file"), 7); + int perms = -1; + int error = -1; + Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(At("file").c_str(), &perms, &error) == TypeRegular && error == 0, "regular file"); + FileStat status{}; + Require(stat_nid_postfix(At("file").c_str(), &status) == 0 && perms == (status.st_mode & 07777), "permission bits"); + Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(root.c_str(), nullptr, &error) == TypeDirectory && error == 0, "directory"); + perms = 1234; + Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(At("missing").c_str(), &perms, &error) == TypeNotFound && error == NoSuchFile && perms == 1234, "missing path"); +} + +void TestDirectories() { + std::filesystem::create_directory(At("made")); + char name[260]; + int error = -1; + int type = -1; + Require(_ZNSt8__sce_v210filesystem9_Open_dirERA260_cPKcRiRNS0_9file_typeE_nid_postfix(name, At("made").c_str(), &error, &type) == nullptr && error == 0 && name[0] == '\0', "an empty directory gives no handle"); + Require(_ZNSt8__sce_v210filesystem9_Open_dirERA260_cPKcRiRNS0_9file_typeE_nid_postfix(name, At("missing").c_str(), &error, &type) == nullptr && error == NoSuchFile, "a missing directory"); + Write(At("made/one"), 1); + Write(At("made/two"), 2); + void* handle = _ZNSt8__sce_v210filesystem9_Open_dirERA260_cPKcRiRNS0_9file_typeE_nid_postfix(name, At("made").c_str(), &error, &type); + Require(handle != nullptr && error == 0 && type == TypeUnknown, "open a directory"); + std::set names{name}; + type = 0; + Require(_ZNSt8__sce_v210filesystem9_Read_dirERA260_cPvRNS0_9file_typeE_nid_postfix(name, handle, &type) == name && type == TypeUnknown, "read a directory"); + names.insert(name); + Require(names == std::set{"one", "two"}, "entries without dot entries"); + Require(_ZNSt8__sce_v210filesystem9_Read_dirERA260_cPvRNS0_9file_typeE_nid_postfix(name, handle, &type)[0] == '\0' && type == TypeUnknown, "end of the directory"); + Require(_ZNSt8__sce_v210filesystem10_Close_dirEPv_nid_postfix(handle) == 0, "close a directory"); +} + +void TestCurrent() { + char current[260]; + std::memset(current, 0x5a, sizeof(current)); + Require(_ZNSt8__sce_v210filesystem12_Current_getERA260_c_nid_postfix(current), "current directory"); + Require(std::strcmp(current, "/") == 0, "current directory is the sandbox root"); + Require(static_cast(current[2]) == 0x5a, "current directory writes two bytes"); +} + +} + +int main() { + root = "anyps5-sce-v2-filesystem-" + std::to_string(std::random_device{}()); + std::filesystem::create_directory(root); + TestStat(); + TestDirectories(); + TestCurrent(); + std::error_code ignored; + std::filesystem::remove_all(root, ignored); + return 0; +} diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 6826a932df..0e121fcef9 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -269,3 +269,4 @@ - [tanhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) is FreeBSD msun's [`s_tanhf.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_tanhf.c) over a full port of msun's [`s_expm1f.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_expm1f.c) (`Expm1f`, with `huge` read at run time), which the exported `expm1f` also uses. It sets no `errno`, as the `libc.prx` that titles ship does not. Floating-point exceptions are not compared. It gives the same bits as the `tanhf` of the `libc.prx` bundled with PPSA07923 and PPSA05916 for all 2^32 arguments, run on a Windows host; the FMA-contracted copy of PPSA13934 differs by up to 3 ulp on 150,846 of them. - [expm1f](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions. It is FreeBSD msun's [`s_expm1f.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_expm1f.c) (`Expm1f`, shared with `tanhf`), and sets `ERANGE` only when a finite argument overflows to infinity, as the titles' `libc.prx` does; large negative arguments return -1 without `errno`. It gives the same bits and `errno` as the `expm1f` bundled with PPSA07923 and PPSA05916 for all 2^32 arguments on a Windows host. The FMA-contracted build bundled with PPSA13934 differs by 1 ulp on 197,464 arguments. - [logb and logbf](../../../core/libs/prx/libc/src/Math.cpp) (libc) do not compare floating-point exceptions, and their double inputs are checked by sampling, not exhaustively. They are FreeBSD msun's [`s_logb.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_logb.c) and [`s_logbf.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_logbf.c): ±0 returns `-inf` without `errno`, and a signalling NaN is returned quieted. The `libc.prx` bundled with eleven titles (PPSA04158 to PPSA35169) gives the same bits and no `errno` on 4,032,768 inputs per function on a Windows host. +- [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Stat`, `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` use the unversioned Dinkumware filesystem implementation, which resolves guest paths and reads through libkernel's directory calls; the console's version loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows (`_Stat` 0xd7 bytes at 0xcb9b0, `_Open_dir` 0xe9 at 0xcb640, `_Read_dir` 0x15b at 0xcb730, `_Close_dir` 0x2b at 0xcb890, `_Current_get` 8 at 0xcb8c0), and maps the same errors. `_Read_dir` reports file type 8 for every entry, as both do. The console's `_Current_get` writes "/" and returns true instead of reading the current directory, and that is reproduced. The two `_Xfilesystem_error` throwers have no console body yet and call `NotImplemented_nid_no_patch`, as the unversioned pair does; `std::__sce_v2::filesystem_error` is not implemented. From 6ea864bf5e3ca2f84ff61263790af9c13da12e9e Mon Sep 17 00:00:00 2001 From: Eduardo Ruiz Date: Sun, 25 Oct 2026 14:41:10 +0100 Subject: [PATCH 299/318] fix(libc): stat the versioned filesystem paths without libkernel _Stat now resolves the guest path and reads the type and permissions through the host filesystem, so libc.prx no longer imports stat_nid_postfix from libkernel.prx. That import broke every host target that links only libc.prx (agc_shader_replay, agc_shader_replay_requests, host_libstdcxx_exception and the libc test executables) with an undefined reference to stat_nid_postfix. Types, permission bits and guest errno keep matching stat_nid_postfix, which guest_sce_v2_filesystem still cross-checks. --- .../libs/prx/libc/src/VersionedFilesystem.cpp | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libc/src/VersionedFilesystem.cpp b/core/libs/prx/libc/src/VersionedFilesystem.cpp index 7a44277709..236b98bfeb 100644 --- a/core/libs/prx/libc/src/VersionedFilesystem.cpp +++ b/core/libs/prx/libc/src/VersionedFilesystem.cpp @@ -1,11 +1,66 @@ #include "prx/libc/include/Api.hpp" +#include "prx/libc/include/FilesystemError.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestFilesystemPrimitives.hpp" +#include +#include +#include + +namespace { + +GuestFilesystem::FileType FromNativeType(std::filesystem::file_type type) { + switch (type) { + case std::filesystem::file_type::none: return GuestFilesystem::TypeNone; + case std::filesystem::file_type::not_found: return GuestFilesystem::TypeNotFound; + case std::filesystem::file_type::regular: return GuestFilesystem::TypeRegular; + case std::filesystem::file_type::directory: return GuestFilesystem::TypeDirectory; + case std::filesystem::file_type::symlink: return GuestFilesystem::TypeSymlink; + case std::filesystem::file_type::block: return GuestFilesystem::TypeBlock; + case std::filesystem::file_type::character: return GuestFilesystem::TypeCharacter; + case std::filesystem::file_type::fifo: return GuestFilesystem::TypeFifo; + case std::filesystem::file_type::socket: return GuestFilesystem::TypeSocket; + default: return GuestFilesystem::TypeUnknown; + } +} + +bool IsNotFound(int code) { + return code == GuestFilesystem::NoSuchFile || code == GuestFilesystem::NotDirectory || + code == GuestFilesystem::Invalid || code == GuestFilesystem::NameTooLong; +} + +int StatType(const char* path, int* perms, int* error) { + errno = 0; + std::filesystem::path native; + int code = ResolvePathError_nid_no_patch(path, native); + int type; + if (code != 0) { + errno = code; + type = IsNotFound(code) ? GuestFilesystem::TypeNotFound : GuestFilesystem::TypeNone; + } else { + std::error_code failure; + const auto status = std::filesystem::status(native, failure); + if (!failure && status.type() != std::filesystem::file_type::not_found) { + errno = 0; + if (perms) *perms = static_cast(status.permissions()) & GuestFilesystem::PermissionBits; + type = FromNativeType(status.type()); + } else { + code = failure ? FilesystemError(failure) : GuestFilesystem::NoSuchFile; + if (code == GuestFilesystem::NoSuchFile) code = NotFoundError_nid_no_patch(native); + errno = code; + type = GuestFilesystem::TypeNotFound; + } + } + if (error) *error = errno; + return type; +} + +} + extern "C" { int APS5_VABI _ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(const char* path, int* perms, int* error) { - return GuestFilesystem::StatType(path, perms, error); + return StatType(path, perms, error); } void* APS5_VABI _ZNSt8__sce_v210filesystem9_Open_dirERA260_cPKcRiRNS0_9file_typeE_nid_postfix(char* target, const char* path, int* error, int* type) { From 2c5b571e6837c4b5b3030025752b07831b08f49a Mon Sep 17 00:00:00 2001 From: Eduardo Ruiz Date: Sun, 25 Oct 2026 20:06:38 +0100 Subject: [PATCH 300/318] fix(libc): return TypeNone for non-not-found stat errors --- core/libs/prx/libc/src/VersionedFilesystem.cpp | 2 +- core/libs/tests/GuestSceV2Filesystem.cpp | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libc/src/VersionedFilesystem.cpp b/core/libs/prx/libc/src/VersionedFilesystem.cpp index 236b98bfeb..01ea53464b 100644 --- a/core/libs/prx/libc/src/VersionedFilesystem.cpp +++ b/core/libs/prx/libc/src/VersionedFilesystem.cpp @@ -48,7 +48,7 @@ int StatType(const char* path, int* perms, int* error) { code = failure ? FilesystemError(failure) : GuestFilesystem::NoSuchFile; if (code == GuestFilesystem::NoSuchFile) code = NotFoundError_nid_no_patch(native); errno = code; - type = GuestFilesystem::TypeNotFound; + type = IsNotFound(code) ? GuestFilesystem::TypeNotFound : GuestFilesystem::TypeNone; } } if (error) *error = errno; diff --git a/core/libs/tests/GuestSceV2Filesystem.cpp b/core/libs/tests/GuestSceV2Filesystem.cpp index d3ed79bf46..c1ad0adf24 100644 --- a/core/libs/tests/GuestSceV2Filesystem.cpp +++ b/core/libs/tests/GuestSceV2Filesystem.cpp @@ -10,14 +10,17 @@ #include #include #include +#include namespace { constexpr int NoSuchFile = 2; constexpr int TypeNotFound = -1; +constexpr int TypeNone = 0; constexpr int TypeRegular = 1; constexpr int TypeDirectory = 2; constexpr int TypeUnknown = 8; +constexpr int Loop = 62; void Require(bool condition, const char* message) { if (condition) return; @@ -48,6 +51,19 @@ void TestStat() { Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(At("missing").c_str(), &perms, &error) == TypeNotFound && error == NoSuchFile && perms == 1234, "missing path"); } +void TestStatError() { + const auto loop = At("loop"); + std::error_code failure; + std::filesystem::create_symlink("loop", loop, failure); + if (failure) return; + std::error_code probe; + (void)std::filesystem::status(loop, probe); + if (probe != std::errc::too_many_symbolic_link_levels) return; + int perms = 1234; + int error = -1; + Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(loop.c_str(), &perms, &error) == TypeNone && error == Loop && perms == 1234, "a symlink loop is not reported as not found"); +} + void TestDirectories() { std::filesystem::create_directory(At("made")); char name[260]; @@ -82,6 +98,7 @@ int main() { root = "anyps5-sce-v2-filesystem-" + std::to_string(std::random_device{}()); std::filesystem::create_directory(root); TestStat(); + TestStatError(); TestDirectories(); TestCurrent(); std::error_code ignored; From 82779f5cc8dcf3990208db8066b02630e05e5f6f Mon Sep 17 00:00:00 2001 From: edubox Date: Mon, 26 Oct 2026 01:17:11 +0100 Subject: [PATCH 301/318] fix(libc): stat long versioned filesystem paths on Windows --- .../libs/prx/libc/src/VersionedFilesystem.cpp | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/libs/prx/libc/src/VersionedFilesystem.cpp b/core/libs/prx/libc/src/VersionedFilesystem.cpp index 01ea53464b..1128589a9b 100644 --- a/core/libs/prx/libc/src/VersionedFilesystem.cpp +++ b/core/libs/prx/libc/src/VersionedFilesystem.cpp @@ -7,6 +7,13 @@ #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + namespace { GuestFilesystem::FileType FromNativeType(std::filesystem::file_type type) { @@ -29,6 +36,20 @@ bool IsNotFound(int code) { code == GuestFilesystem::Invalid || code == GuestFilesystem::NameTooLong; } +#ifdef _WIN32 +std::filesystem::path StatusPath(const std::filesystem::path& path) { + const auto native = path.native(); + if (native.starts_with(L"\\\\?\\") || native.starts_with(L"\\\\.\\")) return path; + std::error_code error; + const auto absolute = std::filesystem::absolute(path, error); + if (error) return path; + const auto normalized = absolute.lexically_normal().make_preferred().native(); + if (normalized.size() < MAX_PATH) return absolute; + return normalized.starts_with(L"\\\\") ? std::filesystem::path(L"\\\\?\\UNC\\" + normalized.substr(2)) + : std::filesystem::path(L"\\\\?\\" + normalized); +} +#endif + int StatType(const char* path, int* perms, int* error) { errno = 0; std::filesystem::path native; @@ -39,7 +60,11 @@ int StatType(const char* path, int* perms, int* error) { type = IsNotFound(code) ? GuestFilesystem::TypeNotFound : GuestFilesystem::TypeNone; } else { std::error_code failure; +#ifdef _WIN32 + const auto status = std::filesystem::status(StatusPath(native), failure); +#else const auto status = std::filesystem::status(native, failure); +#endif if (!failure && status.type() != std::filesystem::file_type::not_found) { errno = 0; if (perms) *perms = static_cast(status.permissions()) & GuestFilesystem::PermissionBits; From 9e4e43d7b27e30bf59f069171441714c539fc884 Mon Sep 17 00:00:00 2001 From: Eduardo Ruiz Date: Wed, 28 Oct 2026 00:05:09 +0100 Subject: [PATCH 302/318] fix(libc): stat app0.zar entries in the versioned filesystem --- core/libs/prx/libc/CMakeLists.txt | 2 +- .../libs/prx/libc/src/VersionedFilesystem.cpp | 11 +++++++++- core/libs/tests/GuestSceV2Filesystem.cpp | 21 ++++++++++++++++--- docs/dev/debt/libc.md | 1 + 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index 91e92874ef..f0b46e04a4 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -234,7 +234,7 @@ if(BUILD_TESTING) add_test_executable(guest_sce_v2_filesystem_tests ${LIBS_INCLUDE_DIR}/tests/GuestSceV2Filesystem.cpp) target_include_directories(guest_sce_v2_filesystem_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_sce_v2_filesystem_tests PRIVATE libc libkernel) + target_link_libraries(guest_sce_v2_filesystem_tests PRIVATE libc libkernel zarchive) configure_windows_unwind(guest_sce_v2_filesystem_tests) add_test(NAME guest_sce_v2_filesystem COMMAND guest_sce_v2_filesystem_tests) set_tests_properties(guest_sce_v2_filesystem PROPERTIES TIMEOUT 20) diff --git a/core/libs/prx/libc/src/VersionedFilesystem.cpp b/core/libs/prx/libc/src/VersionedFilesystem.cpp index 1128589a9b..33051e826d 100644 --- a/core/libs/prx/libc/src/VersionedFilesystem.cpp +++ b/core/libs/prx/libc/src/VersionedFilesystem.cpp @@ -1,10 +1,12 @@ #include "prx/libc/include/Api.hpp" +#include "prx/libc/include/AppArchive.hpp" #include "prx/libc/include/FilesystemError.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestFilesystemPrimitives.hpp" #include #include +#include #include #ifdef _WIN32 @@ -53,11 +55,18 @@ std::filesystem::path StatusPath(const std::filesystem::path& path) { int StatType(const char* path, int* perms, int* error) { errno = 0; std::filesystem::path native; - int code = ResolvePathError_nid_no_patch(path, native); + int code = ResolvePathError_nid_no_patch(path, native, false, true); int type; if (code != 0) { errno = code; type = IsNotFound(code) ? GuestFilesystem::TypeNotFound : GuestFilesystem::TypeNone; + } else if (AppArchiveOnly_nid_no_patch(native)) { + AppArchiveNode node{}; + if (!AppArchiveLookup_nid_no_patch(native, &node)) + throw std::logic_error("sce v2 filesystem: " + native.string() + " left app0.zar while reading its status"); + errno = 0; + if (perms) *perms = node.Directory ? 0555 : 0444; + type = node.Directory ? GuestFilesystem::TypeDirectory : GuestFilesystem::TypeRegular; } else { std::error_code failure; #ifdef _WIN32 diff --git a/core/libs/tests/GuestSceV2Filesystem.cpp b/core/libs/tests/GuestSceV2Filesystem.cpp index c1ad0adf24..c9b01d2e74 100644 --- a/core/libs/tests/GuestSceV2Filesystem.cpp +++ b/core/libs/tests/GuestSceV2Filesystem.cpp @@ -1,3 +1,4 @@ +#include "AppArchiveFixture.hpp" #include "prx/libc/include/Api.hpp" #include "prx/libkernel/include/Api.hpp" @@ -64,6 +65,17 @@ void TestStatError() { Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(loop.c_str(), &perms, &error) == TypeNone && error == Loop && perms == 1234, "a symlink loop is not reported as not found"); } +void TestArchivedStat() { + int perms = -1; + int error = -1; + Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix("/app0/Readme.TXT", &perms, &error) == TypeRegular && error == 0 && perms == 0444, "archived file"); + FileStat status{}; + Require(stat_nid_postfix("/app0/Readme.TXT", &status) == 0 && perms == (status.st_mode & 07777), "archived file permissions"); + perms = -1; + Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix("/app0/Data", &perms, &error) == TypeDirectory && error == 0 && perms == 0555, "archived directory"); + Require(stat_nid_postfix("/app0/Data", &status) == 0 && perms == (status.st_mode & 07777), "archived directory permissions"); +} + void TestDirectories() { std::filesystem::create_directory(At("made")); char name[260]; @@ -95,13 +107,16 @@ void TestCurrent() { } int main() { - root = "anyps5-sce-v2-filesystem-" + std::to_string(std::random_device{}()); - std::filesystem::create_directory(root); + const auto directory = AppArchiveFixture::EnterFreshRoot("anyps5-sce-v2-filesystem"); + root = "."; + AppArchiveFixture::WriteArchive(directory / "app0.zar", {{"Readme.TXT", {'r', 'e', 'a', 'd'}}, {"Data/Sub.txt", {'s'}}}); TestStat(); TestStatError(); + TestArchivedStat(); TestDirectories(); TestCurrent(); std::error_code ignored; - std::filesystem::remove_all(root, ignored); + std::filesystem::current_path(std::filesystem::temp_directory_path(), ignored); + std::filesystem::remove_all(directory, ignored); return 0; } diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 0e121fcef9..46283b776d 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -270,3 +270,4 @@ - [expm1f](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions. It is FreeBSD msun's [`s_expm1f.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_expm1f.c) (`Expm1f`, shared with `tanhf`), and sets `ERANGE` only when a finite argument overflows to infinity, as the titles' `libc.prx` does; large negative arguments return -1 without `errno`. It gives the same bits and `errno` as the `expm1f` bundled with PPSA07923 and PPSA05916 for all 2^32 arguments on a Windows host. The FMA-contracted build bundled with PPSA13934 differs by 1 ulp on 197,464 arguments. - [logb and logbf](../../../core/libs/prx/libc/src/Math.cpp) (libc) do not compare floating-point exceptions, and their double inputs are checked by sampling, not exhaustively. They are FreeBSD msun's [`s_logb.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_logb.c) and [`s_logbf.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_logbf.c): ±0 returns `-inf` without `errno`, and a signalling NaN is returned quieted. The `libc.prx` bundled with eleven titles (PPSA04158 to PPSA35169) gives the same bits and no `errno` on 4,032,768 inputs per function on a Windows host. - [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Stat`, `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` use the unversioned Dinkumware filesystem implementation, which resolves guest paths and reads through libkernel's directory calls; the console's version loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows (`_Stat` 0xd7 bytes at 0xcb9b0, `_Open_dir` 0xe9 at 0xcb640, `_Read_dir` 0x15b at 0xcb730, `_Close_dir` 0x2b at 0xcb890, `_Current_get` 8 at 0xcb8c0), and maps the same errors. `_Read_dir` reports file type 8 for every entry, as both do. The console's `_Current_get` writes "/" and returns true instead of reading the current directory, and that is reproduced. The two `_Xfilesystem_error` throwers have no console body yet and call `NotImplemented_nid_no_patch`, as the unversioned pair does; `std::__sce_v2::filesystem_error` is not implemented. +- [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` use the unversioned Dinkumware filesystem implementation, which resolves guest paths and reads through libkernel's directory calls; `_Stat` resolves paths in libc, reports app0.zar files and directories through libc's archive reader with the 0444 and 0555 modes `sceKernelStat` gives them, and reads disk entries with host `std::filesystem::status`, as libc cannot link libkernel. The console's version loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows (`_Stat` 0xd7 bytes at 0xcb9b0, `_Open_dir` 0xe9 at 0xcb640, `_Read_dir` 0x15b at 0xcb730, `_Close_dir` 0x2b at 0xcb890, `_Current_get` 8 at 0xcb8c0), and maps the same errors. `_Read_dir` reports file type 8 for every entry, as both do. The console's `_Current_get` writes "/" and returns true instead of reading the current directory, and that is reproduced. The two `_Xfilesystem_error` throwers have no console body yet and call `NotImplemented_nid_no_patch`, as the unversioned pair does; `std::__sce_v2::filesystem_error` is not implemented. From 497e4a960ea5fdb16cd46c4246e0991ade27c7fa Mon Sep 17 00:00:00 2001 From: Eduardo Ruiz Date: Wed, 28 Oct 2026 08:09:58 +0100 Subject: [PATCH 303/318] fix(libc): return guest errors for invalid versioned stat paths --- core/libs/prx/libc/src/VersionedFilesystem.cpp | 6 +++++- core/libs/tests/GuestSceV2Filesystem.cpp | 10 ++++++++++ docs/dev/debt/libc.md | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libc/src/VersionedFilesystem.cpp b/core/libs/prx/libc/src/VersionedFilesystem.cpp index 33051e826d..ed22464987 100644 --- a/core/libs/prx/libc/src/VersionedFilesystem.cpp +++ b/core/libs/prx/libc/src/VersionedFilesystem.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #ifdef _WIN32 @@ -54,8 +55,11 @@ std::filesystem::path StatusPath(const std::filesystem::path& path) { int StatType(const char* path, int* perms, int* error) { errno = 0; + std::string copy; + int code = CopyGuestPath_nid_no_patch(path, copy); + if (code == 0 && copy.empty()) code = GuestFilesystem::NoSuchFile; std::filesystem::path native; - int code = ResolvePathError_nid_no_patch(path, native, false, true); + if (code == 0) code = ResolvePathError_nid_no_patch(copy.c_str(), native, false, true); int type; if (code != 0) { errno = code; diff --git a/core/libs/tests/GuestSceV2Filesystem.cpp b/core/libs/tests/GuestSceV2Filesystem.cpp index c9b01d2e74..d12d9de1e2 100644 --- a/core/libs/tests/GuestSceV2Filesystem.cpp +++ b/core/libs/tests/GuestSceV2Filesystem.cpp @@ -16,6 +16,7 @@ namespace { constexpr int NoSuchFile = 2; +constexpr int Fault = 14; constexpr int TypeNotFound = -1; constexpr int TypeNone = 0; constexpr int TypeRegular = 1; @@ -52,6 +53,14 @@ void TestStat() { Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(At("missing").c_str(), &perms, &error) == TypeNotFound && error == NoSuchFile && perms == 1234, "missing path"); } +void TestStatInvalidPath() { + int perms = 1234; + int error = -1; + Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix(nullptr, &perms, &error) == TypeNone && error == Fault && perms == 1234, "a null path"); + error = -1; + Require(_ZNSt8__sce_v210filesystem5_StatEPKcPNS0_5permsEPi_nid_postfix("", &perms, &error) == TypeNotFound && error == NoSuchFile && perms == 1234, "an empty path"); +} + void TestStatError() { const auto loop = At("loop"); std::error_code failure; @@ -111,6 +120,7 @@ int main() { root = "."; AppArchiveFixture::WriteArchive(directory / "app0.zar", {{"Readme.TXT", {'r', 'e', 'a', 'd'}}, {"Data/Sub.txt", {'s'}}}); TestStat(); + TestStatInvalidPath(); TestStatError(); TestArchivedStat(); TestDirectories(); diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 46283b776d..baaaecf304 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -271,3 +271,4 @@ - [logb and logbf](../../../core/libs/prx/libc/src/Math.cpp) (libc) do not compare floating-point exceptions, and their double inputs are checked by sampling, not exhaustively. They are FreeBSD msun's [`s_logb.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_logb.c) and [`s_logbf.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_logbf.c): ±0 returns `-inf` without `errno`, and a signalling NaN is returned quieted. The `libc.prx` bundled with eleven titles (PPSA04158 to PPSA35169) gives the same bits and no `errno` on 4,032,768 inputs per function on a Windows host. - [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Stat`, `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` use the unversioned Dinkumware filesystem implementation, which resolves guest paths and reads through libkernel's directory calls; the console's version loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows (`_Stat` 0xd7 bytes at 0xcb9b0, `_Open_dir` 0xe9 at 0xcb640, `_Read_dir` 0x15b at 0xcb730, `_Close_dir` 0x2b at 0xcb890, `_Current_get` 8 at 0xcb8c0), and maps the same errors. `_Read_dir` reports file type 8 for every entry, as both do. The console's `_Current_get` writes "/" and returns true instead of reading the current directory, and that is reproduced. The two `_Xfilesystem_error` throwers have no console body yet and call `NotImplemented_nid_no_patch`, as the unversioned pair does; `std::__sce_v2::filesystem_error` is not implemented. - [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` use the unversioned Dinkumware filesystem implementation, which resolves guest paths and reads through libkernel's directory calls; `_Stat` resolves paths in libc, reports app0.zar files and directories through libc's archive reader with the 0444 and 0555 modes `sceKernelStat` gives them, and reads disk entries with host `std::filesystem::status`, as libc cannot link libkernel. The console's version loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows (`_Stat` 0xd7 bytes at 0xcb9b0, `_Open_dir` 0xe9 at 0xcb640, `_Read_dir` 0x15b at 0xcb730, `_Close_dir` 0x2b at 0xcb890, `_Current_get` 8 at 0xcb8c0), and maps the same errors. `_Read_dir` reports file type 8 for every entry, as both do. The console's `_Current_get` writes "/" and returns true instead of reading the current directory, and that is reproduced. The two `_Xfilesystem_error` throwers have no console body yet and call `NotImplemented_nid_no_patch`, as the unversioned pair does; `std::__sce_v2::filesystem_error` is not implemented. +- [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` reuse the unversioned Dinkumware implementation and libkernel's directory calls. `_Stat` copies the guest path and maps errors as `stat_nid_postfix` does, reports app0.zar entries through libc's archive reader with the 0444/0555 modes `sceKernelStat` gives, and reads disk entries with host `std::filesystem::status`, as libc cannot link libkernel. The console loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows, and maps the same errors. `_Read_dir` reports type 8 for every entry, as both do; the console's `_Current_get` writes "/" and returns true, reproduced here. The `_Xfilesystem_error` throwers call `NotImplemented_nid_no_patch`; `std::__sce_v2::filesystem_error` is not implemented. From 72775e627bc835de30c40b17271c0efad0108021 Mon Sep 17 00:00:00 2001 From: Eduardo Ruiz Date: Fri, 30 Oct 2026 21:26:59 +0100 Subject: [PATCH 304/318] docs(libc): keep a single versioned filesystem debt entry --- docs/dev/debt/libc.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index baaaecf304..92e7f56b07 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -269,6 +269,4 @@ - [tanhf](../../../core/libs/prx/libc/src/Math.cpp) (libc) is FreeBSD msun's [`s_tanhf.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_tanhf.c) over a full port of msun's [`s_expm1f.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_expm1f.c) (`Expm1f`, with `huge` read at run time), which the exported `expm1f` also uses. It sets no `errno`, as the `libc.prx` that titles ship does not. Floating-point exceptions are not compared. It gives the same bits as the `tanhf` of the `libc.prx` bundled with PPSA07923 and PPSA05916 for all 2^32 arguments, run on a Windows host; the FMA-contracted copy of PPSA13934 differs by up to 3 ulp on 150,846 of them. - [expm1f](../../../core/libs/prx/libc/src/Math.cpp) (libc) does not compare floating-point exceptions. It is FreeBSD msun's [`s_expm1f.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_expm1f.c) (`Expm1f`, shared with `tanhf`), and sets `ERANGE` only when a finite argument overflows to infinity, as the titles' `libc.prx` does; large negative arguments return -1 without `errno`. It gives the same bits and `errno` as the `expm1f` bundled with PPSA07923 and PPSA05916 for all 2^32 arguments on a Windows host. The FMA-contracted build bundled with PPSA13934 differs by 1 ulp on 197,464 arguments. - [logb and logbf](../../../core/libs/prx/libc/src/Math.cpp) (libc) do not compare floating-point exceptions, and their double inputs are checked by sampling, not exhaustively. They are FreeBSD msun's [`s_logb.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_logb.c) and [`s_logbf.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_logbf.c): ±0 returns `-inf` without `errno`, and a signalling NaN is returned quieted. The `libc.prx` bundled with eleven titles (PPSA04158 to PPSA35169) gives the same bits and no `errno` on 4,032,768 inputs per function on a Windows host. -- [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Stat`, `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` use the unversioned Dinkumware filesystem implementation, which resolves guest paths and reads through libkernel's directory calls; the console's version loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows (`_Stat` 0xd7 bytes at 0xcb9b0, `_Open_dir` 0xe9 at 0xcb640, `_Read_dir` 0x15b at 0xcb730, `_Close_dir` 0x2b at 0xcb890, `_Current_get` 8 at 0xcb8c0), and maps the same errors. `_Read_dir` reports file type 8 for every entry, as both do. The console's `_Current_get` writes "/" and returns true instead of reading the current directory, and that is reproduced. The two `_Xfilesystem_error` throwers have no console body yet and call `NotImplemented_nid_no_patch`, as the unversioned pair does; `std::__sce_v2::filesystem_error` is not implemented. -- [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` use the unversioned Dinkumware filesystem implementation, which resolves guest paths and reads through libkernel's directory calls; `_Stat` resolves paths in libc, reports app0.zar files and directories through libc's archive reader with the 0444 and 0555 modes `sceKernelStat` gives them, and reads disk entries with host `std::filesystem::status`, as libc cannot link libkernel. The console's version loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows (`_Stat` 0xd7 bytes at 0xcb9b0, `_Open_dir` 0xe9 at 0xcb640, `_Read_dir` 0x15b at 0xcb730, `_Close_dir` 0x2b at 0xcb890, `_Current_get` 8 at 0xcb8c0), and maps the same errors. `_Read_dir` reports file type 8 for every entry, as both do. The console's `_Current_get` writes "/" and returns true instead of reading the current directory, and that is reproduced. The two `_Xfilesystem_error` throwers have no console body yet and call `NotImplemented_nid_no_patch`, as the unversioned pair does; `std::__sce_v2::filesystem_error` is not implemented. - [Versioned std::__sce_v2::filesystem primitives](../../../core/libs/prx/libc/src/VersionedFilesystem.cpp) (libc) - `_Open_dir`, `_Read_dir`, `_Close_dir` and `_Current_get` reuse the unversioned Dinkumware implementation and libkernel's directory calls. `_Stat` copies the guest path and maps errors as `stat_nid_postfix` does, reports app0.zar entries through libc's archive reader with the 0444/0555 modes `sceKernelStat` gives, and reads disk entries with host `std::filesystem::status`, as libc cannot link libkernel. The console loops over `sceKernelGetdents` itself, as the `libc.prx` bundled with PPSA28416 shows, and maps the same errors. `_Read_dir` reports type 8 for every entry, as both do; the console's `_Current_get` writes "/" and returns true, reproduced here. The `_Xfilesystem_error` throwers call `NotImplemented_nid_no_patch`; `std::__sce_v2::filesystem_error` is not implemented. From 57d601ffb8b16ea6eaf403ef22e95541c25c4ec4 Mon Sep 17 00:00:00 2001 From: liu <58376171+lgqdaxia@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:01:21 +0200 Subject: [PATCH 305/318] fix(agc): wait when the capture trace queue is full Wait for the writer to take a batch instead of throwing when the pending queue is full. Add a controlled-sink test with four producers and check that records stay complete and ordered. AI-assisted: yes --- .../Execution/include/CaptureTrace.hpp | 8 +- .../libSceAgcDriver/tests/CaptureTrace.cpp | 187 ++++++++++++++++++ .../tests/cmake/agc_capture_trace_tests.cmake | 6 +- docs/dev/debt/libSceAgcDriver.md | 2 + 4 files changed, 200 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp index 501c86b224..972db74ac5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp @@ -45,6 +45,8 @@ public: } private: + friend struct CaptureTraceTestAccess; + static CaptureTrace& instance() { static CaptureTrace trace; return trace; @@ -73,8 +75,8 @@ private: void append(std::string line) { if (!output.is_open()) return; const auto elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); - std::lock_guard lock(mutex); - if (pending.size() + line.size() + 64 > 16 * 1024 * 1024) throw std::runtime_error("Capture trace queue exceeded 16 MiB"); + std::unique_lock lock(mutex); + spaceAvailable.wait(lock, [this, &line] { return pending.size() + line.size() + 64 <= 16 * 1024 * 1024; }); pending += std::to_string(++sequence) + " " + std::to_string(elapsed) + "us " + line + "\n"; if (line.starts_with("blit ")) flushRequested = true; if (pending.size() >= 65536 || flushRequested) changed.notify_one(); @@ -91,6 +93,7 @@ private: batch.swap(pending); flushRequested = false; } + spaceAvailable.notify_all(); if (batch.empty()) continue; output.write(batch.data(), static_cast(batch.size())); output.flush(); @@ -104,6 +107,7 @@ private: std::ofstream output; std::mutex mutex; std::condition_variable changed; + std::condition_variable spaceAvailable; std::string pending; std::thread worker; std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); diff --git a/core/libs/prx/libSceAgcDriver/tests/CaptureTrace.cpp b/core/libs/prx/libSceAgcDriver/tests/CaptureTrace.cpp index b263f9e7b1..b7cecec908 100644 --- a/core/libs/prx/libSceAgcDriver/tests/CaptureTrace.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/CaptureTrace.cpp @@ -4,15 +4,29 @@ #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" +#include #include +#include #include #include +#include #include #include #include +#include #include +#include #include #include +#include + +namespace AgcDriver { + +struct CaptureTraceTestAccess { + static void QueueBackpressure(); +}; + +} namespace { @@ -127,6 +141,52 @@ void BestEffortOverflow() { std::remove(configured); } +class PausedTraceBuffer final : public std::streambuf { +public: + bool WaitForWrite() { + std::unique_lock lock(mutex); + return changed.wait_for(lock, std::chrono::seconds(2), [this] { return entered; }); + } + + void Release() { + { + std::lock_guard lock(mutex); + released = true; + } + changed.notify_all(); + } + + std::string Contents() { + std::lock_guard lock(mutex); + return contents; + } + +protected: + std::streamsize xsputn(const char* text, std::streamsize size) override { + std::unique_lock lock(mutex); + entered = true; + changed.notify_all(); + changed.wait(lock, [this] { return released; }); + contents.append(text, static_cast(size)); + return size; + } + + int sync() override { + return 0; + } + +private: + std::mutex mutex; + std::condition_variable changed; + std::string contents; + bool entered = false; + bool released = false; +}; + +std::string QueueRecord(std::size_t producer, std::size_t item) { + return "queue producer=" + std::to_string(producer) + " item=" + std::to_string(item) + " payload=" + std::string(1900, 'x'); +} + void Disabled() { RemoveEnvironmentVariable("APS5_CAPTURE_TRACE"); Check(!AgcDriver::CaptureTrace::Enabled(), @@ -138,6 +198,132 @@ void Disabled() { } +void AgcDriver::CaptureTraceTestAccess::QueueBackpressure() { + constexpr std::size_t Capacity = 16u * 1024u * 1024u; + constexpr std::size_t Producers = 4; + constexpr std::size_t ItemsPerProducer = 64; + const char* configured = RequiredPathVariable(); + PausedTraceBuffer sink; + std::exception_ptr failure; + std::size_t initialItems = 0; + std::atomic bounded = true; + { + CaptureTrace trace; + Check(trace.output.is_open(), "the queue test trace path must be writable"); + auto& output = static_cast(trace.output); + auto* original = output.rdbuf(&sink); + std::vector producers; + std::vector producerErrors(Producers); + std::mutex stateMutex; + std::condition_variable stateChanged; + std::size_t attempted = 0; + std::size_t finished = 0; + auto append = [&](std::size_t producer, std::size_t item) { + trace.append(QueueRecord(producer, item)); + std::lock_guard lock(trace.mutex); + if (trace.pending.size() > Capacity) bounded.store(false); + }; + try { + for (; initialItems < 64; ++initialItems) append(0, initialItems); + Check(sink.WaitForWrite(), "the trace worker did not reach the paused sink"); + for (;;) { + const auto line = QueueRecord(0, initialItems); + { + std::lock_guard lock(trace.mutex); + if (trace.pending.size() + line.size() + 64 > Capacity) break; + } + append(0, initialItems++); + } + for (std::size_t producer = 0; producer < Producers; ++producer) { + producers.emplace_back([&, producer] { + { + std::lock_guard lock(stateMutex); + ++attempted; + } + stateChanged.notify_all(); + try { + for (std::size_t item = 0; item < ItemsPerProducer; ++item) append(producer + 1, item); + } catch (...) { + producerErrors[producer] = std::current_exception(); + } + { + std::lock_guard lock(stateMutex); + ++finished; + } + stateChanged.notify_all(); + }); + } + { + std::unique_lock lock(stateMutex); + Check(stateChanged.wait_for(lock, std::chrono::seconds(2), [&] { return attempted == Producers; }), + "the queue producers did not start"); + Check(!stateChanged.wait_for(lock, std::chrono::milliseconds(200), [&] { return finished != 0; }), + "a full capture queue must wait for its sink instead of failing or growing"); + } + sink.Release(); + { + std::unique_lock lock(stateMutex); + Check(stateChanged.wait_for(lock, std::chrono::seconds(5), [&] { return finished == Producers; }), + "the queue producers did not resume after the sink recovered"); + } + } catch (...) { + failure = std::current_exception(); + } + sink.Release(); + if (failure) { + std::lock_guard lock(trace.mutex); + trace.pending.clear(); + } + trace.spaceAvailable.notify_all(); + for (auto& producer : producers) producer.join(); + { + std::lock_guard lock(trace.mutex); + trace.stopping = true; + } + trace.changed.notify_one(); + trace.worker.join(); + output.rdbuf(original); + if (!failure) { + for (const auto& error : producerErrors) { + if (error) { + failure = error; + break; + } + } + } + } + if (failure) std::rethrow_exception(failure); + Check(bounded.load(), "the pending capture queue exceeded 16 MiB"); + std::istringstream records(sink.Contents()); + std::vector nextItem(Producers + 1, 0); + std::string record; + std::size_t sequence = 0; + while (std::getline(records, record)) { + std::istringstream fields(record); + unsigned long long foundSequence = 0; + std::string elapsed; + std::string label; + std::string producerField; + std::string itemField; + fields >> foundSequence >> elapsed >> label >> producerField >> itemField; + Check(foundSequence == ++sequence, "the recovered trace lost or reordered sequence numbers"); + Check(label == "queue" && producerField.starts_with("producer=") && itemField.starts_with("item="), + "the recovered trace contains an incomplete record"); + const auto producer = std::stoull(producerField.substr(9)); + const auto item = std::stoull(itemField.substr(5)); + Check(producer <= Producers, "the recovered trace contains an unexpected producer"); + Check(item == nextItem[producer]++, "the recovered trace lost or reordered a producer's records"); + const auto prefix = std::to_string(foundSequence) + " " + elapsed + " "; + Check(record == prefix + QueueRecord(producer, item), "the recovered trace changed a record's payload"); + } + Check(sequence == initialItems + Producers * ItemsPerProducer, "the recovered trace did not drain every queued record"); + Check(nextItem[0] == initialItems, "the recovered trace lost the initial queued records"); + for (std::size_t producer = 1; producer <= Producers; ++producer) { + Check(nextItem[producer] == ItemsPerProducer, "the recovered trace lost a producer's queued records"); + } + std::remove(configured); +} + int main(int argc, char** argv) { try { if (argc != 2) throw std::runtime_error("expected exactly one test mode argument"); @@ -149,6 +335,7 @@ int main(int argc, char** argv) { else if (mode == "default-writes") DefaultWrites(); else if (mode == "open-failure") OpenFailure(); else if (mode == "best-effort-overflow") BestEffortOverflow(); + else if (mode == "queue-backpressure") AgcDriver::CaptureTraceTestAccess::QueueBackpressure(); else if (mode == "disabled") Disabled(); else throw std::runtime_error("unknown capture trace test mode: " + mode); std::cout << "capture trace tests passed\n"; diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake index 5edb1c481e..3f00d9ef95 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake @@ -8,10 +8,12 @@ if(BUILD_TESTING) file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/explicit") file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/failure") file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/best-effort") + file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/queue") file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/disabled") set(agcCaptureTraceExplicitPath "${agcCaptureTraceDirectory}/explicit/trace.log") set(agcCaptureTraceFailurePath "${agcCaptureTraceDirectory}/failure/missing-parent/trace.log") set(agcCaptureTraceBestEffortPath "${agcCaptureTraceDirectory}/best-effort/trace.log") + set(agcCaptureTraceQueuePath "${agcCaptureTraceDirectory}/queue/trace.log") add_test(NAME agc_capture_trace_path_default COMMAND agc_capture_trace_tests path-default) add_test(NAME agc_capture_trace_path_empty COMMAND agc_capture_trace_tests path-empty) set_tests_properties(agc_capture_trace_path_empty PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE_PATH=") @@ -25,12 +27,14 @@ if(BUILD_TESTING) set_tests_properties(agc_capture_trace_open_failure PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceFailurePath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/failure") add_test(NAME agc_capture_trace_best_effort_overflow COMMAND agc_capture_trace_tests best-effort-overflow) set_tests_properties(agc_capture_trace_best_effort_overflow PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceBestEffortPath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/best-effort") + add_test(NAME agc_capture_trace_queue_backpressure COMMAND agc_capture_trace_tests queue-backpressure) + set_tests_properties(agc_capture_trace_queue_backpressure PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceQueuePath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/queue" TIMEOUT 15) add_test(NAME agc_capture_trace_disabled COMMAND agc_capture_trace_tests disabled) set_tests_properties(agc_capture_trace_disabled PROPERTIES WORKING_DIRECTORY "${agcCaptureTraceDirectory}/disabled") if(WIN32) set_tests_properties(agc_capture_trace_path_default agc_capture_trace_path_empty agc_capture_trace_path_explicit agc_capture_trace_explicit_writes - agc_capture_trace_default_writes agc_capture_trace_open_failure agc_capture_trace_best_effort_overflow + agc_capture_trace_default_writes agc_capture_trace_open_failure agc_capture_trace_best_effort_overflow agc_capture_trace_queue_backpressure agc_capture_trace_disabled PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 4f9a9a69f8..3d0702281b 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -114,6 +114,8 @@ - [Written staging window on discrete GPUs](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) - a written buffer element of a dispatch that is not swept is staged device-local up to 8 MiB on a discrete GPU (`Context::discreteGpu`) and up to 2 MiB otherwise; `APS5_WRITTEN_SHADOW_MAX_KIB` replaces either bound. The 8 MiB figure comes from one title on one NVIDIA Windows system (PPSA09911: a 2.5 and a 3.3 MiB skin-cache output written in place cost 55 ms of GPU time per frame, 1.8 ms of extra staging copies once staged); the point where the two copies of a larger element cost more than its in-place stores, and whether AMD or Linux drivers move it, has not been measured. The window is per element and per use, so a kernel that streams once through an element inside the window pays the copy-in and the copy-back on every dispatch. - [FS table calls on macOS](../../../core/libs/prx/libSceAgcDriver/Execution/include/Driver/DriverMemory.hpp) (libSceAgcDriver) - macOS keeps the console's 0xfe0000000 for itself, so the driver block, and the FS table at 0xfe0040000 inside it, moves to 0xee0000000 there; `sceAgc*StaticBuffer` and `AgcDriverStaticTable_nid_postfix` use the moved address. A shader that builds the table address from two `s_mov_b32` literals still loads from 0xfe0040000: `ReadTableCallees` (`Driver/Shaders/ShaderRegistry.cpp`) reads that address unchanged, and the shader also loads from it at run time, so those dispatches cannot link on macOS. #6160 lists titles that build this address. `ComputeFsTableCalls` is skipped on macOS as it is under `ANYPS5_ENABLE_ASAN`. Closes when: the driver rewrites that pointer in the shader code for both the callee lookup and the code the GPU runs, or the console address can be mapped on macOS. +- [Capture trace output](../../../core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp) can block logging callers and normal shutdown while its output is stalled. The pending queue waits at 16 MiB instead of discarding records; the batch being written can hold another 16 MiB. File write failures retain the fatal path. Permanently blocked file I/O has no timeout. + - [Frame capture](../../../core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp) (libSceAgcDriver) throws on macOS. Section views, aliases of one direct memory block and file views replay as separate private memory, so a store through one alias is not seen through another. Only write-watched memory records CPU stores during the frames: private memory on Windows, where a store through one alias is recorded only at that alias; registered ranges on Linux, including direct memory unless mapped more than once (libkernel leaves those views unwatched, so no store through any of them is recorded). Title image pages are recorded only when changed and holding a shader header or its arrays. Linux omits no-access pages (Windows records them unreadable) and records file mapping pages past the file's end as unreadable. A capture starts only at a frame boundary where the driver idles within 10 s (five tries), and stops when GPU work in flight at a delta waits on a later CPU store. - [Frame replay](../../../core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp) (libSceAgcDriver) orders the calls of different captured threads only by their start (a call starts once every earlier call has started), so the driver sees a schedule the title could have produced, not necessarily the one it did. CPU stores a title made from threads that never call the driver are known only by the next recorded call, so the replay applies them early or late when a GPU wait needs them and derives a value from the wait's reference when no recorded store satisfies it (`of_them_synthesized`). Pacing compares the number of submissions in flight, not what they hold. Prologue registrations of shaders whose memory the title reused after registering them fail and are counted (`stale_registrations`). The title's image range must be free in the replaying process, so the tool is linked at a high image base on Windows. - [Host-only multisample memory in frame capture](../../../core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp) (libSceAgcDriver) - memory under multisampled color targets, their FMASK and native multisample depth whose samples are only on the host is zero in the snapshot and in deltas (`host_only_sample_bytes`, with a warning), so replay starts those targets without the contents earlier frames left in them. From 537377688eb4129380fc73beb3a55df6c2daa136 Mon Sep 17 00:00:00 2001 From: liu <58376171+lgqdaxia@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:41:51 +0200 Subject: [PATCH 306/318] test(agc): cover fatal capture trace write failures --- .../tests/CaptureTraceFailure.cmake | 54 +++++ .../tests/CaptureTraceFailure.cpp | 211 ++++++++++++++++++ .../tests/cmake/agc_capture_trace_tests.cmake | 15 ++ 3 files changed, 280 insertions(+) create mode 100644 core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake create mode 100644 core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake new file mode 100644 index 0000000000..14444c542a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake @@ -0,0 +1,54 @@ +foreach(required IN ITEMS FIXTURE MODE OUTPUT_ROOT) + if(NOT DEFINED ${required} OR "${${required}}" STREQUAL "") + message(FATAL_ERROR "Missing ${required}") + endif() +endforeach() +if(NOT MODE STREQUAL "write" AND NOT MODE STREQUAL "flush") + message(FATAL_ERROR "Expected write or flush mode, got ${MODE}") +endif() +string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef runId) +set(runDirectory "${OUTPUT_ROOT}/capture-trace-${MODE}-${runId}") +file(MAKE_DIRECTORY "${runDirectory}") +execute_process( + COMMAND "${FIXTURE}" control-abort + WORKING_DIRECTORY "${runDirectory}" + RESULT_VARIABLE abortResult OUTPUT_VARIABLE abortOutput ERROR_VARIABLE abortError TIMEOUT 8 +) +file(WRITE "${runDirectory}/control.log" "result=${abortResult}\n${abortOutput}\n${abortError}") +if(abortResult STREQUAL "0" OR abortResult STREQUAL "87" OR abortResult STREQUAL "99" + OR NOT abortError MATCHES "capture-trace-failure control-abort" + OR abortError MATCHES "setup-failed|watchdog") + message(FATAL_ERROR "Abort control failed: ${abortResult}\n${abortOutput}\n${abortError}") +endif() +if(WIN32) + if(NOT abortResult STREQUAL "3") + message(FATAL_ERROR "The Windows abort control returned an unexpected result: ${abortResult}") + endif() +elseif(NOT abortResult MATCHES "[Aa]bort") + message(FATAL_ERROR "The abort control did not report an abort signal: ${abortResult}") +endif() +set(ENV{APS5_CAPTURE_TRACE} "1") +set(ENV{APS5_CAPTURE_TRACE_PATH} "${runDirectory}/trace.log") +execute_process( + COMMAND "${FIXTURE}" "${MODE}" + WORKING_DIRECTORY "${runDirectory}" + RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error TIMEOUT 8 +) +file(WRITE "${runDirectory}/failure.log" "result=${result}\n${output}\n${error}") +set(armed "capture-trace-failure armed mode=${MODE} queueFull=1 attempted=4 finished=0") +if(MODE STREQUAL "write") + set(sinkFailure "capture-trace-failure sink-short-write") +else() + set(sinkFailure "capture-trace-failure sink-flush-failed") +endif() +set(terminate "capture-trace-failure terminate armed=1 queueFull=1 attempted=4 finished=0 sinkFailed=1") +string(FIND "${error}" "${armed}" armedAt) +string(FIND "${error}" "${sinkFailure}" sinkAt) +string(FIND "${error}" "Capture trace failed:" caughtAt) +string(FIND "${error}" "${terminate}" terminateAt) +if(NOT result STREQUAL abortResult OR result STREQUAL "87" OR result STREQUAL "99" + OR error MATCHES "setup-failed|watchdog|handler-returned|handler-threw" + OR armedAt LESS 0 OR NOT sinkAt GREATER armedAt OR NOT caughtAt GREATER sinkAt OR NOT terminateAt GREATER caughtAt) + message(FATAL_ERROR "Capture trace ${MODE} failure did not reach its saved terminate handler and abort: ${result}\n${output}\n${error}") +endif() +message(STATUS "Capture trace ${MODE} failure reached its saved terminate handler and aborted with four waiting producers") diff --git a/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cpp b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cpp new file mode 100644 index 0000000000..c48c1f80d8 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cpp @@ -0,0 +1,211 @@ +#include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#endif +#ifdef __linux__ +#include +#endif + +namespace AgcDriver { + +struct CaptureTraceTestAccess { + [[noreturn]] static void WriterFailure(bool flush); +}; + +} + +namespace { + +using namespace std::chrono_literals; + +std::atomic armed{false}; +std::atomic queueFull{false}; +std::atomic sinkFailed{false}; +std::atomic attempted{0}; +std::atomic finished{0}; +std::terminate_handler originalTerminate = nullptr; + +[[noreturn]] void SetupFailure(const char* message) { + std::fprintf(stderr, "capture-trace-failure setup-failed: %s\n", message); + std::fflush(stderr); + std::_Exit(87); +} + +void Check(bool condition, const char* message) { + if (!condition) SetupFailure(message); +} + +[[noreturn]] void SavedTerminate() { + std::fprintf(stderr, "capture-trace-failure terminate armed=%d queueFull=%d attempted=%u finished=%u sinkFailed=%d\n", + armed.load() ? 1 : 0, queueFull.load() ? 1 : 0, attempted.load(), finished.load(), sinkFailed.load() ? 1 : 0); + std::fflush(stderr); + try { + originalTerminate(); + std::fputs("capture-trace-failure handler-returned\n", stderr); + } catch (...) { + std::fputs("capture-trace-failure handler-threw\n", stderr); + } + std::fflush(stderr); + std::abort(); +} + +class FailingBuffer final : public std::streambuf { +public: + explicit FailingBuffer(bool failFlush) : failFlush(failFlush) {} + + bool WaitForWriter() { + std::unique_lock lock(mutex); + return changed.wait_for(lock, 2s, [this] { return entered; }); + } + + void Fail() { + { + std::lock_guard lock(mutex); + released = true; + } + changed.notify_all(); + } + +protected: + std::streamsize xsputn(const char*, std::streamsize size) override { + if (failFlush) return size; + wait(); + sinkFailed.store(true); + std::fputs("capture-trace-failure sink-short-write\n", stderr); + std::fflush(stderr); + return 0; + } + + int sync() override { + if (!failFlush) return 0; + wait(); + sinkFailed.store(true); + std::fputs("capture-trace-failure sink-flush-failed\n", stderr); + std::fflush(stderr); + return -1; + } + +private: + void wait() { + std::unique_lock lock(mutex); + entered = true; + changed.notify_all(); + changed.wait(lock, [this] { return released; }); + } + + std::mutex mutex; + std::condition_variable changed; + const bool failFlush; + bool entered = false; + bool released = false; +}; + +std::string Record(unsigned producer, unsigned item) { + char prefix[64]; + std::snprintf(prefix, sizeof(prefix), "failure producer=%02u item=%08u payload=", producer, item); + return std::string(prefix) + std::string(1900, 'x'); +} + +} + +[[noreturn]] void AgcDriver::CaptureTraceTestAccess::WriterFailure(bool flush) { + constexpr std::size_t Capacity = 16u * 1024u * 1024u; + FailingBuffer sink(flush); + CaptureTrace trace; + Check(trace.output.is_open(), "the trace output did not open"); + static_cast(trace.output).rdbuf(&sink); + unsigned item = 0; + for (; item < 64; ++item) trace.append(Record(0, item)); + Check(sink.WaitForWriter(), "the writer did not reach the failing sink"); + const auto line = Record(1, 0); + for (;;) { + { + std::lock_guard lock(trace.mutex); + if (trace.pending.size() + line.size() + 64 > Capacity) break; + } + trace.append(Record(0, item++)); + } + std::vector producers; + for (unsigned producer = 1; producer <= 4; ++producer) { + producers.emplace_back([&, producer] { + attempted.fetch_add(1); + try { + trace.append(Record(producer, 0)); + } catch (...) { + SetupFailure("a producer threw before the writer failed"); + } + finished.fetch_add(1); + }); + } + const auto deadline = std::chrono::steady_clock::now() + 2s; + while (attempted.load() != 4 && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(1ms); + Check(attempted.load() == 4, "the four producers did not start"); + std::this_thread::sleep_for(200ms); + Check(finished.load() == 0, "a producer passed the full queue before the sink was released"); + { + std::lock_guard lock(trace.mutex); + Check(trace.pending.size() + line.size() + 64 > Capacity, "the producers were not held behind a full queue"); + Check(trace.pending.size() <= Capacity, "the queue exceeded its bound"); + queueFull.store(true); + } + armed.store(true); + std::fprintf(stderr, "capture-trace-failure armed mode=%s queueFull=1 attempted=4 finished=0\n", flush ? "flush" : "write"); + std::fflush(stderr); + sink.Fail(); + for (auto& producer : producers) producer.join(); + SetupFailure("all producers returned after a fatal writer error"); +} + +int main(int argc, char** argv) { +#ifdef _WIN32 + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); +#else + const rlimit noCore{0, 0}; + setrlimit(RLIMIT_CORE, &noCore); +#endif +#ifdef __linux__ + prctl(PR_SET_DUMPABLE, 0); +#endif + std::thread([] { + std::this_thread::sleep_for(6s); + std::fputs("capture-trace-failure watchdog\n", stderr); + std::fflush(stderr); + std::_Exit(99); + }).detach(); + Check(argc == 2, "expected write, flush or control-abort mode"); + const std::string mode = argv[1]; + Check(mode == "write" || mode == "flush" || mode == "control-abort", "unknown failure mode"); + if (mode == "control-abort") { + std::fputs("capture-trace-failure control-abort\n", stderr); + std::fflush(stderr); + std::abort(); + } + Check(AgcDriver::CaptureTrace::Enabled(), "capture tracing must be enabled"); + const char* path = std::getenv("APS5_CAPTURE_TRACE_PATH"); + Check(path != nullptr && path[0] != '\0', "the test requires an explicit trace path"); + originalTerminate = std::set_terminate(&SavedTerminate); + Check(originalTerminate != nullptr, "the runtime did not return its original terminate handler"); + AgcDriver::CaptureTraceTestAccess::WriterFailure(mode == "flush"); +} diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake index 3f00d9ef95..94b5c92864 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake @@ -38,4 +38,19 @@ if(BUILD_TESTING) agc_capture_trace_disabled PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + + add_test_executable(agc_capture_trace_failure_tests tests/CaptureTraceFailure.cpp) + target_include_directories(agc_capture_trace_failure_tests PRIVATE ${LIBS_INCLUDE_DIR}) + if(MINGW) + target_compile_options(agc_capture_trace_failure_tests PRIVATE -fasynchronous-unwind-tables) + endif() + foreach(mode IN ITEMS write flush) + add_test(NAME agc_capture_trace_${mode}_failure + COMMAND "${CMAKE_COMMAND}" + "-DFIXTURE=$" + "-DMODE=${mode}" + "-DOUTPUT_ROOT=${agcCaptureTraceDirectory}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/CaptureTraceFailure.cmake") + set_tests_properties(agc_capture_trace_${mode}_failure PROPERTIES TIMEOUT 20) + endforeach() endif() From 545143e3780c2f297628a5b05450a5d94324074b Mon Sep 17 00:00:00 2001 From: liu Date: Sun, 25 Oct 2026 21:54:31 +0100 Subject: [PATCH 307/318] docs(agc): keep capture debt entries in one list Remove the extra blank line between capture trace output and frame capture. The independent changes in #11157 may merge in either order; the second branch keeps both entries when rebasing. AI-assisted: Codex --- docs/dev/debt/libSceAgcDriver.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 3d0702281b..a6f09aea6b 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -115,7 +115,6 @@ - [Written staging window on discrete GPUs](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) - a written buffer element of a dispatch that is not swept is staged device-local up to 8 MiB on a discrete GPU (`Context::discreteGpu`) and up to 2 MiB otherwise; `APS5_WRITTEN_SHADOW_MAX_KIB` replaces either bound. The 8 MiB figure comes from one title on one NVIDIA Windows system (PPSA09911: a 2.5 and a 3.3 MiB skin-cache output written in place cost 55 ms of GPU time per frame, 1.8 ms of extra staging copies once staged); the point where the two copies of a larger element cost more than its in-place stores, and whether AMD or Linux drivers move it, has not been measured. The window is per element and per use, so a kernel that streams once through an element inside the window pays the copy-in and the copy-back on every dispatch. - [FS table calls on macOS](../../../core/libs/prx/libSceAgcDriver/Execution/include/Driver/DriverMemory.hpp) (libSceAgcDriver) - macOS keeps the console's 0xfe0000000 for itself, so the driver block, and the FS table at 0xfe0040000 inside it, moves to 0xee0000000 there; `sceAgc*StaticBuffer` and `AgcDriverStaticTable_nid_postfix` use the moved address. A shader that builds the table address from two `s_mov_b32` literals still loads from 0xfe0040000: `ReadTableCallees` (`Driver/Shaders/ShaderRegistry.cpp`) reads that address unchanged, and the shader also loads from it at run time, so those dispatches cannot link on macOS. #6160 lists titles that build this address. `ComputeFsTableCalls` is skipped on macOS as it is under `ANYPS5_ENABLE_ASAN`. Closes when: the driver rewrites that pointer in the shader code for both the callee lookup and the code the GPU runs, or the console address can be mapped on macOS. - [Capture trace output](../../../core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp) can block logging callers and normal shutdown while its output is stalled. The pending queue waits at 16 MiB instead of discarding records; the batch being written can hold another 16 MiB. File write failures retain the fatal path. Permanently blocked file I/O has no timeout. - - [Frame capture](../../../core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp) (libSceAgcDriver) throws on macOS. Section views, aliases of one direct memory block and file views replay as separate private memory, so a store through one alias is not seen through another. Only write-watched memory records CPU stores during the frames: private memory on Windows, where a store through one alias is recorded only at that alias; registered ranges on Linux, including direct memory unless mapped more than once (libkernel leaves those views unwatched, so no store through any of them is recorded). Title image pages are recorded only when changed and holding a shader header or its arrays. Linux omits no-access pages (Windows records them unreadable) and records file mapping pages past the file's end as unreadable. A capture starts only at a frame boundary where the driver idles within 10 s (five tries), and stops when GPU work in flight at a delta waits on a later CPU store. - [Frame replay](../../../core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp) (libSceAgcDriver) orders the calls of different captured threads only by their start (a call starts once every earlier call has started), so the driver sees a schedule the title could have produced, not necessarily the one it did. CPU stores a title made from threads that never call the driver are known only by the next recorded call, so the replay applies them early or late when a GPU wait needs them and derives a value from the wait's reference when no recorded store satisfies it (`of_them_synthesized`). Pacing compares the number of submissions in flight, not what they hold. Prologue registrations of shaders whose memory the title reused after registering them fail and are counted (`stale_registrations`). The title's image range must be free in the replaying process, so the tool is linked at a high image base on Windows. - [Host-only multisample memory in frame capture](../../../core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp) (libSceAgcDriver) - memory under multisampled color targets, their FMASK and native multisample depth whose samples are only on the host is zero in the snapshot and in deltas (`host_only_sample_bytes`, with a warning), so replay starts those targets without the contents earlier frames left in them. From 03267ec5aa69624cf0cfe1b99a843593b3b0ebe6 Mon Sep 17 00:00:00 2001 From: liu Date: Mon, 26 Oct 2026 00:22:40 +0100 Subject: [PATCH 308/318] fix(agc): drop excess capture records and stop failed tracing Follow the capture policy requested in #10753 and #10760. Keep queue callers running, record loss markers and totals, and disable failed output without relying on stream exceptions. AI-assisted: Codex --- .../Execution/include/CaptureTrace.hpp | 92 +++++-- .../libSceAgcDriver/tests/CaptureTrace.cpp | 229 +++++++++++------- .../tests/CaptureTraceFailure.cmake | 39 +-- .../tests/CaptureTraceFailure.cpp | 206 ++++++++-------- .../tests/cmake/agc_capture_trace_tests.cmake | 33 ++- docs/dev/debt/libSceAgcDriver.md | 2 +- 6 files changed, 366 insertions(+), 235 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp index 972db74ac5..308f4fd998 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -31,8 +30,11 @@ public: if (!Enabled()) return; std::array line{}; const auto size = std::snprintf(line.data(), line.size(), format, args...); - if (size < 0 || static_cast(size) >= line.size()) throw std::runtime_error("Capture trace event exceeds 2047 bytes"); static CaptureTrace& trace = instance(); + if (size < 0 || static_cast(size) >= line.size()) { + trace.drop(); + return; + } trace.append(std::string(line.data(), static_cast(size))); } @@ -59,7 +61,7 @@ private: std::fprintf(stderr, "Capture trace open failed for %s\n", path); return; } - output.exceptions(std::ios::badbit | std::ios::failbit); + active = true; worker = std::thread([this] { run(); }); } @@ -73,45 +75,85 @@ private: } void append(std::string line) { - if (!output.is_open()) return; - const auto elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); - std::unique_lock lock(mutex); - spaceAvailable.wait(lock, [this, &line] { return pending.size() + line.size() + 64 <= 16 * 1024 * 1024; }); - pending += std::to_string(++sequence) + " " + std::to_string(elapsed) + "us " + line + "\n"; + std::lock_guard lock(mutex); + if (!active || stopping) return; + if (droppedPending != 0 || pending.size() + line.size() + 64 > 16 * 1024 * 1024) { + ++droppedPending; + ++droppedTotal; + return; + } + appendRecord(pending, line); if (line.starts_with("blit ")) flushRequested = true; if (pending.size() >= 65536 || flushRequested) changed.notify_one(); } + void drop() { + std::lock_guard lock(mutex); + if (!active || stopping) return; + ++droppedPending; + ++droppedTotal; + } + + void appendRecord(std::string& destination, const std::string& line) { + const auto elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); + destination += std::to_string(++sequence) + " " + std::to_string(elapsed) + "us " + line + "\n"; + } + + void fail(const char* reason) { + unsigned long long dropped; + { + std::lock_guard lock(mutex); + active = false; + pending.clear(); + dropped = droppedTotal; + } + std::fprintf(stderr, "Capture trace failed: %s (dropped records: %llu)\n", reason, dropped); + } + void run() { - try { - for (;;) { - std::string batch; - { - std::unique_lock lock(mutex); - changed.wait_for(lock, std::chrono::milliseconds(100), [this] { return stopping || flushRequested || pending.size() >= 65536; }); - if (stopping && pending.empty()) return; - batch.swap(pending); - flushRequested = false; + for (;;) { + std::string batch; + bool stop; + { + std::unique_lock lock(mutex); + changed.wait_for(lock, std::chrono::milliseconds(100), [this] { return stopping || flushRequested || pending.size() >= 65536; }); + batch.swap(pending); + if (droppedPending != 0) { + appendRecord(batch, "capture-trace-dropped records=" + std::to_string(droppedPending)); + droppedPending = 0; + } + flushRequested = false; + stop = stopping; + if (stop) { + appendRecord(batch, "capture-trace-summary dropped=" + std::to_string(droppedTotal)); + active = false; } - spaceAvailable.notify_all(); - if (batch.empty()) continue; - output.write(batch.data(), static_cast(batch.size())); - output.flush(); } - } catch (const std::exception& error) { - std::fprintf(stderr, "Capture trace failed: %s\n", error.what()); - std::terminate(); + if (batch.empty()) continue; + output.write(batch.data(), static_cast(batch.size())); + if (!output) { + fail("write failed"); + return; + } + output.flush(); + if (!output) { + fail("flush failed"); + return; + } + if (stop) return; } } std::ofstream output; std::mutex mutex; std::condition_variable changed; - std::condition_variable spaceAvailable; std::string pending; std::thread worker; std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); unsigned long long sequence = 0; + unsigned long long droppedPending = 0; + unsigned long long droppedTotal = 0; + bool active = false; bool stopping = false; bool flushRequested = false; }; diff --git a/core/libs/prx/libSceAgcDriver/tests/CaptureTrace.cpp b/core/libs/prx/libSceAgcDriver/tests/CaptureTrace.cpp index b7cecec908..9d74b2a5a8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/CaptureTrace.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/CaptureTrace.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -23,7 +24,7 @@ namespace AgcDriver { struct CaptureTraceTestAccess { - static void QueueBackpressure(); + static void QueueDrop(bool shutdownOnly, bool cumulative = false); }; } @@ -141,6 +142,20 @@ void BestEffortOverflow() { std::remove(configured); } +void FormatOverflow() { + const char* configured = RequiredPathVariable(); + const std::string oversized(4096, 'x'); + AgcDriver::CaptureTrace::Log("capture-trace-format payload=%s", oversized.c_str()); + Check(WaitForRecord(configured, "capture-trace-dropped records=1\n"), + "an oversized record must be counted instead of throwing or writing a partial record"); + AgcDriver::CaptureTrace::Log("%s", "capture-trace-format-recovered"); + Check(WaitForRecord(configured, "capture-trace-format-recovered\n"), + "capture tracing did not recover after an oversized record"); + Check(ReadFile(configured).find("capture-trace-format payload=") == std::string::npos, + "the trace contains a truncated oversized record"); + std::remove(configured); +} + class PausedTraceBuffer final : public std::streambuf { public: bool WaitForWrite() { @@ -156,11 +171,24 @@ public: changed.notify_all(); } + void Pause() { + std::lock_guard lock(mutex); + entered = false; + released = false; + } + std::string Contents() { std::lock_guard lock(mutex); return contents; } + bool WaitForRecord(const std::string& record) { + std::unique_lock lock(mutex); + return changed.wait_for(lock, std::chrono::seconds(2), [this, &record] { + return contents.find(record) != std::string::npos; + }); + } + protected: std::streamsize xsputn(const char* text, std::streamsize size) override { std::unique_lock lock(mutex); @@ -168,6 +196,7 @@ protected: changed.notify_all(); changed.wait(lock, [this] { return released; }); contents.append(text, static_cast(size)); + changed.notify_all(); return size; } @@ -184,7 +213,9 @@ private: }; std::string QueueRecord(std::size_t producer, std::size_t item) { - return "queue producer=" + std::to_string(producer) + " item=" + std::to_string(item) + " payload=" + std::string(1900, 'x'); + char prefix[64]; + std::snprintf(prefix, sizeof(prefix), "queue producer=%02zu item=%08zu payload=", producer, item); + return std::string(prefix) + std::string(1900, 'x'); } void Disabled() { @@ -198,96 +229,104 @@ void Disabled() { } -void AgcDriver::CaptureTraceTestAccess::QueueBackpressure() { +void AgcDriver::CaptureTraceTestAccess::QueueDrop(bool shutdownOnly, bool cumulative) { constexpr std::size_t Capacity = 16u * 1024u * 1024u; constexpr std::size_t Producers = 4; - constexpr std::size_t ItemsPerProducer = 64; const char* configured = RequiredPathVariable(); PausedTraceBuffer sink; std::exception_ptr failure; std::size_t initialItems = 0; std::atomic bounded = true; + const std::size_t rounds = cumulative ? 2 : 1; + std::size_t lostTotal = 0; + std::vector expectedMarkers; + std::vector queuedByRound; + for (std::size_t round = 0; round < rounds; ++round) { + const auto lost = Producers * (round == 0 ? 64u : 17u) + (shutdownOnly ? 0u : 1u); + lostTotal += lost; + expectedMarkers.push_back("capture-trace-dropped records=" + std::to_string(lost)); + } + const auto summary = "capture-trace-summary dropped=" + std::to_string(lostTotal); { CaptureTrace trace; Check(trace.output.is_open(), "the queue test trace path must be writable"); auto& output = static_cast(trace.output); - auto* original = output.rdbuf(&sink); - std::vector producers; - std::vector producerErrors(Producers); - std::mutex stateMutex; - std::condition_variable stateChanged; - std::size_t attempted = 0; - std::size_t finished = 0; + output.rdbuf(&sink); auto append = [&](std::size_t producer, std::size_t item) { trace.append(QueueRecord(producer, item)); std::lock_guard lock(trace.mutex); if (trace.pending.size() > Capacity) bounded.store(false); }; - try { - for (; initialItems < 64; ++initialItems) append(0, initialItems); - Check(sink.WaitForWrite(), "the trace worker did not reach the paused sink"); - for (;;) { - const auto line = QueueRecord(0, initialItems); - { - std::lock_guard lock(trace.mutex); - if (trace.pending.size() + line.size() + 64 > Capacity) break; + for (std::size_t round = 0; round < rounds && !failure; ++round) { + if (round != 0) sink.Pause(); + const std::size_t itemsPerProducer = round == 0 ? 64 : 17; + std::vector producers; + std::vector producerErrors(Producers); + std::mutex stateMutex; + std::condition_variable stateChanged; + std::size_t attempted = 0; + std::size_t finished = 0; + try { + for (std::size_t item = 0; item < 64; ++item) append(0, initialItems++); + Check(sink.WaitForWrite(), "the trace worker did not reach the paused sink"); + for (;;) { + const auto line = QueueRecord(0, initialItems); + { + std::lock_guard lock(trace.mutex); + if (trace.pending.size() + line.size() + 64 > Capacity) break; + } + append(0, initialItems++); } - append(0, initialItems++); - } - for (std::size_t producer = 0; producer < Producers; ++producer) { - producers.emplace_back([&, producer] { - { - std::lock_guard lock(stateMutex); - ++attempted; - } - stateChanged.notify_all(); - try { - for (std::size_t item = 0; item < ItemsPerProducer; ++item) append(producer + 1, item); - } catch (...) { - producerErrors[producer] = std::current_exception(); - } - { - std::lock_guard lock(stateMutex); - ++finished; - } - stateChanged.notify_all(); - }); - } - { - std::unique_lock lock(stateMutex); - Check(stateChanged.wait_for(lock, std::chrono::seconds(2), [&] { return attempted == Producers; }), - "the queue producers did not start"); - Check(!stateChanged.wait_for(lock, std::chrono::milliseconds(200), [&] { return finished != 0; }), - "a full capture queue must wait for its sink instead of failing or growing"); + queuedByRound.push_back(initialItems); + for (std::size_t producer = 0; producer < Producers; ++producer) { + producers.emplace_back([&, producer] { + { + std::lock_guard lock(stateMutex); + ++attempted; + } + stateChanged.notify_all(); + try { + for (std::size_t item = 0; item < itemsPerProducer; ++item) append(producer + 1, item); + } catch (...) { + producerErrors[producer] = std::current_exception(); + } + { + std::lock_guard lock(stateMutex); + ++finished; + } + stateChanged.notify_all(); + }); + } + { + std::unique_lock lock(stateMutex); + Check(stateChanged.wait_for(lock, std::chrono::seconds(2), [&] { return attempted == Producers; }), + "the queue producers did not start"); + Check(stateChanged.wait_for(lock, std::chrono::seconds(2), [&] { return finished == Producers; }), + "a full capture queue must return from every caller while its writer is paused"); + } + if (!shutdownOnly) trace.append("queue-short-during-loss"); + } catch (...) { + failure = std::current_exception(); } sink.Release(); - { - std::unique_lock lock(stateMutex); - Check(stateChanged.wait_for(lock, std::chrono::seconds(5), [&] { return finished == Producers; }), - "the queue producers did not resume after the sink recovered"); + for (auto& producer : producers) producer.join(); + if (!failure) { + for (const auto& error : producerErrors) { + if (error) { + failure = error; + break; + } + } } - } catch (...) { - failure = std::current_exception(); - } - sink.Release(); - if (failure) { - std::lock_guard lock(trace.mutex); - trace.pending.clear(); - } - trace.spaceAvailable.notify_all(); - for (auto& producer : producers) producer.join(); - { - std::lock_guard lock(trace.mutex); - trace.stopping = true; - } - trace.changed.notify_one(); - trace.worker.join(); - output.rdbuf(original); - if (!failure) { - for (const auto& error : producerErrors) { - if (error) { - failure = error; - break; + if (!failure && !shutdownOnly) { + try { + Check(sink.WaitForRecord(expectedMarkers[round] + "\n"), + "the recovered writer did not report the exact dropped-record count"); + append(Producers + 1, round); + Check(sink.WaitForRecord(QueueRecord(Producers + 1, round) + "\n"), + "the recovered writer did not accept a new record"); + } catch (...) { + failure = std::current_exception(); } } } @@ -295,9 +334,12 @@ void AgcDriver::CaptureTraceTestAccess::QueueBackpressure() { if (failure) std::rethrow_exception(failure); Check(bounded.load(), "the pending capture queue exceeded 16 MiB"); std::istringstream records(sink.Contents()); - std::vector nextItem(Producers + 1, 0); std::string record; std::size_t sequence = 0; + std::size_t queued = 0; + std::size_t recovered = 0; + std::size_t markers = 0; + std::size_t summaries = 0; while (std::getline(records, record)) { std::istringstream fields(record); unsigned long long foundSequence = 0; @@ -307,20 +349,40 @@ void AgcDriver::CaptureTraceTestAccess::QueueBackpressure() { std::string itemField; fields >> foundSequence >> elapsed >> label >> producerField >> itemField; Check(foundSequence == ++sequence, "the recovered trace lost or reordered sequence numbers"); + Check(elapsed.ends_with("us"), "the recovered trace lost its elapsed-time field"); + Check(summaries == 0, "the trace contains a record after its shutdown summary"); + const auto prefix = std::to_string(foundSequence) + " " + elapsed + " "; + const auto body = record.substr(prefix.size()); + if (body.starts_with("capture-trace-dropped records=")) { + Check(markers < rounds && body == expectedMarkers[markers], + "the trace did not report each dropped-record batch exactly once"); + Check(queued == queuedByRound[markers] && recovered == markers, + "the dropped marker did not follow the queued records and precede recovery"); + ++markers; + continue; + } + if (body == summary) { + Check(markers == rounds, "the shutdown total appeared before the loss markers"); + ++summaries; + continue; + } Check(label == "queue" && producerField.starts_with("producer=") && itemField.starts_with("item="), "the recovered trace contains an incomplete record"); const auto producer = std::stoull(producerField.substr(9)); const auto item = std::stoull(itemField.substr(5)); - Check(producer <= Producers, "the recovered trace contains an unexpected producer"); - Check(item == nextItem[producer]++, "the recovered trace lost or reordered a producer's records"); - const auto prefix = std::to_string(foundSequence) + " " + elapsed + " "; + if (producer == 0) { + Check(markers == recovered && item == queued++, "the trace lost or reordered an accepted queued record"); + } else { + Check(!shutdownOnly && producer == Producers + 1 && item == recovered && markers == recovered + 1, + "the trace retained a dropped record or lost recovery ordering"); + ++recovered; + } Check(record == prefix + QueueRecord(producer, item), "the recovered trace changed a record's payload"); } - Check(sequence == initialItems + Producers * ItemsPerProducer, "the recovered trace did not drain every queued record"); - Check(nextItem[0] == initialItems, "the recovered trace lost the initial queued records"); - for (std::size_t producer = 1; producer <= Producers; ++producer) { - Check(nextItem[producer] == ItemsPerProducer, "the recovered trace lost a producer's queued records"); - } + Check(queued == initialItems, "the recovered trace lost the initial accepted records"); + Check(recovered == (shutdownOnly ? 0u : rounds), "the trace lost or duplicated its recovery records"); + Check(markers == rounds && summaries == 1, "the trace did not report each loss marker and one shutdown total"); + Check(sequence == initialItems + recovered + rounds + 1, "the trace contains an unexpected number of records"); std::remove(configured); } @@ -335,7 +397,10 @@ int main(int argc, char** argv) { else if (mode == "default-writes") DefaultWrites(); else if (mode == "open-failure") OpenFailure(); else if (mode == "best-effort-overflow") BestEffortOverflow(); - else if (mode == "queue-backpressure") AgcDriver::CaptureTraceTestAccess::QueueBackpressure(); + else if (mode == "format-overflow") FormatOverflow(); + else if (mode == "queue-drop") AgcDriver::CaptureTraceTestAccess::QueueDrop(false); + else if (mode == "queue-drop-shutdown") AgcDriver::CaptureTraceTestAccess::QueueDrop(true); + else if (mode == "queue-drop-cumulative") AgcDriver::CaptureTraceTestAccess::QueueDrop(false, true); else if (mode == "disabled") Disabled(); else throw std::runtime_error("unknown capture trace test mode: " + mode); std::cout << "capture trace tests passed\n"; diff --git a/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake index 14444c542a..e353b83aa5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake @@ -9,24 +9,6 @@ endif() string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef runId) set(runDirectory "${OUTPUT_ROOT}/capture-trace-${MODE}-${runId}") file(MAKE_DIRECTORY "${runDirectory}") -execute_process( - COMMAND "${FIXTURE}" control-abort - WORKING_DIRECTORY "${runDirectory}" - RESULT_VARIABLE abortResult OUTPUT_VARIABLE abortOutput ERROR_VARIABLE abortError TIMEOUT 8 -) -file(WRITE "${runDirectory}/control.log" "result=${abortResult}\n${abortOutput}\n${abortError}") -if(abortResult STREQUAL "0" OR abortResult STREQUAL "87" OR abortResult STREQUAL "99" - OR NOT abortError MATCHES "capture-trace-failure control-abort" - OR abortError MATCHES "setup-failed|watchdog") - message(FATAL_ERROR "Abort control failed: ${abortResult}\n${abortOutput}\n${abortError}") -endif() -if(WIN32) - if(NOT abortResult STREQUAL "3") - message(FATAL_ERROR "The Windows abort control returned an unexpected result: ${abortResult}") - endif() -elseif(NOT abortResult MATCHES "[Aa]bort") - message(FATAL_ERROR "The abort control did not report an abort signal: ${abortResult}") -endif() set(ENV{APS5_CAPTURE_TRACE} "1") set(ENV{APS5_CAPTURE_TRACE_PATH} "${runDirectory}/trace.log") execute_process( @@ -35,20 +17,23 @@ execute_process( RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error TIMEOUT 8 ) file(WRITE "${runDirectory}/failure.log" "result=${result}\n${output}\n${error}") -set(armed "capture-trace-failure armed mode=${MODE} queueFull=1 attempted=4 finished=0") +set(armed "capture-trace-failure armed mode=${MODE} queueFull=1 attempted=4 finished=4") if(MODE STREQUAL "write") set(sinkFailure "capture-trace-failure sink-short-write") else() set(sinkFailure "capture-trace-failure sink-flush-failed") endif() -set(terminate "capture-trace-failure terminate armed=1 queueFull=1 attempted=4 finished=0 sinkFailed=1") +set(survived "capture-trace-failure survived mode=${MODE} queueFull=1 attempted=4 finished=4 inactive=1 empty=1 sinkFailed=1") string(FIND "${error}" "${armed}" armedAt) string(FIND "${error}" "${sinkFailure}" sinkAt) -string(FIND "${error}" "Capture trace failed:" caughtAt) -string(FIND "${error}" "${terminate}" terminateAt) -if(NOT result STREQUAL abortResult OR result STREQUAL "87" OR result STREQUAL "99" - OR error MATCHES "setup-failed|watchdog|handler-returned|handler-threw" - OR armedAt LESS 0 OR NOT sinkAt GREATER armedAt OR NOT caughtAt GREATER sinkAt OR NOT terminateAt GREATER caughtAt) - message(FATAL_ERROR "Capture trace ${MODE} failure did not reach its saved terminate handler and abort: ${result}\n${output}\n${error}") +string(FIND "${error}" "Capture trace failed:" failedAt) +string(FIND "${error}" "${survived}" survivedAt) +string(REGEX MATCHALL "Capture trace failed:" failureMessages "${error}") +list(LENGTH failureMessages failureCount) +if(NOT result STREQUAL "0" + OR error MATCHES "setup-failed|watchdog|unexpected-terminate" + OR NOT failureCount EQUAL 1 + OR armedAt LESS 0 OR NOT sinkAt GREATER armedAt OR NOT failedAt GREATER sinkAt OR NOT survivedAt GREATER failedAt) + message(FATAL_ERROR "Capture trace ${MODE} failure did not stop only tracing and exit normally: ${result}\n${output}\n${error}") endif() -message(STATUS "Capture trace ${MODE} failure reached its saved terminate handler and aborted with four waiting producers") +message(STATUS "Capture trace ${MODE} failure stopped tracing, released all callers before the sink failed and exited normally") diff --git a/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cpp b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cpp index c48c1f80d8..2bfef87ac8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cpp @@ -12,25 +12,10 @@ #include #include -#ifdef _WIN32 -#ifndef NOMINMAX -#define NOMINMAX -#endif -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include -#else -#include -#endif -#ifdef __linux__ -#include -#endif - namespace AgcDriver { struct CaptureTraceTestAccess { - [[noreturn]] static void WriterFailure(bool flush); + static void WriterFailure(bool flush); }; } @@ -39,13 +24,6 @@ namespace { using namespace std::chrono_literals; -std::atomic armed{false}; -std::atomic queueFull{false}; -std::atomic sinkFailed{false}; -std::atomic attempted{0}; -std::atomic finished{0}; -std::terminate_handler originalTerminate = nullptr; - [[noreturn]] void SetupFailure(const char* message) { std::fprintf(stderr, "capture-trace-failure setup-failed: %s\n", message); std::fflush(stderr); @@ -56,20 +34,6 @@ void Check(bool condition, const char* message) { if (!condition) SetupFailure(message); } -[[noreturn]] void SavedTerminate() { - std::fprintf(stderr, "capture-trace-failure terminate armed=%d queueFull=%d attempted=%u finished=%u sinkFailed=%d\n", - armed.load() ? 1 : 0, queueFull.load() ? 1 : 0, attempted.load(), finished.load(), sinkFailed.load() ? 1 : 0); - std::fflush(stderr); - try { - originalTerminate(); - std::fputs("capture-trace-failure handler-returned\n", stderr); - } catch (...) { - std::fputs("capture-trace-failure handler-threw\n", stderr); - } - std::fflush(stderr); - std::abort(); -} - class FailingBuffer final : public std::streambuf { public: explicit FailingBuffer(bool failFlush) : failFlush(failFlush) {} @@ -87,20 +51,34 @@ public: changed.notify_all(); } + unsigned WriteCalls() const { + return writeCalls.load(); + } + + unsigned FlushCalls() const { + return flushCalls.load(); + } + + bool Failed() const { + return failed.load(); + } + protected: std::streamsize xsputn(const char*, std::streamsize size) override { + writeCalls.fetch_add(1); if (failFlush) return size; wait(); - sinkFailed.store(true); + failed.store(true); std::fputs("capture-trace-failure sink-short-write\n", stderr); std::fflush(stderr); return 0; } int sync() override { + flushCalls.fetch_add(1); if (!failFlush) return 0; wait(); - sinkFailed.store(true); + failed.store(true); std::fputs("capture-trace-failure sink-flush-failed\n", stderr); std::fflush(stderr); return -1; @@ -117,6 +95,9 @@ private: std::mutex mutex; std::condition_variable changed; const bool failFlush; + std::atomic writeCalls{0}; + std::atomic flushCalls{0}; + std::atomic failed{false}; bool entered = false; bool released = false; }; @@ -129,83 +110,114 @@ std::string Record(unsigned producer, unsigned item) { } -[[noreturn]] void AgcDriver::CaptureTraceTestAccess::WriterFailure(bool flush) { +void AgcDriver::CaptureTraceTestAccess::WriterFailure(bool flush) { constexpr std::size_t Capacity = 16u * 1024u * 1024u; FailingBuffer sink(flush); - CaptureTrace trace; - Check(trace.output.is_open(), "the trace output did not open"); - static_cast(trace.output).rdbuf(&sink); - unsigned item = 0; - for (; item < 64; ++item) trace.append(Record(0, item)); - Check(sink.WaitForWriter(), "the writer did not reach the failing sink"); - const auto line = Record(1, 0); - for (;;) { + { + CaptureTrace trace; + Check(trace.output.is_open(), "the trace output did not open"); + auto& output = static_cast(trace.output); + auto* original = output.rdbuf(&sink); + unsigned item = 0; + for (; item < 64; ++item) trace.append(Record(0, item)); + Check(sink.WaitForWriter(), "the writer did not reach the failing sink"); + const auto line = Record(1, 0); + for (;;) { + { + std::lock_guard lock(trace.mutex); + if (trace.pending.size() + line.size() + 64 > Capacity) break; + } + trace.append(Record(0, item++)); + } + std::size_t queuedBytes = 0; { std::lock_guard lock(trace.mutex); - if (trace.pending.size() + line.size() + 64 > Capacity) break; + Check(trace.pending.size() <= Capacity, "the queue exceeded its bound"); + Check(trace.droppedTotal == 0, "filling the queue unexpectedly dropped a record"); + queuedBytes = trace.pending.size(); } - trace.append(Record(0, item++)); + std::atomic attempted{0}; + std::mutex stateMutex; + std::condition_variable stateChanged; + unsigned finished = 0; + std::vector producers; + for (unsigned producer = 1; producer <= 4; ++producer) { + producers.emplace_back([&, producer] { + attempted.fetch_add(1); + try { + trace.append(Record(producer, 0)); + } catch (...) { + SetupFailure("a producer threw before the writer failed"); + } + { + std::lock_guard lock(stateMutex); + ++finished; + } + stateChanged.notify_all(); + }); + } + { + std::unique_lock lock(stateMutex); + Check(stateChanged.wait_for(lock, 2s, [&] { return finished == 4; }), + "full-queue producers did not return while the sink was paused"); + } + for (auto& producer : producers) producer.join(); + { + std::lock_guard lock(trace.mutex); + Check(trace.pending.size() == queuedBytes, "a full-queue producer added data to the pending queue"); + Check(trace.droppedTotal == 4, "the four rejected records were not counted exactly once"); + } + Check(attempted.load() == 4, "the four producers did not run"); + Check(!sink.Failed(), "the sink failed before the full-queue producers returned"); + std::fprintf(stderr, "capture-trace-failure armed mode=%s queueFull=1 attempted=4 finished=4\n", flush ? "flush" : "write"); + std::fflush(stderr); + sink.Fail(); + trace.worker.join(); + Check(sink.Failed(), "the writer did not reach the requested stream failure"); + { + std::lock_guard lock(trace.mutex); + Check(!trace.active, "tracing remained active after the stream failure"); + Check(trace.pending.empty(), "failed tracing retained queued records"); + } + const auto writes = sink.WriteCalls(); + const auto flushes = sink.FlushCalls(); + Check(writes == 1, "the failed sink received another write"); + Check(flushes == (flush ? 1u : 0u), "the failed sink received an unexpected flush"); + for (unsigned producer = 1; producer <= 4; ++producer) trace.append(Record(producer, 1)); + { + std::lock_guard lock(trace.mutex); + Check(!trace.active && trace.pending.empty(), "logging after failure reactivated or queued tracing"); + Check(trace.droppedTotal == 4, "logging after failure changed the queue-loss count"); + } + Check(sink.WriteCalls() == writes && sink.FlushCalls() == flushes, + "logging after failure attempted more stream I/O"); + output.exceptions(std::ios::goodbit); + output.clear(); + output.rdbuf(original); } - std::vector producers; - for (unsigned producer = 1; producer <= 4; ++producer) { - producers.emplace_back([&, producer] { - attempted.fetch_add(1); - try { - trace.append(Record(producer, 0)); - } catch (...) { - SetupFailure("a producer threw before the writer failed"); - } - finished.fetch_add(1); - }); - } - const auto deadline = std::chrono::steady_clock::now() + 2s; - while (attempted.load() != 4 && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(1ms); - Check(attempted.load() == 4, "the four producers did not start"); - std::this_thread::sleep_for(200ms); - Check(finished.load() == 0, "a producer passed the full queue before the sink was released"); - { - std::lock_guard lock(trace.mutex); - Check(trace.pending.size() + line.size() + 64 > Capacity, "the producers were not held behind a full queue"); - Check(trace.pending.size() <= Capacity, "the queue exceeded its bound"); - queueFull.store(true); - } - armed.store(true); - std::fprintf(stderr, "capture-trace-failure armed mode=%s queueFull=1 attempted=4 finished=0\n", flush ? "flush" : "write"); + std::fprintf(stderr, "capture-trace-failure survived mode=%s queueFull=1 attempted=4 finished=4 inactive=1 empty=1 sinkFailed=1\n", + flush ? "flush" : "write"); std::fflush(stderr); - sink.Fail(); - for (auto& producer : producers) producer.join(); - SetupFailure("all producers returned after a fatal writer error"); } int main(int argc, char** argv) { -#ifdef _WIN32 - SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); - _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); -#else - const rlimit noCore{0, 0}; - setrlimit(RLIMIT_CORE, &noCore); -#endif -#ifdef __linux__ - prctl(PR_SET_DUMPABLE, 0); -#endif + std::set_terminate([] { + std::fputs("capture-trace-failure unexpected-terminate\n", stderr); + std::fflush(stderr); + std::_Exit(86); + }); std::thread([] { std::this_thread::sleep_for(6s); std::fputs("capture-trace-failure watchdog\n", stderr); std::fflush(stderr); std::_Exit(99); }).detach(); - Check(argc == 2, "expected write, flush or control-abort mode"); + Check(argc == 2, "expected write or flush mode"); const std::string mode = argv[1]; - Check(mode == "write" || mode == "flush" || mode == "control-abort", "unknown failure mode"); - if (mode == "control-abort") { - std::fputs("capture-trace-failure control-abort\n", stderr); - std::fflush(stderr); - std::abort(); - } + Check(mode == "write" || mode == "flush", "unknown failure mode"); Check(AgcDriver::CaptureTrace::Enabled(), "capture tracing must be enabled"); const char* path = std::getenv("APS5_CAPTURE_TRACE_PATH"); Check(path != nullptr && path[0] != '\0', "the test requires an explicit trace path"); - originalTerminate = std::set_terminate(&SavedTerminate); - Check(originalTerminate != nullptr, "the runtime did not return its original terminate handler"); AgcDriver::CaptureTraceTestAccess::WriterFailure(mode == "flush"); + return 0; } diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake index 94b5c92864..4d5abe8cda 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake @@ -8,12 +8,18 @@ if(BUILD_TESTING) file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/explicit") file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/failure") file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/best-effort") + file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/format") file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/queue") + file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/queue-shutdown") + file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/queue-cumulative") file(MAKE_DIRECTORY "${agcCaptureTraceDirectory}/disabled") set(agcCaptureTraceExplicitPath "${agcCaptureTraceDirectory}/explicit/trace.log") set(agcCaptureTraceFailurePath "${agcCaptureTraceDirectory}/failure/missing-parent/trace.log") set(agcCaptureTraceBestEffortPath "${agcCaptureTraceDirectory}/best-effort/trace.log") + set(agcCaptureTraceFormatPath "${agcCaptureTraceDirectory}/format/trace.log") set(agcCaptureTraceQueuePath "${agcCaptureTraceDirectory}/queue/trace.log") + set(agcCaptureTraceQueueShutdownPath "${agcCaptureTraceDirectory}/queue-shutdown/trace.log") + set(agcCaptureTraceQueueCumulativePath "${agcCaptureTraceDirectory}/queue-cumulative/trace.log") add_test(NAME agc_capture_trace_path_default COMMAND agc_capture_trace_tests path-default) add_test(NAME agc_capture_trace_path_empty COMMAND agc_capture_trace_tests path-empty) set_tests_properties(agc_capture_trace_path_empty PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE_PATH=") @@ -27,14 +33,20 @@ if(BUILD_TESTING) set_tests_properties(agc_capture_trace_open_failure PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceFailurePath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/failure") add_test(NAME agc_capture_trace_best_effort_overflow COMMAND agc_capture_trace_tests best-effort-overflow) set_tests_properties(agc_capture_trace_best_effort_overflow PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceBestEffortPath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/best-effort") - add_test(NAME agc_capture_trace_queue_backpressure COMMAND agc_capture_trace_tests queue-backpressure) - set_tests_properties(agc_capture_trace_queue_backpressure PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceQueuePath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/queue" TIMEOUT 15) + add_test(NAME agc_capture_trace_format_overflow COMMAND agc_capture_trace_tests format-overflow) + set_tests_properties(agc_capture_trace_format_overflow PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceFormatPath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/format" TIMEOUT 15) + add_test(NAME agc_capture_trace_queue_drop COMMAND agc_capture_trace_tests queue-drop) + set_tests_properties(agc_capture_trace_queue_drop PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceQueuePath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/queue" TIMEOUT 15) + add_test(NAME agc_capture_trace_queue_drop_shutdown COMMAND agc_capture_trace_tests queue-drop-shutdown) + set_tests_properties(agc_capture_trace_queue_drop_shutdown PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceQueueShutdownPath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/queue-shutdown" TIMEOUT 15) + add_test(NAME agc_capture_trace_queue_drop_cumulative COMMAND agc_capture_trace_tests queue-drop-cumulative) + set_tests_properties(agc_capture_trace_queue_drop_cumulative PROPERTIES ENVIRONMENT "APS5_CAPTURE_TRACE=1;APS5_CAPTURE_TRACE_PATH=${agcCaptureTraceQueueCumulativePath}" WORKING_DIRECTORY "${agcCaptureTraceDirectory}/queue-cumulative" TIMEOUT 15) add_test(NAME agc_capture_trace_disabled COMMAND agc_capture_trace_tests disabled) set_tests_properties(agc_capture_trace_disabled PROPERTIES WORKING_DIRECTORY "${agcCaptureTraceDirectory}/disabled") if(WIN32) set_tests_properties(agc_capture_trace_path_default agc_capture_trace_path_empty agc_capture_trace_path_explicit agc_capture_trace_explicit_writes - agc_capture_trace_default_writes agc_capture_trace_open_failure agc_capture_trace_best_effort_overflow agc_capture_trace_queue_backpressure + agc_capture_trace_default_writes agc_capture_trace_open_failure agc_capture_trace_best_effort_overflow agc_capture_trace_format_overflow agc_capture_trace_queue_drop agc_capture_trace_queue_drop_shutdown agc_capture_trace_queue_drop_cumulative agc_capture_trace_disabled PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() @@ -44,6 +56,10 @@ if(BUILD_TESTING) if(MINGW) target_compile_options(agc_capture_trace_failure_tests PRIVATE -fasynchronous-unwind-tables) endif() + add_test_executable(agc_capture_trace_runtime_failure_tests tests/CaptureTraceFailure.cpp) + target_include_directories(agc_capture_trace_runtime_failure_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(agc_capture_trace_runtime_failure_tests PRIVATE libc) + configure_windows_unwind(agc_capture_trace_runtime_failure_tests) foreach(mode IN ITEMS write flush) add_test(NAME agc_capture_trace_${mode}_failure COMMAND "${CMAKE_COMMAND}" @@ -52,5 +68,16 @@ if(BUILD_TESTING) "-DOUTPUT_ROOT=${agcCaptureTraceDirectory}" -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/CaptureTraceFailure.cmake") set_tests_properties(agc_capture_trace_${mode}_failure PROPERTIES TIMEOUT 20) + add_test(NAME agc_capture_trace_${mode}_failure_runtime + COMMAND "${CMAKE_COMMAND}" + "-DFIXTURE=$" + "-DMODE=${mode}" + "-DOUTPUT_ROOT=${agcCaptureTraceDirectory}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/CaptureTraceFailure.cmake") + set_tests_properties(agc_capture_trace_${mode}_failure_runtime PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(agc_capture_trace_${mode}_failure_runtime PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endforeach() endif() diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index a6f09aea6b..19815561a4 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -114,7 +114,7 @@ - [Written staging window on discrete GPUs](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) - a written buffer element of a dispatch that is not swept is staged device-local up to 8 MiB on a discrete GPU (`Context::discreteGpu`) and up to 2 MiB otherwise; `APS5_WRITTEN_SHADOW_MAX_KIB` replaces either bound. The 8 MiB figure comes from one title on one NVIDIA Windows system (PPSA09911: a 2.5 and a 3.3 MiB skin-cache output written in place cost 55 ms of GPU time per frame, 1.8 ms of extra staging copies once staged); the point where the two copies of a larger element cost more than its in-place stores, and whether AMD or Linux drivers move it, has not been measured. The window is per element and per use, so a kernel that streams once through an element inside the window pays the copy-in and the copy-back on every dispatch. - [FS table calls on macOS](../../../core/libs/prx/libSceAgcDriver/Execution/include/Driver/DriverMemory.hpp) (libSceAgcDriver) - macOS keeps the console's 0xfe0000000 for itself, so the driver block, and the FS table at 0xfe0040000 inside it, moves to 0xee0000000 there; `sceAgc*StaticBuffer` and `AgcDriverStaticTable_nid_postfix` use the moved address. A shader that builds the table address from two `s_mov_b32` literals still loads from 0xfe0040000: `ReadTableCallees` (`Driver/Shaders/ShaderRegistry.cpp`) reads that address unchanged, and the shader also loads from it at run time, so those dispatches cannot link on macOS. #6160 lists titles that build this address. `ComputeFsTableCalls` is skipped on macOS as it is under `ANYPS5_ENABLE_ASAN`. Closes when: the driver rewrites that pointer in the shader code for both the callee lookup and the code the GPU runs, or the console address can be mapped on macOS. -- [Capture trace output](../../../core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp) can block logging callers and normal shutdown while its output is stalled. The pending queue waits at 16 MiB instead of discarding records; the batch being written can hold another 16 MiB. File write failures retain the fatal path. Permanently blocked file I/O has no timeout. +- [Capture trace output](../../../core/libs/prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp) has no timeout for permanently blocked file I/O, so normal shutdown can wait for its writer. The pending queue holds up to 16 MiB, and the batch being written can hold another 16 MiB plus loss markers. Queue overflow drops new records until the next drain; markers and a shutdown total identify the incomplete trace. A failed write or flush disables tracing and reports an error to stderr, but the failed output may not receive its final markers. - [Frame capture](../../../core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp) (libSceAgcDriver) throws on macOS. Section views, aliases of one direct memory block and file views replay as separate private memory, so a store through one alias is not seen through another. Only write-watched memory records CPU stores during the frames: private memory on Windows, where a store through one alias is recorded only at that alias; registered ranges on Linux, including direct memory unless mapped more than once (libkernel leaves those views unwatched, so no store through any of them is recorded). Title image pages are recorded only when changed and holding a shader header or its arrays. Linux omits no-access pages (Windows records them unreadable) and records file mapping pages past the file's end as unreadable. A capture starts only at a frame boundary where the driver idles within 10 s (five tries), and stops when GPU work in flight at a delta waits on a later CPU store. - [Frame replay](../../../core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp) (libSceAgcDriver) orders the calls of different captured threads only by their start (a call starts once every earlier call has started), so the driver sees a schedule the title could have produced, not necessarily the one it did. CPU stores a title made from threads that never call the driver are known only by the next recorded call, so the replay applies them early or late when a GPU wait needs them and derives a value from the wait's reference when no recorded store satisfies it (`of_them_synthesized`). Pacing compares the number of submissions in flight, not what they hold. Prologue registrations of shaders whose memory the title reused after registering them fail and are counted (`stale_registrations`). The title's image range must be free in the replaying process, so the tool is linked at a high image base on Windows. - [Host-only multisample memory in frame capture](../../../core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp) (libSceAgcDriver) - memory under multisampled color targets, their FMASK and native multisample depth whose samples are only on the host is zero in the snapshot and in deltas (`host_only_sample_bytes`, with a warning), so replay starts those targets without the contents earlier frames left in them. From b2ab7b2dc61b35a3b2db107341b3c4ef624fbe9d Mon Sep 17 00:00:00 2001 From: liu Date: Mon, 26 Oct 2026 09:40:27 +0100 Subject: [PATCH 309/318] test(agc): expose capture failure fixtures to registration audit --- .../libSceAgcDriver/tests/CaptureTraceFailure.cmake | 6 ++++++ .../tests/cmake/agc_capture_trace_tests.cmake | 12 ++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake index e353b83aa5..7cca64b3c3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/CaptureTraceFailure.cmake @@ -1,3 +1,9 @@ +if(NOT CMAKE_ARGC EQUAL 6) + message(FATAL_ERROR "Expected fixture, mode and output root arguments") +endif() +set(FIXTURE "${CMAKE_ARGV3}") +set(MODE "${CMAKE_ARGV4}") +set(OUTPUT_ROOT "${CMAKE_ARGV5}") foreach(required IN ITEMS FIXTURE MODE OUTPUT_ROOT) if(NOT DEFINED ${required} OR "${${required}}" STREQUAL "") message(FATAL_ERROR "Missing ${required}") diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake index 4d5abe8cda..1ef979a1eb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_capture_trace_tests.cmake @@ -63,17 +63,13 @@ if(BUILD_TESTING) foreach(mode IN ITEMS write flush) add_test(NAME agc_capture_trace_${mode}_failure COMMAND "${CMAKE_COMMAND}" - "-DFIXTURE=$" - "-DMODE=${mode}" - "-DOUTPUT_ROOT=${agcCaptureTraceDirectory}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/CaptureTraceFailure.cmake") + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/CaptureTraceFailure.cmake" + "$" "${mode}" "${agcCaptureTraceDirectory}") set_tests_properties(agc_capture_trace_${mode}_failure PROPERTIES TIMEOUT 20) add_test(NAME agc_capture_trace_${mode}_failure_runtime COMMAND "${CMAKE_COMMAND}" - "-DFIXTURE=$" - "-DMODE=${mode}" - "-DOUTPUT_ROOT=${agcCaptureTraceDirectory}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/CaptureTraceFailure.cmake") + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/CaptureTraceFailure.cmake" + "$" "${mode}" "${agcCaptureTraceDirectory}") set_tests_properties(agc_capture_trace_${mode}_failure_runtime PROPERTIES TIMEOUT 20) if(WIN32) set_tests_properties(agc_capture_trace_${mode}_failure_runtime PROPERTIES ENVIRONMENT_MODIFICATION From c3768e3488854f177128bb35ae385a2fdf5c8dfb Mon Sep 17 00:00:00 2001 From: Celegans12 <53881228+Celegans12@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:55:07 +0000 Subject: [PATCH 310/318] perf(agc): stop counting the lease of a build that never stores through BDA as a writer PendingWriterSerial counts an address-based build's lease as a writer of every leased range, because such a build marks the pages it stored to through BDA only when its batch completes. A build whose shaders never store through BDA marks nothing at completion; its descriptor writes are already noted as pending writes when it is recorded. Its lease is now noted as ReadKind::AddressBasedReadOnly, which PendingWriterSerial skips and every other read check treats like any pending read. A texture refresh flush over a range such a build leased no longer waits for its batch. APS5_NO_READ_ONLY_LEASE_SKIP=1 notes the lease as AddressBased, as before. LinearTargetDrawBindings runs its scenario after a storing and after a read-only address-based draw: the drawn target's store syncs the batch after the first and is recorded into it after the second, unless the switch is set. AI-assisted: yes --- .../Graphics/include/Recorder.hpp | 2 +- .../libSceAgcDriver/Graphics/src/Recorder.cpp | 2 +- .../Graphics/src/ShaderResources.cpp | 3 +- ...er_linear_target_draw_bindings_tests.cmake | 8 + ..._driver_read_only_lease_writer_tests.cmake | 8 + .../execution/LinearTargetDrawBindings.cpp | 19 ++- .../tests/execution/ReadOnlyLeaseWriter.cpp | 158 ++++++++++++++++++ docs/dev/ENVIRONMENT.md | 1 + 8 files changed, 191 insertions(+), 10 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_linear_target_draw_bindings_tests.cmake create mode 100644 core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_read_only_lease_writer_tests.cmake create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/ReadOnlyLeaseWriter.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index 66216c459b..4c90695187 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -153,7 +153,7 @@ public: // on the open batch, no snapshot: the only reader holds the mutex. `kind` names the reader for // the [recorder] hit counters. APS5_COPY_READ_TRACKING=0 notes nothing (ReadTracking() is then // false and the CPU copy falls back to Idle()). - enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, DrawInput, Count }; + enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, DrawInput, AddressBasedReadOnly, Count }; static bool ReadTracking(); void NotePendingRead(std::uint64_t address, std::size_t bytes, ReadKind kind); void NotePendingReads(std::span> ranges, ReadKind kind); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 0d0f474245..770a1ece02 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -4214,7 +4214,7 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source, bool AgcDriver::ProfilePrint_nid_no_patch("[recorder] completion label stores: %llu run, %llu skipped (no CPU write-back overlapped them); %llu counted pending at a write-back; %llu write-backs over a tracked label; %llu write-back completions pending\n", static_cast(completionStoresRun.load()), static_cast(completionStoresSkipped.load()), static_cast(completionLabelsCountedLate.load()), static_cast(writeBacksOverLabels.load()), static_cast(writeBackCompletions.load())); AgcDriver::ProfilePrint_nid_no_patch("[recorder] submits %llu, vkQueueSubmit mean %.1f us, max %.1f us; command buffers freed at batch release %llu, fences kept for reuse %llu\n", static_cast(submitCount), submitCount != 0 ? submitUs / static_cast(submitCount) : 0.0, submitMaxUs, static_cast(commandBuffersFreed.load()), static_cast(fencesKept.load())); const auto reads = Recorder::ReadCounts(); - AgcDriver::ProfilePrint_nid_no_patch("[recorder] in-place reads: %llu noted, %llu queries, hits by reader: dispatch element %llu, gpu copy %llu, address-based %llu, indirect %llu, storage upload %llu, copy source %llu, draw input %llu; %llu hits on signaled batches ignored\n", static_cast(reads.noted), static_cast(reads.queries), static_cast(reads.hits[0]), static_cast(reads.hits[1]), static_cast(reads.hits[2]), static_cast(reads.hits[3]), static_cast(reads.hits[4]), static_cast(reads.hits[5]), static_cast(reads.hits[6]), static_cast(reads.staleIgnored)); + AgcDriver::ProfilePrint_nid_no_patch("[recorder] in-place reads: %llu noted, %llu queries, hits by reader: dispatch element %llu, gpu copy %llu, address-based %llu, indirect %llu, storage upload %llu, copy source %llu, draw input %llu, address-based read-only %llu; %llu hits on signaled batches ignored\n", static_cast(reads.noted), static_cast(reads.queries), static_cast(reads.hits[0]), static_cast(reads.hits[1]), static_cast(reads.hits[2]), static_cast(reads.hits[3]), static_cast(reads.hits[4]), static_cast(reads.hits[5]), static_cast(reads.hits[6]), static_cast(reads.hits[7]), static_cast(reads.staleIgnored)); } } } report{profile, source, waitStart, *this, signaledAtStart}; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 78ec8ee522..4c8bd60d45 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -5990,7 +5990,8 @@ void ShaderResources::MarkGpuWrites(Recorder& recorder) { // The ranges this use reads in place through their host imports (read-only and written elements // alike, and an address-based build's whole leased heaps), before the writes: a CPU store into // one of them (the copy HLE) must not land before the recorded work read it. - const auto kind = guestMemory.HoldsLease() ? Recorder::ReadKind::AddressBased : Recorder::ReadKind::DispatchElement; + static const bool readOnlyLeaseSkip = std::getenv("APS5_NO_READ_ONLY_LEASE_SKIP") == nullptr; + const auto kind = !guestMemory.HoldsLease() ? Recorder::ReadKind::DispatchElement : readOnlyLeaseSkip && usesBda && bdaReadOnly ? Recorder::ReadKind::AddressBasedReadOnly : Recorder::ReadKind::AddressBased; if (auto split = guestMemory.SplitInPlaceReads()) { recorder.NotePendingReadList(std::move(split->space), kind); recorder.NotePendingReads(split->own, kind); diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_linear_target_draw_bindings_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_linear_target_draw_bindings_tests.cmake new file mode 100644 index 0000000000..a9fd063a5b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_linear_target_draw_bindings_tests.cmake @@ -0,0 +1,8 @@ +if(BUILD_TESTING) + add_test(NAME agc_driver_linear_target_draw_bindings_counted COMMAND agc_driver_linear_target_draw_bindings_tests) + set_tests_properties(agc_driver_linear_target_draw_bindings_counted PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_READ_ONLY_LEASE_SKIP=1" SKIP_RETURN_CODE 77 TIMEOUT 60 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_linear_target_draw_bindings_counted PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_read_only_lease_writer_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_read_only_lease_writer_tests.cmake new file mode 100644 index 0000000000..53e7001579 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_read_only_lease_writer_tests.cmake @@ -0,0 +1,8 @@ +if(BUILD_TESTING) + add_test(NAME agc_driver_read_only_lease_writer_counted COMMAND agc_driver_read_only_lease_writer_tests) + set_tests_properties(agc_driver_read_only_lease_writer_counted PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_READ_ONLY_LEASE_SKIP=1" SKIP_RETURN_CODE 77 TIMEOUT 60 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_read_only_lease_writer_counted PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDrawBindings.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDrawBindings.cpp index 4335e86e2c..32e8a49d47 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDrawBindings.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDrawBindings.cpp @@ -67,6 +67,7 @@ alignas(256) constexpr std::array VertexCode{ alignas(256) constexpr std::array PixelOpaque{0x7e100280u, 0x7e120280u, 0x7e140280u, 0x7e1602f2u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u}; alignas(256) constexpr std::array PixelRead{0x7e1a0280u, 0xdc308000u, 0x0c02000du, 0xbf8c3f70u, 0x7e10030cu, 0x7e120280u, 0x7e140280u, 0x7e160280u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u}; +alignas(256) constexpr std::array PixelReadStore{0x7e1a0280u, 0xdc308000u, 0x0c02000du, 0xbf8c3f70u, 0xdc708000u, 0x00020c0du, 0x7e10030cu, 0x7e120280u, 0x7e140280u, 0x7e160280u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u}; alignas(256) constexpr std::array PixelCopy{0x7e040280u, 0xe0301000u, 0x80010102u, 0xbf8c3f70u, 0xe0701000u, 0x80020102u, 0x7e100301u, 0x7e120280u, 0x7e140280u, 0x7e1602f2u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u}; constexpr std::array, 3> Triangle{{ @@ -250,7 +251,7 @@ void ExpectTexels(const std::uint8_t* target, const char* what, std::uint32_t fi } } -bool Expect(AgcDriver::VulkanDevice& device, bool unwatch) { +bool Expect(AgcDriver::VulkanDevice& device, bool unwatch, bool readOnly) { const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); if (!context.bufferDeviceAddress) { std::puts("skipped, the device lacks bufferDeviceAddress"); @@ -285,9 +286,11 @@ bool Expect(AgcDriver::VulkanDevice& device, bool unwatch) { const bool watched = AgcDriver::GuestMemory::Watched(copied.address, TargetBytes); Require(!unwatch || !watched, "the copied target is still write-watched after its block was taken out of the write watch"); const auto submitted = recorder->Submissions(); + const std::uint64_t synced = readOnly && std::getenv("APS5_NO_READ_ONLY_LEASE_SKIP") == nullptr ? 0u : 1u; + const std::string reader = readOnly ? "the read-only address-based draw" : "the storing address-based draw"; const auto pointed = block.Address() + PointedOffset; const std::array pointer{0u, 0u, static_cast(pointed), static_cast(pointed >> 32u)}; - Draw(device, drawn, PixelRead, pointer, whole); + Draw(device, drawn, readOnly ? std::span(PixelRead) : std::span(PixelReadStore), pointer, whole); Require(pending(drawn.address), "the address-based draw did not leave its linear target pending"); Require(recorder->Recording() && recorder->Submissions() == submitted, "the address-based draw did not stay in the open batch"); const auto setsBefore = context.descriptorCache->Counters().sets; @@ -295,12 +298,12 @@ bool Expect(AgcDriver::VulkanDevice& device, bool unwatch) { Draw(device, copied, PixelCopy, firstCopy, {{0, 0}, {Width / 2u, Height}}); Require(!pending(drawn.address), "the draw writing a copied buffer did not store the drawn linear target first"); Require(pending(copied.address), "the first copying draw did not leave its linear target pending"); - Require(recorder->Recording() && recorder->Submissions() == submitted + 1, "storing the drawn linear target did not submit the batch the address-based draw read it in (" + std::to_string(recorder->Submissions() - submitted) + " submissions)"); + Require(recorder->Recording() && recorder->Submissions() == submitted + synced, "storing the drawn linear target after " + reader + " made " + std::to_string(recorder->Submissions() - submitted) + " submissions, expected " + std::to_string(synced)); const auto setsBetween = context.descriptorCache->Counters().sets; const auto secondCopy = CopyUserData(block.Address() + SecondSourceOffset, SecondOutput); Draw(device, copied, PixelCopy, secondCopy, {{static_cast(Width / 2u), 0}, {Width / 2u, Height}}); - if (watched) Require(recorder->Recording() && recorder->Submissions() == submitted + 1, "the second copying draw submitted recorded work"); - else Require(recorder->Submissions() == submitted + 2, "the second copying draw into a stored target outside the write watch did not submit once to capture its bytes (" + std::to_string(recorder->Submissions() - submitted - 1) + " submissions)"); + if (watched) Require(recorder->Recording() && recorder->Submissions() == submitted + synced, "the second copying draw submitted recorded work after " + reader); + else Require(recorder->Submissions() == submitted + synced + 1, "the second copying draw into a stored target outside the write watch did not submit once to capture its bytes after " + reader + " (" + std::to_string(recorder->Submissions() - submitted - synced) + " submissions)"); const auto setsAfter = context.descriptorCache->Counters().sets; device.WaitIdle(); Require(FirstOutput[0] == FirstSource, "the first copying draw's output holds " + Hex(FirstOutput[0]) + ", not its source's " + Hex(FirstSource)); @@ -323,8 +326,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - if (!Expect(*device, false)) return VulkanTestSkipped; - if (!Expect(*device, true)) return VulkanTestSkipped; + for (const bool readOnly : {false, true}) { + if (!Expect(*device, false, readOnly)) return VulkanTestSkipped; + if (!Expect(*device, true, readOnly)) return VulkanTestSkipped; + } std::puts("linear target draw binding tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ReadOnlyLeaseWriter.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ReadOnlyLeaseWriter.cpp new file mode 100644 index 0000000000..e9957c8c55 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ReadOnlyLeaseWriter.cpp @@ -0,0 +1,158 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::size_t BlockBytes = 1u << 20u; +constexpr std::uint32_t Width = 256; +constexpr std::uint32_t Height = 128; +constexpr std::size_t ImageBytes = Width * Height * 4u; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Initial = 0x3f800000u; +constexpr std::uint32_t Loaded = 0x5a5a1234u; +constexpr std::uint32_t Stored = 0x22222222u; +constexpr std::size_t SourceOffset = BlockBytes / 2u; +constexpr std::size_t LoadedOffset = SourceOffset + 0x40u; +constexpr std::size_t QueryOffset = SourceOffset + 0x10000u; + +alignas(256) constexpr std::array LoadToImageCode{0x7e000200u, 0x7e020201u, 0xdc308040u, 0x027d0000u, 0xbf8c3f70u, 0x7e000280u, 0x7e020280u, 0xf0200108u, 0x00010200u, 0xbf810000u}; +alignas(256) constexpr std::array StoreGlobalCode{0x7e000208u, 0x7e020209u, 0x7e0402ffu, Stored, 0xdc708100u, 0x007d0200u, 0xbf810000u}; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(65536, BlockBytes)); +#endif + Require(block != nullptr, "read-only lease writer: cannot allocate the guest block"); + std::fill(Words(), Words() + BlockBytes / 4u, Initial); + Words()[LoadedOffset / 4u] = Loaded; + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint32_t* Words() const { return reinterpret_cast(block); } + std::uint64_t Address() const { return static_cast(reinterpret_cast(block)); } + +private: + std::uint8_t* block = nullptr; +}; + +ShaderRecompiler::RecompileResult Compile(std::span code, std::span userData, const ShaderRecompiler::SpirvTarget& target) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {64u, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +std::vector LoadUserData(std::uint64_t image, std::uint64_t source) { + std::vector userData(12, 0u); + userData[0] = static_cast(source); + userData[1] = static_cast(source >> 32u); + userData[4] = static_cast(image >> 8u); + userData[5] = static_cast((image >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Width - 1u) & 3u) << 30u); + userData[6] = ((Width - 1u) >> 2u) | ((Height - 1u) << 14u); + userData[7] = 0xfacu | (Type2D << 28u); + return userData; +} + +std::vector StoreUserData(std::uint64_t base) { + std::vector userData(10, 0u); + userData[8] = static_cast(base); + userData[9] = static_cast(base >> 32u); + return userData; +} + +struct Pending { + bool read; + std::uint64_t writer; + std::uint64_t open; +}; + +Pending PendingAt(std::uint64_t address) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + const auto* recorder = AgcDriver::Graphics::Recorder::Active(); + Require(recorder != nullptr, "read-only lease writer: no active recorder"); + return {recorder->DescribePendingRead(address, 64u).has_value(), recorder->PendingWriterSerial(address, 64u), recorder->OpenSerial()}; +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const bool counted = std::getenv("APS5_NO_READ_ONLY_LEASE_SKIP") != nullptr; + GuestBlock guest; + const auto target = device->Target(); + const auto query = guest.Address() + QueryOffset; + device->WaitIdle(); + + const auto reader = Compile(LoadToImageCode, LoadUserData(guest.Address(), guest.Address() + SourceOffset), target); + device->Dispatch(reader, 1u, 1u, 1u, {}, reinterpret_cast(LoadToImageCode.data())); + const auto afterRead = PendingAt(query); + Require(afterRead.read, "read-only lease writer: the build that only loads through BDA noted no lease over the guest block"); + if (counted) Require(afterRead.writer == afterRead.open, "read-only lease writer: APS5_NO_READ_ONLY_LEASE_SKIP did not count the read-only lease as a writer"); + else Require(afterRead.writer == 0, "read-only lease writer: the lease of a build that never stores through BDA counts as a writer (serial " + std::to_string(afterRead.writer) + ")"); + Require(AgcDriver::Graphics::StorageTexture::FlushPending(guest.Address(), ImageBytes, nullptr, "test"), "read-only lease writer: the read-only build's image store left no results pending"); + device->WaitIdle(); + Require(guest.Words()[0] == Loaded, "read-only lease writer: the read-only build did not store the loaded word into its image"); + + const auto storer = Compile(StoreGlobalCode, StoreUserData(guest.Address() + SourceOffset), target); + device->Dispatch(storer, 1u, 1u, 1u, {}, reinterpret_cast(StoreGlobalCode.data())); + const auto afterStore = PendingAt(query); + Require(afterStore.writer == afterStore.open, "read-only lease writer: the lease of a build that stores through BDA does not count as a writer"); + device->WaitIdle(); + Require(guest.Words()[(SourceOffset + 0x100u) / 4u] == Stored, "read-only lease writer: the storing build's word did not reach guest memory"); + + std::puts("read-only lease writer tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 8ecfba5ad4..19e1cdbce9 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -328,6 +328,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_PROC_TABLE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | resolves the device's Vulkan entry points per call instead of filling the function table once | | `APS5_NO_QUEUED_DRAW_READ_ORDER` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp` | | stops queued draws and DMA copies from holding back write-backs over the guest memory they read at commit | | `APS5_NO_QUEUED_LABELS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp` | | disables the queued-label table | +| `APS5_NO_READ_ONLY_LEASE_SKIP` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | counts the lease of an address-based build whose shaders never store through BDA as a pending writer of its leased range instead of only a reader | | `APS5_NO_RECENT_IMPORTS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | looks every host import up in the import map instead of first checking the last four found while the map is unchanged | | `APS5_NO_RECORDED_CLEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | waits for a storage image's fast clear in a batch of its own instead of recording it | | `APS5_NO_RECORDER_SYNC_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | gives the draws the recorder waits for their own batch instead of recording them with the wait | From 366e000c507313fff8cdbe9e5e17246aa59364e5 Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Mon, 2 Nov 2026 01:31:32 +0530 Subject: [PATCH 311/318] refactor(libc): constant-initialize fundamental typeinfo records and drop _ZTIz Point each record's vptr at the exported vtable's destroy slot via a relocation instead of a runtime FundamentalVtable() call, matching the constant void* vtable pattern in StandardExceptions.cpp. Remove _ZTIz: ellipsis is not a type so no compiler emits a _ZTIz reference, and its NID is the only one absent from the aerolib table. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- core/libs/prx/libc/src/exception/TypeInfo.cpp | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/core/libs/prx/libc/src/exception/TypeInfo.cpp b/core/libs/prx/libc/src/exception/TypeInfo.cpp index 31e3eaec25..3de5ad2496 100644 --- a/core/libs/prx/libc/src/exception/TypeInfo.cpp +++ b/core/libs/prx/libc/src/exception/TypeInfo.cpp @@ -90,37 +90,33 @@ LibcException::TypeInfoVtable _ZTVN10__cxxabiv123__fundamental_type_infoE_nid_po } namespace { -struct FundamentalTypeRecord { const void* const* vtable; const char* name; }; -const void* const* FundamentalVtable() { - return reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix) + 2; -} +struct FundamentalTypeRecord { const void* vtable; const char* name; }; } extern "C" { -FundamentalTypeRecord _ZTIv_nid_postfix {FundamentalVtable(), "v"}; -FundamentalTypeRecord _ZTIw_nid_postfix {FundamentalVtable(), "w"}; -FundamentalTypeRecord _ZTIb_nid_postfix {FundamentalVtable(), "b"}; -FundamentalTypeRecord _ZTIc_nid_postfix {FundamentalVtable(), "c"}; -FundamentalTypeRecord _ZTIa_nid_postfix {FundamentalVtable(), "a"}; -FundamentalTypeRecord _ZTIh_nid_postfix {FundamentalVtable(), "h"}; -FundamentalTypeRecord _ZTIs_nid_postfix {FundamentalVtable(), "s"}; -FundamentalTypeRecord _ZTIt_nid_postfix {FundamentalVtable(), "t"}; -FundamentalTypeRecord _ZTIi_nid_postfix {FundamentalVtable(), "i"}; -FundamentalTypeRecord _ZTIj_nid_postfix {FundamentalVtable(), "j"}; -FundamentalTypeRecord _ZTIl_nid_postfix {FundamentalVtable(), "l"}; -FundamentalTypeRecord _ZTIm_nid_postfix {FundamentalVtable(), "m"}; -FundamentalTypeRecord _ZTIx_nid_postfix {FundamentalVtable(), "x"}; -FundamentalTypeRecord _ZTIy_nid_postfix {FundamentalVtable(), "y"}; -FundamentalTypeRecord _ZTIn_nid_postfix {FundamentalVtable(), "n"}; -FundamentalTypeRecord _ZTIo_nid_postfix {FundamentalVtable(), "o"}; -FundamentalTypeRecord _ZTIf_nid_postfix {FundamentalVtable(), "f"}; -FundamentalTypeRecord _ZTId_nid_postfix {FundamentalVtable(), "d"}; -FundamentalTypeRecord _ZTIe_nid_postfix {FundamentalVtable(), "e"}; -FundamentalTypeRecord _ZTIg_nid_postfix {FundamentalVtable(), "g"}; -FundamentalTypeRecord _ZTIz_nid_postfix {FundamentalVtable(), "z"}; -FundamentalTypeRecord _ZTIDn_nid_postfix {FundamentalVtable(), "Dn"}; -FundamentalTypeRecord _ZTIDs_nid_postfix {FundamentalVtable(), "Ds"}; -FundamentalTypeRecord _ZTIDi_nid_postfix {FundamentalVtable(), "Di"}; -FundamentalTypeRecord _ZTIDu_nid_postfix {FundamentalVtable(), "Du"}; -FundamentalTypeRecord _ZTIDh_nid_postfix {FundamentalVtable(), "Dh"}; +FundamentalTypeRecord _ZTIv_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "v"}; +FundamentalTypeRecord _ZTIw_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "w"}; +FundamentalTypeRecord _ZTIb_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "b"}; +FundamentalTypeRecord _ZTIc_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "c"}; +FundamentalTypeRecord _ZTIa_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "a"}; +FundamentalTypeRecord _ZTIh_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "h"}; +FundamentalTypeRecord _ZTIs_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "s"}; +FundamentalTypeRecord _ZTIt_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "t"}; +FundamentalTypeRecord _ZTIi_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "i"}; +FundamentalTypeRecord _ZTIj_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "j"}; +FundamentalTypeRecord _ZTIl_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "l"}; +FundamentalTypeRecord _ZTIm_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "m"}; +FundamentalTypeRecord _ZTIx_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "x"}; +FundamentalTypeRecord _ZTIy_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "y"}; +FundamentalTypeRecord _ZTIn_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "n"}; +FundamentalTypeRecord _ZTIo_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "o"}; +FundamentalTypeRecord _ZTIf_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "f"}; +FundamentalTypeRecord _ZTId_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "d"}; +FundamentalTypeRecord _ZTIe_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "e"}; +FundamentalTypeRecord _ZTIg_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "g"}; +FundamentalTypeRecord _ZTIDn_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "Dn"}; +FundamentalTypeRecord _ZTIDs_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "Ds"}; +FundamentalTypeRecord _ZTIDi_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "Di"}; +FundamentalTypeRecord _ZTIDu_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "Du"}; +FundamentalTypeRecord _ZTIDh_nid_postfix {reinterpret_cast(&_ZTVN10__cxxabiv123__fundamental_type_infoE_nid_postfix.destroy), "Dh"}; } From b21d67cbdc590e37414c1412ea12b6c1014abbc9 Mon Sep 17 00:00:00 2001 From: Alexandre Figueiredo Date: Sun, 1 Nov 2026 20:05:41 +0000 Subject: [PATCH 312/318] fix(libSceNpCppWebApi): keep the object when a Vector pointer is assigned to itself --- core/libs/prx/libSceNpCppWebApi/Export.cpp | 3 +-- core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceNpCppWebApi/Export.cpp b/core/libs/prx/libSceNpCppWebApi/Export.cpp index 038cdfaf97..e2a5740e89 100644 --- a/core/libs/prx/libSceNpCppWebApi/Export.cpp +++ b/core/libs/prx/libSceNpCppWebApi/Export.cpp @@ -5958,8 +5958,7 @@ void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEED1Ev(In } void APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEEaSERS6_(IntrusivePtr* self, const IntrusivePtr* source) { - ReleaseIntrusivePtr(self, ScalarVectorReferenceCountOffset, __func__); - CopyIntrusivePtr(self, source, ScalarVectorReferenceCountOffset); + AssignIntrusivePtr(self, source, ScalarVectorReferenceCountOffset, __func__); } int APS5_VABI _ZNK3sce2Np9CppWebApi6Common6String5emptyEv(const CommonString* self) { diff --git a/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp b/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp index 0627667527..0a4d4e7e85 100644 --- a/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp +++ b/core/libs/tests/GuestNpCppWebApiIntrusivePtr.cpp @@ -118,6 +118,7 @@ static void TestAssign(const AssignCase& testCase) { testCase.assign(&pointer, &pointer); Require(pointer.object == first && pointer.deleter == RecordDelete && pointer.libContext == &context); Require(ReadCount(first, offset) == 1); + Require(deleterCalls == 0); const IntrusivePtr alias{first, nullptr, nullptr}; testCase.assign(&pointer, &alias); @@ -520,6 +521,7 @@ int main(int argc, char** argv) { {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_14SessionManager2V113PlayerSessionEEaSERS7_, 0}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS3_INS1_14SessionManager2V113PlayerSessionEEEEEEaSERKSA_, VectorReferenceCountOffset}, {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS3_INS1_13InGameCatalog2V59ContainerEEEEEEaSERKSA_, VectorReferenceCountOffset}, + {_ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorImEEEaSERS6_, ScalarVectorReferenceCountOffset}, }; for (const auto& testCase : assignments) TestAssign(testCase); From 813cbd96bbff05b8cceb44e1cb430793178fb2b4 Mon Sep 17 00:00:00 2001 From: orezraey Date: Sun, 1 Nov 2026 17:30:49 -0300 Subject: [PATCH 313/318] test(libc): check %lp through snwprintf_s Since #11893 the plain wide forms drop the length on %p before the branch this PR changes, so only the secure forms reach it. Check %lp through snwprintf_s, which returns -1 without the change, and record in the debt entry that no secure form was run with %lp on a console. Co-Authored-By: Claude Opus 5.5 --- core/libs/tests/GuestWideFormatting.cpp | 2 ++ docs/dev/debt/libc.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/libs/tests/GuestWideFormatting.cpp b/core/libs/tests/GuestWideFormatting.cpp index 1613f6c08a..ee79a5239b 100644 --- a/core/libs/tests/GuestWideFormatting.cpp +++ b/core/libs/tests/GuestWideFormatting.cpp @@ -117,6 +117,8 @@ static void CheckPointer() { Require(Format(buffer, 64, u"[%lp]", address(0)) == 3 && buffer == std::u16string(u"[0]"), "vswprintf %lp of a null pointer"); Require(Format(buffer, 64, u"[%5.0p][%05.3p][%.0p]", address(0), address(0x1f), address(0x1f)) == 18 && buffer == std::u16string(u"[ ][ 01f][1f]"), "vswprintf %p width beside a zero precision, 0 beside a precision, and %.0p of a pointer"); + Require(snwprintf_s_nid_postfix(buffer, 16, u"[%lp]", address(0x1f)) == 4 && buffer == std::u16string(u"[1f]"), "snwprintf_s %lp prints the pointer as %p does"); + Require(snwprintf_s_nid_postfix(buffer, 16, u"[%lp]", address(0)) == 3 && buffer == std::u16string(u"[0]"), "snwprintf_s %lp of a null pointer"); } static void CheckCount() { diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 046282f24c..0e8c4e7fd1 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -184,7 +184,7 @@ - [asinh](../../../core/libs/prx/libc/src/Math.cpp) (libc) is FreeBSD msun's [`s_asinh.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_asinh.c): below 2^-28 and up to |x| = 2 it runs over this file's msun `log1p` and gives msun's bits (0 of 2,000,000 random arguments in each of those ranges differ from a locally built msun on a Windows host). Above 2 it calls this file's `log`, which is still the host's, so those results can differ from msun in the last bit until `log` follows msun: 38,719 of 2,000,000 random arguments in (2, 2^28] differ with the Windows UCRT, none above 2^28 there (not measured with glibc). It sets no `errno`, as msun's does not, and was compared with msun built on the host, not with a console: the titles' `libc.prx` and the system `libSceLibcInternal` are unmeasured. - [%a and %A](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) format a finite double as the `libc.prx` that titles ship does, instead of through the host: a subnormal is normalised (`0x1p-1074`), a shorter precision rounds half up on the dropped hexadecimal digits and a carry stays in the leading digit (`%.3a` of `0x0.fffffffffffffp-1022` is `0x2.000p-1023`), and `-` overrides `0`. Checked by running the copies of PPSA13934, PPSA07923 and PPSA05916 side by side on 7 million random formats, not measured on a console. A `long double` is normalised the same way, with 16 hexadecimal digits (`0x1.999999999999999ap-4` for 0.1L). Infinities still go to the host; NaN keeps its sign and flags through the formatter on current main. - [%e, %f and %g](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) format a finite `double` or `long double` as the `libc.prx` that titles ship does, instead of through the host. It keeps the exact decimal digits down to position `4 * floor(k / 4) - 20`, where `k` is the `frexp` exponent times 30103 / 100000 truncated, and prints zeros after them; a shorter precision rounds half up on the kept digits, except that an exact tie rounds down when `k` is at least 8 (`%.0f` of 67108865.5 is `67108865`). The console computes the kept digits in extended precision, so about 7% of outputs longer than 17 significant digits end one unit lower than here (`%f` of 1e22 is `9999999999999999999999.000000`). Checked by running the copies of PPSA07923, PPSA05916 and PPSA13934 side by side, not measured on a console. The wide printf family uses the same digits, as the titles' `swprintf` gives the same text as their `snprintf`. -- [%p and %lp](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) follow the console's `libSceLibcInternal.sprx` measured from a homebrew payload on firmware 12.70 ([#7549](https://github.com/boykopovar/AnyPS5/issues/7549)), not FreeBSD: bare lowercase hex with no `0x` prefix even under `#`, `0` for a null pointer, `+` and the space flag ignored, the width counting only the digits, `-` padding on the right, `0` padding with zeros when there is no precision, the precision as the minimum number of digits, nothing for a null pointer under `%.0p`, and `l` accepted. Only `snprintf` and `swprintf` of that module were measured, from a payload process rather than a title process. On firmware 13.60 a width beside `%.0p` pads an empty field (`%5.0p` of null gives five spaces), `%05.3p` gives ` 01f`, `%.0p` of a non-null pointer prints its digits and `swprintf` prints `%lp` as `%p`, as this formatter does; another length such as `%hp` follows the printf length modifiers entry. Linux routes `%p` and `%lp` to this formatter because glibc prints the FreeBSD form and `(nil)` for a null pointer, which means a format that combines `%p` with a conversion this formatter refuses, such as an argument position above 99, throws there instead of reaching glibc +- [%p and %lp](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) follow `libSceLibcInternal.sprx` as measured from a homebrew payload process, not a title process, on firmware 12.70 ([#7549](https://github.com/boykopovar/AnyPS5/issues/7549)) and 13.60, only through `snprintf` and `swprintf`. No `_s` form was run with `%lp` on a console; the wide `_s` forms print it as the narrow ones here do. Measured: bare lowercase hex with no `0x` even under `#`, `0` for null, `+` and space ignored, the width counting only the digits, `-` padding on the right, `0` padding with zeros when there is no precision, the precision as the minimum digits, nothing for null under `%.0p` (`%5.0p` gives five spaces), `%05.3p` gives ` 01f`, `%.0p` of non-null prints its digits, and `l` accepted. Another length such as `%hp` follows the printf length modifiers entry. Linux routes `%p` and `%lp` here because glibc prints the FreeBSD form and `(nil)`, so a format combining `%p` with a conversion this formatter refuses, such as an argument position above 99, throws there instead of reaching glibc - [collate and collate _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build the facet for the C locale only and throw for any other locale name, the wide entry point following the narrow one it sits next to. The wide facet carries its own vtable, whose `compare`, `transform` and `hash` take the guest's 16-bit characters, so the length of a range is a count of characters: `compare` orders by 16-bit value (the C locale, as Dinkumware's `char_traits::compare` does), `transform` copies the characters into the result and records size and capacity in 16-bit elements (inline capacity 7, heap capacity `length | 7`, `(capacity + 1) * 2` bytes including the terminator), following Dinkumware's [VC12 `basic_string`](https://github.com/ganboing/crt.vc12/blob/f1d47126bf13d1c9dccaa6c53b72217fd0474968/src/xstring#L500-L531), and `hash` mixes one whole character per round, so wide text below 0x100 hashes like its narrow spelling and a character above 0xff hashes as one value instead of the two bytes it holds. The console's `libc.prx` exports both `_Getcat` (NIDs BSVJqITGCyI and 2wz4rthdiy8) with identical 0x19d-byte bodies, believed to build the same 0x20-byte facet; the console bodies themselves are unreadable (obfuscated dynsym, and the text of the relinked `libc.prx` is zeroed), so the wide entries follow the narrow algorithm and the C locale instead of the console's wide code, and no console run checks the two. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size. The narrow result capacity (`length | 15`) and the over-aligned storage of results of 4080 characters or more follow the console's `libc.prx` code and are not checked on hardware. No module of the PPSA59322, PPSA76677 or PPSA97358 dumps in this tree imports either `_Getcat`, so neither has been exercised by a title. - [codecvt, numpunct, num_put and time_put _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build facets matching a bundled console `libc.prx` (dynsym decoded by NID, vtables disassembled): categories 2, 4, 4 and 5, sizes 0x10, 0x30, 0x10 and 0xb8. Unlike `collate`, they ignore the locale argument and always build the C-locale facet; the console's behaviour for a non-C name is unverified. `time_put`'s `do_put` formats through this library's `strftime` instead of the facet's unpopulated `_Locinfo` table, so the `strftime` gaps apply. `codecvt` reproduces the console virtuals (`noconv` conversions, `length` the shorter range); `numpunct` copies its C-locale table (`""`, `.`, `,`, `false`, `true`). The facets share the no-op `destroy`/`deleteObject`/`retain`/`release` base used for `collate`, so `numpunct`'s static C-locale strings are never freed. - [num_put::do_put](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) follows the SDK 7.00 `libc.prx` that PPSA06416 ships, read from its disassembly and run, but formats through this library's `snprintf`. That converts floating point exactly and rounds ties to even, where the module keeps about 18 exact significant digits, prints later ones from an inexact scaling (1e10 at precision 22 is `9999999999.999999999999`) and gives `-3` for -2.5 at precision 0. A precision so negative that the module's buffer of precision + 50 bytes cannot hold the result throws, where the module reads stale bytes. A stream locale without a `ctype` facet leaves digits unwidened, as the console's C-locale facet does. From 246110e5255a1d72d1b5560568193682feaee54f Mon Sep 17 00:00:00 2001 From: orezraey Date: Sun, 1 Nov 2026 17:31:20 -0300 Subject: [PATCH 314/318] docs(libc): keep the %p debt entry within the length limit Co-Authored-By: Claude Opus 5.5 --- docs/dev/debt/libc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index 0e8c4e7fd1..94b3b20876 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -184,7 +184,7 @@ - [asinh](../../../core/libs/prx/libc/src/Math.cpp) (libc) is FreeBSD msun's [`s_asinh.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_asinh.c): below 2^-28 and up to |x| = 2 it runs over this file's msun `log1p` and gives msun's bits (0 of 2,000,000 random arguments in each of those ranges differ from a locally built msun on a Windows host). Above 2 it calls this file's `log`, which is still the host's, so those results can differ from msun in the last bit until `log` follows msun: 38,719 of 2,000,000 random arguments in (2, 2^28] differ with the Windows UCRT, none above 2^28 there (not measured with glibc). It sets no `errno`, as msun's does not, and was compared with msun built on the host, not with a console: the titles' `libc.prx` and the system `libSceLibcInternal` are unmeasured. - [%a and %A](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) format a finite double as the `libc.prx` that titles ship does, instead of through the host: a subnormal is normalised (`0x1p-1074`), a shorter precision rounds half up on the dropped hexadecimal digits and a carry stays in the leading digit (`%.3a` of `0x0.fffffffffffffp-1022` is `0x2.000p-1023`), and `-` overrides `0`. Checked by running the copies of PPSA13934, PPSA07923 and PPSA05916 side by side on 7 million random formats, not measured on a console. A `long double` is normalised the same way, with 16 hexadecimal digits (`0x1.999999999999999ap-4` for 0.1L). Infinities still go to the host; NaN keeps its sign and flags through the formatter on current main. - [%e, %f and %g](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) format a finite `double` or `long double` as the `libc.prx` that titles ship does, instead of through the host. It keeps the exact decimal digits down to position `4 * floor(k / 4) - 20`, where `k` is the `frexp` exponent times 30103 / 100000 truncated, and prints zeros after them; a shorter precision rounds half up on the kept digits, except that an exact tie rounds down when `k` is at least 8 (`%.0f` of 67108865.5 is `67108865`). The console computes the kept digits in extended precision, so about 7% of outputs longer than 17 significant digits end one unit lower than here (`%f` of 1e22 is `9999999999999999999999.000000`). Checked by running the copies of PPSA07923, PPSA05916 and PPSA13934 side by side, not measured on a console. The wide printf family uses the same digits, as the titles' `swprintf` gives the same text as their `snprintf`. -- [%p and %lp](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) follow `libSceLibcInternal.sprx` as measured from a homebrew payload process, not a title process, on firmware 12.70 ([#7549](https://github.com/boykopovar/AnyPS5/issues/7549)) and 13.60, only through `snprintf` and `swprintf`. No `_s` form was run with `%lp` on a console; the wide `_s` forms print it as the narrow ones here do. Measured: bare lowercase hex with no `0x` even under `#`, `0` for null, `+` and space ignored, the width counting only the digits, `-` padding on the right, `0` padding with zeros when there is no precision, the precision as the minimum digits, nothing for null under `%.0p` (`%5.0p` gives five spaces), `%05.3p` gives ` 01f`, `%.0p` of non-null prints its digits, and `l` accepted. Another length such as `%hp` follows the printf length modifiers entry. Linux routes `%p` and `%lp` here because glibc prints the FreeBSD form and `(nil)`, so a format combining `%p` with a conversion this formatter refuses, such as an argument position above 99, throws there instead of reaching glibc +- [%p and %lp](../../../core/libs/prx/libc/include/WindowsFormatting.hpp) (libc) follow `libSceLibcInternal.sprx` as measured from a homebrew payload, not a title, on firmware 12.70 ([#7549](https://github.com/boykopovar/AnyPS5/issues/7549)) and 13.60, only through `snprintf` and `swprintf`. No `_s` form was run with `%lp` on a console; the wide `_s` forms print it as the narrow ones here do. Measured: bare lowercase hex with no `0x` even under `#`, `0` for null, `+` and space ignored, the width counting digits only, `-` padding right, `0` zero-padding without a precision, the precision as the minimum digits, nothing for null under `%.0p` (`%5.0p` gives five spaces), `%05.3p` gives ` 01f`, `%.0p` of non-null prints its digits, and `l` accepted. Another length such as `%hp` follows the printf length modifiers entry. Linux routes `%p` and `%lp` here because glibc prints the FreeBSD form, so a format combining `%p` with a conversion this formatter refuses throws there - [collate and collate _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build the facet for the C locale only and throw for any other locale name, the wide entry point following the narrow one it sits next to. The wide facet carries its own vtable, whose `compare`, `transform` and `hash` take the guest's 16-bit characters, so the length of a range is a count of characters: `compare` orders by 16-bit value (the C locale, as Dinkumware's `char_traits::compare` does), `transform` copies the characters into the result and records size and capacity in 16-bit elements (inline capacity 7, heap capacity `length | 7`, `(capacity + 1) * 2` bytes including the terminator), following Dinkumware's [VC12 `basic_string`](https://github.com/ganboing/crt.vc12/blob/f1d47126bf13d1c9dccaa6c53b72217fd0474968/src/xstring#L500-L531), and `hash` mixes one whole character per round, so wide text below 0x100 hashes like its narrow spelling and a character above 0xff hashes as one value instead of the two bytes it holds. The console's `libc.prx` exports both `_Getcat` (NIDs BSVJqITGCyI and 2wz4rthdiy8) with identical 0x19d-byte bodies, believed to build the same 0x20-byte facet; the console bodies themselves are unreadable (obfuscated dynsym, and the text of the relinked `libc.prx` is zeroed), so the wide entries follow the narrow algorithm and the C locale instead of the console's wide code, and no console run checks the two. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size. The narrow result capacity (`length | 15`) and the over-aligned storage of results of 4080 characters or more follow the console's `libc.prx` code and are not checked on hardware. No module of the PPSA59322, PPSA76677 or PPSA97358 dumps in this tree imports either `_Getcat`, so neither has been exercised by a title. - [codecvt, numpunct, num_put and time_put _Getcat](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) build facets matching a bundled console `libc.prx` (dynsym decoded by NID, vtables disassembled): categories 2, 4, 4 and 5, sizes 0x10, 0x30, 0x10 and 0xb8. Unlike `collate`, they ignore the locale argument and always build the C-locale facet; the console's behaviour for a non-C name is unverified. `time_put`'s `do_put` formats through this library's `strftime` instead of the facet's unpopulated `_Locinfo` table, so the `strftime` gaps apply. `codecvt` reproduces the console virtuals (`noconv` conversions, `length` the shorter range); `numpunct` copies its C-locale table (`""`, `.`, `,`, `false`, `true`). The facets share the no-op `destroy`/`deleteObject`/`retain`/`release` base used for `collate`, so `numpunct`'s static C-locale strings are never freed. - [num_put::do_put](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) follows the SDK 7.00 `libc.prx` that PPSA06416 ships, read from its disassembly and run, but formats through this library's `snprintf`. That converts floating point exactly and rounds ties to even, where the module keeps about 18 exact significant digits, prints later ones from an inexact scaling (1e10 at precision 22 is `9999999999.999999999999`) and gives `-3` for -2.5 at precision 0. A precision so negative that the module's buffer of precision + 50 bytes cannot hold the result throws, where the module reads stale bytes. A stream locale without a `ctype` facet leaves digits unwidened, as the console's C-locale facet does. From 81bc358860e1253ffc37ae0d848c845a7b73722c Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:20:13 +0000 Subject: [PATCH 315/318] fix(libSceNet): take interface-name zones in the scope-ID text forms rooklz read sceNetInetNtopWithScopeId (FUN_00006be0) and sceNetInetPtonWithScopeId (FUN_000072d0) in libSceNet.sprx. The zone after '%' is one of the ten interface names, never a number, and ntop prints the name of the interface whose index is the scope. A null address or text and a short buffer set ENOSPC, and another family sets EAFNOSUPPORT. Pton reads a zone only when the scope output is not null, and answers another family with EAFNOSUPPORT and a null text with EINVAL. The host has none of these interfaces. Names and indexes go through the interface table and host-index mapping of rooklz's "feat(net): resolve interface ids to host indexes", taken with the same text so the two changes merge without conflict there. A null destination throws std::invalid_argument, where the module faults. Co-authored-by: rooklz <175737347+rooklz@users.noreply.github.com> --- core/libs/CMakeLists.txt | 2 +- core/libs/prx/libSceNet/CMakeLists.txt | 4 +- core/libs/prx/libSceNet/Export.cpp | 114 ++++++++++++++++++++----- core/libs/tests/GuestSceNet.cpp | 41 +++++---- docs/dev/debt/libSceNet.md | 2 +- 5 files changed, 124 insertions(+), 39 deletions(-) diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index 0679e4c24b..262c7866ae 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -181,7 +181,7 @@ foreach(LIB_DIR ${LIBS_DIRS}) target_link_options(${LIB_NAME} PRIVATE LINKER:-z,now) endif() if(WIN32 AND LIB_NAME STREQUAL "libSceNet") - target_link_libraries(${LIB_NAME} PRIVATE ws2_32) + target_link_libraries(${LIB_NAME} PRIVATE ws2_32 iphlpapi) target_compile_definitions(${LIB_NAME} PRIVATE _WIN32_WINNT=0x0602) elseif(WIN32 AND LIB_NAME STREQUAL "libSceNetCtl") target_link_libraries(${LIB_NAME} PRIVATE ws2_32 iphlpapi) diff --git a/core/libs/prx/libSceNet/CMakeLists.txt b/core/libs/prx/libSceNet/CMakeLists.txt index acc487792e..f23d20ffa7 100644 --- a/core/libs/prx/libSceNet/CMakeLists.txt +++ b/core/libs/prx/libSceNet/CMakeLists.txt @@ -86,7 +86,7 @@ if(BUILD_TESTING) target_link_libraries(net_blocking_send_tests PRIVATE libkernel libc libSceNetCtl) configure_windows_unwind(net_blocking_send_tests) if(WIN32) - target_link_libraries(net_blocking_send_tests PRIVATE ws2_32) + target_link_libraries(net_blocking_send_tests PRIVATE ws2_32 iphlpapi) endif() foreach(api IN ITEMS send sendto sendmsg) add_test(NAME net_dontwait_blocking_${api} COMMAND net_blocking_send_tests ${api}) @@ -209,7 +209,7 @@ if(BUILD_TESTING) target_link_libraries(net_epoll_lifetime_tests PRIVATE libkernel libc libSceNetCtl) configure_windows_unwind(net_epoll_lifetime_tests) if(WIN32) - target_link_libraries(net_epoll_lifetime_tests PRIVATE ws2_32) + target_link_libraries(net_epoll_lifetime_tests PRIVATE ws2_32 iphlpapi) endif() foreach(operation IN ITEMS reuse control close delete modify replace abort destroy) add_test(NAME net_epoll_lifetime_${operation} COMMAND net_epoll_lifetime_tests ${operation}) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index 9819703ac2..f29211db31 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -6,9 +6,12 @@ #include #include #include +#include #else #include #include +#include +#include #include #include #include @@ -33,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -136,6 +140,8 @@ constexpr std::uint32_t NET_IFF_LOOPBACK = 0x8; constexpr std::uint32_t NET_IFF_RUNNING = 0x40; constexpr std::uint32_t NET_IFF_MULTICAST = 0x8000; constexpr std::uint32_t NET_IF_LO0 = 0; +constexpr std::array NET_INTERFACE_NAMES = { + "lo0", "eth0", "eth1", "dbg0", "wlan0", "wlan1", "gbe0", "bt0", "phone0", "pppoe0"}; constexpr int NET_AF_INET = 2; constexpr int NET_AF_INET6 = 28; @@ -398,6 +404,56 @@ int fail(int err) { return NET_ERROR_BASE | err; } +std::optional HostInterfaceByRole(bool loopback) { +#ifdef _WIN32 + const ULONG flags = GAA_FLAG_SKIP_ANYCAST | GAA_FLAG_SKIP_MULTICAST | GAA_FLAG_SKIP_DNS_SERVER; + ULONG size = 0; + if (GetAdaptersAddresses(AF_UNSPEC, flags, nullptr, nullptr, &size) != ERROR_BUFFER_OVERFLOW) return std::nullopt; + std::vector storage(size); + auto* adapters = reinterpret_cast(storage.data()); + if (GetAdaptersAddresses(AF_UNSPEC, flags, nullptr, adapters, &size) != NO_ERROR) return std::nullopt; + std::optional primary; + for (auto* adapter = adapters; adapter != nullptr; adapter = adapter->Next) { + if (adapter->IfIndex == 0) continue; + if (adapter->IfType == IF_TYPE_SOFTWARE_LOOPBACK) { + if (loopback) return static_cast(adapter->IfIndex); + continue; + } + if (!primary && adapter->OperStatus == IfOperStatusUp && adapter->FirstUnicastAddress != nullptr) + primary = static_cast(adapter->IfIndex); + } + return loopback ? std::nullopt : primary; +#else + ifaddrs* list = nullptr; + if (getifaddrs(&list) != 0) return std::nullopt; + std::uint32_t loop = 0; + std::optional primary; + for (auto* item = list; item != nullptr; item = item->ifa_next) { + if (item->ifa_name == nullptr || !(item->ifa_flags & IFF_UP)) continue; + const unsigned int index = if_nametoindex(item->ifa_name); + if (index == 0) continue; + if ((item->ifa_flags & IFF_LOOPBACK) != 0) { + if (loop == 0) loop = index; + continue; + } + if (!primary && item->ifa_addr != nullptr && + (item->ifa_addr->sa_family == AF_INET || item->ifa_addr->sa_family == AF_INET6)) + primary = static_cast(index); + } + freeifaddrs(list); + if (loopback) return loop == 0 ? std::nullopt : std::optional(loop); + return primary; +#endif +} + +std::optional HostInterfaceIndex(std::uint32_t interfaceId) { + if (interfaceId >= NET_INTERFACE_NAMES.size()) return std::nullopt; + if (interfaceId == 0) return HostInterfaceByRole(true); + if (interfaceId == 1) return HostInterfaceByRole(false); + const unsigned int index = if_nametoindex(NET_INTERFACE_NAMES[interfaceId]); + return index == 0 ? std::nullopt : std::optional(index); +} + void* APS5_VABI RunNetThread(void* param) { const auto* thread = static_cast(param); thread->entry(thread->arg); @@ -735,6 +791,20 @@ bool parse_ipv4_parts(const char* text, std::uint32_t& address) { return true; } +std::optional InterfaceIdOfName(std::string_view name) { + for (std::uint32_t interfaceId = 0; interfaceId < NET_INTERFACE_NAMES.size(); ++interfaceId) { + if (name == NET_INTERFACE_NAMES[interfaceId]) return interfaceId; + } + return std::nullopt; +} + +std::optional InterfaceIdOfHostIndex(std::uint32_t index) { + for (std::uint32_t interfaceId = 0; interfaceId < NET_INTERFACE_NAMES.size(); ++interfaceId) { + if (HostInterfaceIndex(interfaceId) == index) return interfaceId; + } + return std::nullopt; +} + } // namespace void GuestSceNetTest::SetResolverLookupForTests_nid_no_patch(ResolverLookup lookup) noexcept { @@ -2728,13 +2798,11 @@ int APS5_VABI sceNetGetIfListOnce(std::int32_t interfaceId, NetInterfaceStatisti } const char* APS5_VABI sceNetGetIfName(std::uint32_t index) { - static constexpr std::array names = { - "lo0", "eth0", "eth1", "dbg0", "wlan0", "wlan1", "gbe0", "bt0", "phone0", "pppoe0"}; - if (index >= names.size()) { + if (index >= NET_INTERFACE_NAMES.size()) { set_errno(NET_EINVAL); return ""; } - return names[index]; + return NET_INTERFACE_NAMES[index]; } int APS5_VABI sceNetGetIfnameNumList(std::uint32_t* interfaceIds, std::int32_t capacity, std::uint32_t flags) { @@ -2960,14 +3028,22 @@ int APS5_VABI sceNetShowNetstatExForBuffer(char* buffer, std::int32_t bufferSize const char* APS5_VABI sceNetInetNtopWithScopeId(std::int32_t family, const void* address, char* text, std::uint32_t size, std::uint32_t scopeId) { if (address == nullptr || text == nullptr) { - set_errno(NET_EINVAL); + set_errno(NET_ENOSPC); + return nullptr; + } + if (family != NET_AF_INET6) { + set_errno(NET_EAFNOSUPPORT); return nullptr; } - if (family != NET_AF_INET6) throw std::runtime_error("sceNetInetNtopWithScopeId: family " + std::to_string(family) + " is not implemented"); std::array unscoped{}; if (sceNetInetNtop(family, address, unscoped.data(), static_cast(unscoped.size())) == nullptr) return nullptr; std::string scoped(unscoped.data()); - if (scopeId != 0) scoped += "%" + std::to_string(scopeId); + if (scopeId != 0) { + const auto interfaceId = InterfaceIdOfHostIndex(scopeId); + if (!interfaceId) throw std::runtime_error("sceNetInetNtopWithScopeId: scope " + std::to_string(scopeId) + " matches no interface, and the module's answer for it is not known"); + scoped += '%'; + scoped += NET_INTERFACE_NAMES[*interfaceId]; + } if (scoped.size() >= size) { set_errno(NET_ENOSPC); return nullptr; @@ -2977,26 +3053,24 @@ const char* APS5_VABI sceNetInetNtopWithScopeId(std::int32_t family, const void* } int APS5_VABI sceNetInetPtonWithScopeId(std::int32_t family, const char* text, void* address, std::uint32_t* scopeId) { - if (text == nullptr || address == nullptr || scopeId == nullptr) return fail(NET_EINVAL); - if (family != NET_AF_INET6) throw std::runtime_error("sceNetInetPtonWithScopeId: family " + std::to_string(family) + " is not implemented"); - const char* percent = std::strchr(text, '%'); - std::uint64_t zone = 0; + if (family != NET_AF_INET6) return fail(NET_EAFNOSUPPORT); + if (text == nullptr) return fail(NET_EINVAL); + const char* percent = scopeId != nullptr ? std::strchr(text, '%') : nullptr; + std::uint32_t zone = 0; if (percent != nullptr) { - const std::string_view digits(percent + 1); - if (digits.empty()) return 0; - if (!std::all_of(digits.begin(), digits.end(), [](char digit) { return digit >= '0' && digit <= '9'; })) - throw std::runtime_error("sceNetInetPtonWithScopeId: the interface name zone in " + std::string(text) + " is not implemented"); - for (const char digit : digits) { - zone = zone * 10 + static_cast(digit - '0'); - if (zone > 0xffffffffull) return 0; - } + const auto interfaceId = InterfaceIdOfName(percent + 1); + if (!interfaceId) return 0; + const auto index = HostInterfaceIndex(*interfaceId); + if (!index) return 0; + zone = *index; } const std::string unscoped = percent != nullptr ? std::string(text, percent) : std::string(text); std::array parsed{}; const int converted = sceNetInetPton(family, unscoped.c_str(), parsed.data()); if (converted != 1) return converted; + if (address == nullptr) throw std::invalid_argument("sceNetInetPtonWithScopeId: null address"); std::memcpy(address, parsed.data(), parsed.size()); - *scopeId = static_cast(zone); + if (scopeId != nullptr) *scopeId = zone; return 1; } diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 941de1c717..0dc1c73240 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -386,33 +386,44 @@ static void CheckSysctl() { } static void CheckScopeIdText() { - std::array address{}; - std::uint32_t scope = 0xa5a5a5a5u; - Require(sceNetInetPtonWithScopeId(28, "fe80::1%2", address.data(), &scope) == 1 && scope == 2); std::array plain{}; - Require(sceNetInetPton(28, "fe80::1", plain.data()) == 1 && address == plain); + Require(sceNetInetPton(28, "fe80::1", plain.data()) == 1); + std::array address{}; + std::uint32_t scope = 0; + Require(sceNetInetPtonWithScopeId(28, "fe80::1%lo0", address.data(), &scope) == 1 && scope != 0 && address == plain); std::array text{}; Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), text.size(), scope) == text.data()); - Require(std::strcmp(text.data(), "fe80::1%2") == 0); + Require(std::strcmp(text.data(), "fe80::1%lo0") == 0); text.fill('x'); *sceNetErrnoLoc() = 123; - Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), 9, 2) == nullptr && *sceNetErrnoLoc() == 28 && text[0] == 'x'); - Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), 10, 2) == text.data() && std::strcmp(text.data(), "fe80::1%2") == 0); + Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), 11, scope) == nullptr && *sceNetErrnoLoc() == 28 && text[0] == 'x'); + Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), 12, scope) == text.data() && std::strcmp(text.data(), "fe80::1%lo0") == 0); Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), text.size(), 0) == text.data() && std::strcmp(text.data(), "fe80::1") == 0); - scope = 7; - Require(sceNetInetPtonWithScopeId(28, "fe80::1", address.data(), &scope) == 1 && scope == 0 && address == plain); - Require(sceNetInetPtonWithScopeId(28, "ff02::1%4294967295", address.data(), &scope) == 1 && scope == 0xffffffffu); - Require(sceNetInetNtopWithScopeId(28, address.data(), text.data(), text.size(), scope) == text.data()); - Require(std::strcmp(text.data(), "ff02::1%4294967295") == 0); - for (const char* rejected : {"fe80::1%", "fe80::1%4294967296", "fe80::g%2", "1.2.3.4%2"}) { + for (const char* rejected : {"fe80::1%2", "ff02::1%4294967295", "fe80::1%", "fe80::1%eth9", "fe80::g%lo0", "1.2.3.4%lo0"}) { address.fill(0xa5); scope = 0xa5a5a5a5u; Require(sceNetInetPtonWithScopeId(28, rejected, address.data(), &scope) == 0); Require(scope == 0xa5a5a5a5u && address[0] == 0xa5 && address[15] == 0xa5); } - Require(Failed(sceNetInetPtonWithScopeId(28, "fe80::1%2", address.data(), nullptr), 22)); + address.fill(0xa5); + Require(sceNetInetPtonWithScopeId(28, "fe80::1", address.data(), nullptr) == 1 && address == plain); + address.fill(0xa5); + Require(sceNetInetPtonWithScopeId(28, "fe80::1%lo0", address.data(), nullptr) == 0 && address[0] == 0xa5 && address[15] == 0xa5); + Require(Failed(sceNetInetPtonWithScopeId(2, "fe80::1", address.data(), &scope), 47)); + Require(Failed(sceNetInetPtonWithScopeId(28, nullptr, address.data(), &scope), 22)); + bool threw = false; + try { + sceNetInetPtonWithScopeId(28, "fe80::1", nullptr, &scope); + } catch (const std::invalid_argument&) { + threw = true; + } + Require(threw); *sceNetErrnoLoc() = 123; - Require(sceNetInetNtopWithScopeId(28, nullptr, text.data(), text.size(), 2) == nullptr && *sceNetErrnoLoc() == 22); + Require(sceNetInetNtopWithScopeId(28, nullptr, text.data(), text.size(), 0) == nullptr && *sceNetErrnoLoc() == 28); + *sceNetErrnoLoc() = 123; + Require(sceNetInetNtopWithScopeId(28, plain.data(), nullptr, 32, 0) == nullptr && *sceNetErrnoLoc() == 28); + *sceNetErrnoLoc() = 123; + Require(sceNetInetNtopWithScopeId(2, plain.data(), text.data(), text.size(), 0) == nullptr && *sceNetErrnoLoc() == 47); } static void CheckDefaultRoute() { diff --git a/docs/dev/debt/libSceNet.md b/docs/dev/debt/libSceNet.md index 295c2df966..63923a35b7 100644 --- a/docs/dev/debt/libSceNet.md +++ b/docs/dev/debt/libSceNet.md @@ -32,7 +32,7 @@ - [sceNetConfigRoutingShowRoutingConfig](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - the native routine consumes no arguments; the throwing implementation remains unchanged. - [sceNetSocketInternal, sceNetGetsockoptEx and sceNetSetsockoptEx](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - forward to sceNetSocket, sceNetGetsockopt and sceNetSetsockopt. Their restored argument lists match the public ones, and nothing in the tree records how the console tells these entry points apart from the public ones (for example by the calling process). - [sceNetSysctl](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - calls libkernel's sysctl with the FreeBSD sysctl(3) argument list, so it answers exactly the MIBs libkernel models, and reports a failure as the library's other calls do (`0x80410100 | errno`, with the errno set). How the console's routine reports errors is not measured. -- [sceNetInetNtopWithScopeId and sceNetInetPtonWithScopeId](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - convert IPv6 text with a numeric zone (`fe80::1%2`, [RFC 4007 section 11](https://www.rfc-editor.org/rfc/rfc4007#section-11)) through sceNetInetNtop and sceNetInetPton, as FreeBSD 11.4's `getnameinfo` with `NI_NUMERICSCOPE` prints it and `getaddrinfo` parses it. A zone given as an interface name, and `AF_INET`, throw: the tree has no mapping from interface names to the console's interface indexes, and no answer for an IPv4 zone. +- [sceNetInetNtopWithScopeId and sceNetInetPtonWithScopeId](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - console behaviour from the module; slot-to-host-index mapping is ours. A zone name maps to a host index (`lo0` the loopback, `eth0` the first up interface with an address, the others a host interface of that name); a name with no host interface does not convert, and ntop prints the first name, in sceNetGetIfName's order, with the scope's index. A null destination throws `std::invalid_argument`, where the module faults. Not measured: ntop for a scope that matches no interface (throws), exact name comparison, the 0 written to the scope for a plain address, and the order of the family and null checks. Module: rooklz's [reading](https://github.com/boykopovar/AnyPS5/pull/9578#issuecomment-6549946559) of `libSceNet.sprx`, FUN_00006be0 (FUN_000065e0) and FUN_000072d0 (FUN_00006ee0), names through FUN_00005d40, FUN_00005ea0 and FUN_00005f20. - [sceNetConfigGetDefaultRoute](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - reports the IPv4 gateway of the host interface that sceNetCtlGetInfo describes (`DEFAULT_ROUTE`), in network byte order, the order sceNetInetPton writes an `AF_INET` address in. With no gateway, or a null pointer, it throws: the console's answer for a missing route is not measured. ### Functional From 0a4d4e792cb2ee0f9a5ad7ad7d0069965c7b319f Mon Sep 17 00:00:00 2001 From: Yunus Shaikh <75296019+EagleBearerYS@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:21:53 +0000 Subject: [PATCH 316/318] fix(libSceNet): answer sceNetConfigGetDefaultRoute as the module does rooklz read sceNetConfigGetDefaultRoute (NID lFJb+BlPK1c, wrapper at 0x153f0) in libSceNet.sprx. Before sceNetInit it returns 0x804101C8 with errno 200. Without a default route it returns 0x8041130C and leaves the output alone. With one it stores the route and returns 1, not 0. The wrapper never checks the output pointer. The not-initialized answer reads the library's sceNetInit/sceNetTerm flag through fail(), as the other exports do. The route is still the host gateway. A null output with a route throws std::invalid_argument, where the module faults. Co-authored-by: rooklz <175737347+rooklz@users.noreply.github.com> --- core/libs/prx/libSceNet/Export.cpp | 8 +++++--- core/libs/tests/GuestSceNet.cpp | 26 +++++++++++++++----------- docs/dev/debt/libSceNet.md | 2 +- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index f29211db31..14cfadb103 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -133,6 +133,7 @@ constexpr int NET_ENOTINIT = 200; constexpr int NET_ERROR_BASE = static_cast(0x80410100u); constexpr int NET_ERROR_RESOLVER_ENODNS = static_cast(0x804101E1u); constexpr int NET_CONFIG_ERROR_NULL_FLAGS = static_cast(0x8041130Eu); +constexpr int NET_CONFIG_ERROR_NO_ROUTE = static_cast(0x8041130Cu); constexpr int NET_RESOLVER_ABORT_FLAG_NTOA_PRESERVATION = 0x1; constexpr int NET_RESOLVER_ABORT_FLAG_ATON_PRESERVATION = 0x2; constexpr std::uint32_t NET_IFF_UP = 0x1; @@ -3237,11 +3238,12 @@ int APS5_VABI sceNetConfigDelRoute(const std::uint32_t* destination, const std:: } int APS5_VABI sceNetConfigGetDefaultRoute(std::uint32_t* gateway) { - if (!gateway) throw std::runtime_error("sceNetConfigGetDefaultRoute: a null gateway is not supported"); + if (!g_net_inited) return fail(NET_ENOTINIT); std::uint32_t found = 0; - if (!NetCtlHostIpv4Gateway_nid_no_patch(&found)) throw std::runtime_error("sceNetConfigGetDefaultRoute: the answer without an IPv4 default route is not implemented"); + if (!NetCtlHostIpv4Gateway_nid_no_patch(&found)) return NET_CONFIG_ERROR_NO_ROUTE; + if (gateway == nullptr) throw std::invalid_argument("sceNetConfigGetDefaultRoute: null gateway"); *gateway = found; - return 0; + return 1; } int APS5_VABI sceNetDhcpGetAutoipInfo(std::int32_t interfaceId, NetAutoIpInfo* parameter) { diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 0dc1c73240..6958db34d6 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -427,27 +427,31 @@ static void CheckScopeIdText() { } static void CheckDefaultRoute() { + std::uint32_t gateway = 0xa5a5a5a5u; + Require(sceNetTerm() == 0); + Require(Failed(sceNetConfigGetDefaultRoute(&gateway), 200) && gateway == 0xa5a5a5a5u); + Require(Failed(sceNetConfigGetDefaultRoute(nullptr), 200)); + Require(sceNetInit_nid_postfix() == 0); NetCtlInfo info{}; const int result = sceNetCtlGetInfo(16, &info); const char* route = reinterpret_cast(info.opaque); - std::uint32_t gateway = 0xa5a5a5a5u; - const auto throws = [](auto&& call) { - try { - call(); - } catch (const std::runtime_error&) { - return true; - } - return false; - }; - Require(throws([] { sceNetConfigGetDefaultRoute(nullptr); })); if (result != 0 || std::strcmp(route, "0.0.0.0") == 0) { Require(result == 0 || result == static_cast(0x80412108)); + Require(sceNetConfigGetDefaultRoute(&gateway) == static_cast(0x8041130Cu) && gateway == 0xa5a5a5a5u); + Require(sceNetConfigGetDefaultRoute(nullptr) == static_cast(0x8041130Cu)); return; } - Require(sceNetConfigGetDefaultRoute(&gateway) == 0); + Require(sceNetConfigGetDefaultRoute(&gateway) == 1); std::array text{}; Require(sceNetInetNtop(2, &gateway, text.data(), text.size()) == text.data()); Require(std::strcmp(text.data(), route) == 0); + bool threw = false; + try { + sceNetConfigGetDefaultRoute(nullptr); + } catch (const std::invalid_argument&) { + threw = true; + } + Require(threw); } static void CheckRandom() { diff --git a/docs/dev/debt/libSceNet.md b/docs/dev/debt/libSceNet.md index 63923a35b7..f0e1c4889c 100644 --- a/docs/dev/debt/libSceNet.md +++ b/docs/dev/debt/libSceNet.md @@ -33,7 +33,7 @@ - [sceNetSocketInternal, sceNetGetsockoptEx and sceNetSetsockoptEx](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - forward to sceNetSocket, sceNetGetsockopt and sceNetSetsockopt. Their restored argument lists match the public ones, and nothing in the tree records how the console tells these entry points apart from the public ones (for example by the calling process). - [sceNetSysctl](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - calls libkernel's sysctl with the FreeBSD sysctl(3) argument list, so it answers exactly the MIBs libkernel models, and reports a failure as the library's other calls do (`0x80410100 | errno`, with the errno set). How the console's routine reports errors is not measured. - [sceNetInetNtopWithScopeId and sceNetInetPtonWithScopeId](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - console behaviour from the module; slot-to-host-index mapping is ours. A zone name maps to a host index (`lo0` the loopback, `eth0` the first up interface with an address, the others a host interface of that name); a name with no host interface does not convert, and ntop prints the first name, in sceNetGetIfName's order, with the scope's index. A null destination throws `std::invalid_argument`, where the module faults. Not measured: ntop for a scope that matches no interface (throws), exact name comparison, the 0 written to the scope for a plain address, and the order of the family and null checks. Module: rooklz's [reading](https://github.com/boykopovar/AnyPS5/pull/9578#issuecomment-6549946559) of `libSceNet.sprx`, FUN_00006be0 (FUN_000065e0) and FUN_000072d0 (FUN_00006ee0), names through FUN_00005d40, FUN_00005ea0 and FUN_00005f20. -- [sceNetConfigGetDefaultRoute](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - reports the IPv4 gateway of the host interface that sceNetCtlGetInfo describes (`DEFAULT_ROUTE`), in network byte order, the order sceNetInetPton writes an `AF_INET` address in. With no gateway, or a null pointer, it throws: the console's answer for a missing route is not measured. +- [sceNetConfigGetDefaultRoute](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - the route is the host's: the IPv4 gateway of the interface sceNetCtlGetInfo describes (`DEFAULT_ROUTE`), in network byte order, stands for the kernel record's dword at 0x84. A null output with a route throws `std::invalid_argument`, where the module faults. Not measured: that the dword is the gateway in that order, whether `0x8041130C` sets errno (unchanged here), and the getter's other negative results, which have no host source. Module: rooklz's [reading](https://github.com/boykopovar/AnyPS5/pull/9578#issuecomment-6565097123) of `libSceNet.sprx` (NID `lFJb+BlPK1c`, wrapper 0x153f0, route-table getter 0xc70): `0x804101C8` with errno 200 before initialization, `0x8041130C` with the output untouched without a route, and 1 after storing the dword; no null check. ### Functional From c0703637ba1905ee7a2257c843dd4daa7a49b8fe Mon Sep 17 00:00:00 2001 From: xXPBSXx <20614131+xXPBSXx@users.noreply.github.com> Date: Mon, 2 Nov 2026 06:44:29 +0100 Subject: [PATCH 317/318] docs(debt): say that sceVideoOutSubmitChangeBufferAttribute2 throws for a bad set index or option A set index outside 0 to 3, a set that is not registered and an option other than null throw. The new entry says so and what the titles checked pass for the two arguments. Co-Authored-By: Claude Sonnet 5.5 --- docs/dev/debt/libSceVideoOut.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/debt/libSceVideoOut.md b/docs/dev/debt/libSceVideoOut.md index 2eaf556331..708f86c192 100644 --- a/docs/dev/debt/libSceVideoOut.md +++ b/docs/dev/debt/libSceVideoOut.md @@ -19,6 +19,7 @@ - [sceVideoOutAddBufferAttributeOption](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut, NID `MBlv8YWoyOk`) - the signature `(VideoOutBufferAttribute2*, uint64_t option)` comes from [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/60977b29103452b9681ae10c7a9011d44965bf16/src/SharpProspero/Interop/VideoOut/VideoOut.cs#L174-L176) and the call site in PPSA25380, which passes 0x20 and ignores the result. The option is ORed into `attribute->option`; whether the console validates the bits, keeps them elsewhere or fails on an unknown option is unknown. A null attribute throws. Option 0x20 makes `sceVideoOutRegisterBuffers2` throw, as every option other than 0 and 8; what it selects for an overlay buffer is unknown - [sceVideoOutRegisterBuffers2](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - DCC buffers (category 1): `dcc_control` is taken as the CB_DCC_CONTROL block layout, only bits 0x10026c are accepted; `dcc_cb_register_clear_color` as the register-clear texel, only 32-bit values are presented - [sceVideoOutUnregisterBuffers](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - on a main-bus port (bus type 0) the set holding the buffer on screen is refused with `RESOURCE_BUSY` (0x80290009) and stays registered. EVO Player (PPSA99039) [measured](https://github.com/sainsaji/EVO-PLAYER-PS5/blob/2db44b5294657b12bf91666154a05f029ddb5aa2/projects/evoplayer/media/src/evo_agc_runtime.c#L2830-L2838) that, with a re-registration then failing `SLOT_OCCUPIED`, and Prospero Win (PPSA99995) [measured](https://github.com/mpereiraesaa/prospero-win/blob/59c48117119fd57cd33e7bc91c11fa0c8ebf8b3d/native/pw_videoout_ps5.c#L92-L97) it on firmware 12.02 after its flips went idle, with `sceVideoOutClose` succeeding next. Both titles open bus type 0. Not measured: the overlay bus, whose on-screen set is still released and taken off the composited output; a set with a pending flip that is not on screen, which still throws; and the answer after a blank flip. +- [sceVideoOutSubmitChangeBufferAttribute2](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - a set index outside 0 to 3 or a set that is not registered throws `VIDEO_OUT_ERROR_INVALID_INDEX`, and an `option` other than null throws `VIDEO_OUT_ERROR_INVALID_OPTION`; the console's answers for them are not measured. Every direct call in the titles checked with a constant set index passes 0, and every call with a constant option passes null; a few calls are not shown to be a constant for either argument. - [sceVideoOutIsOutputSupported](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - mode 1 is supported; mode 15 is supported when the current host display runs at 120 Hz or above, and otherwise answers 0 while `sceVideoOutConfigureOutput(15)` returns `UNAVAILABLE_OUTPUT_MODE` (0x80290019): [PS5_Vulkan measured](https://github.com/mihawk-99/PS5_Vulkan/blob/f410e67a0241b3e8edcdbcd2a9b285dfc4a412ed/docs/HARDWARE_FINDINGS.md#L3793-L3835) both on a base PS5 whose output was set to 1080p, in a title declaring high-frame-rate output, and 1 and 0 at 1440p. Which code a title without the declaration gets when both apply is not measured; the declaration is checked first here. Configuring mode 15 requires attribute3 bit 0x40; this single-bit mask is inferred from the measured full value 0x80040, not isolated by a console measurement. Vblanks run at 119.88 Hz until mode 1 restores 59.94 Hz. Display modes are read through SDL; a move to a slower display falls back the configured output mode to mode 1, fires the output-mode event and continues at 59.94 Hz. The console's exact fallback behaviour when a 120 Hz display is lost has not been measured; this fallback avoids the abort in #7877 and matches the HDMI output mode renegotiation that a display change requires. Selecting another host display mode and VRR are not implemented. The console answers per display and per title: with a 120 Hz TV it also answers 1 for mode 15 (119.88 Hz), and [PS5_Vulkan](https://github.com/mihawk-99/PS5_Vulkan/blob/5b5e4fc2d80fdb67a4f61ba9e6a8a424026bb8a5/docs/HARDWARE_FINDINGS.md) measured that `sceVideoOutConfigureOutput` accepts 15 only from a title whose param.json declares high-frame-rate output (attribute3 0x80040). Modes 4, 7, 8, 12-14, 16-17 and 19 return `VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE` here, as they did on the measured display. Mode 18 throws, because one console answered 0x80290016 and two others 0x8029001E, the last a PS5 on 13.40 with a 120 Hz TV (measured 2026-10-19 inside a retail title), whose answers for every other mode match the ones here. The name `VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE` for 0x8029001E is ours. Word 3 of the output options is accepted with any value, as on the console, and its meaning is unknown. PPSA12544 probes `0xd000000a`, which the console answers with 0x8029001E, so the title selects 3840 pixels; PPSA26344 at `0xCD0946` passes `(handle, 0xf, nullptr, nullptr, 0)` and branches on the result equalling 1 - [sceVideoOutGetResolutionStatus](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - `screenSizeInInch` is 0, the value EDID uses for an undefined size, since the host display's size is not read; a PS5 (firmware 13.40, measured 2026-10-19 inside a retail title) reported its TV's 60.0f. The 48-byte layout, from the declarations in ps5-opengl and EVO-PLAYER-PS5 (#1578), is confirmed there: full size and pane 3840 x 2160, `refreshRate` 3, `flags` 0 and the rest 0 on the title's handle, 0x8029000B for handles 0 to 8. `flags` is 0 here too; its bits are unknown. The pane equals the full size because `sceVideoOutSetWindowModeMargins` applies no margins - [sceVideoOutGetOutputStatus](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the status after a buffer is registered, a 1440p output, a display below 4K with the console's resolution on Automatic and `flags` in a title that enables HDR output were not measured. A payload process on a PS5 (firmware 12.70) with no buffer registered reported `resolution` 2 right after `sceVideoOutOpen(0xff, 0, 0, NULL)` with the console's resolution on Automatic on a 4K display and 1 with it set to 1080p, and `dynamicRange` 2 with `flags` 0, as here, on an HDR display with HDR on. `resolution` is 2 when the current display mode SDL reports for the window's display is at least 3840 pixels wide or 2160 high and 1 otherwise, read with the refresh rate and independent of the registered buffers; on Wayland SDL's mode is the output's logical size (from SDL's source, not run), so a scaled 3840x2160 output reports 1. `APS5_OUTPUT_RESOLUTION` replaces that size From 975bebaef165a8426309b491ca09f5a32c1587f8 Mon Sep 17 00:00:00 2001 From: xXPBSXx <20614131+xXPBSXx@users.noreply.github.com> Date: Mon, 2 Nov 2026 07:49:31 +0100 Subject: [PATCH 318/318] docs(debt): say that sceNetResolverCreate fails non-zero flags and what the titles pass sceNetResolverCreate returns EINVAL for any non-zero flags, as the module is read in prosper. The new entry says so and that no direct call in the titles checked passes a non-zero value. Co-Authored-By: Claude Sonnet 5.5 --- docs/dev/debt/libSceNet.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/debt/libSceNet.md b/docs/dev/debt/libSceNet.md index 9629a80cfe..82ae4063eb 100644 --- a/docs/dev/debt/libSceNet.md +++ b/docs/dev/debt/libSceNet.md @@ -84,6 +84,7 @@ - [sceNetInetPton](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - an IPv6 conversion with a null `dst` returns `NET_EINVAL` here, where it crashed a payload process on a PS5 (firmware 13.60). On that PS5 and on PS4 firmware 10.71 an IPv4 address ends at the first space, tab, LF, VT, FF or CR and the rest is ignored (`"1.2.3.4 x"` converts), and a failed conversion leaves the destination and `sceNetErrnoLoc` unchanged. On the PS5 the dotted IPv4 part of IPv6 text is read the same way (`"::ffff:1.2.3.4 x"` converts, `"::ffff:01.2.3.4"` fails), and an IPv4 conversion with a null `dst` returns 1 or 0 without writing. Leading whitespace, any other trailing byte (`0x01`, `0xa0`) and whitespace after hexadecimal IPv6 text fail - [sceNetInetPton, sceNetInetNtop](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) convert IPv6 text with libkernel's [inet_pton and inet_ntop](../../../core/libs/prx/libkernel/Socket/src/AddressConversion.cpp), so they take the same FreeBSD 11.4 text forms; beyond the PS5 measurements in the entry above, these are unverified on a console. IPv4 text keeps the host conversion. The checks and errors before the conversion are libSceNet's own and unchanged. In the executables of the 53 titles run here, 36 call `sceNetInetPton` at 107 call sites and 22 call `sceNetInetNtop` at 67; every one of those call sites loads the family 2 (`AF_INET`) into the first argument, none loads 28 (`AF_INET6`) and none leaves it undetermined, so the IPv6 text conversion that is unverified on a console is not reached by a direct call in those executables (read from the instructions, not measured; calls from the titles' library modules and through function pointers are not seen). - [sceNetResolverStartNtoa](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) ignores flag bits 0x1 and 0x10000 and does not apply the timeout or retry count: every lookup runs once, synchronously, through the host resolver. What those two bits do on a console is not known; the sceNetResolverAbort entry above gives shadPS4's names for them. A negative timeout or retry and any other flag bit fail with `EINVAL` before anything else is checked, as [prosper's reading of the libSceNet module](https://github.com/mattias800/prosper/blob/6c09a34a092a00898fcead5f855200ca7c368cd2/prosper/src/hle/net/hle_net.cpp#L579-L587) gives. A timeout from 1 to 999999 also fails with `EINVAL` before the resolver is looked up, and 0 counts as 1000000, as @BunnyFl's [13.42 run of the module](https://github.com/boykopovar/AnyPS5/pull/9727#issuecomment-6362795252) shows. Every direct call in the titles checked passes the constant timeout 0, retry 0 or 1 and flags 0 or 0x1, so none fails these checks. +- [sceNetResolverCreate](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - non-zero `flags` fail with `EINVAL`, as [prosper's reading of the libSceNet module](https://github.com/mattias800/prosper/blob/6c09a34a092a00898fcead5f855200ca7c368cd2/prosper/src/hle/net/hle_net.cpp#L564-L578) gives. Every direct call in the titles checked passes the constant 0 as flags, so none reaches the check; one title also reaches the function through a tail jump, which is not read. - [sceNetDhcpStart, sceNetAddrConfig6Start, sceNetDuplicateIpStart and sceNetInfoDumpStart](../../../core/libs/prx/libSceNet/Export.cpp) - argument widths and callback ABI are recovered. Address callbacks receive user data, a 32-bit event and a 32-bit error. DHCP configuration is mutable; the 512-byte IPv6 configuration is read-only. Their complete field layouts remain unknown. The two 32-bit thread parameters are stored but not consumed by this firmware thread wrapper, so their original names and units are unconfirmed. Information-dump options contain flags, an interval in seconds and a 64-byte path. All four exports remain unimplemented. - [sceNetSetDnsInfo, sceNetSetDns6Info and sceNetGetDns6Info](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - the resolver still uses the host's DNS servers; setters only store servers for the getters. Without an IPv6 override, the getter reports none. Counts follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/0918faa0255e9816b795b62c9b09eaea51a1bc5e/src/core/libraries/net/net.cpp), not hardware measurements. All four DNS calls' initialization and argument errors match a reported [firmware 13.42 module probe](https://github.com/boykopovar/AnyPS5/pull/9506#issuecomment-6390574180): ENOTINIT first, then EINVAL for nonzero flags, null getter outputs or zero primary servers; null setters clear the override. Setters use a checked guest read and return EFAULT for unreadable non-null inputs, an AnyPS5 safety measure absent from the probed module.