fix(libkernel): protect past an unmapped start as a PS5 does

Measured on firmware 13.60: sceKernelMprotect, mprotect and
sceKernelMtypeprotect on direct memory change the mapped parts of a
range and return 0 also when the range starts in a gap or is wholly
unmapped; sceKernelMtypeprotect retypes only the mapped pages. The batch
map and memory pool protect operations keep their unmeasured EINVAL.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp

@@ -1289,7 +1289,7 @@ static int FindImageContaining(dl_phdr_info* image, std::size_t, void* data) {
 }
 #endif
 
-int DoMprotect(const void* addr, size_t len, int prot) {
+int DoMprotect(const void* addr, size_t len, int prot, bool requireMappedStart) {
     Trace("protect %p+0x%zx prot=0x%x", addr, len, prot);
     const auto address = reinterpret_cast<std::uintptr_t>(addr);
     constexpr auto pageMask = static_cast<std::uintptr_t>(PS5_PAGE_SIZE - 1);
@@ -1368,7 +1368,7 @@ int DoMprotect(const void* addr, size_t len, int prot) {
 #endif
     std::vector<std::pair<const void*, std::size_t>> spans;
     mutation.ForRegistered(pointer, bytes, [&](const void* span, std::size_t spanBytes) { spans.emplace_back(span, spanBytes); });
-    if (spans.empty() || reinterpret_cast<std::uintptr_t>(spans.front().first) != first) return SCE_KERNEL_ERROR_EINVAL;
+    if (requireMappedStart && (spans.empty() || reinterpret_cast<std::uintptr_t>(spans.front().first) != first)) return SCE_KERNEL_ERROR_EINVAL;
     for (const auto& [span, spanBytes] : spans) mutation.RequireUnpinned(span, spanBytes);
     for (const auto& [span, spanBytes] : spans) {
         mutation.Protect(span, spanBytes, (prot & 3) != 0, (prot & 2) != 0, (prot & GuestProtGpuReadWrite) != 0, [&] {
@@ -1379,8 +1379,8 @@ int DoMprotect(const void* addr, size_t len, int prot) {
     return 0;
 }
 
-int DoMtypeprotect(const void* addr, size_t len, int type, int prot) {
-    const int result = DoMprotect(addr, len, prot);
+int DoMtypeprotect(const void* addr, size_t len, int type, int prot, bool requireMappedStart) {
+    const int result = DoMprotect(addr, len, prot, requireMappedStart);
     if (result != 0) return result;
     const auto address = reinterpret_cast<std::uintptr_t>(addr);
     constexpr auto pageMask = static_cast<std::uintptr_t>(PS5_PAGE_SIZE - 1);

core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp

@@ -42,8 +42,8 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment = P
 int DoMapFileBacked_nid_no_patch(void** addr, size_t len, int prot, int flags, int descriptor, std::uint64_t offset, size_t alignment);
 void ReleaseFlexibleMappingRange(std::uintptr_t address, size_t len);
 int DoMapShared(void** addr, size_t len, int prot, HostSharedMemory object);
-int DoMprotect(const void* addr, size_t len, int prot);
-int DoMtypeprotect(const void* addr, size_t len, int type, int prot);
+int DoMprotect(const void* addr, size_t len, int prot, bool requireMappedStart = true);
+int DoMtypeprotect(const void* addr, size_t len, int type, int prot, bool requireMappedStart = true);
 int DoMunmap(void* addr, size_t len);
 int DoReserveVirtual(void** addr, size_t len, int flags, size_t alignment);
 bool GuestProtection(uintptr_t addr, int* prot);

core/libs/prx/libkernel/DirectMemory/Export.cpp

@@ -300,7 +300,7 @@ int32_t APS5_VABI sceKernelMapNamedFlexibleMemoryInternal(void** addr_in_out, si
 int APS5_VABI sceKernelMprotect(const void* addr, size_t len, int prot) {
  constexpr int DefinedProtection = 0x3F7;
  if (len == 0) return 0;
- return DoMprotect(addr, len, prot & DefinedProtection);
+ return DoMprotect(addr, len, prot & DefinedProtection, false);
 }
 
 int APS5_VABI sceKernelMunmap(uint64_t vaddr, size_t len) {
@@ -425,7 +425,7 @@ int APS5_VABI sceKernelCheckedReleaseDirectMemory(int64_t start, size_t len) {
 
 int APS5_VABI sceKernelMtypeprotect(const void* addr, size_t len, int type, int prot) {
  if (len == 0) return 0;
- return DoMtypeprotect(addr, len, type, prot);
+ return DoMtypeprotect(addr, len, type, prot, false);
 }
 
 int APS5_VABI sceKernelQueryMemoryProtection(void* addr, void** start, void** end, int* prot) {

core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp

@@ -282,7 +282,7 @@ int APS5_VABI mprotect_nid_postfix(void* address, std::size_t length, int protec
     if (start == 0 || length > std::numeric_limits<std::uintptr_t>::max() - start)
         return failed(GuestInvalid);
     try {
-        if (DoMprotect(address, length, protection & GuestDefinedProtection) != 0) return failed(GuestInvalid);
+        if (DoMprotect(address, length, protection & GuestDefinedProtection, false) != 0) return failed(GuestInvalid);
         return 0;
     } catch (const std::bad_alloc&) {
         return failed(GuestNoMemory);

core/libs/tests/GuestKernelStackProtect.cpp

@@ -251,8 +251,12 @@ static void* APS5_VABI Worker(void* arg) {
     Require(QueryPage(below).raw == belowBefore);
     static_cast<volatile unsigned char*>(start)[0] = 3;
     const auto outside = reinterpret_cast<std::uintptr_t>(start) & ~(Page - 1);
-    if (outside >= Page) Require(mprotect_nid_postfix(reinterpret_cast<void*>(outside - Page), 2 * Page, 3) == -1);
-    if (outside >= 2 * Page) Require(mprotect_nid_postfix(reinterpret_cast<void*>(outside - 2 * Page), Page, 3) == -1);
+#ifdef __linux__
+    constexpr int StraddleResult = -1;
+#else
+    constexpr int StraddleResult = 0;
+#endif
+    if (outside >= Page) Require(mprotect_nid_postfix(reinterpret_cast<void*>(outside - Page), 2 * Page, 3) == StraddleResult);
     const auto stackBytes = reinterpret_cast<std::uintptr_t>(end) - reinterpret_cast<std::uintptr_t>(start);
     Require(mprotect_nid_postfix(start, stackBytes, 0xc2) == 0);
     Require(QueryPage(low).writable);
@@ -263,7 +267,7 @@ static void* APS5_VABI Worker(void* arg) {
 #endif
     void* host = MapHostPages();
     Require(host != nullptr);
-    Require(mprotect_nid_postfix(host, HostBytes, 1) == -1);
+    Require(mprotect_nid_postfix(host, HostBytes, 1) == 0);
     Require(QueryPage(reinterpret_cast<std::uintptr_t>(host)).writable);
     static_cast<volatile unsigned char*>(host)[HostBytes - 1] = 5;
     UnmapHostPages(host);

core/libs/tests/GuestMemory.cpp

@@ -3483,8 +3483,7 @@ int main(int argc, char** argv) {
     Require(madvise_nid_postfix(memory, page, 4) == 0);
     Require(mincore_nid_postfix(memory, page, mincoreVec) == -1 && *__error_nid_postfix() == 12);
     *__error_nid_postfix() = 0;
-    Require(mprotect_nid_postfix(memory, page, 1) == -1 && *__error_nid_postfix() == 22);
-    *__error_nid_postfix() = 0;
+    Require(mprotect_nid_postfix(memory, page, 1) == 0 && *__error_nid_postfix() == 0);
     Require(mprotect_nid_postfix(memory, std::numeric_limits<std::size_t>::max(), 1) == -1 &&
             *__error_nid_postfix() == 22);
     for (int protection : {0, 1, 3, 5}) {

core/libs/tests/GuestMprotectUnmapped.cpp

@@ -5,6 +5,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <exception>
+#include <limits>
 
 extern "C" {
 void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept;
@@ -13,13 +14,16 @@ int APS5_VABI mprotect_nid_postfix(void*, std::size_t, int) noexcept;
 int APS5_VABI sceKernelMprotect(const void*, std::size_t, int);
 int APS5_VABI sceKernelMtypeprotect(const void*, std::size_t, int, int);
 int APS5_VABI sceKernelVirtualQuery(const void*, int, VirtualQueryInfo*, std::uint64_t);
+int APS5_VABI sceKernelAllocateDirectMemory(std::int64_t, std::int64_t, std::size_t, std::size_t, int, std::int64_t*);
+int APS5_VABI sceKernelMapDirectMemory(void**, std::size_t, int, int, std::int64_t, std::size_t);
+int APS5_VABI sceKernelReleaseDirectMemory(std::int64_t, std::size_t);
+int APS5_VABI sceKernelGetDirectMemoryType(std::int64_t, int*, std::int64_t*, std::int64_t*);
 int* APS5_VABI __error_nid_postfix();
 }
 
 namespace {
 
 constexpr std::size_t Page = 0x4000;
-constexpr int ErrorInvalid = 22;
 constexpr int ProtectRead = 1;
 constexpr int ProtectReadWrite = 3;
 
@@ -71,11 +75,14 @@ int main() {
     Require(Protect(memory + Page - 8, 16, ProtectReadWrite) == 0, "an unaligned range that reaches into the unmapped page protects the mapped page");
     Require(Protect(memory + 100, Page - 200, ProtectReadWrite) == 0, "an unaligned range inside the first page must work");
 
-    Require(Protect(memory + Page, Page, ProtectRead) == SCE_KERNEL_ERROR_EINVAL, "a range that is completely unmapped must return EINVAL");
+    Require(Protect(memory + Page, Page, ProtectRead) == 0, "a range that is completely unmapped must succeed");
     Require(Protect(memory + Page * 2, Page, ProtectReadWrite) == 0, "protecting the last page alone must work");
-    Require(Protect(memory + Page, Page * 2, ProtectRead) == SCE_KERNEL_ERROR_EINVAL, "a range whose start is unmapped must return EINVAL even when later pages are mapped");
-    Require(ProtectionAt(memory + Page * 2) == ProtectReadWrite, "a refused range must not change the mapped pages behind its start");
-    Require(mprotect_nid_postfix(memory + Page, Page, ProtectRead) == -1 && *__error_nid_postfix() == ErrorInvalid, "mprotect over a range that is completely unmapped must fail with EINVAL (22)");
+    Require(Protect(memory + Page, Page * 2, ProtectRead) == 0, "a range whose start is unmapped must succeed for the mapped pages behind it");
+    Require(ProtectionAt(memory + Page * 2) == ProtectRead, "the mapped pages behind an unmapped start must be protected");
+    Require(ProtectionAt(memory) == ProtectReadWrite, "a range that starts in a gap must not change the page before it");
+    Require(mprotect_nid_postfix(memory + Page, Page, ProtectRead) == 0, "mprotect over a range that is completely unmapped must succeed");
+    Require(mprotect_nid_postfix(memory + Page, Page * 2, ProtectReadWrite) == 0 && ProtectionAt(memory + Page * 2) == ProtectReadWrite, "mprotect over an unmapped start must protect the mapped pages behind it");
+    Require(sceKernelMtypeprotect(memory + Page, Page, 0, ProtectRead) == 0, "sceKernelMtypeprotect over a range that is completely unmapped must succeed");
 
     Require(munmap_nid_postfix(memory, Page) == 0 && munmap_nid_postfix(memory + Page * 2, Page) == 0, "munmap of the remaining pages failed");
 
@@ -84,5 +91,24 @@ int main() {
     Require(Protect(tail, Page * 2, ProtectRead) == 0, "a range that ends in an unmapped page must succeed for its mapped part");
     Require(ProtectionAt(tail) == ProtectRead, "the mapped part before a trailing gap must be protected");
     Require(munmap_nid_postfix(tail, Page) == 0, "munmap of the remaining page failed");
+    Require(Protect(tail, Page * 2, ProtectRead) == 0, "a range over pages that were all unmapped must succeed");
+
+    std::int64_t physical = -1;
+    Require(sceKernelAllocateDirectMemory(0, std::numeric_limits<std::int64_t>::max(), Page * 3, 0, 0, &physical) == 0, "direct memory allocation failed");
+    void* direct = nullptr;
+    Require(sceKernelMapDirectMemory(&direct, Page * 3, ProtectReadWrite, 0, physical, 0) == 0, "direct memory map failed");
+    auto* directPages = static_cast<unsigned char*>(direct);
+    Require(munmap_nid_postfix(directPages, Page) == 0, "munmap of the first direct page failed");
+    Require(sceKernelMtypeprotect(directPages, Page * 3, 3, ProtectRead) == 0, "sceKernelMtypeprotect over an unmapped start must succeed for the mapped direct pages");
+    Require(ProtectionAt(directPages + Page) == ProtectRead && ProtectionAt(directPages + Page * 2) == ProtectRead, "the mapped direct pages behind an unmapped start must be protected");
+    for (int page = 0; page < 3; ++page) {
+        int type = -1;
+        std::int64_t start = 0;
+        std::int64_t end = 0;
+        Require(sceKernelGetDirectMemoryType(physical + page * static_cast<std::int64_t>(Page), &type, &start, &end) == 0, "direct memory type query failed");
+        Require(type == (page == 0 ? 0 : 3), "only the mapped direct pages must take the new memory type");
+    }
+    Require(munmap_nid_postfix(directPages + Page, Page * 2) == 0, "munmap of the direct pages failed");
+    Require(sceKernelReleaseDirectMemory(physical, Page * 3) == 0, "direct memory release failed");
     return 0;
 }

docs/dev/debt/libkernel.md

@@ -119,7 +119,7 @@
 
 - [scePthreadYield](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) on Windows calls Sleep(1) after 512 consecutive yields that find no other thread ready on the current processor and arrive less than ~10,000 TSC ticks apart (so the span varies with host CPU TSC rate); FreeBSD sched_yield never sleeps.
 - [Guest thread scheduling](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) - the policies and priorities that `sched_get_priority_max`, `sched_get_priority_min`, `pthread_setschedparam` and `scePthreadSetschedparam` accept, and those a created thread takes, follow a PS5 (firmware 13.60), but they are only recorded and reported: they are not applied to host threads, so a title's priority order has no effect on host scheduling
-- [sceKernelMprotect, sceKernelMtypeprotect](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) over a range with a gap that is not registered in the allocation tracker protect the registered parts and skip the gap, and return 0, as FreeBSD 11.4's [`vm_map_protect`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/vm/vm_map.c) does (it has no check for holes); a range whose start is unmapped (also when later pages are mapped) returns `SCE_KERNEL_ERROR_EINVAL` (`mprotect`: `EINVAL`) because an existing test pins that for an unmapped start; FreeBSD answers 0 there too (`vm_map_protect` takes the next entry), so this errno is not taken from the source and was not changed. `sceKernelMprotect` threw for a gap before, which ended the title. Every part is checked for a GPU or image pin before the first one is protected; the parts are then protected one after the other, so a host protection failure in a later part leaves the earlier parts changed (FreeBSD's `vm_map_protect` cannot fail in its second pass). The behaviour on the console is not known. None of this was measured on a console.
+- [sceKernelMprotect, sceKernelMtypeprotect](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - on a PS5 (firmware 13.60) both, and `mprotect`, change the mapped parts of a range, skip its gaps and return 0, also when the range starts in a gap or is wholly unmapped, as here. On the console `sceKernelMtypeprotect` over a range with any flexible page returns `0x8002002d` and changes nothing; here it changes the pages and returns 0, since flexible memory mapped through `mmap` is not tracked. The batch map and memory pool protect operations still return `EINVAL` for an unmapped start, which was not measured. Every part is checked for a GPU or image pin before the first is protected; a host failure in a later part leaves the earlier parts changed.
 - [sceKernelMprotect protection bits](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - bits outside `0x3F7` are dropped instead of refused, in the POSIX `mprotect` as well (FreeBSD 11.4 masks the argument, `sys/vm/vm_mmap.c:613`; the console was measured only through `sceKernelMprotect`), as a PS5 (firmware 13.60) drops them: Onimusha: Way of the Sword (PPSA27836 01.000.001) maps direct memory with `0xf2` and then protects it with `0xfffffff2`, which the console reports as `0xf3` and then `0x3f3`. In a payload process the console also drops the CPU execute bit (`0x4`) of flexible, direct and JIT memory, and POSIX `mprotect` with write and execute reports write only (`0x2`). A title process may keep execute, so it is kept here. The map calls and `sceKernelMtypeprotect` refuse undefined bits on the console with `EINVAL` (`0x80020016`), and the direct map calls refuse execute; here both throw for undefined bits, and the direct map calls map execute
 - [JIT shared memory](../../../core/libs/prx/libkernel/DirectMemory/Jit.cpp) (libkernel) - `sceKernelJitCreateSharedMemory`, `sceKernelJitCreateAliasOfSharedMemory` and `sceKernelJitMapSharedMemory` take the signatures of [Orbital](https://github.com/AlexAltea/orbital)'s `tools/ida/db_types.json` (name, length, maximum protection, descriptor out; descriptor, maximum protection, descriptor out; descriptor, protection, address in and out), which ps4sdk, ps5-jar-loader and the call sites in RetroArch PPSA99169, XPSemu PPSA97358 and VLC PPSA85300 match; SharpProspero declares the alias as returning the descriptor. The object is a host anonymous file shared by its descriptors (a `memfd` on Linux, a delete-on-close temporary file mapped as an executable section on Windows), the alias is a duplicate descriptor, and every map maps the whole object, so writes through one view show through the others; a non-null address is a hint, as in `mmap` without `MAP_FIXED`. Not measured on the console: a length that is zero or not a multiple of 16 KiB and protection bits other than CPU read, write and execute are `EINVAL`; an alias or a map asking for more than the descriptor's maximum protection (write and execute counting as read) is `EACCES`, as FreeBSD 11's `vm_map_protect` treats a protection past `max_protection`; a descriptor that is not a JIT object is `EBADF`. The name is ignored, the objects do not count against the flexible memory budget, `mprotect` of a view is not limited to the maximum protection, and `mmap` of the descriptors maps the object as a [regular file](../../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp), so its views and the JIT views see each other's writes. On Linux the `memfd` is created without `MFD_EXEC`, which Linux 6.3 added and older kernels reject with `EINVAL`, and relies on the protection of the mapping: the flag only sets the file's execute permission bits, and the direct memory `memfd` is mapped executable the same way; a host with `vm.memfd_noexec` set to 2 refuses the creation and the call throws
 - [kqueue and kevent](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) follow FreeBSD 11's [`kern_event.c`](https://github.com/freebsd/freebsd-src/blob/34bd34750187bae02a0086402c78c031c59dd8b3/sys/kern/kern_event.c) (`kqueue_kevent`, `kqueue_register`, `kqueue_scan`, the timer and user filters) over the event queue; nothing is measured on a console. The descriptor is a host null device registered as the kqueue: `close` and `sceKernelClose` close the queue and `dup`/`dup2` of it throw, but `read`, `write` and `fstat` act on the null device instead of failing with `EOPNOTSUPP` or reporting `S_IFIFO`. Read and write events are supported on guest sockets only, plus `EVFILT_READ` on the read end and `EVFILT_WRITE` on the write end of a pipe (see the kevent on a pipe entry) and the timer, user and [vnode](../../../core/libs/prx/libkernel/Equeue/src/FileEvent.cpp) filters and a read event on the host stream on descriptor 0, which never triggers, as `select` never reports descriptor 0 readable; PPSA34015 registers descriptor 0 for reading in a new kqueue at start-up and does not check the result; other filters, `EV_CLEAR` on a socket or a pipe off Linux (see the kevent EV_CLEAR entry), a `NOTE_NSECONDS` period above one second with a fraction (11.0 converts the fraction with the microsecond macro) and any change to an existing timer (11.0 keeps the armed expiry and applies the new period after it, 11.4's `filt_timertouch` restarts it) throw. `EV_DISABLE`, `EV_DISPATCH` and `EV_FORCEONESHOT` follow kern_event.c: a disabled knote stays registered, stops polling or firing and is not reported, a trigger raised while it is disabled is kept and reported after `EV_ENABLE`, where `knote` would drop it, `EV_DISPATCH` reports once and then disables the knote, and `EV_FORCEONESHOT` is ignored on a new knote while on an existing one it sets `EV_ONESHOT` and fires the knote; a negative `nevents` fails with `EINVAL`. A new user event keeps its `fflags` unmasked and is not triggered by `NOTE_TRIGGER`, because `filt_userattach` reads the already cleared `kn_fflags`. The timer limit `kq_calloutmax` (`ENOMEM`) is not enforced, and closing a kqueue wakes a thread waiting on it in `kevent` with `EBADF`, where `kqueue_drain` makes the close wait for that thread. A level-triggered event is reported once per call and moved behind the other events, as in `kqueue_scan`; the `sceKernel*Equeue` calls keep their own collection loop. `NOTE_LOWAT` on a socket sets the low-water mark as FreeBSD 11's [`filt_soread` and `filt_sowrite`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/kern/uipc_socket.c) do, and a later change of the knote replaces it as `kqueue_register` replaces `kn_sdata`: a read event fires once the buffered bytes reach the mark, or at end of file or on a listening socket. A read mark above 1 is taken on stream sockets only: on a datagram socket Linux's `FIONREAD` reports the next datagram where FreeBSD's `sbavail` counts every queued byte, so it throws. A write event fires once the send buffer's free space reaches the mark, as `filt_sowrite` does with `sbspace`; the host reports that space as `SO_SNDBUF` less `SIOCOUTQ` on Linux and as `SO_SNDBUF` alone elsewhere, so off Linux it does not shrink while data is queued. Marks below 1, which make FreeBSD's read filter always fire, throw