From 4c77b6ab65ef797d2f804c90607f70c60b406a05 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 22 Oct 2026 03:40:26 +0100 Subject: [PATCH 01/14] feat(libkernel): support EV_CLEAR on a kqueue watched by another kqueue FreeBSD re-evaluates a kqueue's EVFILT_READ note from kqueue_wakeup, which runs whenever the watched kqueue queues a note, so an EV_CLEAR note fires again only after such an activation. The equeue now counts activations, and the EV_CLEAR note fires when the count moves while notes are queued. --- core/libs/prx/libkernel/Equeue/Equeue.cpp | 20 ++++++++++ core/libs/prx/libkernel/Equeue/Equeue.hpp | 2 + core/libs/prx/libkernel/Equeue/src/Kqueue.cpp | 40 +++++++++++++++++-- core/libs/tests/GuestKqueue.cpp | 33 ++++++++++++++- docs/dev/debt/libkernel.md | 2 +- 5 files changed, 91 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libkernel/Equeue/Equeue.cpp b/core/libs/prx/libkernel/Equeue/Equeue.cpp index c3d1691407..207774faa4 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.cpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.cpp @@ -125,6 +125,7 @@ void KernelEqueuePrivate::TriggerExpiredTimers(uint64_t nowNs) { ev.event.data += static_cast(count); ev.deadlineNs += count * ev.intervalNs; } + if (!ev.triggered) ++m_activations; ev.triggered = true; } } @@ -150,7 +151,9 @@ bool KernelEqueuePrivate::PollEvents() { bool polled = false; for (auto& ev : m_events) { if (ev.enabled && ev.filter.pollFunc != nullptr) { + const bool was = ev.triggered; ev.triggered = ev.filter.pollFunc(&ev); + if (!was && ev.triggered) ++m_activations; polled = true; } } @@ -333,6 +336,7 @@ int KernelEqueuePrivate::CollectTriggered(KernelEvent* ev, int num) { e.event = e.pendingEvents.front(); e.pendingEvents.pop_front(); e.triggered = true; + ++m_activations; } const auto next = std::next(it); if (erase) { @@ -416,6 +420,11 @@ std::size_t KernelEqueuePrivate::CountTriggeredEvents() { return static_cast(std::count_if(m_events.begin(), m_events.end(), [](const KernelEqueueEvent& event) { return event.enabled && event.triggered; })); } +std::uint64_t KernelEqueuePrivate::Activations() { + std::unique_lock lock(m_mutex); + return m_activations; +} + std::vector> KernelEqueuePrivate::WatchedQueues() { std::unique_lock lock(m_mutex); std::vector> queues; @@ -501,6 +510,7 @@ int KernelEqueuePrivate::AddEvent(const KernelEqueueEvent& event) { } } else { m_events.push_back(event); + if (event.triggered) ++m_activations; } m_cond.NotifyOne(); WakePollers(); @@ -521,11 +531,13 @@ int KernelEqueuePrivate::TriggerEvent(uintptr_t ident, int16_t filter, void* tri return SCE_KERNEL_ERROR_ENOENT; } if (it->enabled) { + const bool was = it->triggered; if (it->filter.triggerFunc != nullptr) { it->filter.triggerFunc(&*it, triggerData); } else { it->triggered = true; } + if (!was && it->triggered) ++m_activations; } m_cond.NotifyOne(); WakePollers(); @@ -555,6 +567,14 @@ int KernelEqueuePrivate::EditEvent(uintptr_t ident, int16_t filter, const std::f if (it == m_events.end()) { return SCE_KERNEL_ERROR_ENOENT; } + struct Activation { + KernelEqueuePrivate& queue; + KernelEqueueEvent& event; + bool active; + ~Activation() { + if (!active && event.enabled && event.triggered) ++queue.m_activations; + } + } activation{*this, *it, it->enabled && it->triggered}; edit(*it); m_cond.NotifyOne(); WakePollers(); diff --git a/core/libs/prx/libkernel/Equeue/Equeue.hpp b/core/libs/prx/libkernel/Equeue/Equeue.hpp index 4cace734bd..6c924922b1 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.hpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.hpp @@ -92,6 +92,7 @@ public: int GetTriggeredEvents(KernelEvent* ev, int num); bool HasTriggeredEvents(); std::size_t CountTriggeredEvents(); + std::uint64_t Activations(); std::vector> WatchedQueues(); int WaitForEvents(KernelEvent* ev, int num, uint32_t micros); int WaitForEventsUntil(KernelEvent* ev, int num, std::uint64_t deadlineNanos); @@ -124,6 +125,7 @@ private: KernelEqueue m_handle; bool m_reportOncePerWait; bool m_closed = false; + std::uint64_t m_activations = 0; }; using KernelEqueueRef = std::shared_ptr; diff --git a/core/libs/prx/libkernel/Equeue/src/Kqueue.cpp b/core/libs/prx/libkernel/Equeue/src/Kqueue.cpp index eea3d985ab..2907358440 100644 --- a/core/libs/prx/libkernel/Equeue/src/Kqueue.cpp +++ b/core/libs/prx/libkernel/Equeue/src/Kqueue.cpp @@ -228,6 +228,32 @@ bool PollKqueue(KernelEqueueEvent* e) { return pending != 0; } +struct ClearedKqueueNote { + std::uint64_t seen = 0; + bool active = false; +}; + +bool PollClearedKqueue(KernelEqueueEvent* e) { + auto& note = *static_cast(e->filter.data); + const auto pending = e->filter.watched->CountTriggeredEvents(); + const auto activations = e->filter.watched->Activations(); + if (activations != note.seen) { + note.seen = activations; + if (pending != 0) { + note.active = true; + e->event.data = static_cast(pending); + } + } + return note.active; +} + +void ResetClearedKqueue(KernelEqueueEvent* e) { + static_cast(e->filter.data)->active = false; + e->triggered = false; + e->event.fflags = 0; + e->event.data = 0; +} + bool KqueueReaches(const KernelEqueueRef& from, const KernelEqueuePrivate* target, std::vector& visited) { if (from.get() == target) { return true; @@ -245,9 +271,6 @@ bool KqueueReaches(const KernelEqueueRef& from, const KernelEqueuePrivate* targe } int AddKqueueKnote(const KernelEqueueRef& queue, const KernelEqueueRef& watched, KernelEvent& change, std::uint16_t flags) { - if ((flags & EV_CLEAR) != 0) { - NotImplemented_nid_no_patch("kevent: EV_CLEAR on a kqueue descriptor"); - } std::lock_guard lock(g_kqueueWatchMutex); std::vector visited; if (KqueueReaches(watched, queue.get(), visited)) { @@ -260,6 +283,17 @@ int AddKqueueKnote(const KernelEqueueRef& queue, const KernelEqueueRef& watched, event.event.data = 0; event.filter.pollFunc = PollKqueue; event.filter.watched = watched; + if ((flags & EV_CLEAR) != 0) { + auto note = std::make_shared(); + const auto pending = watched->CountTriggeredEvents(); + note->seen = watched->Activations(); + note->active = pending != 0; + event.event.data = static_cast(pending); + event.filter.data = note.get(); + event.filter.owner = std::move(note); + event.filter.pollFunc = PollClearedKqueue; + event.filter.resetFunc = ResetClearedKqueue; + } change.fflags = 0; change.data = 0; return GuestErrno(queue->AddEvent(event)); diff --git a/core/libs/tests/GuestKqueue.cpp b/core/libs/tests/GuestKqueue.cpp index 7b81a77b4a..55644153b3 100644 --- a/core/libs/tests/GuestKqueue.cpp +++ b/core/libs/tests/GuestKqueue.cpp @@ -570,8 +570,8 @@ static void WatchKqueue() { RequireFailure(Apply(outer, Change(watched, EvfiltWrite, EvAdd)), GuestEinval); RequireFailure(Apply(outer, Change(watched, EvfiltVnode, EvAdd, NoteWrite)), GuestEinval); - Require(ThrownMessage([outer, watched] { Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)); }).find("EV_CLEAR on a kqueue descriptor") != std::string::npos); - RequireFailure(Apply(outer, Change(watched, EvfiltRead, EvDelete)), GuestEnoent); + Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); + Require(Apply(outer, Change(watched, EvfiltRead, EvDelete)) == 0); Require(Apply(outer, Change(watched, EvfiltRead, EvAdd)) == 0); Require(ThrownMessage([inner, outer] { Apply(inner, Change(static_cast(outer), EvfiltRead, EvAdd)); }).find("watches the kqueue") != std::string::npos); Require(Apply(inner, Change(5, EvfiltUser, 0, NoteTrigger)) == 0); @@ -583,6 +583,34 @@ static void WatchKqueue() { Require(close_nid_postfix(outer) == 0); } +static void ClearedKqueue() { + const int inner = kqueue_nid_postfix(); + const int outer = kqueue_nid_postfix(); + Require(inner >= 0 && outer >= 0); + const auto watched = static_cast(inner); + KernelEvent events[4]{}; + Require(Apply(inner, Change(7, EvfiltUser, EvAdd)) == 0); + Require(Apply(inner, Change(8, EvfiltUser, EvAdd)) == 0); + Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); + Require(Poll(outer, events, 4) == 0); + + Require(Apply(inner, Change(7, EvfiltUser, 0, NoteTrigger)) == 0); + Require(Poll(outer, events, 4) == 1 && events[0].ident == watched && events[0].data == 1 && (events[0].flags & EvClear) != 0); + Require(Poll(outer, events, 4) == 0); + Require(Poll(inner, events, 4) == 1); + Require(Poll(outer, events, 4) == 0); + + Require(Apply(inner, Change(8, EvfiltUser, 0, NoteTrigger)) == 0); + Require(Poll(outer, events, 4) == 1 && events[0].data == 2); + Require(Poll(outer, events, 4) == 0); + + Require(Apply(outer, Change(watched, EvfiltRead, EvDelete)) == 0); + Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); + Require(Poll(outer, events, 4) == 1 && events[0].data == 2); + Require(Poll(outer, events, 4) == 0); + Require(close_nid_postfix(outer) == 0 && close_nid_postfix(inner) == 0); +} + static void LowWaterMarks() { const int kq = kqueue_nid_postfix(); int pair[2]; @@ -1497,6 +1525,7 @@ int main() { ReportedKnotesMoveToTail(); Errors(); LowWaterMarks(); + ClearedKqueue(); FileWatch(); #ifndef _WIN32 RenameAndDeleteWatchedFile(); diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 4968f46ca4..67e7e73a8a 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -249,7 +249,7 @@ - [poll on a stream socket](../../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) - on Linux a socket that reports `POLLHUP` reports it without `POLLOUT`, as FreeBSD 11.4's `pollscan` clears `POLLOUT` whenever it reports `POLLHUP`; Linux, like FreeBSD, reports `POLLHUP` only once both of the socket's directions are shut. Windows keeps the flags `WSAPoll` reports. Not measured on a console - [kevent EV_CLEAR on a socket or a pipe](../../../core/libs/prx/libkernel/Equeue/src/EdgeWatch.cpp) (libkernel) - FreeBSD 11 activates an `EV_CLEAR` knote when `kqueue_register` finds it ready, which every `EV_ADD` or `EV_ENABLE` of an enabled knote does again, and on each `KNOTE` the socket or pipe raises (`sorwakeup`, `sowwakeup`, `pipeselwakeup`); [`kqueue_scan`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/kern/kern_event.c) checks the filter again before reporting, drops a knote that is no longer ready and deactivates a reported one until the next activation. On Linux each such knote registers the host socket or pipe with its own epoll instance in edge-triggered mode (`EPOLLIN | EPOLLRDHUP` for `EVFILT_READ`, `EPOLLOUT` for `EVFILT_WRITE`), and a wakeup of the host descriptor with that mask stands for a `KNOTE`: the knote is then checked with the same readiness test as a level-triggered one, `NOTE_LOWAT` mark included, and a wait blocks on the epoll descriptor instead of the socket, so a socket that stays readable does not wake it. The host's wakeups are not FreeBSD's: FreeBSD raises `sowwakeup` whenever send buffer space is freed, while Linux TCP wakes `EPOLLOUT` waiters only once the socket is writable again after a write found it full (`SOCK_NOSPACE`, [`tcp_check_space`](https://github.com/torvalds/linux/blob/v6.8/net/ipv4/tcp_input.c#L5639)), so a TCP writer that never fills its buffer sees fewer `EVFILT_WRITE` events than on FreeBSD; a host stack may also coalesce data wakeups, so the number of times a busy socket fires can differ. Windows and macOS throw, as the host there has no edge-triggered wait over these descriptors that is wired up; `EV_CLEAR` on the timer, user and vnode filters does not use this and works everywhere. - [kevent on a pipe](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - host pipes are one-way, where FreeBSD's [`kern_pipe`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/kern/sys_pipe.c#L422) opens both ends for reading and writing, so `EVFILT_READ` on the write end and `EVFILT_WRITE` on the read end throw; on macOS `EVFILT_WRITE` on a pipe throws, as the host has no call for a pipe's capacity. `EVFILT_WRITE` reports the free space of the host buffer, not of FreeBSD's 16 KiB one, and on Linux fires when the host says the pipe is writable rather than at `PIPE_BUF` (512) free bytes. Windows pipes cannot be waited on, so a kqueue waiting on one checks it every 1 ms. Otherwise [`filt_piperead`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/kern/sys_pipe.c#L1787-L1828) and `filt_pipewrite` are followed: `EV_EOF` once the other end closes, `EPIPE` for a write event added after the read end closed, and `NOTE_LOWAT` ignored. On Windows `fcntl` cannot make a pipe write end non-blocking, so the full-pipe and drain case of `EVFILT_WRITE` is tested on Linux only and the `NtQueryInformationFile` write quota is checked there only for the free space, `EV_EOF` and `EPIPE`. -- [kevent on a kqueue](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - `EVFILT_READ` on a kqueue counts the watched queue's enabled notes that are ready when it is evaluated, where [`filt_kqueue`](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/sys/kern/kern_event.c#L363-L370) reports `kq_count`, the queued notes: a queued note counts until that kqueue is scanned, even once disabled or no longer ready (a socket or pipe read empty, a user note cleared, a watched kqueue drained), so FreeBSD can report it readable where this reports nothing. A queued `EV_ONESHOT` note is reported there with its last count and deleted; here it is evaluated again and, if the watched queue is empty by then, neither reported nor deleted. `EV_CLEAR` throws, as FreeBSD re-arms it only when the watched kqueue queues a note. A cycle of watches throws; a note on the queue itself is `EINVAL`. Not measured on a console +- [kevent on a kqueue](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - `EVFILT_READ` on a kqueue counts the watched queue's enabled notes that are ready when it is evaluated, where [`filt_kqueue`](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/sys/kern/kern_event.c#L363-L370) reports `kq_count`, the queued notes: a queued note counts until that kqueue is scanned, even once disabled or no longer ready (a socket or pipe read empty, a user note cleared, a watched kqueue drained), so FreeBSD can report it readable where this reports nothing. A queued `EV_ONESHOT` note is reported there with its last count and deleted; here it is evaluated again and, if the watched queue is empty by then, neither reported nor deleted. `EV_CLEAR` fires again only after the watched queue activates a note; one undone before the watching queue polls goes unseen. A cycle of watches throws; a note on the queue itself is `EINVAL`. Not measured on a console - [Socket error numbers](../../../core/libs/prx/libkernel/Socket/include/SocketError.hpp) (libkernel) - a host socket error becomes the FreeBSD 11 `errno.h` number of the same name; the numbers are not checked on the console. Winsock `WSAESHUTDOWN` becomes `EPIPE`, as a send after `shutdown` does on FreeBSD, and a host error with no FreeBSD socket counterpart becomes `EIO`. On Windows a refused nonblocking `connect` is never reported (`SO_ERROR` is unsupported), so a repeated one keeps reporting `EALREADY` where FreeBSD reports `ECONNREFUSED`. - [getsockopt, setsockopt](../../../core/libs/prx/libkernel/Socket/src/SocketOptions.cpp) (libkernel) - a guest socket has the SOL_SOCKET level, eight `IPPROTO_TCP` options, `IPV6_V6ONLY` and ten `IPPROTO_IP` options. `SO_ACCEPTFILTER` (0x1000) fails with `EINVAL` for a set and a get on a socket that is not listening, as a payload process on a PS5 (firmware 13.60) did and as FreeBSD 11.4's `do_setopt_accept_filter` and `do_getopt_accept_filter` do; on a listening socket both throw, as accept filters are not modelled; the other options of these levels fail with `ENOPROTOOPT` and were not all measured. `SO_KEEPALIVE`, `SO_DONTROUTE`, `SO_BROADCAST`, `SO_OOBINLINE` and `SO_LINGER` act on the host socket, and a boolean one reads back as its own bit, as there. Winsock does not clear a pending error when `SO_ERROR` reads it, so on Windows a pending error is reported once and again only after a new `connect`; a second failure with the same code without a `connect` in between is hidden - [TCP_NOPUSH, TCP_NOOPT, TCP option 0x80, IP_RECVDSTADDR, IP_RECVIF, IP_RECVTTL](../../../core/libs/prx/libkernel/Socket/src/SocketOptions.cpp) (libkernel) - kept and read back as a payload process on a PS5 (firmware 13.60) did: the TCP ones as the bits 0x1000, 0x8 and 0x10000 (so 0x80 is not FreeBSD's `TCP_KEEPINIT` there), the IP ones as 0 or 1. None of them changes what the socket does: `TCP_NOPUSH` does not hold back sends, and the IP ones add no control message to `recvmsg`. `IP_RECVTTL` was only read, never set, on the console, and an accepted socket takes `TCP_NOPUSH` from its listener, as FreeBSD 11.4's `syncache_socket` does and as measured on that console From ec708dbde2c69c929380d6f71b2b6eb6d3570c76 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 22 Oct 2026 09:24:09 +0100 Subject: [PATCH 02/14] fix(libkernel): take the watched kqueue's count and activations in one snapshot A cleared kqueue watch read the watched queue's triggered count and its activation counter under two separate locks, so an activation between the reads was marked seen without making the watch active, and the wakeup was lost. SnapshotTriggered now returns both under one lock. Every poll of an active watch also rechecks the count, as kqueue_scan reruns filt_kqueue: it drops when the watched queue empties and reports the current count. EV_ADD and EV_ENABLE of an existing watch re-arm it. A separate guest_kqueue_cleared_race test triggers the watched queue from another thread while the watching queue waits. --- core/libs/prx/libkernel/CMakeLists.txt | 4 +- core/libs/prx/libkernel/Equeue/Equeue.cpp | 10 ++- core/libs/prx/libkernel/Equeue/Equeue.hpp | 6 +- core/libs/prx/libkernel/Equeue/src/Kqueue.cpp | 28 +++++---- core/libs/tests/GuestKqueue.cpp | 63 ++++++++++++++++++- 5 files changed, 94 insertions(+), 17 deletions(-) diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 2a70b082f2..dc1aea96cc 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -1752,9 +1752,11 @@ if(BUILD_TESTING) configure_windows_unwind(guest_kqueue_tests) add_test(NAME guest_kqueue COMMAND guest_kqueue_tests) set_tests_properties(guest_kqueue PROPERTIES TIMEOUT 20) + add_test(NAME guest_kqueue_cleared_race COMMAND guest_kqueue_tests cleared-race) + set_tests_properties(guest_kqueue_cleared_race PROPERTIES TIMEOUT 60) if(WIN32) target_link_libraries(guest_kqueue_tests PRIVATE advapi32) - set_tests_properties(guest_kqueue PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_kqueue guest_kqueue_cleared_race PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() add_test_executable(guest_socket_descriptors_tests ${LIBS_INCLUDE_DIR}/tests/GuestSocketDescriptors.cpp) diff --git a/core/libs/prx/libkernel/Equeue/Equeue.cpp b/core/libs/prx/libkernel/Equeue/Equeue.cpp index 207774faa4..17299485ca 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.cpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.cpp @@ -420,9 +420,15 @@ std::size_t KernelEqueuePrivate::CountTriggeredEvents() { return static_cast(std::count_if(m_events.begin(), m_events.end(), [](const KernelEqueueEvent& event) { return event.enabled && event.triggered; })); } -std::uint64_t KernelEqueuePrivate::Activations() { +KernelEqueuePrivate::TriggeredSnapshot KernelEqueuePrivate::SnapshotTriggered() { std::unique_lock lock(m_mutex); - return m_activations; + if (m_closed) { + return {0, m_activations}; + } + TriggerExpiredTimers(MonotonicNs()); + PollEvents(); + const auto count = static_cast(std::count_if(m_events.begin(), m_events.end(), [](const KernelEqueueEvent& event) { return event.enabled && event.triggered; })); + return {count, m_activations}; } std::vector> KernelEqueuePrivate::WatchedQueues() { diff --git a/core/libs/prx/libkernel/Equeue/Equeue.hpp b/core/libs/prx/libkernel/Equeue/Equeue.hpp index 6c924922b1..0be3819f2e 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.hpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.hpp @@ -92,7 +92,11 @@ public: int GetTriggeredEvents(KernelEvent* ev, int num); bool HasTriggeredEvents(); std::size_t CountTriggeredEvents(); - std::uint64_t Activations(); + struct TriggeredSnapshot { + std::size_t count = 0; + std::uint64_t activations = 0; + }; + TriggeredSnapshot SnapshotTriggered(); std::vector> WatchedQueues(); int WaitForEvents(KernelEvent* ev, int num, uint32_t micros); int WaitForEventsUntil(KernelEvent* ev, int num, std::uint64_t deadlineNanos); diff --git a/core/libs/prx/libkernel/Equeue/src/Kqueue.cpp b/core/libs/prx/libkernel/Equeue/src/Kqueue.cpp index 2907358440..803268a6c3 100644 --- a/core/libs/prx/libkernel/Equeue/src/Kqueue.cpp +++ b/core/libs/prx/libkernel/Equeue/src/Kqueue.cpp @@ -235,14 +235,15 @@ struct ClearedKqueueNote { bool PollClearedKqueue(KernelEqueueEvent* e) { auto& note = *static_cast(e->filter.data); - const auto pending = e->filter.watched->CountTriggeredEvents(); - const auto activations = e->filter.watched->Activations(); - if (activations != note.seen) { - note.seen = activations; - if (pending != 0) { - note.active = true; - e->event.data = static_cast(pending); - } + const auto snapshot = e->filter.watched->SnapshotTriggered(); + if (snapshot.activations != note.seen) { + note.seen = snapshot.activations; + note.active = true; + } + if (snapshot.count == 0) { + note.active = false; + } else if (note.active) { + e->event.data = static_cast(snapshot.count); } return note.active; } @@ -285,10 +286,10 @@ int AddKqueueKnote(const KernelEqueueRef& queue, const KernelEqueueRef& watched, event.filter.watched = watched; if ((flags & EV_CLEAR) != 0) { auto note = std::make_shared(); - const auto pending = watched->CountTriggeredEvents(); - note->seen = watched->Activations(); - note->active = pending != 0; - event.event.data = static_cast(pending); + const auto snapshot = watched->SnapshotTriggered(); + note->seen = snapshot.activations; + note->active = snapshot.count != 0; + event.event.data = static_cast(snapshot.count); event.filter.data = note.get(); event.filter.owner = std::move(note); event.filter.pollFunc = PollClearedKqueue; @@ -458,6 +459,9 @@ int ModifyKnote(const KernelEqueueRef& queue, KernelEvent& change) { if (event.enabled && event.filter.pollFunc == PollEdge) { static_cast(event.filter.data)->armed = true; } + if (event.enabled && event.filter.pollFunc == PollClearedKqueue) { + static_cast(event.filter.data)->active = true; + } if (change.filter == EVFILT_USER) { TouchUserKnote(event, change); } else if (change.filter == EVFILT_VNODE) { diff --git a/core/libs/tests/GuestKqueue.cpp b/core/libs/tests/GuestKqueue.cpp index 55644153b3..f6384b029c 100644 --- a/core/libs/tests/GuestKqueue.cpp +++ b/core/libs/tests/GuestKqueue.cpp @@ -611,6 +611,62 @@ static void ClearedKqueue() { Require(close_nid_postfix(outer) == 0 && close_nid_postfix(inner) == 0); } +static void ClearedKqueueRechecksTheWatchedCount() { + const int inner = kqueue_nid_postfix(); + const int outer = kqueue_nid_postfix(); + Require(inner >= 0 && outer >= 0); + const auto watched = static_cast(inner); + KernelEvent events[4]{}; + Require(Apply(inner, Change(7, EvfiltUser, EvAdd | EvClear)) == 0); + Require(Apply(inner, Change(8, EvfiltUser, EvAdd | EvClear)) == 0); + Require(Apply(outer, Change(1, EvfiltUser, EvAdd | EvClear)) == 0); + Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); + + Require(Apply(inner, Change(7, EvfiltUser, 0, NoteTrigger)) == 0); + Require(Apply(outer, Change(1, EvfiltUser, 0, NoteTrigger)) == 0); + Require(Poll(outer, events, 1) == 1 && events[0].ident == 1 && events[0].filter == EvfiltUser); + Require(Poll(inner, events, 4) == 1 && events[0].ident == 7); + Require(Poll(outer, events, 4) == 0); + + Require(Apply(inner, Change(7, EvfiltUser, 0, NoteTrigger)) == 0); + Require(Apply(inner, Change(8, EvfiltUser, 0, NoteTrigger)) == 0); + Require(Apply(outer, Change(1, EvfiltUser, 0, NoteTrigger)) == 0); + Require(Poll(outer, events, 1) == 1 && events[0].ident == 1); + Require(Poll(inner, events, 1) == 1); + Require(Poll(outer, events, 4) == 1 && events[0].ident == watched && events[0].data == 1); + Require(Poll(outer, events, 4) == 0); + + Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); + Require(Poll(outer, events, 4) == 1 && events[0].ident == watched && events[0].data == 1); + Require(Poll(outer, events, 4) == 0); + Require(Apply(outer, Change(watched, EvfiltRead, EvDisable)) == 0); + Require(Apply(outer, Change(watched, EvfiltRead, EvEnable)) == 0); + Require(Poll(outer, events, 4) == 1 && events[0].ident == watched && events[0].data == 1); + Require(Poll(inner, events, 4) == 1); + Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); + Require(Poll(outer, events, 4) == 0); + Require(close_nid_postfix(outer) == 0 && close_nid_postfix(inner) == 0); +} + +static void ClearedKqueueKeepsConcurrentActivations() { + const int inner = kqueue_nid_postfix(); + const int outer = kqueue_nid_postfix(); + Require(inner >= 0 && outer >= 0); + const auto watched = static_cast(inner); + KernelEvent events[4]{}; + Require(Apply(inner, Change(7, EvfiltUser, EvAdd | EvClear)) == 0); + for (std::uintptr_t filler = 100; filler < 20100; ++filler) Require(Apply(inner, Change(filler, EvfiltUser, EvAdd)) == 0); + Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); + for (int round = 0; round < 5000; ++round) { + std::thread trigger([inner] { Require(Apply(inner, Change(7, EvfiltUser, 0, NoteTrigger)) == 0); }); + const int reported = Wait(outer, events, 4); + trigger.join(); + Require(reported == 1 && events[0].ident == watched); + Require(Poll(inner, events, 4) == 1); + } + Require(close_nid_postfix(outer) == 0 && close_nid_postfix(inner) == 0); +} + static void LowWaterMarks() { const int kq = kqueue_nid_postfix(); int pair[2]; @@ -1498,7 +1554,11 @@ static void StandardInput() { Require(close_nid_postfix(kq) == 0); } -int main() { +int main(int argc, char** argv) { + if (argc == 2 && std::string(argv[1]) == "cleared-race") { + ClearedKqueueKeepsConcurrentActivations(); + return 0; + } StandardInputReplacement(); StandardInput(); CreateAndClose(); @@ -1526,6 +1586,7 @@ int main() { Errors(); LowWaterMarks(); ClearedKqueue(); + ClearedKqueueRechecksTheWatchedCount(); FileWatch(); #ifndef _WIN32 RenameAndDeleteWatchedFile(); From b91db60dba943f1e5925e7d151bf43bf6f8a0dc9 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 22 Oct 2026 10:29:52 +0100 Subject: [PATCH 03/14] test(libkernel): run the cleared kqueue race test on one CPU on Linux On a multi-core Linux machine the race test passed with the split reads it is meant to catch, and caught them only when the process ran on one CPU. It now pins itself to its first allowed CPU on Linux. Windows already catches the split reads unpinned, and pinning there hides them, so it stays unpinned. --- core/libs/tests/GuestKqueue.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/libs/tests/GuestKqueue.cpp b/core/libs/tests/GuestKqueue.cpp index f6384b029c..3bdba2a116 100644 --- a/core/libs/tests/GuestKqueue.cpp +++ b/core/libs/tests/GuestKqueue.cpp @@ -16,6 +16,8 @@ #ifdef _WIN32 #include #include +#elif defined(__linux__) +#include #endif #include #ifdef _WIN32 @@ -648,7 +650,23 @@ static void ClearedKqueueRechecksTheWatchedCount() { Require(close_nid_postfix(outer) == 0 && close_nid_postfix(inner) == 0); } +static void RunOnOneCpuOnLinux() { +#ifdef __linux__ + cpu_set_t allowed; + CPU_ZERO(&allowed); + Require(sched_getaffinity(0, sizeof(allowed), &allowed) == 0); + int first = 0; + while (first < CPU_SETSIZE && !CPU_ISSET(first, &allowed)) ++first; + Require(first < CPU_SETSIZE); + cpu_set_t one; + CPU_ZERO(&one); + CPU_SET(first, &one); + Require(sched_setaffinity(0, sizeof(one), &one) == 0); +#endif +} + static void ClearedKqueueKeepsConcurrentActivations() { + RunOnOneCpuOnLinux(); const int inner = kqueue_nid_postfix(); const int outer = kqueue_nid_postfix(); Require(inner >= 0 && outer >= 0); From 7cdb7a8780fb81e40e0e7e77c398d23f0a4d693d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0?= Date: Wed, 28 Oct 2026 13:45:37 +0100 Subject: [PATCH 04/14] test(libkernel): shrink the cleared kqueue race, match watched notes by ident, correct the kqueue debt entry Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libkernel/CMakeLists.txt | 2 +- core/libs/tests/GuestKqueue.cpp | 14 +++++++++----- docs/dev/debt/libkernel.md | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index dc1aea96cc..c778cf3c74 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -1753,7 +1753,7 @@ if(BUILD_TESTING) add_test(NAME guest_kqueue COMMAND guest_kqueue_tests) set_tests_properties(guest_kqueue PROPERTIES TIMEOUT 20) add_test(NAME guest_kqueue_cleared_race COMMAND guest_kqueue_tests cleared-race) - set_tests_properties(guest_kqueue_cleared_race PROPERTIES TIMEOUT 60) + set_tests_properties(guest_kqueue_cleared_race PROPERTIES TIMEOUT 30) if(WIN32) target_link_libraries(guest_kqueue_tests PRIVATE advapi32) set_tests_properties(guest_kqueue guest_kqueue_cleared_race PROPERTIES ENVIRONMENT_MODIFICATION diff --git a/core/libs/tests/GuestKqueue.cpp b/core/libs/tests/GuestKqueue.cpp index 3bdba2a116..c421f32f45 100644 --- a/core/libs/tests/GuestKqueue.cpp +++ b/core/libs/tests/GuestKqueue.cpp @@ -626,16 +626,20 @@ static void ClearedKqueueRechecksTheWatchedCount() { Require(Apply(inner, Change(7, EvfiltUser, 0, NoteTrigger)) == 0); Require(Apply(outer, Change(1, EvfiltUser, 0, NoteTrigger)) == 0); - Require(Poll(outer, events, 1) == 1 && events[0].ident == 1 && events[0].filter == EvfiltUser); Require(Poll(inner, events, 4) == 1 && events[0].ident == 7); + Require(Poll(outer, events, 4) == 1 && events[0].ident == 1 && events[0].filter == EvfiltUser); Require(Poll(outer, events, 4) == 0); Require(Apply(inner, Change(7, EvfiltUser, 0, NoteTrigger)) == 0); Require(Apply(inner, Change(8, EvfiltUser, 0, NoteTrigger)) == 0); Require(Apply(outer, Change(1, EvfiltUser, 0, NoteTrigger)) == 0); - Require(Poll(outer, events, 1) == 1 && events[0].ident == 1); Require(Poll(inner, events, 1) == 1); - Require(Poll(outer, events, 4) == 1 && events[0].ident == watched && events[0].data == 1); + Require(Poll(outer, events, 4) == 2); + const bool watchedFirst = events[0].ident == watched; + const KernelEvent& userEvent = events[watchedFirst ? 1 : 0]; + const KernelEvent& watchedEvent = events[watchedFirst ? 0 : 1]; + Require(userEvent.ident == 1 && userEvent.filter == EvfiltUser); + Require(watchedEvent.ident == watched && watchedEvent.filter == EvfiltRead && watchedEvent.data == 1); Require(Poll(outer, events, 4) == 0); Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); @@ -673,9 +677,9 @@ static void ClearedKqueueKeepsConcurrentActivations() { const auto watched = static_cast(inner); KernelEvent events[4]{}; Require(Apply(inner, Change(7, EvfiltUser, EvAdd | EvClear)) == 0); - for (std::uintptr_t filler = 100; filler < 20100; ++filler) Require(Apply(inner, Change(filler, EvfiltUser, EvAdd)) == 0); + for (std::uintptr_t filler = 100; filler < 12100; ++filler) Require(Apply(inner, Change(filler, EvfiltUser, EvAdd)) == 0); Require(Apply(outer, Change(watched, EvfiltRead, EvAdd | EvClear)) == 0); - for (int round = 0; round < 5000; ++round) { + for (int round = 0; round < 1250; ++round) { std::thread trigger([inner] { Require(Apply(inner, Change(7, EvfiltUser, 0, NoteTrigger)) == 0); }); const int reported = Wait(outer, events, 4); trigger.join(); diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 67e7e73a8a..b9097a1f2c 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -249,7 +249,7 @@ - [poll on a stream socket](../../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) - on Linux a socket that reports `POLLHUP` reports it without `POLLOUT`, as FreeBSD 11.4's `pollscan` clears `POLLOUT` whenever it reports `POLLHUP`; Linux, like FreeBSD, reports `POLLHUP` only once both of the socket's directions are shut. Windows keeps the flags `WSAPoll` reports. Not measured on a console - [kevent EV_CLEAR on a socket or a pipe](../../../core/libs/prx/libkernel/Equeue/src/EdgeWatch.cpp) (libkernel) - FreeBSD 11 activates an `EV_CLEAR` knote when `kqueue_register` finds it ready, which every `EV_ADD` or `EV_ENABLE` of an enabled knote does again, and on each `KNOTE` the socket or pipe raises (`sorwakeup`, `sowwakeup`, `pipeselwakeup`); [`kqueue_scan`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/kern/kern_event.c) checks the filter again before reporting, drops a knote that is no longer ready and deactivates a reported one until the next activation. On Linux each such knote registers the host socket or pipe with its own epoll instance in edge-triggered mode (`EPOLLIN | EPOLLRDHUP` for `EVFILT_READ`, `EPOLLOUT` for `EVFILT_WRITE`), and a wakeup of the host descriptor with that mask stands for a `KNOTE`: the knote is then checked with the same readiness test as a level-triggered one, `NOTE_LOWAT` mark included, and a wait blocks on the epoll descriptor instead of the socket, so a socket that stays readable does not wake it. The host's wakeups are not FreeBSD's: FreeBSD raises `sowwakeup` whenever send buffer space is freed, while Linux TCP wakes `EPOLLOUT` waiters only once the socket is writable again after a write found it full (`SOCK_NOSPACE`, [`tcp_check_space`](https://github.com/torvalds/linux/blob/v6.8/net/ipv4/tcp_input.c#L5639)), so a TCP writer that never fills its buffer sees fewer `EVFILT_WRITE` events than on FreeBSD; a host stack may also coalesce data wakeups, so the number of times a busy socket fires can differ. Windows and macOS throw, as the host there has no edge-triggered wait over these descriptors that is wired up; `EV_CLEAR` on the timer, user and vnode filters does not use this and works everywhere. - [kevent on a pipe](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - host pipes are one-way, where FreeBSD's [`kern_pipe`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/kern/sys_pipe.c#L422) opens both ends for reading and writing, so `EVFILT_READ` on the write end and `EVFILT_WRITE` on the read end throw; on macOS `EVFILT_WRITE` on a pipe throws, as the host has no call for a pipe's capacity. `EVFILT_WRITE` reports the free space of the host buffer, not of FreeBSD's 16 KiB one, and on Linux fires when the host says the pipe is writable rather than at `PIPE_BUF` (512) free bytes. Windows pipes cannot be waited on, so a kqueue waiting on one checks it every 1 ms. Otherwise [`filt_piperead`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/kern/sys_pipe.c#L1787-L1828) and `filt_pipewrite` are followed: `EV_EOF` once the other end closes, `EPIPE` for a write event added after the read end closed, and `NOTE_LOWAT` ignored. On Windows `fcntl` cannot make a pipe write end non-blocking, so the full-pipe and drain case of `EVFILT_WRITE` is tested on Linux only and the `NtQueryInformationFile` write quota is checked there only for the free space, `EV_EOF` and `EPIPE`. -- [kevent on a kqueue](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - `EVFILT_READ` on a kqueue counts the watched queue's enabled notes that are ready when it is evaluated, where [`filt_kqueue`](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/sys/kern/kern_event.c#L363-L370) reports `kq_count`, the queued notes: a queued note counts until that kqueue is scanned, even once disabled or no longer ready (a socket or pipe read empty, a user note cleared, a watched kqueue drained), so FreeBSD can report it readable where this reports nothing. A queued `EV_ONESHOT` note is reported there with its last count and deleted; here it is evaluated again and, if the watched queue is empty by then, neither reported nor deleted. `EV_CLEAR` fires again only after the watched queue activates a note; one undone before the watching queue polls goes unseen. A cycle of watches throws; a note on the queue itself is `EINVAL`. Not measured on a console +- [kevent on a kqueue](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - `EVFILT_READ` on a kqueue counts the watched queue's enabled notes that are ready when it is evaluated, where [`filt_kqueue`](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/sys/kern/kern_event.c#L363-L370) reports `kq_count`, the queued notes: a queued note counts until that kqueue is scanned, even once disabled or no longer ready (a socket or pipe read empty, a user note cleared, a watched kqueue drained), so FreeBSD can report it readable where this reports nothing. A queued `EV_ONESHOT` note is reported there with its last count and deleted; here it is evaluated again and, if the watched queue is empty by then, neither reported nor deleted. `EV_CLEAR` fires again only after the watched queue activates a note; one undone before the watching queue polls is missed. A cycle of watches throws; a note on the queue itself is `EINVAL`. Not measured on a console - [Socket error numbers](../../../core/libs/prx/libkernel/Socket/include/SocketError.hpp) (libkernel) - a host socket error becomes the FreeBSD 11 `errno.h` number of the same name; the numbers are not checked on the console. Winsock `WSAESHUTDOWN` becomes `EPIPE`, as a send after `shutdown` does on FreeBSD, and a host error with no FreeBSD socket counterpart becomes `EIO`. On Windows a refused nonblocking `connect` is never reported (`SO_ERROR` is unsupported), so a repeated one keeps reporting `EALREADY` where FreeBSD reports `ECONNREFUSED`. - [getsockopt, setsockopt](../../../core/libs/prx/libkernel/Socket/src/SocketOptions.cpp) (libkernel) - a guest socket has the SOL_SOCKET level, eight `IPPROTO_TCP` options, `IPV6_V6ONLY` and ten `IPPROTO_IP` options. `SO_ACCEPTFILTER` (0x1000) fails with `EINVAL` for a set and a get on a socket that is not listening, as a payload process on a PS5 (firmware 13.60) did and as FreeBSD 11.4's `do_setopt_accept_filter` and `do_getopt_accept_filter` do; on a listening socket both throw, as accept filters are not modelled; the other options of these levels fail with `ENOPROTOOPT` and were not all measured. `SO_KEEPALIVE`, `SO_DONTROUTE`, `SO_BROADCAST`, `SO_OOBINLINE` and `SO_LINGER` act on the host socket, and a boolean one reads back as its own bit, as there. Winsock does not clear a pending error when `SO_ERROR` reads it, so on Windows a pending error is reported once and again only after a new `connect`; a second failure with the same code without a `connect` in between is hidden - [TCP_NOPUSH, TCP_NOOPT, TCP option 0x80, IP_RECVDSTADDR, IP_RECVIF, IP_RECVTTL](../../../core/libs/prx/libkernel/Socket/src/SocketOptions.cpp) (libkernel) - kept and read back as a payload process on a PS5 (firmware 13.60) did: the TCP ones as the bits 0x1000, 0x8 and 0x10000 (so 0x80 is not FreeBSD's `TCP_KEEPINIT` there), the IP ones as 0 or 1. None of them changes what the socket does: `TCP_NOPUSH` does not hold back sends, and the IP ones add no control message to `recvmsg`. `IP_RECVTTL` was only read, never set, on the console, and an accepted socket takes `TCP_NOPUSH` from its listener, as FreeBSD 11.4's `syncache_socket` does and as measured on that console From 638996829b1a6c9b3800d7aac89a8409af6cbe49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0?= Date: Wed, 28 Oct 2026 13:45:47 +0100 Subject: [PATCH 05/14] docs(libkernel): shorten the kevent on a kqueue debt entry Co-Authored-By: Claude Sonnet 5.5 --- 3rdparty/SPIRV-Headers | 2 +- 3rdparty/SPIRV-Tools | 2 +- docs/dev/debt/libkernel.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/3rdparty/SPIRV-Headers b/3rdparty/SPIRV-Headers index b824a462d4..01e0577914 160000 --- a/3rdparty/SPIRV-Headers +++ b/3rdparty/SPIRV-Headers @@ -1 +1 @@ -Subproject commit b824a462d4256d720bebb40e78b9eb8f78bbb305 +Subproject commit 01e0577914a75a2569c846778c2f93aa8e6feddd diff --git a/3rdparty/SPIRV-Tools b/3rdparty/SPIRV-Tools index 262bdab481..7f2d9ee926 160000 --- a/3rdparty/SPIRV-Tools +++ b/3rdparty/SPIRV-Tools @@ -1 +1 @@ -Subproject commit 262bdab48146c937467f826699a40da0fdfc0f1a +Subproject commit 7f2d9ee926f98fc77a3ed1e1e0f113b8c9c49458 diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index b9097a1f2c..dd97a1905f 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -249,7 +249,7 @@ - [poll on a stream socket](../../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) - on Linux a socket that reports `POLLHUP` reports it without `POLLOUT`, as FreeBSD 11.4's `pollscan` clears `POLLOUT` whenever it reports `POLLHUP`; Linux, like FreeBSD, reports `POLLHUP` only once both of the socket's directions are shut. Windows keeps the flags `WSAPoll` reports. Not measured on a console - [kevent EV_CLEAR on a socket or a pipe](../../../core/libs/prx/libkernel/Equeue/src/EdgeWatch.cpp) (libkernel) - FreeBSD 11 activates an `EV_CLEAR` knote when `kqueue_register` finds it ready, which every `EV_ADD` or `EV_ENABLE` of an enabled knote does again, and on each `KNOTE` the socket or pipe raises (`sorwakeup`, `sowwakeup`, `pipeselwakeup`); [`kqueue_scan`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/kern/kern_event.c) checks the filter again before reporting, drops a knote that is no longer ready and deactivates a reported one until the next activation. On Linux each such knote registers the host socket or pipe with its own epoll instance in edge-triggered mode (`EPOLLIN | EPOLLRDHUP` for `EVFILT_READ`, `EPOLLOUT` for `EVFILT_WRITE`), and a wakeup of the host descriptor with that mask stands for a `KNOTE`: the knote is then checked with the same readiness test as a level-triggered one, `NOTE_LOWAT` mark included, and a wait blocks on the epoll descriptor instead of the socket, so a socket that stays readable does not wake it. The host's wakeups are not FreeBSD's: FreeBSD raises `sowwakeup` whenever send buffer space is freed, while Linux TCP wakes `EPOLLOUT` waiters only once the socket is writable again after a write found it full (`SOCK_NOSPACE`, [`tcp_check_space`](https://github.com/torvalds/linux/blob/v6.8/net/ipv4/tcp_input.c#L5639)), so a TCP writer that never fills its buffer sees fewer `EVFILT_WRITE` events than on FreeBSD; a host stack may also coalesce data wakeups, so the number of times a busy socket fires can differ. Windows and macOS throw, as the host there has no edge-triggered wait over these descriptors that is wired up; `EV_CLEAR` on the timer, user and vnode filters does not use this and works everywhere. - [kevent on a pipe](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - host pipes are one-way, where FreeBSD's [`kern_pipe`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/kern/sys_pipe.c#L422) opens both ends for reading and writing, so `EVFILT_READ` on the write end and `EVFILT_WRITE` on the read end throw; on macOS `EVFILT_WRITE` on a pipe throws, as the host has no call for a pipe's capacity. `EVFILT_WRITE` reports the free space of the host buffer, not of FreeBSD's 16 KiB one, and on Linux fires when the host says the pipe is writable rather than at `PIPE_BUF` (512) free bytes. Windows pipes cannot be waited on, so a kqueue waiting on one checks it every 1 ms. Otherwise [`filt_piperead`](https://github.com/freebsd/freebsd-src/blob/release/11.0.0/sys/kern/sys_pipe.c#L1787-L1828) and `filt_pipewrite` are followed: `EV_EOF` once the other end closes, `EPIPE` for a write event added after the read end closed, and `NOTE_LOWAT` ignored. On Windows `fcntl` cannot make a pipe write end non-blocking, so the full-pipe and drain case of `EVFILT_WRITE` is tested on Linux only and the `NtQueryInformationFile` write quota is checked there only for the free space, `EV_EOF` and `EPIPE`. -- [kevent on a kqueue](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - `EVFILT_READ` on a kqueue counts the watched queue's enabled notes that are ready when it is evaluated, where [`filt_kqueue`](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/sys/kern/kern_event.c#L363-L370) reports `kq_count`, the queued notes: a queued note counts until that kqueue is scanned, even once disabled or no longer ready (a socket or pipe read empty, a user note cleared, a watched kqueue drained), so FreeBSD can report it readable where this reports nothing. A queued `EV_ONESHOT` note is reported there with its last count and deleted; here it is evaluated again and, if the watched queue is empty by then, neither reported nor deleted. `EV_CLEAR` fires again only after the watched queue activates a note; one undone before the watching queue polls is missed. A cycle of watches throws; a note on the queue itself is `EINVAL`. Not measured on a console +- [kevent on a kqueue](../../../core/libs/prx/libkernel/Equeue/src/Kqueue.cpp) (libkernel) - `EVFILT_READ` on a kqueue counts the watched queue's enabled notes that are ready when it is evaluated, where [`filt_kqueue`](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/sys/kern/kern_event.c#L363-L370) reports `kq_count`, the queued notes: a queued note counts until that kqueue is scanned, even once disabled or no longer ready (a socket or pipe read empty, a user note cleared, a watched kqueue drained), so FreeBSD can report it readable where this reports nothing. A queued `EV_ONESHOT` note is reported there with its last count and deleted; here it is evaluated again and, if the watched queue is empty by then, neither reported nor deleted. `EV_CLEAR` fires again only after the watched queue activates a note; one undone before the watching queue polls is missed. A cycle of watches throws; a note on its own queue is `EINVAL`. Not measured on a console - [Socket error numbers](../../../core/libs/prx/libkernel/Socket/include/SocketError.hpp) (libkernel) - a host socket error becomes the FreeBSD 11 `errno.h` number of the same name; the numbers are not checked on the console. Winsock `WSAESHUTDOWN` becomes `EPIPE`, as a send after `shutdown` does on FreeBSD, and a host error with no FreeBSD socket counterpart becomes `EIO`. On Windows a refused nonblocking `connect` is never reported (`SO_ERROR` is unsupported), so a repeated one keeps reporting `EALREADY` where FreeBSD reports `ECONNREFUSED`. - [getsockopt, setsockopt](../../../core/libs/prx/libkernel/Socket/src/SocketOptions.cpp) (libkernel) - a guest socket has the SOL_SOCKET level, eight `IPPROTO_TCP` options, `IPV6_V6ONLY` and ten `IPPROTO_IP` options. `SO_ACCEPTFILTER` (0x1000) fails with `EINVAL` for a set and a get on a socket that is not listening, as a payload process on a PS5 (firmware 13.60) did and as FreeBSD 11.4's `do_setopt_accept_filter` and `do_getopt_accept_filter` do; on a listening socket both throw, as accept filters are not modelled; the other options of these levels fail with `ENOPROTOOPT` and were not all measured. `SO_KEEPALIVE`, `SO_DONTROUTE`, `SO_BROADCAST`, `SO_OOBINLINE` and `SO_LINGER` act on the host socket, and a boolean one reads back as its own bit, as there. Winsock does not clear a pending error when `SO_ERROR` reads it, so on Windows a pending error is reported once and again only after a new `connect`; a second failure with the same code without a `connect` in between is hidden - [TCP_NOPUSH, TCP_NOOPT, TCP option 0x80, IP_RECVDSTADDR, IP_RECVIF, IP_RECVTTL](../../../core/libs/prx/libkernel/Socket/src/SocketOptions.cpp) (libkernel) - kept and read back as a payload process on a PS5 (firmware 13.60) did: the TCP ones as the bits 0x1000, 0x8 and 0x10000 (so 0x80 is not FreeBSD's `TCP_KEEPINIT` there), the IP ones as 0 or 1. None of them changes what the socket does: `TCP_NOPUSH` does not hold back sends, and the IP ones add no control message to `recvmsg`. `IP_RECVTTL` was only read, never set, on the console, and an accepted socket takes `TCP_NOPUSH` from its listener, as FreeBSD 11.4's `syncache_socket` does and as measured on that console From 9b9e87bd729bf20fc29e9d35339bb8ff210bf308 Mon Sep 17 00:00:00 2001 From: simnJS Date: Wed, 28 Oct 2026 14:18:04 +0100 Subject: [PATCH 06/14] feat(libSceNpManager): answer sceNpWaitAsync as sceNpPollAsync does --- core/libs/prx/libSceNpManager/Export.cpp | 7 ++-- core/libs/prx/libSceNpManager/include/Api.hpp | 2 +- core/libs/tests/GuestNpManager.cpp | 33 +++++++++++++++++++ docs/dev/debt/libSceNpManager.md | 2 +- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceNpManager/Export.cpp b/core/libs/prx/libSceNpManager/Export.cpp index 4d60e0869e..046eafb727 100644 --- a/core/libs/prx/libSceNpManager/Export.cpp +++ b/core/libs/prx/libSceNpManager/Export.cpp @@ -887,8 +887,11 @@ int APS5_VABI sceNpPollAsync(int req_id, int* result) { return NP_POLL_ASYNC_FINISHED; } -int APS5_VABI sceNpWaitAsync(void) { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceNpWaitAsync(int req_id, int* result) { + if (!result) return SCE_NP_ERROR_INVALID_ARGUMENT; + std::lock_guard lock(requestMutex); + if (!liveRequests.count(req_id)) return SCE_NP_ERROR_REQUEST_NOT_FOUND; + *result = SCE_NP_ERROR_SIGNED_OUT; return 0; } diff --git a/core/libs/prx/libSceNpManager/include/Api.hpp b/core/libs/prx/libSceNpManager/include/Api.hpp index 0fd7a18a74..85ea6fc53f 100644 --- a/core/libs/prx/libSceNpManager/include/Api.hpp +++ b/core/libs/prx/libSceNpManager/include/Api.hpp @@ -184,7 +184,7 @@ int APS5_VABI sceNpGetOnlineId(int userId, NpOnlineId* onlineId); int APS5_VABI sceNpGetState(int userId, uint32_t* state); int APS5_VABI sceNpHasSignedUp(int userId, bool* hasSignedUp); int APS5_VABI sceNpPollAsync(int reqId, int* result); -int APS5_VABI sceNpWaitAsync(void); +int APS5_VABI sceNpWaitAsync(int reqId, int* result); void APS5_VABI sceNpRegisterGamePresenceCallback(void* callback, void* userdata); int APS5_VABI sceNpRegisterNpReachabilityStateCallback(void* callback, void* userdata); int APS5_VABI sceNpUnregisterNpReachabilityStateCallback(void); diff --git a/core/libs/tests/GuestNpManager.cpp b/core/libs/tests/GuestNpManager.cpp index 6ccef24325..eaa10034b0 100644 --- a/core/libs/tests/GuestNpManager.cpp +++ b/core/libs/tests/GuestNpManager.cpp @@ -225,6 +225,38 @@ void CheckAsyncRequestLifecycle() { Require(sceNpDeleteRequest(unusedAsyncResult) == RequestNotFound, "second delete"); } +void CheckWaitAsync() { + NpCreateAsyncRequestParameter param{}; + const int liveId = sceNpCreateAsyncRequest(¶m); + Require(liveId > 0, "sceNpCreateAsyncRequest must return a positive request id"); + const int deletedId = sceNpCreateAsyncRequest(¶m); + Require(deletedId > 0 && deletedId != liveId, "sceNpCreateAsyncRequest must return distinct positive request ids"); + Require(sceNpDeleteRequest(deletedId) == 0, "initial delete"); + + constexpr int resultSentinel = 0x5a5a5a5a; + for (const int id : {0, -1, 0x7fffffff, deletedId}) { + int result = resultSentinel; + Require(sceNpWaitAsync(id, &result) == RequestNotFound, "wait must reject an absent or deleted id"); + Require(result == resultSentinel, "wait must not touch the result for an absent or deleted id"); + Require(sceNpWaitAsync(id, nullptr) == InvalidArgument, "wait must reject a null result before the id"); + } + Require(sceNpWaitAsync(liveId, nullptr) == InvalidArgument, "wait must reject a null result for a live request"); + + for (int round = 0; round < 2; ++round) { + int result = resultSentinel; + Require(sceNpWaitAsync(liveId, &result) == 0, "wait on a live async request"); + Require(result == SignedOut, "wait must report the signed-out result on a live async request"); + } + int pollResult = resultSentinel; + Require(sceNpPollAsync(liveId, &pollResult) == 0, "poll after wait"); + Require(pollResult == SignedOut, "poll after wait must report the same result"); + + Require(sceNpDeleteRequest(liveId) == 0, "delete after wait"); + int result = resultSentinel; + Require(sceNpWaitAsync(liveId, &result) == RequestNotFound, "wait after delete"); + Require(result == resultSentinel, "wait must not touch the result after delete"); +} + void CheckIdMapperRequests() { for (const int reqId : {0, -1, -0x7fffffff}) { Require(sceNpIdMapperAbortRequest(reqId) == IdMapperRefusedRequestId, "sceNpIdMapperAbortRequest must refuse a request id of 0 or less"); @@ -558,6 +590,7 @@ int main() { CheckLegacyStateCallback(); CheckRequestTimeouts(); CheckAsyncRequestLifecycle(); + CheckWaitAsync(); CheckIdMapperRequests(); CheckManagerIntRequests(); CheckIdMapperLookup("sceNpIdMapperNpIdToAccountId", sceNpIdMapperNpIdToAccountId); diff --git a/docs/dev/debt/libSceNpManager.md b/docs/dev/debt/libSceNpManager.md index 8afcb90846..39afaf5ca5 100644 --- a/docs/dev/debt/libSceNpManager.md +++ b/docs/dev/debt/libSceNpManager.md @@ -6,7 +6,6 @@ ### Unknown function info -- [sceNpWaitAsync](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager, NID `jyi5p9XWUSs`) - the public PS5 3.20 inventory identifies this as a function. Its signature, request completion, output and error behavior are not verified for PS5, so the no-parameter placeholder throws instead of reporting a fabricated completion result. - [libSceNpManager pending exports](../../../core/libs/prx/libSceNpManager/Export.cpp) - 144 named callable exports are present in the local `libSceNpManager.sprx` export table and the public PS5 3.20 symbol list. Their signatures and behavior are unverified; each declaration uses the no-parameter unknown-signature convention and throws through `NotImplemented_nid_no_patch`. - [sceNpGetSandboxInfo](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager, NID `ZVZ++68nNI8`) - the name hashes to the imported NID (found with nidhunt, confirmed with `tools/nid_names.py`). STAR WARS: Galactic Racer (PPSA28416) imports it; its OnlineSubsystemPS5 calls it with one pointer to a 0x50-byte stack buffer and reads the buffer as a string (discussions/3086). The returned string, the exact parameter struct and console errors are unknown, so every call throws without reading or writing caller memory. A working implementation belongs with that title's bring-up. @@ -44,4 +43,5 @@ - [sceNpManagerIntAuthGetAuthorizationCode, sceNpManagerIntAuthGetIdToken, sceNpManagerIntGetAuthorizationCode, sceNpManagerIntGetAuthorizationCodeWithPsnoUri](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager) - the arities and checks are [measured on firmware 13.42 `libSceNpManager.sprx` by BunnyFl](https://github.com/boykopovar/AnyPS5/pull/10890#issuecomment-6397162960), with no libSceNpAuth layout: user id -1 or a required pointer null gives `SCE_NP_ERROR_INVALID_ARGUMENT`, as do the eight-argument call's values outside 0 to 3 and 1 to 3; a mode other than 0, 3 or 5 gives `0x80550004`; the URI form refuses a size outside 1 to 0x801 with `0x80550011`, then the mode, then a URI of 0x2000 bytes or more with `0x80550005` (terminator unmeasured). Past the checks the answer follows main's offline model, not a console measurement: `SCE_NP_ERROR_SIGNED_OUT`, writing nothing, for any request id; the request lookup and the URI form's `sceHttpUriParse` step are not modelled - [sceNpManagerIntGetAccountId, sceNpManagerIntGetAccountIdSdk, sceNpManagerIntGetUserIdByAccountId](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager) - still throw: their answer or their input is an account id, and what a signed-out user's account id is depends on the offline account-id model of [#6990](https://github.com/boykopovar/AnyPS5/pull/6990) and [#7412](https://github.com/boykopovar/AnyPS5/pull/7412), as for `sceNpIdMapperAccountIdToNpId` - [sceNpPollAsync](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager) - a null result pointer returns `SCE_NP_ERROR_INVALID_ARGUMENT` before the id is read, a request id that was never created or was deleted returns `SCE_NP_ERROR_REQUEST_NOT_FOUND` without writing the caller's result, and a live request keeps its signed-out completion `0` with `SCE_NP_ERROR_SIGNED_OUT`, as [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/81be949167d0060b08f26dc3e055a0b640d129c8/src/core/libraries/np/np_manager.cpp) does in `sceNpPollAsync` ([#9277](https://github.com/boykopovar/AnyPS5/issues/9277)). The parts of that model beyond the validation are not implemented: shadPS4 keeps a per-request state machine decorated by `sceNpCheckNpAvailability` and neighbors and refuses `InvalidId` for a ready async request in wait/poll, whose completion publication, poll-versus-wait precedence and already-complete errors remain unattributable to the PS5 +- [sceNpWaitAsync](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager) - not measured on a PS5: how the console waits on a request still running is unknown. No request is ever running here, so it never blocks and gives `sceNpPollAsync`'s answers, with that entry's gaps, as [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/45837a89a617ac394c1844ec95a28471fb79ec08/src/core/libraries/np/np_manager.cpp#L402-L452) does with two identical bodies. The `(request id, result)` signature follows shadPS4 and [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Np/NpAccount.cs#L138-L146) - [sceNpCreateAsyncRequest](../../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager) - the parameter is not read ([NpCreateAsyncRequestParameter](../../../core/libs/SceTypes.hpp) is a 0x40-byte opaque blob, so shadPS4's nonnull check and `size == sizeof` reject ([81be949167d0060b08f26dc3e055a0b640d129c8](https://github.com/shadps4-emu/shadPS4/blob/81be949167d0060b08f26dc3e055a0b640d129c8/src/core/libraries/np/np_manager.cpp)) cannot be applied without inventing a layout) and no async kind or completion state is stored until the console's contract for them is evidenced ([#9277](https://github.com/boykopovar/AnyPS5/issues/9277)) From 43cae067c5517918227343ab2a9ebccf6a0259f1 Mon Sep 17 00:00:00 2001 From: Aniket Tuli Date: Wed, 21 Oct 2026 22:19:52 -0700 Subject: [PATCH 07/14] feat(libScePs2EmuMenuDialog): return the measured firmware 13.42 results Co-Authored-By: Claude Sonnet 5.5 --- .../prx/libScePs2EmuMenuDialog/CMakeLists.txt | 14 ++++++ .../prx/libScePs2EmuMenuDialog/Export.cpp | 29 ++++++------ core/libs/tests/GuestPs2EmuMenuDialog.cpp | 45 +++++++++++++++++++ docs/dev/debt/libScePs2EmuMenuDialog.md | 2 +- 4 files changed, 74 insertions(+), 16 deletions(-) create mode 100644 core/libs/prx/libScePs2EmuMenuDialog/CMakeLists.txt create mode 100644 core/libs/tests/GuestPs2EmuMenuDialog.cpp diff --git a/core/libs/prx/libScePs2EmuMenuDialog/CMakeLists.txt b/core/libs/prx/libScePs2EmuMenuDialog/CMakeLists.txt new file mode 100644 index 0000000000..fac9e8f0bf --- /dev/null +++ b/core/libs/prx/libScePs2EmuMenuDialog/CMakeLists.txt @@ -0,0 +1,14 @@ +add_library(libScePs2EmuMenuDialog SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libScePs2EmuMenuDialog PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libScePs2EmuMenuDialog) + +if(BUILD_TESTING) + add_test_executable(guest_ps2_emu_menu_dialog_tests ${LIBS_INCLUDE_DIR}/tests/GuestPs2EmuMenuDialog.cpp) + target_include_directories(guest_ps2_emu_menu_dialog_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_ps2_emu_menu_dialog_tests PRIVATE libScePs2EmuMenuDialog) + add_test(NAME guest_ps2_emu_menu_dialog COMMAND guest_ps2_emu_menu_dialog_tests) + if(WIN32) + set_tests_properties(guest_ps2_emu_menu_dialog PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libScePs2EmuMenuDialog/Export.cpp b/core/libs/prx/libScePs2EmuMenuDialog/Export.cpp index a27fb265fc..d454772b97 100644 --- a/core/libs/prx/libScePs2EmuMenuDialog/Export.cpp +++ b/core/libs/prx/libScePs2EmuMenuDialog/Export.cpp @@ -2,43 +2,42 @@ #include #include #include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" + +namespace { + +constexpr int PS2_EMU_MENU_DIALOG_ERROR_UNAVAILABLE = static_cast(0x80B8000Fu); +constexpr int COMMON_DIALOG_STATUS_NONE = 0; + +} extern "C" { int APS5_VABI scePs2EmuMenuDialogClose() { - NotImplemented_nid_no_patch(__func__); - return 0; + return PS2_EMU_MENU_DIALOG_ERROR_UNAVAILABLE; } int APS5_VABI scePs2EmuMenuDialogGetResult() { - NotImplemented_nid_no_patch(__func__); - return 0; + return PS2_EMU_MENU_DIALOG_ERROR_UNAVAILABLE; } int APS5_VABI scePs2EmuMenuDialogGetStatus() { - NotImplemented_nid_no_patch(__func__); - return 0; + return COMMON_DIALOG_STATUS_NONE; } int APS5_VABI scePs2EmuMenuDialogInitialize() { - NotImplemented_nid_no_patch(__func__); - return 0; + return PS2_EMU_MENU_DIALOG_ERROR_UNAVAILABLE; } int APS5_VABI scePs2EmuMenuDialogOpen() { - NotImplemented_nid_no_patch(__func__); - return 0; + return PS2_EMU_MENU_DIALOG_ERROR_UNAVAILABLE; } int APS5_VABI scePs2EmuMenuDialogTerminate() { - NotImplemented_nid_no_patch(__func__); - return 0; + return PS2_EMU_MENU_DIALOG_ERROR_UNAVAILABLE; } int APS5_VABI scePs2EmuMenuDialogUpdateStatus() { - NotImplemented_nid_no_patch(__func__); - return 0; + return COMMON_DIALOG_STATUS_NONE; } } diff --git a/core/libs/tests/GuestPs2EmuMenuDialog.cpp b/core/libs/tests/GuestPs2EmuMenuDialog.cpp new file mode 100644 index 0000000000..cb6f6e2fd3 --- /dev/null +++ b/core/libs/tests/GuestPs2EmuMenuDialog.cpp @@ -0,0 +1,45 @@ +#include "prx/libScePs2EmuMenuDialog/include/Api.hpp" +#include +#include +#include +#include + +namespace { + +constexpr int kStatusNone = 0; +constexpr int kErrUnavailable = static_cast(0x80B8000Fu); + +using OneArg = int(APS5_VABI*)(void*); + +void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "PS2EmuMenuDialog: %s\n", message); + std::abort(); + } +} + +void CheckUntouched(OneArg fn, int expected, const char* message) { + std::uint8_t buffer[32]; + for (std::size_t i = 0; i < sizeof(buffer); ++i) buffer[i] = static_cast(0xA5 ^ i); + void* const args[] = {nullptr, buffer, reinterpret_cast(~std::uintptr_t{0})}; + for (void* arg : args) { + Require(fn(arg) == expected, message); + } + for (std::size_t i = 0; i < sizeof(buffer); ++i) { + Require(buffer[i] == static_cast(0xA5 ^ i), message); + } +} + +} + +int main() { + for (int round = 0; round < 2; ++round) { + CheckUntouched(reinterpret_cast(&scePs2EmuMenuDialogInitialize), kErrUnavailable, "initialize"); + CheckUntouched(reinterpret_cast(&scePs2EmuMenuDialogOpen), kErrUnavailable, "open"); + CheckUntouched(reinterpret_cast(&scePs2EmuMenuDialogGetResult), kErrUnavailable, "get result"); + CheckUntouched(reinterpret_cast(&scePs2EmuMenuDialogClose), kErrUnavailable, "close"); + CheckUntouched(reinterpret_cast(&scePs2EmuMenuDialogTerminate), kErrUnavailable, "terminate"); + CheckUntouched(reinterpret_cast(&scePs2EmuMenuDialogGetStatus), kStatusNone, "get status"); + CheckUntouched(reinterpret_cast(&scePs2EmuMenuDialogUpdateStatus), kStatusNone, "update status"); + } +} diff --git a/docs/dev/debt/libScePs2EmuMenuDialog.md b/docs/dev/debt/libScePs2EmuMenuDialog.md index abb7485619..abebbe9810 100644 --- a/docs/dev/debt/libScePs2EmuMenuDialog.md +++ b/docs/dev/debt/libScePs2EmuMenuDialog.md @@ -2,4 +2,4 @@ ### Unknown function info -- [scePs2EmuMenuDialogClose, scePs2EmuMenuDialogGetResult, scePs2EmuMenuDialogGetStatus, scePs2EmuMenuDialogInitialize, scePs2EmuMenuDialogOpen, scePs2EmuMenuDialogTerminate and scePs2EmuMenuDialogUpdateStatus](../../../core/libs/prx/libScePs2EmuMenuDialog/Export.cpp) (libScePs2EmuMenuDialog) - names and NIDs are present in the [PS5 3.20 export inventory](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libScePs2EmuMenuDialog.c), but no credible prototypes were found. The declarations have no parameters solely to resolve known imports; this is not an ABI claim and callers must not call them until signatures are verified +- [scePs2EmuMenuDialogClose, scePs2EmuMenuDialogGetResult, scePs2EmuMenuDialogGetStatus, scePs2EmuMenuDialogInitialize, scePs2EmuMenuDialogOpen, scePs2EmuMenuDialogTerminate and scePs2EmuMenuDialogUpdateStatus](../../../core/libs/prx/libScePs2EmuMenuDialog/Export.cpp) (libScePs2EmuMenuDialog) - names and NIDs are present in the [PS5 3.20 export inventory](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libScePs2EmuMenuDialog.c). Behaviour follows a measurement on firmware 13.42 reported in PR #9634: `Initialize`, `Open`, `GetResult`, `Close` and `Terminate` always return `0x80B8000F` and `UpdateStatus`/`GetStatus` return 0 (`COMMON_DIALOG_STATUS_NONE`), with no state and no writes to the argument, checked with null, small and all-ones arguments. Other firmware versions and the parameter types remain unmeasured; the declarations have no parameters and this is not an ABI claim. From ae305dfd5d11c66483a47ee2d71b8ecb74635a6a Mon Sep 17 00:00:00 2001 From: Kian Date: Wed, 28 Oct 2026 23:05:17 +1000 Subject: [PATCH 08/14] feat(agc): APS5_DUMP_SHADERS takes a list of program addresses A comma-separated list of hex program addresses (optional 0x prefix) now writes only those programs' requests, so one expensive shader can be captured without serializing every request a title prepares. Any other value, such as 1, keeps writing every request. A filtered-out program returns no name, so diagnostics never point at a file that was not written, and it stays eligible for a later list. Co-Authored-By: Claude Opus 5.5 --- .../src/Driver/Shaders/ShaderDiagnostics.cpp | 26 +++++++++++ .../tests/execution/PreparedShaders.cpp | 45 +++++++++++++++++++ docs/dev/ENVIRONMENT.md | 2 +- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderDiagnostics.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderDiagnostics.cpp index 91b97871d7..19a1ca8811 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderDiagnostics.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderDiagnostics.cpp @@ -1,12 +1,37 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Mutex.hpp" #include "ControlFlow/RequestSerializer.hpp" +#include #include #include +#include #include namespace AgcDriver::DriverDetail { +namespace { + +std::optional> selectedPrograms() { + const char* value = std::getenv("APS5_DUMP_SHADERS"); + if (value == nullptr || std::string_view(value) == "1") return std::nullopt; + std::set programs; + std::string_view rest(value); + while (!rest.empty()) { + const auto comma = rest.find(','); + auto token = rest.substr(0, comma); + rest = comma == std::string_view::npos ? std::string_view{} : rest.substr(comma + 1); + if (token.starts_with("0x") || token.starts_with("0X")) token.remove_prefix(2); + std::uint64_t address = 0; + const auto [end, error] = std::from_chars(token.data(), token.data() + token.size(), address, 16); + if (token.empty() || error != std::errc{} || end != token.data() + token.size()) return std::nullopt; + programs.insert(address); + } + if (programs.empty()) return std::nullopt; + return programs; +} + +} + void Driver::reportSkip(const char* kind, const std::string& what) { static AgcDriver::Mutex reportedMutex; static std::set reported; @@ -56,6 +81,7 @@ std::string Driver::dumpRequest(std::uint64_t address, const ShaderRecompiler::R char hex[32]; std::snprintf(hex, sizeof(hex), "%llx", static_cast(address)); const std::string name = "shader_" + std::string(hex) + std::string(suffix) + ".req"; + if (const auto selected = selectedPrograms(); selected.has_value() && !selected->contains(address)) return {}; std::lock_guard lock(dumpMutex); if (!dumped.insert(name).second) return name; try { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp index 94b7c946eb..1e5cd00571 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -195,6 +195,50 @@ void Run(AgcDriver::VulkanDevice& device) { ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request)); }, "does not refer to registered code"); } +void DumpSelection(AgcDriver::VulkanDevice& device) { + using namespace ShaderRecompiler; + const std::array code{0xffffffffu}; + const std::vector header(2048, std::byte{0x5a}); + RecompileRequest request{{ShaderStage::Compute, 0x12345cafeull, code, 0x20000, header}, + {32, 0, {}, ShaderComputeStageInfo{{1, 1, 1}, 0, {}, false, 1, {}}, {}, {}, {}}, device.ComputeTarget(32), {0, 0, 0, 128}}; + const char* previous = std::getenv("APS5_DUMP_SHADERS"); + const std::string saved = previous != nullptr ? previous : ""; + const auto setCapture = [](const char* value) { +#ifdef _WIN32 + Require(_putenv_s("APS5_DUMP_SHADERS", value) == 0, "cannot set shader capture"); +#else + Require(setenv("APS5_DUMP_SHADERS", value, 1) == 0, "cannot set shader capture"); +#endif + }; + const auto written = [](const std::string& name) { + const bool exists = std::ifstream(name, std::ios::binary).is_open(); + std::remove(name.c_str()); + return exists; + }; + const std::string excludedPath = "shader_12345cafe_excluded.req"; + const std::string listedPath = "shader_12345cafe_listed.req"; + const std::string otherPath = "shader_feedface_unlisted.req"; + std::remove(excludedPath.c_str()); + std::remove(listedPath.c_str()); + std::remove(otherPath.c_str()); + setCapture("feedface"); + Require(AgcDriver::DriverDetail::Driver::dumpRequest(request.shader.codeAddress, request, "_excluded").empty(), "a program outside the APS5_DUMP_SHADERS list was named as dumped"); + Require(!written(excludedPath), "a program outside the APS5_DUMP_SHADERS list was dumped"); + setCapture("feedface,0x12345CAFE"); + Require(AgcDriver::DriverDetail::Driver::dumpRequest(request.shader.codeAddress, request, "_excluded") == excludedPath, "a program dropped by an earlier list was not dumped once listed"); + Require(written(excludedPath), "a listed program's request was not written after an earlier list dropped it"); + Require(AgcDriver::DriverDetail::Driver::dumpRequest(request.shader.codeAddress, request, "_listed") == listedPath, "a listed program was not dumped"); + Require(written(listedPath), "a listed program's request was not written"); + setCapture("12345cafe,not-an-address"); + Require(AgcDriver::DriverDetail::Driver::dumpRequest(0xfeedfaceull, request, "_unlisted") == otherPath, "a value that is not an address list did not dump every program"); + Require(written(otherPath), "a value that is not an address list did not write the request"); +#ifdef _WIN32 + Require(_putenv_s("APS5_DUMP_SHADERS", saved.c_str()) == 0, "cannot restore shader capture"); +#else + Require((previous != nullptr ? setenv("APS5_DUMP_SHADERS", saved.c_str(), 1) : unsetenv("APS5_DUMP_SHADERS")) == 0, "cannot restore shader capture"); +#endif +} + void FailureCapture(AgcDriver::VulkanDevice& device) { using namespace ShaderRecompiler; const std::array code{0xffffffffu}; @@ -1543,6 +1587,7 @@ int main(int argc, char** argv) { Run(*device); RegisteredVertexWorkgroup(device->Target()); FailureCapture(*device); + DumpSelection(*device); PendingRegistrationPreparation(*device); AwaitedRegistrationPreparation(); Require(argc != 2 || std::string_view(argv[1]) != "--fail-before-registration", "injected failure before registration"); diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index b74ec48890..cd7a74ae73 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -148,7 +148,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_DUMP_QUEUE` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp` | | dumps the first submissions of the queue with this hex id | | `APS5_DUMP_REJECTED` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/CommandValidation.cpp` | | writes a rejected submission to rejected_submission_.bin | | `APS5_DUMP_SCALE` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | 4 | keeps every n-th pixel in each direction of a dumped frame (1 to 16384; any other value fails) | -| `APS5_DUMP_SHADERS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp` | | writes each prepared shader's request to shader_
.req | +| `APS5_DUMP_SHADERS` | string | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp` | | writes each prepared shader's request to shader_
.req; a comma-separated list of hex program addresses (for example `807b90a100,0x807b925400`) writes only those programs' requests, and any other value (such as `1`) writes every request | | `APS5_DUMP_STORAGE` | string | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | selects storage textures to dump by hex address or WxHxL[:format] extent as storage_
_.raw | | `APS5_DUMP_STORAGE_GATE` | path | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | with `APS5_DUMP_STORAGE`, waits for this path to exist and reads matching live images directly without guest writeback; at most eight addresses, 8 MiB linear storage per image; writes `storage_direct_
_.raw` with the existing pitch, height and VkFormat header and first-mip pixels | | `APS5_TEST_DIRECT_READBACK` | set | `core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp` | | runs the clean-volume gated live-image readback regression in the layered volume execution test | From b0ed32bd6c3d02643c4effaba08ed0172b69aece Mon Sep 17 00:00:00 2001 From: InfinitySpeed <101837966+InfinitySpeed@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:19:20 +0200 Subject: [PATCH 09/14] feat(libSceConvertKeycode): implement sceConvertKeycodeGetCharacterFromKeyboardData --- core/libs/prx/libSceConvertKeycode/Export.cpp | 9 +++++- core/libs/tests/GuestConvertKeycode.cpp | 30 +++++++++++++++++++ docs/dev/debt/libSceConvertKeycode.md | 3 +- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceConvertKeycode/Export.cpp b/core/libs/prx/libSceConvertKeycode/Export.cpp index f96e2deca1..fa2cf6af55 100644 --- a/core/libs/prx/libSceConvertKeycode/Export.cpp +++ b/core/libs/prx/libSceConvertKeycode/Export.cpp @@ -2431,7 +2431,14 @@ int APS5_VABI sceConvertKeycodeGetVirtualKeycode(std::uint16_t keycode, std::uin } int APS5_VABI sceConvertKeycodeGetCharacterFromKeyboardData(const KeyboardData* keyData, std::int32_t index, std::uint32_t state, std::uint32_t keyboardType, char16_t* character) { - NotImplemented_nid_no_patch(__func__); + if (keyData == nullptr || character == nullptr) return kErrInvalidAddress; + if (keyboardType == 0 || keyboardType > 39) return kErrInvalidKeyboardType; + if (keyboardType >= 31) throw std::runtime_error("sceConvertKeycodeGetCharacterFromKeyboardData: guest sdk version is unavailable"); + if (index < 0 || index >= keyData->length || index >= static_cast(KEYBOARD_MAX_KEYCODES)) { + throw std::runtime_error("sceConvertKeycodeGetCharacterFromKeyboardData: result for an index outside the key data is unknown"); + } + const std::uint16_t keycode = keyData->key_code[index]; + *character = ConvertCharacter(keycode, state, keyboardType); return 0; } diff --git a/core/libs/tests/GuestConvertKeycode.cpp b/core/libs/tests/GuestConvertKeycode.cpp index d2d3cb463f..fc100b607b 100644 --- a/core/libs/tests/GuestConvertKeycode.cpp +++ b/core/libs/tests/GuestConvertKeycode.cpp @@ -213,5 +213,35 @@ int main() { Require(sceConvertKeycodeGetCharacter(key, 0xFFFFFFFFu, layout, &character) == 0 && character == 0); } } + KeyboardData keyData{}; + keyData.length = 1; + keyData.key_code[0] = 4; + char16_t kbdChar = u'?'; + Require(sceConvertKeycodeGetCharacterFromKeyboardData(nullptr, 0, 0, 4, &kbdChar) == static_cast(0x80BC0031u)); + Require(sceConvertKeycodeGetCharacterFromKeyboardData(&keyData, 0, 0, 4, nullptr) == static_cast(0x80BC0031u)); + for (std::int32_t badIndex : {-1, 1, 16}) { + kbdChar = u'?'; + bool rejected = false; + try { + sceConvertKeycodeGetCharacterFromKeyboardData(&keyData, badIndex, 0, 4, &kbdChar); + } catch (const std::runtime_error&) { + rejected = true; + } + Require(rejected && kbdChar == u'?'); + } + for (std::uint32_t layout : {31u, 39u}) { + kbdChar = u'?'; + bool rejected = false; + try { + sceConvertKeycodeGetCharacterFromKeyboardData(&keyData, 0, 0, layout, &kbdChar); + } catch (const std::runtime_error&) { + rejected = true; + } + Require(rejected && kbdChar == u'?'); + } + Require(sceConvertKeycodeGetCharacterFromKeyboardData(&keyData, 0, 0, 0, &kbdChar) == static_cast(0x80BC001Du)); + Require(sceConvertKeycodeGetCharacterFromKeyboardData(&keyData, 0, 0, 40, &kbdChar) == static_cast(0x80BC001Du)); + Require(sceConvertKeycodeGetCharacterFromKeyboardData(&keyData, 0, 0, 4, &kbdChar) == 0 && kbdChar == u'a'); + Require(sceConvertKeycodeGetCharacterFromKeyboardData(&keyData, 0, 0x200, 4, &kbdChar) == 0 && kbdChar == u'A'); return 0; } diff --git a/docs/dev/debt/libSceConvertKeycode.md b/docs/dev/debt/libSceConvertKeycode.md index 2c6bae6882..3fbf0e754c 100644 --- a/docs/dev/debt/libSceConvertKeycode.md +++ b/docs/dev/debt/libSceConvertKeycode.md @@ -2,5 +2,6 @@ ### Functional -- [sceConvertKeycodeGetVirtualKeycode](../../../core/libs/prx/libSceConvertKeycode/Export.cpp) and `sceConvertKeycodeGetCharacter`: layouts 31–39 throw until the guest platform and compiled SDK version are exposed. `GetVirtualKeycode` also throws for version-dependent Y/Z mappings in layouts 2, 3 and 12. Mapping tables and modifier behavior are recovered; no SDK version is guessed. Three titles disassembled for this entry call `sceConvertKeycodeGetVirtualKeycode` at four sites (PPSA24779 0xa9199a, PPSA29866 0x2d8bdd, PPSA30601 0x15e4624 and 0x15e47da): the keyboard type is the constant 4 at 0xa9199a and 0x15e47da and a run-time value at the other two, and no value read is 2, 3, 12 or in 31 to 39. +- [sceConvertKeycodeGetVirtualKeycode](../../../core/libs/prx/libSceConvertKeycode/Export.cpp) `sceConvertKeycodeGetCharacter` and `sceConvertKeycodeGetCharacterFromKeyboardData`: layouts 31–39 throw until the guest platform and compiled SDK version are exposed. `GetVirtualKeycode` also throws for version-dependent Y/Z mappings in layouts 2, 3 and 12. Mapping tables and modifier behavior are recovered; no SDK version is guessed. Three titles disassembled for this entry call `sceConvertKeycodeGetVirtualKeycode` at four sites (PPSA24779 0xa9199a, PPSA29866 0x2d8bdd, PPSA30601 0x15e4624 and 0x15e47da): the keyboard type is the constant 4 at 0xa9199a and 0x15e47da and a run-time value at the other two, and no value read is 2, 3, 12 or in 31 to 39. +- [sceConvertKeycodeGetCharacterFromKeyboardData](../../../core/libs/prx/libSceConvertKeycode/Export.cpp): an index outside `keyData->length` or `KEYBOARD_MAX_KEYCODES` throws; no reference gives the real result or validation order. - [sceConvertKeycodeGetImeKeyboardType](../../../core/libs/prx/libSceConvertKeycode/Export.cpp): implemented exactly as the repository owner (boykopovar) specified the UserService keyboard type (plus the Japanese variant for type 37 and the Traditional Chinese variant for type 39) to IME keyboard type table, the SDK compatibility rules (38 becomes 4 for PS5 SDK below 0x06000000 or PS4 SDK below 0x10000000; 39 becomes 29 for PS5 SDK below 0x08000000 and for every PS4 client), the error codes and the unmatched-type result (success, IME type 0). The PS4/PS5 client rule reads the title ID prefix from `param.sfo` or `param.json` (`CUSA` is PS4); a missing, malformed or oversized ID or title, as in homebrew, is treated as PS5. Reliable detection inspects the original `eboot.bin`, which only the relinker reads, so it is not done. UserService reports the keyboard type of the configured system language, so the function returns the owner's IME type for that language (see libSceUserService.md). From 0a071016498f3a6b09a6bfb4c11fd448c046c5bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0?= Date: Wed, 28 Oct 2026 14:27:12 +0100 Subject: [PATCH 10/14] docs(libSceConvertKeycode): fix debt entry punctuation --- 3rdparty/SPIRV-Headers | 2 +- 3rdparty/SPIRV-Tools | 2 +- 3rdparty/vorbis | 2 +- docs/dev/debt/libSceConvertKeycode.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/3rdparty/SPIRV-Headers b/3rdparty/SPIRV-Headers index b824a462d4..01e0577914 160000 --- a/3rdparty/SPIRV-Headers +++ b/3rdparty/SPIRV-Headers @@ -1 +1 @@ -Subproject commit b824a462d4256d720bebb40e78b9eb8f78bbb305 +Subproject commit 01e0577914a75a2569c846778c2f93aa8e6feddd diff --git a/3rdparty/SPIRV-Tools b/3rdparty/SPIRV-Tools index 262bdab481..7f2d9ee926 160000 --- a/3rdparty/SPIRV-Tools +++ b/3rdparty/SPIRV-Tools @@ -1 +1 @@ -Subproject commit 262bdab48146c937467f826699a40da0fdfc0f1a +Subproject commit 7f2d9ee926f98fc77a3ed1e1e0f113b8c9c49458 diff --git a/3rdparty/vorbis b/3rdparty/vorbis index c2aa86b05e..0657aee69d 160000 --- a/3rdparty/vorbis +++ b/3rdparty/vorbis @@ -1 +1 @@ -Subproject commit c2aa86b05e981c96bf381fc6aa11cdd03eccc2fb +Subproject commit 0657aee69dec8508a0011f47f3b69d7538e9d262 diff --git a/docs/dev/debt/libSceConvertKeycode.md b/docs/dev/debt/libSceConvertKeycode.md index 3fbf0e754c..22a13f06a2 100644 --- a/docs/dev/debt/libSceConvertKeycode.md +++ b/docs/dev/debt/libSceConvertKeycode.md @@ -2,6 +2,6 @@ ### Functional -- [sceConvertKeycodeGetVirtualKeycode](../../../core/libs/prx/libSceConvertKeycode/Export.cpp) `sceConvertKeycodeGetCharacter` and `sceConvertKeycodeGetCharacterFromKeyboardData`: layouts 31–39 throw until the guest platform and compiled SDK version are exposed. `GetVirtualKeycode` also throws for version-dependent Y/Z mappings in layouts 2, 3 and 12. Mapping tables and modifier behavior are recovered; no SDK version is guessed. Three titles disassembled for this entry call `sceConvertKeycodeGetVirtualKeycode` at four sites (PPSA24779 0xa9199a, PPSA29866 0x2d8bdd, PPSA30601 0x15e4624 and 0x15e47da): the keyboard type is the constant 4 at 0xa9199a and 0x15e47da and a run-time value at the other two, and no value read is 2, 3, 12 or in 31 to 39. +- [sceConvertKeycodeGetVirtualKeycode](../../../core/libs/prx/libSceConvertKeycode/Export.cpp), `sceConvertKeycodeGetCharacter` and `sceConvertKeycodeGetCharacterFromKeyboardData`: layouts 31–39 throw until the guest platform and compiled SDK version are exposed. `GetVirtualKeycode` also throws for version-dependent Y/Z mappings in layouts 2, 3 and 12. Mapping tables and modifier behavior are recovered; no SDK version is guessed. Three titles disassembled for this entry call `sceConvertKeycodeGetVirtualKeycode` at four sites (PPSA24779 0xa9199a, PPSA29866 0x2d8bdd, PPSA30601 0x15e4624 and 0x15e47da): the keyboard type is the constant 4 at 0xa9199a and 0x15e47da and a run-time value at the other two, and no value read is 2, 3, 12 or in 31 to 39. - [sceConvertKeycodeGetCharacterFromKeyboardData](../../../core/libs/prx/libSceConvertKeycode/Export.cpp): an index outside `keyData->length` or `KEYBOARD_MAX_KEYCODES` throws; no reference gives the real result or validation order. - [sceConvertKeycodeGetImeKeyboardType](../../../core/libs/prx/libSceConvertKeycode/Export.cpp): implemented exactly as the repository owner (boykopovar) specified the UserService keyboard type (plus the Japanese variant for type 37 and the Traditional Chinese variant for type 39) to IME keyboard type table, the SDK compatibility rules (38 becomes 4 for PS5 SDK below 0x06000000 or PS4 SDK below 0x10000000; 39 becomes 29 for PS5 SDK below 0x08000000 and for every PS4 client), the error codes and the unmatched-type result (success, IME type 0). The PS4/PS5 client rule reads the title ID prefix from `param.sfo` or `param.json` (`CUSA` is PS4); a missing, malformed or oversized ID or title, as in homebrew, is treated as PS5. Reliable detection inspects the original `eboot.bin`, which only the relinker reads, so it is not done. UserService reports the keyboard type of the configured system language, so the function returns the owner's IME type for that language (see libSceUserService.md). From 4ababcbcfa7218122c5d1f19c639766c7796f1d6 Mon Sep 17 00:00:00 2001 From: dom Date: Wed, 28 Oct 2026 09:35:23 -0400 Subject: [PATCH 11/14] fix(tools): ignore no-op relocations in module annotations --- tools/annotate_module.py | 3 ++ tools/tests/test_annotate_module.py | 58 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/tools/annotate_module.py b/tools/annotate_module.py index a4ef16d003..7074dc3593 100644 --- a/tools/annotate_module.py +++ b/tools/annotate_module.py @@ -32,6 +32,7 @@ STT_TLS = 6 DT_GNU_HASH = 0x6ffffef5 SYMBOL_SIZE = 24 RELA_SIZE = 24 +R_X86_64_NONE = 0 LINE_ADDRESS = re.compile(r"^\s*([0-9a-f]+):") STUB = re.compile(r"^\s*([0-9a-f]+):\s+(?:\w+\s+)?jmpq?\s+\*(?:-?0x[0-9a-f]+)?\(%rip\)\s+# 0x([0-9a-f]+)") @@ -159,6 +160,8 @@ class Module: start = self.address(table) for position in range(start, start + size, RELA_SIZE): offset, info, _ = self.unpack("> 32: slots[offset] = self.symbol(info >> 32)[0] return slots diff --git a/tools/tests/test_annotate_module.py b/tools/tests/test_annotate_module.py index 542e44b022..98354f6179 100644 --- a/tools/tests/test_annotate_module.py +++ b/tools/tests/test_annotate_module.py @@ -97,6 +97,44 @@ def instruction(listing, mnemonic): return next(line for line in listing if pattern.search(line)) +def with_relocation(module, table_offset, index, slot, symbol_index, kind): + image = bytearray(module) + struct.pack_into(" " + IMPORT_NAME)) + def test_none_relocation_does_not_annotate_a_stub_or_call(self): + module = with_relocation(build_module(), 0x2280, 0, SLOT, 1, 0) + summary = self.annotate(module) + self.assertIn("0 imports, 1 exports, 0 tls, 0 stubs", summary) + self.assertEqual(self.read(".imports.txt"), "") + listing = self.read(".asm") + self.assertNotIn("; GOT ", listing) + self.assertNotIn("; -> ", listing) + self.assertIn("; ===== export " + EXPORT_NAME, listing) + + def test_none_relocation_keeps_the_real_import_annotation(self): + symbols = DEFAULT_SYMBOLS + [(SECOND_IMPORT_NAME, 0, 0x12, 0)] + module = build_module(symbols=symbols, relocations=((SLOT, 1), (SLOT, 3))) + module = with_relocation(module, 0x2280, 1, SLOT, 3, 0) + self.annotate(module) + self.assertEqual(self.read(".imports.txt"), "%#x %s %s\n" % (SLOT, symbol_name(IMPORT_NAME), IMPORT_NAME)) + listing = self.read(".asm").splitlines() + self.assertTrue(instruction(listing, "jmp").endswith("; GOT " + IMPORT_NAME)) + self.assertTrue(instruction(listing, "call").endswith("; -> " + IMPORT_NAME)) + def test_tls_symbol_is_listed_but_not_an_address(self): symbols = DEFAULT_SYMBOLS + [(TLS_NAME, 1, 0x16, EXPORT_ADDRESS)] summary = self.annotate(build_module(symbols=symbols)) From 4620e29426e904841ccc9bb1d16363cfcc08919a Mon Sep 17 00:00:00 2001 From: cgrd <20777498+cgrd@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:48:58 -0700 Subject: [PATCH 12/14] fix(shader): drop vector stores through TYPE 2/3 and FORMAT 0 buffer V#s as measured A buffer V# whose TYPE is not 0 threw "buffer descriptor uses an unsupported type" for every access. Measured on a PS5 (CFI-1115B, firmware 12.20): through a V# with TYPE 2 or 3 and FORMAT 0 or a valid format, buffer_store_dword and buffer_store_format_x write nothing and don't fault, so a buffer a shader only stores to through one is bound as a null V#. Loads and atomics through those V#s leave the destination VGPR unchanged, which a null V# can't reproduce, so they still throw. Through a TYPE 0, FORMAT 0 V#, vector loads return 0 and stores are dropped, and those buffers are bound as a null V# too. TYPE 1 faults the GPU for every access measured and keeps throwing. Co-Authored-By: Claude Opus 5.5 --- .../execution/BufferDescriptorRuntime.cpp | 78 +++++++++++++++++++ .../Optimization/src/ResourceMaterializer.cpp | 23 ++++++ docs/dev/debt/shader.md | 1 + 3 files changed, 102 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp index 0a1624612e..0f8ea4f99b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp @@ -4,6 +4,7 @@ #include "prx/libc/include/GuestAllocations.hpp" #include "Optimization/ResourceProgram.hpp" #include "VulkanTestDevice.hpp" +#include #include #include #include @@ -177,6 +178,82 @@ void ScalarReads(AgcDriver::VulkanDevice& device) { GuestAllocations::Mutation().Remove(Input.data()); } +void DisabledDescriptors(AgcDriver::VulkanDevice& device) { + constexpr std::uint32_t Value = 0x12345678u; + alignas(256) static constexpr std::array formattedStore{0x7e0202ffu, Value, 0xe0102000u, 0x80000100u, 0xbf810000u}; + alignas(256) static constexpr std::array rawStore{0x34020082u, 0x7e0402ffu, Value, 0xe0701000u, 0x80000201u, 0xbf810000u}; + alignas(256) static constexpr std::array formattedLoad{0x34020082u, 0xe0002000u, 0x80000200u, 0xbf8c3f70u, 0xe0701000u, 0x80010201u, 0xbf810000u}; + alignas(256) static constexpr std::array atomicAdd{0x34020082u, 0xe0c81000u, 0x80000201u, 0xbf810000u}; + std::array userData{}; + auto request = Request(userData, device.ComputeTarget(Lanes), {}); + request.useCache = false; + const auto output = BufferDescriptor(Output.data(), sizeof(Output)); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const auto dispatch = [&](std::span code, const std::array& descriptor) { + request.shader.code = code; + request.shader.codeAddress = reinterpret_cast(code.data()); + std::copy(descriptor.begin(), descriptor.end(), userData.begin()); + Output.fill(Fill); + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1); + device.WaitIdle(); + }; + const auto all = [](std::uint32_t expected) { return std::all_of(Output.begin(), Output.end(), [expected](std::uint32_t value) { return value == expected; }); }; + const auto raw = [](const void* data, std::uint32_t bytes, std::uint32_t word3) { + auto descriptor = BufferDescriptor(data, bytes); + descriptor[3] = word3; + return descriptor; + }; + const auto strided = [](const void* data, std::uint32_t word3) { + auto descriptor = BufferDescriptor(data, Lanes); + descriptor[1] |= 4u << 16u; + descriptor[3] = word3; + return descriptor; + }; + std::array input{}; + std::memcpy(input.data(), Input.data(), sizeof(input)); + dispatch(Code, raw(Input.data(), InputBytes, 0x31016facu)); + Require(Output == input, "raw load through a TYPE 0 32_FLOAT V# did not read the buffer"); + dispatch(formattedLoad, strided(Input.data(), 0x31014facu)); + Require(Output == input, "formatted load through a TYPE 0 32_UINT V# did not read the buffer"); + dispatch(rawStore, raw(Output.data(), sizeof(Output), 0x31016facu)); + Require(all(Value), "raw store through a TYPE 0 32_FLOAT V# did not write every lane"); + dispatch(formattedStore, strided(Output.data(), 0x31016facu)); + Require(all(Value), "formatted store through a TYPE 0 32_FLOAT V# did not write every lane"); + dispatch(Code, raw(Input.data(), InputBytes, 0x31000facu)); + Require(all(0u), "raw load through a TYPE 0 FORMAT 0 V# did not return 0"); + dispatch(formattedLoad, strided(Input.data(), 0x31000facu)); + Require(all(0u), "formatted load through a TYPE 0 FORMAT 0 V# did not return 0"); + for (const auto word3 : {0x31000facu, 0x91000facu, 0x91016facu, 0x91b00facu, 0xd1000facu, 0xd1016facu}) { + dispatch(rawStore, raw(Output.data(), sizeof(Output), word3)); + Require(all(Fill), "raw store through V# word 3 " + Hex(word3) + " wrote memory"); + dispatch(formattedStore, strided(Output.data(), word3)); + Require(all(Fill), "formatted store through V# word 3 " + Hex(word3) + " wrote memory"); + } + dispatch(formattedStore, {0x02100b00u, 0xc3c00000u, 0x010dc1dfu, 0x91b00facu}); + Require(all(Fill), "store through an image T# read as a V# wrote memory"); + const auto rejects = [&](std::span code, std::uint32_t word3, const char* access) { + bool rejected = false; + try { + dispatch(code, raw(Output.data(), sizeof(Output), word3)); + } catch (const std::runtime_error& error) { + Require(std::string(error.what()).find("buffer descriptor uses an unsupported type") != std::string::npos, std::string(access) + " through V# word 3 " + Hex(word3) + " failed for another reason: " + error.what()); + rejected = true; + } + Require(rejected, std::string(access) + " accepted V# word 3 " + Hex(word3)); + }; + for (const auto word3 : {0x71000facu, 0x71016facu, 0xb107ffacu, 0xf107ffacu}) { + rejects(Code, word3, "a raw load"); + rejects(rawStore, word3, "a raw store"); + rejects(formattedStore, word3, "a formatted store"); + } + for (const auto word3 : {0x91000facu, 0x91016facu, 0xd1000facu, 0xd1016facu}) { + rejects(Code, word3, "a raw load"); + rejects(formattedLoad, word3, "a formatted load"); + rejects(atomicAdd, word3, "an atomic"); + } +} + } int main() { @@ -187,6 +264,7 @@ int main() { Run(*device, false); Run(*device, true); ScalarReads(*device); + DisabledDescriptors(*device); std::puts("buffer descriptor runtime tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index f7d85442af..cb7db83ee4 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -1613,9 +1613,32 @@ std::uint32_t emulatedCompareState(const ShaderInfo& info, const ResourceSnapsho return *samplerState | (reference << EmulatedCompare::ReferenceShift) | (singleLevel ? EmulatedCompare::SingleLevel : 0u); } +bool disabledBufferDescriptor(const BufferResource& buffer, const DescriptorValue& descriptor) { + const auto decoded = decodeBufferDescriptor(descriptor); + const auto format = decoded.RawFormat(); + if (decoded.Type() == 0u) return format == 0u; + return (decoded.Type() == 2u || decoded.Type() == 3u) && format <= static_cast(IrBufferFormat::Format32_32_32_32Float) && !buffer.read; +} + +bool disabledBufferAccesses(const IrResourcePlan& plan, std::uint32_t index, const DescriptorValue& descriptor) { + const auto& buffer = plan.info.buffers[index]; + if (GcnGuestShaders() || buffer.atomic || buffer.scalar || buffer.imageAlias != BufferResource::NoImageAlias || !disabledBufferDescriptor(buffer, descriptor)) return false; + bool accessed = false; + for (const auto& memory : plan.memoryInfo) { + if (memory.kind != ResourceKind::Buffer || memory.resource != index) continue; + if (memory.typed || memory.gpuDescriptor) return false; + accessed = true; + } + return accessed; +} + void materializeTables(const IrResourcePlan& plan, ResourceSnapshot& snapshot) { for (std::uint32_t i = 0u; i < plan.info.buffers.size(); ++i) { const auto& buffer = plan.info.buffers[i]; + if (disabledBufferAccesses(plan, i, snapshot.buffers.at(i))) { + auto& words = snapshot.buffers.at(i).dwords; + std::fill(words.begin(), words.begin() + 4, 0u); + } const bool scalarOnly = buffer.scalar && !buffer.written && !buffer.atomic && !buffer.formatted && buffer.imageAlias == BufferResource::NoImageAlias && std::none_of(plan.memoryInfo.begin(), plan.memoryInfo.end(), [i](const MemoryInfo& memory) { return memory.kind == ResourceKind::Buffer && memory.resource == i; }); if (scalarOnly) { diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index d9a9964ce3..9d270dc76d 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -31,6 +31,7 @@ ### Functional +- [Buffer descriptor type](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) - a buffer V# whose TYPE (word 3 bits 30-31) is not 0 throws `buffer descriptor uses an unsupported type`, except for vector stores through TYPE 2 or 3. Measured on a PS5 (CFI-1115B, firmware 12.20): through TYPE 2 or 3 with FORMAT 0 or a valid format, `buffer_store_dword` and `buffer_store_format_x` wrote nothing and did not fault, also with an unmapped base; loads and GLC atomics left the destination VGPR unchanged, which a null V# can't reproduce, so they throw. TYPE 0 with FORMAT 0 returned 0 for loads and wrote nothing. TYPE 1 faulted the GPU for every access and throws. Unmeasured and still throwing for TYPE 2 and 3: `tbuffer_*`, scalar loads, FORMAT above 77, GCN shaders and V#s loaded at run time; multi-dword stores are assumed to match the one-dword ones. Closes when: those cases are measured on a PS5. - [Pixel interpolators past `SPI_PS_IN_CONTROL.NUM_INTERP`](../../../core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp) (shader recompiler) - when the IR reads an attribute at or past the guest interpolator count, the missing `SPI_PS_INPUT_CNTL` slots are filled as DEFAULT (`OFFSET` 0x20, constant 0/1) before info collection, and SPIR-V emission of `GetInterpolationParameter` for a missing binding returns the same defaults instead of throwing, matching Mesa's treatment of a PS input with no VS output. Component and mode bounds still throw. A retail title prepares a pixel shader with `NUM_INTERP` 0 while it still issues `v_interp` on attribute 0 (`undefined attribute` after the collector fill alone). Console behaviour for that pairing is not measured. - [Windows cache housekeeping](../../../core/shader/recompiler/ShaderDiskCache.cpp): eviction of old versions and temporary files is skipped for `\\?\` and `\\.\` cache roots because the supported WinLibs directory iterator can enumerate the process working directory for long prefixed paths. The active version's `last-used` stamp update still runs; it fails for long `\\.\` aliases with the supported toolchain. Ordinary Windows directory enumeration remains subject to MAX_PATH. From 123f7f3b605fe8468c874e034f6a64d4354669ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0?= Date: Wed, 28 Oct 2026 14:42:32 +0100 Subject: [PATCH 13/14] fix(shader): keep multi-dword TYPE 2/3 stores throwing and move the failed-capture fixture to TYPE 1 --- .../tests/execution/BufferDescriptorRuntime.cpp | 5 ----- .../libSceAgcDriver/tests/execution/FailedCaptureDump.cpp | 4 ++-- .../recompiler/Optimization/src/ResourceMaterializer.cpp | 2 ++ docs/dev/debt/shader.md | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp index 0f8ea4f99b..15aa7c77a2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp @@ -247,11 +247,6 @@ void DisabledDescriptors(AgcDriver::VulkanDevice& device) { rejects(rawStore, word3, "a raw store"); rejects(formattedStore, word3, "a formatted store"); } - for (const auto word3 : {0x91000facu, 0x91016facu, 0xd1000facu, 0xd1016facu}) { - rejects(Code, word3, "a raw load"); - rejects(formattedLoad, word3, "a formatted load"); - rejects(atomicAdd, word3, "an atomic"); - } } } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FailedCaptureDump.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FailedCaptureDump.cpp index 0382270371..8697d8b1ba 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FailedCaptureDump.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FailedCaptureDump.cpp @@ -88,7 +88,7 @@ void Run() { const std::string first = std::string(prefix) + ".req"; const std::string failed = std::string(prefix) + "_failed.req"; Valid = BufferDescriptor(0x00014204u); - Unsupported = BufferDescriptor(0xa1b00facu); + Unsupported = BufferDescriptor(0x61b00facu); Output.fill(Untouched); const auto valid = Pointer(Valid.data()); Dispatch(valid); @@ -101,7 +101,7 @@ void Run() { } catch (const std::exception& error) { message = error.what(); } - Require(message.find("unsupported type") != std::string::npos, "the dispatch through a type 2 descriptor failed for another reason: " + message); + Require(message.find("unsupported type") != std::string::npos, "the dispatch through a type 1 descriptor failed for another reason: " + message); Require(message.find("(" + failed + ")") != std::string::npos, "the failure does not name the failing dispatch's request: " + message); const auto firstText = Read(first); const auto failedText = Read(failed); diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index cb7db83ee4..6944079956 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -1623,10 +1623,12 @@ bool disabledBufferDescriptor(const BufferResource& buffer, const DescriptorValu bool disabledBufferAccesses(const IrResourcePlan& plan, std::uint32_t index, const DescriptorValue& descriptor) { const auto& buffer = plan.info.buffers[index]; if (GcnGuestShaders() || buffer.atomic || buffer.scalar || buffer.imageAlias != BufferResource::NoImageAlias || !disabledBufferDescriptor(buffer, descriptor)) return false; + const bool measuredOnly = decodeBufferDescriptor(descriptor).Type() != 0u; bool accessed = false; for (const auto& memory : plan.memoryInfo) { if (memory.kind != ResourceKind::Buffer || memory.resource != index) continue; if (memory.typed || memory.gpuDescriptor) return false; + if (measuredOnly && (memory.dataDwords != 1u || memory.componentCount != 1u)) return false; accessed = true; } return accessed; diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 9d270dc76d..674d43958d 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -31,7 +31,7 @@ ### Functional -- [Buffer descriptor type](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) - a buffer V# whose TYPE (word 3 bits 30-31) is not 0 throws `buffer descriptor uses an unsupported type`, except for vector stores through TYPE 2 or 3. Measured on a PS5 (CFI-1115B, firmware 12.20): through TYPE 2 or 3 with FORMAT 0 or a valid format, `buffer_store_dword` and `buffer_store_format_x` wrote nothing and did not fault, also with an unmapped base; loads and GLC atomics left the destination VGPR unchanged, which a null V# can't reproduce, so they throw. TYPE 0 with FORMAT 0 returned 0 for loads and wrote nothing. TYPE 1 faulted the GPU for every access and throws. Unmeasured and still throwing for TYPE 2 and 3: `tbuffer_*`, scalar loads, FORMAT above 77, GCN shaders and V#s loaded at run time; multi-dword stores are assumed to match the one-dword ones. Closes when: those cases are measured on a PS5. +- [Buffer descriptor type](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) - a buffer V# whose TYPE (word 3 bits 30-31) is not 0 throws `buffer descriptor uses an unsupported type`, except for vector stores through TYPE 2 or 3. Measured on a PS5 (CFI-1115B, firmware 12.20): through TYPE 2 or 3 with FORMAT 0 or a valid format, `buffer_store_dword` and `buffer_store_format_x` wrote nothing and did not fault, also with an unmapped base; loads and GLC atomics left the destination VGPR unchanged, which a null V# can't reproduce, so they throw. TYPE 0 with FORMAT 0 returned 0 for loads and wrote nothing. TYPE 1 faulted the GPU for every access and throws. Unmeasured and still throwing for TYPE 2 and 3: `tbuffer_*`, scalar loads, FORMAT above 77, GCN shaders and V#s loaded at run time; multi-dword and multi-component stores through TYPE 2 and 3 are unmeasured and throw. Closes when: those cases are measured on a PS5. - [Pixel interpolators past `SPI_PS_IN_CONTROL.NUM_INTERP`](../../../core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp) (shader recompiler) - when the IR reads an attribute at or past the guest interpolator count, the missing `SPI_PS_INPUT_CNTL` slots are filled as DEFAULT (`OFFSET` 0x20, constant 0/1) before info collection, and SPIR-V emission of `GetInterpolationParameter` for a missing binding returns the same defaults instead of throwing, matching Mesa's treatment of a PS input with no VS output. Component and mode bounds still throw. A retail title prepares a pixel shader with `NUM_INTERP` 0 while it still issues `v_interp` on attribute 0 (`undefined attribute` after the collector fill alone). Console behaviour for that pairing is not measured. - [Windows cache housekeeping](../../../core/shader/recompiler/ShaderDiskCache.cpp): eviction of old versions and temporary files is skipped for `\\?\` and `\\.\` cache roots because the supported WinLibs directory iterator can enumerate the process working directory for long prefixed paths. The active version's `last-used` stamp update still runs; it fails for long `\\.\` aliases with the supported toolchain. Ordinary Windows directory enumeration remains subject to MAX_PATH. From 11bdd371fa80f9cf3834c34372eccbbe1644ee2b Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Thu, 29 Oct 2026 21:35:14 +1300 Subject: [PATCH 14/14] fix(web): accept web release archives that predate newer web files The Pages deploy has failed since #11228 added failure-report.mjs to WEB_FILES: build_web.py --release required the release zip's file set to equal main's current WEB_FILES, but the latest release (v0.1.3) was packaged before that file existed, so every run stopped with "Invalid web release archive layout". Require the core files the site needs (index.html, app.mjs, worker.mjs and native/relinker_wasm.*) plus web-release.json, and allow only names from WEB_FILES. Unknown or duplicate entries, encrypted or empty files, the size limits and the tag/revision check are unchanged. The release is served as shipped, so it stays self-consistent. --- tools/build_web.py | 7 +++- tools/tests/test_build_web.py | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/tools/build_web.py b/tools/build_web.py index c8f964bb76..3cd131ae04 100644 --- a/tools/build_web.py +++ b/tools/build_web.py @@ -14,6 +14,7 @@ import zipfile from package_release import WEB_FILES ROOT = Path(__file__).resolve().parent.parent +REQUIRED_WEB_FILES = ("index.html", "app.mjs", "worker.mjs", "native/relinker_wasm.mjs", "native/relinker_wasm.wasm") MAX_EXPANDED_BYTES = 512 * 1024 * 1024 @@ -169,8 +170,10 @@ def publish_release(output): staging.mkdir() with zipfile.ZipFile(io.BytesIO(data)) as archive: files = archive.infolist() - expected = {*WEB_FILES, "web-release.json"} - if len(files) != len(expected) or {file.filename for file in files} != expected: + names = [file.filename for file in files] + allowed = {*WEB_FILES, "web-release.json"} + if len(set(names)) != len(names) or not allowed.issuperset(names) \ + or not {*REQUIRED_WEB_FILES, "web-release.json"}.issubset(names): raise ValueError("Invalid web release archive layout") if any(file.flag_bits & 1 for file in files): raise ValueError(f"Encrypted release archive {asset['name']} is not supported") diff --git a/tools/tests/test_build_web.py b/tools/tests/test_build_web.py index 0b46ed03cb..d44dff4472 100644 --- a/tools/tests/test_build_web.py +++ b/tools/tests/test_build_web.py @@ -306,6 +306,81 @@ class ReleaseMetadataTests(unittest.TestCase): +class WebReleaseLayoutTests(unittest.TestCase): + TAG, REVISION = "v1", "a" * 40 + CORE = ("index.html", "app.mjs", "worker.mjs", "native/relinker_wasm.mjs", "native/relinker_wasm.wasm") + + def release(self, names, extra=(), metadata=None): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for name in (*names, *extra): + archive.writestr(name, b"content " + name.encode()) + archive.writestr("web-release.json", json.dumps(metadata or {"tag": self.TAG, "revision": self.REVISION})) + return buffer.getvalue() + + def publish(self, data): + url = "https://example.invalid/web.zip" + release = {"tag_name": self.TAG, "html_url": "https://example.invalid/v1", + "assets": [{"name": f"web-{self.TAG}.zip", "size": len(data), "browser_download_url": url, + "digest": "sha256:" + hashlib.sha256(data).hexdigest()}]} + responses = {"https://api.github.com/repos/boykopovar/AnyPS5/releases/latest": json.dumps(release).encode(), url: data} + + def libraries(staging, _release): + (staging / "release.json").write_text(json.dumps({"tag": self.TAG}), encoding="utf-8") + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "web" + with patch.object(build_web, "request", side_effect=lambda address: io.BytesIO(responses[address])), \ + patch.object(build_web, "library_assets"), patch.object(build_web, "publish_libraries", side_effect=libraries), \ + contextlib.redirect_stdout(io.StringIO()): + try: + build_web.publish_release(output) + finally: + self.published = sorted(str(file.relative_to(output)) for file in output.rglob("*") if file.is_file()) if output.exists() else [] + + def test_current_release_with_every_web_file_is_published(self): + self.publish(self.release(build_web.WEB_FILES)) + self.assertIn("failure-report.mjs", self.published) + + def test_release_lacking_a_newer_web_file_is_published_as_shipped(self): + older = [name for name in build_web.WEB_FILES if name != "failure-report.mjs"] + self.publish(self.release(older)) + self.assertNotIn("failure-report.mjs", self.published) + self.assertIn("app.mjs", self.published) + self.assertIn("release.json", self.published) + + def test_release_with_an_unknown_file_is_rejected(self): + for extra in ("unlisted.mjs", "native/extra.wasm", "../escape.mjs", "app.mjs.bak"): + with self.subTest(extra=extra): + with self.assertRaisesRegex(ValueError, "Invalid web release archive layout"): + self.publish(self.release(build_web.WEB_FILES, extra=(extra,))) + self.assertEqual(self.published, []) + + def test_release_missing_a_core_file_is_rejected(self): + for missing in self.CORE: + with self.subTest(missing=missing): + with self.assertRaisesRegex(ValueError, "Invalid web release archive layout"): + self.publish(self.release([name for name in build_web.WEB_FILES if name != missing])) + self.assertEqual(self.published, []) + + def test_release_with_a_duplicate_entry_is_rejected(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + data = self.release(build_web.WEB_FILES, extra=("app.mjs",)) + with self.assertRaisesRegex(ValueError, "Invalid web release archive layout"): + self.publish(data) + + def test_release_with_an_empty_file_or_mismatched_metadata_is_rejected(self): + with self.assertRaisesRegex(ValueError, "does not match the selected release"): + self.publish(self.release(build_web.WEB_FILES, metadata={"tag": "v0", "revision": self.REVISION})) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name in build_web.WEB_FILES: + archive.writestr(name, b"" if name == "app.mjs" else b"x") + archive.writestr("web-release.json", json.dumps({"tag": self.TAG, "revision": self.REVISION})) + with self.assertRaisesRegex(ValueError, "Invalid expanded web release archive size"): + self.publish(buffer.getvalue()) + + class LocalNativeArtifactTests(unittest.TestCase): def test_empty_or_missing_native_input_does_not_copy_sources(self): with tempfile.TemporaryDirectory() as directory: