From f3ac61000d90e1f576e1be3fea4330d5fa3e8a49 Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:27:49 -0700 Subject: [PATCH 01/93] feat(runtime): guest arena, kernel thread/sync/file APIs, crash hooks - libc: reserve guest memory in one arena below the PS5 map limit with MEM_WRITE_WATCH (GuestArena) and a caching guest heap; the allocation registry gains a generation counter, an invalidator callback for the driver's page cache and a pin waiter for leased ranges. This replaces the fault-based GuestMemoryTracking/GuestMemoryBacking layer, whose files are removed. - libkernel: mmap/mprotect/munmap on the arena, flexible memory budget, real sceKernelVirtualQuery; POSIX pthread layer, rwlocks, timed locks, TSD, event flags; 0.5 ms scheduler tick and high-resolution sleeps; TSC/clock/time functions; file APIs return SCE errno codes; the APR file-streaming API. - libc: vectored crash reporter with backtraces for fatal exceptions, SIGABRT, UCRT invalid parameters and process exit; SSE4a EXTRQ/INSERTQ emulation for Intel hosts; %ls/%lc/%S/%C with the guest's 16-bit wchar_t; hand-written setjmp/longjmp for the guest jmp_buf; a path alias table for short save-data mount points; more string, stdio and math exports. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- .../Execution/include/SwapchainState.hpp | 20 - .../Execution/src/DeviceSynchronization.cpp | 17 - .../Execution/src/GuestGpuRange.cpp | 18 - .../Execution/src/PresentationImage.cpp | 29 -- .../Execution/src/SwapchainState.cpp | 28 -- .../Graphics/include/DescriptorCache.hpp | 46 --- .../Graphics/include/DrawQueue.hpp | 39 -- .../Graphics/include/RenderCache.hpp | 55 --- .../Graphics/src/ColorMemoryWriteback.cpp | 28 -- .../Graphics/src/CommandCompletion.cpp | 15 - .../Graphics/src/DescriptorCache.cpp | 65 --- .../Graphics/src/DrawCompletion.cpp | 56 --- .../Graphics/src/DrawQueue.cpp | 48 --- .../Graphics/src/RenderCache.cpp | 131 ------ .../Graphics/src/RenderMemoryOwnership.cpp | 62 --- .../Graphics/src/RenderTexture.cpp | 73 ---- core/libs/prx/libc/CMakeLists.txt | 14 +- core/libs/prx/libc/Export.cpp | 36 ++ core/libs/prx/libc/include/General.hpp | 4 + .../prx/libc/include/GuestAllocations.hpp | 14 + core/libs/prx/libc/include/GuestArena.hpp | 27 ++ .../prx/libc/include/GuestHeapStorage.hpp | 13 - .../prx/libc/include/GuestMemoryBacking.hpp | 18 - .../prx/libc/include/GuestMemoryTracking.hpp | 40 -- .../libc/include/MemoryBackingPlatform.hpp | 22 -- .../libc/include/MemoryTrackingPlatform.hpp | 24 -- .../prx/libc/include/WindowsFormatting.hpp | 44 ++- core/libs/prx/libc/src/General.cpp | 71 +++- core/libs/prx/libc/src/GuestAllocations.cpp | 141 ++++++- core/libs/prx/libc/src/GuestArena.cpp | 154 ++++++++ core/libs/prx/libc/src/GuestHeap.cpp | 204 +++++++++- core/libs/prx/libc/src/GuestHeapStorage.cpp | 116 ------ core/libs/prx/libc/src/GuestMemoryBacking.cpp | 96 ----- .../libs/prx/libc/src/GuestMemoryTracking.cpp | 173 -------- core/libs/prx/libc/src/JumpBuffer.cpp | 52 ++- core/libs/prx/libc/src/LocaleSupport.cpp | 69 +++- core/libs/prx/libc/src/Math.cpp | 38 ++ core/libs/prx/libc/src/MemoryBackingLinux.cpp | 70 ---- .../prx/libc/src/MemoryBackingWindows.cpp | 72 ---- .../libs/prx/libc/src/MemoryTrackingLinux.cpp | 99 ----- .../prx/libc/src/MemoryTrackingWindows.cpp | 78 ---- core/libs/prx/libc/src/Process.cpp | 8 + core/libs/prx/libc/src/RuntimeSupport.cpp | 1 + core/libs/prx/libc/src/StdioExtras.cpp | 160 ++++++++ core/libs/prx/libc/src/Strings.cpp | 115 ++++++ core/libs/prx/libc/src/Time.cpp | 46 ++- core/libs/prx/libc/src/exception/Runtime.cpp | 58 ++- core/libs/prx/libc/src/exception/Unwind.cpp | 50 +-- .../src/specifics/windows/CrashReport.cpp | 372 ++++++++++++++++++ .../src/specifics/windows/Sse4aEmulation.hpp | 136 +++++++ .../Apr/include/AprCommandBuffer.hpp | 55 +++ core/libs/prx/libkernel/Apr/src/Apr.cpp | 196 +++++++++ core/libs/prx/libkernel/CMakeLists.txt | 2 + .../libkernel/DirectMemory/DirectMemory.cpp | 209 +++++++++- .../prx/libkernel/DirectMemory/Export.cpp | 177 +++++++-- core/libs/prx/libkernel/Equeue/Equeue.cpp | 5 + .../prx/libkernel/EventFlag/src/EventFlag.cpp | 184 +++++++-- .../prx/libkernel/File/include/NativeStat.hpp | 1 + .../prx/libkernel/File/src/NativeStat.cpp | 19 + core/libs/prx/libkernel/File/src/Open.cpp | 21 +- core/libs/prx/libkernel/File/src/Stdio.cpp | 54 ++- .../libs/prx/libkernel/Pthread/Posix/Attr.cpp | 119 +++--- .../prx/libkernel/Pthread/Posix/Common.hpp | 34 ++ .../libs/prx/libkernel/Pthread/Posix/Cond.cpp | 72 ++-- .../prx/libkernel/Pthread/Posix/Mutex.cpp | 116 ++++-- .../prx/libkernel/Pthread/Posix/Rwlock.cpp | 43 +- core/libs/prx/libkernel/Pthread/Posix/Sem.cpp | 111 ++++-- .../prx/libkernel/Pthread/Posix/Thread.cpp | 83 ++-- core/libs/prx/libkernel/Pthread/Posix/Tsd.cpp | 27 +- .../prx/libkernel/Pthread/include/Pthread.hpp | 21 + core/libs/prx/libkernel/Pthread/src/Attr.cpp | 91 ++--- core/libs/prx/libkernel/Pthread/src/Cond.cpp | 60 +-- core/libs/prx/libkernel/Pthread/src/Mutex.cpp | 43 +- .../libs/prx/libkernel/Pthread/src/Rwlock.cpp | 117 ++++-- .../libs/prx/libkernel/Pthread/src/Thread.cpp | 122 ++++-- core/libs/prx/libkernel/Pthread/src/Tsd.cpp | 86 +++- .../prx/libkernel/Semaphore/src/Semaphore.cpp | 8 + .../prx/libkernel/System/src/Coredump.cpp | 19 +- .../libs/prx/libkernel/System/src/Process.cpp | 34 +- core/libs/prx/libkernel/Time/Time.cpp | 204 ++++++++-- core/libs/prx/libkernel/Time/include/Time.hpp | 5 + core/libs/prx/libkernel/Unimplemented.cpp | 88 +++++ 82 files changed, 3586 insertions(+), 2205 deletions(-) delete mode 100644 core/libs/prx/libSceAgcDriver/Execution/include/SwapchainState.hpp delete mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/DeviceSynchronization.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/GuestGpuRange.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/PresentationImage.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/SwapchainState.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/ColorMemoryWriteback.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/CommandCompletion.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/DescriptorCache.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/DrawCompletion.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/RenderMemoryOwnership.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/RenderTexture.cpp create mode 100644 core/libs/prx/libc/include/GuestArena.hpp delete mode 100644 core/libs/prx/libc/include/GuestHeapStorage.hpp delete mode 100644 core/libs/prx/libc/include/GuestMemoryBacking.hpp delete mode 100644 core/libs/prx/libc/include/GuestMemoryTracking.hpp delete mode 100644 core/libs/prx/libc/include/MemoryBackingPlatform.hpp delete mode 100644 core/libs/prx/libc/include/MemoryTrackingPlatform.hpp create mode 100644 core/libs/prx/libc/src/GuestArena.cpp delete mode 100644 core/libs/prx/libc/src/GuestHeapStorage.cpp delete mode 100644 core/libs/prx/libc/src/GuestMemoryBacking.cpp delete mode 100644 core/libs/prx/libc/src/GuestMemoryTracking.cpp delete mode 100644 core/libs/prx/libc/src/MemoryBackingLinux.cpp delete mode 100644 core/libs/prx/libc/src/MemoryBackingWindows.cpp delete mode 100644 core/libs/prx/libc/src/MemoryTrackingLinux.cpp delete mode 100644 core/libs/prx/libc/src/MemoryTrackingWindows.cpp create mode 100644 core/libs/prx/libc/src/StdioExtras.cpp create mode 100644 core/libs/prx/libc/src/specifics/windows/CrashReport.cpp create mode 100644 core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp create mode 100644 core/libs/prx/libkernel/Apr/include/AprCommandBuffer.hpp create mode 100644 core/libs/prx/libkernel/Apr/src/Apr.cpp create mode 100644 core/libs/prx/libkernel/Pthread/Posix/Common.hpp create mode 100644 core/libs/prx/libkernel/Unimplemented.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/SwapchainState.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/SwapchainState.hpp deleted file mode 100644 index 4ffc3a574c..0000000000 --- a/core/libs/prx/libSceAgcDriver/Execution/include/SwapchainState.hpp +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SWAPCHAINSTATE_HPP -#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SWAPCHAINSTATE_HPP - -#include - -namespace AgcDriver { - -class SwapchainState { -public: - bool NeedsRecreation() const; - void Recreated(); - bool ProcessResult(VkResult result, const char* operation); - -private: - bool outdated = false; -}; - -} - -#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/DeviceSynchronization.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/DeviceSynchronization.cpp deleted file mode 100644 index fbefc93ec3..0000000000 --- a/core/libs/prx/libSceAgcDriver/Execution/src/DeviceSynchronization.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" - -namespace AgcDriver { - -void VulkanDevice::AcquireGpuMemory() { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - PerformanceTimer timing("Vulkan.AcquireGpuMemory"); - const auto context = graphicsContext(); - Graphics::Require(context.drawQueue != nullptr, "GPU memory barrier requires a draw queue"); - context.drawQueue->RecordMemoryBarrier(context); - timing.Mark("barrier_record"); -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestGpuRange.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestGpuRange.cpp deleted file mode 100644 index b0da605d1c..0000000000 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestGpuRange.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" -#include -#include - -namespace AgcDriver::GuestMemory { - -void CheckGpuRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable) { - const auto address = reinterpret_cast(pointer); - if (alignment == 0 || address == 0 || address % alignment != 0 || bytes > std::numeric_limits::max() - address) throw std::invalid_argument("invalid guest GPU memory range"); - const MemoryAccessScope suspended(nullptr, nullptr); - GuestMemoryTracking::GuestMemoryTrackingValidate_nid_postfix(address, bytes, [writable](std::uint64_t first, std::size_t count) { - CheckRange(reinterpret_cast(first), count, 1, writable); - }); -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/PresentationImage.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/PresentationImage.cpp deleted file mode 100644 index 9224bbc757..0000000000 --- a/core/libs/prx/libSceAgcDriver/Execution/src/PresentationImage.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp" - -namespace AgcDriver { - -void PresentationScaler::RecordImage(VkCommandBuffer commands, VkImage image) { - Graphics::Require(sourceImage != VK_NULL_HANDLE && image != VK_NULL_HANDLE, "presentation image is unavailable"); - const auto pipelineBarrier = context.Function("vkCmdPipelineBarrier"); - VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; - barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; - barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = sourceImage; - barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); - VkImageCopy copy{}; - copy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; - copy.dstSubresource = copy.srcSubresource; - copy.extent = {sourceWidth, sourceHeight, 1}; - context.Function("vkCmdCopyImage")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); - barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; - pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/SwapchainState.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/SwapchainState.cpp deleted file mode 100644 index df90426511..0000000000 --- a/core/libs/prx/libSceAgcDriver/Execution/src/SwapchainState.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "prx/libSceAgcDriver/Execution/include/SwapchainState.hpp" -#include -#include - -namespace AgcDriver { - -bool SwapchainState::NeedsRecreation() const { - return outdated; -} - -void SwapchainState::Recreated() { - outdated = false; -} - -bool SwapchainState::ProcessResult(VkResult result, const char* operation) { - if (result == VK_ERROR_OUT_OF_DATE_KHR) { - outdated = true; - return false; - } - if (result == VK_SUBOPTIMAL_KHR) { - outdated = true; - return true; - } - if (result != VK_SUCCESS) throw std::runtime_error(std::string(operation) + ": Vulkan result " + std::to_string(result)); - return true; -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp deleted file mode 100644 index 56f0c83070..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DESCRIPTORCACHE_HPP -#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DESCRIPTORCACHE_HPP - -#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" -#include -#include -#include -#include - -namespace AgcDriver::Graphics { - -class DescriptorAllocation { -public: - DescriptorAllocation(const Context& context, std::span key, std::span bindings, std::span sizes); - ~DescriptorAllocation(); - DescriptorAllocation(const DescriptorAllocation&) = delete; - DescriptorAllocation& operator=(const DescriptorAllocation&) = delete; - VkDescriptorSetLayout Layout() const { return layout; } - VkDescriptorSet Set() const { return set; } - bool Matches(std::span candidate) const; - -private: - void release() noexcept; - VkDevice device; - PFN_vkDestroyDescriptorPool destroyPool; - PFN_vkDestroyDescriptorSetLayout destroyLayout; - std::vector key; - VkDescriptorSetLayout layout = VK_NULL_HANDLE; - VkDescriptorSet set = VK_NULL_HANDLE; - VkDescriptorPool pool = VK_NULL_HANDLE; -}; - -class DescriptorCache { -public: - std::unique_ptr Take(std::span key); - void Put(std::unique_ptr allocation); - -private: - std::mutex mutex; - std::array, 256> available; - std::size_t cursor = 0; -}; - -} - -#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp deleted file mode 100644 index fc5a053ade..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DRAWQUEUE_HPP -#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DRAWQUEUE_HPP - -#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" - -namespace AgcDriver::Graphics { - -class DrawQueue { -public: - ~DrawQueue(); - VkCommandBuffer Begin(const Context& context); - void Enqueue(std::shared_ptr resources, std::shared_ptr storage); - void Flush(); - void Resolve(std::uint64_t address, std::size_t bytes); - void Wait(); - void WaitGpu(); - void Collect(); - void RecordMemoryBarrier(const Context& context); - -private: - struct Entry { - std::shared_ptr storage; - std::shared_ptr resources; - }; - struct Batch { - std::vector entries; - std::unique_ptr commands; - bool hasBarrier = false; - }; - void retire(Batch batch); - Batch recording; - std::vector pending; - std::vector> available; - std::size_t drawCount = 0; -}; - -} - -#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp deleted file mode 100644 index 601c5cbaf6..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RENDERCACHE_HPP -#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RENDERCACHE_HPP - -#include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" -#include - -namespace AgcDriver::Graphics { - -class ResidentColor { -public: - ResidentColor(const Context& context, const ColorTarget& color); - ~ResidentColor(); - void Begin(VkCommandBuffer commands); - void Download(VkCommandBuffer commands); - void Commit(); - void Transition(VkCommandBuffer commands, VkImageLayout layout); - RenderTarget& Target() { return transfer.Target(color, false); } - const ColorTarget& Description() const { return color; } - bool Valid() const { return valid; } - bool Dirty() const { return dirty; } - std::uint64_t Generation() const { return generation; } - void Invalidate(); - void ReleaseMemory(); - bool SharesPages(const ColorTarget& other) const; - -private: - void resolveCpuAccess(GuestMemoryTracking::Access access); - Context context; - ColorTarget color; - GpuColorTransfer transfer; - VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED; - bool valid = false; - bool dirty = false; - std::uint64_t generation = 0; - std::unique_ptr memoryWatch; -}; - -class RenderCache { -public: - explicit RenderCache(const Context& context) : context(context) {} - ~RenderCache(); - std::shared_ptr Get(const ColorTarget& color, bool blending); - std::shared_ptr Find(std::uint64_t address) const; - void Resolve(std::uint64_t address, std::size_t bytes, bool writable); - void Flush(); - -private: - Context context; - std::map> entries; -}; - -} - -#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorMemoryWriteback.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorMemoryWriteback.cpp deleted file mode 100644 index 8dbbf04a59..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorMemoryWriteback.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" -#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" -#include "prx/libc/include/GuestMemoryBacking.hpp" - -namespace AgcDriver::Graphics { - -void GpuColorTransfer::WriteBack(std::uint64_t address) { - writeBack(address, false); -} - -void GpuColorTransfer::WriteBackTracked(std::uint64_t address) { - writeBack(address, true); -} - -void GpuColorTransfer::writeBack(std::uint64_t address, bool tracked) { - PerformanceTimer timing("ColorTransfer.WriteBack"); - Require(tiled != nullptr && readback != nullptr, "color transfer is not prepared for writeback"); - const ColorTargetLayout layout(width, height, mode); - Require(address != 0 && address % layout.Alignment() == 0, "misaligned color writeback address"); - timing.Mark("validate"); - readback->Invalidate(); - if (tracked) GuestMemoryBacking::GuestMemoryBackingWrite_nid_postfix(address, readback->Bytes().data(), readback->Bytes().size()); - else GuestMemory::Write(address, readback->Bytes(), layout.Alignment()); - timing.Mark("guest_write", layout.Bytes()); -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/CommandCompletion.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/CommandCompletion.cpp deleted file mode 100644 index 283c14f851..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/CommandCompletion.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" - -namespace AgcDriver::Graphics { - -bool CommandBatch::IsComplete() { - Require(submitted, "cannot query an unsubmitted command batch"); - if (!pending) return true; - const auto result = context.Function("vkGetFenceStatus")(context.device, fence); - if (result == VK_NOT_READY) return false; - Check(result, "vkGetFenceStatus graphics"); - pending = false; - return true; -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DescriptorCache.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DescriptorCache.cpp deleted file mode 100644 index 2f40339b78..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DescriptorCache.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp" -#include -#include - -namespace AgcDriver::Graphics { - -DescriptorAllocation::DescriptorAllocation(const Context& context, std::span key, std::span bindings, std::span sizes) : device(context.device), destroyPool(context.Function("vkDestroyDescriptorPool")), destroyLayout(context.Function("vkDestroyDescriptorSetLayout")), key(key.begin(), key.end()) { - try { - VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; - info.bindingCount = static_cast(bindings.size()); - info.pBindings = bindings.data(); - Check(context.Function("vkCreateDescriptorSetLayout")(device, &info, nullptr, &layout), "vkCreateDescriptorSetLayout"); - if (bindings.empty()) return; - VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - poolInfo.maxSets = 1; - poolInfo.poolSizeCount = static_cast(sizes.size()); - poolInfo.pPoolSizes = sizes.data(); - Check(context.Function("vkCreateDescriptorPool")(device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); - VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; - allocation.descriptorPool = pool; - allocation.descriptorSetCount = 1; - allocation.pSetLayouts = &layout; - Check(context.Function("vkAllocateDescriptorSets")(device, &allocation, &set), "vkAllocateDescriptorSets"); - } catch (...) { - release(); - throw; - } -} - -DescriptorAllocation::~DescriptorAllocation() { - release(); -} - -void DescriptorAllocation::release() noexcept { - if (pool) destroyPool(device, pool, nullptr); - if (layout) destroyLayout(device, layout, nullptr); -} - -bool DescriptorAllocation::Matches(std::span candidate) const { - return std::equal(key.begin(), key.end(), candidate.begin(), candidate.end()); -} - -std::unique_ptr DescriptorCache::Take(std::span key) { - std::lock_guard lock(mutex); - for (auto& allocation : available) { - if (allocation && allocation->Matches(key)) return std::move(allocation); - } - return nullptr; -} - -void DescriptorCache::Put(std::unique_ptr allocation) { - Require(allocation != nullptr, "cannot cache an empty descriptor allocation"); - std::lock_guard lock(mutex); - for (std::size_t offset = 0; offset < available.size(); ++offset) { - const auto index = (cursor + offset) % available.size(); - if (available[index]) continue; - available[index] = std::move(allocation); - cursor = (index + 1) % available.size(); - return; - } - available[cursor] = std::move(allocation); - cursor = (cursor + 1) % available.size(); -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DrawCompletion.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DrawCompletion.cpp deleted file mode 100644 index 460eecaf3d..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DrawCompletion.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" - -namespace AgcDriver::Graphics { - -void DrawQueue::retire(Batch batch) { - PerformanceTimer timing("Graphics.DrawQueue.Retire"); - const GuestMemory::MemoryAccessScope suspended(nullptr, nullptr); - Require(drawCount >= batch.entries.size(), "draw queue completion count underflow"); - drawCount -= batch.entries.size(); - for (auto& entry : batch.entries) entry.resources->WriteBack(); - timing.Mark("resources_writeback"); - batch.entries.clear(); - available.push_back(std::move(batch.commands)); - timing.Mark("resources_release"); -} - -void DrawQueue::Collect() { - while (!pending.empty() && pending.front().commands->IsComplete()) { - auto batch = std::move(pending.front()); - pending.erase(pending.begin()); - retire(std::move(batch)); - } -} - -void DrawQueue::WaitGpu() { - Flush(); - for (auto& batch : pending) batch.commands->Wait(); -} - -void DrawQueue::Wait() { - if (pending.empty() && !recording.commands) return; - PerformanceTimer timing("Graphics.DrawQueue.Wait"); - Flush(); - timing.Mark("submit"); - while (!pending.empty()) { - pending.front().commands->Wait(); - timing.Mark("fence_wait"); - auto batch = std::move(pending.front()); - pending.erase(pending.begin()); - retire(std::move(batch)); - timing.Mark("retire"); - } -} - -void DrawQueue::RecordMemoryBarrier(const Context& context) { - const auto commands = Begin(context); - VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); - recording.hasBarrier = true; -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp deleted file mode 100644 index 722df65bf2..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" -#include - -namespace AgcDriver::Graphics { - -DrawQueue::~DrawQueue() { - recording.commands.reset(); - for (auto& batch : pending) batch.commands.reset(); -} - -VkCommandBuffer DrawQueue::Begin(const Context& context) { - Collect(); - if (drawCount >= 64) Wait(); - if (!recording.commands) { - if (available.empty()) recording.commands = std::make_unique(context); - else { - recording.commands = std::move(available.back()); - available.pop_back(); - recording.commands->Reset(); - } - } - return recording.commands->Handle(); -} - -void DrawQueue::Enqueue(std::shared_ptr resources, std::shared_ptr storage) { - Require(recording.commands != nullptr && resources != nullptr && storage != nullptr, "draw batch is incomplete"); - recording.entries.push_back({std::move(storage), std::move(resources)}); - ++drawCount; - if (recording.entries.size() >= 8) Flush(); -} - -void DrawQueue::Flush() { - if (!recording.commands) return; - Require(!recording.entries.empty() || recording.hasBarrier, "cannot submit an incomplete draw batch"); - pending.push_back(std::move(recording)); - recording = Batch{}; - pending.back().commands->Submit(); -} - -void DrawQueue::Resolve(std::uint64_t address, std::size_t bytes) { - const auto overlaps = [&](const auto& entry) { return entry.resources->WritesOverlap(address, bytes); }; - if (std::any_of(recording.entries.begin(), recording.entries.end(), overlaps) || std::any_of(pending.begin(), pending.end(), [&](const auto& batch) { return std::any_of(batch.entries.begin(), batch.entries.end(), overlaps); })) Wait(); -} - - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp deleted file mode 100644 index ef51eac565..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" -#include - -namespace AgcDriver::Graphics { - -void ResidentColor::Transition(VkCommandBuffer commands, VkImageLayout next) { - VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; - barrier.srcAccessMask = layout == VK_IMAGE_LAYOUT_UNDEFINED ? 0u : VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; - barrier.oldLayout = layout; - barrier.newLayout = next; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = Target().Image(); - barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); - layout = next; -} - -void ResidentColor::Begin(VkCommandBuffer commands) { - PerformanceTimer timing("Graphics.ResidentColor.Begin"); - Require(generation != std::numeric_limits::max(), "resident color generation overflow"); - ++generation; - Require(memoryWatch != nullptr, "render target memory ownership was released"); - if (!valid) { - memoryWatch->Protect(GuestMemoryTracking::Protection::Read); - const GuestMemory::MemoryAccessScope suspended(nullptr, nullptr); - transfer.Upload(color.address, color.extent.width, color.extent.height, color.tileMode); - transfer.Detile(commands); - Transition(commands, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); - VkBufferImageCopy copy{}; - copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; - copy.imageExtent = {color.extent.width, color.extent.height, 1}; - context.Function("vkCmdCopyBufferToImage")(commands, transfer.LinearBuffer(), Target().Image(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); - timing.Mark("upload", color.bytes); - } else { - timing.Mark("reuse"); - } - Transition(commands, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); - valid = true; - dirty = true; - memoryWatch->Protect(GuestMemoryTracking::Protection::None); -} - -void ResidentColor::Download(VkCommandBuffer commands) { - if (!dirty) return; - Transition(commands, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); - VkMemoryBarrier reuse{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - reuse.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; - reuse.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &reuse, 0, nullptr, 0, nullptr); - VkBufferImageCopy copy{}; - copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; - copy.imageExtent = {color.extent.width, color.extent.height, 1}; - context.Function("vkCmdCopyImageToBuffer")(commands, Target().Image(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, transfer.LinearBuffer(), 1, ©); - transfer.Tile(commands); -} - -void ResidentColor::Commit() { - if (!dirty) return; - const GuestMemory::MemoryAccessScope suspended(nullptr, nullptr); - transfer.WriteBackTracked(color.address); - dirty = false; - memoryWatch->Protect(GuestMemoryTracking::Protection::Read); -} - -std::shared_ptr RenderCache::Get(const ColorTarget& color, bool blending) { - Require(color.address != 0 && color.bytes != 0 && color.bytes <= std::numeric_limits::max() - color.address, "invalid resident color range"); - if (context.drawQueue) context.drawQueue->Resolve(color.address, color.bytes); - if (blending) { - VkFormatProperties properties{}; - context.formatProperties(context.physical, color.format, &properties); - Require((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT) != 0, "render target format does not support blending"); - } - for (auto it = entries.begin(); it != entries.end();) { - const auto& previous = it->second->Description(); - if (!it->second->SharesPages(color)) { - ++it; - continue; - } - if (previous.address == color.address && previous.bytes == color.bytes && previous.extent.width == color.extent.width && previous.extent.height == color.extent.height && previous.format == color.format && previous.tileMode == color.tileMode) return it->second; - Resolve(previous.address, previous.bytes, true); - it->second->ReleaseMemory(); - it = entries.erase(it); - } - if (entries.size() >= 64) { - Flush(); - for (const auto& [address, resident] : entries) resident->ReleaseMemory(); - entries.clear(); - } - auto entry = std::make_shared(context, color); - entries.emplace(color.address, entry); - return entry; -} - -std::shared_ptr RenderCache::Find(std::uint64_t address) const { - const auto it = entries.find(address); - if (it != entries.end() && context.drawQueue) context.drawQueue->Resolve(address, it->second->Description().bytes); - return it != entries.end() && it->second->Valid() ? it->second : nullptr; -} - -void RenderCache::Resolve(std::uint64_t address, std::size_t bytes, bool writable) { - std::vector> affected; - for (const auto& [base, entry] : entries) { - const auto& color = entry->Description(); - if (address >= base + color.bytes || base >= address + bytes) continue; - if (entry->Dirty()) affected.push_back(entry); - else if (writable) entry->Invalidate(); - } - if (affected.empty()) return; - PerformanceTimer timing("Graphics.RenderCache.Resolve"); - if (context.drawQueue) context.drawQueue->Wait(); - CommandBatch batch(context); - for (const auto& entry : affected) entry->Download(batch.Handle()); - batch.SubmitAndWait(); - timing.Mark("download_wait"); - for (const auto& entry : affected) { - entry->Commit(); - if (writable) entry->Invalidate(); - } - timing.Mark("guest_writeback"); -} - -void RenderCache::Flush() { - Resolve(0, std::numeric_limits::max(), true); -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderMemoryOwnership.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/RenderMemoryOwnership.cpp deleted file mode 100644 index f76dd122dc..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderMemoryOwnership.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" -#include - -namespace AgcDriver::Graphics { - -ResidentColor::ResidentColor(const Context& context, const ColorTarget& color) : context(context), color(color), transfer(context) { - memoryWatch = std::make_unique(color.address, color.bytes, this, [](void* owner, GuestMemoryTracking::Access access) { - static_cast(owner)->resolveCpuAccess(access); - }); -} - -ResidentColor::~ResidentColor() = default; - -void ResidentColor::Invalidate() { - Require(!dirty, "cannot discard GPU-owned render target contents"); - if (memoryWatch) memoryWatch->Protect(GuestMemoryTracking::Protection::ReadWrite); - valid = false; -} - -void ResidentColor::ReleaseMemory() { - Require(!dirty, "cannot release GPU-owned render target memory"); - Invalidate(); - memoryWatch.reset(); -} - -bool ResidentColor::SharesPages(const ColorTarget& other) const { - const auto pageSize = GuestMemoryTracking::GuestMemoryTrackingPageSize_nid_postfix(); - Require(color.bytes != 0 && other.bytes != 0, "empty render target page range"); - Require(color.bytes <= std::numeric_limits::max() - color.address && other.bytes <= std::numeric_limits::max() - other.address, "render target page range overflow"); - const auto first = color.address / pageSize; - const auto last = (color.address + color.bytes - 1) / pageSize; - const auto otherFirst = other.address / pageSize; - const auto otherLast = (other.address + other.bytes - 1) / pageSize; - return first <= otherLast && otherFirst <= last; -} - -void ResidentColor::resolveCpuAccess(GuestMemoryTracking::Access access) { - PerformanceTimer timing("Graphics.RenderMemory.CpuAccess"); - Require(memoryWatch != nullptr && context.drawQueue != nullptr, "render target memory resolver is unavailable"); - if (dirty || access == GuestMemoryTracking::Access::Invalidate) { - context.drawQueue->WaitGpu(); - timing.Mark("draw_wait"); - } - if (dirty) { - CommandBatch batch(context); - Download(batch.Handle()); - batch.SubmitAndWait(); - timing.Mark("download_wait"); - Commit(); - timing.Mark("guest_writeback"); - } - if (access != GuestMemoryTracking::Access::Read) Invalidate(); - if (access == GuestMemoryTracking::Access::Invalidate) context.drawQueue->Wait(); -} - -RenderCache::~RenderCache() { - for (const auto& [address, entry] : entries) entry->ReleaseMemory(); -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderTexture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/RenderTexture.cpp deleted file mode 100644 index a8bdbac8fa..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderTexture.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" - -namespace AgcDriver::Graphics { - -Texture::Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components) : context(context), source(source) { - try { - Require(source != nullptr && descriptor.dimension == TextureDimension::k2D && descriptor.mipCount == 1 && descriptor.baseLevel == 0 && descriptor.baseArray == 0, "invalid resident texture view"); - const auto format = ResolveTextureFormat(descriptor.format); - Require(!IsBlockCompressed(descriptor.format) && BytesPerElement(descriptor.format) == 4, "resident texture copy requires a 32-bit texel format"); - VkFormatProperties properties{}; - context.formatProperties(context.physical, format, &properties); - Require((properties.optimalTilingFeatures & (VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) == (VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT), "resident texture format does not support sampling and copies"); - VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; - info.imageType = VK_IMAGE_TYPE_2D; - info.format = format; - info.extent = {descriptor.width, descriptor.height, 1}; - info.mipLevels = 1; - info.arrayLayers = 1; - info.samples = VK_SAMPLE_COUNT_1_BIT; - info.tiling = VK_IMAGE_TILING_OPTIMAL; - info.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; - info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - Check(context.Function("vkCreateImage")(context.device, &info, nullptr, &image), "vkCreateImage resident texture"); - VkMemoryRequirements requirements{}; - context.Function("vkGetImageMemoryRequirements")(context.device, image, &requirements); - VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; - allocation.allocationSize = requirements.size; - allocationBytes = requirements.size; - allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory resident texture"); - Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory resident texture"); - VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; - viewInfo.image = image; - viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; - viewInfo.format = format; - viewInfo.components = components; - viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView resident texture"); - if (context.drawQueue) context.drawQueue->Flush(); - upload = std::make_unique(context); - const auto commands = upload->Handle(); - source->Transition(commands, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); - VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; - barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; - barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = image; - barrier.subresourceRange = viewInfo.subresourceRange; - const auto pipelineBarrier = context.Function("vkCmdPipelineBarrier"); - pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); - VkImageCopy copy{}; - copy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; - copy.dstSubresource = copy.srcSubresource; - copy.extent = info.extent; - context.Function("vkCmdCopyImage")(commands, source->Target().Image(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); - barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; - barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - barrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); - upload->Submit(); - } catch (...) { - release(); - throw; - } -} - -} diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index 3851f85630..872f8dfb65 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -2,15 +2,15 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/General.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/gcc/StackGuard.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/windows/CrashReport.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/linux/DlIteratePhdr.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Strings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Math.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Time.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/FileAndHeap.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/StdioExtras.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestAllocations.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestMemoryTracking.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestMemoryBacking.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestHeapStorage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestArena.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestHeap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ApplicationHeap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ApplicationProcessParameters.cpp @@ -23,14 +23,6 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/exception/Exports.cpp ) -if(WIN32) - target_sources(libc PRIVATE src/MemoryTrackingWindows.cpp src/MemoryBackingWindows.cpp) -elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$") - target_sources(libc PRIVATE src/MemoryTrackingLinux.cpp src/MemoryBackingLinux.cpp) -else() - message(FATAL_ERROR "Guest GPU memory tracking requires Windows or x86-64 Linux") -endif() - target_include_directories(libc PRIVATE ${LIBS_INCLUDE_DIR} ) diff --git a/core/libs/prx/libc/Export.cpp b/core/libs/prx/libc/Export.cpp index a7bacbfdec..76c3e114b3 100644 --- a/core/libs/prx/libc/Export.cpp +++ b/core/libs/prx/libc/Export.cpp @@ -61,4 +61,40 @@ extern "C" { return 0; } +int APS5_VABI _ZNKSt9exception6_RaiseEv_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI fputwc_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI fputws_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI fscanf_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sscanf_s_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI vswprintf_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("Pu0Ecyk-7FU", libcUnknown_Pu0Ecyk_M7FU); +int APS5_VABI libcUnknown_Pu0Ecyk_M7FU() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + } diff --git a/core/libs/prx/libc/include/General.hpp b/core/libs/prx/libc/include/General.hpp index 05e86a048b..f1ed6ca204 100644 --- a/core/libs/prx/libc/include/General.hpp +++ b/core/libs/prx/libc/include/General.hpp @@ -11,6 +11,10 @@ extern "C" void NotImplemented_nid_no_patch(const char* funcName); extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path); +// Guest path aliases: a guest prefix such as "/_sm/0" (a save-data mount point) resolves to a host +// directory instead of the run directory; the prefix matches whole path components only. +extern "C" void AddPathAlias_nid_no_patch(const char* guestPrefix, const char* hostPath); +extern "C" void RemovePathAlias_nid_no_patch(const char* guestPrefix); #define APS5_INVALID_ARG_EX throw std::invalid_argument(std::string(__func__) + ": invalid argument") diff --git a/core/libs/prx/libc/include/GuestAllocations.hpp b/core/libs/prx/libc/include/GuestAllocations.hpp index db5f0394f6..4a34751488 100644 --- a/core/libs/prx/libc/include/GuestAllocations.hpp +++ b/core/libs/prx/libc/include/GuestAllocations.hpp @@ -31,11 +31,24 @@ void GuestAllocationsEnd_nid_postfix(void* mutation) noexcept; void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t bytes, bool readable, bool writable); void GuestAllocationsRequireUnpinned_nid_postfix(void* mutation, const void* pointer, std::size_t bytes); void GuestAllocationsRequireAvailable_nid_postfix(void* mutation, const void* pointer, std::size_t bytes); +bool GuestAllocationsCovers_nid_postfix(void* mutation, const void* pointer, std::size_t bytes); Range GuestAllocationsFind_nid_postfix(void* mutation, const void* pointer); void GuestAllocationsRemove_nid_postfix(void* mutation, const void* pointer); void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, const std::function& apply); void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function& apply); Lease GuestAllocationsAcquire_nid_postfix(); +// Changes whenever a mutation ends, so callers can cache facts about guest mappings between changes. +std::uint64_t GuestAllocationsGeneration_nid_postfix(); +// Precise invalidation for such caches: the callback runs, after the generation changed, for every +// range whose mapping or protection changed (registry mutations and guest heap decommits). +void GuestAllocationsSetInvalidator_nid_postfix(void (*callback)(std::uintptr_t address, std::size_t bytes)); +void GuestAllocationsInvalidate_nid_postfix(std::uintptr_t address, std::size_t bytes); +// A mutation of a range that GPU work still leases waits for the lease (RequireUnpinned). The +// waiter, set by the GPU driver, is called on the mutating thread with the registry lock released +// and finishes the GPU work that holds leases; it returns whether it finished any (a round that found +// nothing to wait for counts against the same bound as a plain spin). Without one the mutation spins +// until the lease is dropped by another thread. +void GuestAllocationsSetPinWaiter_nid_postfix(bool (*callback)()); } class Mutation { @@ -50,6 +63,7 @@ public: void Add(void* pointer, std::size_t bytes, bool readable, bool writable) { GuestAllocationsAdd_nid_postfix(handle, pointer, bytes, readable, writable); } void RequireUnpinned(const void* pointer, std::size_t bytes) const { GuestAllocationsRequireUnpinned_nid_postfix(handle, pointer, bytes); } void RequireAvailable(const void* pointer, std::size_t bytes) const { GuestAllocationsRequireAvailable_nid_postfix(handle, pointer, bytes); } + bool Covers(const void* pointer, std::size_t bytes) const { return GuestAllocationsCovers_nid_postfix(handle, pointer, bytes); } Range Find(const void* pointer) const { return GuestAllocationsFind_nid_postfix(handle, pointer); } void Remove(const void* pointer) { GuestAllocationsRemove_nid_postfix(handle, pointer); } void Unmap(const void* pointer, std::size_t bytes, const std::function& apply) { GuestAllocationsUnmap_nid_postfix(handle, pointer, bytes, apply); } diff --git a/core/libs/prx/libc/include/GuestArena.hpp b/core/libs/prx/libc/include/GuestArena.hpp new file mode 100644 index 0000000000..0311cbd7e2 --- /dev/null +++ b/core/libs/prx/libc/include/GuestArena.hpp @@ -0,0 +1,27 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTARENA_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTARENA_HPP + +#include +#include + +// Guest virtual memory is placed inside one reserved arena below the PS5 application map limit +// (0xFC_0000_0000), in ascending first-fit order like the PS5 kernel. Guest code indexes tables by +// absolute address and breaks on host addresses outside that range. +namespace GuestArena { + +extern "C" { + +bool GuestArenaAvailable_nid_postfix(); +bool GuestArenaContains_nid_postfix(const void* pointer, std::size_t bytes); +void* GuestArenaAllocate_nid_postfix(std::size_t bytes, std::size_t alignment); +void GuestArenaMarkUsed_nid_postfix(const void* pointer, std::size_t bytes); +void GuestArenaRelease_nid_postfix(const void* pointer, std::size_t bytes); +// The reserved range, and whether it was reserved with page write watching (Windows MEM_WRITE_WATCH). +void GuestArenaRange_nid_postfix(std::uintptr_t* base, std::size_t* bytes); +bool GuestArenaWriteWatched_nid_postfix(); + +} + +} + +#endif diff --git a/core/libs/prx/libc/include/GuestHeapStorage.hpp b/core/libs/prx/libc/include/GuestHeapStorage.hpp deleted file mode 100644 index 22600050d5..0000000000 --- a/core/libs/prx/libc/include/GuestHeapStorage.hpp +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTHEAPSTORAGE_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTHEAPSTORAGE_HPP - -#include - -namespace GuestHeapStorage { - -void* Allocate(std::size_t alignment, std::size_t bytes); -void Free(void* pointer); - -} - -#endif diff --git a/core/libs/prx/libc/include/GuestMemoryBacking.hpp b/core/libs/prx/libc/include/GuestMemoryBacking.hpp deleted file mode 100644 index 1ac03e17af..0000000000 --- a/core/libs/prx/libc/include/GuestMemoryBacking.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTMEMORYBACKING_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTMEMORYBACKING_HPP - -#include -#include - -namespace GuestMemoryBacking { - -extern "C" { -void* GuestMemoryBackingMap_nid_postfix(void* address, std::size_t bytes, std::size_t alignment, int protection); -void GuestMemoryBackingUnmap_nid_postfix(void* address, std::size_t bytes); -void GuestMemoryBackingRequire_nid_postfix(std::uint64_t address, std::size_t bytes); -void GuestMemoryBackingWrite_nid_postfix(std::uint64_t address, const void* source, std::size_t bytes); -} - -} - -#endif diff --git a/core/libs/prx/libc/include/GuestMemoryTracking.hpp b/core/libs/prx/libc/include/GuestMemoryTracking.hpp deleted file mode 100644 index 669543a673..0000000000 --- a/core/libs/prx/libc/include/GuestMemoryTracking.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTMEMORYTRACKING_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTMEMORYTRACKING_HPP - -#include -#include -#include -#include - -namespace GuestMemoryTracking { - -enum class Access { Read, Write, Invalidate }; -enum class Protection { None, Read, ReadWrite }; -using Resolver = void (*)(void*, Access); - -extern "C" { -std::recursive_mutex& GuestMemoryTrackingMutex_nid_postfix(); -std::size_t GuestMemoryTrackingPageSize_nid_postfix(); -void* GuestMemoryTrackingCreate_nid_postfix(std::uint64_t address, std::size_t bytes, void* context, Resolver resolver); -void GuestMemoryTrackingDestroy_nid_postfix(void* handle) noexcept; -void GuestMemoryTrackingProtect_nid_postfix(void* handle, Protection protection); -void GuestMemoryTrackingResolve_nid_postfix(std::uint64_t address, std::size_t bytes, bool writable); -void GuestMemoryTrackingInvalidate_nid_postfix(std::uint64_t address, std::size_t bytes); -void GuestMemoryTrackingValidate_nid_postfix(std::uint64_t address, std::size_t bytes, const std::function& validate); -} - -class Watch { -public: - Watch(std::uint64_t address, std::size_t bytes, void* context, Resolver resolver) : handle(GuestMemoryTrackingCreate_nid_postfix(address, bytes, context, resolver)) {} - ~Watch() { GuestMemoryTrackingDestroy_nid_postfix(handle); } - Watch(const Watch&) = delete; - Watch& operator=(const Watch&) = delete; - void Protect(Protection protection) { GuestMemoryTrackingProtect_nid_postfix(handle, protection); } - -private: - void* handle; -}; - -} - -#endif diff --git a/core/libs/prx/libc/include/MemoryBackingPlatform.hpp b/core/libs/prx/libc/include/MemoryBackingPlatform.hpp deleted file mode 100644 index 952c8f6d1e..0000000000 --- a/core/libs/prx/libc/include/MemoryBackingPlatform.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_MEMORYBACKINGPLATFORM_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_MEMORYBACKINGPLATFORM_HPP - -#include -#include - -namespace GuestMemoryBacking::Platform { - -struct Mapping { - std::uint64_t address; - std::size_t bytes; - void* alias; - std::uintptr_t handle; -}; - -Mapping Map(void* address, std::size_t bytes, std::size_t alignment, int protection); -void Unmap(const Mapping& mapping); -void Deactivate(std::uint64_t address, std::size_t bytes); - -} - -#endif diff --git a/core/libs/prx/libc/include/MemoryTrackingPlatform.hpp b/core/libs/prx/libc/include/MemoryTrackingPlatform.hpp deleted file mode 100644 index 1165819518..0000000000 --- a/core/libs/prx/libc/include/MemoryTrackingPlatform.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_MEMORYTRACKINGPLATFORM_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_MEMORYTRACKINGPLATFORM_HPP - -#include "prx/libc/include/GuestMemoryTracking.hpp" -#include - -namespace GuestMemoryTracking::Platform { - -struct Region { - std::uint64_t address; - std::size_t bytes; - std::uint64_t protection; -}; - -using FaultHandler = bool (*)(std::uint64_t, bool); -std::size_t PageSize(); -void Install(FaultHandler handler); -std::vector Query(std::uint64_t address, std::size_t bytes); -void Protect(std::uint64_t address, std::size_t bytes, Protection protection); -void Restore(const std::vector& regions); - -} - -#endif diff --git a/core/libs/prx/libc/include/WindowsFormatting.hpp b/core/libs/prx/libc/include/WindowsFormatting.hpp index 4d8655a93a..2b2775270b 100644 --- a/core/libs/prx/libc/include/WindowsFormatting.hpp +++ b/core/libs/prx/libc/include/WindowsFormatting.hpp @@ -100,8 +100,30 @@ public: int Count() const { return static_cast(count); } }; +// The guest's wchar_t is 16 bits (UTF-16, as on the PS4/PS5 toolchain); wide conversions are +// formatted as UTF-8 through %s. +inline void AppendUtf8(std::string& utf8, char32_t code) { + if (code > 0x10ffff) code = 0xfffd; + if (code < 0x80) { + utf8 += static_cast(code); + } else if (code < 0x800) { + utf8 += static_cast(0xc0 | (code >> 6)); + utf8 += static_cast(0x80 | (code & 0x3f)); + } else if (code < 0x10000) { + utf8 += static_cast(0xe0 | (code >> 12)); + utf8 += static_cast(0x80 | ((code >> 6) & 0x3f)); + utf8 += static_cast(0x80 | (code & 0x3f)); + } else { + utf8 += static_cast(0xf0 | (code >> 18)); + utf8 += static_cast(0x80 | ((code >> 12) & 0x3f)); + utf8 += static_cast(0x80 | ((code >> 6) & 0x3f)); + utf8 += static_cast(0x80 | (code & 0x3f)); + } +} + inline int FormatWindows(char* buffer, size_t size, const char* format, const void* source, std::string* complete = nullptr) { if (!format || !source) throw std::invalid_argument("Null formatting argument"); + const char* const formatStart = format; FormatArguments args(source); FormatOutput output(buffer, size, complete); while (*format) { @@ -173,12 +195,26 @@ inline int FormatWindows(char* buffer, size_t size, const char* format, const vo if (!length.empty() && length != "l") throw std::invalid_argument("Invalid floating length"); output.Value(spec + conversion, args.Next()); } - } else if (conversion == 'c' && length.empty()) { + } else if (conversion == 'c' && (length.empty() || length == "h")) { output.Value(spec + conversion, args.Next()); - } else if (conversion == 's' && length.empty()) { + } else if ((conversion == 'c' && length == "l") || (conversion == 'C' && length.empty())) { + std::string utf8; + AppendUtf8(utf8, static_cast(args.Next())); + output.Value(spec + 's', utf8.c_str()); + } else if (conversion == 's' && (length.empty() || length == "h")) { const char* value = args.Next(); if (!value) throw std::invalid_argument("Null formatted string"); - output.Value(spec + conversion, value); + output.Value(spec + 's', value); + } else if ((conversion == 's' && length == "l") || (conversion == 'S' && length.empty())) { + const char16_t* value = args.Next(); + if (!value) throw std::invalid_argument("Null formatted wide string"); + std::string utf8; + while (*value) { + char32_t code = *value++; + if (code >= 0xd800 && code < 0xdc00 && *value >= 0xdc00 && *value < 0xe000) code = 0x10000 + ((code - 0xd800) << 10) + (*value++ - 0xdc00); + AppendUtf8(utf8, code); + } + output.Value(spec + 's', utf8.c_str()); } else if (conversion == 'p' && length.empty()) { output.Value(spec + conversion, args.Next()); } else if (conversion == 'n' && integerLength && spec == "%") { @@ -190,7 +226,7 @@ inline int FormatWindows(char* buffer, size_t size, const char* format, const vo else if (length.empty()) *static_cast(pointer) = count; else *static_cast(pointer) = count; } else { - throw std::invalid_argument("Unsupported format conversion"); + throw std::invalid_argument("Unsupported format conversion '" + spec + length + conversion + "' in \"" + std::string(formatStart).substr(0, 160) + "\""); } } return output.Count(); diff --git a/core/libs/prx/libc/src/General.cpp b/core/libs/prx/libc/src/General.cpp index 7abd12819a..cfe33fcd04 100644 --- a/core/libs/prx/libc/src/General.cpp +++ b/core/libs/prx/libc/src/General.cpp @@ -1,13 +1,27 @@ +#include #include #include +#include +#include #include #include "prx/libc/include/General.hpp" -extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path) { - if (path == nullptr) { - APS5_INVALID_ARG_EX; - } +namespace { + +// Guest prefixes (without leading slashes) mapped to host directories, e.g. save-data mount points: +// the PS5 hands the title a short mount point ("/_sm/0") whose files live under _sd/. +struct PathAliases { + std::mutex mutex; + std::vector> entries; +}; + +PathAliases& Aliases() { + static PathAliases aliases; + return aliases; +} + +std::string TrimSlashes(const char* path) { std::string s(path); std::size_t start = 0; while (start < s.size() && (s[start] == '/' || s[start] == '\\')) { @@ -17,7 +31,54 @@ extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path) { while (end > start && (s[end - 1] == '/' || s[end - 1] == '\\')) { --end; } - std::filesystem::path result = std::filesystem::current_path() / std::filesystem::path(s.substr(start, end - start)); + return s.substr(start, end - start); +} + +} + +extern "C" void AddPathAlias_nid_no_patch(const char* guestPrefix, const char* hostPath) { + if (guestPrefix == nullptr || hostPath == nullptr) { + APS5_INVALID_ARG_EX; + } + auto& aliases = Aliases(); + std::lock_guard lock(aliases.mutex); + const auto prefix = TrimSlashes(guestPrefix); + for (auto& entry : aliases.entries) { + if (entry.first == prefix) { + entry.second = hostPath; + return; + } + } + aliases.entries.emplace_back(prefix, hostPath); +} + +extern "C" void RemovePathAlias_nid_no_patch(const char* guestPrefix) { + if (guestPrefix == nullptr) { + APS5_INVALID_ARG_EX; + } + auto& aliases = Aliases(); + std::lock_guard lock(aliases.mutex); + const auto prefix = TrimSlashes(guestPrefix); + std::erase_if(aliases.entries, [&](const auto& entry) { return entry.first == prefix; }); +} + +extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path) { + if (path == nullptr) { + APS5_INVALID_ARG_EX; + } + const auto relative = TrimSlashes(path); + { + auto& aliases = Aliases(); + std::lock_guard lock(aliases.mutex); + for (const auto& [prefix, host] : aliases.entries) { + if (relative.size() < prefix.size() || relative.compare(0, prefix.size(), prefix) != 0) continue; + if (relative.size() == prefix.size()) return std::filesystem::path(host).make_preferred(); + if (relative[prefix.size()] != '/' && relative[prefix.size()] != '\\') continue; + std::filesystem::path result = std::filesystem::path(host) / std::filesystem::path(relative.substr(prefix.size() + 1)); + return result.make_preferred(); + } + } + std::filesystem::path result = std::filesystem::current_path() / std::filesystem::path(relative); return result.make_preferred(); } diff --git a/core/libs/prx/libc/src/GuestAllocations.cpp b/core/libs/prx/libc/src/GuestAllocations.cpp index bcac0d0cf5..571d25743a 100644 --- a/core/libs/prx/libc/src/GuestAllocations.cpp +++ b/core/libs/prx/libc/src/GuestAllocations.cpp @@ -1,5 +1,8 @@ #include "prx/libc/include/GuestAllocations.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" +#include +#include +#include +#include #include #include #include @@ -15,7 +18,7 @@ namespace GuestAllocations { namespace { struct Registry { - std::recursive_mutex& mutex = GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix(); + std::mutex mutex; std::map> ranges; bool mainImageRegistered = false; }; @@ -25,18 +28,56 @@ Registry& registry() { return value; } +std::atomic generation{1}; +std::atomic invalidator{nullptr}; +std::atomic pinWaiter{nullptr}; + void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(reason); } +// A mutation holds the registry lock and remembers the ranges it changed, which are reported to the +// invalidator once the generation has moved on, so a cache that observed the old state meanwhile +// notices either way. +struct MutationState { + std::unique_lock lock; + std::vector> changed; +}; + +void recordChange(void* mutation, const void* pointer, std::size_t bytes) { + if (mutation != nullptr && bytes != 0) static_cast(mutation)->changed.emplace_back(reinterpret_cast(pointer), bytes); +} + } void* GuestAllocationsBegin_nid_postfix() { - return new std::unique_lock(registry().mutex); + return new MutationState{std::unique_lock(registry().mutex), {}}; } void GuestAllocationsEnd_nid_postfix(void* mutation) noexcept { - delete static_cast*>(mutation); + auto* state = static_cast(mutation); + generation.fetch_add(1, std::memory_order_release); + if (const auto callback = invalidator.load(std::memory_order_acquire)) { + for (const auto& [address, bytes] : state->changed) callback(address, bytes); + } + delete state; +} + +std::uint64_t GuestAllocationsGeneration_nid_postfix() { + return generation.load(std::memory_order_acquire); +} + +void GuestAllocationsSetInvalidator_nid_postfix(void (*callback)(std::uintptr_t, std::size_t)) { + invalidator.store(callback, std::memory_order_release); +} + +void GuestAllocationsInvalidate_nid_postfix(std::uintptr_t address, std::size_t bytes) { + generation.fetch_add(1, std::memory_order_release); + if (const auto callback = invalidator.load(std::memory_order_acquire)) callback(address, bytes); +} + +void GuestAllocationsSetPinWaiter_nid_postfix(bool (*callback)()) { + pinWaiter.store(callback, std::memory_order_release); } #ifdef _WIN32 @@ -77,7 +118,8 @@ void GuestAllocationsRegisterMainImage_nid_postfix(void*) { } #endif -void GuestAllocationsAdd_nid_postfix(void*, void* pointer, std::size_t bytes, bool readable, bool writable) { +void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t bytes, bool readable, bool writable) { + recordChange(mutation, pointer, bytes); const auto address = reinterpret_cast(pointer); require(address != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest allocation range"); require(!writable || readable, "writable guest allocation must be readable"); @@ -91,14 +133,55 @@ void GuestAllocationsAdd_nid_postfix(void*, void* pointer, std::size_t bytes, bo ranges.emplace(address, std::make_shared(Range{address, bytes, readable, writable, address, bytes})); } -void GuestAllocationsRequireUnpinned_nid_postfix(void*, const void* pointer, std::size_t bytes) { +void GuestAllocationsRequireUnpinned_nid_postfix(void* mutation, const void* pointer, std::size_t bytes) { const auto address = reinterpret_cast(pointer); require(bytes <= std::numeric_limits::max() - address, "guest allocation range overflow"); - GuestMemoryTracking::GuestMemoryTrackingInvalidate_nid_postfix(address, bytes); const auto end = address + bytes; - for (const auto& [base, range] : registry().ranges) { - if (base >= end && base != address) break; - if ((address < base + range->bytes && base < end) || base == address) require(range.use_count() == 1, "guest allocation is owned by an active GPU command"); + auto& ranges = registry().ranges; + // GPU work leases allocations (a lease copies the shared_ptr) until the work completed, so wait + // for it rather than failing the guest. The wait releases the registry lock: the driver's waiter + // finishes that work under the device lock, under which the driver's workers acquire leases from + // this registry, so waiting with the lock held would deadlock them. The scan restarts after the + // lock is retaken, since another mutation may have run meanwhile. Without a waiter the lease is + // dropped by another thread (a synchronous draw or dispatch), so yielding suffices. A waiter + // round that finished nothing (the holder is not recorded GPU work) counts as a spin, so a lease + // nobody releases fails as fast as it did without a waiter; rounds that did finish work are + // bounded by the deadline only, as each one is a GPU wait. + auto* state = static_cast(mutation); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60); + int idleRounds = 0; + int syncedRounds = 0; + for (;;) { + bool pinned = false; + auto it = ranges.upper_bound(address); + if (it != ranges.begin()) --it; + for (; it != ranges.end(); ++it) { + const auto& [base, range] = *it; + if (base >= end && base != address) break; + if (((address < base + range->bytes && base < end) || base == address) && range.use_count() != 1) { + pinned = true; + break; + } + } + if (!pinned) return; + require(std::chrono::steady_clock::now() < deadline, "guest allocation is owned by an active GPU command"); + const auto waiter = pinWaiter.load(std::memory_order_acquire); + bool progressed = false; + if (waiter != nullptr && state != nullptr && state->lock.owns_lock()) { + state->lock.unlock(); + progressed = waiter(); + state->lock.lock(); + } else { + std::this_thread::yield(); + } + if (progressed) { + // The lease's work finished, yet the range is pinned again: between the waiter's return + // and the re-lock a driver worker took a new lease (every address-based build pins every + // registered range). Rare per round, but each round is a GPU wait, so a repeat is reported. + if (++syncedRounds == 8) std::fprintf(stderr, "[gpu] leased allocation 0x%llx+0x%llx still pinned after %d recorder syncs\n", static_cast(address), static_cast(bytes), syncedRounds); + continue; + } + require(++idleRounds < 1000000, "guest allocation is owned by an active GPU command"); } } @@ -107,12 +190,37 @@ void GuestAllocationsRequireAvailable_nid_postfix(void*, const void* pointer, st require(address != 0 && bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid fixed guest mapping"); for (const auto& [base, range] : registry().ranges) { if (base >= address + bytes) break; - require(base + range->bytes <= address, "fixed mapping overlaps a registered guest allocation"); + if (base + range->bytes > address) { + char message[160]; + std::snprintf(message, sizeof(message), "fixed mapping 0x%llx+0x%llx overlaps a registered guest allocation 0x%llx+0x%llx", static_cast(address), static_cast(bytes), static_cast(base), static_cast(range->bytes)); + throw std::runtime_error(message); + } } } +bool GuestAllocationsCovers_nid_postfix(void*, const void* pointer, std::size_t bytes) { + const auto address = reinterpret_cast(pointer); + require(address != 0 && bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest allocation range"); + const auto end = address + bytes; + auto cursor = address; + for (const auto& [base, range] : registry().ranges) { + const auto finish = base + range->bytes; + if (finish <= cursor) continue; + if (base > cursor || !range->releasable) return false; + cursor = finish; + if (cursor >= end) return true; + } + return false; +} + Range GuestAllocationsFind_nid_postfix(void*, const void* pointer) { const auto address = reinterpret_cast(pointer); + const auto exact = registry().ranges.find(address); + if (exact != registry().ranges.end() && exact->second->allocationAddress == address && exact->second->allocationBytes == exact->second->bytes) { + const auto& range = *exact->second; + require(range.releasable, "guest image memory is not a releasable allocation"); + return {address, range.allocationBytes, range.readable, range.writable, address, range.allocationBytes, range.releasable}; + } for (const auto& [base, range] : registry().ranges) { if (range->allocationAddress == address) { require(range->releasable, "guest image memory is not a releasable allocation"); @@ -125,7 +233,14 @@ Range GuestAllocationsFind_nid_postfix(void*, const void* pointer) { void GuestAllocationsRemove_nid_postfix(void* mutation, const void* pointer) { const auto range = GuestAllocationsFind_nid_postfix(mutation, pointer); GuestAllocationsRequireUnpinned_nid_postfix(mutation, pointer, range.bytes); - std::erase_if(registry().ranges, [&](const auto& entry) { return entry.second->allocationAddress == range.address; }); + recordChange(mutation, pointer, range.bytes); + auto& ranges = registry().ranges; + const auto exact = ranges.find(range.address); + if (exact != ranges.end() && exact->second->allocationBytes == exact->second->bytes) { + ranges.erase(exact); + return; + } + std::erase_if(ranges, [&](const auto& entry) { return entry.second->allocationAddress == range.address; }); } namespace { @@ -160,6 +275,7 @@ std::map> replaceRange(const void* p void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, const std::function& apply) { GuestAllocationsRequireUnpinned_nid_postfix(mutation, pointer, bytes); + recordChange(mutation, pointer, bytes); auto replacement = replaceRange(pointer, bytes, false, readable, writable); apply(); registry().ranges.swap(replacement); @@ -167,6 +283,7 @@ void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, st void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function& apply) { GuestAllocationsRequireUnpinned_nid_postfix(mutation, pointer, bytes); + recordChange(mutation, pointer, bytes); const auto address = reinterpret_cast(pointer); const auto found = registry().ranges.upper_bound(address); require(found != registry().ranges.begin(), "unmap address is not registered"); diff --git a/core/libs/prx/libc/src/GuestArena.cpp b/core/libs/prx/libc/src/GuestArena.cpp new file mode 100644 index 0000000000..4baedeeaf6 --- /dev/null +++ b/core/libs/prx/libc/src/GuestArena.cpp @@ -0,0 +1,154 @@ +#include "prx/libc/include/GuestArena.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +namespace GuestArena { +namespace { + +constexpr std::uintptr_t PreferredBase = 0x0000001000000000ull; +constexpr std::size_t MaximumSize = 0x0000007000000000ull; +constexpr std::size_t MinimumSize = 0x0000001000000000ull; +constexpr std::uintptr_t MapAreaEnd = 0x000000FC00000000ull; + +std::uintptr_t alignUp(std::uintptr_t value, std::size_t alignment) { + return (value + alignment - 1) & ~(static_cast(alignment) - 1); +} + +class Arena { +public: + static Arena& Get() { + static Arena arena; + return arena; + } + + bool Available() const { + return _base != 0; + } + + bool Contains(const void* pointer, std::size_t bytes) const { + const auto address = reinterpret_cast(pointer); + return _base != 0 && address >= _base && bytes <= _end - _base && address - _base <= (_end - _base) - bytes; + } + + void* Allocate(std::size_t bytes, std::size_t alignment) { + if (_base == 0) throw std::runtime_error("guest address space arena is unavailable"); + if (alignment == 0 || (alignment & (alignment - 1)) != 0) throw std::invalid_argument("invalid guest arena alignment"); + std::lock_guard lock(_lock); + std::uintptr_t candidate = alignUp(_base, alignment); + for (const auto& [start, end] : _used) { + if (candidate + bytes <= start) break; + candidate = std::max(candidate, alignUp(end, alignment)); + } + if (bytes > _end - candidate) throw std::runtime_error("guest address space arena exhausted"); + _used.emplace(candidate, candidate + bytes); + return reinterpret_cast(candidate); + } + + void MarkUsed(const void* pointer, std::size_t bytes) { + std::lock_guard lock(_lock); + const auto start = reinterpret_cast(pointer); + const auto next = _used.lower_bound(start); + if ((next != _used.end() && next->first < start + bytes) || (next != _used.begin() && std::prev(next)->second > start)) + throw std::runtime_error("fixed guest mapping overlaps a guest arena range"); + _used.emplace(start, start + bytes); + } + + void Release(const void* pointer, std::size_t bytes) { + std::lock_guard lock(_lock); + const auto start = reinterpret_cast(pointer); + const auto end = start + bytes; + auto it = _used.upper_bound(start); + if (it != _used.begin()) --it; + while (it != _used.end() && it->first < end) { + const auto rangeStart = it->first; + const auto rangeEnd = it->second; + if (rangeEnd <= start) { + ++it; + continue; + } + it = _used.erase(it); + if (rangeStart < start) _used.emplace(rangeStart, start); + if (rangeEnd > end) _used.emplace(end, rangeEnd); + } + } + +private: + Arena() { +#ifdef _WIN32 + // Windows places other reservations randomly, so take the lowest base and largest size that fit. + for (std::uintptr_t base = PreferredBase; base + MinimumSize <= MapAreaEnd; base += MinimumSize) { + for (std::size_t size = MaximumSize; size >= MinimumSize; size /= 2) { + if (base + size > MapAreaEnd) continue; + // Write watching lets the GPU driver learn which pages the CPU wrote instead of comparing + // whole resources; the plain reservation is the fallback. Debug aid: APS5_NO_WRITE_WATCH=1 + // skips it, to measure what the write faults it re-arms cost the game's threads. + static const bool noWriteWatch = std::getenv("APS5_NO_WRITE_WATCH") != nullptr; + void* reserved = noWriteWatch ? nullptr : VirtualAlloc(reinterpret_cast(base), size, MEM_RESERVE | MEM_WRITE_WATCH, PAGE_NOACCESS); + _writeWatched = reserved != nullptr; + if (!reserved) reserved = VirtualAlloc(reinterpret_cast(base), size, MEM_RESERVE, PAGE_NOACCESS); + if (!reserved) continue; + _base = reinterpret_cast(reserved); + _end = _base + size; + return; + } + } + std::fprintf(stderr, "[memory] guest arena unavailable below 0x%llx\n", static_cast(MapAreaEnd)); +#endif + } + + std::mutex _lock; + std::map _used; + std::uintptr_t _base = 0; + std::uintptr_t _end = 0; + bool _writeWatched = false; + +public: + std::uintptr_t Base() const { return _base; } + std::size_t Size() const { return _end - _base; } + bool WriteWatched() const { return _writeWatched; } +}; + +const bool g_reserved = (Arena::Get(), true); + +} + +bool GuestArenaAvailable_nid_postfix() { + return Arena::Get().Available(); +} + +bool GuestArenaContains_nid_postfix(const void* pointer, std::size_t bytes) { + return Arena::Get().Contains(pointer, bytes); +} + +void* GuestArenaAllocate_nid_postfix(std::size_t bytes, std::size_t alignment) { + return Arena::Get().Allocate(bytes, alignment); +} + +void GuestArenaMarkUsed_nid_postfix(const void* pointer, std::size_t bytes) { + Arena::Get().MarkUsed(pointer, bytes); +} + +void GuestArenaRelease_nid_postfix(const void* pointer, std::size_t bytes) { + Arena::Get().Release(pointer, bytes); +} + +void GuestArenaRange_nid_postfix(std::uintptr_t* base, std::size_t* bytes) { + *base = Arena::Get().Base(); + *bytes = Arena::Get().Size(); +} + +bool GuestArenaWriteWatched_nid_postfix() { + return Arena::Get().WriteWatched(); +} + +} diff --git a/core/libs/prx/libc/src/GuestHeap.cpp b/core/libs/prx/libc/src/GuestHeap.cpp index 51e8699a83..f47016d029 100644 --- a/core/libs/prx/libc/src/GuestHeap.cpp +++ b/core/libs/prx/libc/src/GuestHeap.cpp @@ -1,24 +1,214 @@ #include "prx/libc/include/GuestHeap.hpp" #include "prx/libc/include/GuestAllocations.hpp" -#include "prx/libc/include/GuestHeapStorage.hpp" +#include "prx/libc/include/GuestArena.hpp" #include +#include +#include #include #include #include +#include +#include #include +#include +#include #include +#include + +#ifdef _WIN32 +#include +#endif namespace GuestHeap { namespace { +// Each block is preceded by a header holding the raw block address and its size in bytes. +constexpr std::size_t HeaderBytes = 2 * sizeof(void*); +constexpr std::size_t MinimumAlignment = 16; +constexpr std::size_t PageBytes = 0x4000; +constexpr std::size_t SpanBytes = 1u << 20; +constexpr std::size_t MaximumSmallBytes = 64u * 1024u; +// Large blocks are rounded up to this so that freed blocks are reused by later requests of similar size. +constexpr std::size_t LargeGranuleBytes = 64u * 1024u; + +std::size_t alignUp(std::size_t value, std::size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +class ArenaHeap { +public: + static ArenaHeap& Get() { + static ArenaHeap heap; + return heap; + } + + void* Allocate(std::size_t total, std::size_t& blockBytes) { + if (total > MaximumSmallBytes) { + blockBytes = alignUp(total, LargeGranuleBytes); + if (void* cached = _large.Take(blockBytes)) return cached; + void* raw = GuestArena::GuestArenaAllocate_nid_postfix(blockBytes, PageBytes); + commit(raw, blockBytes); + return raw; + } + auto& sizeClass = _classes[classIndex(total)]; + blockBytes = sizeClass.bytes; + std::lock_guard lock(sizeClass.lock); + if (sizeClass.freeList) { + void* block = sizeClass.freeList; + sizeClass.freeList = *static_cast(block); + return block; + } + if (sizeClass.bump == nullptr || sizeClass.bytes > static_cast(sizeClass.bumpEnd - sizeClass.bump)) { + auto* span = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(SpanBytes, PageBytes)); + commit(span, SpanBytes); + sizeClass.bump = span; + sizeClass.bumpEnd = span + SpanBytes; + } + void* block = sizeClass.bump; + sizeClass.bump += sizeClass.bytes; + return block; + } + + void Free(void* raw, std::size_t blockBytes) { + if (blockBytes > MaximumSmallBytes) { + _large.Put(raw, blockBytes); + return; + } + auto& sizeClass = _classes[classIndex(blockBytes)]; + std::lock_guard lock(sizeClass.lock); + *static_cast(raw) = sizeClass.freeList; + sizeClass.freeList = raw; + } + +private: + struct SizeClass { + std::mutex lock; + std::size_t bytes = 0; + void* freeList = nullptr; + std::uint8_t* bump = nullptr; + std::uint8_t* bumpEnd = nullptr; + }; + + static constexpr std::size_t ClassCount = 64; + + ArenaHeap() { + std::size_t bytes = 32; + for (std::size_t index = 0; index < ClassCount; ++index) { + _classes[index].bytes = std::min(bytes, MaximumSmallBytes); + bytes = bytes < 256 ? bytes + 32 : alignUp(bytes + bytes / 4, 64); + } + _classes[ClassCount - 1].bytes = MaximumSmallBytes; + } + + std::size_t classIndex(std::size_t bytes) const { + const auto found = std::lower_bound(_classes.begin(), _classes.end(), bytes, [](const SizeClass& sizeClass, std::size_t value) { return sizeClass.bytes < value; }); + return static_cast(found - _classes.begin()); + } + + static void commit(void* pointer, std::size_t bytes) { +#ifdef _WIN32 + if (!VirtualAlloc(pointer, bytes, MEM_COMMIT, PAGE_READWRITE)) throw std::bad_alloc(); +#else + (void)pointer; + (void)bytes; +#endif + } + + // Freed large blocks stay committed and are handed out again for the same block size: the game + // frees and re-allocates hundreds of MB of temporary buffers every frame, and decommitting each + // one made every re-touch a demand-zero page fault plus a TLB shootdown across every thread + // (over 100K faults per second, a third of the main thread's time in the kernel). The cache is + // bounded (APS5_HEAP_CACHE_MIB, default 4096); past the budget the oldest blocks are decommitted + // and released, which is also the only time the GPU driver's page cache needs invalidating. + class LargeCache { + public: + void* Take(std::size_t bytes) { + std::lock_guard lock(_lock); + const auto found = _bySize.find(bytes); + if (found == _bySize.end() || found->second.empty()) return nullptr; + const auto entry = found->second.back(); + found->second.pop_back(); + void* raw = entry->raw; + _cachedBytes -= entry->bytes; + _order.erase(entry); + return raw; + } + + void Put(void* raw, std::size_t bytes) { + std::lock_guard lock(_lock); + _order.push_back({raw, bytes}); + _bySize[bytes].push_back(std::prev(_order.end())); + _cachedBytes += bytes; + while (_cachedBytes > budget() && !_order.empty()) { + const auto oldest = _order.front(); + auto& same = _bySize[oldest.bytes]; + same.erase(std::find(same.begin(), same.end(), _order.begin())); + _order.pop_front(); + _cachedBytes -= oldest.bytes; + release(oldest.raw, oldest.bytes); + } + } + + private: + struct Entry { + void* raw; + std::size_t bytes; + }; + + static std::size_t budget() { + static const std::size_t value = [] { + const char* text = std::getenv("APS5_HEAP_CACHE_MIB"); + return (text ? static_cast(std::strtoull(text, nullptr, 10)) : std::size_t{4096}) << 20; + }(); + return value; + } + + static void release(void* raw, std::size_t bytes) { +#ifdef _WIN32 + if (!VirtualFree(raw, bytes, MEM_DECOMMIT)) throw std::system_error(static_cast(GetLastError()), std::system_category(), "guest heap decommit failed"); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast(raw), bytes); +#endif + GuestArena::GuestArenaRelease_nid_postfix(raw, bytes); + } + + std::mutex _lock; + std::list _order; + std::unordered_map::iterator>> _bySize; + std::size_t _cachedBytes = 0; + }; + + std::array _classes; + LargeCache _large; +}; + +void* rawAllocate(std::size_t total, std::size_t& blockBytes) { + if (GuestArena::GuestArenaAvailable_nid_postfix()) return ArenaHeap::Get().Allocate(total, blockBytes); + blockBytes = total; + void* raw = std::malloc(total); + if (raw == nullptr) throw std::bad_alloc(); + return raw; +} + +void rawFree(void* raw, std::size_t blockBytes) { + if (GuestArena::GuestArenaAvailable_nid_postfix()) ArenaHeap::Get().Free(raw, blockBytes); + else std::free(raw); +} + void* allocate(GuestAllocations::Mutation& mutation, std::size_t alignment, std::size_t bytes) { if (alignment == 0 || (alignment & (alignment - 1)) != 0) throw std::invalid_argument("invalid guest heap alignment"); - alignment = std::max(alignment, alignof(void*)); - void* pointer = GuestHeapStorage::Allocate(alignment, bytes); + alignment = std::max(alignment, MinimumAlignment); + const std::size_t padding = alignment > MinimumAlignment ? alignment : 0; + if (bytes > std::numeric_limits::max() - HeaderBytes - padding) throw std::length_error("guest heap allocation overflow"); + std::size_t blockBytes = 0; + void* raw = rawAllocate(bytes + HeaderBytes + padding, blockBytes); + const auto address = alignUp(reinterpret_cast(raw) + HeaderBytes, alignment); + auto* pointer = reinterpret_cast(address); + reinterpret_cast(pointer)[-2] = raw; + reinterpret_cast(pointer)[-1] = blockBytes; try { mutation.Add(pointer, bytes, true, true); } catch (...) { - GuestHeapStorage::Free(pointer); + rawFree(raw, blockBytes); throw; } return pointer; @@ -26,14 +216,14 @@ void* allocate(GuestAllocations::Mutation& mutation, std::size_t alignment, std: void free(GuestAllocations::Mutation& mutation, void* pointer) { mutation.Remove(pointer); - GuestHeapStorage::Free(pointer); + rawFree(reinterpret_cast(pointer)[-2], reinterpret_cast(pointer)[-1]); } } void* GuestHeapAllocate_nid_postfix(std::size_t bytes) { GuestAllocations::Mutation mutation; - return allocate(mutation, alignof(std::max_align_t), bytes); + return allocate(mutation, MinimumAlignment, bytes); } void GuestHeapFree_nid_postfix(void* pointer) { @@ -53,7 +243,7 @@ void* GuestHeapReallocate_nid_postfix(void* pointer, std::size_t bytes) { free(mutation, pointer); return nullptr; } - void* result = allocate(mutation, alignof(std::max_align_t), bytes); + void* result = allocate(mutation, MinimumAlignment, bytes); std::memcpy(result, pointer, std::min(bytes, range.bytes)); free(mutation, pointer); return result; diff --git a/core/libs/prx/libc/src/GuestHeapStorage.cpp b/core/libs/prx/libc/src/GuestHeapStorage.cpp deleted file mode 100644 index e1dc44e7ff..0000000000 --- a/core/libs/prx/libc/src/GuestHeapStorage.cpp +++ /dev/null @@ -1,116 +0,0 @@ -#include "prx/libc/include/GuestHeapStorage.hpp" -#include "prx/libc/include/GuestMemoryBacking.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace GuestHeapStorage { -namespace { - -struct Chunk { - void* address; - std::size_t bytes; - std::map free; -}; - -struct Allocation { - Chunk* chunk; - std::size_t bytes; -}; - -struct Arena { - std::vector> chunks; - std::map allocations; -}; - -Arena& arena() { - static auto* value = new Arena; - return *value; -} - -} - -void* Allocate(std::size_t alignment, std::size_t bytes) { - if (alignment == 0 || (alignment & (alignment - 1)) != 0) throw std::invalid_argument("invalid shared heap alignment"); - bytes = std::max(bytes, std::size_t{1}); - if (bytes > std::numeric_limits::max() - alignment) throw std::length_error("shared heap allocation overflow"); - std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - auto& state = arena(); - for (std::size_t index = 0;; ++index) { - if (index == state.chunks.size()) { - constexpr std::size_t chunkSize = 64 * 1024 * 1024; - const auto granularity = std::max(alignment, std::size_t{65536}); - const auto requested = std::max(bytes, chunkSize); - if (requested > std::numeric_limits::max() - (granularity - 1)) throw std::length_error("shared heap chunk overflow"); - const auto size = (requested + granularity - 1) & ~(granularity - 1); - void* memory = GuestMemoryBacking::GuestMemoryBackingMap_nid_postfix(nullptr, size, granularity, 3); - try { - auto chunk = std::make_unique(); - chunk->address = memory; - chunk->bytes = size; - chunk->free.emplace(reinterpret_cast(memory), size); - state.chunks.push_back(std::move(chunk)); - } catch (...) { - GuestMemoryBacking::GuestMemoryBackingUnmap_nid_postfix(memory, size); - throw; - } - } - auto& chunk = *state.chunks[index]; - for (auto it = chunk.free.begin(); it != chunk.free.end(); ++it) { - const auto first = it->first; - const auto available = it->second; - if (first > std::numeric_limits::max() - (alignment - 1)) throw std::overflow_error("shared heap address overflow"); - const auto address = (first + alignment - 1) & ~(static_cast(alignment) - 1); - const auto prefix = address - first; - if (prefix > available || bytes > available - prefix) continue; - if (!state.allocations.emplace(address, Allocation{&chunk, bytes}).second) throw std::runtime_error("duplicate shared heap allocation"); - const auto suffix = available - prefix - bytes; - try { - if (suffix != 0) chunk.free.emplace(address + bytes, suffix); - } catch (...) { - state.allocations.erase(address); - throw; - } - if (prefix != 0) it->second = prefix; - else chunk.free.erase(it); - return reinterpret_cast(address); - } - } -} - -void Free(void* pointer) { - std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - auto& allocations = arena().allocations; - const auto address = reinterpret_cast(pointer); - const auto found = allocations.find(address); - if (found == allocations.end()) throw std::invalid_argument("unknown shared heap allocation"); - auto* chunk = found->second.chunk; - auto& free = chunk->free; - auto [entry, inserted] = free.emplace(address, found->second.bytes); - if (!inserted) throw std::runtime_error("shared heap allocation was already freed"); - allocations.erase(found); - if (entry != free.begin()) { - auto previous = std::prev(entry); - if (previous->first + previous->second == entry->first) { - previous->second += entry->second; - free.erase(entry); - entry = previous; - } - } - const auto next = std::next(entry); - if (next != free.end() && entry->first + entry->second == next->first) { - entry->second += next->second; - free.erase(next); - } - if (free.size() == 1 && entry->first == reinterpret_cast(chunk->address) && entry->second == chunk->bytes) { - GuestMemoryBacking::GuestMemoryBackingUnmap_nid_postfix(chunk->address, chunk->bytes); - std::erase_if(arena().chunks, [chunk](const auto& value) { return value.get() == chunk; }); - } -} - -} diff --git a/core/libs/prx/libc/src/GuestMemoryBacking.cpp b/core/libs/prx/libc/src/GuestMemoryBacking.cpp deleted file mode 100644 index f3facf723a..0000000000 --- a/core/libs/prx/libc/src/GuestMemoryBacking.cpp +++ /dev/null @@ -1,96 +0,0 @@ -#include "prx/libc/include/GuestMemoryBacking.hpp" -#include "prx/libc/include/MemoryBackingPlatform.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" -#include -#include -#include -#include -#include -#include - -namespace GuestMemoryBacking { -namespace { - -struct Allocation { - Platform::Mapping mapping; - std::map ranges; -}; - -std::map& allocations() { - static auto* value = new std::map; - return *value; -} - -Allocation& find(std::uint64_t address, std::size_t bytes) { - if (address == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) throw std::invalid_argument("invalid guest backing range"); - auto found = allocations().upper_bound(address); - if (found == allocations().begin()) throw std::runtime_error("guest memory has no shared backing"); - auto& allocation = std::prev(found)->second; - auto range = allocation.ranges.upper_bound(address); - if (range == allocation.ranges.begin() || address + bytes > std::prev(range)->second) throw std::runtime_error("guest memory backing range is unmapped"); - return allocation; -} - -} - -void* GuestMemoryBackingMap_nid_postfix(void* address, std::size_t bytes, std::size_t alignment, int protection) { - const auto pageSize = GuestMemoryTracking::GuestMemoryTrackingPageSize_nid_postfix(); - if (bytes == 0 || bytes % pageSize != 0 || alignment < pageSize || (alignment & (alignment - 1)) != 0 || (protection & ~7) != 0) throw std::invalid_argument("invalid shared guest memory mapping"); - if (address != nullptr && reinterpret_cast(address) % alignment != 0) throw std::invalid_argument("misaligned fixed guest memory mapping"); - std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - const auto mapping = Platform::Map(address, bytes, alignment, protection); - try { - if (mapping.bytes > std::numeric_limits::max() - mapping.address) throw std::overflow_error("guest backing mapping overflow"); - const auto next = allocations().lower_bound(mapping.address); - if (next != allocations().end() && next->first < mapping.address + bytes) throw std::runtime_error("overlapping guest backing mappings"); - if (next != allocations().begin()) { - const auto& previous = std::prev(next)->second.mapping; - if (previous.address + previous.bytes > mapping.address) throw std::runtime_error("guest mapping overlaps a retained backing reservation"); - } - Allocation allocation{mapping, {{mapping.address, mapping.address + bytes}}}; - if (!allocations().emplace(mapping.address, std::move(allocation)).second) throw std::runtime_error("duplicate guest backing mapping"); - } catch (...) { - Platform::Unmap(mapping); - throw; - } - return reinterpret_cast(mapping.address); -} - -void GuestMemoryBackingUnmap_nid_postfix(void* pointer, std::size_t bytes) { - const auto address = reinterpret_cast(pointer); - const auto pageSize = GuestMemoryTracking::GuestMemoryTrackingPageSize_nid_postfix(); - if (address % pageSize != 0 || bytes % pageSize != 0) throw std::invalid_argument("misaligned guest backing unmap"); - std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - auto& allocation = find(address, bytes); - auto replacement = allocation.ranges; - const auto found = std::prev(replacement.upper_bound(address)); - const auto first = found->first; - const auto last = found->second; - replacement.erase(found); - if (first < address) replacement.emplace(first, address); - if (address + bytes < last) replacement.emplace(address + bytes, last); - GuestMemoryTracking::GuestMemoryTrackingInvalidate_nid_postfix(address, bytes); - if (replacement.empty()) { - const auto base = allocation.mapping.address; - Platform::Unmap(allocation.mapping); - allocations().erase(base); - } else { - Platform::Deactivate(address, bytes); - allocation.ranges.swap(replacement); - } -} - -void GuestMemoryBackingRequire_nid_postfix(std::uint64_t address, std::size_t bytes) { - std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - static_cast(find(address, bytes)); -} - -void GuestMemoryBackingWrite_nid_postfix(std::uint64_t address, const void* source, std::size_t bytes) { - if (source == nullptr) throw std::invalid_argument("missing guest backing write source"); - std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - auto& allocation = find(address, bytes); - auto* destination = static_cast(allocation.mapping.alias) + (address - allocation.mapping.address); - std::memcpy(destination, source, bytes); -} - -} diff --git a/core/libs/prx/libc/src/GuestMemoryTracking.cpp b/core/libs/prx/libc/src/GuestMemoryTracking.cpp deleted file mode 100644 index d11f274a29..0000000000 --- a/core/libs/prx/libc/src/GuestMemoryTracking.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include "prx/libc/include/GuestMemoryTracking.hpp" -#include "prx/libc/include/MemoryTrackingPlatform.hpp" -#include "prx/libc/include/GuestMemoryBacking.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace GuestMemoryTracking { -namespace { - -struct Entry { - std::uint64_t address; - std::size_t bytes; - void* context; - Resolver resolver; - Protection protection = Protection::ReadWrite; - std::vector original; - bool resolving = false; - bool active = false; -}; - -struct Registry { - std::recursive_mutex mutex; - std::map> entries; - bool installed = false; -}; - -Registry& registry() { - static auto* value = new Registry; - return *value; -} - -std::uint64_t checkedEnd(std::uint64_t address, std::size_t bytes) { - if (address == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) throw std::invalid_argument("invalid tracked guest memory range"); - return address + bytes; -} - -void resolve(const std::shared_ptr& entry, Access access) { - if (entry->resolving) throw std::runtime_error("recursive guest memory ownership resolution"); - entry->resolving = true; - try { - entry->resolver(entry->context, access); - if (entry->protection == Protection::None || (access != Access::Read && entry->protection != Protection::ReadWrite)) throw std::runtime_error("guest memory resolver did not release CPU access"); - entry->resolving = false; - } catch (...) { - entry->resolving = false; - throw; - } -} - -std::vector> overlapping(std::uint64_t address, std::size_t bytes) { - const auto end = checkedEnd(address, bytes); - auto& entries = registry().entries; - auto first = entries.upper_bound(address); - if (first != entries.begin()) --first; - std::vector> result; - for (auto it = first; it != entries.end() && it->first < end; ++it) { - if (it->first + it->second->bytes > address) result.push_back(it->second); - } - return result; -} - -bool fault(std::uint64_t address, bool writable) { - std::lock_guard lock(registry().mutex); - const auto entries = overlapping(address, 1); - if (entries.empty()) return false; - bool handled = false; - for (const auto& entry : entries) { - if (!entry->active) continue; - if (entry->protection == Protection::None || (writable && entry->protection == Protection::Read)) resolve(entry, writable ? Access::Write : Access::Read); - handled = true; - } - return handled; -} - -} - -std::recursive_mutex& GuestMemoryTrackingMutex_nid_postfix() { - return registry().mutex; -} - -std::size_t GuestMemoryTrackingPageSize_nid_postfix() { - return Platform::PageSize(); -} - -void* GuestMemoryTrackingCreate_nid_postfix(std::uint64_t address, std::size_t bytes, void* context, Resolver resolver) { - const auto end = checkedEnd(address, bytes); - const auto pageSize = Platform::PageSize(); - if (context == nullptr || resolver == nullptr) throw std::invalid_argument("missing guest memory ownership resolver"); - if (end > std::numeric_limits::max() - (pageSize - 1)) throw std::overflow_error("tracked guest memory page range overflow"); - const auto first = address - address % pageSize; - const auto last = (end + pageSize - 1) / pageSize * pageSize; - std::lock_guard lock(registry().mutex); - GuestMemoryBacking::GuestMemoryBackingRequire_nid_postfix(first, static_cast(last - first)); - if (!overlapping(first, static_cast(last - first)).empty()) throw std::runtime_error("overlapping guest memory ownership pages"); - static_cast(Platform::Query(first, static_cast(last - first))); - if (!registry().installed) { - Platform::Install(fault); - registry().installed = true; - } - auto entry = std::make_shared(); - entry->address = first; - entry->bytes = static_cast(last - first); - entry->context = context; - entry->resolver = resolver; - auto handle = std::make_unique>(entry); - registry().entries.emplace(first, std::move(entry)); - return handle.release(); -} - -void GuestMemoryTrackingDestroy_nid_postfix(void* handle) noexcept { - if (handle == nullptr) std::terminate(); - std::lock_guard lock(registry().mutex); - std::unique_ptr> owner(static_cast*>(handle)); - const auto& entry = **owner; - Platform::Restore(entry.original); - registry().entries.erase(entry.address); -} - -void GuestMemoryTrackingProtect_nid_postfix(void* handle, Protection protection) { - if (handle == nullptr) throw std::invalid_argument("missing guest memory watch"); - std::lock_guard lock(registry().mutex); - auto& entry = **static_cast*>(handle); - if (entry.protection == protection) return; - if (protection == Protection::ReadWrite) { - Platform::Restore(entry.original); - entry.original.clear(); - } else { - if (entry.original.empty()) entry.original = Platform::Query(entry.address, entry.bytes); - Platform::Protect(entry.address, entry.bytes, protection); - entry.active = true; - } - entry.protection = protection; -} - -void GuestMemoryTrackingResolve_nid_postfix(std::uint64_t address, std::size_t bytes, bool writable) { - if (bytes == 0) return; - std::lock_guard lock(registry().mutex); - for (const auto& entry : overlapping(address, bytes)) { - if (entry->protection == Protection::None || (writable && entry->protection == Protection::Read)) resolve(entry, writable ? Access::Write : Access::Read); - } -} - -void GuestMemoryTrackingInvalidate_nid_postfix(std::uint64_t address, std::size_t bytes) { - if (bytes == 0) return; - std::lock_guard lock(registry().mutex); - for (const auto& entry : overlapping(address, bytes)) { - resolve(entry, Access::Invalidate); - entry->active = false; - } -} - -void GuestMemoryTrackingValidate_nid_postfix(std::uint64_t address, std::size_t bytes, const std::function& validate) { - if (bytes == 0) return; - if (!validate) throw std::invalid_argument("missing native memory range validator"); - const auto end = checkedEnd(address, bytes); - std::lock_guard lock(registry().mutex); - for (const auto& entry : overlapping(address, bytes)) { - if (address < entry->address) validate(address, static_cast(entry->address - address)); - const auto first = std::max(address, entry->address); - const auto last = std::min(end, entry->address + entry->bytes); - if (entry->protection == Protection::ReadWrite) validate(first, static_cast(last - first)); - else if (entry->original.empty()) throw std::runtime_error("protected guest memory has no native permission record"); - address = last; - } - if (address < end) validate(address, static_cast(end - address)); -} - -} diff --git a/core/libs/prx/libc/src/JumpBuffer.cpp b/core/libs/prx/libc/src/JumpBuffer.cpp index d225005bd8..5a1d2fa6c7 100644 --- a/core/libs/prx/libc/src/JumpBuffer.cpp +++ b/core/libs/prx/libc/src/JumpBuffer.cpp @@ -1,15 +1,43 @@ -#include - #include "prx/libc/include/General.hpp" -extern "C" { +// setjmp/longjmp must capture the guest's own frame, so they are written directly in assembly with +// the guest (System V) calling convention. The saved state fits the guest's 96-byte jmp_buf: +// return address, rbx, rsp, rbp, r12-r15, MXCSR and the x87 control word. +#ifdef _WIN32 +#define APS5_ASM_FUNCTION(name) ".globl " name "\n.def " name "; .scl 2; .type 32; .endef\n" name ":\n" +#else +#define APS5_ASM_FUNCTION(name) ".globl " name "\n.type " name ", @function\n" name ":\n" +#endif -int APS5_VABI setjmp_nid_postfix(std::jmp_buf env) { - return setjmp(env); -} - -[[noreturn]] void longjmp_nid_postfix(std::jmp_buf env, int val) { - std::longjmp(env, val); -} - -} +asm(".text\n" + APS5_ASM_FUNCTION("setjmp_nid_postfix") + " mov (%rsp), %rax\n" + " mov %rax, 0(%rdi)\n" + " mov %rbx, 8(%rdi)\n" + " lea 8(%rsp), %rax\n" + " mov %rax, 16(%rdi)\n" + " mov %rbp, 24(%rdi)\n" + " mov %r12, 32(%rdi)\n" + " mov %r13, 40(%rdi)\n" + " mov %r14, 48(%rdi)\n" + " mov %r15, 56(%rdi)\n" + " stmxcsr 64(%rdi)\n" + " fnstcw 68(%rdi)\n" + " xor %eax, %eax\n" + " ret\n" + APS5_ASM_FUNCTION("longjmp_nid_postfix") + " mov %esi, %eax\n" + " test %eax, %eax\n" + " jnz 1f\n" + " inc %eax\n" + "1:\n" + " mov 8(%rdi), %rbx\n" + " mov 16(%rdi), %rsp\n" + " mov 24(%rdi), %rbp\n" + " mov 32(%rdi), %r12\n" + " mov 40(%rdi), %r13\n" + " mov 48(%rdi), %r14\n" + " mov 56(%rdi), %r15\n" + " ldmxcsr 64(%rdi)\n" + " fldcw 68(%rdi)\n" + " jmp *0(%rdi)\n"); diff --git a/core/libs/prx/libc/src/LocaleSupport.cpp b/core/libs/prx/libc/src/LocaleSupport.cpp index b16a1bb7c7..66a128c394 100644 --- a/core/libs/prx/libc/src/LocaleSupport.cpp +++ b/core/libs/prx/libc/src/LocaleSupport.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -37,13 +38,32 @@ GuestLocale::Facet* APS5_VABI ReleaseLocale(GuestLocale::Facet* self) { if (self == nullptr) throw std::invalid_argument("locale release: null facet"); std::atomic_ref references(self->references); auto count = references.load(); + // The classic locale is immortal: a release that would drop it to zero keeps its last reference. do { - if (count <= 1) throw std::runtime_error("classic locale: unbalanced release"); + if (count <= 1) return nullptr; } while (!references.compare_exchange_weak(count, count - 1)); return nullptr; } +// Copies of the classic locale (see _Locimp::_Locimp(const _Locimp&)) count down to zero but are never +// freed either, since their facets are shared with the classic locale. +void APS5_VABI DestroyCopiedLocale(GuestLocale::Facet*) {} + +void APS5_VABI RetainCopiedLocale(GuestLocale::Facet* self) { + if (self == nullptr) throw std::invalid_argument("locale retain: null facet"); + std::atomic_ref(self->references).fetch_add(1); +} + +GuestLocale::Facet* APS5_VABI ReleaseCopiedLocale(GuestLocale::Facet* self) { + if (self == nullptr) throw std::invalid_argument("locale release: null facet"); + std::atomic_ref references(self->references); + auto count = references.load(); + while (count != 0 && !references.compare_exchange_weak(count, count - 1)) {} + return nullptr; +} + const GuestLocale::FacetVtable g_localeVtable{DestroyClassicLocale, DestroyClassicLocale, RetainLocale, ReleaseLocale}; +const GuestLocale::FacetVtable g_copiedLocaleVtable{DestroyCopiedLocale, DestroyCopiedLocale, RetainCopiedLocale, ReleaseCopiedLocale}; GuestLocale::Facet* g_classicFacets[1]{}; GuestLocale::Implementation g_classicLocale{{&g_localeVtable, 1, 0}, g_classicFacets, 1, 0, false, "C"}; @@ -103,6 +123,7 @@ void APS5_VABI _ZNSt8ios_baseD2Ev_nid_postfix(GuestLocale::IosBase* self) { self->locale = nullptr; } +// Dinkumware's static locale::_Init(bool) returns the global _Locimp, which the caller copies. GuestLocale::Implementation* APS5_VABI _ZNSt6locale5_InitEv_nid_postfix() { std::lock_guard lock(g_localeInitMutex); if (!g_localeInitialized) { @@ -182,6 +203,52 @@ wint_t APS5_VABI _Towctrans_nid_postfix(wint_t c, wctrans_t desc) { return std::towctrans(c, desc); } +// Locale ids and stream objects the title imports besides the ones above; the streams are zeroed +// storage (the guest constructs Dinkumware streams itself) so address-taking code links and only a +// use of them would misbehave. +std::uint64_t _ZNSt7codecvtIwc9_MbstatetE2idE_nid_postfix = 0; +alignas(16) unsigned char _ZSt4cout_nid_postfix[0x400] {}; +alignas(16) unsigned char _ZSt4cerr_nid_postfix[0x400] {}; +alignas(16) unsigned char _ZSt3cin_nid_postfix[0x400] {}; +alignas(16) unsigned char _ZSt5wcout_nid_postfix[0x400] {}; +alignas(16) unsigned char _ZSt5wcerr_nid_postfix[0x400] {}; +alignas(16) unsigned char _ZSt4wcin_nid_postfix[0x400] {}; + +// Facet vectors are read by inlined guest code, so they come from the guest application heap. +GuestLocale::Facet** AllocateFacetVector(std::size_t count) { + auto* vector = static_cast(ApplicationHeapAllocate_nid_no_patch(count * sizeof(GuestLocale::Facet*))); + if (vector == nullptr) throw std::runtime_error("locale: facet vector allocation failed"); + std::memset(vector, 0, count * sizeof(GuestLocale::Facet*)); + return vector; +} + +// _Locimp::_Locimp(const _Locimp&): the copy shares the source's facets; guest locales are never freed. +void APS5_VABI _ZNSt6locale7_LocimpC1ERKS0__nid_postfix(GuestLocale::Implementation* self, const GuestLocale::Implementation* source) { + if (self == nullptr || source == nullptr) throw std::invalid_argument("_Locimp copy: null object"); + std::lock_guard lock(g_localeInitMutex); + *self = *source; + self->base.vtable = &g_copiedLocaleVtable; + self->base.references = 1; + if (source->facetCount != 0) { + self->facets = AllocateFacetVector(source->facetCount); + std::memcpy(self->facets, source->facets, source->facetCount * sizeof(GuestLocale::Facet*)); + } +} + +// _Locimp::_Addfac(facet*, size_t id): installs a facet at its id, growing the vector as needed. +void APS5_VABI _ZNSt6locale7_Locimp7_AddfacEPNS_5facetEm_nid_postfix(GuestLocale::Implementation* self, GuestLocale::Facet* facet, std::size_t id) { + if (self == nullptr) throw std::invalid_argument("_Locimp::_Addfac: null object"); + std::lock_guard lock(g_localeInitMutex); + if (id >= self->facetCount) { + const std::size_t count = std::max(id + 1, 40); + auto* grown = AllocateFacetVector(count); + if (self->facetCount != 0) std::memcpy(grown, self->facets, self->facetCount * sizeof(GuestLocale::Facet*)); + self->facets = grown; + self->facetCount = count; + } + self->facets[id] = facet; +} + void APS5_VABI _init_env_nid_postfix() { ApplicationHeapInitialize_nid_no_patch(ApplicationProcessParameters_nid_no_patch()); } diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index 87253bcf0e..48968b7262 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include "prx/libc/include/General.hpp" @@ -47,4 +49,40 @@ int APS5_VABI __isfinite_nid_postfix(double x) { return std::isfinite(x) ? 1 : 0 int APS5_VABI __isnan_nid_postfix(double x) { return std::isnan(x) ? 1 : 0; } int APS5_VABI __signbit_nid_postfix(double x) { return std::signbit(x) ? 1 : 0; } +double APS5_VABI modf_nid_postfix(double x, double* integral) { return std::modf(x, integral); } +float APS5_VABI modff_nid_postfix(float x, float* integral) { return std::modf(x, integral); } +double APS5_VABI frexp_nid_postfix(double x, int* exponent) { return std::frexp(x, exponent); } +float APS5_VABI frexpf_nid_postfix(float x, int* exponent) { return std::frexp(x, exponent); } +double APS5_VABI ldexp_nid_postfix(double x, int exponent) { return std::ldexp(x, exponent); } +float APS5_VABI fmodf_nid_postfix(float x, float y) { return std::fmod(x, y); } +float APS5_VABI atan2f_nid_postfix(float y, float x) { return std::atan2(y, x); } +float APS5_VABI asinf_nid_postfix(float x) { return std::asin(x); } +float APS5_VABI acosf_nid_postfix(float x) { return std::acos(x); } +float APS5_VABI tanf_nid_postfix(float x) { return std::tan(x); } +double APS5_VABI tanh_nid_postfix(double x) { return std::tanh(x); } +float APS5_VABI tanhf_nid_postfix(float x) { return std::tanh(x); } +float APS5_VABI log10f_nid_postfix(float x) { return std::log10(x); } +float APS5_VABI _FSinh_nid_postfix(float x, float y) { return y * std::sinh(x); } +float APS5_VABI _FCosh_nid_postfix(float x, float y) { return y * std::cosh(x); } +int APS5_VABI __isnanf_nid_postfix(float x) { return std::isnan(x) ? 1 : 0; } +int APS5_VABI __isinff_nid_postfix(float x) { return std::isinf(x) ? 1 : 0; } +int APS5_VABI __signbitf_nid_postfix(float x) { return std::signbit(x) ? 1 : 0; } + +static std::mutex g_randLock; +static std::uint32_t g_randState = 1; + +int APS5_VABI rand_nid_postfix() { + std::lock_guard lock(g_randLock); + const std::int64_t x = static_cast(g_randState % 0x7ffffffeu) + 1; + std::int64_t next = 16807 * (x % 127773) - 2836 * (x / 127773); + if (next < 0) next += 0x7fffffff; + g_randState = static_cast(next - 1); + return static_cast(next - 1); +} + +void APS5_VABI srand_nid_postfix(unsigned int seed) { + std::lock_guard lock(g_randLock); + g_randState = seed; +} + } diff --git a/core/libs/prx/libc/src/MemoryBackingLinux.cpp b/core/libs/prx/libc/src/MemoryBackingLinux.cpp deleted file mode 100644 index 5c788aa741..0000000000 --- a/core/libs/prx/libc/src/MemoryBackingLinux.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "prx/libc/include/MemoryBackingPlatform.hpp" -#include -#include -#include -#include -#include -#include - -namespace GuestMemoryBacking::Platform { -namespace { - -void check(bool success, const char* operation) { - if (!success) throw std::system_error(errno, std::generic_category(), operation); -} - -} - -Mapping Map(void* address, std::size_t bytes, std::size_t alignment, int protection) { - if (bytes > static_cast(std::numeric_limits::max()) || bytes > std::numeric_limits::max() - alignment) throw std::overflow_error("guest backing size overflow"); - int descriptor = memfd_create("AnyPS5 guest memory", MFD_CLOEXEC); - check(descriptor >= 0, "memfd_create guest backing"); - void* alias = MAP_FAILED; - void* guest = MAP_FAILED; - void* reservation = MAP_FAILED; - std::size_t reservationBytes = 0; - try { - check(ftruncate(descriptor, static_cast(bytes)) == 0, "ftruncate guest backing"); - alias = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, descriptor, 0); - check(alias != MAP_FAILED, "mmap guest backing alias"); - if (address == nullptr) { - reservationBytes = bytes + alignment; - reservation = mmap(nullptr, reservationBytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - } else { - reservationBytes = bytes; - reservation = mmap(address, reservationBytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); - } - check(reservation != MAP_FAILED, "mmap guest backing reservation"); - if (address != nullptr && reservation != address) throw std::runtime_error("fixed guest backing reservation address mismatch"); - const auto first = reinterpret_cast(reservation); - const auto aligned = (first + alignment - 1) & ~(static_cast(alignment) - 1); - guest = mmap(reinterpret_cast(aligned), bytes, protection, MAP_SHARED | MAP_FIXED, descriptor, 0); - check(guest != MAP_FAILED, "mmap guest backing view"); - const auto prefix = aligned - first; - const auto suffix = reservationBytes - prefix - bytes; - if (prefix != 0) check(munmap(reservation, prefix) == 0, "munmap guest backing prefix"); - if (suffix != 0) check(munmap(reinterpret_cast(aligned + bytes), suffix) == 0, "munmap guest backing suffix"); - reservation = MAP_FAILED; - const auto closeResult = close(descriptor); - descriptor = -1; - check(closeResult == 0, "close guest backing descriptor"); - return {aligned, bytes, alias, 0}; - } catch (...) { - if (reservation != MAP_FAILED) check(munmap(reservation, reservationBytes) == 0, "munmap failed guest reservation"); - else if (guest != MAP_FAILED) check(munmap(guest, bytes) == 0, "munmap failed guest view"); - if (alias != MAP_FAILED) check(munmap(alias, bytes) == 0, "munmap failed guest alias"); - if (descriptor >= 0) check(close(descriptor) == 0, "close failed guest backing"); - throw; - } -} - -void Unmap(const Mapping& mapping) { - check(munmap(reinterpret_cast(mapping.address), mapping.bytes) == 0, "munmap guest view"); - check(munmap(mapping.alias, mapping.bytes) == 0, "munmap guest alias"); -} - -void Deactivate(std::uint64_t address, std::size_t bytes) { - check(mprotect(reinterpret_cast(address), bytes, PROT_NONE) == 0, "mprotect guest backing unmap"); -} - -} diff --git a/core/libs/prx/libc/src/MemoryBackingWindows.cpp b/core/libs/prx/libc/src/MemoryBackingWindows.cpp deleted file mode 100644 index 8b0c964569..0000000000 --- a/core/libs/prx/libc/src/MemoryBackingWindows.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "prx/libc/include/MemoryBackingPlatform.hpp" -#include -#include -#include -#include -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include - -namespace GuestMemoryBacking::Platform { -namespace { - -DWORD nativeProtection(int protection) { - if ((protection & 4) != 0) return (protection & 2) != 0 ? PAGE_EXECUTE_READWRITE : (protection & 1) != 0 ? PAGE_EXECUTE_READ : PAGE_EXECUTE; - if ((protection & 2) != 0) return PAGE_READWRITE; - return (protection & 1) != 0 ? PAGE_READONLY : PAGE_NOACCESS; -} - -void check(bool success, const char* operation) { - if (!success) throw std::system_error(static_cast(GetLastError()), std::system_category(), operation); -} - -} - -Mapping Map(void* address, std::size_t bytes, std::size_t alignment, int protection) { - SYSTEM_INFO info{}; - GetSystemInfo(&info); - alignment = std::max(alignment, static_cast(info.dwAllocationGranularity)); - if (address != nullptr && reinterpret_cast(address) % info.dwAllocationGranularity != 0) throw std::invalid_argument("fixed guest view is not aligned to native allocation granularity"); - const auto size = static_cast(bytes); - HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, static_cast(size >> 32u), static_cast(size), nullptr); - check(section != nullptr, "CreateFileMapping guest backing"); - void* alias = nullptr; - void* guest = nullptr; - try { - alias = MapViewOfFile(section, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, bytes); - check(alias != nullptr, "MapViewOfFile guest backing alias"); - if (address == nullptr && alignment > info.dwAllocationGranularity) { - if (bytes > std::numeric_limits::max() - alignment) throw std::overflow_error("aligned guest backing reservation overflow"); - void* reservation = VirtualAlloc(nullptr, bytes + alignment, MEM_RESERVE, PAGE_NOACCESS); - check(reservation != nullptr, "VirtualAlloc guest backing reservation"); - const auto first = reinterpret_cast(reservation); - address = reinterpret_cast((first + alignment - 1) & ~(static_cast(alignment) - 1)); - check(VirtualFree(reservation, 0, MEM_RELEASE) != FALSE, "VirtualFree guest backing reservation"); - } - guest = MapViewOfFileEx(section, FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE, 0, 0, bytes, address); - check(guest != nullptr, "MapViewOfFileEx guest memory"); - if (reinterpret_cast(guest) % alignment != 0 || (address != nullptr && guest != address)) throw std::runtime_error("guest backing view address mismatch"); - DWORD previous = 0; - check(VirtualProtect(guest, bytes, nativeProtection(protection), &previous) != FALSE, "VirtualProtect guest backing view"); - return {reinterpret_cast(guest), bytes, alias, reinterpret_cast(section)}; - } catch (...) { - if (guest != nullptr) check(UnmapViewOfFile(guest) != FALSE, "UnmapViewOfFile failed guest view"); - if (alias != nullptr) check(UnmapViewOfFile(alias) != FALSE, "UnmapViewOfFile failed guest alias"); - check(CloseHandle(section) != FALSE, "CloseHandle failed guest backing"); - throw; - } -} - -void Unmap(const Mapping& mapping) { - check(UnmapViewOfFile(reinterpret_cast(mapping.address)) != FALSE, "UnmapViewOfFile guest view"); - check(UnmapViewOfFile(mapping.alias) != FALSE, "UnmapViewOfFile guest alias"); - check(CloseHandle(reinterpret_cast(mapping.handle)) != FALSE, "CloseHandle guest backing"); -} - -void Deactivate(std::uint64_t address, std::size_t bytes) { - DWORD previous = 0; - check(VirtualProtect(reinterpret_cast(address), bytes, PAGE_NOACCESS, &previous) != FALSE, "VirtualProtect guest backing unmap"); -} - -} diff --git a/core/libs/prx/libc/src/MemoryTrackingLinux.cpp b/core/libs/prx/libc/src/MemoryTrackingLinux.cpp deleted file mode 100644 index 54b9ca0dd4..0000000000 --- a/core/libs/prx/libc/src/MemoryTrackingLinux.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include "prx/libc/include/MemoryTrackingPlatform.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace GuestMemoryTracking::Platform { -namespace { - -FaultHandler faultHandler = nullptr; -struct sigaction previousAction{}; - -void handleFault(int signal, siginfo_t* info, void* context) { - const auto* native = static_cast(context); - const auto error = native->uc_mcontext.gregs[REG_ERR]; - if (info->si_code == SEGV_ACCERR && (error & 16) == 0) { - try { - if (faultHandler(reinterpret_cast(info->si_addr), (error & 2) != 0)) return; - } catch (...) { - std::terminate(); - } - } - if (previousAction.sa_handler == SIG_DFL || previousAction.sa_handler == SIG_IGN) { - if (sigaction(signal, &previousAction, nullptr) != 0) std::terminate(); - if (raise(signal) != 0) std::terminate(); - return; - } - if ((previousAction.sa_flags & SA_SIGINFO) != 0) previousAction.sa_sigaction(signal, info, context); - else previousAction.sa_handler(signal); -} - -void protect(std::uint64_t address, std::size_t bytes, int protection) { - if (mprotect(reinterpret_cast(address), bytes, protection) != 0) throw std::system_error(errno, std::generic_category(), "guest memory tracking mprotect failed"); -} - -} - -std::size_t PageSize() { - static const auto size = [] { - const auto result = sysconf(_SC_PAGESIZE); - if (result <= 0) throw std::runtime_error("invalid native page size"); - return static_cast(result); - }(); - return size; -} - -void Install(FaultHandler handler) { - if (handler == nullptr || faultHandler != nullptr) throw std::runtime_error("invalid guest memory fault handler installation"); - faultHandler = handler; - struct sigaction action{}; - action.sa_flags = SA_SIGINFO; - action.sa_sigaction = handleFault; - if (sigemptyset(&action.sa_mask) != 0 || sigaction(SIGSEGV, &action, &previousAction) != 0) { - const auto error = errno; - faultHandler = nullptr; - throw std::system_error(error, std::generic_category(), "guest memory fault handler installation failed"); - } -} - -std::vector Query(std::uint64_t address, std::size_t bytes) { - std::ifstream maps("/proc/self/maps"); - if (!maps) throw std::runtime_error("cannot query tracked guest memory mappings"); - std::vector regions; - const auto end = address + bytes; - std::string line; - while (address < end && std::getline(maps, line)) { - std::istringstream input(line); - std::uint64_t first = 0; - std::uint64_t last = 0; - char separator = 0; - std::string permissions; - if (!(input >> std::hex >> first >> separator >> last >> permissions) || separator != '-' || permissions.size() < 3 || first >= last) throw std::runtime_error("invalid tracked guest memory mapping"); - if (last <= address) continue; - if (first > address || permissions[0] != 'r' || permissions[1] != 'w' || permissions[2] != '-') throw std::runtime_error("tracked render memory must be mapped, writable and non-executable"); - const auto next = std::min(end, last); - regions.push_back({address, static_cast(next - address), static_cast(PROT_READ | PROT_WRITE | (permissions[2] == 'x' ? PROT_EXEC : 0))}); - address = next; - } - if (address != end) throw std::runtime_error("tracked guest memory range is not mapped"); - return regions; -} - -void Protect(std::uint64_t address, std::size_t bytes, Protection protection) { - if (protection != Protection::None && protection != Protection::Read) throw std::invalid_argument("invalid tracked page protection"); - protect(address, bytes, protection == Protection::None ? PROT_NONE : PROT_READ); -} - -void Restore(const std::vector& regions) { - for (const auto& region : regions) protect(region.address, region.bytes, static_cast(region.protection)); -} - -} diff --git a/core/libs/prx/libc/src/MemoryTrackingWindows.cpp b/core/libs/prx/libc/src/MemoryTrackingWindows.cpp deleted file mode 100644 index d504f0fd6b..0000000000 --- a/core/libs/prx/libc/src/MemoryTrackingWindows.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "prx/libc/include/MemoryTrackingPlatform.hpp" -#include -#include -#include -#include -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include - -namespace GuestMemoryTracking::Platform { -namespace { - -FaultHandler faultHandler = nullptr; - -LONG CALLBACK handleException(EXCEPTION_POINTERS* exception) { - const auto* record = exception->ExceptionRecord; - if (record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || record->NumberParameters < 2 || record->ExceptionInformation[0] > 1) return EXCEPTION_CONTINUE_SEARCH; - try { - if (faultHandler(record->ExceptionInformation[1], record->ExceptionInformation[0] == 1)) return EXCEPTION_CONTINUE_EXECUTION; - } catch (...) { - std::terminate(); - } - return EXCEPTION_CONTINUE_SEARCH; -} - -void protect(std::uint64_t address, std::size_t bytes, DWORD protection) { - DWORD previous = 0; - if (!VirtualProtect(reinterpret_cast(address), bytes, protection, &previous)) throw std::system_error(static_cast(GetLastError()), std::system_category(), "guest memory tracking VirtualProtect failed"); -} - -} - -std::size_t PageSize() { - static const auto size = [] { - SYSTEM_INFO info{}; - GetSystemInfo(&info); - if (info.dwPageSize == 0) throw std::runtime_error("invalid native page size"); - return static_cast(info.dwPageSize); - }(); - return size; -} - -void Install(FaultHandler handler) { - if (handler == nullptr || faultHandler != nullptr) throw std::runtime_error("invalid guest memory fault handler installation"); - faultHandler = handler; - if (AddVectoredExceptionHandler(1, handleException) == nullptr) { - const auto error = GetLastError(); - faultHandler = nullptr; - throw std::system_error(static_cast(error), std::system_category(), "guest memory fault handler installation failed"); - } -} - -std::vector Query(std::uint64_t address, std::size_t bytes) { - std::vector regions; - const auto end = address + bytes; - while (address < end) { - MEMORY_BASIC_INFORMATION info{}; - if (VirtualQuery(reinterpret_cast(address), &info, sizeof(info)) != sizeof(info)) throw std::system_error(static_cast(GetLastError()), std::system_category(), "tracked guest memory query failed"); - if (info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) throw std::runtime_error("tracked render memory must be committed, writable and non-executable"); - const auto next = std::min(end, reinterpret_cast(info.BaseAddress) + info.RegionSize); - if (next <= address) throw std::runtime_error("invalid tracked guest memory mapping"); - regions.push_back({address, static_cast(next - address), info.Protect}); - address = next; - } - return regions; -} - -void Protect(std::uint64_t address, std::size_t bytes, Protection protection) { - if (protection != Protection::None && protection != Protection::Read) throw std::invalid_argument("invalid tracked page protection"); - protect(address, bytes, protection == Protection::None ? PAGE_NOACCESS : PAGE_READONLY); -} - -void Restore(const std::vector& regions) { - for (const auto& region : regions) protect(region.address, region.bytes, static_cast(region.protection)); -} - -} diff --git a/core/libs/prx/libc/src/Process.cpp b/core/libs/prx/libc/src/Process.cpp index 5e8bf25b3c..622d60321f 100644 --- a/core/libs/prx/libc/src/Process.cpp +++ b/core/libs/prx/libc/src/Process.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -58,6 +59,11 @@ extern "C" void LibcRunShutdown_nid_postfix() { extern "C" { void APS5_VABI exit_nid_postfix(int code) { + static const bool trace = std::getenv("APS5_TRACE_EXIT") != nullptr; + if (trace) { + std::fprintf(stderr, "[libc] exit(%d) called from %p\n", code, __builtin_return_address(0)); + std::fflush(stderr); + } LibcRunShutdown_nid_postfix(); std::exit(code); } @@ -68,6 +74,8 @@ void APS5_VABI exit_nid_postfix(int code) { ) { (void)arg0; (void)arg1; (void)arg2; (void)arg3; (void)arg4; (void)arg5; + std::fprintf(stderr, "[libc] abort() called from %p\n", __builtin_return_address(0)); + std::fflush(nullptr); std::abort(); } diff --git a/core/libs/prx/libc/src/RuntimeSupport.cpp b/core/libs/prx/libc/src/RuntimeSupport.cpp index 582f30ab68..c2772ac47c 100644 --- a/core/libs/prx/libc/src/RuntimeSupport.cpp +++ b/core/libs/prx/libc/src/RuntimeSupport.cpp @@ -21,6 +21,7 @@ extern "C" { FileStream _Stderr_nid_postfix{stderr}; FileStream _Stdout_nid_postfix{stdout}; +FileStream _Stdin_nid_postfix{stdin}; int APS5_VABI __cxa_atexit_nid_postfix(void (*func)(void*), void* arg, void* dsoHandle) { (void)dsoHandle; diff --git a/core/libs/prx/libc/src/StdioExtras.cpp b/core/libs/prx/libc/src/StdioExtras.cpp new file mode 100644 index 0000000000..e9c9896605 --- /dev/null +++ b/core/libs/prx/libc/src/StdioExtras.cpp @@ -0,0 +1,160 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "prx/libc/include/FileStream.hpp" +#include "prx/libc/include/GuestHeap.hpp" +#include "prx/libc/include/General.hpp" +#include "SceTypes.hpp" + +#ifdef _WIN32 +#include "prx/libc/include/WindowsFormatting.hpp" +#endif + +namespace { + +using GuestCompare = int (APS5_VABI*)(const void*, const void*); +using GuestNewHandler = void (APS5_VABI*)(); + +GuestNewHandler g_newHandler = nullptr; + +#ifdef _WIN32 +// Formats with a guest (SysV) va_list into a host string. +std::string FormatGuest(const char* format, const void* args) { + std::string text; + LibcDetail::FormatWindows(nullptr, 0, format, args, &text); + return text; +} +#endif + +} + +extern "C" { + +void* APS5_VABI bsearch_nid_postfix(const void* key, const void* base, size_t count, size_t size, GuestCompare compare) { + const auto* bytes = static_cast(base); + size_t low = 0; + size_t high = count; + while (low < high) { + const size_t middle = low + (high - low) / 2; + const void* element = bytes + middle * size; + const int order = compare(key, element); + if (order == 0) return const_cast(element); + if (order < 0) high = middle; + else low = middle + 1; + } + return nullptr; +} + +void APS5_VABI clearerr_nid_postfix(FileStream* stream) { + std::clearerr(GetNativeStream(stream)); +} + +int APS5_VABI feof_nid_postfix(FileStream* stream) { + return std::feof(GetNativeStream(stream)); +} + +int APS5_VABI ferror_nid_postfix(FileStream* stream) { + return std::ferror(GetNativeStream(stream)); +} + +int APS5_VABI fputc_nid_postfix(int character, FileStream* stream) { + return std::fputc(character, GetNativeStream(stream)); +} + +int APS5_VABI getc_nid_postfix(FileStream* stream) { + return std::getc(GetNativeStream(stream)); +} + +int APS5_VABI ungetc_nid_postfix(int character, FileStream* stream) { + return std::ungetc(character, GetNativeStream(stream)); +} + +char* APS5_VABI fgets_nid_postfix(char* buffer, int count, FileStream* stream) { + return std::fgets(buffer, count, GetNativeStream(stream)); +} + +int APS5_VABI putchar_nid_postfix(int character) { + return std::putchar(character); +} + +void APS5_VABI perror_nid_postfix(const char* prefix) { + if (prefix && *prefix) std::fprintf(stderr, "%s: %s\n", prefix, std::strerror(errno)); + else std::fprintf(stderr, "%s\n", std::strerror(errno)); +} + +int APS5_VABI remove_nid_postfix(const char* path) { + std::error_code error; + return std::filesystem::remove(ResolvePath_nid_no_patch(path), error) ? 0 : -1; +} + +// FreeBSD and the host CRT agree on the classic errno values the messages are looked up by. +char* APS5_VABI strerror_nid_postfix(int error) { + return std::strerror(error); +} + +#ifdef _WIN32 + +int APS5_VABI fprintf_nid_postfix(FileStream* stream, const char* format, ...) { + __builtin_sysv_va_list args; + __builtin_sysv_va_start(args, format); + const auto text = FormatGuest(format, args); + __builtin_sysv_va_end(args); + return static_cast(std::fwrite(text.data(), 1, text.size(), GetNativeStream(stream))); +} + +int APS5_VABI vsprintf_nid_postfix(char* buffer, const char* format, VaList* args) { + return LibcDetail::FormatWindows(buffer, SIZE_MAX, format, args); +} + +int APS5_VABI vsprintf_s_nid_postfix(char* buffer, size_t size, const char* format, VaList* args) { + return LibcDetail::FormatWindows(buffer, size, format, args); +} + +int APS5_VABI sprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) { + __builtin_sysv_va_list args; + __builtin_sysv_va_start(args, format); + const int result = LibcDetail::FormatWindows(buffer, size, format, args); + __builtin_sysv_va_end(args); + return result; +} + +#endif + +[[noreturn]] void APS5_VABI _Assert_nid_postfix(const char* message, const char* location) { + std::fprintf(stderr, "[libc] guest assertion failed: %s (%s)\n", message ? message : "?", location ? location : "?"); + std::fflush(stderr); + std::abort(); +} + +[[noreturn]] void APS5_VABI _Exit_nid_postfix(int status) { + static const bool trace = std::getenv("APS5_TRACE_EXIT") != nullptr; + if (trace) std::fprintf(stderr, "[libc] _Exit(%d) called from %p\n", status, __builtin_return_address(0)); + std::fflush(nullptr); + std::_Exit(status); +} + +// Only the "C" locale exists. +const char* APS5_VABI setlocale_nid_postfix(int category, const char* locale) { + (void)category; + if (locale == nullptr || locale[0] == 0 || std::strcmp(locale, "C") == 0 || std::strcmp(locale, "POSIX") == 0) return "C"; + return nullptr; +} + +GuestNewHandler APS5_VABI _ZSt15set_new_handlerPFvvE_nid_postfix(GuestNewHandler handler) { + const auto previous = g_newHandler; + g_newHandler = handler; + return previous; +} + +void APS5_VABI _ZdlPvSt11align_val_t_nid_postfix(void* pointer, std::size_t alignment) { + (void)alignment; + GuestHeap::GuestHeapFree_nid_postfix(pointer); +} + +} diff --git a/core/libs/prx/libc/src/Strings.cpp b/core/libs/prx/libc/src/Strings.cpp index 667ce9dd04..f46cee0491 100644 --- a/core/libs/prx/libc/src/Strings.cpp +++ b/core/libs/prx/libc/src/Strings.cpp @@ -150,4 +150,119 @@ char* APS5_VABI strdup_nid_postfix(const char* s) { return copy; } +int APS5_VABI bcmp_nid_postfix(const void* s1, const void* s2, size_t n) { + return std::memcmp(s1, s2, n); +} + +size_t APS5_VABI strspn_nid_postfix(const char* s, const char* accept) { + return std::strspn(s, accept); +} + +size_t APS5_VABI strcspn_nid_postfix(const char* s, const char* reject) { + return std::strcspn(s, reject); +} + +const char* APS5_VABI strpbrk_nid_postfix(const char* s, const char* accept) { + return std::strpbrk(s, accept); +} + +char* APS5_VABI strncat_nid_postfix(char* dest, const char* src, size_t n) { + return std::strncat(dest, src, n); +} + +int APS5_VABI strcoll_nid_postfix(const char* s1, const char* s2) { + return std::strcmp(s1, s2); +} + +int APS5_VABI strncpy_s_nid_postfix(char* dest, size_t destsz, const char* src, size_t count) { + constexpr int GuestEinval = 22; + constexpr int GuestErange = 34; + if (!dest || destsz == 0) return GuestEinval; + if (!src) { + dest[0] = '\0'; + return GuestEinval; + } + size_t length = 0; + while (length < count && src[length] != '\0') ++length; + if (length >= destsz) { + dest[0] = '\0'; + return GuestErange; + } + std::memcpy(dest, src, length); + dest[length] = '\0'; + return 0; +} + +float APS5_VABI strtof_nid_postfix(const char* str, char** endptr) { + return std::strtof(str, endptr); +} + +double APS5_VABI atof_nid_postfix(const char* str) { + return std::atof(str); +} + +unsigned long long APS5_VABI _Stoull_nid_postfix(const char* str, char** endptr, int base) { + return std::strtoull(str, endptr, base); +} + +size_t APS5_VABI wcslen_nid_postfix(const wchar_t* s) { + return std::wcslen(s); +} + +int APS5_VABI wcscmp_nid_postfix(const wchar_t* s1, const wchar_t* s2) { + return std::wcscmp(s1, s2); +} + +int APS5_VABI wcsncmp_nid_postfix(const wchar_t* s1, const wchar_t* s2, size_t n) { + return std::wcsncmp(s1, s2, n); +} + +wchar_t* APS5_VABI wcscpy_nid_postfix(wchar_t* dest, const wchar_t* src) { + return std::wcscpy(dest, src); +} + +wchar_t* APS5_VABI wcsncpy_nid_postfix(wchar_t* dest, const wchar_t* src, size_t n) { + return std::wcsncpy(dest, src, n); +} + +const wchar_t* APS5_VABI wcschr_nid_postfix(const wchar_t* s, wchar_t c) { + return std::wcschr(s, c); +} + +const wchar_t* APS5_VABI wcsrchr_nid_postfix(const wchar_t* s, wchar_t c) { + return std::wcsrchr(s, c); +} + +const wchar_t* APS5_VABI wcsstr_nid_postfix(const wchar_t* haystack, const wchar_t* needle) { + return std::wcsstr(haystack, needle); +} + +const wchar_t* APS5_VABI wcspbrk_nid_postfix(const wchar_t* s, const wchar_t* accept) { + return std::wcspbrk(s, accept); +} + +size_t APS5_VABI wcsspn_nid_postfix(const wchar_t* s, const wchar_t* accept) { + return std::wcsspn(s, accept); +} + +wchar_t* APS5_VABI wmemset_nid_postfix(wchar_t* s, wchar_t c, size_t n) { + return std::wmemset(s, c, n); +} + +double APS5_VABI wcstod_nid_postfix(const wchar_t* str, wchar_t** endptr) { + return std::wcstod(str, endptr); +} + +float APS5_VABI wcstof_nid_postfix(const wchar_t* str, wchar_t** endptr) { + return std::wcstof(str, endptr); +} + +long long APS5_VABI wcstol_nid_postfix(const wchar_t* str, wchar_t** endptr, int base) { + return std::wcstoll(str, endptr, base); +} + +long long APS5_VABI wcstoll_nid_postfix(const wchar_t* str, wchar_t** endptr, int base) { + return std::wcstoll(str, endptr, base); +} + } diff --git a/core/libs/prx/libc/src/Time.cpp b/core/libs/prx/libc/src/Time.cpp index 85bedd1c98..6d03d5fb1a 100644 --- a/core/libs/prx/libc/src/Time.cpp +++ b/core/libs/prx/libc/src/Time.cpp @@ -3,6 +3,10 @@ #include #include +#ifdef _WIN32 +#include +#endif + #include "prx/libc/include/General.hpp" extern "C" { @@ -47,20 +51,30 @@ std::tm* APS5_VABI localtime_nid_postfix(const int64_t* timer) { return libc_localtime_nid_postfix(timer); } -int APS5_VABI localtime_s_nid_postfix(std::tm* result, const int64_t* timer) { +std::tm* APS5_VABI localtime_s_nid_postfix(const int64_t* timer, std::tm* result) { + if (timer == nullptr || result == nullptr) return nullptr; const std::time_t t = static_cast(*timer); - const std::tm* converted = std::localtime(&t); - if (converted == nullptr) return -1; - *result = *converted; - return 0; +#ifdef _WIN32 + if (localtime_s(result, &t) != 0) return nullptr; +#else + if (localtime_r(&t, result) == nullptr) return nullptr; +#endif + return result; } -int APS5_VABI gmtime_s_nid_postfix(std::tm* result, const int64_t* timer) { +std::tm* APS5_VABI gmtime_s_nid_postfix(const int64_t* timer, std::tm* result) { + if (timer == nullptr || result == nullptr) return nullptr; const std::time_t t = static_cast(*timer); - const std::tm* converted = std::gmtime(&t); - if (converted == nullptr) return -1; - *result = *converted; - return 0; +#ifdef _WIN32 + if (gmtime_s(result, &t) != 0) return nullptr; +#else + if (gmtime_r(&t, result) == nullptr) return nullptr; +#endif + return result; +} + +std::tm* APS5_VABI gmtime_nid_postfix(const int64_t* timer) { + return libc_gmtime_nid_postfix(timer); } int64_t APS5_VABI libc_mktime_nid_postfix(std::tm* timeptr) { @@ -79,4 +93,16 @@ size_t APS5_VABI strftime_nid_postfix(char* str, size_t count, const char* forma return std::strftime(str, count, format, timeptr); } +// The guest's CLOCKS_PER_SEC is 1000000: clock() reports process CPU time in microseconds. +int64_t APS5_VABI clock_nid_postfix() { +#ifdef _WIN32 + FILETIME creation, exit, kernel, user; + if (!GetProcessTimes(GetCurrentProcess(), &creation, &exit, &kernel, &user)) return -1; + const auto toTicks = [](const FILETIME& time) { return (static_cast(time.dwHighDateTime) << 32) | time.dwLowDateTime; }; + return static_cast((toTicks(kernel) + toTicks(user)) / 10); +#else + return static_cast(static_cast(std::clock()) * 1000000.0 / CLOCKS_PER_SEC); +#endif +} + } diff --git a/core/libs/prx/libc/src/exception/Runtime.cpp b/core/libs/prx/libc/src/exception/Runtime.cpp index 6e1c52cffb..c26045b6b0 100644 --- a/core/libs/prx/libc/src/exception/Runtime.cpp +++ b/core/libs/prx/libc/src/exception/Runtime.cpp @@ -1,9 +1,31 @@ #include "prx/libc/include/exceptions/Runtime.hpp" +#include #include #include +#ifdef _WIN32 +#include +#endif namespace LibcException { +// Writes straight to the stderr handle: terminate paths must not wait on the CRT stream lock, +// which another (possibly suspended) thread may hold. +static void RawLog(const char* format, ...) { + char buffer[1024]; + va_list args; + va_start(args, format); + int length = std::vsnprintf(buffer, sizeof(buffer), format, args); + va_end(args); + if (length <= 0) return; + if (length > static_cast(sizeof(buffer)) - 1) length = sizeof(buffer) - 1; +#ifdef _WIN32 + DWORD written = 0; + WriteFile(GetStdHandle(STD_ERROR_HANDLE), buffer, static_cast(length), &written, nullptr); +#else + std::fwrite(buffer, 1, length, stderr); +#endif +} + [[noreturn]] static void DefaultTerminate() { if (globals.caught && Native(globals.caught->unwind.exception_class)) { Header* primary = Primary(globals.caught); @@ -25,12 +47,12 @@ namespace LibcException { char* demangled = abi::__cxa_demangle(typeName, nullptr, nullptr, &status); const char* displayName = (status == 0 && demangled) ? demangled : typeName; if (what) - std::fprintf(stderr, "terminate called after throwing an instance of '%s'\n what(): %s\n", displayName, what); + RawLog("terminate called after throwing an instance of '%s'\n what(): %s\n", displayName, what); else - std::fprintf(stderr, "terminate called after throwing an instance of '%s'\n", displayName); + RawLog("terminate called after throwing an instance of '%s'\n", displayName); std::free(demangled); } else { - std::fprintf(stderr, "terminate called without an active exception\n"); + RawLog("terminate called without an active exception\n"); } std::abort(); } @@ -206,7 +228,7 @@ void APS5_VABI __cxa_free_exception_nid_postfix(void* object) { std::free(allocation); } -[[noreturn]] void __cxa_throw_nid_postfix(void* object, std::type_info* type, void (*destructor)(void*)) { +[[noreturn]] void APS5_VABI __cxa_throw_nid_postfix(void* object, std::type_info* type, void (*destructor)(void*)) { using namespace LibcException; auto* header = FromObject(object); header->type = type; @@ -217,6 +239,8 @@ void APS5_VABI __cxa_free_exception_nid_postfix(void* object) { header->unwind.exception_cleanup = Cleanup; ++globals.uncaught; _Unwind_RaiseException_nid_postfix(&header->unwind); + RawLog("[libc] unhandled exception '%s' thrown from %p (caller %p)\n", type ? type->name() : "?", __builtin_return_address(0), + static_cast(static_cast(__builtin_frame_address(0))[0])[1]); __cxa_begin_catch_nid_postfix(&header->unwind); InvokeTerminate(header->terminate); } @@ -261,7 +285,7 @@ void APS5_VABI __cxa_end_catch_nid_postfix() { } } -[[noreturn]] void __cxa_rethrow_nid_postfix() { +[[noreturn]] void APS5_VABI __cxa_rethrow_nid_postfix() { using namespace LibcException; auto* header = globals.caught; if (!header) Terminate(); @@ -280,23 +304,23 @@ std::type_info* APS5_VABI __cxa_current_exception_type_nid_postfix() { using namespace LibcException; return globals.caught && Native(globals.caught->unwind.exception_class) ? Primary(globals.caught)->type : nullptr; } -void* __cxa_get_globals_nid_postfix() { return &LibcException::globals; } -void* __cxa_get_globals_fast_nid_postfix() { return &LibcException::globals; } -bool __cxa_uncaught_exception_nid_postfix() { return LibcException::globals.uncaught != 0; } -unsigned __cxa_uncaught_exceptions_nid_postfix() { return LibcException::globals.uncaught; } -bool _ZSt18uncaught_exceptionv_nid_postfix() { return __cxa_uncaught_exception_nid_postfix(); } -int _ZSt19uncaught_exceptionsv_nid_postfix() { return static_cast(__cxa_uncaught_exceptions_nid_postfix()); } -[[noreturn]] void _ZSt9terminatev_nid_postfix() { LibcException::Terminate(); } +void* APS5_VABI __cxa_get_globals_nid_postfix() { return &LibcException::globals; } +void* APS5_VABI __cxa_get_globals_fast_nid_postfix() { return &LibcException::globals; } +bool APS5_VABI __cxa_uncaught_exception_nid_postfix() { return LibcException::globals.uncaught != 0; } +unsigned APS5_VABI __cxa_uncaught_exceptions_nid_postfix() { return LibcException::globals.uncaught; } +bool APS5_VABI _ZSt18uncaught_exceptionv_nid_postfix() { return __cxa_uncaught_exception_nid_postfix(); } +int APS5_VABI _ZSt19uncaught_exceptionsv_nid_postfix() { return static_cast(__cxa_uncaught_exceptions_nid_postfix()); } +[[noreturn]] void APS5_VABI _ZSt9terminatev_nid_postfix() { LibcException::Terminate(); } using LibcTerminateHandler = void(*)(); LibcTerminateHandler APS5_VABI _ZSt13set_terminatePFvvE_nid_postfix(LibcTerminateHandler handler) { return LibcException::terminateHandler.exchange(handler ? handler : std::abort); } -LibcTerminateHandler _ZSt13get_terminatev_nid_postfix() { return LibcException::terminateHandler.load(); } +LibcTerminateHandler APS5_VABI _ZSt13get_terminatev_nid_postfix() { return LibcException::terminateHandler.load(); } void APS5_VABI __cxa_increment_exception_refcount_nid_postfix(void* object) { if (object) LibcException::AllocationOf(LibcException::FromObject(object))->references.fetch_add(1, std::memory_order_relaxed); } -void __cxa_decrement_exception_refcount_nid_postfix(void* object) { LibcException::Release(object); } +void APS5_VABI __cxa_decrement_exception_refcount_nid_postfix(void* object) { LibcException::Release(object); } void* APS5_VABI __cxa_current_primary_exception_nid_postfix() { using namespace LibcException; if (!globals.caught || !Native(globals.caught->unwind.exception_class)) return nullptr; @@ -362,10 +386,10 @@ void* __dynamic_cast_nid_postfix(const void* source, const __cxxabiv1::__class_t return publicTarget.count == 1 ? publicTarget.found : nullptr; } -[[noreturn]] void __cxa_call_terminate_nid_postfix(void* exception) { +[[noreturn]] void APS5_VABI __cxa_call_terminate_nid_postfix(void* exception) { __cxa_begin_catch_nid_postfix(exception); LibcException::Terminate(); } -[[noreturn]] void __cxa_pure_virtual_nid_postfix() { LibcException::Terminate(); } -[[noreturn]] void __cxa_deleted_virtual_nid_postfix() { LibcException::Terminate(); } +[[noreturn]] void APS5_VABI __cxa_pure_virtual_nid_postfix() { LibcException::Terminate(); } +[[noreturn]] void APS5_VABI __cxa_deleted_virtual_nid_postfix() { LibcException::Terminate(); } } diff --git a/core/libs/prx/libc/src/exception/Unwind.cpp b/core/libs/prx/libc/src/exception/Unwind.cpp index 371803ac8c..f37d0a5242 100644 --- a/core/libs/prx/libc/src/exception/Unwind.cpp +++ b/core/libs/prx/libc/src/exception/Unwind.cpp @@ -435,7 +435,7 @@ _Unwind_Reason_Code APS5_VABI _Unwind_RaiseException_nid_postfix(_Unwind_Excepti return _URC_FATAL_PHASE1_ERROR; } -[[noreturn]] void _Unwind_Resume_nid_postfix(_Unwind_Exception* exception) { +[[noreturn]] void APS5_VABI _Unwind_Resume_nid_postfix(_Unwind_Exception* exception) { _Unwind_Context context; LibcCaptureRegisters(context.registers); if (!LibcUnwind::Step(context)) std::abort(); @@ -460,14 +460,14 @@ void APS5_VABI _Unwind_SetGR_nid_postfix(_Unwind_Context* context, int index, _U if (index < 0 || index >= 17) std::abort(); context->registers[index] = value; } -_Unwind_Ptr _Unwind_GetIP_nid_postfix(_Unwind_Context* context) { return context->registers[16]; } -void _Unwind_SetIP_nid_postfix(_Unwind_Context* context, _Unwind_Ptr value) { context->registers[16] = value; } -_Unwind_Ptr _Unwind_GetIPInfo_nid_postfix(_Unwind_Context* context, int* before) { if (before) *before = context->signalFrame; return context->registers[16]; } -_Unwind_Word _Unwind_GetCFA_nid_postfix(_Unwind_Context* context) { return context->cfa; } -_Unwind_Ptr _Unwind_GetLanguageSpecificData_nid_postfix(_Unwind_Context* context) { return context->lsda; } -_Unwind_Ptr _Unwind_GetRegionStart_nid_postfix(_Unwind_Context* context) { return context->region; } -_Unwind_Ptr _Unwind_GetDataRelBase_nid_postfix(_Unwind_Context* context) { return context->dataBase; } -_Unwind_Ptr _Unwind_GetTextRelBase_nid_postfix(_Unwind_Context* context) { return context->textBase; } +_Unwind_Ptr APS5_VABI _Unwind_GetIP_nid_postfix(_Unwind_Context* context) { return context->registers[16]; } +void APS5_VABI _Unwind_SetIP_nid_postfix(_Unwind_Context* context, _Unwind_Ptr value) { context->registers[16] = value; } +_Unwind_Ptr APS5_VABI _Unwind_GetIPInfo_nid_postfix(_Unwind_Context* context, int* before) { if (before) *before = context->signalFrame; return context->registers[16]; } +_Unwind_Word APS5_VABI _Unwind_GetCFA_nid_postfix(_Unwind_Context* context) { return context->cfa; } +_Unwind_Ptr APS5_VABI _Unwind_GetLanguageSpecificData_nid_postfix(_Unwind_Context* context) { return context->lsda; } +_Unwind_Ptr APS5_VABI _Unwind_GetRegionStart_nid_postfix(_Unwind_Context* context) { return context->region; } +_Unwind_Ptr APS5_VABI _Unwind_GetDataRelBase_nid_postfix(_Unwind_Context* context) { return context->dataBase; } +_Unwind_Ptr APS5_VABI _Unwind_GetTextRelBase_nid_postfix(_Unwind_Context* context) { return context->textBase; } _Unwind_Reason_Code APS5_VABI _Unwind_ForcedUnwind_nid_postfix(_Unwind_Exception* exception, _Unwind_Stop_Fn stop, void* argument) { if (!stop) return _URC_FATAL_PHASE2_ERROR; @@ -497,22 +497,22 @@ _Unwind_Reason_Code APS5_VABI _Unwind_Backtrace_nid_postfix(_Unwind_Trace_Fn tra #else extern "C" { -_Unwind_Reason_Code _Unwind_RaiseException_nid_postfix(_Unwind_Exception*) { std::abort(); } -[[noreturn]] void _Unwind_Resume_nid_postfix(_Unwind_Exception*) { std::abort(); } -_Unwind_Reason_Code _Unwind_Resume_or_Rethrow_nid_postfix(_Unwind_Exception*) { std::abort(); } -void _Unwind_DeleteException_nid_postfix(_Unwind_Exception* e) { if (e && e->exception_cleanup) e->exception_cleanup(_URC_FOREIGN_EXCEPTION_CAUGHT, e); } -_Unwind_Word _Unwind_GetGR_nid_postfix(_Unwind_Context*, int) { std::abort(); } -void _Unwind_SetGR_nid_postfix(_Unwind_Context*, int, _Unwind_Word) { std::abort(); } -_Unwind_Ptr _Unwind_GetIP_nid_postfix(_Unwind_Context*) { std::abort(); } -void _Unwind_SetIP_nid_postfix(_Unwind_Context*, _Unwind_Ptr) { std::abort(); } -_Unwind_Ptr _Unwind_GetIPInfo_nid_postfix(_Unwind_Context*, int*) { std::abort(); } -_Unwind_Word _Unwind_GetCFA_nid_postfix(_Unwind_Context*) { std::abort(); } -_Unwind_Ptr _Unwind_GetLanguageSpecificData_nid_postfix(_Unwind_Context*) { std::abort(); } -_Unwind_Ptr _Unwind_GetRegionStart_nid_postfix(_Unwind_Context*) { std::abort(); } -_Unwind_Ptr _Unwind_GetDataRelBase_nid_postfix(_Unwind_Context*) { std::abort(); } -_Unwind_Ptr _Unwind_GetTextRelBase_nid_postfix(_Unwind_Context*) { std::abort(); } -_Unwind_Reason_Code _Unwind_ForcedUnwind_nid_postfix(_Unwind_Exception*, _Unwind_Stop_Fn, void*) { std::abort(); } -_Unwind_Reason_Code _Unwind_Backtrace_nid_postfix(_Unwind_Trace_Fn, void*) { std::abort(); } +_Unwind_Reason_Code APS5_VABI _Unwind_RaiseException_nid_postfix(_Unwind_Exception*) { std::abort(); } +[[noreturn]] void APS5_VABI _Unwind_Resume_nid_postfix(_Unwind_Exception*) { std::abort(); } +_Unwind_Reason_Code APS5_VABI _Unwind_Resume_or_Rethrow_nid_postfix(_Unwind_Exception*) { std::abort(); } +void APS5_VABI _Unwind_DeleteException_nid_postfix(_Unwind_Exception* e) { if (e && e->exception_cleanup) e->exception_cleanup(_URC_FOREIGN_EXCEPTION_CAUGHT, e); } +_Unwind_Word APS5_VABI _Unwind_GetGR_nid_postfix(_Unwind_Context*, int) { std::abort(); } +void APS5_VABI _Unwind_SetGR_nid_postfix(_Unwind_Context*, int, _Unwind_Word) { std::abort(); } +_Unwind_Ptr APS5_VABI _Unwind_GetIP_nid_postfix(_Unwind_Context*) { std::abort(); } +void APS5_VABI _Unwind_SetIP_nid_postfix(_Unwind_Context*, _Unwind_Ptr) { std::abort(); } +_Unwind_Ptr APS5_VABI _Unwind_GetIPInfo_nid_postfix(_Unwind_Context*, int*) { std::abort(); } +_Unwind_Word APS5_VABI _Unwind_GetCFA_nid_postfix(_Unwind_Context*) { std::abort(); } +_Unwind_Ptr APS5_VABI _Unwind_GetLanguageSpecificData_nid_postfix(_Unwind_Context*) { std::abort(); } +_Unwind_Ptr APS5_VABI _Unwind_GetRegionStart_nid_postfix(_Unwind_Context*) { std::abort(); } +_Unwind_Ptr APS5_VABI _Unwind_GetDataRelBase_nid_postfix(_Unwind_Context*) { std::abort(); } +_Unwind_Ptr APS5_VABI _Unwind_GetTextRelBase_nid_postfix(_Unwind_Context*) { std::abort(); } +_Unwind_Reason_Code APS5_VABI _Unwind_ForcedUnwind_nid_postfix(_Unwind_Exception*, _Unwind_Stop_Fn, void*) { std::abort(); } +_Unwind_Reason_Code APS5_VABI _Unwind_Backtrace_nid_postfix(_Unwind_Trace_Fn, void*) { std::abort(); } } #endif diff --git a/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp b/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp new file mode 100644 index 0000000000..53d33430b0 --- /dev/null +++ b/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp @@ -0,0 +1,372 @@ +#ifdef _WIN32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Sse4aEmulation.hpp" + +namespace { + +// The reporter runs on a faulting thread that may itself hold the CRT stream lock (a fault inside +// printf), so it writes straight to the stderr handle. +void Report(const char* format, ...) { + char buffer[1024]; + va_list args; + va_start(args, format); + int length = std::vsnprintf(buffer, sizeof(buffer), format, args); + va_end(args); + if (length <= 0) return; + if (length > static_cast(sizeof(buffer)) - 1) length = sizeof(buffer) - 1; + DWORD written = 0; + WriteFile(GetStdHandle(STD_ERROR_HANDLE), buffer, static_cast(length), &written, nullptr); +} + +void DescribeAddress(std::uint64_t address, char* buffer, std::size_t size) { + HMODULE module = nullptr; + char path[MAX_PATH] = "?"; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(address), &module) && module) { + GetModuleFileNameA(module, path, sizeof(path)); + const char* name = path; + for (const char* cursor = path; *cursor; ++cursor) { + if (*cursor == '\\' || *cursor == '/') name = cursor + 1; + } + std::snprintf(buffer, size, "0x%016llx %s+0x%llx", static_cast(address), name, static_cast(address - reinterpret_cast(module))); + return; + } + std::snprintf(buffer, size, "0x%016llx", static_cast(address)); +} + +bool IsExecutable(std::uint64_t address) { + MEMORY_BASIC_INFORMATION memory{}; + if (VirtualQuery(reinterpret_cast(address), &memory, sizeof(memory)) != sizeof(memory)) return false; + if (memory.State != MEM_COMMIT) return false; + const DWORD protection = memory.Protect & 0xff; + return protection == PAGE_EXECUTE || protection == PAGE_EXECUTE_READ || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY; +} + +bool IsReadable(std::uint64_t address) { + MEMORY_BASIC_INFORMATION memory{}; + if (VirtualQuery(reinterpret_cast(address), &memory, sizeof(memory)) != sizeof(memory)) return false; + return memory.State == MEM_COMMIT && (memory.Protect & (PAGE_NOACCESS | PAGE_GUARD)) == 0; +} + +bool IsFatal(DWORD code) { + switch (code) { + case EXCEPTION_ACCESS_VIOLATION: + case EXCEPTION_ILLEGAL_INSTRUCTION: + case EXCEPTION_PRIV_INSTRUCTION: + case EXCEPTION_STACK_OVERFLOW: + case EXCEPTION_INT_DIVIDE_BY_ZERO: + case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: + case EXCEPTION_IN_PAGE_ERROR: + return true; + default: + return false; + } +} + +// Debug aid: APS5_WATCH_WRITE= reports every write to that 8-byte location by +// making its page read-only and single-stepping each trapped write. +std::uintptr_t g_watchAddress = 0; +std::uintptr_t g_watchPage = 0; +thread_local bool t_watchStepping = false; +thread_local bool t_watchHit = false; +thread_local unsigned long long t_watchBefore = 0; +thread_local std::uintptr_t t_watchFault = 0; +thread_local char t_watchWhere[MAX_PATH + 64]; + +void ProtectWatchPage(DWORD protection) { + DWORD old; + VirtualProtect(reinterpret_cast(g_watchPage), 0x1000, protection, &old); +} + +bool HandleWatch(EXCEPTION_POINTERS* info) { + if (g_watchPage == 0) return false; + const auto* record = info->ExceptionRecord; + auto* context = info->ContextRecord; + if (record->ExceptionCode == EXCEPTION_SINGLE_STEP && t_watchStepping) { + t_watchStepping = false; + ProtectWatchPage(PAGE_READONLY); + const auto current = *reinterpret_cast(g_watchAddress); + if (t_watchHit || current != t_watchBefore) { + if (!t_watchHit) Report("[watch] thread %lu changed the value with a write starting at 0x%llx from %s\n", GetCurrentThreadId(), static_cast(t_watchFault), t_watchWhere); + Report("[watch] new value 0x%016llx at %lld ms\n", current, static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count())); + t_watchHit = false; + } + return true; + } + if (record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || record->NumberParameters < 2 || record->ExceptionInformation[0] != 1) return false; + const auto address = static_cast(record->ExceptionInformation[1]); + if (address < g_watchPage || address >= g_watchPage + 0x1000) return false; + // Block writes (memset, memcpy) fault on their first byte, so compare the value after the step + // as well as matching the fault address. + t_watchBefore = *reinterpret_cast(g_watchAddress); + t_watchFault = address; + DescribeAddress(context->Rip, t_watchWhere, sizeof(t_watchWhere)); + if (address + 32 > g_watchAddress && address < g_watchAddress + 8) { + char line[MAX_PATH + 64]; + DescribeAddress(context->Rip, line, sizeof(line)); + Report("[watch] thread %lu writes 0x%llx at %s rcx=%llx rdx=%llx r8=%llx\n", GetCurrentThreadId(), static_cast(address), line, context->Rcx, context->Rdx, context->R8); + for (std::uint64_t slot = context->Rsp; slot < context->Rsp + 0x80 && IsReadable(slot); slot += 8) { + const auto value = *reinterpret_cast(slot); + if (!IsExecutable(value)) continue; + DescribeAddress(value, line, sizeof(line)); + Report("[watch] [rsp+0x%llx] %s\n", static_cast(slot - context->Rsp), line); + } + std::uint64_t frame = context->Rbp; + for (int depth = 0; depth < 6 && frame != 0 && (frame & 7) == 0 && IsReadable(frame) && IsReadable(frame + 8); ++depth) { + const auto returnAddress = reinterpret_cast(frame)[1]; + if (!IsExecutable(returnAddress)) break; + DescribeAddress(returnAddress, line, sizeof(line)); + Report("[watch] #%d %s\n", depth, line); + const auto next = reinterpret_cast(frame)[0]; + if (next <= frame) break; + frame = next; + } + t_watchHit = true; + } + ProtectWatchPage(PAGE_READWRITE); + context->EFlags |= 0x100; + t_watchStepping = true; + return true; +} + +void InstallWatch() { + // APS5_WATCH_ADDR= watches guest data that is mapped later: the page is + // protected once it has been committed. + if (const char* absolute = std::getenv("APS5_WATCH_ADDR")) { + g_watchAddress = std::strtoull(absolute, nullptr, 16); + const auto page = g_watchAddress & ~static_cast(0xfff); + CreateThread(nullptr, 0, [](void* parameter) -> DWORD { + const auto page = reinterpret_cast(parameter); + for (;;) { + MEMORY_BASIC_INFORMATION memory{}; + if (VirtualQuery(reinterpret_cast(page), &memory, sizeof(memory)) == sizeof(memory) && memory.State == MEM_COMMIT) break; + Sleep(1); + } + g_watchPage = page; + ProtectWatchPage(PAGE_READONLY); + Report("[watch] watching 0x%llx (page 0x%llx)\n", static_cast(g_watchAddress), static_cast(page)); + return 0; + }, reinterpret_cast(page), 0, nullptr); + return; + } + const char* value = std::getenv("APS5_WATCH_WRITE"); + if (!value) return; + const auto elfAddress = std::strtoull(value, nullptr, 0); + g_watchAddress = reinterpret_cast(GetModuleHandleW(nullptr)) + 0x10000 + elfAddress; + g_watchPage = g_watchAddress & ~static_cast(0xfff); + ProtectWatchPage(PAGE_READONLY); + Report("[watch] watching 0x%llx (page 0x%llx)\n", static_cast(g_watchAddress), static_cast(g_watchPage)); +} + +// SSE4a (EXTRQ / INSERTQ) is AMD-only: the PS5's Zen 2 has it and titles use it, so on an Intel host the +// instruction raises STATUS_ILLEGAL_INSTRUCTION and is emulated on the faulting thread's CONTEXT. +// APS5_NO_SSE4A_EMULATION=1 disables this (the fault is then reported as fatal as before) and +// APS5_TRACE_SSE4A=1 prints each emulated instruction once per rip. +bool g_sse4aEmulation = true; +bool g_sse4aTrace = false; +std::atomic g_sse4aEmulated{0}; + +bool IsEnvironmentSet(const char* name) { + const char* value = std::getenv(name); + return value != nullptr && *value != '\0' && !(value[0] == '0' && value[1] == '\0'); +} + +// Copies the bytes at `address` that are mapped; the read stays within readable pages. +std::size_t ReadCode(std::uint64_t address, std::uint8_t* buffer, std::size_t size) { + std::size_t count = 0; + while (count < size) { + const auto cursor = address + count; + if ((cursor & 0xfff) == 0 || count == 0) { + if (!IsReadable(cursor)) break; + } + buffer[count++] = *reinterpret_cast(cursor); + } + return count; +} + +void TraceSse4a(std::uint64_t rip, const sse4a::Instruction& instruction, const sse4a::Field& field, unsigned long long count) { + static std::mutex mutex; + static std::uint64_t seen[128]; + static std::size_t seenCount = 0; + { + std::lock_guard lock(mutex); + for (std::size_t i = 0; i < seenCount; ++i) { + if (seen[i] == rip) return; + } + if (seenCount < sizeof(seen) / sizeof(seen[0])) seen[seenCount++] = rip; + } + char line[MAX_PATH + 64]; + DescribeAddress(rip, line, sizeof(line)); + const char* mnemonic = instruction.op == sse4a::Op::Extrq ? "extrq" : "insertq"; + char operands[64]; + if (instruction.op == sse4a::Op::Extrq && !instruction.registerForm) { + std::snprintf(operands, sizeof(operands), "xmm%u, %u, %u", instruction.destination, instruction.length, instruction.index); + } else if (!instruction.registerForm) { + std::snprintf(operands, sizeof(operands), "xmm%u, xmm%u, %u, %u", instruction.destination, instruction.source, instruction.length, instruction.index); + } else { + std::snprintf(operands, sizeof(operands), "xmm%u, xmm%u", instruction.destination, instruction.source); + } + Report("[sse4a] #%llu %s %s (length %u, index %u, %u bytes) at %s on thread %lu\n", count, mnemonic, operands, field.length, field.index, static_cast(instruction.size), line, GetCurrentThreadId()); +} + +bool HandleSse4a(EXCEPTION_POINTERS* info) { + if (!g_sse4aEmulation) return false; + const auto* record = info->ExceptionRecord; + if (record->ExceptionCode != EXCEPTION_ILLEGAL_INSTRUCTION) return false; + auto* context = info->ContextRecord; + const auto rip = context->Rip; + std::uint8_t bytes[sse4a::kMaxInstructionSize]; + const std::size_t available = ReadCode(rip, bytes, sizeof(bytes)); + sse4a::Instruction instruction; + sse4a::Field field; + if (!sse4a::Emulate(bytes, available, *context, &instruction, &field)) return false; + const auto count = ++g_sse4aEmulated; + if (g_sse4aTrace) TraceSse4a(rip, instruction, field, count); + return true; +} + +LONG WINAPI ReportCrash(EXCEPTION_POINTERS* info) { + static std::atomic reported{false}; + if (HandleWatch(info)) return EXCEPTION_CONTINUE_EXECUTION; + if (HandleSse4a(info)) return EXCEPTION_CONTINUE_EXECUTION; + const auto* record = info->ExceptionRecord; + if (!IsFatal(record->ExceptionCode)) return EXCEPTION_CONTINUE_SEARCH; + if (reported.exchange(true)) { + // Another thread is already reporting; let it finish before this fault ends the process. + Sleep(INFINITE); + } + const auto* context = info->ContextRecord; + char line[MAX_PATH + 64]; + char threadName[128] = ""; + PWSTR description = nullptr; + if (SUCCEEDED(GetThreadDescription(GetCurrentThread(), &description)) && description) { + WideCharToMultiByte(CP_UTF8, 0, description, -1, threadName, sizeof(threadName), nullptr, nullptr); + LocalFree(description); + } + Report("\nFATAL: unhandled exception 0x%08lx on thread %lu '%s'\n", record->ExceptionCode, GetCurrentThreadId(), threadName); + DescribeAddress(context->Rip, line, sizeof(line)); + Report(" rip %s\n", line); + if (record->ExceptionCode == EXCEPTION_ILLEGAL_INSTRUCTION) { + std::uint8_t bytes[8]; + const std::size_t available = ReadCode(context->Rip, bytes, sizeof(bytes)); + char hex[3 * sizeof(bytes) + 1] = ""; + for (std::size_t i = 0; i < available; ++i) std::snprintf(hex + 3 * i, sizeof(hex) - 3 * i, "%02x ", bytes[i]); + Report(" bytes %s(sse4a emulation %s, %llu emulated so far)\n", hex, g_sse4aEmulation ? "on" : "off", g_sse4aEmulated.load()); + } + if (record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && record->NumberParameters >= 2) { + const char* kind = record->ExceptionInformation[0] == 0 ? "read" : record->ExceptionInformation[0] == 1 ? "write" : "execute"; + Report(" %s of 0x%016llx\n", kind, static_cast(record->ExceptionInformation[1])); + } + Report(" rax %016llx rbx %016llx rcx %016llx rdx %016llx\n", context->Rax, context->Rbx, context->Rcx, context->Rdx); + Report(" rsi %016llx rdi %016llx rbp %016llx rsp %016llx\n", context->Rsi, context->Rdi, context->Rbp, context->Rsp); + Report(" r8 %016llx r9 %016llx r10 %016llx r11 %016llx\n", context->R8, context->R9, context->R10, context->R11); + Report(" r12 %016llx r13 %016llx r14 %016llx r15 %016llx\n", context->R12, context->R13, context->R14, context->R15); + // The words each register points at, to recognise which structure a bad pointer came from. + { + const std::pair pointers[] = {{"rax", context->Rax}, {"rbx", context->Rbx}, {"rcx", context->Rcx}, {"rdx", context->Rdx}, {"rsi", context->Rsi}, {"rdi", context->Rdi}, {"r12", context->R12}, {"r13", context->R13}, {"r14", context->R14}, {"r15", context->R15}}; + for (const auto& [name, value] : pointers) { + if (value < 0x10000 || (value & 7) != 0 || !IsReadable(value) || !IsReadable(value + 0x38)) continue; + const auto* words = reinterpret_cast(value); + Report(" [%s] %016llx %016llx %016llx %016llx %016llx %016llx %016llx %016llx\n", name, words[0], words[1], words[2], words[3], words[4], words[5], words[6], words[7]); + } + } + Report(" frame pointer chain:\n"); + std::uint64_t frame = context->Rbp; + for (int depth = 0; depth < 32 && frame != 0 && (frame & 7) == 0 && IsReadable(frame) && IsReadable(frame + 8); ++depth) { + const auto returnAddress = reinterpret_cast(frame)[1]; + if (!IsExecutable(returnAddress)) break; + DescribeAddress(returnAddress, line, sizeof(line)); + Report(" #%d %s\n", depth, line); + const auto next = reinterpret_cast(frame)[0]; + if (next <= frame) break; + frame = next; + } + Report(" stack return address candidates:\n"); + int printed = 0; + for (std::uint64_t slot = context->Rsp; printed < 16 && slot < context->Rsp + 0x2000; slot += 8) { + if (!IsReadable(slot)) break; + const auto value = *reinterpret_cast(slot); + if (!IsExecutable(value)) continue; + DescribeAddress(value, line, sizeof(line)); + Report(" [rsp+0x%llx] %s\n", static_cast(slot - context->Rsp), line); + ++printed; + } + if (context->Rip == 0 || !IsExecutable(context->Rip)) { + // A jump into nothing usually follows a return through a clobbered frame; the frames that just + // returned are still below rsp. + Report(" recently popped return address candidates:\n"); + for (std::uint64_t slot = context->Rsp - 8; slot >= context->Rsp - 0x800; slot -= 8) { + if (!IsReadable(slot)) break; + const auto value = *reinterpret_cast(slot); + if (!IsExecutable(value)) continue; + DescribeAddress(value, line, sizeof(line)); + Report(" [rsp-0x%llx] %s\n", static_cast(context->Rsp - slot), line); + } + // Registers often still point into the stack the code ran on before the bad return. + const std::pair registers[] = {{"rdi", context->Rdi}, {"rsi", context->Rsi}, {"rbx", context->Rbx}, {"r12", context->R12}, {"r13", context->R13}, {"r14", context->R14}, {"r15", context->R15}}; + for (const auto& [name, value] : registers) { + if (value < 0x10000 || (value & 7) != 0 || !IsReadable(value)) continue; + Report(" code addresses above %s (0x%llx):\n", name, static_cast(value)); + int found = 0; + for (std::uint64_t slot = value; found < 24 && slot < value + 0x1000; slot += 8) { + if (!IsReadable(slot)) break; + const auto candidate = *reinterpret_cast(slot); + if (!IsExecutable(candidate)) continue; + DescribeAddress(candidate, line, sizeof(line)); + Report(" [+0x%llx] %s\n", static_cast(slot - value), line); + ++found; + } + } + } + std::fflush(stderr); + return EXCEPTION_CONTINUE_SEARCH; +} + +// abort() and the UCRT's invalid-parameter path end the process with a silent fast fail +// (0xc0000409); both are reported with the calling thread's stack first. +void ReportBacktrace(const char* what) { + void* frames[48]; + const auto count = RtlCaptureStackBackTrace(0, 48, frames, nullptr); + Report("FATAL: %s on thread %lu\n", what, static_cast(GetCurrentThreadId())); + char line[256]; + for (USHORT i = 0; i < count; ++i) { + DescribeAddress(reinterpret_cast(frames[i]), line, sizeof(line)); + Report(" #%u %s\n", static_cast(i), line); + } + std::fflush(stderr); +} + +void AbortSignalHandler(int) { + ReportBacktrace("abort() (SIGABRT)"); +} + +void InvalidParameterHandler(const wchar_t*, const wchar_t*, const wchar_t*, unsigned, std::uintptr_t) { + ReportBacktrace("UCRT invalid parameter"); +} + +// Static destruction runs on the thread that ends the process (ExitProcess -> DLL_PROCESS_DETACH), +// so its stack names the exit's caller when no libc exit/abort path was taken. +struct ExitReporter { + ~ExitReporter() { ReportBacktrace("process exit (static destruction)"); } +} g_exitReporter; + +const bool g_crashReportInstalled = [] { + g_sse4aEmulation = !IsEnvironmentSet("APS5_NO_SSE4A_EMULATION"); + g_sse4aTrace = IsEnvironmentSet("APS5_TRACE_SSE4A"); + AddVectoredExceptionHandler(1, ReportCrash); + std::signal(SIGABRT, AbortSignalHandler); + _set_invalid_parameter_handler(InvalidParameterHandler); + InstallWatch(); + return true; +}(); + +} +#endif diff --git a/core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp b/core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp new file mode 100644 index 0000000000..166b7754be --- /dev/null +++ b/core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp @@ -0,0 +1,136 @@ +#pragma once +// Emulation of the AMD SSE4a bit-field instructions EXTRQ and INSERTQ on a thread CONTEXT. +// +// The PS5's Zen 2 CPU has SSE4a and titles use it (Demon's Souls packs bytes with vpinsrb + insertq), +// but Intel hosts raise STATUS_ILLEGAL_INSTRUCTION. The crash reporter's vectored handler decodes the +// faulting bytes with this header, performs the operation on the CONTEXT's XMM registers and resumes. +// Kept header-only and free of process state so a host test can drive it with crafted CONTEXTs. +// +// Encodings (AMD64 Architecture Programmer's Manual vol. 4); only register-direct ModRM forms exist: +// EXTRQ xmm1, imm8(length), imm8(index) 66 [REX] 0F 78 /0 ib ib +// EXTRQ xmm1, xmm2 66 [REX] 0F 79 /r (length = xmm2[5:0], index = xmm2[13:8]) +// INSERTQ xmm1, xmm2, imm8(length), imm8(index) F2 [REX] 0F 78 /r ib ib +// INSERTQ xmm1, xmm2 F2 [REX] 0F 79 /r (length = xmm2[69:64], index = xmm2[77:72]) +// A length of 0 means 64. EXTRQ zero-extends the extracted field into xmm1[63:0]; INSERTQ writes the +// low `length` bits of xmm2[63:0] into xmm1[63:0] at bit `index`, keeping the bits outside the field. +// xmm1[127:64] is undefined after both and is left untouched here. +#ifdef _WIN32 +#include +#include +#include + +namespace sse4a { + +enum class Op : std::uint8_t { Extrq, Insertq }; + +struct Instruction { + Op op = Op::Extrq; + bool registerForm = false; // 0F 79: length and index come from the source register + unsigned destination = 0; // xmm register number 0-15 (xmm1 in the manual) + unsigned source = 0; // xmm register number 0-15 (xmm2; equals destination for EXTRQ imm) + std::uint8_t length = 0; // raw imm8 (immediate forms only) + std::uint8_t index = 0; // raw imm8 (immediate forms only) + std::size_t size = 0; // encoded length in bytes (4 to 7) +}; + +// The effective field after masking and the length-0-means-64 rule. +struct Field { + unsigned length = 0; + unsigned index = 0; +}; + +constexpr std::size_t kMaxInstructionSize = 7; // prefix, REX, 0F, 78, ModRM, imm8, imm8 + +inline bool Decode(const std::uint8_t* bytes, std::size_t available, Instruction& out) { + std::size_t cursor = 0; + if (available < 4) return false; + const std::uint8_t prefix = bytes[cursor++]; + if (prefix != 0x66 && prefix != 0xf2) return false; + std::uint8_t rex = 0; + if ((bytes[cursor] & 0xf0) == 0x40) rex = bytes[cursor++]; + if (cursor + 3 > available) return false; + if (bytes[cursor++] != 0x0f) return false; + const std::uint8_t opcode = bytes[cursor++]; + if (opcode != 0x78 && opcode != 0x79) return false; + const std::uint8_t modrm = bytes[cursor++]; + if ((modrm >> 6) != 3) return false; // register-direct only + const unsigned regField = (modrm >> 3) & 7; + const unsigned reg = regField | ((rex & 0x4) ? 8u : 0u); + const unsigned rm = (modrm & 7) | ((rex & 0x1) ? 8u : 0u); + Instruction result; + result.op = prefix == 0x66 ? Op::Extrq : Op::Insertq; + result.registerForm = opcode == 0x79; + if (opcode == 0x78) { + if (cursor + 2 > available) return false; + result.length = bytes[cursor++]; + result.index = bytes[cursor++]; + if (result.op == Op::Extrq) { + if (regField != 0) return false; // /0: the reg field is an opcode extension + result.destination = result.source = rm; + } else { + result.destination = reg; + result.source = rm; + } + } else { + result.destination = reg; + result.source = rm; + } + result.size = cursor; + out = result; + return true; +} + +inline M128A& Register(CONTEXT& context, unsigned number) { + return context.FltSave.XmmRegisters[number & 15]; +} + +inline Field Resolve(const Instruction& instruction, const CONTEXT& context) { + Field field; + if (instruction.registerForm) { + const M128A& source = context.FltSave.XmmRegisters[instruction.source & 15]; + const auto control = instruction.op == Op::Extrq ? static_cast(source.Low) : static_cast(source.High); + field.length = static_cast(control & 0x3f); + field.index = static_cast((control >> 8) & 0x3f); + } else { + field.length = instruction.length & 0x3f; + field.index = instruction.index & 0x3f; + } + if (field.length == 0) field.length = 64; + return field; +} + +inline std::uint64_t FieldMask(unsigned length) { + return length >= 64 ? ~std::uint64_t{0} : ((std::uint64_t{1} << length) - 1); +} + +// Applies the decoded instruction to the CONTEXT's XMM registers (Rip is not touched). +inline Field Execute(const Instruction& instruction, CONTEXT& context) { + const Field field = Resolve(instruction, context); + const std::uint64_t mask = FieldMask(field.length); + M128A& destination = Register(context, instruction.destination); + const M128A& source = Register(context, instruction.source); + const auto low = static_cast(destination.Low); + if (instruction.op == Op::Extrq) { + destination.Low = (low >> field.index) & mask; + } else { + const std::uint64_t hole = mask << field.index; + const std::uint64_t bits = (static_cast(source.Low) & mask) << field.index; + destination.Low = (low & ~hole) | bits; + } + return field; +} + +// Emulates the EXTRQ / INSERTQ at `bytes` on `context` and advances Rip past it. Returns false, leaving +// the context untouched, when the bytes are not one of those instructions. +inline bool Emulate(const std::uint8_t* bytes, std::size_t available, CONTEXT& context, Instruction* decoded = nullptr, Field* field = nullptr) { + Instruction instruction; + if (!Decode(bytes, available, instruction)) return false; + const Field resolved = Execute(instruction, context); + context.Rip += instruction.size; + if (decoded) *decoded = instruction; + if (field) *field = resolved; + return true; +} + +} +#endif diff --git a/core/libs/prx/libkernel/Apr/include/AprCommandBuffer.hpp b/core/libs/prx/libkernel/Apr/include/AprCommandBuffer.hpp new file mode 100644 index 0000000000..cb88f48f48 --- /dev/null +++ b/core/libs/prx/libkernel/Apr/include/AprCommandBuffer.hpp @@ -0,0 +1,55 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_APR_APRCOMMANDBUFFER_HPP +#define CORE_LIBS_PRX_LIBKERNEL_APR_APRCOMMANDBUFFER_HPP + +#include +#include + +// Shared between libSceAmpr (which records commands) and libkernel (which executes them). +// The guest treats both the command buffer object and its memory as opaque, so the encoding is ours. +namespace Apr { + +enum class BufferType : std::uint32_t { + Generic = 0, + Apr = 1, +}; + +struct CommandBufferObject { + std::uint8_t* base; + std::uint32_t size; + std::uint32_t offset; + std::uint32_t numCommands; + BufferType type; +}; +static_assert(sizeof(CommandBufferObject) == 0x18, "guest reserves 0x18 bytes for sce::Ampr::CommandBuffer"); + +enum class Opcode : std::uint32_t { + Nop = 0, + ReadFile = 1, + WriteAddress = 2, +}; + +struct CommandHeader { + Opcode opcode; + std::uint32_t bytes; +}; + +struct ReadFileCommand { + CommandHeader header; + std::uint32_t fileId; + std::uint32_t reserved; + std::uint64_t destination; + std::uint64_t size; + std::uint64_t offset; +}; + +struct WriteAddressCommand { + CommandHeader header; + std::uint64_t address; + std::uint64_t value; + std::uint32_t flags; + std::uint32_t reserved; +}; + +} + +#endif diff --git a/core/libs/prx/libkernel/Apr/src/Apr.cpp b/core/libs/prx/libkernel/Apr/src/Apr.cpp new file mode 100644 index 0000000000..8d25f8f30b --- /dev/null +++ b/core/libs/prx/libkernel/Apr/src/Apr.cpp @@ -0,0 +1,196 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" +#include "prx/libkernel/Apr/include/AprCommandBuffer.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#endif + +namespace { + +constexpr int GUEST_ENOENT = 2; +constexpr int GUEST_EINVAL = 22; + +struct AprFile { + std::filesystem::path path; + std::uint64_t size; +}; + +std::mutex g_filesLock; +std::vector g_files; +std::unordered_map g_idsByPath; + +// File sizes by directory, listed on first use. Titles resolve thousands of package paths at startup, +// and one size query per path took over 20 s; a listing costs one directory read. Names are compared +// case-insensitively, like the host file system. +std::mutex g_directoriesLock; +std::unordered_map> g_directories; + +std::string _foldCase(std::string text) { + for (auto& character : text) { + if (character >= 'A' && character <= 'Z') character = static_cast(character - 'A' + 'a'); + } + return text; +} + +bool _fileSize(const std::filesystem::path& path, std::uint64_t& bytes) { + { + std::lock_guard lock(g_directoriesLock); + const auto directory = _foldCase(path.parent_path().string()); + auto listed = g_directories.find(directory); + if (listed == g_directories.end()) { + std::unordered_map sizes; +#ifdef _WIN32 + // The find data carries each size; std::filesystem would query every entry again. + WIN32_FIND_DATAW entry{}; + const HANDLE find = FindFirstFileExW((path.parent_path() / L"*").c_str(), FindExInfoBasic, &entry, FindExSearchNameMatch, nullptr, FIND_FIRST_EX_LARGE_FETCH); + if (find != INVALID_HANDLE_VALUE) { + do { + if ((entry.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) continue; + const auto size = (static_cast(entry.nFileSizeHigh) << 32u) | entry.nFileSizeLow; + sizes.emplace(_foldCase(std::filesystem::path(entry.cFileName).string()), size); + } while (FindNextFileW(find, &entry)); + FindClose(find); + } +#else + std::error_code error; + for (std::filesystem::directory_iterator it(path.parent_path(), error), end; !error && it != end; it.increment(error)) { + std::error_code entryError; + if (!it->is_regular_file(entryError)) continue; + const auto size = it->file_size(entryError); + if (!entryError) sizes.emplace(_foldCase(it->path().filename().string()), size); + } +#endif + listed = g_directories.emplace(directory, std::move(sizes)).first; + } + if (const auto file = listed->second.find(_foldCase(path.filename().string())); file != listed->second.end()) { + bytes = file->second; + return true; + } + } + // Not in the listing (created since, or an unusual name): ask the file system directly. + std::error_code error; + bytes = std::filesystem::file_size(path, error); + return !error; +} + +int _fail(int guestErrno) { + errno = guestErrno; + return -1; +} + +bool _resolve(const char* guestPath, std::uint32_t* id, std::uint64_t* size) { + if (!guestPath) return false; + auto hostPath = ResolvePath_nid_no_patch(guestPath); + std::uint64_t bytes = 0; + if (!_fileSize(hostPath, bytes)) return false; + std::lock_guard lock(g_filesLock); + const auto key = hostPath.string(); + auto found = g_idsByPath.find(key); + if (found == g_idsByPath.end()) { + found = g_idsByPath.emplace(key, static_cast(g_files.size())).first; + g_files.push_back({std::move(hostPath), bytes}); + } + if (id) *id = found->second; + if (size) *size = bytes; + return true; +} + +AprFile _file(std::uint32_t id) { + std::lock_guard lock(g_filesLock); + if (id >= g_files.size()) throw std::runtime_error("APR: unknown file id " + std::to_string(id)); + return g_files[id]; +} + +void _readFile(const Apr::ReadFileCommand& command) { + const auto file = _file(command.fileId); + static const bool trace = std::getenv("APS5_TRACE_APR") != nullptr; + if (trace) std::fprintf(stderr, "[apr] read %s offset=0x%llx size=0x%llx -> 0x%llx\n", file.path.string().c_str(), static_cast(command.offset), static_cast(command.size), static_cast(command.destination)); + std::ifstream stream(file.path, std::ios::binary); + if (!stream) throw std::runtime_error("APR: cannot open " + file.path.string()); + stream.seekg(static_cast(command.offset)); + stream.read(reinterpret_cast(command.destination), static_cast(command.size)); + if (stream.bad()) throw std::runtime_error("APR: read failed for " + file.path.string()); +} + +void _writeAddress(const Apr::WriteAddressCommand& command) { + if (command.flags != 0) throw std::runtime_error("APR: WriteAddress flags " + std::to_string(command.flags) + " not implemented"); + std::atomic_ref(*reinterpret_cast(command.address)).store(command.value, std::memory_order_release); +} + +void _execute(const Apr::CommandBufferObject& buffer) { + std::uint32_t cursor = 0; + for (std::uint32_t index = 0; index < buffer.numCommands; ++index) { + if (cursor + sizeof(Apr::CommandHeader) > buffer.offset) throw std::runtime_error("APR: truncated command buffer"); + Apr::CommandHeader header; + std::memcpy(&header, buffer.base + cursor, sizeof(header)); + if (header.bytes < sizeof(header) || cursor + header.bytes > buffer.offset) throw std::runtime_error("APR: malformed command"); + switch (header.opcode) { + case Apr::Opcode::Nop: + break; + case Apr::Opcode::ReadFile: { + Apr::ReadFileCommand command; + std::memcpy(&command, buffer.base + cursor, sizeof(command)); + _readFile(command); + break; + } + case Apr::Opcode::WriteAddress: { + Apr::WriteAddressCommand command; + std::memcpy(&command, buffer.base + cursor, sizeof(command)); + _writeAddress(command); + break; + } + default: + throw std::runtime_error("APR: unknown opcode " + std::to_string(static_cast(header.opcode))); + } + cursor += header.bytes; + } +} + +} + +extern "C" { + +int APS5_VABI sceKernelAprResolveFilepathsToIds(const char** paths, uint32_t count, uint32_t* ids, uint32_t* error_index) { + if (!paths || !ids) return _fail(GUEST_EINVAL); + for (uint32_t index = 0; index < count; ++index) { + if (!_resolve(paths[index], &ids[index], nullptr)) { + if (error_index) *error_index = index; + return _fail(GUEST_ENOENT); + } + } + return 0; +} + +int APS5_VABI sceKernelAprResolveFilepathsToIdsAndFileSizes(const char** paths, uint32_t count, uint32_t* ids, uint64_t* sizes, uint32_t* error_index) { + if (!paths || !ids || !sizes) return _fail(GUEST_EINVAL); + for (uint32_t index = 0; index < count; ++index) { + if (!_resolve(paths[index], &ids[index], &sizes[index])) { + if (error_index) *error_index = index; + return _fail(GUEST_ENOENT); + } + } + return 0; +} + +int APS5_VABI sceKernelAprSubmitCommandBuffer(const Apr::CommandBufferObject* buffer, uint32_t priority) { + (void)priority; + if (!buffer || buffer->type != Apr::BufferType::Apr) return _fail(GUEST_EINVAL); + _execute(*buffer); + return 0; +} + +} diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 3593468597..cf688c1547 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -1,4 +1,6 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Apr/src/Apr.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Equeue/Equeue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Equeue/src/EventAccessors.cpp ${CMAKE_CURRENT_SOURCE_DIR}/File/src/NativeStat.cpp diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index 69be5eb95f..9ff2811c0e 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -1,7 +1,14 @@ -#include "prx/libkernel/DirectMemory/DirectMemory.hpp" +#include "DirectMemory.hpp" #include "prx/libc/include/GuestAllocations.hpp" -#include "prx/libc/include/GuestMemoryBacking.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -15,6 +22,10 @@ static constexpr int PROT_NONE = 0; static constexpr int PROT_READ = 1; static constexpr int PROT_WRITE = 2; static constexpr int PROT_EXEC = 4; +static constexpr int MAP_PRIVATE = 0x02; +static constexpr int MAP_ANONYMOUS = 0x20; +static constexpr int MAP_FIXED = 0x10; +static void* const MAP_FAILED = reinterpret_cast(-1); static DWORD WinProtFromPosix(int prot) { if (prot == PROT_NONE) return PAGE_NOACCESS; @@ -25,10 +36,92 @@ static DWORD WinProtFromPosix(int prot) { return PAGE_READONLY; } +// Guest mappings share libc's guest address space arena so they stay inside the PS5 map area. +struct KernelArena { + static KernelArena& Get() { + static KernelArena arena; + return arena; + } + bool Contains(const void* pointer, size_t len) const { return GuestArena::GuestArenaContains_nid_postfix(pointer, len); } + void* Allocate(size_t len, size_t alignment) { return GuestArena::GuestArenaAllocate_nid_postfix(len, alignment); } + void MarkUsed(const void* pointer, size_t len) { GuestArena::GuestArenaMarkUsed_nid_postfix(pointer, len); } + void Release(const void* pointer, size_t len) { GuestArena::GuestArenaRelease_nid_postfix(pointer, len); } +}; + +static void CommitArenaRange(void* addr, size_t len, DWORD winProt) { + if (!VirtualAlloc(addr, len, MEM_COMMIT, winProt)) + throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualAlloc commit failed"); +} + +static void* mmap_aligned(size_t len, int prot, size_t alignment) { + auto& arena = KernelArena::Get(); + void* result = arena.Allocate(len, alignment); + if (prot != PROT_NONE) { + try { + CommitArenaRange(result, len, WinProtFromPosix(prot)); + } catch (...) { + arena.Release(result, len); + throw; + } + } + return result; +} + +static void* mmap(void* addr, size_t len, int prot, int flags, int, int) { + DWORD winProt = WinProtFromPosix(prot); + if (flags & MAP_FIXED) { + auto& arena = KernelArena::Get(); + if (arena.Contains(addr, len)) { + arena.MarkUsed(addr, len); + if (prot != PROT_NONE) { + try { + CommitArenaRange(addr, len, winProt); + } catch (...) { + arena.Release(addr, len); + throw; + } + } + return addr; + } + void* result = VirtualAlloc(addr, len, MEM_RESERVE | MEM_COMMIT, winProt); + if (!result) throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualAlloc fixed failed"); + return result; + } + return mmap_aligned(len, prot, PS5_PAGE_SIZE); +} + +static int munmap(void* addr, size_t len) { + if (!VirtualFree(addr, len, MEM_DECOMMIT)) + throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualFree decommit failed"); + auto& arena = KernelArena::Get(); + if (arena.Contains(addr, len)) arena.Release(addr, len); + return 0; +} + +static int munmap_release(void* addr) { + if (!VirtualFree(addr, 0, MEM_RELEASE)) + throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualFree release failed"); + return 0; +} + static int mprotect(void* addr, size_t len, int prot) { - DWORD old; - if (!VirtualProtect(addr, len, WinProtFromPosix(prot), &old)) - throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualProtect failed"); + if (prot != PROT_NONE && KernelArena::Get().Contains(addr, len)) CommitArenaRange(addr, len, WinProtFromPosix(prot)); + auto cursor = reinterpret_cast(addr); + const auto end = cursor + len; + while (cursor < end) { + MEMORY_BASIC_INFORMATION memory{}; + if (VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) != sizeof(memory)) + throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualQuery failed"); + const auto regionEnd = std::min(end, reinterpret_cast(memory.BaseAddress) + memory.RegionSize); + if (memory.State == MEM_COMMIT) { + DWORD old; + if (!VirtualProtect(reinterpret_cast(cursor), regionEnd - cursor, WinProtFromPosix(prot), &old)) + throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualProtect failed"); + } else if (prot != PROT_NONE) { + throw std::runtime_error("mprotect of uncommitted memory outside the guest arena"); + } + cursor = regionEnd; + } return 0; } #endif @@ -61,9 +154,9 @@ void ValidateRange(const void* addr, size_t len, size_t alignment) { } int LinuxProtFromSce(int prot) { - if ((prot & ~0x37) != 0) { + if ((prot & ~0xF7) != 0) { // return SCE_KERNEL_ERROR_EINVAL; - throw std::invalid_argument("Unsupported memory protection bits"); + throw std::invalid_argument("Unsupported memory protection bits: " + std::to_string(prot)); } int result = PROT_NONE; if (prot & 1) result |= PROT_READ; @@ -72,19 +165,92 @@ int LinuxProtFromSce(int prot) { return result; } +bool TraceEnabled() { + static const bool enabled = std::getenv("APS5_TRACE_MEMORY") != nullptr; + return enabled; +} + +void Trace(const char* format, ...) { + if (!TraceEnabled()) return; + va_list args; + va_start(args, format); + std::fprintf(stderr, "[memory] "); + std::vfprintf(stderr, format, args); + std::fputc('\n', stderr); + va_end(args); +} + +bool RemapFixedIntoRegistered(GuestAllocations::Mutation& mutation, void* addr, size_t len, int prot, int flags, int64_t physStart = -1) { + constexpr int GuestMapFixed = 0x10; + if (addr == nullptr || (flags & GuestMapFixed) == 0 || !mutation.Covers(addr, len)) return false; + ValidateRange(addr, len, PS5_PAGE_SIZE); + Trace("remap fixed %p+0x%zx prot=0x%x phys=0x%llx", addr, len, prot, static_cast(physStart)); + const auto nativeProtection = LinuxProtFromSce(prot); + mutation.Protect(addr, len, (prot & 3) != 0, (prot & 2) != 0, [&] { +#ifdef _WIN32 + if (!VirtualAlloc(addr, len, MEM_COMMIT, WinProtFromPosix(nativeProtection))) throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualAlloc commit failed"); +#endif + if (mprotect(addr, len, nativeProtection) != 0) throw std::system_error(errno, std::generic_category(), "mprotect failed"); + }); + return true; +} + void Unmap(void* addr, size_t len) { - GuestMemoryBacking::GuestMemoryBackingUnmap_nid_postfix(addr, len); +#if defined(__linux__) + if (munmap(addr, len) != 0) throw std::system_error(errno, std::generic_category(), "munmap failed"); +#else + if (KernelArena::Get().Contains(addr, len)) munmap(addr, len); + else munmap_release(addr); +#endif } void* MapAligned(void* addr, size_t len, int prot, int flags, size_t alignment) { ValidateLength(len); alignment = ValidateAlignment(alignment); - constexpr int guestMapFixed = 0x10; - constexpr int guestMapNoCoalesce = 0x400000; - if ((flags & ~(guestMapFixed | guestMapNoCoalesce)) != 0) throw std::invalid_argument("Unsupported memory mapping flags"); - if ((flags & guestMapFixed) != 0) ValidateRange(addr, len, alignment); - else if (addr != nullptr) throw std::invalid_argument("Non-fixed mapping address hints are not implemented"); - return GuestMemoryBacking::GuestMemoryBackingMap_nid_postfix(addr, len, alignment, prot); + constexpr int GuestMapFixed = 0x10; + constexpr int GuestMapNoCoalesce = 0x400000; + if ((flags & ~(GuestMapFixed | GuestMapNoCoalesce)) != 0) { + throw std::invalid_argument("Unsupported memory mapping flags"); + } + if ((flags & GuestMapFixed) != 0) { + ValidateRange(addr, len, alignment); + void* result = mmap(addr, len, prot, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (result == MAP_FAILED) { + // return SCE_KERNEL_ERROR_ENOMEM; + throw std::system_error(errno, std::generic_category(), "Fixed mmap failed"); + } + return result; + } + if (addr) { + throw std::invalid_argument("Non-fixed mapping address hints are not implemented"); + } +#ifdef _WIN32 + return mmap_aligned(len, prot, alignment); +#endif + if (len > std::numeric_limits::max() - alignment) { + throw std::overflow_error("Aligned mapping size overflow"); + } + const size_t allocLen = len + alignment; + void* result = mmap(nullptr, allocLen, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (result == MAP_FAILED) { + // return SCE_KERNEL_ERROR_ENOMEM; + throw std::system_error(errno, std::generic_category(), "Aligned mmap failed"); + } + const auto raw = reinterpret_cast(result); + const size_t prefix = (alignment - (raw & (alignment - 1))) & (alignment - 1); + void* aligned = reinterpret_cast(raw + prefix); + const size_t suffix = allocLen - prefix - len; + if (prefix != 0 && munmap(result, prefix) != 0) { + const int error = errno; + Unmap(result, allocLen); + throw std::system_error(error, std::generic_category(), "Mapping prefix munmap failed"); + } + if (suffix != 0 && munmap(reinterpret_cast(raw + prefix + len), suffix) != 0) { + const int error = errno; + Unmap(aligned, allocLen - prefix); + throw std::system_error(error, std::generic_category(), "Mapping suffix munmap failed"); + } + return aligned; } void ValidateOutput(void** addr) { @@ -103,6 +269,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, return SCE_KERNEL_ERROR_EINVAL; } GuestAllocations::Mutation mutation; + if (RemapFixedIntoRegistered(mutation, *addr, len, prot, flags)) return 0; if (*addr != nullptr) mutation.RequireAvailable(*addr, len); void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, alignment); try { @@ -112,6 +279,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, throw; } *addr = mapped; + Trace("map direct %p+0x%zx phys=0x%llx prot=0x%x flags=0x%x align=0x%zx", mapped, len, static_cast(physStart), prot, flags, alignment); return 0; } @@ -119,6 +287,7 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags) { ValidateOutput(addr); if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL; GuestAllocations::Mutation mutation; + if (RemapFixedIntoRegistered(mutation, *addr, len, prot, flags)) return 0; if (*addr != nullptr) mutation.RequireAvailable(*addr, len); void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, PS5_PAGE_SIZE); try { @@ -128,10 +297,12 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags) { throw; } *addr = mapped; + Trace("map anon %p+0x%zx prot=0x%x flags=0x%x", mapped, len, prot, flags); return 0; } int DoMprotect(const void* addr, size_t len, int prot) { + Trace("protect %p+0x%zx prot=0x%x", addr, len, prot); const auto address = reinterpret_cast(addr); constexpr auto pageMask = static_cast(PS5_PAGE_SIZE - 1); const auto limit = std::numeric_limits::max(); @@ -157,10 +328,17 @@ int DoMprotect(const void* addr, size_t len, int prot) { } int DoMunmap(void* addr, size_t len) { + Trace("unmap %p+0x%zx", addr, len); if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0 || !addr) return SCE_KERNEL_ERROR_EINVAL; GuestAllocations::Mutation mutation; - mutation.Unmap(addr, len, [&](const void*, bool) { + mutation.Unmap(addr, len, [&](const void* allocation, bool last) { +#if defined(__linux__) Unmap(addr, len); +#else + if (KernelArena::Get().Contains(addr, len)) munmap(addr, len); + else if (last) munmap_release(const_cast(allocation)); + else munmap(addr, len); +#endif }); return 0; } @@ -177,5 +355,6 @@ int DoReserveVirtual(void** addr, size_t len, size_t alignment) { throw; } *addr = mapped; + Trace("reserve %p+0x%zx align=0x%zx", mapped, len, alignment); return 0; } diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 0ef1a9b988..fd263732aa 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -3,7 +3,68 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/GuestAllocations.hpp" #include "DirectMemory.hpp" +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t FLEXIBLE_MEMORY_SIZE = 448ULL * 1024 * 1024; + +constexpr int PRT_APERTURE_COUNT = 3; + +struct PrtAperture { + void* address; + size_t length; +}; + +std::mutex g_prtLock; +PrtAperture g_prtApertures[PRT_APERTURE_COUNT] = {}; + +std::mutex g_flexibleLock; +std::map g_flexibleRanges; + +size_t _flexibleUsedLocked() { + size_t used = 0; + for (const auto& [start, len] : g_flexibleRanges) used += len; + return used; +} + +int _mapFlexible(void** addr, size_t len, int prot, int flags) { + std::lock_guard lock(g_flexibleLock); + if (len > FLEXIBLE_MEMORY_SIZE - std::min(FLEXIBLE_MEMORY_SIZE, _flexibleUsedLocked())) { + std::fprintf(stderr, "[memory] flexible memory exhausted: request 0x%zx with 0x%zx of 0x%zx in use\n", len, _flexibleUsedLocked(), static_cast(FLEXIBLE_MEMORY_SIZE)); + return SCE_KERNEL_ERROR_ENOMEM; + } + const int result = DoMapAnon(addr, len, prot, flags); + if (result == 0) g_flexibleRanges[reinterpret_cast(*addr)] = len; + return result; +} + +void _releaseFlexible(uintptr_t start, size_t len) { + std::lock_guard lock(g_flexibleLock); + const uintptr_t end = start + len; + auto it = g_flexibleRanges.upper_bound(start); + if (it != g_flexibleRanges.begin()) --it; + while (it != g_flexibleRanges.end() && it->first < end) { + const uintptr_t rangeStart = it->first; + const uintptr_t rangeEnd = rangeStart + it->second; + if (rangeEnd <= start) { ++it; continue; } + it = g_flexibleRanges.erase(it); + if (rangeStart < start) g_flexibleRanges[rangeStart] = start - rangeStart; + if (rangeEnd > end) g_flexibleRanges[end] = rangeEnd - end; + } +} + +} extern "C" { @@ -58,7 +119,7 @@ int APS5_VABI sceKernelMapDirectMemory2(void** addr, size_t len, int type, int p } int APS5_VABI sceKernelMapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags) { - return DoMapAnon(addr_in_out, len, prot, flags); + return _mapFlexible(addr_in_out, len, prot, flags); } int APS5_VABI sceKernelMapNamedDirectMemory(void** addr, size_t len, int prot, int flags, int64_t direct_memory_start, size_t alignment, const char* name) { @@ -68,7 +129,7 @@ int APS5_VABI sceKernelMapNamedDirectMemory(void** addr, size_t len, int prot, i int32_t APS5_VABI sceKernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags, const char* name) { (void)name; - return DoMapAnon(addr_in_out, len, prot, flags); + return _mapFlexible(addr_in_out, len, prot, flags); } int APS5_VABI sceKernelMprotect(const void* addr, size_t len, int prot) { @@ -76,7 +137,9 @@ int APS5_VABI sceKernelMprotect(const void* addr, size_t len, int prot) { } int APS5_VABI sceKernelMunmap(uint64_t vaddr, size_t len) { - return DoMunmap(reinterpret_cast(vaddr), len); + const int result = DoMunmap(reinterpret_cast(vaddr), len); + if (result == 0) _releaseFlexible(static_cast(vaddr), len); + return result; } int APS5_VABI sceKernelReleaseDirectMemory(int64_t start, size_t len) { @@ -91,14 +154,40 @@ int APS5_VABI sceKernelReserveVirtualRange(void** addr, size_t len, int flags, s } int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, uint64_t info_size) { - (void)flags; if (!info || info_size < sizeof(VirtualQueryInfo)) return SCE_KERNEL_ERROR_EINVAL; memset(info, 0, sizeof(VirtualQueryInfo)); - uintptr_t ptr = reinterpret_cast(addr); - info->start = ptr & ~static_cast(PS5_PAGE_SIZE - 1); - info->end = info->start + PS5_PAGE_SIZE; - info->is_direct = 1; - info->protection = 3; + const auto address = reinterpret_cast(addr); + // The mapping that contains the address, or with SCE_KERNEL_VQ_FIND_NEXT (flags bit 0) the first + // mapping at or above it: titles walk their mappings and check that a mapping covers a whole + // allocation, so the answer must be the registered allocation, not a page. + constexpr int findNext = 1; + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + const GuestAllocations::Range* best = nullptr; + for (const auto& range : lease) { + const auto begin = range->allocationAddress; + const auto end = begin + range->allocationBytes; + if (address >= begin && address < end) { best = range.get(); break; } + if ((flags & findNext) != 0 && begin > address && (best == nullptr || begin < best->allocationAddress)) best = range.get(); + } + if (best != nullptr) { + info->start = best->allocationAddress; + info->end = best->allocationAddress + best->allocationBytes; + info->protection = (best->readable ? 1 : 0) | (best->writable ? 2 : 0) | (!best->releasable ? 4 : 0); + info->is_direct = best->releasable ? 1u : 0u; + info->is_committed = 1; + return 0; + } + // Memory the registry does not know (the title's own heap blocks, stacks): the host's committed + // region around the address is the honest extent; unmapped memory is an error, as on the PS5. + MEMORY_BASIC_INFORMATION host{}; + if (VirtualQuery(addr, &host, sizeof(host)) == 0 || host.State != MEM_COMMIT) return SCE_KERNEL_ERROR_EACCES; + info->start = reinterpret_cast(host.BaseAddress); + info->end = info->start + host.RegionSize; + const bool writable = (host.Protect & (PAGE_READWRITE | PAGE_EXECUTE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_WRITECOPY)) != 0; + const bool executable = (host.Protect & (PAGE_EXECUTE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY)) != 0; + info->protection = 1 | (writable ? 2 : 0) | (executable ? 4 : 0); + info->is_flexible = 1; + info->is_committed = 1; return 0; } @@ -140,14 +229,15 @@ int APS5_VABI sceKernelIsStack(void* addr, void** start, void** end) { } int APS5_VABI sceKernelAvailableFlexibleMemorySize(size_t* size) { - (void)size; - NotImplemented_nid_no_patch(__func__); + if (!size) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(g_flexibleLock); + *size = FLEXIBLE_MEMORY_SIZE - std::min(FLEXIBLE_MEMORY_SIZE, _flexibleUsedLocked()); return 0; } int APS5_VABI sceKernelConfiguredFlexibleMemorySize(size_t* size) { - (void)size; - NotImplemented_nid_no_patch(__func__); + if (!size) return SCE_KERNEL_ERROR_EINVAL; + *size = FLEXIBLE_MEMORY_SIZE; return 0; } @@ -169,36 +259,59 @@ int APS5_VABI sceKernelGetPageTableStats(int* cpu_total, int* cpu_available, int } int APS5_VABI sceKernelGetPrtAperture(int index, void** addr, size_t* len) { - (void)index; - (void)addr; - (void)len; - NotImplemented_nid_no_patch(__func__); + if (index < 0 || index >= PRT_APERTURE_COUNT || !addr || !len) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(g_prtLock); + *addr = g_prtApertures[index].address; + *len = g_prtApertures[index].length; return 0; } int APS5_VABI sceKernelSetPrtAperture(int index, void* addr, size_t len) { - (void)index; - (void)addr; - (void)len; - NotImplemented_nid_no_patch(__func__); + if (index < 0 || index >= PRT_APERTURE_COUNT) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(g_prtLock); + g_prtApertures[index] = {addr, len}; return 0; } +int APS5_VABI sceKernelBatchMap2(KernelBatchMapEntry* entries, int num_entries, int* num_entries_out, int flags); + int APS5_VABI sceKernelBatchMap(KernelBatchMapEntry* entries, int num_entries, int* num_entries_out) { - (void)entries; - (void)num_entries; - (void)num_entries_out; - NotImplemented_nid_no_patch(__func__); - return 0; + constexpr int MapFixed = 0x10; + return sceKernelBatchMap2(entries, num_entries, num_entries_out, MapFixed); } int APS5_VABI sceKernelBatchMap2(KernelBatchMapEntry* entries, int num_entries, int* num_entries_out, int flags) { - (void)entries; - (void)num_entries; - (void)num_entries_out; - (void)flags; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!entries || num_entries < 0) return SCE_KERNEL_ERROR_EINVAL; + constexpr int OpMapDirect = 0; + constexpr int OpUnmap = 1; + constexpr int OpProtect = 2; + constexpr int OpMapFlexible = 3; + constexpr int OpTypeProtect = 4; + int processed = 0; + int result = 0; + for (; processed < num_entries; ++processed) { + auto& entry = entries[processed]; + switch (entry.operation) { + case OpMapDirect: + result = DoMapDirect(&entry.start, entry.length, static_cast(entry.protection), flags, static_cast(entry.offset), 0); + break; + case OpUnmap: + result = sceKernelMunmap(reinterpret_cast(entry.start), entry.length); + break; + case OpProtect: + case OpTypeProtect: + result = DoMprotect(entry.start, entry.length, static_cast(entry.protection)); + break; + case OpMapFlexible: + result = _mapFlexible(&entry.start, entry.length, static_cast(entry.protection), flags); + break; + default: + throw std::invalid_argument("sceKernelBatchMap2: unsupported operation " + std::to_string(entry.operation)); + } + if (result != 0) break; + } + if (num_entries_out) *num_entries_out = processed; + return result; } } diff --git a/core/libs/prx/libkernel/Equeue/Equeue.cpp b/core/libs/prx/libkernel/Equeue/Equeue.cpp index 3b9b49b276..4ccab694a1 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.cpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.cpp @@ -1,4 +1,5 @@ #include "Equeue.hpp" +#include "prx/libkernel/Time/include/Time.hpp" #include #include @@ -311,6 +312,7 @@ int APS5_VABI sceKernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, int if (num < 1 || out == nullptr) { return EQUEUE_ERROR_EINVAL; } + const auto waitStart = std::chrono::steady_clock::now(); if (timo == nullptr) { *out = owner->WaitForEvents(ev, num, 0); } else if (*timo == 0) { @@ -318,6 +320,9 @@ int APS5_VABI sceKernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, int } else { *out = owner->WaitForEvents(ev, num, *timo); } + if (timo == nullptr || *timo != 0) { + KernelTraceWait_nid_postfix("equeue", __builtin_return_address(0), static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), *out == 0); + } if (*out == EQUEUE_ERROR_EBADF) { return EQUEUE_ERROR_EBADF; } diff --git a/core/libs/prx/libkernel/EventFlag/src/EventFlag.cpp b/core/libs/prx/libkernel/EventFlag/src/EventFlag.cpp index a08cca7ab1..9706587659 100644 --- a/core/libs/prx/libkernel/EventFlag/src/EventFlag.cpp +++ b/core/libs/prx/libkernel/EventFlag/src/EventFlag.cpp @@ -2,64 +2,176 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/Time/include/Time.hpp" +#include +#include +#include +#include +#include +#include + +static constexpr int SCE_OK = 0; +static constexpr int SCE_KERNEL_ERROR_EPERM = 0x80020001; +static constexpr int SCE_KERNEL_ERROR_ESRCH = 0x80020003; +static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; +static constexpr int SCE_KERNEL_ERROR_EACCES = 0x8002000D; +static constexpr int SCE_KERNEL_ERROR_EBUSY = 0x80020010; +static constexpr int SCE_KERNEL_ERROR_EINVAL = 0x80020016; +static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = 0x8002003C; +static constexpr int SCE_KERNEL_ERROR_ECANCELED = 0x80020055; + +static constexpr uint32_t EVF_ATTR_SINGLE = 0x10; +static constexpr uint32_t EVF_ATTR_MULTI = 0x20; +static constexpr uint32_t EVF_WAITMODE_AND = 0x01; +static constexpr uint32_t EVF_WAITMODE_OR = 0x02; +static constexpr uint32_t EVF_WAITMODE_CLEAR_ALL = 0x10; +static constexpr uint32_t EVF_WAITMODE_CLEAR_PAT = 0x20; + +struct KernelEventFlagPrivate { + std::mutex _mutex; + std::condition_variable _changed; + std::string _name; + uint64_t _pattern = 0; + bool _multipleWaiters = false; + bool _deleted = false; + int _waiters = 0; + uint64_t _cancelGeneration = 0; +}; + +static bool TraceSync() { + static const bool enabled = std::getenv("APS5_TRACE_SYNC") != nullptr; + return enabled; +} + +static bool IsValidWaitMode(uint32_t waitMode) { + const uint32_t match = waitMode & (EVF_WAITMODE_AND | EVF_WAITMODE_OR); + const uint32_t clear = waitMode & (EVF_WAITMODE_CLEAR_ALL | EVF_WAITMODE_CLEAR_PAT); + if ((waitMode & ~(EVF_WAITMODE_AND | EVF_WAITMODE_OR | EVF_WAITMODE_CLEAR_ALL | EVF_WAITMODE_CLEAR_PAT)) != 0) return false; + return (match == EVF_WAITMODE_AND || match == EVF_WAITMODE_OR) && clear != (EVF_WAITMODE_CLEAR_ALL | EVF_WAITMODE_CLEAR_PAT); +} + +static bool IsSatisfied(uint64_t pattern, uint64_t bitPattern, uint32_t waitMode) { + return (waitMode & EVF_WAITMODE_AND) ? (pattern & bitPattern) == bitPattern : (pattern & bitPattern) != 0; +} + +static void Consume(KernelEventFlagPrivate* flag, uint64_t bitPattern, uint32_t waitMode, uint64_t* resultPattern) { + if (resultPattern) *resultPattern = flag->_pattern; + if (waitMode & EVF_WAITMODE_CLEAR_ALL) flag->_pattern = 0; + else if (waitMode & EVF_WAITMODE_CLEAR_PAT) flag->_pattern &= ~bitPattern; +} extern "C" { int APS5_VABI sceKernelCreateEventFlag(KernelEventFlag* ef, const char* name, uint32_t attr, uint64_t init_pattern, const void* param) { - (void)ef; - (void)name; - (void)attr; - (void)init_pattern; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)param; + if (!ef || !name) return SCE_KERNEL_ERROR_EINVAL; + if ((attr & EVF_ATTR_SINGLE) && (attr & EVF_ATTR_MULTI)) return SCE_KERNEL_ERROR_EINVAL; + auto* flag = new (std::nothrow) KernelEventFlagPrivate(); + if (!flag) return SCE_KERNEL_ERROR_ENOMEM; + flag->_name = name; + flag->_pattern = init_pattern; + flag->_multipleWaiters = (attr & EVF_ATTR_MULTI) != 0; + *ef = flag; + if (TraceSync()) std::fprintf(stderr, "[evf] create %p '%s' attr=0x%x init=0x%llx\n", static_cast(flag), name, attr, static_cast(init_pattern)); + return SCE_OK; } int APS5_VABI sceKernelDeleteEventFlag(KernelEventFlag ef) { - (void)ef; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!ef) return SCE_KERNEL_ERROR_ESRCH; + { + std::lock_guard lock(ef->_mutex); + if (ef->_deleted) return SCE_KERNEL_ERROR_ESRCH; + ef->_deleted = true; + if (ef->_waiters != 0) { + ef->_changed.notify_all(); + return SCE_OK; + } + } + delete ef; + return SCE_OK; } int APS5_VABI sceKernelSetEventFlag(KernelEventFlag ef, uint64_t bit_pattern) { - (void)ef; - (void)bit_pattern; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!ef) return SCE_KERNEL_ERROR_ESRCH; + std::lock_guard lock(ef->_mutex); + if (ef->_deleted) return SCE_KERNEL_ERROR_ESRCH; + ef->_pattern |= bit_pattern; + if (TraceSync()) std::fprintf(stderr, "[evf] set %p '%s' |=0x%llx -> 0x%llx\n", static_cast(ef), ef->_name.c_str(), static_cast(bit_pattern), static_cast(ef->_pattern)); + ef->_changed.notify_all(); + return SCE_OK; } int APS5_VABI sceKernelClearEventFlag(KernelEventFlag ef, uint64_t bit_pattern) { - (void)ef; - (void)bit_pattern; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!ef) return SCE_KERNEL_ERROR_ESRCH; + std::lock_guard lock(ef->_mutex); + if (ef->_deleted) return SCE_KERNEL_ERROR_ESRCH; + ef->_pattern &= bit_pattern; + return SCE_OK; } int APS5_VABI sceKernelCancelEventFlag(KernelEventFlag ef, uint64_t set_pattern, int* num_wait_threads) { - (void)ef; - (void)set_pattern; - (void)num_wait_threads; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!ef) return SCE_KERNEL_ERROR_ESRCH; + std::lock_guard lock(ef->_mutex); + if (ef->_deleted) return SCE_KERNEL_ERROR_ESRCH; + if (num_wait_threads) *num_wait_threads = ef->_waiters; + ef->_pattern = set_pattern; + ++ef->_cancelGeneration; + ef->_changed.notify_all(); + return SCE_OK; } int APS5_VABI sceKernelPollEventFlag(KernelEventFlag ef, uint64_t bit_pattern, uint32_t wait_mode, uint64_t* result_pat) { - (void)ef; - (void)bit_pattern; - (void)wait_mode; - (void)result_pat; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!ef) return SCE_KERNEL_ERROR_ESRCH; + if (bit_pattern == 0 || !IsValidWaitMode(wait_mode)) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(ef->_mutex); + if (ef->_deleted) return SCE_KERNEL_ERROR_ESRCH; + if (!ef->_multipleWaiters && ef->_waiters != 0) return SCE_KERNEL_ERROR_EPERM; + if (!IsSatisfied(ef->_pattern, bit_pattern, wait_mode)) { + if (result_pat) *result_pat = ef->_pattern; + return SCE_KERNEL_ERROR_EBUSY; + } + Consume(ef, bit_pattern, wait_mode, result_pat); + return SCE_OK; } int APS5_VABI sceKernelWaitEventFlag(KernelEventFlag ef, uint64_t bit_pattern, uint32_t wait_mode, uint64_t* result_pat, KernelUseconds* timeout) { - (void)ef; - (void)bit_pattern; - (void)wait_mode; - (void)result_pat; - (void)timeout; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!ef) return SCE_KERNEL_ERROR_ESRCH; + if (bit_pattern == 0 || !IsValidWaitMode(wait_mode)) return SCE_KERNEL_ERROR_EINVAL; + std::unique_lock lock(ef->_mutex); + if (ef->_deleted) return SCE_KERNEL_ERROR_ESRCH; + if (!ef->_multipleWaiters && ef->_waiters != 0) return SCE_KERNEL_ERROR_EPERM; + const uint64_t generation = ef->_cancelGeneration; + if (TraceSync()) std::fprintf(stderr, "[evf] wait %p '%s' want=0x%llx mode=0x%x have=0x%llx timeout=%d caller=%p outer=%p\n", static_cast(ef), ef->_name.c_str(), static_cast(bit_pattern), wait_mode, static_cast(ef->_pattern), timeout ? static_cast(*timeout) : -1, __builtin_return_address(0), static_cast(static_cast(__builtin_frame_address(0))[0])[1]); + const auto released = [&] { return ef->_deleted || ef->_cancelGeneration != generation || IsSatisfied(ef->_pattern, bit_pattern, wait_mode); }; + ++ef->_waiters; + bool timedOut = false; + const auto waitStart = std::chrono::steady_clock::now(); + if (timeout) { + const auto deadline = waitStart + std::chrono::microseconds(*timeout); + timedOut = !ef->_changed.wait_until(lock, deadline, released); + const auto remaining = std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()).count(); + *timeout = remaining > 0 ? static_cast(remaining) : 0; + } else { + ef->_changed.wait(lock, released); + } + KernelTraceWait_nid_postfix("evf", __builtin_return_address(0), static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), timedOut); + --ef->_waiters; + if (ef->_deleted) { + const bool last = ef->_waiters == 0; + lock.unlock(); + if (last) delete ef; + return SCE_KERNEL_ERROR_EACCES; + } + if (ef->_cancelGeneration != generation) { + if (result_pat) *result_pat = ef->_pattern; + return SCE_KERNEL_ERROR_ECANCELED; + } + if (timedOut) { + if (result_pat) *result_pat = ef->_pattern; + return SCE_KERNEL_ERROR_ETIMEDOUT; + } + Consume(ef, bit_pattern, wait_mode, result_pat); + return SCE_OK; } } diff --git a/core/libs/prx/libkernel/File/include/NativeStat.hpp b/core/libs/prx/libkernel/File/include/NativeStat.hpp index 8ea664c419..7a84604992 100644 --- a/core/libs/prx/libkernel/File/include/NativeStat.hpp +++ b/core/libs/prx/libkernel/File/include/NativeStat.hpp @@ -7,6 +7,7 @@ namespace File { void FillFileStat(const std::filesystem::path& nativePath, FileStat* sb); +bool FillFileStatFromDescriptor(int fd, FileStat* sb); } diff --git a/core/libs/prx/libkernel/File/src/NativeStat.cpp b/core/libs/prx/libkernel/File/src/NativeStat.cpp index 5f92973b89..8791dc71f4 100644 --- a/core/libs/prx/libkernel/File/src/NativeStat.cpp +++ b/core/libs/prx/libkernel/File/src/NativeStat.cpp @@ -10,21 +10,40 @@ using NativeStat = struct __stat64; static int DoStat(const std::filesystem::path& p, NativeStat* st) { return _wstat64(p.wstring().c_str(), st); } +static int DoFstat(int fd, NativeStat* st) { + return _fstat64(fd, st); +} #else #include using NativeStat = struct stat; static int DoStat(const std::filesystem::path& p, NativeStat* st) { return ::stat(p.c_str(), st); } +static int DoFstat(int fd, NativeStat* st) { + return ::fstat(fd, st); +} #endif namespace File { +static void FillFromNative(const NativeStat& st, FileStat* sb); + void FillFileStat(const std::filesystem::path& nativePath, FileStat* sb) { NativeStat st{}; if (DoStat(nativePath, &st) != 0) { throw std::runtime_error(std::string("FillFileStat: stat failed for ") + nativePath.string()); } + FillFromNative(st, sb); +} + +bool FillFileStatFromDescriptor(int fd, FileStat* sb) { + NativeStat st{}; + if (DoFstat(fd, &st) != 0) return false; + FillFromNative(st, sb); + return true; +} + +static void FillFromNative(const NativeStat& st, FileStat* sb) { *sb = FileStat{}; sb->st_mode = static_cast(st.st_mode); sb->st_size = static_cast(st.st_size); diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp index d1f3a9cbef..0ddf0be894 100644 --- a/core/libs/prx/libkernel/File/src/Open.cpp +++ b/core/libs/prx/libkernel/File/src/Open.cpp @@ -87,6 +87,12 @@ static int MapFlags(int sceFlags) { } #endif +static int SceErrorFromErrno(int error) { + constexpr int GuestEio = 5; + const int guest = error > 0 && error <= 34 ? error : GuestEio; + return static_cast(0x80020000u | static_cast(guest)); +} + extern "C" { int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) { @@ -94,11 +100,7 @@ int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) { auto native = ResolvePath_nid_no_patch(path); int fd = NativeOpen(native, MapFlags(flags), mode); if (fd < 0) { - const int error = errno; - if (error == ENOENT) { - return SCE_KERNEL_ERROR_ENOENT; - } - throw std::runtime_error(std::string(__func__) + ": failed to open " + native.string() + ", errno=" + std::to_string(error)); + return SceErrorFromErrno(errno); } return fd; } @@ -153,7 +155,12 @@ int APS5_VABI sceKernelStat(const char* path, FileStat* sb) { if (sb == nullptr) { throw std::invalid_argument(std::string(__func__) + ": sb is null"); } - File::FillFileStat(ResolvePath_nid_no_patch(path), sb); + const auto native = ResolvePath_nid_no_patch(path); + std::error_code error; + if (!std::filesystem::exists(native, error)) { + return SceErrorFromErrno(2); + } + File::FillFileStat(native, sb); return 0; } @@ -163,7 +170,7 @@ int APS5_VABI sceKernelUnlink(const char* path) { } auto native = ResolvePath_nid_no_patch(path); if (NativeUnlink(native) != 0) { - throw std::runtime_error(std::string(__func__) + ": unlink failed for " + native.string() + ", errno=" + std::to_string(errno)); + return SceErrorFromErrno(errno); } return 0; } diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index f3dc9fa597..14f9a91bd8 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -2,6 +2,20 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/File/include/NativeStat.hpp" +#include +#include +#include + +static constexpr int GUEST_ENOENT = 2; +static constexpr int GUEST_EIO = 5; +static constexpr int GUEST_EEXIST = 17; +static constexpr int GUEST_ENOTDIR = 20; +static constexpr int GUEST_ENOTEMPTY = 66; + +static int SceErrorFromErrno(int error) { + return static_cast(0x80020000u | static_cast(error > 0 && error <= 34 ? error : error == GUEST_ENOTEMPTY ? error : GUEST_EIO)); +} extern "C" { @@ -110,10 +124,9 @@ int APS5_VABI sceKernelCheckReachability(const char* path) { } int APS5_VABI sceKernelFstat(int d, FileStat* sb) { - (void)d; - (void)sb; - NotImplemented_nid_no_patch(__func__); - return 0; + if (sb == nullptr) throw std::invalid_argument("sceKernelFstat: sb is null"); + if (!File::FillFileStatFromDescriptor(d, sb)) return SceErrorFromErrno(errno); + return 0; } int APS5_VABI sceKernelFsync(int fd) { @@ -140,10 +153,14 @@ int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* bas } int APS5_VABI sceKernelMkdir(const char* path, uint16_t mode) { - (void)path; - (void)mode; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)mode; + if (path == nullptr) throw std::invalid_argument("sceKernelMkdir: path is null"); + const auto native = ResolvePath_nid_no_patch(path); + std::error_code error; + if (std::filesystem::exists(native, error)) return SceErrorFromErrno(GUEST_EEXIST); + if (!std::filesystem::exists(native.parent_path(), error)) return SceErrorFromErrno(GUEST_ENOENT); + if (!std::filesystem::create_directory(native, error)) return SceErrorFromErrno(error.value() ? error.value() : GUEST_EIO); + return 0; } int64_t APS5_VABI sceKernelPread(int d, void* buf, size_t nbytes, int64_t offset) { @@ -165,16 +182,23 @@ int64_t APS5_VABI sceKernelPwrite(int d, const void* buf, size_t nbytes, int64_t } int APS5_VABI sceKernelRename(const char* from, const char* to) { - (void)from; - (void)to; - NotImplemented_nid_no_patch(__func__); - return 0; + if (from == nullptr || to == nullptr) throw std::invalid_argument("sceKernelRename: path is null"); + const auto source = ResolvePath_nid_no_patch(from); + std::error_code error; + if (!std::filesystem::exists(source, error)) return SceErrorFromErrno(GUEST_ENOENT); + std::filesystem::rename(source, ResolvePath_nid_no_patch(to), error); + if (error) return SceErrorFromErrno(GUEST_EIO); + return 0; } int APS5_VABI sceKernelRmdir(const char* path) { - (void)path; - NotImplemented_nid_no_patch(__func__); - return 0; + if (path == nullptr) throw std::invalid_argument("sceKernelRmdir: path is null"); + const auto native = ResolvePath_nid_no_patch(path); + std::error_code error; + if (!std::filesystem::is_directory(native, error)) return SceErrorFromErrno(std::filesystem::exists(native, error) ? GUEST_ENOTDIR : GUEST_ENOENT); + if (!std::filesystem::is_empty(native, error)) return SceErrorFromErrno(GUEST_ENOTEMPTY); + if (!std::filesystem::remove(native, error)) return SceErrorFromErrno(GUEST_EIO); + return 0; } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Attr.cpp b/core/libs/prx/libkernel/Pthread/Posix/Attr.cpp index c3c9af8a8a..d9308e40c7 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Attr.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Attr.cpp @@ -2,111 +2,112 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "../include/Pthread.hpp" +#include "Common.hpp" + +extern "C" { +int APS5_VABI scePthreadAttrInit(PthreadAttr* attr); +int APS5_VABI scePthreadAttrDestroy(PthreadAttr* attr); +int APS5_VABI scePthreadAttrSetstacksize(PthreadAttr* attr, std::size_t stacksize); +} + +static bool Valid(const PthreadAttr* attr) { + return attr && *attr; +} extern "C" { int APS5_VABI pthread_attr_destroy_nid_postfix(PthreadAttr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr)) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadAttrDestroy(attr)); } int APS5_VABI pthread_attr_get_np_nid_postfix(Pthread thread, PthreadAttr* attr) { - (void)thread; - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!thread || !Valid(attr)) return PosixThread::GUEST_EINVAL; + auto* p = *attr; + p->_stacksize = thread->stackSize; + p->stackAddress = thread->stackAddress; + p->_detachstate = thread->_detached ? 1 : 0; + p->_schedpriority = thread->priority.load(std::memory_order_relaxed); + p->_affinity = thread->affinity.load(std::memory_order_relaxed); + return 0; } int APS5_VABI pthread_attr_getdetachstate_nid_postfix(const PthreadAttr* attr, int* state) { - (void)attr; - (void)state; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr) || !state) return PosixThread::GUEST_EINVAL; + *state = (*attr)->_detachstate; + return 0; } int APS5_VABI pthread_attr_getguardsize_nid_postfix(const PthreadAttr* attr, size_t* guard_size) { - (void)attr; - (void)guard_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr) || !guard_size) return PosixThread::GUEST_EINVAL; + *guard_size = (*attr)->_guardsize; + return 0; } int APS5_VABI pthread_attr_getschedparam_nid_postfix(const PthreadAttr* attr, KernelSchedParam* param) { - (void)attr; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr) || !param) return PosixThread::GUEST_EINVAL; + param->sched_priority = (*attr)->_schedpriority; + return 0; } int APS5_VABI pthread_attr_getschedpolicy_nid_postfix(const PthreadAttr* attr, int* policy) { - (void)attr; - (void)policy; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr) || !policy) return PosixThread::GUEST_EINVAL; + *policy = (*attr)->_schedpolicy; + return 0; } int APS5_VABI pthread_attr_getstack_nid_postfix(const PthreadAttr* __restrict attr, void** __restrict stack_addr, size_t* __restrict stack_size) { - (void)attr; - (void)stack_addr; - (void)stack_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr) || !stack_addr || !stack_size) return PosixThread::GUEST_EINVAL; + *stack_addr = (*attr)->stackAddress; + *stack_size = (*attr)->_stacksize; + return 0; } int APS5_VABI pthread_attr_getstacksize_nid_postfix(const PthreadAttr* attr, size_t* stack_size) { - (void)attr; - (void)stack_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr) || !stack_size) return PosixThread::GUEST_EINVAL; + *stack_size = (*attr)->_stacksize; + return 0; } int APS5_VABI pthread_attr_init_nid_postfix(PthreadAttr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadAttrInit(attr)); } int APS5_VABI pthread_attr_setdetachstate_nid_postfix(PthreadAttr* attr, int state) { - (void)attr; - (void)state; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr) || (state != 0 && state != 1)) return PosixThread::GUEST_EINVAL; + (*attr)->_detachstate = state; + return 0; } int APS5_VABI pthread_attr_setguardsize_nid_postfix(PthreadAttr* attr, size_t guard_size) { - (void)attr; - (void)guard_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr)) return PosixThread::GUEST_EINVAL; + (*attr)->_guardsize = guard_size; + return 0; } int APS5_VABI pthread_attr_setinheritsched_nid_postfix(PthreadAttr* attr, int inherit_sched) { - (void)attr; - (void)inherit_sched; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr)) return PosixThread::GUEST_EINVAL; + (*attr)->_inheritsched = inherit_sched; + return 0; } int APS5_VABI pthread_attr_setschedparam_nid_postfix(PthreadAttr* attr, const KernelSchedParam* param) { - (void)attr; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr) || !param) return PosixThread::GUEST_EINVAL; + (*attr)->_schedpriority = param->sched_priority; + return 0; } int APS5_VABI pthread_attr_setschedpolicy_nid_postfix(PthreadAttr* attr, int policy) { - (void)attr; - (void)policy; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr)) return PosixThread::GUEST_EINVAL; + (*attr)->_schedpolicy = policy; + return 0; } int APS5_VABI pthread_attr_setstacksize_nid_postfix(PthreadAttr* attr, size_t stack_size) { - (void)attr; - (void)stack_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Valid(attr)) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadAttrSetstacksize(attr, stack_size)); } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Common.hpp b/core/libs/prx/libkernel/Pthread/Posix/Common.hpp new file mode 100644 index 0000000000..3b5734120b --- /dev/null +++ b/core/libs/prx/libkernel/Pthread/Posix/Common.hpp @@ -0,0 +1,34 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_PTHREAD_POSIX_COMMON_HPP +#define CORE_LIBS_PRX_LIBKERNEL_PTHREAD_POSIX_COMMON_HPP + +#include +#include +#include +#include "SceTypes.hpp" + +extern "C" int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp); + +namespace PosixThread { + +constexpr int GUEST_EINVAL = 22; +constexpr int GUEST_ETIMEDOUT = 60; + +inline int ToErrno(int sceResult) { + return sceResult == 0 ? 0 : static_cast(static_cast(sceResult) & 0xFFFFu); +} + +// Converts an absolute deadline on the guest clock into a relative wait, clamped to what the sce layer accepts. +inline bool RelativeMicroseconds(int clockId, const KernelTimespec* abstime, KernelUseconds* usec) { + if (!abstime || abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) return false; + KernelTimespec now{}; + clock_gettime_nid_postfix(clockId, &now); + const auto deadline = std::chrono::seconds(abstime->tv_sec) + std::chrono::nanoseconds(abstime->tv_nsec); + const auto current = std::chrono::seconds(now.tv_sec) + std::chrono::nanoseconds(now.tv_nsec); + const auto remaining = std::chrono::duration_cast(deadline - current).count(); + *usec = remaining <= 0 ? 0 : remaining >= std::numeric_limits::max() ? std::numeric_limits::max() : static_cast(remaining); + return true; +} + +} + +#endif diff --git a/core/libs/prx/libkernel/Pthread/Posix/Cond.cpp b/core/libs/prx/libkernel/Pthread/Posix/Cond.cpp index b860a2262b..7055cae0c4 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Cond.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Cond.cpp @@ -2,60 +2,74 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "../include/Pthread.hpp" +#include "Common.hpp" +#include +#include +#include + +extern "C" { +int APS5_VABI scePthreadCondattrInit(PthreadCondattr* attr); +int APS5_VABI scePthreadCondattrDestroy(PthreadCondattr* attr); +int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char* name); +int APS5_VABI scePthreadCondSignal(PthreadCond* cond); +int APS5_VABI scePthreadCondBroadcast(PthreadCond* cond); +int APS5_VABI scePthreadCondWait(PthreadCond* cond, PthreadMutex* mutex); +int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, unsigned int usec); +} + +static void InitializeStatic(PthreadCond* cond, const char* funcName) { + if (!cond) throw std::runtime_error(std::string(funcName) + ": null cond"); + std::atomic_ref slot(*cond); + PthreadCond current = slot.load(std::memory_order_acquire); + if (current != nullptr) return; + auto* created = new PthreadCondPrivate(); + if (!slot.compare_exchange_strong(current, created, std::memory_order_acq_rel, std::memory_order_acquire)) + delete created; +} extern "C" { int APS5_VABI pthread_cond_broadcast_nid_postfix(PthreadCond* cond) { - (void)cond; - NotImplemented_nid_no_patch(__func__); - return 0; + InitializeStatic(cond, __func__); + return PosixThread::ToErrno(scePthreadCondBroadcast(cond)); } int APS5_VABI pthread_cond_init_nid_postfix(PthreadCond* cond, const PthreadCondattr* attr) { - (void)cond; - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return PosixThread::ToErrno(scePthreadCondInit(cond, attr, nullptr)); } int APS5_VABI pthread_cond_signal_nid_postfix(PthreadCond* cond) { - (void)cond; - NotImplemented_nid_no_patch(__func__); - return 0; + InitializeStatic(cond, __func__); + return PosixThread::ToErrno(scePthreadCondSignal(cond)); } int APS5_VABI pthread_cond_timedwait_nid_postfix(PthreadCond* cond, PthreadMutex* mutex, const KernelTimespec* abstime) { - (void)cond; - (void)mutex; - (void)abstime; - NotImplemented_nid_no_patch(__func__); - return 0; + InitializeStatic(cond, __func__); + KernelUseconds usec = 0; + if (!PosixThread::RelativeMicroseconds((*cond)->_clockid, abstime, &usec)) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadCondTimedwait(cond, mutex, usec)); } int APS5_VABI pthread_cond_wait_nid_postfix(PthreadCond* cond, PthreadMutex* mutex) { - (void)cond; - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + InitializeStatic(cond, __func__); + return PosixThread::ToErrno(scePthreadCondWait(cond, mutex)); } int APS5_VABI pthread_condattr_destroy_nid_postfix(PthreadCondattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadCondattrDestroy(attr)); } int APS5_VABI pthread_condattr_init_nid_postfix(PthreadCondattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadCondattrInit(attr)); } int APS5_VABI pthread_condattr_setclock_nid_postfix(PthreadCondattr* attr, KernelClockid clock_id) { - (void)attr; - (void)clock_id; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) return PosixThread::GUEST_EINVAL; + (*attr)->_clockid = clock_id; + return 0; } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp b/core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp index 1409b3450a..2f56763b1b 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp @@ -2,71 +2,119 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "../include/Pthread.hpp" +#include +#include +#include +#include + +extern "C" { +int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attr); +int APS5_VABI scePthreadMutexattrDestroy(PthreadMutexattr* attr); +int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attr, int type); +int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr, const char* name); +int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec); +int APS5_VABI scePthreadMutexTrylock(PthreadMutex* mutex); +} + +namespace { + +constexpr int POSIX_EINVAL = 22; +constexpr int POSIX_MUTEX_ADAPTIVE_NP = 4; +constexpr int POSIX_PRIO_PROTECT = 2; +constexpr std::uintptr_t POSIX_ADAPTIVE_MUTEX_INITIALIZER = 1; + +int _toErrno(int sceResult) { + return sceResult == 0 ? 0 : static_cast(static_cast(sceResult) & 0xFFFFu); +} + +bool _isStaticInitializer(PthreadMutex mutex) { + const auto value = reinterpret_cast(mutex); + return value == 0 || value == POSIX_ADAPTIVE_MUTEX_INITIALIZER; +} + +void _initializeStatic(PthreadMutex* mutex, const char* funcName) { + if (!mutex) throw std::runtime_error(std::string(funcName) + ": null mutex"); + std::atomic_ref slot(*mutex); + PthreadMutex current = slot.load(std::memory_order_acquire); + if (!_isStaticInitializer(current)) return; + auto* created = new PthreadMutexPrivate(); + created->_type = reinterpret_cast(current) == POSIX_ADAPTIVE_MUTEX_INITIALIZER ? MutexType::Normal : MutexType::ErrorCheck; + if (!slot.compare_exchange_strong(current, created, std::memory_order_acq_rel, std::memory_order_acquire)) + delete created; +} + +} extern "C" { int APS5_VABI pthread_mutex_destroy_nid_postfix(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!mutex) throw std::runtime_error("pthread_mutex_destroy: null mutex"); + if (_isStaticInitializer(*mutex)) { + *mutex = nullptr; + return 0; + } + return _toErrno(scePthreadMutexDestroy(mutex)); } int APS5_VABI pthread_mutex_init_nid_postfix(PthreadMutex* mutex, const PthreadMutexattr* attr) { - (void)mutex; - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + const int result = scePthreadMutexInit(mutex, attr, nullptr); + if (result == 0 && (!attr || !*attr)) (*mutex)->_type = MutexType::ErrorCheck; + return _toErrno(result); } int APS5_VABI pthread_mutex_lock_nid_postfix(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + _initializeStatic(mutex, __func__); + return _toErrno(scePthreadMutexLock(mutex)); } int APS5_VABI pthread_mutex_timedlock_nid_postfix(PthreadMutex* mutex, const KernelTimespec* abstime) { - (void)mutex; - (void)abstime; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!abstime) throw std::runtime_error("pthread_mutex_timedlock: null abstime"); + if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) return POSIX_EINVAL; + _initializeStatic(mutex, __func__); + const auto deadline = std::chrono::seconds(abstime->tv_sec) + std::chrono::nanoseconds(abstime->tv_nsec); + const auto now = std::chrono::system_clock::now().time_since_epoch(); + const auto remaining = std::chrono::duration_cast(deadline - now).count(); + const auto usec = remaining <= 0 ? 0 : remaining >= std::numeric_limits::max() ? std::numeric_limits::max() : static_cast(remaining); + return _toErrno(scePthreadMutexTimedlock(mutex, usec)); } int APS5_VABI pthread_mutex_trylock_nid_postfix(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + _initializeStatic(mutex, __func__); + return _toErrno(scePthreadMutexTrylock(mutex)); } int APS5_VABI pthread_mutex_unlock_nid_postfix(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + _initializeStatic(mutex, __func__); + return _toErrno(scePthreadMutexUnlock(mutex)); } int APS5_VABI pthread_mutexattr_destroy_nid_postfix(PthreadMutexattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return _toErrno(scePthreadMutexattrDestroy(attr)); } int APS5_VABI pthread_mutexattr_init_nid_postfix(PthreadMutexattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + const int result = scePthreadMutexattrInit(attr); + if (result == 0) (*attr)->type = MutexType::ErrorCheck; + return _toErrno(result); } int APS5_VABI pthread_mutexattr_setprotocol_nid_postfix(PthreadMutexattr* attr, int protocol) { - (void)attr; - (void)protocol; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error("pthread_mutexattr_setprotocol: null attr"); + if (protocol < 0 || protocol > POSIX_PRIO_PROTECT) return POSIX_EINVAL; + return 0; } int APS5_VABI pthread_mutexattr_settype_nid_postfix(PthreadMutexattr* attr, int type) { - (void)attr; - (void)type; - NotImplemented_nid_no_patch(__func__); - return 0; + if (type == POSIX_MUTEX_ADAPTIVE_NP) { + if (!attr || !*attr) throw std::runtime_error("pthread_mutexattr_settype: null attr"); + (*attr)->type = MutexType::Normal; + return 0; + } + return _toErrno(scePthreadMutexattrSettype(attr, type)); } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Rwlock.cpp b/core/libs/prx/libkernel/Pthread/Posix/Rwlock.cpp index 2f1ee6a983..c15248a607 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Rwlock.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Rwlock.cpp @@ -2,26 +2,49 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "../include/Pthread.hpp" +#include +#include + +extern "C" { +int APS5_VABI scePthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr, const char* name); +int APS5_VABI scePthreadRwlockDestroy(PthreadRwlock* rwlock); +int APS5_VABI scePthreadRwlockWrlock(PthreadRwlock* rwlock); +} + +namespace { + +int _toErrno(int sceResult) { + return sceResult == 0 ? 0 : static_cast(static_cast(sceResult) & 0xFFFFu); +} + +void _initializeStatic(PthreadRwlock* rwlock, const char* funcName) { + if (!rwlock) throw std::runtime_error(std::string(funcName) + ": null rwlock"); + std::atomic_ref slot(*rwlock); + PthreadRwlock current = slot.load(std::memory_order_acquire); + if (current != nullptr) return; + auto* created = new PthreadRwlockPrivate(); + if (!slot.compare_exchange_strong(current, created, std::memory_order_acq_rel, std::memory_order_acquire)) + delete created; +} + +} extern "C" { int APS5_VABI pthread_rwlock_destroy_nid_postfix(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!rwlock) throw std::runtime_error("pthread_rwlock_destroy: null rwlock"); + if (*rwlock == nullptr) return 0; + return _toErrno(scePthreadRwlockDestroy(rwlock)); } int APS5_VABI pthread_rwlock_init_nid_postfix(PthreadRwlock* rwlock, const PthreadRwlockattr* attr) { - (void)rwlock; - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return _toErrno(scePthreadRwlockInit(rwlock, attr, nullptr)); } int APS5_VABI pthread_rwlock_wrlock_nid_postfix(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; + _initializeStatic(rwlock, __func__); + return _toErrno(scePthreadRwlockWrlock(rwlock)); } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp b/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp index ad7fb14933..bb76c83d40 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp @@ -2,60 +2,113 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "Common.hpp" +#include +#include +#include +#include +#include + +// The guest sem_t is at least 8 bytes; its first word holds a pointer to the host semaphore. +struct PosixSemaphore { + std::mutex lock; + std::condition_variable available; + unsigned int count; +}; + +static constexpr int GUEST_EAGAIN = 35; +static constexpr int GUEST_REALTIME_CLOCK = 0; + +static int Fail(int error) { + errno = error; + return -1; +} + +static PosixSemaphore* Get(void* sem) { + if (!sem) return nullptr; + PosixSemaphore* semaphore = nullptr; + std::memcpy(&semaphore, sem, sizeof(semaphore)); + return semaphore; +} + +static int WaitFor(PosixSemaphore* semaphore, const KernelUseconds* usec) { + std::unique_lock lock(semaphore->lock); + const auto ready = [&] { return semaphore->count > 0; }; + if (usec) { + if (!semaphore->available.wait_for(lock, std::chrono::microseconds(*usec), ready)) return Fail(PosixThread::GUEST_ETIMEDOUT); + } else { + semaphore->available.wait(lock, ready); + } + --semaphore->count; + return 0; +} extern "C" { int APS5_VABI sem_init_nid_postfix(void* sem, int pshared, unsigned int value) { - (void)sem; - (void)pshared; - (void)value; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)pshared; + if (!sem) return Fail(PosixThread::GUEST_EINVAL); + auto* semaphore = new PosixSemaphore(); + semaphore->count = value; + std::memcpy(sem, &semaphore, sizeof(semaphore)); + return 0; } int APS5_VABI sem_destroy_nid_postfix(void* sem) { - (void)sem; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* semaphore = Get(sem); + if (!semaphore) return Fail(PosixThread::GUEST_EINVAL); + delete semaphore; + std::memset(sem, 0, sizeof(semaphore)); + return 0; } int APS5_VABI sem_wait_nid_postfix(void* sem) { - (void)sem; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* semaphore = Get(sem); + if (!semaphore) return Fail(PosixThread::GUEST_EINVAL); + return WaitFor(semaphore, nullptr); } int APS5_VABI sem_trywait_nid_postfix(void* sem) { - (void)sem; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* semaphore = Get(sem); + if (!semaphore) return Fail(PosixThread::GUEST_EINVAL); + std::lock_guard lock(semaphore->lock); + if (semaphore->count == 0) return Fail(GUEST_EAGAIN); + --semaphore->count; + return 0; } int APS5_VABI sem_timedwait_nid_postfix(void* sem, const KernelTimespec* abstime) { - (void)sem; - (void)abstime; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* semaphore = Get(sem); + if (!semaphore) return Fail(PosixThread::GUEST_EINVAL); + KernelUseconds usec = 0; + if (!PosixThread::RelativeMicroseconds(GUEST_REALTIME_CLOCK, abstime, &usec)) return Fail(PosixThread::GUEST_EINVAL); + return WaitFor(semaphore, &usec); } int APS5_VABI sem_reltimedwait_np_nid_postfix(void* sem, uint32_t usec) { - (void)sem; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* semaphore = Get(sem); + if (!semaphore) return Fail(PosixThread::GUEST_EINVAL); + const KernelUseconds timeout = usec; + return WaitFor(semaphore, &timeout); } int APS5_VABI sem_post_nid_postfix(void* sem) { - (void)sem; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* semaphore = Get(sem); + if (!semaphore) return Fail(PosixThread::GUEST_EINVAL); + { + std::lock_guard lock(semaphore->lock); + ++semaphore->count; + } + semaphore->available.notify_one(); + return 0; } int APS5_VABI sem_getvalue_nid_postfix(void* sem, int* value) { - (void)sem; - (void)value; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* semaphore = Get(sem); + if (!semaphore || !value) return Fail(PosixThread::GUEST_EINVAL); + std::lock_guard lock(semaphore->lock); + *value = static_cast(semaphore->count); + return 0; } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp b/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp index da9083fc49..fc7b245e7c 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp @@ -2,91 +2,76 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "../include/Pthread.hpp" +#include "Common.hpp" +#include +extern "C" { +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadDetach(Pthread thread); +void APS5_VABI scePthreadExit(void* retval); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +int APS5_VABI scePthreadRename(Pthread thread, const char* name); +Pthread APS5_VABI scePthreadSelf(); +int APS5_VABI scePthreadSetcancelstate(int state, int* old_state); +int APS5_VABI scePthreadSetprio(Pthread thread, int prio); +int APS5_VABI scePthreadGetprio(Pthread thread, int* prio); +} + +static constexpr int GUEST_SCHED_FIFO = 1; extern "C" { -int APS5_VABI pthread_create_nid_postfix(Pthread* thread, const PthreadAttr* attr, pthread_entry_func_t entry, void* arg) { - (void)thread; - (void)attr; - (void)entry; - (void)arg; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI pthread_create_nid_postfix(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg) { + return PosixThread::ToErrno(scePthreadCreate(thread, attr, entry, arg, nullptr)); } -int APS5_VABI pthread_create_name_np_nid_postfix(Pthread* thread, const PthreadAttr* attr, pthread_entry_func_t entry, void* arg, const char* name) { - (void)thread; - (void)attr; - (void)entry; - (void)arg; - (void)name; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI pthread_create_name_np_nid_postfix(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name) { + return PosixThread::ToErrno(scePthreadCreate(thread, attr, entry, arg, name)); } int APS5_VABI pthread_detach_nid_postfix(Pthread thread) { - (void)thread; - NotImplemented_nid_no_patch(__func__); - return 0; + return PosixThread::ToErrno(scePthreadDetach(thread)); } void APS5_VABI pthread_exit_nid_postfix(void* value) { - (void)value; - NotImplemented_nid_no_patch(__func__); + scePthreadExit(value); } int APS5_VABI pthread_getschedparam_nid_postfix(Pthread thread, int* policy, KernelSchedParam* param) { - (void)thread; - (void)policy; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!policy || !param) return PosixThread::GUEST_EINVAL; + *policy = GUEST_SCHED_FIFO; + return PosixThread::ToErrno(scePthreadGetprio(thread, ¶m->sched_priority)); } int APS5_VABI pthread_join_nid_postfix(Pthread thread, void** value) { - (void)thread; - (void)value; - NotImplemented_nid_no_patch(__func__); - return 0; + return PosixThread::ToErrno(scePthreadJoin(thread, value)); } int APS5_VABI pthread_rename_np_nid_postfix(Pthread thread, const char* name) { - (void)thread; - (void)name; - NotImplemented_nid_no_patch(__func__); - return 0; + return PosixThread::ToErrno(scePthreadRename(thread, name)); } Pthread APS5_VABI pthread_self_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return {}; + return scePthreadSelf(); } int APS5_VABI pthread_setcancelstate_nid_postfix(int state, int* old_state) { - (void)state; - (void)old_state; - NotImplemented_nid_no_patch(__func__); - return 0; + return PosixThread::ToErrno(scePthreadSetcancelstate(state, old_state)); } int APS5_VABI pthread_setprio_nid_postfix(Pthread thread, int prio) { - (void)thread; - (void)prio; - NotImplemented_nid_no_patch(__func__); - return 0; + return PosixThread::ToErrno(scePthreadSetprio(thread, prio)); } int APS5_VABI pthread_setschedparam_nid_postfix(Pthread thread, int policy, const KernelSchedParam* param) { - (void)thread; - (void)policy; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)policy; + if (!param) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadSetprio(thread, param->sched_priority)); } void APS5_VABI pthread_yield_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); + std::this_thread::yield(); } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Tsd.cpp b/core/libs/prx/libkernel/Pthread/Posix/Tsd.cpp index a5e15b3cc5..d46ac1d42d 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Tsd.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Tsd.cpp @@ -2,33 +2,32 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "Common.hpp" + +extern "C" { +int APS5_VABI scePthreadKeyCreate(PthreadKey* key, pthread_key_destructor_func_t destructor); +int APS5_VABI scePthreadKeyDelete(PthreadKey key); +void* APS5_VABI scePthreadGetspecific(PthreadKey key); +int APS5_VABI scePthreadSetspecific(PthreadKey key, void* value); +} extern "C" { void* APS5_VABI pthread_getspecific_nid_postfix(PthreadKey key) { - (void)key; - NotImplemented_nid_no_patch(__func__); - return nullptr; + return scePthreadGetspecific(key); } int APS5_VABI pthread_setspecific_nid_postfix(PthreadKey key, void* value) { - (void)key; - (void)value; - NotImplemented_nid_no_patch(__func__); - return 0; + return PosixThread::ToErrno(scePthreadSetspecific(key, value)); } int APS5_VABI pthread_key_create_nid_postfix(PthreadKey* key, pthread_key_destructor_func_t destructor) { - (void)key; - (void)destructor; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!key) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadKeyCreate(key, destructor)); } int APS5_VABI pthread_key_delete_nid_postfix(PthreadKey key) { - (void)key; - NotImplemented_nid_no_patch(__func__); - return 0; + return PosixThread::ToErrno(scePthreadKeyDelete(key)); } } diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index e0adf92b8d..c436f91fbc 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -31,14 +32,27 @@ struct PthreadMutexPrivate { PthreadMutexPrivate() : _type(MutexType::Normal), _count(0) {} }; +struct PthreadRwlockattrPrivate { + int type; +}; + +struct PthreadRwlockPrivate { + std::shared_timed_mutex _lock; + std::atomic _writer; +}; + struct PthreadCondattrPrivate { int _clockid; }; struct PthreadCondPrivate { std::condition_variable_any _cv; + int _clockid = 0; }; +static constexpr KernelCpumask DEFAULT_THREAD_AFFINITY = 0x1FFF; +static constexpr int DEFAULT_THREAD_PRIORITY = 700; + struct PthreadAttrPrivate { void* stackAddress = nullptr; std::size_t _stacksize; @@ -46,6 +60,9 @@ struct PthreadAttrPrivate { int _schedpriority; int _schedpolicy; int _inheritsched; + KernelCpumask _affinity = DEFAULT_THREAD_AFFINITY; + std::size_t _guardsize = 0x1000; + int _solosched = 0; }; struct PthreadPrivate { @@ -58,6 +75,10 @@ struct PthreadPrivate { #endif void* stackAddress = nullptr; std::size_t stackSize = 0; + std::atomic affinity{DEFAULT_THREAD_AFFINITY}; + std::atomic priority{DEFAULT_THREAD_PRIORITY}; + std::mutex nameLock; + std::string name; std::atomic _finished; void* _retval; bool _detached; diff --git a/core/libs/prx/libkernel/Pthread/src/Attr.cpp b/core/libs/prx/libkernel/Pthread/src/Attr.cpp index 7f71adb808..5de4fe8d7a 100644 --- a/core/libs/prx/libkernel/Pthread/src/Attr.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Attr.cpp @@ -1,6 +1,7 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" #include +#include static constexpr int SCE_OK = 0; static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; @@ -86,87 +87,81 @@ int APS5_VABI scePthreadAttrGet(Pthread thread, PthreadAttr* attr) { } int APS5_VABI scePthreadAttrGetaffinity(const PthreadAttr* attr, KernelCpumask* mask) { - (void)attr; - (void)mask; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr || !mask) throw std::runtime_error("scePthreadAttrGetaffinity: null argument"); + *mask = (*attr)->_affinity; + return SCE_OK; } int APS5_VABI scePthreadAttrGetdetachstate(const PthreadAttr* attr, int* state) { - (void)attr; - (void)state; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + if (!state) throw std::runtime_error(std::string(__func__) + ": null output"); + *state = (*attr)->_detachstate; + return SCE_OK; } int APS5_VABI scePthreadAttrGetguardsize(const PthreadAttr* attr, size_t* guard_size) { - (void)attr; - (void)guard_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + if (!guard_size) throw std::runtime_error(std::string(__func__) + ": null output"); + *guard_size = (*attr)->_guardsize; + return SCE_OK; } int APS5_VABI scePthreadAttrGetschedparam(const PthreadAttr* attr, KernelSchedParam* param) { - (void)attr; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + if (!param) throw std::runtime_error(std::string(__func__) + ": null output"); + param->sched_priority = (*attr)->_schedpriority; + return SCE_OK; } int APS5_VABI scePthreadAttrGetsolosched(const PthreadAttr* attr, int* solosched) { - (void)attr; - (void)solosched; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + if (!solosched) throw std::runtime_error(std::string(__func__) + ": null output"); + *solosched = (*attr)->_solosched; + return SCE_OK; } int APS5_VABI scePthreadAttrGetstackaddr(const PthreadAttr* attr, void** stack_addr) { - (void)attr; - (void)stack_addr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + if (!stack_addr) throw std::runtime_error(std::string(__func__) + ": null output"); + *stack_addr = (*attr)->stackAddress; + return SCE_OK; } int APS5_VABI scePthreadAttrGetstacksize(const PthreadAttr* attr, size_t* stack_size) { - (void)attr; - (void)stack_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + if (!stack_size) throw std::runtime_error(std::string(__func__) + ": null output"); + *stack_size = (*attr)->_stacksize; + return SCE_OK; } int APS5_VABI scePthreadAttrSetaffinity(PthreadAttr* attr, KernelCpumask mask) { - (void)attr; - (void)mask; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error("scePthreadAttrSetaffinity: null attr"); + (*attr)->_affinity = mask; + return SCE_OK; } int APS5_VABI scePthreadAttrSetguardsize(PthreadAttr* attr, size_t guard_size) { - (void)attr; - (void)guard_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + (*attr)->_guardsize = guard_size; + return SCE_OK; } int APS5_VABI scePthreadAttrSetinheritsched(PthreadAttr* attr, int inherit_sched) { - (void)attr; - (void)inherit_sched; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + (*attr)->_inheritsched = inherit_sched; + return SCE_OK; } int APS5_VABI scePthreadAttrSetschedpolicy(PthreadAttr* attr, int policy) { - (void)attr; - (void)policy; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + (*attr)->_schedpolicy = policy; + return SCE_OK; } int APS5_VABI scePthreadAttrSetsolosched(PthreadAttr* attr, int solosched) { - (void)attr; - (void)solosched; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + (*attr)->_solosched = solosched; + return SCE_OK; } int APS5_VABI scePthreadAttrSetstack(PthreadAttr* attr, void* addr, size_t size) { diff --git a/core/libs/prx/libkernel/Pthread/src/Cond.cpp b/core/libs/prx/libkernel/Pthread/src/Cond.cpp index 754a076f32..2f1a2267aa 100644 --- a/core/libs/prx/libkernel/Pthread/src/Cond.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Cond.cpp @@ -1,11 +1,38 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/Time/include/Time.hpp" #include +#include +#include #include static constexpr int SCE_OK = 0; static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; -static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = 0x80020062; +static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = 0x8002003C; + +static int WaitOn(PthreadCondPrivate* cond, PthreadMutexPrivate* mutex, std::optional timeout, const void* caller) { + const auto self = std::this_thread::get_id(); + bool timedOut = false; + const auto waitStart = std::chrono::steady_clock::now(); + struct Trace { + const void* caller; const bool& timedOut; std::chrono::steady_clock::time_point start; + ~Trace() { KernelTraceWait_nid_postfix("cond", caller, static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()), timedOut); } + } trace{caller, timedOut, waitStart}; + mutex->_owner.store(std::thread::id{}, std::memory_order_relaxed); + if (mutex->_type == MutexType::Recursive) { + std::unique_lock lock(mutex->_rmtx, std::adopt_lock); + if (timeout) timedOut = cond->_cv.wait_for(lock, *timeout) == std::cv_status::timeout; + else cond->_cv.wait(lock); + lock.release(); + } else { + std::unique_lock lock(mutex->_mtx, std::adopt_lock); + if (timeout) timedOut = cond->_cv.wait_for(lock, *timeout) == std::cv_status::timeout; + else cond->_cv.wait(lock); + lock.release(); + } + mutex->_owner.store(self, std::memory_order_relaxed); + return timedOut ? SCE_KERNEL_ERROR_ETIMEDOUT : SCE_OK; +} extern "C" { @@ -24,10 +51,11 @@ int APS5_VABI scePthreadCondattrDestroy(PthreadCondattr* attr) { return SCE_OK; } -int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr*, const char*) { +int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char*) { if (!cond) throw std::runtime_error("scePthreadCondInit: null cond"); auto* p = new (std::nothrow) PthreadCondPrivate{}; if (!p) return SCE_KERNEL_ERROR_ENOMEM; + if (attr && *attr) p->_clockid = (*attr)->_clockid; *cond = p; return SCE_OK; } @@ -54,32 +82,20 @@ int APS5_VABI scePthreadCondBroadcast(PthreadCond* cond) { int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, unsigned int usec) { if (!cond || !*cond || !mutex || !*mutex) throw std::runtime_error("scePthreadCondTimedwait: null arg"); - auto* m = *mutex; - auto* c = *cond; - if (m->_type == MutexType::Recursive) { - std::unique_lock lk(m->_rmtx, std::adopt_lock); - auto res = c->_cv.wait_for(lk, std::chrono::microseconds(usec)); - lk.release(); - return res == std::cv_status::timeout ? SCE_KERNEL_ERROR_ETIMEDOUT : SCE_OK; - } - std::unique_lock lk(m->_mtx, std::adopt_lock); - auto res = c->_cv.wait_for(lk, std::chrono::microseconds(usec)); - lk.release(); - return res == std::cv_status::timeout ? SCE_KERNEL_ERROR_ETIMEDOUT : SCE_OK; + return WaitOn(*cond, *mutex, std::chrono::microseconds(usec), __builtin_return_address(0)); } int APS5_VABI scePthreadCondSignalto(PthreadCond* cond, Pthread thread) { - (void)cond; - (void)thread; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)thread; + if (!cond || !*cond) throw std::runtime_error("scePthreadCondSignalto: null cond"); + (*cond)->_cv.notify_all(); + return SCE_OK; } int APS5_VABI scePthreadCondWait(PthreadCond* cond, PthreadMutex* mutex) { - (void)cond; - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!cond || !*cond || !mutex || !*mutex) + throw std::runtime_error("scePthreadCondWait: null arg"); + return WaitOn(*cond, *mutex, std::nullopt, __builtin_return_address(0)); } } diff --git a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp index cc9ac18813..baf24a6a91 100644 --- a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp @@ -3,11 +3,14 @@ #include #include #include +#include static constexpr int SCE_OK = 0; static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; -static constexpr int SCE_KERNEL_ERROR_EDEADLK = 0x80020023; +static constexpr int SCE_KERNEL_ERROR_EDEADLK = 0x8002000B; static constexpr int SCE_KERNEL_ERROR_EPERM = 0x80020001; +static constexpr int SCE_KERNEL_ERROR_EBUSY = 0x80020010; +static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = 0x8002003C; extern "C" { @@ -32,7 +35,8 @@ int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attr, int type) { case 1: (*attr)->type = MutexType::ErrorCheck; break; case 2: (*attr)->type = MutexType::Recursive; break; case 3: (*attr)->type = MutexType::Normal; break; - default: throw std::runtime_error("scePthreadMutexattrSettype: invalid type"); + case 4: (*attr)->type = MutexType::Normal; break; + default: throw std::runtime_error("scePthreadMutexattrSettype: invalid type " + std::to_string(type)); } return SCE_OK; } @@ -91,16 +95,37 @@ int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex) { } int APS5_VABI scePthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec) { - (void)mutex; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!mutex || !*mutex) throw std::runtime_error("scePthreadMutexTimedlock: null mutex"); + auto* m = *mutex; + const auto tid = std::this_thread::get_id(); + const auto timeout = std::chrono::microseconds(usec); + if (m->_type == MutexType::Recursive) { + if (!m->_rmtx.try_lock_for(timeout)) return SCE_KERNEL_ERROR_ETIMEDOUT; + m->_owner.store(tid, std::memory_order_relaxed); + ++m->_count; + return SCE_OK; + } + if (m->_type == MutexType::ErrorCheck) { + if (m->_owner.load(std::memory_order_acquire) == tid) return SCE_KERNEL_ERROR_EDEADLK; + } + if (!m->_mtx.try_lock_for(timeout)) return SCE_KERNEL_ERROR_ETIMEDOUT; + m->_owner.store(tid, std::memory_order_relaxed); + return SCE_OK; } int APS5_VABI scePthreadMutexTrylock(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!mutex || !*mutex) throw std::runtime_error("scePthreadMutexTrylock: null mutex"); + auto* m = *mutex; + const auto tid = std::this_thread::get_id(); + if (m->_type == MutexType::Recursive) { + if (!m->_rmtx.try_lock()) return SCE_KERNEL_ERROR_EBUSY; + m->_owner.store(tid, std::memory_order_relaxed); + ++m->_count; + return SCE_OK; + } + if (!m->_mtx.try_lock()) return SCE_KERNEL_ERROR_EBUSY; + m->_owner.store(tid, std::memory_order_relaxed); + return SCE_OK; } int APS5_VABI scePthreadMutexattrSetprotocol(PthreadMutexattr* attr, int protocol) { diff --git a/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp b/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp index 12a6040a28..424a889b75 100644 --- a/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp @@ -1,69 +1,112 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" +#include +#include + +static constexpr int SCE_OK = 0; +static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; +static constexpr int SCE_KERNEL_ERROR_EDEADLK = 0x8002000B; +static constexpr int SCE_KERNEL_ERROR_EBUSY = 0x80020010; +static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = 0x8002003C; + +static PthreadRwlockPrivate* RequireRwlock(PthreadRwlock* rwlock, const char* funcName) { + if (!rwlock || !*rwlock) throw std::runtime_error(std::string(funcName) + ": null rwlock"); + return *rwlock; +} + +static bool OwnsWrite(const PthreadRwlockPrivate* lock) { + return lock->_writer.load(std::memory_order_acquire) == std::this_thread::get_id(); +} extern "C" { int APS5_VABI scePthreadRwlockDestroy(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; + delete RequireRwlock(rwlock, __func__); + *rwlock = nullptr; + return SCE_OK; } int APS5_VABI scePthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr, const char* name) { - (void)rwlock; - (void)attr; - (void)name; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)attr; + (void)name; + if (!rwlock) throw std::runtime_error("scePthreadRwlockInit: null rwlock"); + auto* p = new (std::nothrow) PthreadRwlockPrivate(); + if (!p) return SCE_KERNEL_ERROR_ENOMEM; + *rwlock = p; + return SCE_OK; } int APS5_VABI scePthreadRwlockRdlock(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* lock = RequireRwlock(rwlock, __func__); + if (OwnsWrite(lock)) return SCE_KERNEL_ERROR_EDEADLK; + lock->_lock.lock_shared(); + return SCE_OK; } int APS5_VABI scePthreadRwlockTryrdlock(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* lock = RequireRwlock(rwlock, __func__); + return lock->_lock.try_lock_shared() ? SCE_OK : SCE_KERNEL_ERROR_EBUSY; } -int APS5_VABI scePthreadRwlockTrywrlock(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI scePthreadRwlockUnlock(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI scePthreadRwlockTimedrdlock(PthreadRwlock* rwlock, KernelUseconds usec) { + auto* lock = RequireRwlock(rwlock, __func__); + if (OwnsWrite(lock)) return SCE_KERNEL_ERROR_EDEADLK; + return lock->_lock.try_lock_shared_for(std::chrono::microseconds(usec)) ? SCE_OK : SCE_KERNEL_ERROR_ETIMEDOUT; } int APS5_VABI scePthreadRwlockWrlock(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* lock = RequireRwlock(rwlock, __func__); + if (OwnsWrite(lock)) return SCE_KERNEL_ERROR_EDEADLK; + lock->_lock.lock(); + lock->_writer.store(std::this_thread::get_id(), std::memory_order_release); + return SCE_OK; +} + +int APS5_VABI scePthreadRwlockTrywrlock(PthreadRwlock* rwlock) { + auto* lock = RequireRwlock(rwlock, __func__); + if (!lock->_lock.try_lock()) return SCE_KERNEL_ERROR_EBUSY; + lock->_writer.store(std::this_thread::get_id(), std::memory_order_release); + return SCE_OK; +} + +int APS5_VABI scePthreadRwlockTimedwrlock(PthreadRwlock* rwlock, KernelUseconds usec) { + auto* lock = RequireRwlock(rwlock, __func__); + if (OwnsWrite(lock)) return SCE_KERNEL_ERROR_EDEADLK; + if (!lock->_lock.try_lock_for(std::chrono::microseconds(usec))) return SCE_KERNEL_ERROR_ETIMEDOUT; + lock->_writer.store(std::this_thread::get_id(), std::memory_order_release); + return SCE_OK; +} + +int APS5_VABI scePthreadRwlockUnlock(PthreadRwlock* rwlock) { + auto* lock = RequireRwlock(rwlock, __func__); + if (OwnsWrite(lock)) { + lock->_writer.store(std::thread::id{}, std::memory_order_release); + lock->_lock.unlock(); + } else { + lock->_lock.unlock_shared(); + } + return SCE_OK; } int APS5_VABI scePthreadRwlockattrDestroy(PthreadRwlockattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error("scePthreadRwlockattrDestroy: null attr"); + delete *attr; + *attr = nullptr; + return SCE_OK; } int APS5_VABI scePthreadRwlockattrInit(PthreadRwlockattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr) throw std::runtime_error("scePthreadRwlockattrInit: null attr"); + auto* p = new (std::nothrow) PthreadRwlockattrPrivate{0}; + if (!p) return SCE_KERNEL_ERROR_ENOMEM; + *attr = p; + return SCE_OK; } int APS5_VABI scePthreadRwlockattrSettype(PthreadRwlockattr* attr, int type) { - (void)attr; - (void)type; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error("scePthreadRwlockattrSettype: null attr"); + (*attr)->type = type; + return SCE_OK; } } diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index b594ba7a15..2cff28a931 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -1,6 +1,10 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" +#include #include +#include +#include +#include #include #include #include @@ -16,6 +20,7 @@ static constexpr int SCE_KERNEL_ERROR_EINVAL = 0x80020016; static constexpr std::size_t DEFAULT_STACK_SIZE = 1u << 20; static constexpr int DETACH_DETACHED = 1; +static constexpr std::size_t THREAD_NAME_CAPACITY = 32; #ifdef _WIN32 #include @@ -82,6 +87,10 @@ static unsigned __stdcall StartNativeThread(void* opaque) { } self->threadId = std::this_thread::get_id(); currentThread = self; + if (!self->name.empty()) { + const std::wstring description(self->name.begin(), self->name.end()); + SetThreadDescription(GetCurrentThread(), description.c_str()); + } args->initialized.set_value(); } catch (...) { args->initialized.set_exception(std::current_exception()); @@ -100,7 +109,7 @@ static unsigned __stdcall StartNativeThread(void* opaque) { extern "C" { -int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char*) { +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name) { if (!thread || !entry) throw std::runtime_error("scePthreadCreate: null arg"); if (attr && !*attr) throw std::runtime_error("scePthreadCreate: null attributes"); auto p = std::make_unique(); @@ -108,6 +117,11 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread if (attr && *attr) detached = ((*attr)->_detachstate == DETACH_DETACHED); p->_detached = detached; p->stackSize = attr ? (*attr)->_stacksize : DEFAULT_STACK_SIZE; + if (attr) { + p->affinity.store((*attr)->_affinity, std::memory_order_relaxed); + p->priority.store((*attr)->_schedpriority, std::memory_order_relaxed); + } + if (name) p->name = name; std::promise start; auto args = std::make_unique(ThreadArgs{entry, arg, p.get()}); #ifdef _WIN32 @@ -199,6 +213,17 @@ void APS5_VABI scePthreadExit(void* retval) { Pthread APS5_VABI scePthreadSelf() { #ifdef _WIN32 + if (!currentThread) { + auto adopted = std::make_unique(); + HANDLE handle = nullptr; + if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), &handle, 0, FALSE, DUPLICATE_SAME_ACCESS)) + throw std::system_error(GetLastError(), std::system_category(), "Adopting guest thread"); + adopted->nativeHandle = handle; + adopted->threadId = std::this_thread::get_id(); + adopted->_detached = true; + adopted->references.store(1, std::memory_order_relaxed); + currentThread = adopted.release(); + } return currentThread; #else return nullptr; @@ -216,71 +241,90 @@ int APS5_VABI scePthreadCancel(Pthread thread) { } int APS5_VABI scePthreadEqual(Pthread thread1, Pthread thread2) { - (void)thread1; - (void)thread2; - NotImplemented_nid_no_patch(__func__); - return 0; + return thread1 == thread2 ? 1 : 0; } int APS5_VABI scePthreadGetaffinity(Pthread thread, KernelCpumask* mask) { - (void)thread; - (void)mask; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!thread || !mask) return SCE_KERNEL_ERROR_EINVAL; + *mask = thread->affinity.load(std::memory_order_relaxed); + return SCE_OK; } int APS5_VABI scePthreadGetname(Pthread thread, char* name) { - (void)thread; - (void)name; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!thread || !name) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(thread->nameLock); + const std::size_t length = std::min(thread->name.size(), THREAD_NAME_CAPACITY - 1); + std::memcpy(name, thread->name.data(), length); + name[length] = '\0'; + return SCE_OK; } int APS5_VABI scePthreadGetprio(Pthread thread, int* prio) { - (void)thread; - (void)prio; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!thread || !prio) return SCE_KERNEL_ERROR_EINVAL; + *prio = thread->priority.load(std::memory_order_relaxed); + return SCE_OK; } int APS5_VABI scePthreadGetthreadid(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +#ifdef _WIN32 + return static_cast(GetCurrentThreadId()); +#else + return static_cast(std::hash{}(std::this_thread::get_id()) & 0x7fffffff); +#endif } int APS5_VABI scePthreadRename(Pthread thread, const char* name) { - (void)thread; - (void)name; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!thread || !name) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(thread->nameLock); + thread->name = name; +#ifdef _WIN32 + const std::wstring description(thread->name.begin(), thread->name.end()); + SetThreadDescription(static_cast(thread->nativeHandle), description.c_str()); +#endif + return SCE_OK; } int APS5_VABI scePthreadSetaffinity(Pthread thread, KernelCpumask mask) { - (void)thread; - (void)mask; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!thread || mask == 0) return SCE_KERNEL_ERROR_EINVAL; + thread->affinity.store(mask, std::memory_order_relaxed); + return SCE_OK; } int APS5_VABI scePthreadSetcancelstate(int state, int* old_state) { - (void)state; - (void)old_state; - NotImplemented_nid_no_patch(__func__); - return 0; + static thread_local int cancelState = 0; + if (old_state) *old_state = cancelState; + cancelState = state; + return SCE_OK; } int APS5_VABI scePthreadSetcanceltype(int type, int* old_type) { - (void)type; - (void)old_type; - NotImplemented_nid_no_patch(__func__); - return 0; + static thread_local int cancelType = 0; + if (old_type) *old_type = cancelType; + cancelType = type; + return SCE_OK; } int APS5_VABI scePthreadSetprio(Pthread thread, int prio) { - (void)thread; - (void)prio; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!thread) return SCE_KERNEL_ERROR_EINVAL; + thread->priority.store(prio, std::memory_order_relaxed); + return SCE_OK; +} + +int APS5_VABI scePthreadOnce(int32_t* once, void (APS5_VABI* init)(void)) { + if (!once || !init) return SCE_KERNEL_ERROR_EINVAL; + constexpr int32_t Never = 0; + constexpr int32_t Done = 1; + constexpr int32_t Running = 2; + std::atomic_ref state(*once); + int32_t expected = Never; + if (state.compare_exchange_strong(expected, Running, std::memory_order_acq_rel)) { + init(); + state.store(Done, std::memory_order_release); + state.notify_all(); + return SCE_OK; + } + while ((expected = state.load(std::memory_order_acquire)) == Running) state.wait(Running, std::memory_order_acquire); + return SCE_OK; } } diff --git a/core/libs/prx/libkernel/Pthread/src/Tsd.cpp b/core/libs/prx/libkernel/Pthread/src/Tsd.cpp index adc9ce1215..8f6442f8af 100644 --- a/core/libs/prx/libkernel/Pthread/src/Tsd.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Tsd.cpp @@ -1,32 +1,90 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" +#include +#include +#include + +static constexpr int SCE_OK = 0; +static constexpr int SCE_KERNEL_ERROR_EINVAL = 0x80020016; +static constexpr int SCE_KERNEL_ERROR_EAGAIN = 0x80020023; +static constexpr int MAX_KEYS = 512; +static constexpr int DESTRUCTOR_ITERATIONS = 4; + +using GuestKeyDestructor = void (APS5_VABI*)(void*); + +struct KeySlot { + bool used = false; + GuestKeyDestructor destructor = nullptr; +}; + +static std::mutex g_keyLock; +static std::array g_keys; + +struct ThreadValues { + std::array values{}; + + ~ThreadValues() { + for (int round = 0; round < DESTRUCTOR_ITERATIONS; ++round) { + bool called = false; + for (int key = 0; key < MAX_KEYS; ++key) { + void* value = values[key]; + if (!value) continue; + GuestKeyDestructor destructor; + { + std::lock_guard lock(g_keyLock); + destructor = g_keys[key].used ? g_keys[key].destructor : nullptr; + } + values[key] = nullptr; + if (destructor) { + destructor(value); + called = true; + } + } + if (!called) return; + } + } +}; + +static thread_local ThreadValues g_values; + +static bool IsValidKey(PthreadKey key) { + if (key < 0 || key >= MAX_KEYS) return false; + std::lock_guard lock(g_keyLock); + return g_keys[key].used; +} extern "C" { int APS5_VABI scePthreadKeyCreate(PthreadKey* key, pthread_key_destructor_func_t destructor) { - (void)key; - (void)destructor; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!key) throw std::runtime_error("scePthreadKeyCreate: null key"); + std::lock_guard lock(g_keyLock); + for (int index = 0; index < MAX_KEYS; ++index) { + if (!g_keys[index].used) { + g_keys[index] = {true, reinterpret_cast(destructor)}; + *key = index; + return SCE_OK; + } + } + return SCE_KERNEL_ERROR_EAGAIN; } int APS5_VABI scePthreadKeyDelete(PthreadKey key) { - (void)key; - NotImplemented_nid_no_patch(__func__); - return 0; + if (key < 0 || key >= MAX_KEYS) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(g_keyLock); + if (!g_keys[key].used) return SCE_KERNEL_ERROR_EINVAL; + g_keys[key] = {}; + return SCE_OK; } void* APS5_VABI scePthreadGetspecific(PthreadKey key) { - (void)key; - NotImplemented_nid_no_patch(__func__); - return nullptr; + if (key < 0 || key >= MAX_KEYS) return nullptr; + return g_values.values[key]; } int APS5_VABI scePthreadSetspecific(PthreadKey key, void* value) { - (void)key; - (void)value; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!IsValidKey(key)) return SCE_KERNEL_ERROR_EINVAL; + g_values.values[key] = value; + return SCE_OK; } } diff --git a/core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp b/core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp index ac822495fd..9ac0b16a59 100644 --- a/core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp +++ b/core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp @@ -1,4 +1,6 @@ #include "prx/libkernel/Semaphore/include/Semaphore.hpp" +#include "prx/libkernel/Time/include/Time.hpp" +#include #include #include @@ -53,14 +55,20 @@ int APS5_VABI sceKernelWaitSema(KernelSema sem, int need, KernelUseconds* time) } std::unique_lock lock(sem->mutex); + const auto waitStart = std::chrono::steady_clock::now(); + const auto traceWait = [&](bool timedOut) { + KernelTraceWait_nid_postfix("sema", __builtin_return_address(0), static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), timedOut); + }; if (time == nullptr) { sem->condition.wait(lock, [&] { return sem->tokenCount >= need; }); + traceWait(false); sem->tokenCount -= need; return KERNEL_SEMA_OK; } auto timeout = std::chrono::microseconds(*time); bool acquired = sem->condition.wait_for(lock, timeout, [&] { return sem->tokenCount >= need; }); + traceWait(!acquired); if (!acquired) { return KERNEL_SEMA_ERROR_ETIMEDOUT; } diff --git a/core/libs/prx/libkernel/System/src/Coredump.cpp b/core/libs/prx/libkernel/System/src/Coredump.cpp index f8df644e0f..9c6cc39866 100644 --- a/core/libs/prx/libkernel/System/src/Coredump.cpp +++ b/core/libs/prx/libkernel/System/src/Coredump.cpp @@ -1,21 +1,26 @@ #include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +// The handler is recorded but never invoked: host crashes are not turned into guest core dumps. +static std::atomic g_coredumpHandler{0}; +static std::atomic g_coredumpContext{0}; + extern "C" { int APS5_VABI sceCoredumpRegisterCoredumpHandler(uint64_t handler, size_t stack_size, uint64_t context) { - (void)handler; - (void)stack_size; - (void)context; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)stack_size; + g_coredumpHandler.store(handler, std::memory_order_relaxed); + g_coredumpContext.store(context, std::memory_order_relaxed); + return 0; } int APS5_VABI sceCoredumpUnregisterCoredumpHandler(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + g_coredumpHandler.store(0, std::memory_order_relaxed); + g_coredumpContext.store(0, std::memory_order_relaxed); + return 0; } } diff --git a/core/libs/prx/libkernel/System/src/Process.cpp b/core/libs/prx/libkernel/System/src/Process.cpp index a9d28ee5e9..3b453878d6 100644 --- a/core/libs/prx/libkernel/System/src/Process.cpp +++ b/core/libs/prx/libkernel/System/src/Process.cpp @@ -2,6 +2,11 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include +#include +#include + +static constexpr int SCE_KERNEL_ERROR_EINVAL = static_cast(0x80020016); extern "C" { @@ -16,8 +21,7 @@ const char** APS5_VABI getargv_nid_postfix(void) { } int APS5_VABI getpagesize_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0x4000; } int APS5_VABI getpid_nid_postfix(void) { @@ -36,13 +40,11 @@ int APS5_VABI sceKernelGetCurrentCpu(void) { } uint64_t APS5_VABI sceKernelGetGPI(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } void APS5_VABI sceKernelSetGPO(uint32_t bits) { - (void)bits; - NotImplemented_nid_no_patch(__func__); + (void)bits; } int APS5_VABI sceKernelGetOpenPsId(void* open_ps_id) { @@ -56,10 +58,18 @@ void* APS5_VABI sceKernelGetProcParam(void) { return nullptr; } -int APS5_VABI sceKernelUuidCreate(uint32_t* uuid) { - (void)uuid; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceKernelUuidCreate(uint8_t* uuid) { + if (!uuid) return SCE_KERNEL_ERROR_EINVAL; + static std::mutex lock; + static std::mt19937_64 generator{std::random_device{}()}; + std::lock_guard guard(lock); + for (int index = 0; index < 16; index += 8) { + const uint64_t bits = generator(); + std::memcpy(uuid + index, &bits, 8); + } + uuid[6] = static_cast((uuid[6] & 0x0F) | 0x40); + uuid[8] = static_cast((uuid[8] & 0x3F) | 0x80); + return 0; } void APS5_VABI sceKernelSync(void) { @@ -78,4 +88,8 @@ int APS5_VABI sched_get_priority_min_nid_postfix(int policy) { return 0; } +int APS5_VABI sceKernelIsTrinityMode(void) { + return 0; +} + } diff --git a/core/libs/prx/libkernel/Time/Time.cpp b/core/libs/prx/libkernel/Time/Time.cpp index aeb9cafb9f..3465b0704a 100644 --- a/core/libs/prx/libkernel/Time/Time.cpp +++ b/core/libs/prx/libkernel/Time/Time.cpp @@ -1,10 +1,18 @@ #include "prx/libkernel/Time/include/Time.hpp" #include "prx/libc/include/General.hpp" +#include #include #include +#include +#include +#include +#include +#include +#include #include #include +#include #ifdef _WIN32 #include @@ -13,7 +21,7 @@ #include #endif -static std::uint64_t GetMonotonicNanos() { +static std::uint64_t RawMonotonicNanos() { #ifdef _WIN32 static const std::uint64_t freq = [] { LARGE_INTEGER f{}; @@ -31,20 +39,79 @@ static std::uint64_t GetMonotonicNanos() { #endif } +// Debug aid: APS5_TIME_SCALE= slows (or speeds) the guest's monotonic clocks and TSC, so a +// title running far below its frame rate sees plausible frame times. Sleeps still take real time. +static double TimeScale() { + static const double scale = [] { + const char* value = std::getenv("APS5_TIME_SCALE"); + const double parsed = value ? std::strtod(value, nullptr) : 1.0; + return parsed > 0.0 ? parsed : 1.0; + }(); + return scale; +} + +static std::uint64_t ClockOrigin() { + static const std::uint64_t origin = RawMonotonicNanos(); + return origin; +} + +static std::uint64_t GetMonotonicNanos() { + const auto raw = RawMonotonicNanos(); + if (TimeScale() == 1.0) return raw; + const auto origin = ClockOrigin(); + return origin + static_cast(static_cast(raw - origin) * TimeScale()); +} + static std::uint64_t GetStartNanos() { static const std::uint64_t start = GetMonotonicNanos(); return start; } +#ifdef _WIN32 +// Every relative wait in the process (winpthreads nanosleep, std::condition_variable::wait_for, the +// GPU driver's poll sleeps) rounds up to the scheduler tick, 15.6 ms by default; a console game and +// its driver hand labels between queues dozens of times per frame, so the tick is raised to 0.5 ms at +// load, and Windows 11 is told not to drop it while the window is unfocused. APS5_NO_TIMER_RESOLUTION=1 +// keeps the default for comparison. +static const bool g_timerResolutionRaised = [] { + if (std::getenv("APS5_NO_TIMER_RESOLUTION") != nullptr) return false; + using NtSetTimerResolutionFn = LONG(WINAPI*)(ULONG, BOOLEAN, PULONG); + const HMODULE ntdll = GetModuleHandleW(L"ntdll.dll"); + const auto setResolution = ntdll ? reinterpret_cast(reinterpret_cast(GetProcAddress(ntdll, "NtSetTimerResolution"))) : nullptr; + ULONG previous = 0; + const bool raised = setResolution != nullptr && setResolution(5000, TRUE, &previous) >= 0; + PROCESS_POWER_THROTTLING_STATE throttling{}; + throttling.Version = PROCESS_POWER_THROTTLING_CURRENT_VERSION; + throttling.ControlMask = PROCESS_POWER_THROTTLING_IGNORE_TIMER_RESOLUTION; + throttling.StateMask = 0; + SetProcessInformation(GetCurrentProcess(), ProcessPowerThrottling, &throttling, sizeof(throttling)); + std::fprintf(stderr, "[time] scheduler tick %s (was %.2f ms)\n", raised ? "raised to 0.5 ms" : "unchanged", previous / 10000.0); + return raised; +}(); +#endif + static void SleepNanos(std::uint64_t nanos) { if (nanos == 0) { return; } #ifdef _WIN32 - const DWORD millis = static_cast(nanos / 1000000ULL); - if (millis > 0) { - Sleep(millis); + // Guest spin/backoff loops sleep for a few microseconds; Sleep() would either not sleep at all + // or round up to a whole scheduler tick, so short sleeps yield and longer ones use a + // high-resolution waitable timer. + if (nanos < 50000ULL) { + SwitchToThread(); + return; } + thread_local HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); + if (timer) { + LARGE_INTEGER due{}; + due.QuadPart = -static_cast(nanos / 100ULL); + if (SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE)) { + WaitForSingleObject(timer, INFINITE); + return; + } + } + Sleep(static_cast((nanos + 999999ULL) / 1000000ULL)); #else struct timespec req{}; req.tv_sec = static_cast(nanos / 1000000000ULL); @@ -67,7 +134,71 @@ std::uint64_t APS5_VABI sceKernelGetProcessTimeCounterFrequency() { return 1000000000ULL; } +// Debug aid: APS5_TRACE_USLEEP reports every 2000 calls which guest call sites sleep, so a CPU +// thread polling for GPU or I/O completion can be found (offsets are relative to the executable). +static void TraceSleep(const void* caller, std::uint64_t microseconds) { + static const bool enabled = std::getenv("APS5_TRACE_USLEEP") != nullptr; + if (!enabled) return; + static std::mutex mutex; + static std::unordered_map> sites; + static std::uint64_t calls = 0; + std::lock_guard lock(mutex); + auto& site = sites[reinterpret_cast(caller)]; + ++site.first; + site.second += microseconds; + if (++calls % 2000 != 0) return; +#ifdef _WIN32 + const auto image = reinterpret_cast(GetModuleHandleW(nullptr)); +#else + const std::uintptr_t image = 0; +#endif + std::vector>> hot(sites.begin(), sites.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.first > b.second.first; }); + std::fprintf(stderr, "[usleep] %llu calls:", static_cast(calls)); + for (std::size_t i = 0; i < hot.size() && i < 6; ++i) std::fprintf(stderr, " exe+0x%llx x%llu (%llu us)", static_cast(hot[i].first - image), static_cast(hot[i].second.first), static_cast(hot[i].second.second)); + std::fprintf(stderr, "\n"); +} + +void KernelTraceWait_nid_postfix(const char* kind, const void* caller, std::uint64_t waitedNanos, bool timedOut) { + static const bool enabled = std::getenv("APS5_TRACE_WAITS") != nullptr; + if (!enabled) return; + struct Site { std::uint64_t calls = 0; std::uint64_t timeouts = 0; std::uint64_t nanos = 0; std::uint64_t longest = 0; unsigned long lastThread = 0; }; + static std::mutex mutex; + static std::unordered_map sites; + static std::uint64_t lastReport = 0; + char key[96]; +#ifdef _WIN32 + const auto image = reinterpret_cast(GetModuleHandleW(nullptr)); + const unsigned long thread = GetCurrentThreadId(); +#else + const std::uintptr_t image = 0; + const unsigned long thread = 0; +#endif + std::snprintf(key, sizeof(key), "%s exe+0x%llx", kind, static_cast(reinterpret_cast(caller) - image)); + std::lock_guard lock(mutex); + auto& site = sites[key]; + ++site.calls; + site.timeouts += timedOut ? 1u : 0u; + site.nanos += waitedNanos; + site.longest = std::max(site.longest, waitedNanos); + site.lastThread = thread; + const auto now = GetMonotonicNanos(); + if (lastReport == 0) lastReport = now; + if (now - lastReport < 3000000000ULL) return; + lastReport = now; + std::vector> hot(sites.begin(), sites.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.nanos > b.second.nanos; }); + std::fprintf(stderr, "[waits]"); + for (std::size_t i = 0; i < hot.size() && i < 8; ++i) { + const auto& s = hot[i].second; + std::fprintf(stderr, " | %s x%llu %.1fs (avg %.1f ms, max %.0f ms, %llu timeouts, tid %lu)", hot[i].first.c_str(), static_cast(s.calls), s.nanos / 1e9, s.nanos / 1e6 / static_cast(s.calls), s.longest / 1e6, static_cast(s.timeouts), s.lastThread); + } + std::fprintf(stderr, "\n"); + for (auto& [name, s] : sites) s = Site{}; +} + int APS5_VABI sceKernelUsleep_nid_postfix(KernelUseconds microseconds) { + TraceSleep(__builtin_return_address(0), microseconds); SleepNanos(static_cast(microseconds) * 1000ULL); return 0; } @@ -240,59 +371,62 @@ int APS5_VABI clock_getres_nid_postfix(int clockId, KernelTimespec* res) { // Moved as-is (not yet implemented) from the monolithic libkernel/Export.cpp. // --------------------------------------------------------------------------- +// The sce* clock calls share the POSIX implementations above. The console clock is kept in UTC: +// the time zone reported here and by gettimeofday has no offset and no daylight saving. +static constexpr int SceKernelErrorEfault = static_cast(0x8002000e); + int APS5_VABI sceKernelClockGetres(KernelClockid clock_id, KernelTimespec* tp) { - (void)clock_id; - (void)tp; - NotImplemented_nid_no_patch(__func__); - return 0; + if (tp == nullptr) return SceKernelErrorEfault; + return clock_getres_nid_postfix(static_cast(clock_id), tp); } int APS5_VABI sceKernelClockGettime(KernelClockid clock_id, KernelTimespec* tp) { - (void)clock_id; - (void)tp; - NotImplemented_nid_no_patch(__func__); - return 0; + if (tp == nullptr) return SceKernelErrorEfault; + return clock_gettime_nid_postfix(static_cast(clock_id), tp); } int APS5_VABI sceKernelConvertLocaltimeToUtc(int64_t local_time, int64_t reserved, int64_t* utc_time, KernelTimezone* timezone, int32_t* dst_seconds) { - (void)local_time; - (void)reserved; - (void)utc_time; - (void)timezone; - (void)dst_seconds; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)reserved; + if (utc_time != nullptr) *utc_time = local_time; + if (timezone != nullptr) *timezone = {0, 0}; + if (dst_seconds != nullptr) *dst_seconds = 0; + return 0; } int APS5_VABI sceKernelConvertUtcToLocaltime(int64_t utc_time, int64_t* local_time, KernelTimesec* st, uint64_t* dst_sec) { - (void)utc_time; - (void)local_time; - (void)st; - (void)dst_sec; - NotImplemented_nid_no_patch(__func__); - return 0; + if (local_time != nullptr) *local_time = utc_time; + if (st != nullptr) *st = {utc_time, 0u, 0u}; + if (dst_sec != nullptr) *dst_sec = 0; + return 0; } int APS5_VABI sceKernelGettimeofday(KernelTimeval* tp) { - (void)tp; - NotImplemented_nid_no_patch(__func__); - return 0; + if (tp == nullptr) return SceKernelErrorEfault; + return gettimeofday_nid_postfix(tp, nullptr); } int APS5_VABI sceKernelGettimezone(KernelTimezone* tz) { - (void)tz; - NotImplemented_nid_no_patch(__func__); - return 0; + if (tz == nullptr) return SceKernelErrorEfault; + *tz = {0, 0}; + return 0; } uint64_t APS5_VABI sceKernelReadTsc(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + if (TimeScale() == 1.0) return __rdtsc(); + static const std::uint64_t origin = __rdtsc(); + return origin + static_cast(static_cast(__rdtsc() - origin) * TimeScale()); } uint64_t APS5_VABI sceKernelGetTscFrequency(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + static const std::uint64_t frequency = [] { + const std::uint64_t startNanos = RawMonotonicNanos(); + const std::uint64_t startTicks = __rdtsc(); + SleepNanos(20000000ULL); + const std::uint64_t elapsedNanos = RawMonotonicNanos() - startNanos; + const std::uint64_t elapsedTicks = __rdtsc() - startTicks; + return static_cast(static_cast(elapsedTicks) * 1000000000.0L / static_cast(elapsedNanos)); + }(); + return frequency; } unsigned int APS5_VABI sceKernelSleep(unsigned int seconds) { diff --git a/core/libs/prx/libkernel/Time/include/Time.hpp b/core/libs/prx/libkernel/Time/include/Time.hpp index 71427c05df..dc1f2a0d85 100644 --- a/core/libs/prx/libkernel/Time/include/Time.hpp +++ b/core/libs/prx/libkernel/Time/include/Time.hpp @@ -17,6 +17,11 @@ extern "C" { int APS5_VABI clock_getres_nid_postfix(int clockId, KernelTimespec* res); int APS5_VABI gettimeofday_nid_postfix(KernelTimeval* tv, KernelTimezone* tz); + // Debug aid: APS5_TRACE_WAITS=1 aggregates blocking waits (event flags, semaphores, condition + // variables, event queues) per guest call site and prints the sites that wait longest every 3 s, + // with their timeout rate; a site that times out once per frame is a lost wake-up. + void KernelTraceWait_nid_postfix(const char* kind, const void* caller, std::uint64_t waitedNanos, bool timedOut); + } #endif diff --git a/core/libs/prx/libkernel/Unimplemented.cpp b/core/libs/prx/libkernel/Unimplemented.cpp new file mode 100644 index 0000000000..5edc7a3e2f --- /dev/null +++ b/core/libs/prx/libkernel/Unimplemented.cpp @@ -0,0 +1,88 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI rename_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceCoredumpWriteUserData() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAioPollRequest() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprGetFileSize() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprGetFileStat() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprResolveFilepathsToIdsAndFileSizesForEach() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprResolveFilepathsToIdsForEach() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprResolveFilepathsWithPrefixToIds() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprResolveFilepathsWithPrefixToIdsAndFileSizes() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprResolveFilepathsWithPrefixToIdsAndFileSizesForEach() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprResolveFilepathsWithPrefixToIdsForEach() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprSubmitCommandBufferAndGetId() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprSubmitCommandBufferAndGetResult() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelAprWaitCommandBuffer() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelFtruncate() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI shutdown_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} From 87911b3bb5229e119bec4d106501615062e6f77c Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:27:49 -0700 Subject: [PATCH 02/93] fix(recompiler): saveexec operand order, atomic-zero rewrite, cache keys - s_*_saveexec reads its source before writing the old EXEC into the destination, so the aliased form (vcc, vcc) computes exec & vcc; the per-lane exec bit comes from the lane's own bit. Every Bink 2 decoder kernel uses that form; this was the video corruption. - buffer atomics add/sub/or/xor with a zero operand become an atomic load or nothing. - DescriptorBinding carries bufferWritten/bufferAtomic/imageWritten; results expose a variantId and share SPIR-V copy-on-write; source keys use a length plus 64-bit hash; captures can be reused. - wave-LDS ordering barriers, v_movrels/v_movreld lowering, SMEM with a VCC base, f16 conversion opcode fix, a per-function SPIRV-Tools pass list, env-gated IR dumps and probes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/shader/recompiler/BdaAbi.hpp | 2 +- core/shader/recompiler/CacheKey.hpp | 26 ++- core/shader/recompiler/Deserialize.cpp | 2 +- .../src/IrBuilder/IrBuilderCore.cpp | 4 +- .../include/Optimization/ResourceProgram.hpp | 14 ++ .../SrtWalker/SrtDescriptorEvaluation.hpp | 5 + .../src/DescriptorBindingBuilder.cpp | 48 ++++- .../Optimization/src/ReadLaneEliminator.cpp | 92 +++++++++ .../Optimization/src/RequestMemoryView.cpp | 13 +- .../Optimization/src/ResourceTracker.cpp | 23 +++ .../src/SrtWalker/SrtDescriptorEvaluation.cpp | 49 ++++- .../src/SrtWalker/SrtEvaluator.cpp | 5 + .../Optimization/src/SrtWalker/SrtWalker.cpp | 4 +- .../RdnaDecoder/src/RdnaMemoryOpDecoder.cpp | 14 +- core/shader/recompiler/Recompiler.cpp | 191 +++++++++++++----- core/shader/recompiler/Recompiler.hpp | 76 ++++++- .../include/SpirvBackend/SpirvBda.hpp | 3 + .../SpirvBackend/SpirvEmitterHelpers.hpp | 27 +++ .../SpirvBackend/SpirvEmitterState.hpp | 10 + .../SpirvBackend/src/RectListShaders.cpp | 13 +- .../SpirvBackend/src/SpirvBdaFault.cpp | 1 + .../SpirvBackend/src/SpirvBdaRead.cpp | 40 +++- .../SpirvBackend/src/SpirvEmitter.cpp | 34 ++++ .../SpirvBackend/src/SpirvFlowEmitter.cpp | 95 ++++++++- .../SpirvBackend/src/SpirvImageEmitter.cpp | 15 +- .../src/SpirvMemory/SpirvFormatConvert.cpp | 6 + .../SpirvMemory/SpirvMemoryInstructions.cpp | 98 ++++++++- .../SpirvBackend/src/SpirvOptimizer.cpp | 37 +++- .../Translation/ShaderInputInfoBuilder.hpp | 4 +- .../Translation/TranslationContext.hpp | 13 +- .../src/ControlFlowInstructions.cpp | 51 ++++- .../Translation/src/DispatchInstructions.cpp | 42 ++++ .../MemoryInstructions/BufferInstructions.cpp | 6 +- .../MemoryInstructions/ImageInstructions.cpp | 8 + .../MemoryResourceHelpers.cpp | 11 +- .../ScalarBufferInstructions.cpp | 6 +- .../src/ShaderInputInfoBuilder.cpp | 5 +- .../src/TranslationContext/SubwordPacking.cpp | 6 +- 38 files changed, 982 insertions(+), 117 deletions(-) diff --git a/core/shader/recompiler/BdaAbi.hpp b/core/shader/recompiler/BdaAbi.hpp index 3172960da4..148746e9db 100644 --- a/core/shader/recompiler/BdaAbi.hpp +++ b/core/shader/recompiler/BdaAbi.hpp @@ -27,7 +27,7 @@ struct Range { }; enum class FaultState : std::uint32_t { Empty, Writing, Ready }; -enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle }; +enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit }; struct Fault { FaultState state; diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index df7af4478a..c11a390c83 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -2,6 +2,7 @@ #define CORE_SHADER_RECOMPILER_CACHEKEY_HPP #include "Recompiler.hpp" +#include #include #include @@ -12,7 +13,16 @@ public: static void Build(const RecompileRequest& request, std::vector& key) { key.clear(); append(key, request.shader.stage); - append(key, request.shader.code); + // The code enters as a hash rather than word by word: the key is built, hashed and compared + // on every dispatch and draw. The cache verifies a match against the code it stored. + // Debug aid: APS5_NO_CODE_HASH_KEY=1 appends every code word, as before. + static const bool hashedCode = std::getenv("APS5_NO_CODE_HASH_KEY") == nullptr; + if (hashedCode) { + append(key, request.shader.code.size()); + append(key, HashCode(request.shader.code)); + } else { + append(key, request.shader.code); + } append(key, request.context.waveSize); append(key, request.context.userDataBaseRegister); append(key, request.context.userData.size()); @@ -20,6 +30,20 @@ public: append(key, request.context.pixel); append(key, request.context.vertex); append(key, request.target); + append(key, DebugProbeActive()); + } + + // A 64-bit hash of the code, two dwords per step; collisions are resolved by comparing the code. + static std::uint64_t HashCode(std::span code) { + std::uint64_t hash = 0x9e3779b97f4a7c15ull ^ (static_cast(code.size()) * 0x100000001b3ull); + const auto mix = [&](std::uint64_t chunk) { + hash = (hash ^ chunk) * 0x9e3779b97f4a7c15ull; + hash ^= hash >> 29u; + }; + std::size_t index = 0; + for (; index + 2 <= code.size(); index += 2) mix(code[index] | (static_cast(code[index + 1]) << 32u)); + if (index < code.size()) mix(code[index]); + return hash; } private: diff --git a/core/shader/recompiler/Deserialize.cpp b/core/shader/recompiler/Deserialize.cpp index 1e5759510a..377bebb3a6 100644 --- a/core/shader/recompiler/Deserialize.cpp +++ b/core/shader/recompiler/Deserialize.cpp @@ -152,7 +152,7 @@ int main(int argc, char** argv) { std::cout << "Disassembly written to disasm.txt (" << disasm.size() << " bytes)\n"; const ShaderStageKind stageKind = ToShaderStageKind(request.shader.stage); - const auto inputInfo = BuildShaderStageInputInfo(stageKind, request.context); + const auto inputInfo = BuildShaderStageInputInfo(stageKind, request.context, request.target.subgroupSize); GraphBuilder graphBuilder; auto cfg = graphBuilder.Build(decoded); diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrBuilder/IrBuilderCore.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrBuilder/IrBuilderCore.cpp index db7a8e94c2..3e41ba7529 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrBuilder/IrBuilderCore.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrBuilder/IrBuilderCore.cpp @@ -25,7 +25,7 @@ IrValue& IrBuilder::Emit(IrOpcode opcode, IrType type, std::initializer_listType() != IrOpcodeArgumentType(opcode, index)) { - throw std::invalid_argument("IrBuilder::Emit argument type mismatch at index " + std::to_string(index)); + throw std::invalid_argument("IrBuilder::Emit argument type mismatch at index " + std::to_string(index) + " of " + std::string(IrOpcodeName(opcode)) + ": expected " + TypeToString(IrOpcodeArgumentType(opcode, index)) + ", got " + TypeToString(argument->Type()) + " from " + std::string(IrOpcodeName(argument->Opcode()))); } value.AddArgument(argument); ++index; @@ -48,7 +48,7 @@ IrValue& IrBuilder::Emit(IrOpcode opcode, IrType type, std::initializer_listType() != IrOpcodeArgumentType(opcode, index)) { - throw std::invalid_argument("IrBuilder::Emit argument type mismatch at index " + std::to_string(index)); + throw std::invalid_argument("IrBuilder::Emit argument type mismatch at index " + std::to_string(index) + " of " + std::string(IrOpcodeName(opcode)) + ": expected " + TypeToString(IrOpcodeArgumentType(opcode, index)) + ", got " + TypeToString(argument->Type()) + " from " + std::string(IrOpcodeName(argument->Opcode()))); } value.AddArgument(argument); ++index; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index 7f02d33b8f..e22701682b 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -3,12 +3,26 @@ #include "Recompiler.hpp" #include "IntermediateRepresentation/IrProgram.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include namespace ShaderRecompiler { [[nodiscard]] IrProgram PrepareResourceProgram(const RecompileRequest& request); [[nodiscard]] std::shared_ptr GetResourcePlan(const RecompileRequest& request); +// What a driver's capture of a request produces: the plan and the materialization of the words the +// runtime read through it. Recompile(request, capture) compiles from these without a second walk. +struct SourceEntry; +struct ResourceCapture { + std::shared_ptr plan; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + // The cache entry the plan belongs to; null when the request bypasses the cache. + std::shared_ptr source; +}; +[[nodiscard]] std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime); + } #endif diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp index f962868dad..c1b008e9b5 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp @@ -8,8 +8,13 @@ #include #include +#include + namespace ShaderRecompiler::Detail { +// Why the last EvaluateRuntimeSourcesImpl call on this thread returned false. +const std::string& RuntimeSourceFailureReason(); + bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources); } diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index ba7c3fe4f6..eac82b77ce 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -1,6 +1,9 @@ #include "Optimization/DescriptorBindingBuilder.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include +#include +#include +#include #include #include #include @@ -14,6 +17,19 @@ namespace { throw std::runtime_error(message); } +// Debug aid: APS5_TRACE_BUFFER_WRITTEN=1 prints, after every Populate, how many guest buffer +// elements the program may store to and how many it only loads (this call and cumulative), so the +// replay tool shows what a driver gains from DescriptorBinding::bufferWritten. +bool bufferWrittenTraceEnabled() { + static const bool enabled = std::getenv("APS5_TRACE_BUFFER_WRITTEN") != nullptr; + return enabled; +} +struct BufferWrittenCounts { + std::atomic written{0}; + std::atomic readOnly{0}; +}; +BufferWrittenCounts bufferWrittenCounts; + DescriptorKind PhysicalKindFor(DescriptorBindingKind kind) { if (kind == DescriptorBindingKind::Samplers) { return DescriptorKind::Sampler; @@ -71,11 +87,10 @@ DescriptorImageShape ImageShapeForResource(const ImageResource& image) { return DescriptorImageShape::Image3D; } if (info.spirvDimension == spv::Dim2D) { - if (image.cube) { - if (info.arrayed == 0u) { - fail("DescriptorBindingBuilder::Populate cube image resource is not arrayed"); - } - return DescriptorImageShape::ImageCube; + // Cube images are declared and addressed as 2D arrays of faces (the backend converts + // cube coordinates to face layers), so they bind as 2D arrays. + if (image.cube && info.arrayed == 0u) { + fail("DescriptorBindingBuilder::Populate cube image resource is not arrayed"); } return info.arrayed != 0u ? DescriptorImageShape::Image2DArray : DescriptorImageShape::Image2D; } @@ -187,6 +202,8 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con std::vector bindings; bindings.reserve(layout.descriptors.size()); + std::size_t writtenHere = 0; + std::size_t readOnlyHere = 0; for (const IrDescriptorBinding& logical : layout.descriptors) { DescriptorBinding physical; physical.descriptorSet = 0u; @@ -199,10 +216,24 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con switch (physical.role) { case DescriptorRole::GuestBuffers: physical.guestDescriptor = GuestBuffersDescriptor(logical.resources, snapshot); + for (const std::uint32_t resource : logical.resources) { + const auto& buffer = info.buffers.at(resource); + physical.bufferAtomic.push_back(buffer.atomic); + // The tracker merges every buffer access of a source into its resource + // (ResourceTracker::Merge), so an element without a store or atomic is read-only + // over its whole extent; stores through pointers (BDA) never bind a V#. + physical.bufferWritten.push_back(buffer.written || buffer.atomic); + if (buffer.written || buffer.atomic) ++writtenHere; + else ++readOnlyHere; + } break; case DescriptorRole::GuestImages: physical.guestDescriptor = GuestImagesDescriptor(logical.resources, snapshot); physical.imageShape = ImageShapeFor(info.images, logical.resources); + for (const std::uint32_t resource : logical.resources) { + const auto& image = info.images.at(resource); + physical.imageWritten.push_back(image.written || image.atomic); + } break; case DescriptorRole::GuestSamplers: physical.guestDescriptor = GuestSamplersDescriptor(logical.resources, snapshot); @@ -243,6 +274,13 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con bindings.push_back(std::move(physical)); } + if (bufferWrittenTraceEnabled()) { + // Cumulative counts include every Populate of the process (the replay tool populates + // each program twice), so the per-call counts are the ones to sum per program. + const auto written = bufferWrittenCounts.written.fetch_add(writtenHere) + writtenHere; + const auto readOnly = bufferWrittenCounts.readOnly.fetch_add(readOnlyHere) + readOnlyHere; + std::fprintf(stderr, "[bindings] guest buffer elements: %zu written, %zu read-only (total so far: %llu / %llu)\n", writtenHere, readOnlyHere, written, readOnly); + } allocation.bindings = std::move(bindings); allocation.pushConstants.clear(); if (layout.UsesPushData()) { diff --git a/core/shader/recompiler/Optimization/src/ReadLaneEliminator.cpp b/core/shader/recompiler/Optimization/src/ReadLaneEliminator.cpp index ae887548ef..b4ec817c1a 100644 --- a/core/shader/recompiler/Optimization/src/ReadLaneEliminator.cpp +++ b/core/shader/recompiler/Optimization/src/ReadLaneEliminator.cpp @@ -98,6 +98,91 @@ IrValue* getRealValue(IrProgram& program, PhiMap& phiMap, IrValue* source, std:: return φ } +// Spilled SGPRs written with v_writelane inside divergent control flow reach their v_readlane through +// per-lane exec selects and loop phis. Lane `lane` of such a vector can only hold what some WriteLane +// stored to that lane, so the read is collected as the set of those stored values; operands that +// enter through a phi without any lane write (the spill slot's initial contents) are never read back. +struct LaneValues { + std::unordered_set values; + std::unordered_set visited; + bool failed = false; +}; + +void collectLaneValues(IrValue* value, std::uint32_t lane, std::uint32_t waveSize, bool throughPhi, LaneValues& out) { + if (out.failed) return; + value = value->Resolve(); + if (!out.visited.insert(value).second) return; + if (out.visited.size() > 4096u) { + out.failed = true; + return; + } + switch (value->Opcode()) { + case IrOpcode::WriteLane: { + IrValue* selector = value->Argument(1)->Resolve(); + if (!isImmediateU32(*selector)) { + out.failed = true; + return; + } + if (selector->ImmediateU32() % waveSize == lane) { + out.values.insert(value->Argument(0)->Resolve()); + return; + } + collectLaneValues(value->Argument(2), lane, waveSize, throughPhi, out); + return; + } + case IrOpcode::SelectU32: + collectLaneValues(value->Argument(1), lane, waveSize, throughPhi, out); + collectLaneValues(value->Argument(2), lane, waveSize, throughPhi, out); + return; + default: + if (value->IsPhi()) { + for (std::size_t index = 0; index < value->ArgumentCount(); index++) collectLaneValues(value->Argument(index), lane, waveSize, true, out); + return; + } + if (!throughPhi) out.failed = true; + return; + } +} + +// A stored value computed from the read itself (a spilled loop counter) is not a fixed value. +bool dependsOn(IrValue* value, IrValue* target) { + std::vector pending{value}; + std::unordered_set visited; + while (!pending.empty()) { + IrValue* current = pending.back()->Resolve(); + pending.pop_back(); + if (current == target) return true; + if (!visited.insert(current).second || visited.size() > 65536u) continue; + for (std::size_t index = 0; index < current->ArgumentCount(); index++) pending.push_back(current->Argument(index)); + } + return false; +} + +// A scalar phi whose operands are all `read` itself or members of `candidates` adds nothing new. +IrValue* uniqueLaneValue(IrValue* read, std::unordered_set candidates) { + std::unordered_set expanded; + for (bool changed = true; changed;) { + changed = false; + for (IrValue* candidate : std::vector(candidates.begin(), candidates.end())) { + if (!candidate->IsPhi() || !expanded.insert(candidate).second) continue; + std::vector operands; + bool closed = true; + for (std::size_t index = 0; index < candidate->ArgumentCount(); index++) { + IrValue* operand = candidate->Argument(index)->Resolve(); + if (operand == read || operand == candidate || expanded.contains(operand)) continue; + operands.push_back(operand); + } + if (operands.empty()) closed = false; + if (!closed) continue; + candidates.erase(candidate); + for (IrValue* operand : operands) candidates.insert(operand); + changed = true; + } + } + candidates.erase(read); + return candidates.size() == 1u ? *candidates.begin() : nullptr; +} + } ReadLaneEliminationStats ReadLaneEliminator::Eliminate(IrProgram& program, std::uint32_t waveSize) const { @@ -124,6 +209,13 @@ ReadLaneEliminationStats ReadLaneEliminator::Eliminate(IrProgram& program, std:: continue; } if (!chain.value->IsPhi() || !isPossibleToEliminate(chain.value, lane, waveSize)) { + LaneValues values; + collectLaneValues(inst->Argument(0), lane, waveSize, false, values); + if (values.failed || values.values.empty()) continue; + if (IrValue* unique = uniqueLaneValue(inst, values.values); unique != nullptr && !dependsOn(unique, inst)) { + inst->ReplaceUsesWith(unique, true); + stats.rewrittenReads++; + } continue; } diff --git a/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp b/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp index bd51e770e6..249778f064 100644 --- a/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp +++ b/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp @@ -1,5 +1,7 @@ #include "Optimization/RequestMemoryView.hpp" #include +#include +#include #include #include #include @@ -37,19 +39,24 @@ RequestMemoryView::RequestMemoryView(std::span memoryRegions bool RequestMemoryView::ReadGuestMemory(void* userContext, std::uint64_t address, std::uint32_t* value) { const auto* self = static_cast(userContext); + static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr; + const auto miss = [&] { + if (debug) std::fprintf(stderr, "[srt] guest read 0x%llx is outside the request memory\n", static_cast(address)); + return false; + }; const auto it = std::upper_bound(self->regions.begin(), self->regions.end(), address, [](std::uint64_t addressValue, const MemoryRegion& region) { return addressValue < region.guestAddress; }); if (it == self->regions.begin()) { - return false; + return miss(); } const MemoryRegion& region = *std::prev(it); if (address < region.guestAddress) { - return false; + return miss(); } const std::uint64_t offset = address - region.guestAddress; if (offset > region.bytes.size() || region.bytes.size() - offset < sizeof(std::uint32_t)) { - return false; + return miss(); } std::uint32_t word = 0; std::memcpy(&word, region.bytes.data() + offset, sizeof(word)); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index f38ef9bba8..6e11c7f2b8 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -4,6 +4,8 @@ #include #include +#include +#include #include #include #include @@ -20,6 +22,13 @@ constexpr std::uint32_t samplerDword3ReservedMask = 0x3ffff000u; throw std::runtime_error(message); } +// Debug aid: APS5_TRACE_BDA=1 names the accesses that make a program address-based (see Collect). +bool bdaTraceEnabled() { + static const bool enabled = std::getenv("APS5_TRACE_BDA") != nullptr; + return enabled; +} +constexpr unsigned bdaTraceLimit = 8; + std::string formatHex32(std::uint32_t value) { static constexpr char digits[] = "0123456789abcdef"; std::string hex(8u, '0'); @@ -101,6 +110,7 @@ public: Collect(*inst); } } + if (m_bdaTraces > bdaTraceLimit) std::fprintf(stderr, "[bda] %u more address accesses in this program not shown\n", m_bdaTraces - bdaTraceLimit); LinkImageAliases(); for (const auto& patch : m_handlePatches) { patch.handle->SetFlags(patch.resource); @@ -632,6 +642,17 @@ private: return; } ValidateAddressHandle(inst.Argument(0)); + // Debug aid: APS5_TRACE_BDA=1 names every access that makes the program address-based + // (a raw scalar load the SRT walker left in place, or a flat/global access), so the + // reason a stage takes the BDA path can be read off without a shader dump. + // The first few accesses of a program are printed (a Bink kernel has hundreds); Run + // reports how many more there were. + if (bdaTraceEnabled() && ++m_bdaTraces <= bdaTraceLimit) { + const auto* kind = memory.kind == ResourceKind::ScalarAddress ? "scalar address" : memory.kind == ResourceKind::Global ? "global" : "flat"; + const IrValue* offset = inst.ArgumentCount() > 1 ? inst.Argument(1)->Resolve() : nullptr; + const bool immediateOffset = offset != nullptr && offset->HasImmediate(); + std::fprintf(stderr, "[bda] %s at pc 0x%08x: %s access, offset %s%s\n", std::string(IrOpcodeName(op)).c_str(), flags.pc, kind, immediateOffset ? "immediate" : "dynamic", memory.kind == ResourceKind::ScalarAddress && !immediateOffset ? " (a register offset is not planned by the SRT walker)" : ""); + } m_info.usesDma = true; return; } @@ -703,6 +724,8 @@ private: std::vector m_sources; std::vector m_handlePatches; std::vector m_memoryPatches; + // APS5_TRACE_BDA: address accesses seen by Collect (the first bdaTraceLimit are printed). + unsigned m_bdaTraces = 0; std::vector m_indirectImages; }; diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp index 2f676ce9b8..89b1a36703 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp @@ -1,12 +1,36 @@ #include "Optimization/SrtWalker/SrtDescriptorEvaluation.hpp" #include "Optimization/SrtWalker/SrtEvaluator.hpp" +#include +#include #include +#include namespace ShaderRecompiler::Detail { namespace { +thread_local std::string g_failureReason; + +std::string DescribeValue(const IrValue* value, std::uint32_t depth) { + if (value == nullptr) return "null"; + value = value->Resolve(); + std::string text(IrOpcodeName(value->Opcode())); + if (value->HasImmediate() && value->Type() == IrType::U32) return text + "(" + std::to_string(value->ImmediateU32()) + ")"; + if (depth == 0 || value->ArgumentCount() == 0) return text; + text += "("; + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) { + if (index != 0) text += ", "; + text += DescribeValue(value->Argument(index), depth - 1); + } + return text + ")"; +} + +bool Fail(std::string reason) { + g_failureReason = std::move(reason); + return false; +} + const DescriptorSource* Source(const IrResourcePlan& program, std::uint32_t source) { if (source >= program.descriptorSources.size()) { return nullptr; @@ -17,11 +41,16 @@ const DescriptorSource* Source(const IrResourcePlan& program, std::uint32_t sour } bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources) { + g_failureReason.clear(); + static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr; + if (debug) { + for (std::size_t slot = 0; slot < program.srtReads.size(); ++slot) std::fprintf(stderr, "[srt] slot %zu = %s" "\n", slot, DescribeValue(program.srtReads[slot].value, 6).c_str()); + } if (!program.srtPlanComplete) { - return false; + return Fail("SRT plan is incomplete"); } if (std::any_of(cleanFlatSlots.begin(), cleanFlatSlots.end(), [](std::uint8_t clean) { return clean != 0u; }) && runtime.readSpecializationMemory == nullptr) { - return false; + return Fail("clean flat slots need specialization memory"); } SrtRuntime cleanRuntime = runtime; cleanRuntime.readMemory = runtime.readSpecializationMemory; @@ -64,14 +93,20 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::spandwordCount; if (!evaluateFlat || active[sourceIndex]) { for (std::uint32_t index = 0; index < source->dwordCount; index++) { if (!evaluator.Evaluate(source->dwords[index], value.dwords[index])) { - return false; + std::string detail = DescribeValue(source->dwords[index], 4); + const IrValue* dword = source->dwords[index]->Resolve(); + if (dword->Opcode() == IrOpcode::ReadConst && dword->ArgumentCount() == 2 && dword->Argument(1)->Resolve()->HasImmediate()) { + const auto slot = dword->Argument(1)->Resolve()->ImmediateU32(); + if (slot < program.srtReads.size()) detail += " where slot " + std::to_string(slot) + " = " + DescribeValue(program.srtReads[slot].value, 8); + } + return Fail("descriptor source " + std::to_string(sourceIndex) + " dword " + std::to_string(index) + ": " + detail); } } } @@ -84,7 +119,7 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span= flattened.size() || !selected.Evaluate(read.value, flattened[read.flatOffset])) { - return false; + return Fail(std::string(clean ? "clean " : "") + "SRT read at flat offset " + std::to_string(read.flatOffset) + ": " + DescribeValue(read.value, 4)); } } } @@ -96,4 +131,8 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span +#include +#include #include #include #include @@ -51,6 +54,8 @@ bool Evaluator::EvaluateWide(IrValue* raw, std::uint64_t& result) { const bool evaluated = EvaluateInst(*inst, out); _visiting.pop_back(); if (!evaluated) { + static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr; + if (debug) std::fprintf(stderr, "[srt] cannot evaluate %s (%zu arguments)\n", std::string(IrOpcodeName(inst->Opcode())).c_str(), inst->ArgumentCount()); return false; } _cache.emplace(inst, out); diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp index 34edcda128..2000b1c900 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp @@ -45,13 +45,13 @@ void SrtWalker::EvaluateDescriptorSources(const IrResourcePlan& program, std::sp std::vector ignored; std::vector active; if (!Detail::EvaluateRuntimeSourcesImpl(program, sources, runtime, results, ignored, false, {}, active)) { - throw std::runtime_error("SrtWalker::EvaluateDescriptorSources failed to evaluate descriptor sources"); + throw std::runtime_error("SrtWalker::EvaluateDescriptorSources failed to evaluate descriptor sources: " + Detail::RuntimeSourceFailureReason()); } } void SrtWalker::EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources) const { if (!Detail::EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, cleanFlatSlots, activeSources)) { - throw std::runtime_error("SrtWalker::EvaluateRuntimeSources failed to evaluate runtime sources"); + throw std::runtime_error("SrtWalker::EvaluateRuntimeSources failed to evaluate runtime sources: " + Detail::RuntimeSourceFailureReason()); } } diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp index 129fcce12e..04d2f8eceb 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp @@ -1,3 +1,4 @@ +#include #include "RdnaDecoder/RdnaMemoryOpDecoder.hpp" #include #include @@ -382,7 +383,18 @@ RdnaInstruction DecodeRdnaSmem(std::uint32_t programCounter, std::span 104u) { + char reason[96]; + std::snprintf(reason, sizeof(reason), "SMEM base register range overflow (sbase s%u at pc 0x%x, words %08x %08x)", sbase * 2u, programCounter, word0, word1); + throw std::runtime_error(reason); + } else { + instruction.source0 = scalarDescriptorBase(sbase * 2u, 2u, "SMEM base register range overflow"); + } instruction.source1 = scalarSource(soffsetCode); instruction.sourceCount = 2; return instruction; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 2b9ef46a5a..52be8f178d 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -1,4 +1,8 @@ #include "Recompiler.hpp" +#include +#include +#include +#include #include "CacheKey.hpp" #include #include @@ -59,9 +63,26 @@ ShaderStageKind toShaderStageKind(ShaderStage stage) { } +namespace { + +// The host subgroup width wave64 programs are laid out for. Debug aid: APS5_SINGLE_LANE= keeps the listed programs at one guest lane per invocation. +std::uint32_t HostSubgroupSize(const RecompileRequest& request) { + static const std::string list = [] { const char* text = std::getenv("APS5_SINGLE_LANE"); return text ? std::string(text) : std::string(); }(); + if (!list.empty()) { + if (list == "all") return 64u; + char address[32]; + std::snprintf(address, sizeof(address), "%llx", static_cast(request.shader.codeAddress)); + if (list.find(address) != std::string::npos) return 64u; + } + return request.target.subgroupSize; +} + +} + IrProgram PrepareResourceProgram(const RecompileRequest& request) { const auto stageKind = toShaderStageKind(request.shader.stage); - const auto inputInfo = BuildShaderStageInputInfo(stageKind, request.context); + const auto inputInfo = BuildShaderStageInputInfo(stageKind, request.context, HostSubgroupSize(request)); constexpr RdnaInstructionDecoder decoder; const auto decoded = decoder.Decode(request.shader.code); @@ -91,9 +112,20 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request) { translateOptions.embeddedFetch = embeddedFetch.loads.empty() ? nullptr : &embeddedFetch; auto program = translator.Translate(decoded, cfg, translateOptions); + // Debug aid: APS5_DUMP_IR= (or "all") prints the program after each front-end pass. + const auto dumpIr = [&](const char* pass) { + static const std::string list = [] { const char* text = std::getenv("APS5_DUMP_IR"); return text ? std::string(text) : std::string(); }(); + if (list.empty()) return; + char address[32]; + std::snprintf(address, sizeof(address), "%llx", static_cast(request.shader.codeAddress)); + if (list != "all" && list.find(address) == std::string::npos) return; + std::fprintf(stderr, "==== IR 0x%s after %s\n%s\n", address, pass, ProgramToString(program).c_str()); + }; + dumpIr("translate"); constexpr SsaBuilder ssaBuilder; ssaBuilder.Rewrite(program); + dumpIr("ssa"); constexpr ConstantFolder constantFolder; constexpr DeadCodeEliminator deadCodeEliminator; @@ -102,6 +134,7 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request) { ResolveControlFlowIdentities(program); deadCodeEliminator.RemoveIdentities(program); deadCodeEliminator.Eliminate(program); + dumpIr("fold"); constexpr ReadLaneEliminator readLaneEliminator; const auto readLaneStats = readLaneEliminator.Eliminate(program, translateOptions.waveSize); @@ -115,16 +148,16 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request) { constexpr SrtWalker srtWalker; srtWalker.BuildPlan(program); deadCodeEliminator.Eliminate(program); + dumpIr("srt"); constexpr ResourceTracker resourceTracker; resourceTracker.Track(program); deadCodeEliminator.Eliminate(program); + dumpIr("resources"); return program; } -namespace { - struct CompiledVariant { ResourceSpecialization specialization; BindingLayout layout; @@ -133,6 +166,18 @@ struct CompiledVariant { RecompileResult result; }; +struct SourceEntry { + std::mutex mutex; + // The code the entry was built for: the key carries only a hash of it, so a candidate entry is + // accepted only when its code matches word for word. Owned here because the request's span + // points into a registration the driver may replace while the entry lives on. + std::vector code; + std::shared_ptr plan; + std::vector> variants; +}; + +namespace { + struct ResourceProgram { explicit ResourceProgram(const RecompileRequest& request) : program(PrepareResourceProgram(request)), plan(ResourceMaterializer{}.ExtractPlan(program)) {} @@ -145,12 +190,6 @@ std::shared_ptr makeResourcePlan(const RecompileRequest& r return std::shared_ptr(resource, &resource->plan); } -struct SourceEntry { - std::mutex mutex; - std::shared_ptr plan; - std::vector> variants; -}; - struct SourceKeyHash { std::size_t operator()(const std::vector& key) const { std::size_t hash = 0; @@ -164,22 +203,39 @@ struct SourceKeyHash { std::shared_ptr getSource(const RecompileRequest& request) { static std::shared_mutex mutex; - static std::unordered_map, std::shared_ptr, SourceKeyHash> sources; + // Entries whose code hashes alike share a bucket; the code comparison picks the right one. + static std::unordered_map, std::vector>, SourceKeyHash> sources; thread_local std::vector key; RecompileCacheKey::Build(request, key); + const auto find = [&]() -> std::shared_ptr { + const auto found = sources.find(key); + if (found == sources.end()) return nullptr; + for (const auto& entry : found->second) { + if (std::equal(entry->code.begin(), entry->code.end(), request.shader.code.begin(), request.shader.code.end())) return entry; + } + return nullptr; + }; std::shared_ptr source; { std::shared_lock lock(mutex); - const auto found = sources.find(key); - if (found != sources.end()) source = found->second; + source = find(); } if (source == nullptr) { std::unique_lock lock(mutex); - const auto found = sources.find(key); - if (found != sources.end()) source = found->second; - else { + source = find(); + if (source == nullptr) { source = std::make_shared(); - sources.emplace(key, source); + source->code.assign(request.shader.code.begin(), request.shader.code.end()); + auto& bucket = sources[key]; + if (!bucket.empty()) { + // A second entry under one key is a code hash collision (or the unhashed key with + // identical code, which cannot happen); each is reported under the profile switch. + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static std::uint64_t collisions = 0; + ++collisions; + if (profile) std::fprintf(stderr, "[recompile] source key collision %llu: %zu entries share a key (%zu code words)\n", static_cast(collisions), bucket.size() + 1, request.shader.code.size()); + } + bucket.push_back(source); } } { @@ -192,7 +248,7 @@ std::shared_ptr getSource(const RecompileRequest& request) { } CompiledVariant compileVariant(const RecompileRequest& request, IrProgram program, const ResourceSnapshot& resourceSnapshot, const ResourceSpecialization& resourceSpecialization) { - const auto inputInfo = BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context); + const auto inputInfo = BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request)); constexpr DeadCodeEliminator deadCodeEliminator; constexpr ResourceMaterializer resourceMaterializer; resourceMaterializer.Apply(program, resourceSpecialization); @@ -219,6 +275,8 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra constexpr SpirvEmitter spirvEmitter; RecompileResult result; + static std::atomic variants{0}; + result.variantId = variants.fetch_add(1, std::memory_order_relaxed) + 1; result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions); #if ANYPS5_ENABLE_SPIRV_TOOLS @@ -269,8 +327,32 @@ bool sameLayout(const BindingLayout& left, const BindingLayout& right) { return left.descriptorSet == right.descriptorSet && left.firstBinding == right.firstBinding && left.pushConstantOffsetBytes == right.pushConstantOffsetBytes && left.pushConstantSizeBytes == right.pushConstantSizeBytes; } +// The cached variant of `source` for the specialization, compiled on first use. +RecompileResult materializeVariant(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, const ResourceSpecialization& specialization) { + std::shared_ptr variant; + bool cacheHit = false; + { + std::lock_guard lock(source.mutex); + for (const auto& candidate : source.variants) { + if (sameLayout(candidate->layout, request.layout) && candidate->specialization == specialization) { + variant = candidate; + cacheHit = true; + break; + } + } + if (variant == nullptr) { + auto program = PrepareResourceProgram(request); + variant = std::make_shared(compileVariant(request, std::move(program), snapshot, specialization)); + source.variants.push_back(variant); + } + } + auto result = materializeResult(*variant, request, snapshot); + result.cacheHit = cacheHit; + return result; +} + RecompileResult RecompileImpl(const RecompileRequest& request) { - static_cast(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context)); + static_cast(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request))); RequestMemoryView memory(request.context.memory); const auto runtime = memory.MakeRuntime(request.context.userData, request.shader.codeAddress); ResourceSnapshot snapshot; @@ -285,39 +367,24 @@ RecompileResult RecompileImpl(const RecompileRequest& request) { } const auto source = getSource(request); materializer.Materialize(*source->plan, runtime, snapshot, specialization); - std::shared_ptr variant; - bool cacheHit = false; - { - std::lock_guard lock(source->mutex); - for (const auto& candidate : source->variants) { - if (sameLayout(candidate->layout, request.layout) && candidate->specialization == specialization) { - variant = candidate; - cacheHit = true; - break; - } - } - if (variant == nullptr) { - auto program = PrepareResourceProgram(request); - variant = std::make_shared(compileVariant(request, std::move(program), snapshot, specialization)); - source->variants.push_back(variant); - } + return materializeVariant(*source, request, snapshot, specialization); +} + +// The capture already resolved the source entry (stage input validation included) and materialized +// the request over exactly the words the driver captured, so neither is repeated here. +RecompileResult RecompileImpl(const RecompileRequest& request, const ResourceCapture& capture) { + if (!request.useCache || capture.source == nullptr) { + auto program = PrepareResourceProgram(request); + const auto variant = compileVariant(request, std::move(program), capture.snapshot, capture.specialization); + return materializeResult(variant, request, capture.snapshot); } - auto result = materializeResult(*variant, request, snapshot); - result.cacheHit = cacheHit; - return result; + return materializeVariant(*capture.source, request, capture.snapshot, capture.specialization); } -} - -std::shared_ptr GetResourcePlan(const RecompileRequest& request) { - static_cast(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context)); - if (request.useCache) return getSource(request)->plan; - return makeResourcePlan(request); -} - -RecompileResult Recompile(const RecompileRequest& request) { +template +RecompileResult recompileReporting(const RecompileRequest& request, Impl&& impl) { try { - return RecompileImpl(request); + return impl(); } catch (const std::exception& e) { constexpr auto requestSerializer = RequestSerializer{}; const auto inputInfo = "\nRecompileRequest:\n" + requestSerializer.Serialize(request); @@ -328,3 +395,33 @@ RecompileResult Recompile(const RecompileRequest& request) { } } + +std::shared_ptr GetResourcePlan(const RecompileRequest& request) { + static_cast(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request))); + if (request.useCache) return getSource(request)->plan; + return makeResourcePlan(request); +} + +std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime) { + // Validates the stage inputs once per request, as GetResourcePlan and Recompile(request) do. + static_cast(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request))); + auto capture = std::make_shared(); + if (request.useCache) { + capture->source = getSource(request); + capture->plan = capture->source->plan; + } else { + capture->plan = makeResourcePlan(request); + } + ResourceMaterializer{}.Materialize(*capture->plan, runtime, capture->snapshot, capture->specialization); + return capture; +} + +RecompileResult Recompile(const RecompileRequest& request) { + return recompileReporting(request, [&] { return RecompileImpl(request); }); +} + +RecompileResult Recompile(const RecompileRequest& request, const ResourceCapture& capture) { + return recompileReporting(request, [&] { return RecompileImpl(request, capture); }); +} + +} diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index e160db4065..251fe2fd4a 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -240,6 +242,18 @@ struct DescriptorBinding { bool readOnly = false; std::optional imageShape; std::vector samplerDepthCompare; + // Guest image elements the shader stores to (or updates atomically); the others are only read. + std::vector imageWritten; + // Guest buffer elements the shader updates atomically (one entry per element of a GuestBuffers + // binding, empty otherwise). An atomic on a host-imported range is a serialized PCIe round trip + // (~0.4-0.5 us each on NVIDIA), so a driver may keep these elements in device-local memory. + std::vector bufferAtomic; + // Guest buffer elements the shader may store to through this V# (any store or atomic in the + // program, whatever its offset), one entry per element of a GuestBuffers binding, empty + // otherwise. A false entry is proved: every access of that element is a load. A driver may then + // skip the write-back and the pending-write note for the element; an element beyond the vector + // (a producer that does not fill it) must be treated as written. + std::vector bufferWritten; }; struct VertexAttribute { @@ -256,8 +270,53 @@ struct FragmentParameter { bool perVertex; }; +// Compiled SPIR-V shared between a cached variant and every result materialized from it: results +// are copied per dispatch and draw, so the words are reference counted and only duplicated when a +// holder writes to them (tests and tools patch modules in place). Reads look like a vector. +// The non-const data(), operator[], begin() and end() count as writes: on a shared holder (every +// cached result) they clone the module, so a consumer that only reads takes the result by const +// reference, or the per-dispatch copy this class removes comes back without a compiler hint. +class SharedSpirv { +public: + SharedSpirv() = default; + SharedSpirv(std::vector words) : words(std::make_shared>(std::move(words))) {} + SharedSpirv& operator=(std::vector other) { + words = std::make_shared>(std::move(other)); + return *this; + } + + [[nodiscard]] const std::vector& Words() const { return words ? *words : Empty(); } + operator const std::vector&() const { return Words(); } + [[nodiscard]] std::size_t size() const { return Words().size(); } + [[nodiscard]] bool empty() const { return Words().empty(); } + [[nodiscard]] const std::uint32_t* data() const { return Words().data(); } + [[nodiscard]] std::uint32_t* data() { return Mutable().data(); } + [[nodiscard]] const std::uint32_t& operator[](std::size_t index) const { return Words()[index]; } + [[nodiscard]] std::uint32_t& operator[](std::size_t index) { return Mutable()[index]; } + [[nodiscard]] std::vector::const_iterator begin() const { return Words().begin(); } + [[nodiscard]] std::vector::const_iterator end() const { return Words().end(); } + [[nodiscard]] std::vector::iterator begin() { return Mutable().begin(); } + [[nodiscard]] std::vector::iterator end() { return Mutable().end(); } + void resize(std::size_t count) { Mutable().resize(count); } + std::vector::iterator insert(std::vector::const_iterator where, std::initializer_list values) { return Mutable().insert(where, values); } + friend bool operator==(const SharedSpirv& left, const SharedSpirv& right) { return left.words == right.words || left.Words() == right.Words(); } + +private: + static const std::vector& Empty() { + static const std::vector empty; + return empty; + } + // Copy on write: a holder whose words are shared gets its own copy before the first write. + std::vector& Mutable() { + if (words == nullptr || words.use_count() != 1) words = std::make_shared>(Words()); + return *words; + } + + std::shared_ptr> words; +}; + struct RecompileResult { - std::vector spirv; + SharedSpirv spirv; std::vector bindings; std::vector pushConstants; std::uint32_t bdaAbiVersion = 0; @@ -267,10 +326,25 @@ struct RecompileResult { std::vector parameterExports; std::vector fragmentParameters; bool cacheHit = false; + // Identifies the compiled variant the result came from: equal ids mean identical SPIR-V and + // bindings, so drivers can reuse pipeline objects. Zero when unknown. + std::uint64_t variantId = 0; }; [[nodiscard]] RecompileResult Recompile(const RecompileRequest& request); +// The resource plan, snapshot and specialization a driver captured for the request (see +// CaptureResources in Optimization/ResourceProgram.hpp): this overload reuses them instead of +// materializing the request's memory regions again, and is otherwise Recompile(request). +struct ResourceCapture; +[[nodiscard]] RecompileResult Recompile(const RecompileRequest& request, const ResourceCapture& capture); + +// Debug aid (see DebugProbe in Translation/TranslationContext.hpp): the APS5_PROBE register probe is +// only applied while a driver has it active, so it can be limited to one dispatch; the recompile +// cache keys on it. +void SetDebugProbeActive(bool active); +[[nodiscard]] bool DebugProbeActive(); + struct RectListShaders { RecompileResult control; RecompileResult evaluation; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index 57f3205fda..00bca4f89a 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -14,6 +14,9 @@ std::uint32_t BdaLoadAddress(SpirvEmitterState& state, std::uint32_t index); void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target); +// Whether a faulting BDA access may end its invocation. Programs with workgroup barriers must keep +// every invocation running, so their faulting reads return zero instead. +bool BdaInvocationsMayStop(const IrProgram& program); void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition); std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits); std::uint32_t AddBdaAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, bool subtract); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index 99bb88e29a..f52867e06c 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -179,6 +179,33 @@ std::uint32_t EmitValueOrZeroIfCondition(SpirvEmitterState& state, std::uint32_t return EmitValueOrDefaultIfCondition(state, condition, TypeU32(state), ConstantU32(state, 0u), std::forward(function)); } +// Same structure with both arms emitting code; each arm gets its own exit block so the phi stays +// valid when an arm opens nested selections. +template +std::uint32_t EmitValueIfElse(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t type, TThen&& thenFunction, TElse&& elseFunction) { + const auto thenLabel = state.module.AllocateId(); + const auto thenExit = state.module.AllocateId(); + const auto elseLabel = state.module.AllocateId(); + const auto elseExit = state.module.AllocateId(); + const auto mergeLabel = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, mergeLabel, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, condition, thenLabel, elseLabel); + EmitLabel(state, thenLabel); + const auto thenValue = thenFunction(); + state.module.AddFunction(spv::OpBranch, thenExit); + EmitLabel(state, thenExit); + state.module.AddFunction(spv::OpBranch, mergeLabel); + EmitLabel(state, elseLabel); + const auto elseValue = elseFunction(); + state.module.AddFunction(spv::OpBranch, elseExit); + EmitLabel(state, elseExit); + state.module.AddFunction(spv::OpBranch, mergeLabel); + EmitLabel(state, mergeLabel); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpPhi, type, value, thenValue, thenExit, elseValue, elseExit); + return value; +} + template std::uint32_t AtomicUpdate(SpirvEmitterState& state, std::uint32_t pointer, ResourceKind kind, TFunction&& function) { const auto scope = kind == ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 09da195cfe..0693b53e89 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -84,6 +84,16 @@ struct SpirvEmitterState { std::uint32_t bdaPagetableVariable = 0; std::uint32_t faultBufferVariable = 0; std::uint32_t bdaPointerFunction = 0; + // False for programs with workgroup barriers: faulting BDA accesses then continue (see BdaInvocationsMayStop). + bool bdaStopsInvocations = true; + // Execution scope of the barriers that keep one guest wave's LDS accesses in program order across + // host invocations (see WaveLdsScope); 0 when none are emitted. + std::uint32_t waveLdsScope = 0; + // Debug aid (APS5_LOOP_GUARD=): after that many evaluated loop exits an invocation leaves its + // loops, and the first loop that ran out is reported as a LoopLimit fault instead of hanging the GPU. + std::uint32_t loopGuardLimit = 0; + std::uint32_t loopGuardVisits = 0; + std::uint32_t loopGuardPc = 0; std::uint32_t gdsVariable = 0; std::uint32_t gdsLength = 0; std::uint32_t pushConstantVariable = 0; diff --git a/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp b/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp index ae4e75cddf..6bfd5e6349 100644 --- a/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp +++ b/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp @@ -19,6 +19,9 @@ struct Parameter { std::uint32_t inputLocation; std::uint32_t outputLocation; bool flat; + // The vertex shader never exports this parameter; the fragment shader reads zero, as it would from + // an unwritten parameter cache entry. + bool missing = false; }; void require(bool condition, const char* message) { @@ -114,6 +117,11 @@ public: store(access(ptrOutputVec4Float, glOut, invocation, intConstant(0)), position); for (std::uint32_t i = 0; i < parameters.size(); i++) { + if (parameters[i].missing) { + const auto zero = constant(floatType, 0); + store(access(ptrOutputVec4Float, outputs[i], invocation), result(spv::OpCompositeConstruct, vec4FloatType, zero, zero, zero, zero)); + continue; + } const auto input0 = load(vec4FloatType, access(ptrInputVec4Float, inputs[i], intConstant(0))); if (parameters[i].flat) { store(access(ptrOutputVec4Float, outputs[i], invocation), input0); @@ -291,6 +299,7 @@ private: inputs.resize(parameters.size()); std::array locations {}; for (std::uint32_t i = 0; i < parameters.size(); i++) { + if (tessControl && parameters[i].missing) continue; const auto location = tessControl ? parameters[i].inputLocation : parameters[i].outputLocation; if (tessControl && locations[location] != 0) { inputs[i] = locations[location]; @@ -374,8 +383,8 @@ RectListShaders BuildRectListShaders(const RecompileResult& vertex, const Recomp for (const auto& input : fragment.fragmentParameters) { require(input.location < 32 && input.sourceLocation < 32 && locations.insert(input.location).second, "invalid fragment parameter location"); require(!input.perVertex, "custom per-vertex interpolation is unsupported"); - require(std::find(vertex.parameterExports.begin(), vertex.parameterExports.end(), input.sourceLocation) != vertex.parameterExports.end(), "fragment input has no vertex export"); - parameters.push_back({input.sourceLocation, input.location, input.flat}); + const bool exported = std::find(vertex.parameterExports.begin(), vertex.parameterExports.end(), input.sourceLocation) != vertex.parameterExports.end(); + parameters.push_back({input.sourceLocation, input.location, input.flat, !exported}); } const auto components = static_cast((parameters.size() + 1) * 4); const auto& limits = *target.tessellation; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index 7205c0b3d4..7116197934 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -61,6 +61,7 @@ void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std:: } void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition) { + if (!state.bdaStopsInvocations) return; const auto failed = state.module.AllocateId(); const auto next = state.module.AllocateId(); state.module.AddFunction(spv::OpSelectionMerge, next, spv::SelectionControlMaskNone); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index bcc5ee8935..8314af119c 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -26,11 +26,15 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe if (std::find(target.supportedExtensions.begin(), target.supportedExtensions.end(), extension) == target.supportedExtensions.end()) throw std::runtime_error(std::string("BDA requires unsupported extension ") + extension); } if (program.Resources().stage == IrShaderStage::Mesh || program.Resources().stage == IrShaderStage::TessellationControl) throw std::runtime_error("BDA fault termination requires a barrier-safe mesh or tessellation-control execution protocol"); +} + +bool BdaInvocationsMayStop(const IrProgram& program) { for (const auto* block : program.BlockOrder()) { for (const auto* instruction : block->Instructions()) { - if (instruction->Opcode() == IrOpcode::Barrier) throw std::runtime_error("BDA fault termination cannot bypass a workgroup barrier"); + if (instruction->Opcode() == IrOpcode::Barrier) return false; } } + return true; } std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits) { @@ -48,12 +52,34 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte)); const auto physical = state.module.AllocateId(); state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaPointerFunction, guest, ConstantU32(state, 1u), instruction); - StopBdaInvocationIf(state, Binary(state, spv::OpIEqual, TypeBool(state), physical, BdaConstant(state, 0u))); - const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpConvertUToPtr, bytePointer, pointer, physical); - const auto loaded = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, byteType, loaded, pointer, spv::MemoryAccessAlignedMask, 1u); - const auto value = Unary(state, spv::OpUConvert, TypeU32(state), loaded); + std::uint32_t value = 0; + if (state.bdaStopsInvocations) { + StopBdaInvocationIf(state, Binary(state, spv::OpIEqual, TypeBool(state), physical, BdaConstant(state, 0u))); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, bytePointer, pointer, physical); + const auto loaded = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, byteType, loaded, pointer, spv::MemoryAccessAlignedMask, 1u); + value = Unary(state, spv::OpUConvert, TypeU32(state), loaded); + } else { + // Unmapped bytes read as zero; the fault is recorded and every invocation reaches the + // program's barriers. + const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); + const auto before = state.currentLabel; + const auto loadLabel = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, mapped, loadLabel, merge); + EmitLabel(state, loadLabel); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, bytePointer, pointer, physical); + const auto loaded = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, byteType, loaded, pointer, spv::MemoryAccessAlignedMask, 1u); + const auto widened = Unary(state, spv::OpUConvert, TypeU32(state), loaded); + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, merge); + value = state.module.AllocateId(); + state.module.AddFunction(spv::OpPhi, TypeU32(state), value, widened, loadLabel, ConstantU32(state, 0u), before); + } result = Binary(state, spv::OpBitwiseOr, TypeU32(state), result, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), value, ConstantU32(state, byte * 8u))); } return result; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 6c53d4192a..aaf3edb5bd 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -6,7 +6,9 @@ #include "SpirvBackend/SpirvMemory/SpirvModuleSetup.hpp" #include #include +#include #include +#include #include #include #include @@ -36,6 +38,34 @@ const ShaderWorkgroupInputInfo* ShaderWorkgroupInputFor(const SpirvEmitterState& } } +// Lanes of one guest wave run in lockstep, so a lane reads LDS another lane of its wave wrote without a +// barrier. Host invocations need one; it can be issued wherever the guest wave's control flow is +// uniform across the barrier's scope: the host subgroup when it holds exactly one guest wave, or the +// workgroup when the workgroup is a single wave. +std::uint32_t WaveLdsScope(const IrProgram& program, const ShaderWorkgroupInputInfo* workgroup, std::uint32_t laneCount) { + if (program.Resources().stage != IrShaderStage::Compute || workgroup == nullptr) return 0; + bool writes = false; + for (const auto* block : program.BlockOrder()) { + for (const auto* instruction : block->Instructions()) { + const auto access = SharedAccessOf(instruction->Opcode()); + writes |= access != SharedAccess::None && access != SharedAccess::Read; + } + } + if (!writes) return 0; + const auto threads = std::max(workgroup->threadsNum[0], 1u) * std::max(workgroup->threadsNum[1], 1u) * std::max(workgroup->threadsNum[2], 1u); + // A single-wave workgroup gets workgroup-scope barriers: on NVIDIA a subgroup-scope + // OpControlBarrier did not make one lane's LDS writes visible to the others (Bink's decode + // shaders read half their block as zeros), while the workgroup scope does. Debug aid: + // APS5_WAVE_LDS_SUBGROUP=1 restores the subgroup scope for comparison. + static const bool preferSubgroup = std::getenv("APS5_WAVE_LDS_SUBGROUP") != nullptr; + if (!preferSubgroup && threads <= program.WaveSize()) return spv::ScopeWorkgroup; + // A wave64 program kept at one lane per invocation (APS5_SINGLE_LANE reports a 64-wide host) spans + // two real subgroups, so only the workgroup scope covers it; hosts wider than 32 lanes are not + // distinguished from that case and get the same, still correct, scope. + if (laneCount == 2u || (program.WaveSize() == workgroup->hostSubgroupSize && workgroup->hostSubgroupSize <= 32u)) return spv::ScopeSubgroup; + return threads <= program.WaveSize() ? spv::ScopeWorkgroup : 0u; +} + } SpirvEmitterState::SpirvEmitterState(const IrProgram& program, const ShaderStageInputInfo& inputInfo) : module(program.Resources().stage == IrShaderStage::Mesh ? 0x00010400u : 0x00010300u), program(program), inputInfo(inputInfo), requirements(AnalyzeProgramRequirements(program)) { @@ -215,6 +245,10 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.module.RequireVersion(target.spirvVersion); const auto* workgroup = ShaderWorkgroupInputFor(state); state.laneCount = workgroup != nullptr && program.WaveSize() == 64u && workgroup->hostSubgroupSize == 32u ? 2u : 1u; + state.waveLdsScope = WaveLdsScope(program, workgroup, state.laneCount); + if (const char* guard = std::getenv("APS5_LOOP_GUARD")) state.loopGuardLimit = static_cast(std::strtoul(guard, nullptr, 0)); + // Stopped invocations would leave the wave LDS barriers incomplete. + state.bdaStopsInvocations = state.waveLdsScope == 0 && BdaInvocationsMayStop(program); EmitModuleHeader(state, bindings); EmitProgram(state); state.module.EmitEntryPoint(ExecutionModelForStage(state.program.Resources().stage), state.mainFunc, "main", state.interfaceVariables); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 52773a7d5a..0539474c38 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -1,5 +1,9 @@ #include "SpirvBackend/SpirvFlowEmitter.hpp" +#include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" +#include +#include +#include #include #include #include @@ -32,6 +36,8 @@ void EmitKillIfPixelValidMaskInactive(SpirvEmitterState& state) { EmitKillIfBoolFalse(state, active); } +// Block metadata is paired with BlockOrder by position (as IrProgram validation does); terminator +// targets name control-flow ids, which are not IR block ids. const IrBlock* TargetBlock(const IrProgram& program, std::uint32_t id) { const auto& blocks = program.BlockOrder(); const auto& blockInfo = program.Metadata().blockInfo; @@ -67,8 +73,43 @@ const BlockInfo* BlockInfoFor(const IrProgram& program, const IrBlock* block) { } void EmitReturnTerminator(SpirvValueEmitContext& ctx) { - EmitKillIfPixelValidMaskInactive(ctx.state); - ctx.state.module.AddFunction(spv::OpReturn); + auto& state = ctx.state; + if (state.loopGuardLimit != 0 && state.faultBufferVariable != 0) { + const auto pc = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), pc, state.loopGuardPc); + EmitIfCondition(state, Binary(state, spv::OpINotEqual, TypeBool(state), pc, ConstantU32(state, 0u)), [&] { + RecordBdaFault(state, BdaConstant(state, 0u), ConstantU32(state, state.loopGuardLimit), EmitBinaryU32(state, spv::OpISub, pc, ConstantU32(state, 1u)), BdaAbi::FaultReason::LoopLimit); + }); + } + EmitKillIfPixelValidMaskInactive(state); + state.module.AddFunction(spv::OpReturn); +} + +// The loop guard (see SpirvEmitterState::loopGuardLimit) at a branch that may leave a loop: counts the +// evaluation and, past the limit, takes the exit and remembers the block's PC (plus one, 0 = none). +std::uint32_t GuardLoopExit(SpirvEmitterState& state, const BlockInfo& info, std::uint32_t condition, bool exitWhenTrue) { + const auto visits = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), visits, state.loopGuardVisits); + const auto next = EmitBinaryU32(state, spv::OpIAdd, visits, ConstantU32(state, 1u)); + state.module.AddFunction(spv::OpStore, state.loopGuardVisits, next); + const auto tripped = Binary(state, spv::OpUGreaterThan, TypeBool(state), next, ConstantU32(state, state.loopGuardLimit)); + const auto previous = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), previous, state.loopGuardPc); + const auto first = Binary(state, spv::OpLogicalAnd, TypeBool(state), tripped, Binary(state, spv::OpIEqual, TypeBool(state), previous, ConstantU32(state, 0u))); + const auto pc = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelect, TypeU32(state), pc, first, ConstantU32(state, info.startPc + 1u), previous); + state.module.AddFunction(spv::OpStore, state.loopGuardPc, pc); + if (exitWhenTrue) return Binary(state, spv::OpLogicalOr, TypeBool(state), condition, tripped); + const auto stay = state.module.AllocateId(); + state.module.AddFunction(spv::OpLogicalNot, TypeBool(state), stay, tripped); + return Binary(state, spv::OpLogicalAnd, TypeBool(state), condition, stay); +} + +bool IsLoopMerge(const IrProgram& program, std::uint32_t block) { + for (const auto& info : program.Metadata().blockInfo) { + if (info.terminator.loopHeader && info.terminator.mergeBlock == block) return true; + } + return false; } std::uint32_t EmitBranchCondition(SpirvValueEmitContext& ctx, const BlockInfo& info) { @@ -123,7 +164,18 @@ void EmitStructuredTerminator(SpirvValueEmitContext& ctx, const IrProgram& progr EmitReturnTerminator(ctx); return; } - const auto condition = EmitBranchCondition(ctx, info); + if (const IrValue* resolved = info.condition->Resolve(); !resolved->HasImmediate() && !ctx.definitions.contains(resolved)) { + const std::string where = resolved->Parent() == nullptr ? std::string("no block") : "block " + std::to_string(resolved->Parent()->Id()); + const std::string message = "branch condition " + std::string(IrOpcodeName(resolved->Opcode())) + " of block " + std::to_string(info.id) + " (defined in " + where + ") was not emitted before the branch"; + ctx.Fail(message.c_str()); + } + auto condition = EmitBranchCondition(ctx, info); + if (state.loopGuardLimit != 0 && term.trueBlock != term.falseBlock) { + if (IsLoopMerge(program, term.trueBlock)) condition = GuardLoopExit(state, info, condition, true); + else if (IsLoopMerge(program, term.falseBlock)) condition = GuardLoopExit(state, info, condition, false); + } + static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr; + if (debug) std::fprintf(stderr, "[spirv] block %u branches on %%%u = %s (kind %d)\n", static_cast(info.id), condition, std::string(IrOpcodeName(info.condition->Resolve()->Opcode())).c_str(), static_cast(info.terminator.condition)); emitMerge(); state.module.AddFunction(spv::OpBranchConditional, condition, ctx.Label(trueBlock), ctx.Label(falseBlock)); return; @@ -480,6 +532,10 @@ void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& fu state.currentBlock = block; EmitLabel(state, ctx.Label(block)); bool emittedNonPhi = false; + // Wave LDS ordering (see WaveLdsScope): a barrier separates an LDS write from the next LDS access + // and a read from the next write. The block may be entered right after a write. + bool ldsWritten = true; + bool ldsRead = false; for (const IrValue* inst : block->Instructions()) { if (inst->Opcode() == IrOpcode::Phi) { if (emittedNonPhi) { @@ -488,6 +544,22 @@ void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& fu } else { emittedNonPhi = true; } + if (state.waveLdsScope != 0) { + const auto access = SharedAccessOf(inst->Opcode()); + if (inst->Opcode() == IrOpcode::Barrier) { + ldsWritten = false; + ldsRead = false; + } else if (access != SharedAccess::None) { + const bool writes = access != SharedAccess::Read; + if (ldsWritten || (writes && ldsRead)) { + state.module.AddFunction(spv::OpControlBarrier, ConstantU32(state, state.waveLdsScope), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask)); + ldsWritten = false; + ldsRead = false; + } + ldsWritten |= writes; + ldsRead |= access != SharedAccess::Write; + } + } for (std::uint32_t half = 0; half < state.laneCount; half++) { if (half != 0 && ctx.otherHalf == nullptr) { ctx.Fail(*inst, "requires a second lane context"); @@ -528,7 +600,24 @@ void EmitControlFlow(SpirvValueEmitContext& context, StructuredFunctionState& fu if (blocks.empty() || blocks.front() == nullptr) { context.Fail("structured control flow requires at least one block"); } + if (state.loopGuardLimit != 0) { + const auto pointer = TypePointer(state, spv::StorageClassPrivate, TypeU32(state)); + if (state.loopGuardVisits == 0) state.loopGuardVisits = state.module.DefineGlobalVariable(pointer, spv::StorageClassPrivate); + if (state.loopGuardPc == 0) state.loopGuardPc = state.module.DefineGlobalVariable(pointer, spv::StorageClassPrivate); + state.module.AddFunction(spv::OpStore, state.loopGuardVisits, ConstantU32(state, 0u)); + state.module.AddFunction(spv::OpStore, state.loopGuardPc, ConstantU32(state, 0u)); + } state.module.AddFunction(spv::OpBranch, context.Label(blocks.front())); + static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr; + if (debug) { + const auto& infos = program.Metadata().blockInfo; + for (std::size_t index = 0; index < blocks.size(); ++index) { + const auto* byId = BlockInfoFor(program, blocks[index]); + std::fprintf(stderr, "[spirv] order %zu: IR block %u, positional info id %u, id-matched info %s (kind %d true %u false %u)\n", index, blocks[index]->Id(), + index < infos.size() ? infos[index].id : 0xffffffffu, byId ? std::to_string(byId->id).c_str() : "none", + byId ? static_cast(byId->terminator.kind) : -1, byId ? byId->terminator.trueBlock : 0u, byId ? byId->terminator.falseBlock : 0u); + } + } for (const IrBlock* block : blocks) { if (block == nullptr) { context.Fail("structured control flow contains a null block"); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 0db1bbbd17..3107be84d9 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include @@ -659,7 +661,18 @@ void EmitGatherOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con if (setup.dref) { resultNumericClass = IrTextureNumericClass::Float; } - ctx.Define(access.inst, ResultVector(ctx, access, UnpackImageGather(ctx, access, sample), resultNumericClass, false, true)); + // Debug aid: APS5_GATHER_CONST= makes every non-depth gather return that integer (or its + // float value) in all four texels, to separate a wrong gather result from wrong math after it. + static const char* gatherConstText = std::getenv("APS5_GATHER_CONST"); + auto gathered = sample; + if (gatherConstText != nullptr && !setup.dref) { + const auto value = static_cast(std::strtoul(gatherConstText, nullptr, 0)); + const bool integer = resultNumericClass == IrTextureNumericClass::Uint || resultNumericClass == IrTextureNumericClass::Sint; + const auto component = integer ? ConstantU32(state, value) : ConstantF32(state, std::bit_cast(static_cast(value))); + gathered = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, ImageVectorType(state, resultNumericClass, 4), gathered, component, component, component, component); + } + ctx.Define(access.inst, ResultVector(ctx, access, UnpackImageGather(ctx, access, gathered), resultNumericClass, false, true)); } std::uint32_t EmitIndirectImageSelector(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, std::uint32_t key) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp index c98c29f60e..92b2588cd3 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -102,6 +103,11 @@ std::uint32_t NormalizeFormatComponent(SpirvEmitterState& state, const SpirvBuff } void EmitDeviceAtomicMemoryBarrier(SpirvEmitterState& state) { + // GCN atomics imply no fence; this device-scope barrier is the acquire side of lock/publish + // patterns (atomic, then loads of data other waves wrote). It also makes every non-returning + // atomic wait for its completion. Experiment switch: APS5_NO_ATOMIC_BARRIER=1 drops it. + static const bool disabled = std::getenv("APS5_NO_ATOMIC_BARRIER") != nullptr; + if (disabled) return; const auto semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask; state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, semantics)); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index dc73c40eef..edba86edd6 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -4,7 +4,11 @@ #include #include #include +#include +#include #include +#include +#include #include #include @@ -703,19 +707,93 @@ std::uint32_t AtomicDecrement(SpirvEmitterState& state, std::uint32_t old, std:: return Select(state, TypeU32(state), wrap, limit, next); } +// Buffer atomics whose operand is the identity element (add, sub, or, xor of 0) leave memory +// unchanged. Tile-classification kernels issue one such atomic per bin per wave with a mostly zero +// count, and every atomic on a host-imported range is a serialized PCIe round trip (Demon's Souls +// 0x…88aa800 / 0x…88ab300: 24 + 29 ms of GPU per frame for ~60K atomics each). An unused result +// skips the operation; a used one takes a relaxed atomic load of the current value instead, which is +// what the no-op update would have returned. APS5_NO_ATOMIC_ZERO_SKIP=1 restores the plain atomics. +bool HasZeroIdentity(IrOpcode opcode) { + switch (opcode) { + case IrOpcode::BufferAtomicIAdd32: + case IrOpcode::BufferAtomicISub32: + case IrOpcode::BufferAtomicOr32: + case IrOpcode::BufferAtomicXor32: + return true; + default: + return false; + } +} + +bool AtomicZeroSkipEnabled() { + static const bool disabled = std::getenv("APS5_NO_ATOMIC_ZERO_SKIP") != nullptr; + return !disabled; +} + +// Emission sites by kind under APS5_PROFILE_DRAW (a two-lane wave64 program counts each site twice). +// Printing is emission-driven: the line appears with the first site compiled at least 10 s after the +// previous line (the first line 10 s after the first site), so it lags the true totals until the +// next compile that emits a buffer atomic; the last sites of a run may never be printed. +void NoteBufferAtomicSite(bool zeroSkip, bool zeroLoad) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + static std::atomic plain{0}, skipped{0}, loads{0}; + static std::atomic lastReport{0}; + (zeroSkip ? skipped : zeroLoad ? loads : plain).fetch_add(1, std::memory_order_relaxed); + const auto now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = lastReport.load(std::memory_order_relaxed); + if (last == 0) { + // First site: start the 10 s window instead of printing a one-site line. + lastReport.compare_exchange_strong(last, now); + return; + } + if (now - last < 10 || !lastReport.compare_exchange_strong(last, now)) return; + std::fprintf(stderr, "[recompile] buffer atomic sites: %llu plain, %llu zero-skipped (unused result), %llu zero-load\n", static_cast(plain.load()), static_cast(skipped.load()), static_cast(loads.load())); +} + std::uint32_t Atomic32(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { auto& state = ctx.state; const bool lds = mem.kind == ResourceKind::Lds; - return EmitAtomicAccess(ctx, inst, mem, [&](std::uint32_t pointer) { - const std::uint32_t scope = lds ? spv::ScopeWorkgroup : spv::ScopeDevice; - const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); - if (lds) { - const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; - state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)); - } else { - EmitDeviceAtomicMemoryBarrier(state); - } - return old; + const std::uint32_t scope = lds ? spv::ScopeWorkgroup : spv::ScopeDevice; + // Buffer atomic arguments are {resource, index, offset, soffset, value, exec}. A non-zero + // immediate operand (counters bumped by 1) can never take the zero path, so it keeps the plain + // atomic instead of a compare and a dead branch the main build has no optimizer to fold. + const auto nonZeroImmediateOperand = [&]() { + const IrValue* operand = inst.Argument(inst.ArgumentCount() - 2u)->Resolve(); + return operand->HasImmediate() && operand->Type() == IrType::U32 && operand->ImmediateU32() != 0u; + }; + const bool zeroIdentity = !lds && AtomicZeroSkipEnabled() && HasZeroIdentity(inst.Opcode()) && !nonZeroImmediateOperand(); + const bool zeroSkip = zeroIdentity && !inst.HasUses(); + std::uint32_t active = ActiveArgument(ctx, inst); + std::uint32_t nonZero = 0; + if (zeroIdentity) { + nonZero = Binary(state, spv::OpINotEqual, TypeBool(state), ctx.Arg(inst, inst.ArgumentCount() - 2u), ConstantU32(state, 0u)); + if (zeroSkip) active = AndCondition(state, active, nonZero); + } + if (mem.kind == ResourceKind::Buffer) NoteBufferAtomicSite(zeroSkip, zeroIdentity && !zeroSkip); + return EmitValueOrZeroIfCondition(state, active, [&]() { + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index), [&]() { + const auto pointer = EmitMemoryElementPointer(state, access.resource, access.index); + const auto operation = [&]() { + const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); + if (lds) { + const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; + state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)); + } else { + EmitDeviceAtomicMemoryBarrier(state); + } + return old; + }; + if (!zeroIdentity || zeroSkip) return operation(); + return EmitValueIfElse(state, nonZero, TypeU32(state), operation, [&]() { + // The barrier stays so an "add 0" used as a coherent read keeps its acquire. + const auto current = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicLoad, TypeU32(state), current, pointer, ConstantU32(state, scope), ConstantU32(state, spv::MemorySemanticsMaskNone)); + EmitDeviceAtomicMemoryBarrier(state); + return current; + }); + }); }); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp index ae013026a1..aba07a8a9b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp @@ -1,6 +1,7 @@ #include "SpirvBackend/SpirvOptimizer.hpp" #include #include +#include #include #include @@ -39,8 +40,40 @@ std::vector ValidateAndOptimizeSpirv(std::span(spirv.begin(), spirv.end()); + // The recompiler emits helpers (BDA lookup, fault reporting) as functions called from every memory + // access. Exhaustive inlining multiplies module size by ~10x and optimization time by ~20x, so the + // default pipeline keeps the performance passes that work per function and leaves inlining to the driver. + // APS5_SPIRV_OPT=full restores the SPIRV-Tools performance pipeline, =none skips optimization. + if (mode == nullptr || std::string(mode) != "full") { + optimizer.RegisterPass(spvtools::CreateWrapOpKillPass()) + .RegisterPass(spvtools::CreateDeadBranchElimPass()) + .RegisterPass(spvtools::CreateAggressiveDCEPass(true)) + .RegisterPass(spvtools::CreatePrivateToLocalPass()) + .RegisterPass(spvtools::CreateLocalSingleBlockLoadStoreElimPass()) + .RegisterPass(spvtools::CreateLocalSingleStoreElimPass()) + .RegisterPass(spvtools::CreateAggressiveDCEPass(true)) + .RegisterPass(spvtools::CreateScalarReplacementPass(0)) + .RegisterPass(spvtools::CreateLocalAccessChainConvertPass()) + .RegisterPass(spvtools::CreateLocalSingleBlockLoadStoreElimPass()) + .RegisterPass(spvtools::CreateLocalSingleStoreElimPass()) + .RegisterPass(spvtools::CreateAggressiveDCEPass(true)) + .RegisterPass(spvtools::CreateLocalMultiStoreElimPass()) + .RegisterPass(spvtools::CreateAggressiveDCEPass(true)) + .RegisterPass(spvtools::CreateCCPPass()) + .RegisterPass(spvtools::CreateAggressiveDCEPass(true)) + .RegisterPass(spvtools::CreateDeadBranchElimPass()) + .RegisterPass(spvtools::CreateRedundancyEliminationPass()) + .RegisterPass(spvtools::CreateSimplificationPass()) + .RegisterPass(spvtools::CreateAggressiveDCEPass(true)) + .RegisterPass(spvtools::CreateBlockMergePass()) + .RegisterPass(spvtools::CreateSimplificationPass()); + } else { + optimizer.RegisterPerformancePasses(true); + } + // Validating after every pass costs more than the passes on large shaders; the result is validated below. + optimizer.SetValidateAfterAll(false); spvtools::OptimizerOptions options; options.set_preserve_bindings(true); options.set_preserve_spec_constants(true); diff --git a/core/shader/recompiler/Translation/include/Translation/ShaderInputInfoBuilder.hpp b/core/shader/recompiler/Translation/include/Translation/ShaderInputInfoBuilder.hpp index 169e434115..1096eb2e71 100644 --- a/core/shader/recompiler/Translation/include/Translation/ShaderInputInfoBuilder.hpp +++ b/core/shader/recompiler/Translation/include/Translation/ShaderInputInfoBuilder.hpp @@ -7,7 +7,9 @@ namespace ShaderRecompiler { -ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const GuestContext& context); +// hostSubgroupSize is the device subgroup width; wave64 workgroups on 32-wide subgroups run each +// invocation as two guest lanes so a guest wave stays one host subgroup. +ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const GuestContext& context, std::uint32_t hostSubgroupSize); } diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 147fa5181d..2aa9f3c363 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -74,7 +74,7 @@ private: IrValue* makeImageAddress(const RdnaInstruction& inst, const RdnaOperand& base); IrValue* constructU32x4(const RdnaOperand& base, std::uint32_t count); void writeImageComponents(const RdnaOperand& dst, IrValue* value, const MemoryInfo& memory, std::uint32_t componentLimit); - BufferAddress readBufferAddress(const RdnaInstruction& inst, std::uint32_t sourceOffset); + BufferAddress readBufferAddress(const RdnaInstruction& inst); IrU32 widenSubdword(IrValue* value, std::uint32_t bits, bool sign); IrValue* narrowSubdword(IrU32 value, std::uint32_t bits); bool sLoad(const RdnaInstruction& inst, bool raw); @@ -235,6 +235,17 @@ private: std::uint32_t currentVectorLimit = 1; }; +// Debug aid: APS5_PROBE=:[:] copies VGPR into v255 right after the +// instruction at is translated, and every image store then writes (v255 >> shift) instead of its +// data, so a dumped output image is a per-lane trace of that register. +struct DebugProbe { + bool enabled = false; + std::uint32_t programCounter = 0; + std::uint32_t vgpr = 0; + std::uint32_t shift = 0; +}; +[[nodiscard]] DebugProbe DebugProbeConfig(); + } #endif diff --git a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp index 58d8423d42..bcf67a5733 100644 --- a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp @@ -2,6 +2,7 @@ #include "Translation/TranslationContext.hpp" #include #include +#include #include namespace ShaderRecompiler { @@ -14,11 +15,25 @@ void TranslationContext::sSubvectorLoop(const RdnaInstruction& inst, bool begin) throw std::runtime_error("GNM subvector loop instructions are not supported by this translator"); } +// s_*_saveexec reads its source before it writes the old exec into the destination, so +// `s_and_saveexec_b64 vcc, vcc` (the Bink 2 decoder kernels gate every sparse coefficient load with +// it) masks exec by the old vcc. Reading the source after the destination write turned that form into +// exec = exec & exec. APS5_SAVEEXEC_WRITE_FIRST=1 restores the old order. +bool SaveexecWritesDestinationFirst() { + static const bool writeFirst = std::getenv("APS5_SAVEEXEC_WRITE_FIRST") != nullptr; + return writeFirst; +} + void TranslationContext::sSaveexec(const RdnaInstruction& inst, IrOpcode operation, bool negateExec, bool negateSource, bool write64) { + // Callers name the lane-wise operation; the exec mask words themselves combine bitwise. + if (operation == IrOpcode::LogicalAnd) operation = IrOpcode::BitwiseAnd32; + else if (operation == IrOpcode::LogicalOr) operation = IrOpcode::BitwiseOr32; + const bool writeFirst = SaveexecWritesDestinationFirst(); if (write64) { const std::array oldExec{IrU32(ir.GetExecLo()), IrU32(ir.GetExecHi())}; - writeU32Pair(inst.destination, oldExec); + if (writeFirst) writeU32Pair(inst.destination, oldExec); const std::array source = readU32Pair(sourceAt(inst, 0u)); + if (!writeFirst) writeU32Pair(inst.destination, oldExec); IrValue& lowExecOperand = negateExec ? ir.BitwiseNot(oldExec[0].Value()) : oldExec[0].Value(); IrValue& lowSourceOperand = negateSource ? ir.BitwiseNot(source[0].Value()) : source[0].Value(); IrValue& highExecOperand = negateExec ? ir.BitwiseNot(oldExec[1].Value()) : oldExec[1].Value(); @@ -29,19 +44,20 @@ void TranslationContext::sSaveexec(const RdnaInstruction& inst, IrOpcode operati ir.SetExecHi(newExecHi.Value()); const IrU1 nonZero(ir.LogicalOr(ir.INotEqual(newExecLo.Value(), ir.Constant(0u)), ir.INotEqual(newExecHi.Value(), ir.Constant(0u)))); ir.SetScc(nonZero.Value()); - ir.SetExec(nonZero.Value()); + ir.SetExec(threadBit({newExecLo, newExecHi}).Value()); return; } const IrU32 oldExec(ir.GetExecLo()); - writeRawU32(inst.destination, oldExec); + if (writeFirst) writeRawU32(inst.destination, oldExec); const IrU32 source = readU32(sourceAt(inst, 0u)); + if (!writeFirst) writeRawU32(inst.destination, oldExec); IrValue& execOperand = negateExec ? ir.BitwiseNot(oldExec.Value()) : oldExec.Value(); IrValue& sourceOperand = negateSource ? ir.BitwiseNot(source.Value()) : source.Value(); const IrU32 newExec(ir.Emit(operation, IrType::U32, {&execOperand, &sourceOperand})); ir.SetExecLo(newExec.Value()); const IrU1 nonZero(ir.INotEqual(newExec.Value(), ir.Constant(0u))); ir.SetScc(nonZero.Value()); - ir.SetExec(nonZero.Value()); + ir.SetExec(threadBit({newExec, IrU32(ir.GetExecHi())}).Value()); } void TranslationContext::addU32(const RdnaInstruction& inst, bool vector, bool useCarryIn) { @@ -220,12 +236,35 @@ void TranslationContext::sWqm(const RdnaInstruction& inst, bool wide) { writeRawU32(inst.destination, extractU64(result)[0]); } +// M0 holds a uniform register offset. Vector registers are SSA values with fixed indices, so an indexed +// access becomes a select over every register the program references at or past the base. void TranslationContext::vMovrelsB32(const RdnaInstruction& inst) { - throw std::runtime_error("dynamic M0-relative vector register addressing is not supported by this translator"); + const RdnaOperand& source = sourceAt(inst, 0u); + if (source.kind != RdnaOperandKind::VectorRegister) { + throw std::runtime_error("v_movrels_b32 source is not a vector register"); + } + IrValue& offset = ir.GetM0(); + IrU32 result(ir.GetVectorReg(static_cast(source.reg))); + for (std::uint32_t reg = source.reg + 1u; reg < currentVectorLimit; ++reg) { + IrValue& hit = ir.IEqual(offset, ir.Constant(reg - source.reg)); + result = IrU32(ir.Select(hit, ir.GetVectorReg(static_cast(reg)), result.Value())); + } + writeRawU32(inst.destination, result); } void TranslationContext::vMovreldB32(const RdnaInstruction& inst) { - throw std::runtime_error("dynamic M0-relative vector register addressing is not supported by this translator"); + const RdnaOperand destination = plainOperand(inst.destination); + if (destination.kind != RdnaOperandKind::VectorRegister) { + throw std::runtime_error("v_movreld_b32 destination is not a vector register"); + } + const IrU32 value = readU32(sourceAt(inst, 0u)); + IrValue& offset = ir.GetM0(); + for (std::uint32_t reg = destination.reg; reg < currentVectorLimit; ++reg) { + RdnaOperand target = destination; + target.reg = reg; + IrValue& hit = ir.IEqual(offset, ir.Constant(reg - destination.reg)); + writeRawU32(target, IrU32(ir.Select(hit, value.Value(), ir.GetVectorReg(static_cast(reg))))); + } } void TranslationContext::vReadfirstlaneB32(const RdnaInstruction& inst) { diff --git a/core/shader/recompiler/Translation/src/DispatchInstructions.cpp b/core/shader/recompiler/Translation/src/DispatchInstructions.cpp index 208248dabf..66a4ecbb0e 100644 --- a/core/shader/recompiler/Translation/src/DispatchInstructions.cpp +++ b/core/shader/recompiler/Translation/src/DispatchInstructions.cpp @@ -1,5 +1,8 @@ #include "Translation/DispatchInstructions.hpp" #include "Translation/TranslationContext.hpp" +#include "Recompiler.hpp" +#include +#include #include #include @@ -55,6 +58,45 @@ void TranslationContext::TranslateInstruction(const RdnaInstruction& instruction if (!translated) { throw std::runtime_error("opcode has no IR translation at pc " + std::to_string(instruction.programCounter)); } + if (const DebugProbe probe = DebugProbeConfig(); probe.enabled && instruction.programCounter == probe.programCounter) { + RdnaOperand source{}; + source.kind = RdnaOperandKind::VectorRegister; + source.reg = probe.vgpr; + RdnaOperand target{}; + target.kind = RdnaOperandKind::VectorRegister; + target.reg = 255u; + writeOperand(target, &readRawU32(source).Value()); + } +} + +namespace { +std::atomic g_debugProbeActive{false}; +} + +void SetDebugProbeActive(bool active) { + g_debugProbeActive.store(active); +} + +bool DebugProbeActive() { + return g_debugProbeActive.load(); +} + +DebugProbe DebugProbeConfig() { + static const DebugProbe parsed = [] { + DebugProbe result; + const char* text = std::getenv("APS5_PROBE"); + if (text == nullptr) return result; + char* end = nullptr; + result.programCounter = static_cast(std::strtoul(text, &end, 16)); + if (end == nullptr || *end != ':') return result; + result.vgpr = static_cast(std::strtoul(end + 1, &end, 10)); + if (end != nullptr && *end == ':') result.shift = static_cast(std::strtoul(end + 1, nullptr, 10)); + result.enabled = result.vgpr < 255u; + return result; + }(); + DebugProbe probe = parsed; + probe.enabled = parsed.enabled && DebugProbeActive(); + return probe; } void DispatchInstruction(TranslationContext& context, const RdnaInstruction& instruction) { diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/BufferInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/BufferInstructions.cpp index 6cdc9b30d1..92b69df705 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/BufferInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/BufferInstructions.cpp @@ -64,7 +64,7 @@ bool TranslationContext::bufferLoad(const RdnaInstruction& inst) { return false; } IrValue* resource = getBufferResource(memory); - const BufferAddress address = readBufferAddress(inst, 0u); + const BufferAddress address = readBufferAddress(inst); IrValue& exec = ir.GetExec(); IrValue& loaded = ir.Emit(opcode, IrOpcodeType(opcode), {resource, &address.index.Value(), &address.offset.Value(), &address.soffset.Value(), &exec}, addMemoryInfo(memory, inst.programCounter)); if (memory.dataBits != 32u) { @@ -82,7 +82,7 @@ bool TranslationContext::bufferLoad(const RdnaInstruction& inst) { bool TranslationContext::bufferStore(const RdnaInstruction& inst) { const MemoryInfo memory = bufferMemoryInfoFromInstruction(inst); IrValue* resource = getBufferResource(memory); - const BufferAddress address = readBufferAddress(inst, 1u); + const BufferAddress address = readBufferAddress(inst); const IrU32 data = readU32(inst.destination); IrOpcode opcode; IrValue* value; @@ -137,7 +137,7 @@ bool TranslationContext::bufferStore(const RdnaInstruction& inst) { bool TranslationContext::bufferAtomic(const RdnaInstruction& inst, IrOpcode opcode) { const MemoryInfo memory = bufferMemoryInfoFromInstruction(inst); IrValue* resource = getBufferResource(memory); - const BufferAddress address = readBufferAddress(inst, 1u); + const BufferAddress address = readBufferAddress(inst); const MemoryFlags flags = addMemoryInfo(memory, inst.programCounter); IrValue& exec = ir.GetExec(); IrValue* result; diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp index 20347c255c..edaf4c2700 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp @@ -87,6 +87,14 @@ bool TranslationContext::imageStore(const RdnaInstruction& inst) { IrValue* resource = getImageResource(memory); IrValue* address = makeImageAddress(inst, inst.source0); IrValue* data = constructU32x4(inst.destination, memory.dataDwords); + if (const DebugProbe probe = DebugProbeConfig(); probe.enabled) { + RdnaOperand probeReg{}; + probeReg.kind = RdnaOperandKind::VectorRegister; + probeReg.reg = 255u; + IrValue& probed = ir.ShiftRightLogical(readRawU32(probeReg).Value(), ir.Constant(probe.shift)); + IrValue& zero = ir.Constant(0u); + data = &ir.Emit(IrOpcode::CompositeConstructU32x4, IrOpcodeType(IrOpcode::CompositeConstructU32x4), {&probed, &zero, &zero, &zero}); + } IrValue& exec = ir.GetExec(); (void)ir.Emit(IrOpcode::ImageWrite, IrOpcodeType(IrOpcode::ImageWrite), {resource, address, data, &exec}, addMemoryInfo(memory, inst.programCounter)); return true; diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryResourceHelpers.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryResourceHelpers.cpp index d687c97fff..88e1afe14b 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryResourceHelpers.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryResourceHelpers.cpp @@ -142,12 +142,11 @@ void TranslationContext::writeImageComponents(const RdnaOperand& dst, IrValue* v } } -TranslationContext::BufferAddress TranslationContext::readBufferAddress(const RdnaInstruction& inst, std::uint32_t sourceOffset) { - std::uint32_t cursor = sourceOffset; - const auto next = [&]() { return readU32(sourceAt(inst, cursor++)); }; - const IrU32 index = inst.idxen ? next() : IrU32(ir.Constant(0u)); - const IrU32 offset = inst.offen ? next() : IrU32(ir.Constant(0u)); - const IrU32 soffset = next(); +// MUBUF/MTBUF sources: VADDR (the index first with IDXEN, then the offset with OFFEN), the resource, SOFFSET. +TranslationContext::BufferAddress TranslationContext::readBufferAddress(const RdnaInstruction& inst) { + const IrU32 index = inst.idxen ? readU32(inst.source0) : IrU32(ir.Constant(0u)); + const IrU32 offset = inst.offen ? readU32(offsetOperand(inst.source0, inst.idxen ? 1u : 0u)) : IrU32(ir.Constant(0u)); + const IrU32 soffset = readU32(inst.source2); return BufferAddress{index, offset, soffset}; } diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/ScalarBufferInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/ScalarBufferInstructions.cpp index e30aa4099c..7ae52737ca 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/ScalarBufferInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/ScalarBufferInstructions.cpp @@ -11,7 +11,8 @@ MemoryInfo scalarMemoryInfoFromInstruction(const RdnaInstruction& inst, bool raw if (inst.family != RdnaInstructionFamily::SMEM) { throw std::runtime_error("scalarMemoryInfoFromInstruction requires an SMEM instruction"); } - if (inst.source0.kind != RdnaOperandKind::ScalarRegister) { + // Raw address loads may take their 64-bit base from VCC; descriptor loads need four SGPRs. + if (inst.source0.kind != RdnaOperandKind::ScalarRegister && !(raw && inst.source0.kind == RdnaOperandKind::VccLo)) { throw std::runtime_error("scalar memory base must be a scalar register"); } MemoryInfo memory; @@ -29,7 +30,8 @@ MemoryInfo scalarMemoryInfoFromInstruction(const RdnaInstruction& inst, bool raw bool TranslationContext::sLoad(const RdnaInstruction& inst, bool raw) { const MemoryInfo memory = scalarMemoryInfoFromInstruction(inst, raw); - IrValue* resource = raw ? getScalarAddressResource(inst.source0.reg) : getBufferResource(memory); + const std::uint32_t baseCode = inst.source0.kind == RdnaOperandKind::VccLo ? 106u : inst.source0.reg; + IrValue* resource = raw ? getScalarAddressResource(baseCode) : getBufferResource(memory); const IrU32 offset = readU32(inst.source1); std::array loaded{}; for (std::uint32_t component = 0u; component < memory.dataDwords; ++component) { diff --git a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp index 1aaf5e1eff..92fe17ef04 100644 --- a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp +++ b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp @@ -67,7 +67,7 @@ void _detectVertexBuffers(ShaderVertexInputInfo& info) { } -ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const GuestContext& context) { +ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const GuestContext& context, std::uint32_t hostSubgroupSize) { switch (stage) { case ShaderStageKind::Compute: { if (!context.compute.has_value()) { @@ -80,11 +80,14 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues computeStorage.threadsNum[2] = compute.numThreads[2]; computeStorage.ldsSizeDwords = compute.ldsSizeDwords; computeStorage.waveSize = context.waveSize; + computeStorage.hostSubgroupSize = hostSubgroupSize; computeStorage.groupId[0] = compute.groupIdEnable[0]; computeStorage.groupId[1] = compute.groupIdEnable[1]; computeStorage.groupId[2] = compute.groupIdEnable[2]; computeStorage.tgSizeEn = compute.tgSizeEnable; computeStorage.threadIdsNum = static_cast(compute.threadIdComponentCount); + // Workgroup ids (and the thread-group size word) follow the user SGPRs. + computeStorage.workgroupRegister = static_cast(context.userDataBaseRegister + context.userData.size()); ShaderStageInputInfo result; result.compute = &computeStorage; return result; diff --git a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp index e1ba9818f5..2179313bf1 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp @@ -30,7 +30,7 @@ IrF32 TranslationContext::readF16LaneAsF32(const RdnaOperand& operand, bool high const IrU32 raw = readU16LaneRaw(operand, selectHigh); const IrU16 bits(ir.Emit(IrOpcode::ConvertU16U32, IrType::U16, {&raw.Value()})); const IrF16 half(ir.Emit(IrOpcode::BitCastF16U16, IrType::F16, {&bits.Value()})); - IrF32 value(ir.Emit(IrOpcode::ConvertF16F32, IrType::F32, {&half.Value()})); + IrF32 value(ir.Emit(IrOpcode::ConvertF32F16, IrType::F32, {&half.Value()})); if (operand.absolute) { value = IrF32(ir.Emit(IrOpcode::FPAbs32, IrType::F32, {&value.Value()})); } @@ -45,7 +45,7 @@ IrF32 TranslationContext::readF16AsF32(const RdnaOperand& operand) { const IrU32 raw = applyBitSourceModifiers(operand, readRawU32(operand)); const IrU16 bits(ir.Emit(IrOpcode::ConvertU16U32, IrType::U16, {&raw.Value()})); const IrF16 half(ir.Emit(IrOpcode::BitCastF16U16, IrType::F16, {&bits.Value()})); - IrF32 value(ir.Emit(IrOpcode::ConvertF16F32, IrType::F32, {&half.Value()})); + IrF32 value(ir.Emit(IrOpcode::ConvertF32F16, IrType::F32, {&half.Value()})); if (operand.absolute) { value = IrF32(ir.Emit(IrOpcode::FPAbs32, IrType::F32, {&value.Value()})); } @@ -104,7 +104,7 @@ void TranslationContext::write16Bits(const RdnaOperand& operand, IrU32 value) { } void TranslationContext::writeF16(const RdnaOperand& operand, IrF32 value) { - const IrF16 half(ir.Emit(IrOpcode::ConvertF32F16, IrType::F16, {&value.Value()})); + const IrF16 half(ir.Emit(IrOpcode::ConvertF16F32, IrType::F16, {&value.Value()})); const IrU16 bits(ir.Emit(IrOpcode::BitCastU16F16, IrType::U16, {&half.Value()})); write16Bits(operand, IrU32(ir.Emit(IrOpcode::ConvertU32U16, IrType::U32, {&bits.Value()}))); } From 8a69fefe1346fccc7c2f9c0912fa44b783a42462 Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:27:49 -0700 Subject: [PATCH 03/93] perf(agc): batch GPU work in a recorder with host-imported guest memory Replaces the synchronous per-command execution model, and with it the upstream render-target, pipeline-cache, tile-buffer, batched-draw and memory-ownership commits (5a280ce, 7ed5a0b, 46c799d, 598caac, 40df528), whose files are removed. - Recorder: one open command buffer per device; dispatches, resident draws, uploads, write-backs, labels, fills and copies record into it; timeline-semaphore serials let drains wait without the GPU mutex; completions run in order at reap. - one worker thread per PM4 queue; labels and stores written by the GPU into the host import; GPU-side DISPATCH_INDIRECT; the buffer-fill kernel replaced by vkCmdFillBuffer; a dispatch cache validated by write watch; filler packets tolerated; rejected submissions dumped. - guest memory: page-state cache, 64 KiB write generations, host imports with VK_EXT_external_memory_host, persistent mirrors of the executable image, GPU copies of misaligned sub-ranges. - textures: shader-written images stay resident with deferred write-back (adjacent-generation rule for packed video planes), sampled views of storage images, DCC key handling, addrlib swizzle equations, bidirectional detiler. - draws: MRT, more formats, dynamic viewport/scissor, cached pipelines, resident 64 KiB targets, draws recorded into the recorder. - ShaderResources: two-stage build, descriptor and sampler caches, a content-keyed resource cache; presentation from the resident image. - a label deferred behind a completion is no longer re-stored by the CPU at reap time unless a write-back overwrote it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 19 +- .../Execution/include/GuestMemory.hpp | 155 +- .../libSceAgcDriver/Execution/include/Pm4.hpp | 51 + .../Execution/include/PresentationScaler.hpp | 11 + .../Execution/include/ShaderMemory.hpp | 30 +- .../Execution/include/VulkanDevice.hpp | 96 +- .../Execution/include/WorkerSampler.hpp | 11 + .../Execution/src/DisplayBuffer.cpp | 41 +- .../libSceAgcDriver/Execution/src/Driver.cpp | 2182 +++++++++++++++-- .../Execution/src/GuestMemory.cpp | 1085 +++++++- .../prx/libSceAgcDriver/Execution/src/Pm4.cpp | 247 +- .../Execution/src/PresentationScaler.cpp | 46 +- .../Execution/src/ShaderMemory.cpp | 120 +- .../Execution/src/VulkanDevice.cpp | 1353 ++++++++-- .../Execution/src/WorkerSampler.cpp | 181 ++ .../Graphics/include/BdaResources.hpp | 13 +- .../Graphics/include/BufferPool.hpp | 69 +- .../Graphics/include/ColorTargetLayout.hpp | 12 +- .../Graphics/include/Context.hpp | 12 +- .../Graphics/include/DccMetadata.hpp | 53 + .../libSceAgcDriver/Graphics/include/Draw.hpp | 17 + .../Graphics/include/GpuColorTransfer.hpp | 13 +- .../include/GraphicsPipelineCache.hpp | 30 - .../Graphics/include/GuestBufferMemory.hpp | 157 +- .../Graphics/include/GuestTextureResource.hpp | 27 +- .../Graphics/include/Pipeline.hpp | 63 +- .../Graphics/include/Recorder.hpp | 262 ++ .../Graphics/include/Resources.hpp | 35 +- .../Graphics/include/Sampler.hpp | 29 +- .../Graphics/include/ShaderResources.hpp | 297 ++- .../Graphics/include/State.hpp | 9 + .../Graphics/include/Texture.hpp | 145 +- .../Graphics/include/TextureDetiler.hpp | 8 +- .../include/TextureSwizzleEquations.hpp | 63 + .../Graphics/include/TextureTiling.hpp | 38 + .../Graphics/shaders/ColorTransfer.comp | 2 + .../Graphics/shaders/TextureDetile.comp | 117 +- .../Graphics/src/BdaResources.cpp | 141 +- .../Graphics/src/BufferPool.cpp | 136 +- .../Graphics/src/ColorTargetLayout.cpp | 55 +- .../Graphics/src/ColorTargetTransfer.cpp | 4 +- .../Graphics/src/DccMetadata.cpp | 433 ++++ .../prx/libSceAgcDriver/Graphics/src/Draw.cpp | 926 ++++++- .../Graphics/src/GpuColorTransfer.cpp | 72 +- .../Graphics/src/GraphicsPipelineCache.cpp | 138 -- .../Graphics/src/GuestBufferMemory.cpp | 1435 ++++++++++- .../Graphics/src/GuestSamplerResource.cpp | 11 +- .../Graphics/src/GuestTextureResource.cpp | 68 +- .../libSceAgcDriver/Graphics/src/Pipeline.cpp | 381 ++- .../libSceAgcDriver/Graphics/src/Recorder.cpp | 1592 ++++++++++++ .../Graphics/src/RegisteredGpuMemory.cpp | 21 - .../Graphics/src/Resources.cpp | 105 +- .../libSceAgcDriver/Graphics/src/Sampler.cpp | 36 +- .../Graphics/src/ShaderInputState.cpp | 20 +- .../Graphics/src/ShaderResources.cpp | 1677 ++++++++++++- .../Graphics/src/ShaderValidation.cpp | 118 +- .../libSceAgcDriver/Graphics/src/State.cpp | 289 ++- .../libSceAgcDriver/Graphics/src/Texture.cpp | 1141 ++++++++- .../Graphics/src/TextureCache.cpp | 30 +- .../Graphics/src/TextureDetiler.cpp | 58 +- .../src/TextureDetilerDescriptors.cpp | 6 + .../Graphics/src/TextureTiling.cpp | 117 +- .../prx/libSceAgcDriver/PostCompile.cmake | 26 +- .../Resource/src/Registration.cpp | 33 +- .../prx/libSceAgcDriver/Unimplemented.cpp | 112 + .../tests/GuestSamplerResource.cpp | 25 +- .../tests/GuestTextureResource.cpp | 16 +- .../libSceAgcDriver/tools/ShaderReplay.cpp | 125 + 68 files changed, 14986 insertions(+), 1460 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/GraphicsPipelineCache.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp delete mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/RegisteredGpuMemory.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Unimplemented.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 8cb5e76be2..8cd0d4edf3 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -47,14 +47,15 @@ add_custom_command( ) add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/DisplayBuffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/WorkerSampler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ShaderMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/BdaFeatures.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Pm4.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/VulkanDevice.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/SwapchainState.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/AspectFit.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PresentationScaler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Draw.cpp @@ -62,6 +63,7 @@ add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetTransfer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Pipeline.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderInputState.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderResources.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderAddressResources.cpp @@ -73,6 +75,7 @@ add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/GuestTextureResource.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/GuestSamplerResource.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureFormat.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DccMetadata.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureTiling.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureDetiler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Texture.cpp @@ -208,6 +211,7 @@ add_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL Graphics/src/GuestTextureResource.cpp Graphics/src/GuestSamplerResource.cpp Graphics/src/TextureFormat.cpp + Graphics/src/DccMetadata.cpp Graphics/src/TextureTiling.cpp Graphics/src/TextureDetiler.cpp Graphics/src/Texture.cpp @@ -282,7 +286,6 @@ if(AGC_BUILD_VISUAL_TEST) add_executable(agc_driver_visual_test EXCLUDE_FROM_ALL tests/Visual.cpp Execution/src/VulkanDevice.cpp - Execution/src/SwapchainState.cpp Execution/src/AspectFit.cpp Execution/src/PresentationScaler.cpp Execution/src/GuestMemory.cpp @@ -301,6 +304,7 @@ if(AGC_BUILD_VISUAL_TEST) Graphics/src/GuestTextureResource.cpp Graphics/src/GuestSamplerResource.cpp Graphics/src/TextureFormat.cpp + Graphics/src/DccMetadata.cpp Graphics/src/TextureTiling.cpp Graphics/src/TextureDetiler.cpp Graphics/src/Texture.cpp @@ -363,3 +367,14 @@ if(AGC_BUILD_VISUAL_TEST) endif() include(PostCompile.cmake) +# Replays shader_*.req files the driver writes when a compute dispatch cannot be recompiled. +add_executable(agc_shader_replay EXCLUDE_FROM_ALL + tools/ShaderReplay.cpp +) +target_include_directories(agc_shader_replay PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(agc_shader_replay PRIVATE shader_recompiler_agc_driver libc) +if(ANYPS5_ENABLE_SPIRV_TOOLS) + target_link_libraries(agc_shader_replay PRIVATE SPIRV-Tools-static) +endif() +set_target_properties(agc_shader_replay PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_shader_replay) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index ab0b5fe4ff..7528b331df 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -1,16 +1,169 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_GUESTMEMORY_HPP #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_GUESTMEMORY_HPP +#include #include #include +#include #include +#include +#include namespace AgcDriver::GuestMemory { void CheckRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable = false); -void CheckGpuRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable = false); +// CheckRange without throwing: whether the whole range is mapped with the requested access. +bool Accessible(const void* pointer, std::size_t bytes, bool writable = false); +// The accessible parts [begin, end) of a range, in address order. GPU heaps are often bound whole while +// the guest commits their pages on demand, so a binding can cover reserved but uncommitted pages. +std::vector> CommittedRanges(std::uint64_t address, std::size_t bytes, bool writable = false); +// Both facts from one walk of the page tables: the accessible parts of a range and whether they cover +// all of it (`whole`, what Accessible answers). Reserved pages are queried each time, so asking twice +// costs two VirtualQuery per run. +struct Commitment { + std::vector> ranges; + bool whole = false; +}; +Commitment DescribeCommitted(std::uint64_t address, std::size_t bytes, bool writable = false); void Read(std::uint64_t address, std::span destination, std::size_t alignment = 1); +// Variants for resources in heaps the guest commits on demand (partially resident textures, GPU +// scratch heaps): uncommitted pages read as zeros, are not compared, and are never stored. +void ReadCommitted(std::uint64_t address, std::span destination); +bool EqualsCommitted(std::uint64_t address, std::span bytes); +void WriteChangedCommitted(std::uint64_t address, std::span current, std::span original); void Write(std::uint64_t address, std::span source, std::size_t alignment = 1); +// Stores the parts of `current` that differ from `original` (the guest bytes the GPU started from), so +// guest writes made meanwhile to untouched bytes survive. Compares in 256-byte blocks. +void WriteChanged(std::uint64_t address, std::span current, std::span original); + +// Write tracking. The guest arena is reserved with page write watching, so CollectWrites learns which +// pages of a range the CPU wrote since they were last collected, stamps their 64 KiB blocks with a new +// generation and returns it; a snapshot of the range taken after that call is current while +// UnchangedSince(range, that generation) holds. Every validation collects its own range first, so +// resources sharing pages see each other's writes. GPU writes into host-imported memory bypass the +// page tables and are reported with MarkWritten. Without write watching CollectWrites returns 0 and +// UnchangedSince is always false, so callers fall back to comparing bytes. +std::uint64_t CollectWrites(std::uint64_t address, std::size_t bytes); +bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation); +void MarkWritten(std::uint64_t address, std::size_t bytes); +// Collect epoch: within one epoch a range already collected is not walked again, CollectWrites +// returns the current generation instead. A guest write landing between two collects of the same +// epoch is seen by the next epoch, which is the ordering real hardware gives a CPU write made while a +// submission runs. The driver's own writes (Write, WriteChanged, MarkWritten) stamp their blocks, so +// they are never hidden. APS5_NO_COLLECT_MEMO=1 disables the memo. CollectWritesUncached always walks: +// for decisions about which blocks the CPU wrote while a GPU result was pending (a stale answer there +// stores old texels over reused memory). +// The epoch is per queue worker: BumpCollectEpoch advances the calling thread's epoch to a value no +// other thread ever holds (one worker's walk never satisfies another's check) at the hardware +// ordering points of its queue: the start of a submission, a WAIT_REG_MEM the CPU may have satisfied +// (at entry, polled, timed out, or by another queue's recorded label; a label this queue recorded +// itself orders no CPU write), a device drain, and before reaping completions outside a submission. +// A thread that never bumps (the presenter, game threads inside the flush hook) gets a fresh epoch +// for every collect: nothing is reused without an ordering point. APS5_PACKET_EPOCH=1 makes the +// workers bump before every packet as before. CollectEpochBumps counts the bumps ([guestmem] line). +void BumpCollectEpoch(); +std::uint64_t CollectEpochBumps(); +std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes); + +// Serializes device work: draws, dispatches, presentation and the deferred write-backs below. The +// mutex is recursive; it is wrapped so every acquisition (std::lock_guard at any site) measures how +// long it waited: the graphics worker's frame is mostly spent waiting for this lock behind the +// compute workers, and that wait has to stay visible. Under APS5_PROFILE_DRAW each thread reports its +// waits in a [lock] line every 10 s (APS5_NO_LOCK_PROFILE=1 turns the timing off); a queue worker tags +// its thread with TagGpuLockThread so the line names the queue. +class GpuMutexType { +public: + void lock(); + bool try_lock(); + void unlock(); + // Whether the calling thread holds the mutex (at any depth): the APS5_ASSERT_GPU_LOCK checks + // (AssertGpuLockHeld) and code that must know whether it runs inside the device lock. + bool HeldByThisThread() const; + // The calling thread's recursion depth (0 when it does not hold the mutex): 1 means its + // current acquisition is the outermost, so code that releases the mutex around a wait + // (Recorder::SyncThrough for the flush hook) gives up no caller's hold. + std::uint32_t DepthOnThisThread() const; + +private: + void acquired(); + std::recursive_mutex mutex; + // The holder's thread token and recursion depth, written by the holder only; another thread + // reads the owner just to see that it is not itself. + std::atomic owner{nullptr}; + std::uint32_t depth = 0; +}; +GpuMutexType& GpuMutex(); +// Called on the thread that just gave up its outermost hold, right after the mutex was released: +// work the hold deferred to run without it (the recorder's release of the objects completed batches +// kept). One hook, set once; it must not take GpuMutex itself. +void SetGpuUnlockHook(void (*hook)()); +// Debug aid (APS5_ASSERT_GPU_LOCK=1): aborts with a [lock] line naming `where` when the calling +// thread does not hold GpuMutex. Recording, submits, command batches and the detiler's pools must +// only be reached under it (a resource build's stage A runs without it, see ShaderResources). +void AssertGpuLockHeld(const char* where); +void TagGpuLockThread(std::uint32_t queue); +// The queue tag of the calling thread (0xffffffff when untagged), for reports that name the thread. +std::uint32_t GpuLockThreadTag(); +// Lock sites (APS5_PROFILE_DRAW): a caller names the site of its next lock() so the [lock] line +// splits acquisitions and waits per site (whose holds queue 0 waits behind is then measurable). The +// tag is consumed by that acquisition, so a nested lock inside the flush hook is attributed to the +// hook, and an acquisition nobody tagged counts as "other". Try-only acquisitions are not counted +// as acquisitions; the hold they start is timed under "try". Holds are timed per site as well (the +// outermost acquisition of a thread until its unlock; APS5_NO_HOLD_PROFILE=1 turns that off), so +// the line shows who holds the mutex, not only who waits for it. +enum class GpuLockSite : std::uint8_t { Other = 0, Dispatch, Indirect, Draw, Hook, Wait, Label, Flush, Present, Fill, Try, Count }; +void TagGpuLockSite(GpuLockSite site); +// A code address (a return address) as an offset into its module, in the form of the [guestmem] +// callers (symbolized with nm against the driver's image). Takes the loader lock: for reports only. +unsigned long long CodeOffset(const void* address); + +// Deferred GPU writes. The graphics layer keeps the results of storage images on the GPU until guest +// memory is read; it registers a hook that stores every pending result overlapping a range, which +// every read and store through this interface calls first. Code that reads guest memory directly +// calls FlushGpuWrites itself. +void SetFlushHook(void (*hook)(std::uint64_t address, std::size_t bytes)); +void FlushGpuWrites(std::uint64_t address, std::size_t bytes); + +// Attribution of guest memory accesses (the [hooksync] line in Recorder.cpp and the read-site +// counts of the [guestmem] line, APS5_PROFILE_DRAW): a sync the flush hook makes for an access has +// to be charged to the PM4 packet and the driver code that made the access. A queue worker names +// the packet it executes at the top of its packet loop (SetCurrentPacket; opcode 0xffff is the +// flip); a thread that runs no packets (the presenter, game threads) reads back NoPacket. The main +// read paths name themselves with a ReadSiteScope; an access made by none of them is attributed by +// its return addresses instead (CaptureCallerOffsets). +struct PacketTag { + std::uint32_t opcode; + std::uint32_t queue; +}; +constexpr std::uint32_t NoPacket = 0xfffffffeu; +void SetCurrentPacket(std::uint32_t opcode, std::uint32_t queue); +PacketTag CurrentPacket(); + +// Store: a CPU write (Write/WriteChanged) rather than a read; a hook sync made for one orders a +// write after a pending GPU write, so whether the bytes change says nothing about its necessity. +// MirrorRefresh: an address-based build bringing an image mirror up to date with guest memory +// (GuestBufferMemory.cpp: the flush before a writable mirror's compare, a new mirror's fill). +// Every enumerator needs its name in ReadSiteName's table (GuestMemory.cpp). +enum class ReadSite : std::uint8_t { Unknown = 0, Capture, DispatchCache, TextureCompare, TextureRead, BufferUpload, IndexBuffer, VertexBuffer, Registers, IndirectArguments, Wait, Label, Scanout, Store, MirrorRefresh, Count }; +const char* ReadSiteName(ReadSite site); +// Sets the calling thread's read site and returns the previous one (nested scopes restore it). +ReadSite SetReadSite(ReadSite site); +ReadSite CurrentReadSite(); +class ReadSiteScope { +public: + explicit ReadSiteScope(ReadSite site) : previous(SetReadSite(site)) {} + ~ReadSiteScope() { SetReadSite(previous); } + ReadSiteScope(const ReadSiteScope&) = delete; + ReadSiteScope& operator=(const ReadSiteScope&) = delete; + +private: + ReadSite previous; +}; +// Return addresses above the caller as offsets into the driver's module (the same form as the +// [guestmem] callers, symbolized with nm against the driver's image; an address outside the module +// is written as 0), newest first, skipping `skip` frames above the caller; returns how many were +// written. Stack unwinding, so only for rare, profiled events. +std::size_t CaptureCallerOffsets(std::span frames, unsigned skip); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp index 88c1055664..475995cccd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp @@ -3,6 +3,9 @@ #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4Opcodes.hpp" +#include +#include +#include #include #include #include @@ -21,11 +24,59 @@ struct DrawParameters { }; std::string Name(std::uint32_t header); +// A PM4 type-2 packet is a one-dword filler (command-buffer padding); type 3 and type 0 carry a +// dword count in bits 29:16. Type 1 is undefined. +inline bool FillerPacket(std::uint32_t header) { return (header >> 30u) == 2u; } +inline std::size_t PacketWords(std::uint32_t header) { return FillerPacket(header) ? 1u : static_cast((header >> 16u) & 0x3fffu) + 2u; } std::string_view UnsupportedReason(std::uint32_t header); void Validate(std::span packet, std::uint32_t queue); void Execute(std::span packet, QueueState& queue); bool AccessesMemory(std::uint32_t header); +// Whether an ACQUIRE_MEM packet asks only for GPU cache actions (no CPU-visible memory +// synchronization): such a packet needs a pipeline barrier, not a device drain. bool UsesGpuCacheBarrier(std::span packet); +bool IsTagMarker(std::span packet); +bool WaitSatisfied(std::span packet); +// WaitSatisfied for a polling loop: the caller has validated the address once, so the value is read +// directly instead of through the checked guest memory path. +bool WaitSatisfiedUnchecked(std::span packet); +// Whether a WAIT_REG_MEM's compare holds for `value` (the 4 or 8 bytes a label the recorder still +// holds will store), without reading memory. +bool WaitComparesValue(std::span packet, std::uint64_t value); +// A label write (RELEASE_MEM with a data select, WRITE_DATA to memory): the destination and the +// bytes it stores, so the write can be recorded on the GPU behind the work it signals. Packets +// without a memory destination decode to nothing. No allocation per label (tens of thousands per +// second): WRITE_DATA's bytes are a view of the packet (valid while the packet is), a RELEASE_MEM +// value is held inline. +struct LabelWrite { + std::uint64_t address; + std::span packetBytes; + std::array inlineBytes{}; + std::size_t inlineSize = 0; + std::span Bytes() const { return inlineSize != 0 ? std::span(inlineBytes).first(inlineSize) : packetBytes; } +}; +std::optional DecodeLabelWrite(std::span packet); +// A memory store the CPU can resolve before the GPU runs it (COPY_DATA and DMA_DATA to memory, +// DUMP_CONST_RAM): the destination and the bytes it stores, so the driver can record the store on +// the GPU like a label instead of draining the device and storing on the CPU. Immediate and constant +// RAM bytes are known at once; a memory source is read here through the checked guest memory path +// (which waits only for recorded GPU work that writes the source). Nothing when the packet is not +// such a store, when it stores more than `limit` bytes, or when its destination or size is not a +// multiple of 4 (the GPU store needs both; the source is then not read: the caller keeps its CPU +// path). A store of zero bytes resolves with empty bytes. +struct StoreWrite { + std::uint64_t address; + std::span viewBytes; + std::vector ownedBytes; + std::span Bytes() const { return ownedBytes.empty() ? viewBytes : std::span(ownedBytes); } +}; +std::optional ResolveStore(std::span packet, const QueueState& queue, std::size_t limit); +// A DISPATCH_INDIRECT's arguments: the guest address of its three group-count dwords (no memory +// access, so the GPU can read them in place: VulkanDevice::DispatchIndirect), the DISPATCH_DIRECT +// packet made by reading them there (through the checked guest memory path, which waits for +// recorded GPU work that writes them), and both steps in one for the packet. +std::uint64_t DispatchArgumentAddress(std::span packet, const QueueState& queue); +std::array ReadDispatchArguments(std::uint64_t arguments, std::uint32_t initiator); std::array ResolveDispatch(std::span packet, const QueueState& queue); DrawParameters ResolveDraw(std::span packet, const QueueState& queue); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp index 307ceeb481..c3fb1e15cb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp @@ -18,7 +18,18 @@ public: void EnsureSourceImage(std::uint32_t width, std::uint32_t height); void RecordUpload(VkCommandBuffer commands, VkBuffer uploadBuffer); void RecordImage(VkCommandBuffer commands, VkImage image); + // Fills the source image from another image of the same extent (any blittable format, layout + // unchanged), leaving the source image ready to blit or read back. + void RecordBlitInto(VkCommandBuffer commands, VkImage image, VkImageLayout layout, VkFilter filter); + // Copies the source image's texels (tightly packed rows of its format) into a host-readable buffer + // and makes them visible to the host once the submission completes. + void RecordReadback(VkCommandBuffer commands, VkBuffer destination); void RecordBlit(VkCommandBuffer commands, VkImage destinationImage, std::uint32_t destinationWidth, std::uint32_t destinationHeight); + // The aspect-fitting blit of RecordBlit from any image: presenting a resident render target needs + // no copy into the source image first. + static void RecordBlitFrom(const Graphics::Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout layout, std::uint32_t width, std::uint32_t height, VkFilter filter, VkImage destinationImage, std::uint32_t destinationWidth, std::uint32_t destinationHeight); + std::uint32_t SourceWidth() const { return sourceWidth; } + std::uint32_t SourceHeight() const { return sourceHeight; } private: void release() noexcept; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index 25693df61c..ce2ab7bfa5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -2,19 +2,45 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERMEMORY_HPP #include "Recompiler.hpp" +#include +#include +#include #include +#include +#include +#include namespace AgcDriver { +// Records the guest words a shader's resource analysis reads, as the memory regions its recompile +// request carries. Guest memory is fetched a 4 KiB page at a time; the regions report exactly the +// dwords read, so cache keys built from them do not change with unrelated bytes nearby. class ShaderMemory { public: explicit ShaderMemory(std::span initial); - void Capture(const ShaderRecompiler::RecompileRequest& request); + // Returns what the capture resolved (plan, snapshot, specialization) for + // ShaderRecompiler::Recompile(request, capture), which then skips its own materialization. One + // result per call: the draw path captures several stages on one ShaderMemory. + std::shared_ptr Capture(const ShaderRecompiler::RecompileRequest& request); [[nodiscard]] std::vector Regions() const; private: + static constexpr std::size_t PageBytes = 4096; + static constexpr std::size_t PageWords = PageBytes / sizeof(std::uint32_t); + + struct Page { + std::array words{}; + std::bitset valid; + std::bitset read; + }; + static bool read(void* context, std::uint64_t address, std::uint32_t* value); - std::map> regions; + Page& page(std::uint64_t base); + + // Regions given at construction (the registered shader's code and header), referenced as given: + // the caller keeps them alive for as long as the capture is used. + std::map> initial; + std::map pages; }; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 50f172636d..2c1fef9ace 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -8,9 +8,17 @@ #include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include +#include namespace AgcDriver { +namespace Graphics { +class StorageTexture; +} + +// Stage A of a dispatch's resource build (see VulkanDevice::PrepareDispatch); opaque to callers. +struct PreparedDispatch; + class VulkanDevice { public: explicit VulkanDevice(const PresentationWindow* window = nullptr); @@ -18,23 +26,99 @@ public: VulkanDevice(const VulkanDevice&) = delete; VulkanDevice& operator=(const VulkanDevice&) = delete; ShaderRecompiler::SpirvTarget Target() const; + // Drains everything under the caller's GpuMutex: recorder Sync plus vkDeviceWaitIdle. For suspend, + // resize, device replacement, CPU fill fallback and APS5_DRAIN_ALL; the packet-loop drains use the + // three-step form below so the GPU wait happens without the mutex. void WaitIdle(); - void WaitDraws(); - void AcquireGpuMemory(); + // Sends recorded work to the GPU without waiting for it. With `reapFirst` it first retires batches + // that already finished, so the in-flight list stays short (APS5_NO_OPPORTUNISTIC_REAP=1 skips + // that). A reap runs completion actions, and a write-back can wait for a later batch under the + // mutex through the flush hook, so a thread submitting on another queue's behalf (a WAIT_REG_MEM + // poller, a compute worker's between-packet flush) passes false: it must only submit, never wait. + void SubmitRecorded(bool reapFirst = true); + // Unlocked drain: SubmitAndEpoch (under the mutex) returns the serial covering every recorded + // batch (0: nothing in flight); WaitRecorded(serial) waits for it WITHOUT the mutex (the caller + // holds a shared_ptr to this device); ReapRecorded(serial) (under the mutex) runs the completions + // of the batches up to it. CanWaitUnlocked is false without timeline semaphores: use WaitIdle. + bool CanWaitUnlocked() const; + std::uint64_t SubmitAndEpoch(); + void WaitRecorded(std::uint64_t serial); + void ReapRecorded(std::uint64_t serial); + // Completes the batches that already finished (non-blocking); under the mutex. + void ReapRecorded(); + // Records a store of `bytes` at a guest address behind the recorded work, so the value appears + // once that work completed; the batch is submitted by the queue worker's flush rules (or at once + // with APS5_LABEL_SUBMIT_NOW=1). `stamp` is the record-order stamp and `queue` the recording + // queue for the recorder's pending-label table. Returns 0 when recorded on the GPU, 5 when kept + // as a completion action behind pending write-backs and 6 when kept as one because the memory is + // not host-imported so the GPU cannot store it (both land when their batch is reaped), else why + // the CPU must write it: + // 1 nothing recorded (the write is already ordered), 2 write-backs pending and 3 memory not + // imported (both only with APS5_DRAIN_COMPLETION_LABELS=1), 4 unsuitable size or alignment. + // `reapFirst` retires finished batches before the checks; a caller recording a group of labels + // under one lock passes it for the first label only. + int WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true); + // Pending-label table lookup and open-batch overlap test for WAIT_REG_MEM (see Recorder). + std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; + bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + // Fills [address, address + bytes) of host-imported guest memory with a repeating 16-byte pattern, + // recorded behind the open batch; false when the range is not imported (the caller stores it). + // Recorded GPU stores over the range are ordered before the fill by its barrier; finished + // batches are retired first, and it waits only for stores a batch's completion makes on the + // CPU (a copied buffer's write-back, a deferred label). Debug aid: APS5_FILL_SYNC=1 waits for + // every recorded store over the range. + bool FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern); void ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable); void* Window() const; void Resize(std::uint32_t width, std::uint32_t height); bool Presentable() const; - void PresentClear(std::uint32_t width, std::uint32_t height, bool opaque); + // A presentation is four steps so the presenter holds GuestMemory::GpuMutex only while it touches + // the queue: AcquireImage takes the next swapchain image (no mutex needed: the swapchain and its + // fences are the presenter's own, and in FIFO mode this is where a frame waits for the vblank); + // PresentClear/PresentDisplayBuffer record and submit the frame (under the mutex) and return + // whether one is in flight; FinishPresent waits for its render fence (no mutex); QueuePresent + // hands the image to the swapchain (under the mutex again). AcquireImage returns false when the + // swapchain is out of date (the frame is dropped; the next Resize recreates it); a present without + // a prior AcquireImage acquires itself. PresentPixels does all steps itself. + bool AcquireImage(); + bool PresentClear(std::uint32_t width, std::uint32_t height, bool opaque); void PresentPixels(std::uint32_t width, std::uint32_t height, std::span pixels); - void PresentDisplayBuffer(const DisplayBuffer& buffer); - void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}); + bool PresentDisplayBuffer(const DisplayBuffer& buffer); + void FinishPresent(); + void QueuePresent(); + // Stage A of a dispatch's resource build, run WITHOUT GuestMemory::GpuMutex before Dispatch or + // DispatchIndirect: the binding plan, data buffers, imports already serving the guest buffers, + // read-only copies and the descriptor set (see ShaderResources). The dispatch completes it + // under the mutex (stage B: texture lookups, the rest of the upload, descriptor writes) and + // records. Null when nothing is prepared: the resource cache may serve the dispatch (its + // Revalidate stays under the mutex), or APS5_LOCKED_BUILD=1 keeps the whole build under it as + // before. `shader` and `snapshots` must outlive the dispatch. + std::shared_ptr PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots); + void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr); + // A dispatch whose group counts are the three dwords at `arguments` in guest memory + // (DISPATCH_INDIRECT): the GPU reads them in place from the host import, ordered after everything + // recorded before, so the CPU never waits for the shader that wrote them. When the GPU could not + // see the current bytes the counts are read on the CPU instead (through the flush hook, as the + // driver resolved every indirect dispatch before) and the dispatch is recorded as a direct one: + // cpuReason 1 storage-image results were pending over them, 2 a recorded dispatch writes them + // through a copied buffer (its CPU write-back lands only when the batch is reaped) or a label + // over those dwords is pending, 3 the memory is not host-imported; 0 when recorded GPU-side. argumentReadMs is + // what that CPU read (its sync) took. The device limit on group counts is not checked GPU-side. + // Debug aid: APS5_NO_GPU_INDIRECT=1 (in the driver) keeps every indirect dispatch on the CPU path. + struct IndirectOutcome { + int cpuReason; + double argumentReadMs; + }; + IndirectOutcome DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr); void Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); void EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); private: Graphics::Context graphicsContext() const; - void present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display = nullptr); + // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups. + IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared); + bool present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display = nullptr, const std::shared_ptr& resident = nullptr, VkFilter residentFilter = VK_FILTER_LINEAR, bool dumpFrame = false); + void writeFrameDump(); struct State; std::unique_ptr state; }; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp new file mode 100644 index 0000000000..65b2726717 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp @@ -0,0 +1,11 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_WORKERSAMPLER_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_WORKERSAMPLER_HPP + +namespace AgcDriver { + +// Starts sampling the calling thread when APS5_SAMPLE_WORKER names an output file. +void StartWorkerSampler(); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp index 7de87a4382..e9294fea79 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp @@ -1,6 +1,9 @@ #include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include +#include +#include #include #include #include @@ -12,6 +15,15 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(reason); } +// Bit 56 selects the 10-bit (A2B10G10R10) variant of the format. +constexpr std::uint64_t PixelFormatUnormBit = 0x0100000000000000ull; +constexpr std::uint64_t PixelFormatB8G8R8A8 = 0x8000000000000000ull; +constexpr std::uint64_t PixelFormatR8G8B8A8 = 0x8000000022000000ull; + +std::uint64_t baseFormat(std::uint64_t pixelFormat) { + return pixelFormat & ~PixelFormatUnormBit; +} + std::uint32_t tileOffset(std::uint32_t x, std::uint32_t y) { return ((y << 4u) & 0x0070u) ^ ((y << 5u) & 0x0f00u) ^ ((y << 9u) & 0x1000u) ^ ((y << 8u) & 0x4000u) ^ ((x << 2u) & 0x000cu) ^ ((x << 5u) & 0x0380u) ^ ((x << 4u) & 0x0400u) ^ ((x << 6u) & 0x0800u) ^ ((x << 9u) & 0xa000u); } @@ -20,7 +32,11 @@ std::uint32_t tileOffset(std::uint32_t x, std::uint32_t y) { std::size_t DisplayBufferSize(const DisplayBuffer& buffer) { require(buffer.width != 0 && buffer.height != 0 && buffer.width <= 16384 && buffer.height <= 16384, "VideoOut: invalid display buffer dimensions"); - require(buffer.pixelFormat == 0x8000000000000000ull || buffer.pixelFormat == 0x8000000022000000ull, "VideoOut: unsupported display pixel format"); + if (baseFormat(buffer.pixelFormat) != PixelFormatB8G8R8A8 && baseFormat(buffer.pixelFormat) != PixelFormatR8G8B8A8) { + char message[96]; + std::snprintf(message, sizeof(message), "VideoOut: unsupported display pixel format 0x%016llx", static_cast(buffer.pixelFormat)); + throw std::runtime_error(message); + } require(buffer.address != 0 && (buffer.address & 65535u) == 0, "VideoOut: display buffer requires 64 KiB alignment"); const auto size = static_cast((buffer.width + 127u) / 128u) * ((buffer.height + 127u) / 128u) * 65536u; require(size <= std::numeric_limits::max() && size <= std::numeric_limits::max() - buffer.address, "VideoOut: display buffer range overflow"); @@ -32,12 +48,27 @@ std::vector DecodeDisplayBuffer(const DisplayBuffer& buffer, std::spa require(source.size() == DisplayBufferSize(buffer), "VideoOut: invalid tiled display buffer size"); std::vector pixels(static_cast(buffer.width) * buffer.height * 4); const auto blocksPerRow = (buffer.width + 127u) / 128u; - const bool rgba = buffer.pixelFormat == 0x8000000022000000ull; + const bool rgba = baseFormat(buffer.pixelFormat) == PixelFormatR8G8B8A8; + // Bit 56 marks the 10-bit variant: A2B10G10R10 with red in the low bits (the scanout draws render + // COLOR_2_10_10_10 into it). Presented as 8-bit BGRA by dropping the low two bits of each channel. + const bool tenBit = (buffer.pixelFormat & PixelFormatUnormBit) != 0; timing.Mark("validate_allocate"); for (std::uint32_t y = 0; y < buffer.height; ++y) { for (std::uint32_t x = 0; x < buffer.width; ++x) { const auto tiled = (static_cast(y / 128u) * blocksPerRow + x / 128u) * 65536u + tileOffset(x, y); const auto linear = (static_cast(y) * buffer.width + x) * 4; + if (tenBit) { + std::uint32_t word = 0; + std::memcpy(&word, source.data() + tiled, 4); + const auto red = static_cast((word >> 2u) & 0xffu); + const auto green = static_cast((word >> 12u) & 0xffu); + const auto blue = static_cast((word >> 22u) & 0xffu); + pixels[linear] = rgba ? blue : red; + pixels[linear + 1] = green; + pixels[linear + 2] = rgba ? red : blue; + pixels[linear + 3] = std::byte{0xff}; + continue; + } pixels[linear] = source[tiled + (rgba ? 2 : 0)]; pixels[linear + 1] = source[tiled + 1]; pixels[linear + 2] = source[tiled + (rgba ? 0 : 2)]; @@ -50,6 +81,12 @@ std::vector DecodeDisplayBuffer(const DisplayBuffer& buffer, std::spa std::vector ReadDisplayBuffer(const DisplayBuffer& buffer) { const auto size = DisplayBufferSize(buffer); + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + if (traceGpu) { + static const auto start = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[gpu] scanout display buffer 0x%llx %ux%u format 0x%016llx at %.1f s\n", static_cast(buffer.address), buffer.width, buffer.height, static_cast(buffer.pixelFormat), std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + } + GuestMemory::FlushGpuWrites(buffer.address, size); GuestMemory::CheckRange(reinterpret_cast(buffer.address), size, 65536); const auto* source = reinterpret_cast(buffer.address); return DecodeDisplayBuffer(buffer, {source, size}); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index efd60cf2e8..2a82386bb1 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -6,30 +6,71 @@ #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp" +#include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libc/include/Shutdown.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" +#include "ControlFlow/RequestSerializer.hpp" +#ifdef _WIN32 +#include +#endif +#include #include +#include +#include +#include +#include +#include #include #include +#include #include #include #include #include #include +#include #include #include #include #include +#include #include #include +#include #include +#include #include namespace AgcDriver { namespace { +// Milliseconds since the first trace line, for the APS5_TRACE_GPU timeline. +double TraceMs() { + static const auto origin = std::chrono::steady_clock::now(); + return std::chrono::duration(std::chrono::steady_clock::now() - origin).count(); +} + +// A short sleep between polls of a label another queue or the CPU writes. std::this_thread::sleep_for +// goes through winpthreads' nanosleep and rounds up to the scheduler tick (15.6 ms unless raised), +// which multiplied by the dozens of queue-to-queue hand-offs in a frame; a high-resolution waitable +// timer sleeps for the 200 us asked. +void PollSleep() { +#ifdef _WIN32 + thread_local HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); + if (timer != nullptr) { + LARGE_INTEGER due{}; + due.QuadPart = -2000; // 200 us in 100 ns units + if (SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE)) { + WaitForSingleObject(timer, INFINITE); + return; + } + } +#endif + std::this_thread::sleep_for(std::chrono::microseconds(200)); +} + void require(bool condition, const char* reason) { if (!condition) { throw std::runtime_error(std::string("AGC driver: ") + reason); @@ -51,11 +92,9 @@ struct Submission { std::map> shaders; std::map> flips; bool suspend = false; - FrameTiming::Clock::time_point received; - FrameTiming::Clock::time_point copied; - FrameTiming::Clock::time_point validated; - FrameTiming::Clock::time_point enqueued; - FrameTiming::Clock::time_point dequeued; + // Record-order stamp (Driver::eventSerial) taken when the game submitted: a WAIT_REG_MEM of this + // submission trusts only labels the recorder noted with a newer stamp (see Recorder::NoteLabel). + std::uint64_t received = 0; }; std::uint32_t readRegister(const Registers& registers, std::uint32_t offset) { @@ -64,6 +103,161 @@ std::uint32_t readRegister(const Registers& registers, std::uint32_t offset) { return it->second; } +// The last packets of a submission, for the WAIT_REG_MEM timeout report and the draw-failure dump. +// Only offsets are kept per packet: formatting every packet's line cost more than most packets +// themselves, and the reports are rare. The commands outlive the history (both live in execute). +struct PacketHistory { + std::span commands; + std::array offsets{}; + std::size_t count = 0; + + void Record(std::size_t offset) { + offsets[count % offsets.size()] = offset; + ++count; + } + + std::string Format(std::size_t offset) const { + const auto header = commands[offset]; + const auto words = std::min(Pm4::PacketWords(header), commands.size() - offset); + char line[160]; + int length = std::snprintf(line, sizeof(line), "%s", Pm4::Name(header).c_str()); + for (std::size_t i = 1; i < words && i < 9 && length < 140; ++i) length += std::snprintf(line + length, sizeof(line) - length, " %08x", commands[offset + i]); + return line; + } + + // Visits the recorded packets' lines oldest to newest. + template + void Each(Visit&& visit) const { + const auto shown = std::min(count, offsets.size()); + const auto first = count > shown ? count % offsets.size() : 0; + for (std::size_t i = 0; i < shown; ++i) visit(Format(offsets[(first + i) % offsets.size()])); + } +}; + +// The driver's device. It is created, replaced and reset under GuestMemory::GpuMutex only; the +// atomic lets a dispatch's prologue (capture, cache validation, recompile), which runs without that +// lock, fetch it without waiting behind another queue's device phase. Arrow access is for holders of +// the GpuMutex, under which the pointee cannot be released. +class DevicePointer { +public: + std::shared_ptr load() const { return pointer.load(std::memory_order_acquire); } + operator std::shared_ptr() const { return load(); } + DevicePointer& operator=(std::shared_ptr value) { + pointer.store(std::move(value), std::memory_order_release); + return *this; + } + void reset() { *this = nullptr; } + VulkanDevice* operator->() const { return load().get(); } + explicit operator bool() const { return load() != nullptr; } + bool operator==(std::nullptr_t) const { return load() == nullptr; } + +private: + std::atomic> pointer; +}; + +// [sync] statistics (APS5_PROFILE_DRAW): device drains by packet, and the label outcomes of +// VulkanDevice::WriteLabelOnGpu (reason 0 GPU, 5 GPU store plus a completion action behind +// write-backs, 6 completion action because the memory is not host-imported, 1-4 CPU fallbacks by +// reason). Not-imported labels +// are counted apart: a title whose CPU-side label polling stalls looks there first. +// Counted from the label path and the drain sites of Driver::execute; the report itself is printed +// under GuestMemory::GpuMutex only (lastSyncReport is plain). +std::atomic gpuLabels{0}, completionLabels{0}, notImportedLabels{0}, noOpLabels{0}, unlockedDrains{0}; +std::atomic labelFallbacks[5] = {}; +// Labels queued in a worker's deferred list, the groups they were recorded in, and labels that +// took the locked path at once (too large for the list, or APS5_LABEL_LOCK_EACH=1). +std::atomic queuedLabels{0}, labelGroups{0}, immediateLabels{0}; +// Label groups recorded and label batches submitted at the following packet's own GPU mutex +// acquisition (recordLabelsForPacket), groups and submits a failed try in flushBetweenPackets left +// to that acquisition (counted apart: a compute worker's try is often submit-only), groups a +// self-locking packet recorded right after its capture (a try that succeeded there, or a queued +// label overlapping the capture, which the packet then redoes), and suspend points (all in the +// [labels] line). +std::atomic packetLockRecords{0}, packetLockSubmits{0}, packetLockDeferred{0}, packetSubmitDeferred{0}, captureTryRecords{0}, captureRetries{0}, suspendPoints{0}; +// APS5_PROFILE_DRAW: the time (microseconds) the record and the submit take at the packet's lock. +std::atomic packetLockRecordUs{0}, packetLockSubmitUs{0}; +// Non-label stores the driver makes for the title (COPY_DATA, DMA_DATA, DUMP_CONST_RAM; see the +// store path of Driver::execute): recorded on the GPU, recorded on the GPU and stored again by a +// completion action (behind a pending write-back, or memory the GPU has no view of: reasons 5 and +// 6, kept apart so a stale GPU read of a store is looked for there first), stored by the CPU under +// the mutex because the recorder was idle (nothing to order after), or stored by the CPU behind a +// device drain as before (too large, misaligned, not decodable, no device, or APS5_CPU_STORES=1). +std::atomic storesOnGpu{0}, storesBehindCompletions{0}, storesOnCpu{0}, storesDrained{0}; +std::map drainCounts; +std::uint64_t drainTotal = 0; +std::chrono::steady_clock::time_point lastSyncReport = std::chrono::steady_clock::now(); +void reportSync() { + std::string report; + for (const auto& [code, count] : drainCounts) report += " " + (code == 0xffffu ? std::string("flip") : Pm4::Name(code << 8u)) + "=" + std::to_string(count); + std::fprintf(stderr, "[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu deferred not imported, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained)\n", static_cast(drainTotal), report.c_str(), static_cast(unlockedDrains.load()), static_cast(gpuLabels.load()), static_cast(completionLabels.load()), static_cast(notImportedLabels.load()), static_cast(noOpLabels.load()), static_cast(labelFallbacks[1].load()), static_cast(labelFallbacks[2].load()), static_cast(labelFallbacks[3].load()), static_cast(labelFallbacks[4].load()), static_cast(queuedLabels.load()), static_cast(labelGroups.load()), static_cast(immediateLabels.load()), static_cast(storesOnGpu.load()), static_cast(storesBehindCompletions.load()), static_cast(storesOnCpu.load()), static_cast(storesDrained.load())); +} +void countLabelOutcome(int reason) { + if (reason == 0) ++gpuLabels; + else if (reason == 5) ++completionLabels; + else if (reason == 6) ++notImportedLabels; + else ++labelFallbacks[std::min(reason, 4)]; +} + +// A worker's deferred labels. A label packet (RELEASE_MEM, WRITE_DATA) no longer takes the GPU +// mutex to record its store: the label is queued here, on the worker's own thread, and the whole +// group is recorded under the mutex the next time this worker needs it anyway (before any packet +// of its queue that touches memory or the device: the label still precedes that work in queue +// order), or at the label flush deadline (APS5_LABEL_FLUSH_US) checked between packets. Other +// queues and game threads poll memory for the value, so their latency stays bounded by the +// deadline. Debug aid: APS5_LABEL_LOCK_EACH=1 records every label under the mutex as before. +struct DeferredLabel { + static constexpr std::size_t Capacity = 32; + std::uint64_t address; + std::size_t size; + std::array bytes; +}; +struct DeferredLabels { + std::vector labels; + // When the first label of the group was queued (the deadline counts from here). + std::chrono::steady_clock::time_point since; +}; +DeferredLabels& deferredLabels() { + static thread_local DeferredLabels deferred; + return deferred; +} +bool DeferLabels() { + // APS5_DRAIN_COMPLETION_LABELS=1 implies the locked path too: a label that must drain then + // keeps the packet path's unlocked timeline wait instead of a WaitIdle under the mutex, so + // that switch bisects the same way it did before labels were deferred. + static const bool defer = std::getenv("APS5_LABEL_LOCK_EACH") == nullptr && std::getenv("APS5_DRAIN_COMPLETION_LABELS") == nullptr; + return defer; +} +// Whether a packet needs this worker's deferred labels recorded before it runs: everything except +// pure register and state writes, NOP, and the events the driver does not execute. A wait, dispatch, +// draw, copy, indirect register load, constant RAM dump or flip either reads memory a label may +// write (through the flush hook, which only knows recorded stores) or records device work that must +// follow the label in queue order. +bool NeedsRecordedLabels(std::uint32_t header) { + if (header == FlipPacketHeader) return true; + switch ((header >> 8u) & 0xffu) { + case 0x10: case 0x11: case 0x12: case 0x13: case 0x26: case 0x28: case 0x2a: case 0x2f: + case 0x42: case 0x46: case 0x58: case 0x68: case 0x69: case 0x76: case 0x78: case 0x79: case 0x7a: case 0x81: + return false; + default: return true; + } +} +// Packets whose own execution takes GuestMemory::GpuMutex and records this worker's deferred labels +// first thing inside it (Driver::recordLabelsForPacket): dispatches (direct and indirect, including +// the fill HLE), draws and the flip. Their unlocked prologue (capture, resource stage A) reads guest +// memory through a flush hook that does not know a still-queued label, so a capture is checked +// against the queued labels once it is known what it read (recordQueuedLabelsAfterCapture: an +// overlap records them and the packet starts over), and a CPU read of dispatch arguments records +// the queue's labels first (recordQueuedLabelsBeforeRead). Stores (COPY_DATA, DMA_DATA, +// DUMP_CONST_RAM) lock too but resolve their source before that lock, so they keep the record +// ahead of them: a copy of a label value this queue just wrote must see it. +bool PacketLocksItself(std::uint32_t header) { + if (header == FlipPacketHeader) return true; + switch ((header >> 8u) & 0xffu) { + case 0x15: case 0x16: case 0x2d: case 0x35: return true; + default: return false; + } +} + class Driver { public: static Driver& Get() { @@ -82,20 +276,24 @@ public: private: void stop() { - require(std::this_thread::get_id() != worker.get_id(), "worker cannot stop itself"); + require(!OnWorkerThread(), "worker cannot stop itself"); std::lock_guard shutdownLock(shutdownMutex); { std::lock_guard lock(mutex); stopping = true; } changed.notify_all(); - if (worker.joinable()) worker.join(); + // No worker is added once `stopping` is set, so the map is stable while the threads finish. + for (auto& [queue, worker] : workers) { + if (worker.thread.joinable()) worker.thread.join(); + } + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + device.reset(); } public: void Submit(const Packet* packet, std::uint32_t queue) { - const auto received = FrameTiming::Clock::now(); CheckFailure(); require(queue == 0 || (queue >= 0x20 && queue < 0x58), "unsupported compute queue"); GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet)); @@ -103,15 +301,14 @@ public: require(descriptor.flags == 0, "nonzero submission flags are not implemented"); Submission submission{}; submission.queue = queue; - submission.received = received; if (descriptor.dw_num != 0) { require(descriptor.dw_num <= std::numeric_limits::max() / sizeof(std::uint32_t), "command size overflow"); GuestMemory::CheckRange(descriptor.addr, static_cast(descriptor.dw_num) * sizeof(std::uint32_t), alignof(std::uint32_t)); submission.commands.assign(descriptor.addr, descriptor.addr + descriptor.dw_num); } - submission.copied = FrameTiming::Clock::now(); - validate(submission.commands, queue); - submission.validated = FrameTiming::Clock::now(); + validate(submission.commands, queue, descriptor.addr); + static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr; + if (trace) std::fprintf(stderr, "[gpu] %.1f submit queue=0x%x dwords=%zu at %p\n", TraceMs(), queue, submission.commands.size(), static_cast(descriptor.addr)); { std::lock_guard lock(mutex); rethrowFailure(); @@ -127,19 +324,21 @@ public: require(request != nullptr, "video output returned a null flip reservation"); submission.flips.emplace(cursor, std::move(request)); } - cursor += static_cast((words[0] >> 16u) & 0x3fffu) + 2; + cursor += Pm4::PacketWords(words[0]); } submission.shaders = shaders; submission.serial = accepted + 1; - submission.enqueued = FrameTiming::Clock::now(); - pending.push_back(std::move(submission)); + // Every CPU store the game made before this call precedes the stamp; labels recorded + // later carry a larger one. + submission.received = ++eventSerial; + enqueue(std::move(submission)); ++accepted; } changed.notify_all(); } void WaitIdle() { - require(std::this_thread::get_id() != worker.get_id(), "worker cannot wait for itself"); + require(!OnWorkerThread(), "worker cannot wait for itself"); std::unique_lock lock(mutex); const auto target = accepted; changed.wait(lock, [&] { return failure != nullptr || completed >= target; }); @@ -147,8 +346,7 @@ public: } void SuspendPoint() { - const auto received = FrameTiming::Clock::now(); - require(std::this_thread::get_id() != worker.get_id(), "worker cannot suspend itself"); + require(!OnWorkerThread(), "worker cannot suspend itself"); std::unique_lock lock(mutex); rethrowFailure(); require(!stopping, "suspend during shutdown"); @@ -156,15 +354,13 @@ public: Submission boundary{}; boundary.serial = accepted + 1; boundary.suspend = true; - boundary.received = received; - boundary.copied = received; - boundary.validated = received; - boundary.enqueued = FrameTiming::Clock::now(); - pending.push_back(std::move(boundary)); - const auto target = ++accepted; + // The boundary resets graphics state, so it runs in order with the graphics queue. + boundary.queue = 0; + enqueue(std::move(boundary)); + ++accepted; + // The suspend point only marks where the system may suspend the title; it does not wait for + // the GPU. Blocking here deadlocks frames whose GPU work waits on labels the CPU writes later. changed.notify_all(); - changed.wait(lock, [&] { return failure != nullptr || completed >= target; }); - rethrowFailure(); } void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr& output) { @@ -182,7 +378,10 @@ public: outputs.erase(it); } + // Called per packet and per poll of a wait: the flag keeps the driver mutex out of those loops + // until a failure exists, and `failure` itself is still read under the mutex. void CheckFailure() { + if (!failed.load(std::memory_order_acquire)) return; std::lock_guard lock(mutex); rethrowFailure(); } @@ -192,26 +391,41 @@ public: { std::lock_guard lock(mutex); if (!failure) failure = error; + failed.store(true, std::memory_order_release); for (const auto& [handle, output] : outputs) output->Fail(failure); - for (const auto& item : pending) { - for (const auto& [offset, flip] : item.flips) flip->Fail(failure); + for (auto& [queue, worker] : workers) { + for (const auto& item : worker.pending) { + for (const auto& [offset, flip] : item.flips) flip->Fail(failure); + } + worker.pending.clear(); } - pending.clear(); } changed.notify_all(); } void Present(const PresentationWindow& window, const DisplayBuffer* buffer, bool opaque, void (*gpuReady)(void*), void* context) { + // The presenter's frame record travels in the window descriptor: PerformanceContext is a + // thread_local per DLL, so the one the video-out library set is invisible here. PerformanceContext timingContext(window.timing.get()); PerformanceTimer timing("Driver.Present"); CheckFailure(); require(gpuReady != nullptr && context != nullptr, "missing GPU completion callback"); require(window.getDrawableSize != nullptr, "missing window drawable size query"); + // The device is held only while the presentation is recorded and submitted; the swapchain + // acquire (where FIFO mode waits for the vblank) and the render fence wait happen outside + // GpuMutex so the queue workers are not blocked for the present's GPU or display time (the + // presenter submits on the same VkQueue as the recorder, so queue order makes the frame's + // batches complete first). Debug aid: APS5_SYNC_FLIP=1 drains the device and presents + // synchronously under one mutex hold, as before. + static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; std::shared_ptr presenting; timing.Mark("validate"); try { + bool submitted = false; + bool presentable = false; { - std::lock_guard lock(gpuMutex); + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); + std::lock_guard lock(GuestMemory::GpuMutex()); timing.Mark("gpu_mutex_wait"); if (device == nullptr || device->Window() == nullptr) { if (device) device->WaitIdle(); @@ -225,18 +439,46 @@ public: window.getDrawableSize(window.context, &drawableWidth, &drawableHeight); presenting->Resize(drawableWidth, drawableHeight); timing.Mark("resize"); - if (presenting->Presentable()) { - if (buffer != nullptr) { - require(buffer->width == window.width && buffer->height == window.height, "display buffer extent differs from output"); - presenting->WaitDraws(); - timing.Mark("draw_wait"); - presenting->PresentDisplayBuffer(*buffer); - timing.Mark("present_display_buffer"); - } else { - presenting->PresentClear(window.width, window.height, opaque); - timing.Mark("present_clear"); + presentable = presenting->Presentable(); + if (buffer != nullptr) require(buffer->width == window.width && buffer->height == window.height, "display buffer extent differs from output"); + } + // The swapchain, its fences and semaphores are the presenter's own: only the VkQueue needs + // the mutex, so the acquire (and its vblank wait) is taken first without it. + if (presentable && !syncFlip) { + presentable = presenting->AcquireImage(); + timing.Mark("acquire_image"); + } + if (presentable) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); + std::lock_guard lock(GuestMemory::GpuMutex()); + timing.Mark("gpu_mutex_wait"); + if (buffer != nullptr) { + if (syncFlip) { + presenting->WaitIdle(); + timing.Mark("device_idle_wait"); } + submitted = presenting->PresentDisplayBuffer(*buffer); + timing.Mark("present_display_buffer"); + } else { + submitted = presenting->PresentClear(window.width, window.height, opaque); + timing.Mark("present_clear"); } + if (submitted && syncFlip) { + presenting->FinishPresent(); + timing.Mark("render_fence_wait"); + presenting->QueuePresent(); + timing.Mark("queue_present"); + submitted = false; + } + } + if (submitted) { + presenting->FinishPresent(); + timing.Mark("render_fence_wait"); + // The single VkQueue is shared with the recorder: presenting on it needs the mutex. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); + std::lock_guard lock(GuestMemory::GpuMutex()); + presenting->QueuePresent(); + timing.Mark("queue_present"); } gpuReady(context); timing.Mark("release_and_callback"); @@ -248,7 +490,7 @@ public: } void ReleaseWindow(void* window) { - std::lock_guard lock(gpuMutex); + std::lock_guard lock(GuestMemory::GpuMutex()); if (device && device->Window() == window) device.reset(); } @@ -266,6 +508,15 @@ public: std::memcpy(snapshot.code.data(), code, shader->shader_size); snapshot.header.resize(shader->header_size); std::memcpy(snapshot.header.data(), shader, shader->header_size); + // Debug aid: APS5_TRACE_SHADER_REGS= (or "all") prints a shader's register lists. + static const char* traceRegs = std::getenv("APS5_TRACE_SHADER_REGS"); + if (traceRegs != nullptr && (std::string(traceRegs) == "all" || std::strtoull(traceRegs, nullptr, 16) == snapshot.codeAddress)) { + std::fprintf(stderr, "[shader] 0x%llx type %u cx", static_cast(snapshot.codeAddress), shader->type); + for (std::uint32_t i = 0; i < shader->num_cx_registers && shader->cx_registers != nullptr; ++i) std::fprintf(stderr, " %x=%08x", shader->cx_registers[i].offset, shader->cx_registers[i].value); + std::fprintf(stderr, " sh"); + for (std::uint32_t i = 0; i < shader->num_sh_registers && shader->sh_registers != nullptr; ++i) std::fprintf(stderr, " %x=%08x", shader->sh_registers[i].offset, shader->sh_registers[i].value); + std::fprintf(stderr, "\n"); + } std::lock_guard lock(mutex); rethrowFailure(); const auto address = snapshot.codeAddress; @@ -276,22 +527,68 @@ private: std::mutex mutex; std::mutex shutdownMutex; std::condition_variable changed; - std::deque pending; - std::map> shaders; + // Each queue runs on its own thread, so a WAIT_REG_MEM blocks only its own queue as on the GPU. + // Draws and dispatches take GuestMemory::GpuMutex, so device work stays serialized. + struct QueueWorker { + std::deque pending; + std::thread thread; + }; + std::map workers; + std::uint64_t frameSerial = 0; + // Packets executing now and packets finished, across all queues; waits time out only when neither + // moves. On a cache line of their own: the waiting queues poll them while the others update them. + alignas(64) std::atomic packetsInFlight{0}; + std::atomic packetsDone{0}; + // Orders the game's submit calls against the labels the workers record (Submission::received). + std::atomic eventSerial{0}; + alignas(64) std::map> shaders; + // Dispatch cache: a program with the same user data and shader registers whose captured memory + // (SRT chains, descriptors) is unchanged reuses its capture and recompile result. Guarded by + // dispatchCacheMutex; entries are evicted least recently used. + struct DispatchEntry { + std::shared_ptr memory; + std::vector captured; + std::shared_ptr compiled; + std::vector> spans; + std::uint64_t generation = 0; + // The registered shader whose code and header the capture references. + std::shared_ptr shader; + // This entry's position in dispatchOrder. + std::list::iterator order; + }; + static constexpr std::size_t DispatchCacheEntries = 4096; + std::unordered_map dispatchCache; + // Keys most recently used first. + std::list dispatchOrder; + std::mutex dispatchCacheMutex; + std::uint64_t dispatchCacheHits = 0; + std::uint64_t dispatchCacheEvictions = 0; std::map queues; std::map> outputs; - std::recursive_mutex& gpuMutex = GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix(); - std::shared_ptr device; + DevicePointer device; std::uint64_t accepted = 0; std::uint64_t completed = 0; + std::set completedOutOfOrder; std::exception_ptr failure; + // Set once `failure` is, so hot loops can check for one without the mutex. + std::atomic failed{false}; bool stopping = false; bool resetGraphics = false; - std::shared_ptr frameTiming; - std::uint64_t frameSerial = 0; - std::thread worker; - Driver() : worker([this] { run(); }) { + static bool& OnWorkerThread() { + static thread_local bool worker = false; + return worker; + } + + // Caller holds `mutex`. + void enqueue(Submission submission) { + const auto queue = submission.queue; + auto& worker = workers[queue]; + worker.pending.push_back(std::move(submission)); + if (!worker.thread.joinable()) worker.thread = std::thread([this, queue] { run(queue); }); + } + + Driver() { try { LibcRegisterShutdown_nid_postfix([] { Driver::Get().Shutdown(); }); } catch (...) { @@ -306,25 +603,227 @@ private: } } - static void validate(std::span commands, std::uint32_t queue) { + // Summarizes a rejected submission (packet names with counts and the first rejection reason per name). + static void dumpPackets(std::span commands, const std::uint32_t* guest = nullptr) { + std::map> summary; + std::vector> walk; for (std::size_t cursor = 0; cursor < commands.size();) { const auto header = commands[cursor]; - require((header & 0xc0000000u) == 0xc0000000u, "unsupported PM4 packet type"); - const auto count = static_cast((header >> 16u) & 0x3fffu) + 2; + if (Pm4::FillerPacket(header)) { ++cursor; continue; } + if ((header & 0xc0000000u) != 0xc0000000u) break; + const auto count = Pm4::PacketWords(header); + if (count > commands.size() - cursor) break; + walk.emplace_back(cursor, header); + auto& entry = summary[Pm4::Name(header)]; + if (entry.first++ == 0) { + try { + Pm4::Validate(commands.subspan(cursor, count), 0); + } catch (const std::exception& error) { + entry.second = error.what(); + } + } + cursor += count; + } + std::fprintf(stderr, "[gpu] rejected submission of %zu dwords at guest %p:\n", commands.size(), static_cast(guest)); + for (const auto& [name, entry] : summary) std::fprintf(stderr, "[gpu] %-28s x%-5zu %s\n", name.c_str(), entry.first, entry.second.c_str()); + // The last packets walked before the parse broke and, with APS5_DUMP_REJECTED=1, the whole + // stream for offline analysis. + for (std::size_t i = walk.size() > 24 ? walk.size() - 24 : 0; i < walk.size(); ++i) { + const auto [offset, header] = walk[i]; + std::fprintf(stderr, "[gpu] at DWORD %-6zu header 0x%08x %-24s %zu dwords:", offset, header, Pm4::Name(header).c_str(), Pm4::PacketWords(header)); + for (std::size_t j = offset + 1; j < std::min(commands.size(), offset + std::min(Pm4::PacketWords(header), 12)); ++j) std::fprintf(stderr, " %08x", commands[j]); + std::fprintf(stderr, "\n"); + } + static const bool dumpRejected = std::getenv("APS5_DUMP_REJECTED") != nullptr; + if (!dumpRejected) return; + static int dumps = 0; + char fileName[64]; + std::snprintf(fileName, sizeof(fileName), "rejected_submission_%d.bin", dumps++); + if (FILE* file = std::fopen(fileName, "wb")) { + std::fwrite(commands.data(), sizeof(std::uint32_t), commands.size(), file); + std::fclose(file); + std::fprintf(stderr, "[gpu] rejected submission written to %s\n", fileName); + } + } + + static void validate(std::span commands, std::uint32_t queue, const std::uint32_t* guest = nullptr) { + for (std::size_t cursor = 0; cursor < commands.size();) { + const auto header = commands[cursor]; + if (Pm4::FillerPacket(header)) { ++cursor; continue; } + if ((header & 0xc0000000u) != 0xc0000000u) { + char what[96]; + std::snprintf(what, sizeof(what), "unsupported PM4 packet type: header 0x%08x at DWORD %zu of %zu", header, cursor, commands.size()); + // The packets before it (and the dwords around it) show which packet was mis-sized. + dumpPackets(commands, guest); + std::fprintf(stderr, "[gpu] dwords %zu..%zu:", cursor >= 8 ? cursor - 8 : 0, std::min(commands.size(), cursor + 8)); + for (std::size_t i = cursor >= 8 ? cursor - 8 : 0; i < std::min(commands.size(), cursor + 8); ++i) std::fprintf(stderr, " %08x", commands[i]); + std::fprintf(stderr, "\n"); + throw std::runtime_error(std::string("AGC driver: ") + what); + } + const auto count = Pm4::PacketWords(header); require(count <= commands.size() - cursor, "truncated PM4 packet"); try { Pm4::Validate(commands.subspan(cursor, count), queue); } catch (const std::exception& error) { + dumpPackets(commands, guest); throw std::runtime_error("AGC driver: " + Pm4::Name(header) + " at DWORD " + std::to_string(cursor) + ": " + error.what()); } cursor += count; } } - void dispatch(QueueState& queue, std::span packet, const Submission& submission) { - PerformanceTimer timing("Driver.Dispatch"); - std::lock_guard gpuLock(gpuMutex); - timing.Mark("gpu_mutex_wait"); + // Worker time per packet class, reported every 10 s (APS5_PROFILE_DRAW) so throughput problems + // show where time goes. + struct WorkerProfile { + double dispatchMs = 0; + double drawMs = 0; + double waitMs = 0; + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + std::chrono::steady_clock::time_point reported = start; + }; + template + static void timed(double WorkerProfile::*bucket, Work&& work) { + static thread_local WorkerProfile profile; + const auto begin = std::chrono::steady_clock::now(); + work(); + const auto end = std::chrono::steady_clock::now(); + profile.*bucket += std::chrono::duration(end - begin).count(); + static const bool report = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (report && end - profile.reported > std::chrono::seconds(10)) { + profile.reported = end; + std::fprintf(stderr, "[gpu] worker at %.0f s: dispatch %.1f s, draw %.1f s, wait %.1f s\n", std::chrono::duration(end - profile.start).count(), profile.dispatchMs / 1000, profile.drawMs / 1000, profile.waitMs / 1000); + } + } + + // A draw or dispatch the translator cannot handle is skipped so the rest of the frame still runs; + // each distinct failure is reported once. + template + void tolerate(const char* kind, Work&& work) { + try { + work(); + } catch (const std::exception& error) { + static std::mutex reportedMutex; + static std::set reported; + std::lock_guard lock(reportedMutex); + // APS5_TRACE_SKIPS reports every skip (shortened), not only the first per reason. + static const bool traceSkips = std::getenv("APS5_TRACE_SKIPS") != nullptr; + if (reported.insert(error.what()).second) std::fprintf(stderr, "[gpu] skipped %s: %s\n", kind, error.what()); + else if (traceSkips) std::fprintf(stderr, "[gpu] skipped %s again: %.100s\n", kind, error.what()); + } + } + + // Saves a request once per shader address so it can be replayed with agc_shader_replay. + static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request) { + static std::mutex dumpMutex; + static std::set dumped; + char name[64]; + std::snprintf(name, sizeof(name), "shader_%llx.req", static_cast(address)); + std::lock_guard lock(dumpMutex); + if (!dumped.insert(address).second) return name; + try { + const auto text = ShaderRecompiler::RequestSerializer{}.Serialize(request); + if (std::FILE* file = std::fopen(name, "wb")) { + std::fwrite(text.data(), 1, text.size(), file); + std::fclose(file); + } + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] could not serialize request for 0x%llx: %s\n", static_cast(address), error.what()); + } + return name; + } + + // Capture and recompile run without the GPU lock, so the compute queues prepare their dispatches + // in parallel; only resource building and recording are serialized. + // The engine clears and initializes buffers with a nine-dword compute kernel (every thread stores one + // 16-byte record of a 32_32_32_32_UINT typed buffer: v_lshl_add_u32 v4, s8, 6, v0; four v_mov from + // the pattern in user data; buffer_store_format_xyzw v[0:3], v4, s[0:3] idxen). Demon's Souls runs + // it ~13 times per frame over ~80 MB, and typed 16-byte stores into host-imported system memory took + // ~22 ms each on the GPU while every drain waited for them. The fill is done as a transfer instead + // (or a CPU store when the memory is not imported). Debug aid: APS5_NO_FILL_HLE=1 runs the kernel. + // Whether `code` is that kernel with a V# the transfer can reproduce: everything fillBuffer + // decides on except the group count, so an indirect dispatch can tell before reading its count. + static bool matchesFillKernel(std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) { + static const bool enabled = std::getenv("APS5_NO_FILL_HLE") == nullptr; + if (!enabled || userData.size() < 8 || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1) return false; + static constexpr std::array fillKernel{0xd7460004u, 0x04010c08u, 0x7e000204u, 0x7e020205u, 0x7e040206u, 0x7e060207u, 0xe01c2000u, 0x80000004u, 0xbf810000u}; + if (code.size() < fillKernel.size() || !std::equal(fillKernel.begin(), fillKernel.end(), code.begin())) return false; + // V# in user[0..3]: base, stride 16, record count, 32_32_32_32_UINT with an identity swizzle. + const auto stride = (userData[1] >> 16u) & 0x3fffu; + const bool swizzled = ((userData[1] >> 31u) & 1u) != 0; + const auto dstSel = userData[3] & 0xfffu; + const bool addTid = ((userData[3] >> 23u) & 1u) != 0; + const auto type = userData[3] >> 30u; + const auto format = (userData[3] >> 12u) & 0x7fu; + return type == 0 && stride == 16 && !swizzled && !addTid && dstSel == 0xfacu && format == 0x4bu; + } + + bool fillBuffer(QueueState& queue, std::uint32_t queueId, std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::shared_ptr& localDevice) { + if (!matchesFillKernel(code, userData, compute)) return false; + const auto numRecords = userData[2]; + std::array groups{packet[1], packet[2], packet[3]}; + if ((packet[4] & 0x20u) != 0) { + for (std::uint32_t axis = 0; axis < 3; ++axis) { + const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u); + groups[axis] = (groups[axis] + threads - 1) / threads; + } + } + if (groups[1] != 1 || groups[2] != 1) return false; + const auto records = std::min(static_cast(groups[0]) * 64u, numRecords); + const auto base = userData[0] | (static_cast(userData[1] & 0xffffu) << 32u); + const auto bytes = static_cast(records * 16u); + const std::array pattern{userData[4], userData[5], userData[6], userData[7]}; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static std::atomic fills{0}, filledBytes{0}, cpuFills{0}; + ++fills; + filledBytes += bytes; + if (bytes != 0) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + // This queue's labels first (queue order), as at every dispatch's lock. + recordLabelsForPacket(localDevice.get(), queueId); + GuestMemory::CheckRange(reinterpret_cast(base), bytes, 16, true); + // Results still on the GPU for the range would be stored over the fill later. + Graphics::StorageTexture::FlushPending(base, bytes, nullptr, "buffer fill"); + if (!localDevice->FillBuffer(base, bytes, pattern)) { + ++cpuFills; + // Ordered after recorded work like any CPU store from the queue. + localDevice->WaitIdle(); + static thread_local std::vector block; + const auto chunk = std::min(bytes, 1u << 20u); + block.resize(chunk); + for (std::size_t at = 0; at < chunk; at += 16) std::memcpy(block.data() + at, pattern.data(), 16); + for (std::size_t done = 0; done < bytes; done += chunk) GuestMemory::Write(base + done, std::span(block).first(std::min(chunk, bytes - done)), 16); + } + } + if (profile && fills % 500 == 0) std::fprintf(stderr, "[fill] %llu buffer fills (%.0f MiB), %llu stored by the CPU\n", static_cast(fills.load()), filledBytes.load() / 1048576.0, static_cast(cpuFills.load())); + return true; + } + + // How DISPATCH_INDIRECT group counts were resolved, for the [indirect] line (APS5_PROFILE_DRAW, + // every 10 s). 0..3 are VulkanDevice::DispatchIndirect's outcomes (0 recorded GPU-side), the rest + // are decided here before the dispatch is prepared. `readMs` is the CPU read's time (its sync). + enum IndirectPath { IndirectGpu = 0, IndirectPendingImage = 1, IndirectCopiedWrite = 2, IndirectNotImported = 3, IndirectThreadDimensions = 4, IndirectFillKernel = 5, IndirectMisaligned = 6, IndirectDisabled = 7, IndirectPaths = 8 }; + static void countIndirect(int path, double readMs) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile || path < 0 || path >= IndirectPaths) return; + static std::mutex countsMutex; + static std::uint64_t counts[IndirectPaths] = {}; + static double readWaitedMs = 0; + static auto lastReport = std::chrono::steady_clock::now(); + std::lock_guard lock(countsMutex); + ++counts[path]; + readWaitedMs += readMs; + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::uint64_t cpu = 0; + for (int i = 1; i < IndirectPaths; ++i) cpu += counts[i]; + std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), readWaitedMs / 1000); + } + + // `indirectArguments` (non-zero): a DISPATCH_INDIRECT whose group counts, at that guest address, + // are not read here; `packet` then carries only the initiator (see dispatchIndirect). + void dispatch(QueueState& queue, std::span packet, const Submission& submission, std::uint64_t indirectArguments = 0) { const auto address = (static_cast(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast(readRegister(queue.shader, 0x20d) & 0xffu) << 40u); auto it = submission.shaders.upper_bound(address); require(it != submission.shaders.begin(), "compute program does not belong to a registered shader"); @@ -339,39 +838,390 @@ private: } const auto compute = Graphics::DecodeComputeStageInfo(queue.shader); const std::array memory{{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}; - if (device == nullptr) { - device = std::make_shared(); + // The GpuMutex is only taken to create the device: taking it just to copy the pointer made + // every dispatch wait behind another queue's whole device phase before its lock-free prologue. + // Debug aid: APS5_NO_UNLOCKED_DEVICE=1 takes the lock to read it as before. + static const bool unlockedDevice = std::getenv("APS5_NO_UNLOCKED_DEVICE") == nullptr; + std::shared_ptr localDevice = unlockedDevice ? device.load() : nullptr; + if (localDevice == nullptr) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Dispatch); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (device == nullptr) device = std::make_shared(); + localDevice = device; } - const GuestMemory::MemoryAccessScope memoryScope(device.get(), [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - static_cast(context)->ResolveMemory(address, bytes, writable); - }); const auto codeOffset = static_cast((address - snapshot.codeAddress) / sizeof(std::uint32_t)); + std::array resolved{}; + if (indirectArguments != 0 && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) { + // The fill HLE consumes the group count on the CPU: read it as before (waiting for the + // producer through the flush hook) and continue as a direct dispatch. + recordQueuedLabelsBeforeRead(submission.queue); + const auto readStart = std::chrono::steady_clock::now(); + resolved = Pm4::ReadDispatchArguments(indirectArguments, packet[4]); + countIndirect(IndirectFillKernel, std::chrono::duration(std::chrono::steady_clock::now() - readStart).count()); + packet = resolved; + indirectArguments = 0; + } + if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) return; ShaderRecompiler::RecompileRequest request{ {ShaderRecompiler::ShaderStage::Compute, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header}, {(packet[4] & 0x8000u) != 0 ? 32u : 64u, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device->Target(), + localDevice->Target(), {0, 0, 0, 128} }; - ShaderMemory shaderMemory(memory); - timing.Mark("prepare"); - shaderMemory.Capture(request); - timing.Mark("shader_memory_capture"); - const auto captured = shaderMemory.Regions(); - request.context.memory = captured; - timing.Mark("request_memory"); - const auto compiled = ShaderRecompiler::Recompile(request); - timing.Mark(compiled.cacheHit ? "shader_cache_hit" : "shader_compile"); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static double captureMs = 0, keyMs = 0, recompileMs = 0, deviceMs = 0; + static std::uint64_t cacheHits = 0; + static std::uint64_t dispatches = 0; + auto lap = std::chrono::steady_clock::now(); + const auto elapsed = [&] { + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - lap).count(); + lap = now; + return ms; + }; + // Debug aid: APS5_NO_DISPATCH_CACHE=1 captures and recompiles every dispatch. + static const bool noDispatchCacheEnv = std::getenv("APS5_NO_DISPATCH_CACHE") != nullptr; + // Debug aid: APS5_PROBE_DISPATCH=: applies the recompiler's APS5_PROBE + // register probe to the n-th (0-based) dispatch of that program only; that dispatch is + // recompiled outside the dispatch cache. + static const std::pair probeDispatch = [] { + const char* text = std::getenv("APS5_PROBE_DISPATCH"); + if (text == nullptr) return std::pair{0, 0}; + char* end = nullptr; + const auto probeAddress = std::strtoull(text, &end, 16); + const auto index = end != nullptr && *end == ':' ? std::strtoull(end + 1, nullptr, 10) : 0ull; + return std::pair{probeAddress, index}; + }(); + bool probeThis = false; + // The heap base differs between runs (0x10…, 0x20…, 0x30…), so only the low 36 bits are compared. + if (probeDispatch.first != 0 && (address & 0xfffffffffull) == (probeDispatch.first & 0xfffffffffull)) { + static std::atomic dispatchesSeen{0}; + probeThis = dispatchesSeen.fetch_add(1) == probeDispatch.second; + if (probeThis) std::fprintf(stderr, "[gpu] probing dispatch %llu of 0x%llx\n", static_cast(probeDispatch.second), static_cast(address)); + } + const bool noDispatchCache = noDispatchCacheEnv || probeThis; + std::uint64_t key = 0xcbf29ce484222325ull; + const auto mix = [&](std::uint64_t value) { + key ^= value; + key *= 0x100000001b3ull; + }; + mix(address); + mix(packet[4] & 0x8000u); + for (const auto word : userData) mix(word); + // Only the shader registers the request reads (thread counts and RSRC1/2; the program + // address is `address`): on queue 0 the bank also holds the graphics stages' registers, + // which every draw rewrites. Debug aid: APS5_NO_DISPATCH_KEY_HYGIENE=1 mixes the whole bank. + static const bool keyHygiene = std::getenv("APS5_NO_DISPATCH_KEY_HYGIENE") == nullptr; + if (keyHygiene) { + for (const auto offset : {0x207u, 0x208u, 0x209u, 0x212u, 0x213u}) { + const auto found = queue.shader.find(offset); + // An absent register mixes a value no 32-bit register can hold. + mix(found == queue.shader.end() ? (1ull << 32u) : found->second); + } + } else { + for (const auto& [offset, value] : queue.shader) { + mix(offset); + mix(value); + } + } + std::shared_ptr compiledResult; + std::shared_ptr keepMemory; + // The captured code and header spans point into the entry's snapshot, which may differ + // from this submission's (re-registered, identical bytes); a hit keeps it past eviction. + std::shared_ptr keepShader; + std::vector captured; + bool cached = false; + // Debug aid: APS5_TRACE_DISPATCH_CACHE reports, per program, what changed between dispatches. + static const bool traceCache = std::getenv("APS5_TRACE_DISPATCH_CACHE") != nullptr; + if (traceCache) { + std::lock_guard traceLock(dispatchCacheMutex); + struct Last { std::vector userData; std::map shader; std::uint64_t key; }; + static std::map last; + static int reports = 0; + auto& previous = last[address]; + if (previous.key != 0 && previous.key != key && reports < 200) { + std::string what; + for (std::size_t i = 0; i < userData.size(); ++i) { + if (i >= previous.userData.size() || previous.userData[i] != userData[i]) { + char text[48]; + std::snprintf(text, sizeof(text), " user[%zu] %08x->%08x", i, i < previous.userData.size() ? previous.userData[i] : 0u, userData[i]); + what += text; + } + } + for (const auto& [offset, value] : queue.shader) { + const auto old = previous.shader.find(offset); + if (old == previous.shader.end() || old->second != value) { + char text[48]; + std::snprintf(text, sizeof(text), " sh[%x] %08x->%08x", offset, old == previous.shader.end() ? 0u : old->second, value); + what += text; + } + } + ++reports; + std::fprintf(stderr, "[dispatch-cache] 0x%llx key changed:%s\n", static_cast(address), what.c_str()); + } + previous.userData = userData; + previous.shader = std::map(queue.shader.begin(), queue.shader.end()); + previous.key = key; + } + if (!noDispatchCache) { + // The entry is copied out and validated without the cache lock: CollectWrites and + // EqualsCommitted (which can wait for the GpuMutex through the flush hook) would + // otherwise stall every other queue's lookup behind this one. The write watch keeps + // concurrent validations of one entry correct, so the re-lock only has to skip an entry + // another worker changed meanwhile. Debug aid: APS5_NO_UNLOCKED_VALIDATE=1 holds the lock. + static const bool validateUnlocked = std::getenv("APS5_NO_UNLOCKED_VALIDATE") == nullptr; + std::unique_lock cacheLock(dispatchCacheMutex); + if (const auto found = dispatchCache.find(key); found != dispatchCache.end()) { + auto entry = found->second; + if (validateUnlocked) cacheLock.unlock(); + bool current = true; + for (const auto& [begin, bytes] : entry.spans) { + GuestMemory::CollectWrites(begin, bytes); + if (!GuestMemory::UnchangedSince(begin, bytes, entry.generation)) { + current = false; + break; + } + } + std::uint64_t restamped = 0; + if (!current) { + // A write landed in a watched span (spans merge regions up to 64 KiB apart, so it + // is often unrelated): the captured bytes themselves decide, and an unchanged + // capture is re-stamped at a fresh generation. The generation is collected + // BEFORE the compare, as on insert: a write landing after the collect dirties + // the pages again and fails the next validation, whereas one landing between a + // compare and a later collect would be stamped into the entry and never seen. + std::uint64_t generation = 0; + for (const auto& [begin, bytes] : entry.spans) generation = std::max(generation, GuestMemory::CollectWrites(begin, bytes)); + bool same = generation != 0; + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); + for (const auto& region : entry.captured) { + if (!same) break; + same = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + } + if (same) { + restamped = generation; + current = true; + } + } + if (!cacheLock.owns_lock()) cacheLock.lock(); + // Only an entry still at the copied generation is re-stamped or erased; the validated + // copy is used either way. + const auto again = dispatchCache.find(key); + const bool untouched = again != dispatchCache.end() && again->second.generation == entry.generation; + if (current) { + compiledResult = std::move(entry.compiled); + keepMemory = std::move(entry.memory); + keepShader = std::move(entry.shader); + captured = std::move(entry.captured); + cached = true; + ++dispatchCacheHits; + if (untouched) { + if (restamped != 0) again->second.generation = restamped; + dispatchOrder.splice(dispatchOrder.begin(), dispatchOrder, again->second.order); + } + } else { + if (traceCache) std::fprintf(stderr, "[dispatch-cache] 0x%llx captured memory changed\n", static_cast(address)); + if (untouched) { + dispatchOrder.erase(again->second.order); + dispatchCache.erase(again); + } + } + } + } + if (cached) { + captureMs += elapsed(); + } else { + auto shaderMemory = std::make_shared(memory); + // Debug aid: APS5_DUMP_SHADERS=1 saves every request, failing ones included, for agc_shader_replay. + static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; + try { + // The probe is part of the recompiler's cache key, so it covers the capture's plan + // lookup as well as the recompile that reuses it. + struct ProbeScope { + bool active; + explicit ProbeScope(bool active) : active(active) { if (active) ShaderRecompiler::SetDebugProbeActive(true); } + ~ProbeScope() { if (active) ShaderRecompiler::SetDebugProbeActive(false); } + } probeScope{probeThis}; + const auto capture = shaderMemory->Capture(request); + captured = shaderMemory->Regions(); + request.context.memory = captured; + captureMs += elapsed(); + if (dumpShaders) static_cast(dumpRequest(address, request)); + const auto started = std::chrono::steady_clock::now(); + // The recompile reuses the capture's plan and materialization rather than walking + // the captured regions again. Debug aid: APS5_NO_CAPTURE_REUSE=1 walks them again. + static const bool reuseCapture = std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; + compiledResult = std::make_shared(reuseCapture ? ShaderRecompiler::Recompile(request, *capture) : ShaderRecompiler::Recompile(request)); + if (compiledResult->cacheHit) ++cacheHits; + const auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + static double totalMs = 0; + totalMs += elapsed; + if (profile && elapsed > 200) std::fprintf(stderr, "[gpu] compute shader 0x%llx recompile took %.0f ms (%zu SPIR-V words, %zu captured regions, total %.1f s)\n", static_cast(address), elapsed, compiledResult->spirv.size(), captured.size(), totalMs / 1000); + } catch (const std::exception& error) { + const auto dump = dumpShaders ? dumpRequest(address, request) : std::string{}; + // Recompile errors append the serialized request; keep only the first line in the report. + std::string reason = error.what(); + if (const auto newline = reason.find('\n'); newline != std::string::npos) reason.resize(newline); + char where[96]; + if (dump.empty()) std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast(address)); + else std::snprintf(where, sizeof(where), "compute shader 0x%llx (%s): ", static_cast(address), dump.c_str()); + throw std::runtime_error(where + reason); + } + recompileMs += elapsed(); + if (!noDispatchCache) { + // The captured regions, merged into spans when close, are watched for writes from now + // on; bytes that already differ show a write racing the capture, and nothing is kept. + DispatchEntry fresh{shaderMemory, captured, compiledResult, {}, 0, it->second}; + for (const auto& region : captured) { + const auto begin = region.guestAddress; + const auto end = begin + region.bytes.size(); + if (!fresh.spans.empty() && begin <= fresh.spans.back().first + fresh.spans.back().second + 65536) { + auto& last = fresh.spans.back(); + last.second = static_cast(std::max(last.first + last.second, end) - last.first); + } else { + fresh.spans.emplace_back(begin, static_cast(end - begin)); + } + } + std::uint64_t generation = 0; + for (const auto& [begin, bytes] : fresh.spans) generation = std::max(generation, GuestMemory::CollectWrites(begin, bytes)); + bool stable = generation != 0; + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); + for (const auto& region : captured) { + if (!stable) break; + stable = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + } + if (stable) { + fresh.generation = generation; + std::lock_guard cacheLock(dispatchCacheMutex); + // Another worker may have inserted the key meanwhile; its entry is as good. + const auto [inserted, isNew] = dispatchCache.try_emplace(key, std::move(fresh)); + if (isNew) { + dispatchOrder.push_front(key); + inserted->second.order = dispatchOrder.begin(); + } + // Debug aid: APS5_NO_DISPATCH_LRU=1 clears the cache when it fills, as before. + static const bool lru = std::getenv("APS5_NO_DISPATCH_LRU") == nullptr; + if (dispatchCache.size() > DispatchCacheEntries) { + if (lru) { + dispatchCache.erase(dispatchOrder.back()); + dispatchOrder.pop_back(); + ++dispatchCacheEvictions; + } else { + dispatchCacheEvictions += dispatchCache.size(); + dispatchCache.clear(); + dispatchOrder.clear(); + } + } + } + } + } + // A label this queue still has queued over a captured region: recorded now, and the dispatch + // starts over (the cache entry just validated or inserted fails its next validation, the + // label store marks the tracker). The queue is empty after, so this happens once. + if (recordQueuedLabelsAfterCapture(submission.queue, captured)) { + dispatch(queue, packet, submission, indirectArguments); + return; + } + const auto& compiled = *compiledResult; std::vector snapshots; for (const auto& region : captured) snapshots.push_back({region.guestAddress, region.bytes}); - timing.Mark("snapshots"); - device->Dispatch(compiled, packet[1], packet[2], packet[3], snapshots); - timing.Mark("dispatch_and_resource_release"); + std::array groups{packet[1], packet[2], packet[3]}; + if (indirectArguments == 0 && (packet[4] & 0x20u) != 0) { + // USE_THREAD_DIMENSIONS: the packet counts threads; launch enough whole groups to cover them. + for (std::uint32_t axis = 0; axis < 3; ++axis) { + const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u); + groups[axis] = (groups[axis] + threads - 1) / threads; + } + } + static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr; + if (traceIo) { + // APS5_TRACE_DISPATCH_IO=2 also lists the user data. + std::string words; + if (std::getenv("APS5_TRACE_DISPATCH_IO")[0] == '2') { + for (const auto word : userData) { + char text[12]; + std::snprintf(text, sizeof(text), " %08x", word); + words += text; + } + } + std::fprintf(stderr, "[dispatch-io] shader 0x%llx%s\n", static_cast(address), words.c_str()); + } + // A failure of the build or the device work is reported against the shader. + const auto rethrow = [&](const std::exception& error) { + char where[64]; + std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast(address)); + throw std::runtime_error(where + std::string(error.what())); + }; + // Stage A of the resource build (copies, buffers, the descriptor set) runs before the + // lock, which then covers only stage B (texture lookups, imports, descriptor writes) and + // the record: a compute queue's build no longer stalls the graphics queue for its whole + // resources phase. Null when the resource cache may serve this dispatch, or with + // APS5_LOCKED_BUILD=1 (the whole build under the lock as before). + std::shared_ptr prepared; + try { + prepared = localDevice->PrepareDispatch(compiled, snapshots); + } catch (const std::exception& error) { + rethrow(error); + } + GuestMemory::TagGpuLockSite(indirectArguments != 0 ? GuestMemory::GpuLockSite::Indirect : GuestMemory::GpuLockSite::Dispatch); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + // This queue's labels first (queue order), and their batch out before the dispatch. Not + // under the shader's name: a label that fails to record is not a shader failure. + recordLabelsForPacket(localDevice.get(), submission.queue); + try { + if (indirectArguments != 0) { + const auto outcome = localDevice->DispatchIndirect(compiled, indirectArguments, snapshots, address, std::move(prepared)); + countIndirect(outcome.cpuReason, outcome.argumentReadMs); + } else { + localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared)); + } + } catch (const std::exception& error) { + rethrow(error); + } + deviceMs += elapsed(); + if (profile && ++dispatches % 100 == 0) std::fprintf(stderr, "[gpu] %llu dispatches (%llu dispatch cache hits, %llu evictions, %llu recompile cache hits): capture %.1f s, cache key %.1f s, recompile %.1f s, device %.1f s\n", static_cast(dispatches), static_cast(dispatchCacheHits), static_cast(dispatchCacheEvictions), static_cast(cacheHits), captureMs / 1000, keyMs / 1000, recompileMs / 1000, deviceMs / 1000); + } + + // DISPATCH_INDIRECT: the group counts are three dwords a shader usually wrote, and reading them + // on the CPU waited for that shader (the flush hook syncs on its batch) before this dispatch could + // even be prepared. The GPU reads them in place instead (VulkanDevice::DispatchIndirect, which + // still falls back to a CPU read when it cannot see the current bytes); the CPU path stays for + // thread-dimension initiators (the counts are rounded to groups here) and the fill HLE (needs the + // count, decided in dispatch once the code is known). Debug aid: APS5_NO_GPU_INDIRECT=1 reads + // every count on the CPU as before. + void dispatchIndirect(QueueState& queue, std::span packet, const Submission& submission) { + static const bool gpuIndirect = std::getenv("APS5_NO_GPU_INDIRECT") == nullptr; + const auto arguments = Pm4::DispatchArgumentAddress(packet, queue); + const auto initiator = packet.back(); + int path = IndirectGpu; + if (!gpuIndirect) path = IndirectDisabled; + else if ((initiator & 0x20u) != 0) path = IndirectThreadDimensions; + else if (arguments % 4 != 0) path = IndirectMisaligned; + if (path != IndirectGpu) { + recordQueuedLabelsBeforeRead(submission.queue); + const auto readStart = std::chrono::steady_clock::now(); + const auto direct = Pm4::ReadDispatchArguments(arguments, initiator); + countIndirect(path, std::chrono::duration(std::chrono::steady_clock::now() - readStart).count()); + dispatch(queue, direct, submission); + return; + } + // A DISPATCH_DIRECT shape with the counts unknown: dispatch reads only the initiator from it. + const std::array unresolved{0xc0031500u, 0, 0, 0, initiator}; + dispatch(queue, unresolved, submission, arguments); } void draw(QueueState& queue, std::span packet, const Submission& submission) { PerformanceTimer timing("Driver.Draw"); auto drawParameters = Pm4::ResolveDraw(packet, queue); if (!drawParameters.indexed && (drawParameters.indexCount == 0 || drawParameters.instanceCount == 0)) return; + // Depth/stencil-only passes (no color writes, no pixel shader) have no effect without depth + // targets, which are not emulated. + { + const auto targetMask = queue.context.find(0x8e); + const auto shaderMask = queue.context.find(0x8f); + const bool colorWrites = targetMask != queue.context.end() && shaderMask != queue.context.end() && (targetMask->second & shaderMask->second) != 0; + if (!colorWrites && !queue.shader.contains(0x8)) return; + } const auto graphics = Graphics::DecodeState(queue); struct Program { ShaderRecompiler::ShaderBinary binary; @@ -458,13 +1308,32 @@ private: linked.push_back({roles[i], program.binary, program.userDataBase, program.firstUserSgpr, program.userData}); } timing.Mark("prepare"); - std::lock_guard gpuLock(gpuMutex); - timing.Mark("gpu_mutex_wait"); - if (device == nullptr) device = std::make_shared(); + // As in dispatch, the capture, the recompiles and the rect-list shaders run without the + // GpuMutex (guest memory is read through the self-locking flush hook, the recompiler has its + // own locks); only the device creation and the device Draw take it, so the compute workers + // do not pile up behind a draw's whole preparation. The device is held by this shared_ptr + // meanwhile (DevicePointer's arrow form is for holders of the mutex only). Debug aid: + // APS5_LOCKED_DRAW_PREPARE=1 holds the mutex across the preparation as before. + static const bool lockedPrepare = std::getenv("APS5_LOCKED_DRAW_PREPARE") != nullptr; + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + std::shared_ptr localDevice; + if (lockedPrepare) { + // The device is created under the lock already held: one "draw" acquisition per draw, + // as in the baseline, so the [lock] site counts stay comparable. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); + gpuLock.lock(); + timing.Mark("gpu_mutex_wait"); + if (device == nullptr) device = std::make_shared(); + localDevice = device; + // This queue's labels first (queue order), as at the unlocked path's acquisition below. + recordLabelsForPacket(localDevice.get(), submission.queue); + } else if ((localDevice = device.load()) == nullptr) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); + std::lock_guard createLock(GuestMemory::GpuMutex()); + if (device == nullptr) device = std::make_shared(); + localDevice = device; + } timing.Mark("device_setup"); - const GuestMemory::MemoryAccessScope memoryScope(device.get(), [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - static_cast(context)->ResolveMemory(address, bytes, writable); - }); ShaderMemory shaderMemory(memory); std::vector results; std::vector stages; @@ -478,18 +1347,40 @@ private: ShaderRecompiler::RecompileRequest request{ program.binary, {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, program.binary.stage == Stage::Fragment ? std::nullopt : std::optional(Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData)), memory}, - device->Target(), + localDevice->Target(), {0, 0, pushCursorBytes, Graphics::PipelinePushConstantBytes - pushCursorBytes}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}} }; - PerformanceTimer shaderTiming("Driver.GraphicsShader"); - shaderMemory.Capture(request); - shaderTiming.Mark("memory_capture"); + const auto capture = shaderMemory.Capture(request); memory = shaderMemory.Regions(); request.context.memory = memory; - shaderTiming.Mark("request_memory"); - results.push_back(ShaderRecompiler::Recompile(request)); - shaderTiming.Mark(results.back().cacheHit ? "cache_hit" : "compile"); + // Debug aid: APS5_DUMP_DRAW_SHADERS= saves the requests of draws into that + // target as shader_
.req for agc_shader_replay. + static const std::uint64_t dumpTarget = [] { const char* text = std::getenv("APS5_DUMP_DRAW_SHADERS"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); + if (dumpTarget != 0) { + // Match the first enabled target or the raw slot-0 base, which skip reports name. + const auto slot0 = (static_cast(readRegister(queue.context, 0x390)) << 40u) | (static_cast(readRegister(queue.context, 0x318)) << 8u); + if ((graphics.hasColorTarget && graphics.color.address == dumpTarget) || slot0 == dumpTarget) static_cast(dumpRequest(program.binary.codeAddress, request)); + } + // Debug aid: APS5_DUMP_DRAW_SLOT1= saves the requests and registers of draws whose + // second color target (slot 1) is at that address, as shader_
.req and draw_slot1.regs. + static const std::uint64_t dumpSlot1 = [] { const char* text = std::getenv("APS5_DUMP_DRAW_SLOT1"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); + if (dumpSlot1 != 0) { + const auto value = [&](std::uint32_t offset) -> std::uint64_t { const auto it = queue.context.find(offset); return it == queue.context.end() ? 0u : it->second; }; + const auto slot1 = (value(0x391) << 40u) | (value(0x327) << 8u); + if (slot1 == dumpSlot1) { + static_cast(dumpRequest(program.binary.codeAddress, request)); + if (std::FILE* file = std::fopen("draw_slot1.regs", "w")) { + for (const auto& [offset, value] : queue.context) std::fprintf(file, "context %x %08x\n", offset, value); + for (const auto& [offset, value] : queue.userConfig) std::fprintf(file, "uconfig %x %08x\n", offset, value); + for (const auto& [offset, value] : queue.shader) std::fprintf(file, "shader %x %08x\n", offset, value); + std::fclose(file); + } + } + } + // As in dispatch: the recompile reuses the capture (APS5_NO_CAPTURE_REUSE=1 restores). + static const bool reuseCapture = std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; + results.push_back(reuseCapture ? ShaderRecompiler::Recompile(request, *capture) : ShaderRecompiler::Recompile(request)); const auto& result = results.back(); if (!drawParameters.indexed && i == 0) { const auto offsetValue = [&](std::int32_t sgpr) { @@ -505,10 +1396,16 @@ private: stages.push_back({program.binary.stage, &result, result.pushConstants.empty() ? 0u : pushCursorBytes}); pushCursorBytes += static_cast(result.pushConstants.size()); } - timing.Mark("shaders"); + timing.Mark("shader_compile_and_link"); + // As in dispatch: a label this queue still has queued over a captured region is recorded + // now and the draw starts over (nothing queued on the locked-prepare path, recorded above). + if (recordQueuedLabelsAfterCapture(submission.queue, memory)) { + draw(queue, packet, submission); + return; + } if (graphics.rectList) { require(stages.size() == 2, "rect-list requires vertex and fragment programs"); - auto rectangle = ShaderRecompiler::BuildRectListShaders(results[0], results[1], device->Target()); + auto rectangle = ShaderRecompiler::BuildRectListShaders(results[0], results[1], localDevice->Target()); results.push_back(std::move(rectangle.control)); results.push_back(std::move(rectangle.evaluation)); stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[2], 0}, {Stage::TessellationEvaluation, &results[3], 0}}); @@ -516,151 +1413,1070 @@ private: std::vector snapshots; for (const auto& region : memory) snapshots.push_back({region.guestAddress, region.bytes}); timing.Mark("post_compile_prepare"); - device->EnqueueDraw(graphics, drawParameters, stages, snapshots); + if (!gpuLock.owns_lock()) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); + gpuLock.lock(); + timing.Mark("gpu_mutex_wait"); + // Present replaces a windowless device by a windowed one (after WaitIdle) while draws + // may be preparing unlocked: the draw then records on the current device, whose + // recorder Recorder::Active() already is, rather than mixing the old device's images + // and pipelines into the new recorder's batch. The preparation is device-independent + // (SPIR-V for the same physical device; pipelines are created inside Draw). + if (auto current = device.load(); current != nullptr && current != localDevice) { + static std::atomic replaced{0}; + std::fprintf(stderr, "[draw] device replaced during unlocked preparation (%llu)\n", static_cast(++replaced)); + localDevice = std::move(current); + } + // This queue's labels first (queue order), and their batch out before the draw. + recordLabelsForPacket(localDevice.get(), submission.queue); + } + localDevice->Draw(graphics, drawParameters, stages, snapshots); timing.Mark("draw_and_resource_release"); } - void includeSubmission(const Submission& submission, bool firstSegment) { - if (frameTiming == nullptr) { - require(frameSerial != std::numeric_limits::max(), "frame serial overflow"); - frameTiming = std::make_shared(++frameSerial); + // Serials complete out of order when a wait runs other queues' work; `completed` stays the + // highest serial below which everything has finished. Caller holds `mutex`. + void markCompleted(std::uint64_t serial) { + completedOutOfOrder.insert(serial); + while (!completedOutOfOrder.empty() && *completedOutOfOrder.begin() == completed + 1) { + completed = *completedOutOfOrder.begin(); + completedOutOfOrder.erase(completedOutOfOrder.begin()); } - const auto dequeued = firstSegment ? submission.dequeued : FrameTiming::Clock::now(); - frameTiming->IncludeSubmission(submission.serial, submission.received, submission.enqueued, dequeued, firstSegment); - if (firstSegment) { - frameTiming->Add(frameTiming->Get("Submission", "copy"), submission.copied - submission.received, submission.commands.size() * sizeof(std::uint32_t)); - frameTiming->Add(frameTiming->Get("Submission", "validate"), submission.validated - submission.copied); - frameTiming->Add(frameTiming->Get("Submission", "reserve_enqueue"), submission.enqueued - submission.validated); + } + + // Debug aid: APS5_TRACE_LABEL= logs every GPU write to or wait on memory within 256 + // bytes of that address. + static void traceLabel(std::span packet, std::uint32_t queue) { + static const std::uint64_t watched = [] { + const char* value = std::getenv("APS5_TRACE_LABEL"); + return value ? std::strtoull(value, nullptr, 16) : 0ull; + }(); + const auto opcode = (packet[0] >> 8u) & 0xffu; + std::uint64_t target = 0; + const char* kind = nullptr; + if (opcode == 0x49 && packet.size() >= 7) { target = packet[3] | (static_cast(packet[4]) << 32u); kind = "RELEASE_MEM"; } + else if (opcode == 0x37 && packet.size() >= 5) { target = packet[2] | (static_cast(packet[3]) << 32u); kind = "WRITE_DATA"; } + else if ((opcode == 0x3c || opcode == 0x93) && packet.size() >= 7) { target = packet[2] | (static_cast(packet[3]) << 32u); kind = "WAIT_REG_MEM"; } + else if (opcode == 0x40 && packet.size() >= 6) { target = packet[4] | (static_cast(packet[5]) << 32u); kind = "COPY_DATA"; } + std::uint64_t length = 4; + if (opcode == 0x50 && packet.size() >= 7) { + target = packet[4] | (static_cast(packet[5]) << 32u); + length = packet[6] & 0x3ffffffu; + kind = "DMA_DATA"; } + if (kind == nullptr) return; + if (std::string_view(kind) != "WAIT_REG_MEM") { + static std::mutex historyMutex; + std::lock_guard lock(historyMutex); + auto& entry = writeHistory()[writeCursor()++ % writeHistory().size()]; + entry = {target, length, queue, opcode}; + } + if (watched == 0 || target + length + 0x100 < watched || target > watched + 0x100) return; + std::fprintf(stderr, "[label] %lld ms queue 0x%x %s 0x%llx:", static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()), queue, kind, static_cast(target)); + for (std::size_t i = 1; i < packet.size() && i < 9; ++i) std::fprintf(stderr, " %08x", packet[i]); + std::fprintf(stderr, "\n"); + } + + struct WriteRecord { + std::uint64_t target; + std::uint64_t length; + std::uint32_t queue; + std::uint32_t opcode; + }; + static std::array& writeHistory() { + static std::array history{}; + return history; + } + static std::size_t& writeCursor() { + static std::size_t cursor = 0; + return cursor; + } + + static int WaitTimeoutMs() { + static const int value = [] { const char* text = std::getenv("APS5_GPU_WAIT_TIMEOUT_MS"); return text ? std::atoi(text) : 1000; }(); + return value; + } + + // How WAIT_REG_MEM packets were satisfied, per queue worker (reported in its [packets] line): + // already true when reached; from the recorder's pending-label table (and how many of those by + // a label of this same queue); by polling memory (the GPU, another queue or the CPU wrote it); + // timed out. Plus the submits and reaps the poll loop made on behalf of the producer. + // fromRecorderUnlocked: table hits (at entry or polling) that took no GPU mutex; entriesUnlocked: + // waits that went to polling without the GPU mutex (nothing recorded writes the range); + // entryTriesFailed: overlapping entries whose try of the mutex failed and went to polling + // instead of waiting for it (see waitMemory). + struct WaitOutcomes { + std::uint64_t atEntry = 0, fromRecorder = 0, fromRecorderSameQueue = 0, fromRecorderPolling = 0, polled = 0, timedOut = 0, pollSubmits = 0, pollReaps = 0; + std::uint64_t fromRecorderUnlocked = 0, entriesUnlocked = 0, entryTriesFailed = 0; + }; + static WaitOutcomes& waitOutcomes() { + static thread_local WaitOutcomes outcomes; + return outcomes; + } + + // Collect-epoch bumps by ordering point, per queue worker (reported in its [packets] line, see + // GuestMemory::BumpCollectEpoch): the start of a submission, a wait the CPU may have satisfied, + // a drain, a reap outside a submission, and (APS5_PACKET_EPOCH=1 only) every packet. + struct EpochBumps { + std::uint64_t submissions = 0, waits = 0, drains = 0, reaps = 0, packets = 0; + }; + static EpochBumps& epochBumps() { + static thread_local EpochBumps bumps; + return bumps; + } + static void bumpEpoch(std::uint64_t EpochBumps::*counter) { + GuestMemory::BumpCollectEpoch(); + ++(epochBumps().*counter); + } + static bool PacketEpoch() { + static const bool packet = std::getenv("APS5_PACKET_EPOCH") != nullptr; + return packet; + } + + // Labels recorded on the GPU are not submitted one by one (see VulkanDevice::WriteLabelOnGpu): + // the open batch goes out at the next non-label packet of the recording worker, and at the + // latest APS5_LABEL_FLUSH_US (default 250) after its first label, checked by every worker + // between packets and by waiting workers inside their poll loop, so a game thread polling the + // label is never left waiting on an unsubmitted batch for long. The deadline is read lock-free. + static std::chrono::microseconds LabelFlushDeadline() { + static const std::chrono::microseconds value = [] { + const char* text = std::getenv("APS5_LABEL_FLUSH_US"); + return std::chrono::microseconds(text ? std::atoi(text) : 250); + }(); + return value; + } + static bool LabelFlushDue() { + const auto since = Graphics::Recorder::PendingLabelSince(); + return since.has_value() && std::chrono::steady_clock::now() - *since >= LabelFlushDeadline(); + } + // Submits the open batch after this many dispatches/draws even without a label, so the GPU + // starts on them while the CPU records the rest (APS5_BATCH_CAP, default 16; 0 disables). + static std::uint64_t BatchCap() { + static const std::uint64_t value = [] { + const char* text = std::getenv("APS5_BATCH_CAP"); + return text ? std::strtoull(text, nullptr, 10) : 16ull; + }(); + return value; + } + + // Records this worker's deferred labels (see DeferredLabels) under GuestMemory::GpuMutex, in + // queue order, each as VulkanDevice::WriteLabelOnGpu would have at its packet: on the GPU, as a + // completion action, or by a CPU store when nothing is recorded (the store is ordered already). + // The stamp is taken here, right before the label is noted, so it orders after every submission + // the game made before this point (a wait received later never trusts the label). Only an + // unusable recorder (reason 4) can still ask for a drain here (APS5_DRAIN_COMPLETION_LABELS=1 + // disables deferral, see DeferLabels); it waits idle under the mutex, which is cheap then. + // A throw (lost device) leaves nothing queued: the worker fails with the packet, or, when the + // record ran inside a tolerated dispatch or draw, that packet is skipped; the dropped group is + // reported either way, so a WAIT_REG_MEM timeout later is not the first sign of it. + void recordDeferredLabels(VulkanDevice* localDevice, std::uint32_t queue) { + auto& deferred = deferredLabels(); + if (deferred.labels.empty()) return; + struct Clear { + std::vector& labels; + std::uint32_t queue; + ~Clear() { + if (std::uncaught_exceptions() != 0) std::fprintf(stderr, "[gpu] queue 0x%x dropped %zu queued labels: their record failed\n", queue, labels.size()); + labels.clear(); + } + } clear{deferred.labels, queue}; + bool first = true; + for (const auto& label : deferred.labels) { + const auto bytes = std::span(label.bytes).first(label.size); + const int reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label.address, bytes, ++eventSerial, queue, first) : 4; + first = false; + countLabelOutcome(reason); + if (reason == 0 || reason == 5 || reason == 6) continue; + if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle(); + GuestMemory::Write(label.address, bytes, 4); + } + ++labelGroups; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // The [sync] line every 10 s (checked every 64 groups; drains alone are too rare now). + if (profile && (labelGroups.load(std::memory_order_relaxed) & 63u) == 0 && std::chrono::steady_clock::now() - lastSyncReport > std::chrono::seconds(10)) { + lastSyncReport = std::chrono::steady_clock::now(); + reportSync(); + } + } + + // The first thing a packet does under its own GpuMutex hold (dispatch, draw, flip: see + // PacketLocksItself), before its work is recorded: this worker's deferred labels go in, in + // queue order ahead of that work, and the open batch is submitted when it carries labels, + // exactly what flushBetweenPackets did under an acquisition of its own before the packet + // (APS5_LABEL_OWN_LOCK=1 restores that). One acquisition instead of two: the label one was + // queue 0's first after a run of unlocked packets and absorbed the compute queues' whole + // critical sections ([lock] site 'label'), after which the packet's own came uncontended. + // Returns whether the batch was submitted (the flip then needs no submit of its own). + bool recordLabelsForPacket(VulkanDevice* localDevice, std::uint32_t queue) { + static const bool packetFlush = std::getenv("APS5_NO_LABEL_PACKET_FLUSH") == nullptr; + // APS5_PROFILE_DRAW: what this costs inside the packet's own hold ([labels] line): the + // record (with queue 0's reap of finished batches before the first label) and the submit + // (with another reap on queue 0), both of which run completions, so a dispatch's or + // indirect dispatch's hold ([lock] line) is partly this. + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + if (!deferredLabels().labels.empty()) { + recordDeferredLabels(localDevice, queue); + ++packetLockRecords; + if (profile) packetLockRecordUs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); + } + if (localDevice == nullptr || !packetFlush || !Graphics::Recorder::PendingLabelSince().has_value()) return false; + // As in flushBetweenPackets: a compute worker submits without reaping. + const auto submitStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + localDevice->SubmitRecorded(queue == 0); + ++packetLockSubmits; + if (profile) packetLockSubmitUs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - submitStart).count()); + return true; + } + + // Before a packet's CPU read of guest memory outside any lock (the group counts of a + // DISPATCH_INDIRECT resolved on the CPU): a label this queue still has queued may write those + // bytes and the flush hook only knows recorded stores, so the queued labels are recorded first. + // Rare (a label is normally followed by a wait or a dispatch that records it) and cheap. + void recordQueuedLabelsBeforeRead(std::uint32_t queue) { + if (deferredLabels().labels.empty()) return; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + recordDeferredLabels(localDevice.get(), queue); + } + + // After a self-locking packet's unlocked capture (dispatch, draw), when this worker still has + // labels queued (the try in flushBetweenPackets failed): a queued label whose bytes lie in a + // captured region (a fence value in SRT-chased data, a patched constant) was read before its + // write, and the hook could not sync on it, so the labels are recorded outright now and the + // caller redoes the packet from the top (returns true; once, the queue is empty after). With no + // overlap the labels are recorded if the mutex happens to be free, so a poller of the label + // does not wait for the rest of this prologue (stage A takes milliseconds for a BDA build); + // otherwise the packet's own acquisition records them as planned. The check itself is a few + // dozen compares: a group holds a handful of labels, a capture a few dozen regions. + bool recordQueuedLabelsAfterCapture(std::uint32_t queue, std::span regions) { + const auto& labels = deferredLabels().labels; + if (labels.empty()) return false; + for (const auto& label : labels) { + for (const auto& region : regions) { + if (label.address < region.guestAddress + region.bytes.size() && region.guestAddress < label.address + label.size) { + recordQueuedLabelsBeforeRead(queue); + ++captureRetries; + return true; + } + } + } + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + if (!gpuLock.try_lock()) return false; + const auto localDevice = device.load(); + recordDeferredLabels(localDevice.get(), queue); + ++captureTryRecords; + return false; + } + + // Between packets: records this worker's deferred labels before a packet that needs them (or + // once their deadline passed), submits the open batch when a label is pending (at once on a + // non-label packet, else once the deadline passed) or the batch holds enough work, and reaps + // finished batches whose completion actions still hold labels. The mutex is taken outright when + // the packet needs the labels or a deadline passed, and by the graphics worker for a pending + // submit; otherwise it is only tried (a register-only packet must not queue behind a 2-4 ms + // dispatch for this) and the next packet retries. A packet that takes the mutex itself + // (PacketLocksItself) gets both the record and the submit at its own acquisition + // (recordLabelsForPacket), so for it the mutex is only tried here as well: a try that succeeds + // records at once (the packet's unlocked prologue then reads through a hook that knows the + // labels, as before), one that fails leaves the group to the packet. Only a deadline still + // locks outright before such a packet: its prologue may take milliseconds, and pollers of the + // label are promised the deadline. APS5_NO_LABEL_PACKET_FLUSH=1 leaves recorded labels pending + // until the deadline (the batch then absorbs the following work); APS5_LABEL_OWN_LOCK=1 takes + // the mutex outright before every packet that needs the labels, as before. + void flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool labelPacket) { + static const bool packetFlush = std::getenv("APS5_NO_LABEL_PACKET_FLUSH") == nullptr; + static const bool ownLock = std::getenv("APS5_LABEL_OWN_LOCK") != nullptr; + static std::atomic packetFlushes{0}, deadlineFlushes{0}, capFlushes{0}, boundaryReaps{0}; + auto& deferred = deferredLabels(); + const bool queued = !deferred.labels.empty(); + const bool selfLocking = !ownLock && PacketLocksItself(header); + bool submit = false, record = false, mustLock = false, deferredDue = false, pendingDue = false; + const auto pending = Graphics::Recorder::PendingLabelSince(); + // The clock is read only while some label waits (every packet passes here). + const auto now = queued || pending.has_value() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + if (queued) { + if (!labelPacket) { + record = true; + mustLock = NeedsRecordedLabels(header) && !selfLocking; + } + if (now - deferred.since >= LabelFlushDeadline()) record = mustLock = deferredDue = true; + } + if (pending.has_value()) { + if (!labelPacket && packetFlush) submit = true; + if (now - *pending >= LabelFlushDeadline()) submit = pendingDue = true; + } + const bool capped = !submit && !record && BatchCap() != 0 && Graphics::Recorder::RecordedWorkSinceSubmit() >= BatchCap(); + const bool reap = !submit && !record && !capped && Graphics::Recorder::PendingCompletionLabels() != 0; + if (!submit && !record && !capped && !reap) return; + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + // The site names the hold too (a try's hold is charged to it; a reap alone shows as 'try'). + if (record) GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + else if (submit || capped) GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + // A reap alone is never worth waiting for the mutex (the next packet retries); neither is a + // submit the self-locking packet makes itself, unless the label's deadline passed. + if (mustLock || (queue == 0 && (capped || (submit && (!selfLocking || pendingDue))))) { + gpuLock.lock(); + } else if (!gpuLock.try_lock()) { + if (selfLocking && record) ++packetLockDeferred; + else if (selfLocking && submit) ++packetSubmitDeferred; + return; + } + const auto localDevice = device.load(); + if (record) { + recordDeferredLabels(localDevice.get(), queue); + // The group goes out with the packet flush (or at once past the deadline), as a label + // recorded at its own packet would have. + if ((!labelPacket && packetFlush) || deferredDue) submit = true; + } + if (localDevice == nullptr) return; + if (submit || capped) { + // Rechecked under the lock: another worker may have submitted meanwhile. A compute + // worker only submits (no reap: a write-back's sync would hold the mutex against queue 0). + if (!Graphics::Recorder::PendingLabelSince().has_value() && (BatchCap() == 0 || Graphics::Recorder::RecordedWorkSinceSubmit() < BatchCap())) return; + localDevice->SubmitRecorded(queue == 0); + if (capped) ++capFlushes; + else if (!labelPacket && packetFlush) ++packetFlushes; + else ++deadlineFlushes; + } else if (reap) { + localDevice->ReapRecorded(); + ++boundaryReaps; + } + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static std::atomic total{0}; + if (profile && (++total & 4095u) == 0) std::fprintf(stderr, "[labels] batch flushes between packets: %llu at the next packet, %llu by deadline, %llu by size cap; %llu boundary reaps; at the packet's own lock: %llu label groups (%.0f ms), %llu submits (%.0f ms incl. queue 0's reaps), left to it by a failed try: %llu groups, %llu submits; after a capture: %llu groups recorded by a try, %llu packets redone for a queued label over the capture; %llu suspend points\n", static_cast(packetFlushes.load()), static_cast(deadlineFlushes.load()), static_cast(capFlushes.load()), static_cast(boundaryReaps.load()), static_cast(packetLockRecords.load()), packetLockRecordUs.load() / 1000.0, static_cast(packetLockSubmits.load()), packetLockSubmitUs.load() / 1000.0, static_cast(packetLockDeferred.load()), static_cast(packetSubmitDeferred.load()), static_cast(captureTryRecords.load()), static_cast(captureRetries.load()), static_cast(suspendPoints.load())); + } + + // WAIT_REG_MEM: another queue or the CPU produces the value; the other queues run on their own threads. + void waitMemory(std::span packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received) { + // The timeout counts from the last sign of GPU progress: while any queue is executing packets, + // the producer may still be on its way (shader compiles alone take hundreds of milliseconds). + auto start = std::chrono::steady_clock::now(); + auto lastDone = packetsDone.load(); + // Names this worker's thread in the [lock] GpuMutex wait report. + GuestMemory::TagGpuLockThread(queue); + auto& outcomes = waitOutcomes(); + const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; + const std::uint64_t awaited = packet[2] | (static_cast(packet[3]) << 32u); + const std::size_t awaitedBytes = wide ? 8 : 4; + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + if (traceGpu) std::fprintf(stderr, "[gpu] %.1f queue 0x%x waits 0x%llx == 0x%x (now 0x%x)\n", TraceMs(), queue, static_cast(packet[2] | (static_cast(packet[3]) << 32u)), packet[4], + *reinterpret_cast(packet[2] | (static_cast(packet[3]) << 32u))); + struct WaitTrace { + bool enabled; std::uint32_t queue; std::uint64_t address; std::chrono::steady_clock::time_point begin; + ~WaitTrace() { + if (!enabled) return; + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - begin).count(); + if (ms >= 0.5) std::fprintf(stderr, "[gpu] %.1f queue 0x%x wait on 0x%llx done after %.1f ms\n", TraceMs(), queue, static_cast(address), ms); + } + } waitTrace{traceGpu, queue, packet[2] | (static_cast(packet[3]) << 32u), std::chrono::steady_clock::now()}; + bool warned = false; + // The collect epoch advances once the wait is over (at entry, polled, timed out, or from a + // label another queue recorded: the CPU's writes before that queue's submission are ordered + // through it), unless a label this queue itself recorded satisfied it, which orders no CPU + // write. After the wait, never before: completions run inside pollService make memo entries + // during the wait that may predate the CPU's store of the flag. + struct EpochPoint { + bool bump = true; + ~EpochPoint() { + if (bump) bumpEpoch(&EpochBumps::waits); + } + } epochPoint; + GuestMemory::CheckRange(reinterpret_cast(awaited), awaitedBytes, awaitedBytes); + if (Pm4::WaitSatisfiedUnchecked(packet)) { + ++outcomes.atEntry; + return; + } + // Debug aids: APS5_NO_LABEL_SHORTCUT=1 never satisfies a wait from the pending-label table; + // APS5_NO_WAIT_OVERLAP_SUBMIT=1 submits the open batch at every wait, as before, instead of + // only when it writes the awaited range. + static const bool labelShortcut = std::getenv("APS5_NO_LABEL_SHORTCUT") == nullptr; + static const bool overlapSubmit = std::getenv("APS5_NO_WAIT_OVERLAP_SUBMIT") == nullptr; + // APS5_WAIT_LOCK=1 takes the GPU mutex at every wait's entry and poll check, as before. + static const bool waitLock = std::getenv("APS5_WAIT_LOCK") != nullptr; + // Read before the checks below: a write noted before the (locked or snapshot) check is seen + // by that check, one noted after it changes the generation the poll loop compares against. + std::uint64_t seenGeneration = Graphics::Recorder::WriteGeneration(); + // Without the GPU mutex first: the label table has its own small mutex, and the lock-free + // pending-write snapshot says whether any recorded work writes the range at all. Only an + // overlap needs the mutex (to submit the open batch); a miss goes straight to polling (its + // producer is the CPU, an in-flight batch, or a queue that has not recorded the label yet: + // the poll loop watches the write generation for that). The snapshot may lag the generation + // (Recorder::noteWrite bumps it before publishing, and NoteLabel follows the note), so a + // decision made from the snapshot never consumes a generation: an unlocked entry makes the + // poll loop's first check a locked (try_lock) one, which sees the producer's consistent view. + const bool unlocked = !waitLock && overlapSubmit; + if (unlocked) { + if (labelShortcut) { + std::uint32_t producer = 0; + if (const auto value = Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++outcomes.fromRecorder; + ++outcomes.fromRecorderUnlocked; + if (producer == queue) { + ++outcomes.fromRecorderSameQueue; + epochPoint.bump = false; + } + return; + } + } + } + const bool lockedEntry = !unlocked || Graphics::Recorder::SnapshotWriteOverlaps(awaited, awaitedBytes); + if (!lockedEntry) ++outcomes.entriesUnlocked; + // An overlapping entry has one thing to do under the mutex: submit the open batch when it + // writes the range (the table lookup already ran without it). It only TRIES the mutex: the + // wait's hold is microseconds, but waiting for the mutex behind a compute queue's build cost + // queue 0 ~4 ms per entry ([lock] 'wait' site). A failed try goes to the poll loop, whose + // first service call repeats the entry's checks under a try of its own (`recheck`) and every + // later one re-tries as well, so the submit happens as soon as the mutex is free, while the + // value may arrive meanwhile (an in-flight batch, another queue, the CPU) and end the wait + // without any hold. Debug aid: APS5_WAIT_LOCK_ENTRY=1 waits for the mutex at entry as before. + static const bool blockingEntry = std::getenv("APS5_WAIT_LOCK_ENTRY") != nullptr; + bool recheck = !lockedEntry; + bool entryTryFailed = false; + if (lockedEntry) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Wait); + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + if (blockingEntry || !unlocked) { + gpuLock.lock(); + } else if (!gpuLock.try_lock()) { + ++outcomes.entryTriesFailed; + recheck = true; + entryTryFailed = true; + } + if (const auto localDevice = gpuLock.owns_lock() ? device.load() : std::shared_ptr{}) { + // A label the recorder holds for this range, recorded after the game submitted this + // packet, will store the value in order (see Recorder::NoteLabel): the wait is over + // without the GPU round trip. The label's batch is submitted by the flush rules. + std::uint32_t producer = 0; + if (labelShortcut) { + if (const auto value = localDevice->PendingLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++outcomes.fromRecorder; + if (producer == queue) { + ++outcomes.fromRecorderSameQueue; + epochPoint.bump = false; + } + return; + } + } + // Recorded work may write the awaited value: send it to the GPU; the poll below sees + // the store once it lands in the imported memory. Work not writing the range stays + // in the open batch (its producer is the CPU, an in-flight batch or a queue that has + // not recorded the label yet; the poll loop submits it once it does). Submit only, + // no reap: this thread must not wait for the GPU under the mutex on another queue's + // behalf (a reaped write-back can sync a later batch through the flush hook). + if (!overlapSubmit || localDevice->OpenWriteOverlaps(awaited, awaitedBytes)) localDevice->SubmitRecorded(false); + } + } + // Consumer-driven submission: the producer's label may be recorded into the open batch after + // this wait began, and the producing worker submits it only at its next packet, so the + // poller (idle anyway) watches the recorder's write generation and submits the batch itself + // when it writes the awaited range, or when any label passed its deadline. In the sleeping + // phase it also reaps finished batches when labels wait in completion actions (a wait on + // such a label converges without a reaper thread). The mutex is only tried: a failed try + // keeps the generation unconsumed and the next check retries. Returns true when the label + // table satisfied the wait meanwhile: a label recorded after this wait began (the common + // queue 0 -> compute -> queue 0 chain when queue 0 reaches its wait first) is accepted on + // the same stamp argument as at entry, sparing the GPU round trip. Every submit here is + // reap-free: a poller never waits for the GPU under the mutex on the producer's behalf. + // An unlocked entry decided from the snapshot alone (or one whose try failed): the first + // check re-does the entry's overlap test under the mutex (a note whose snapshot was not yet + // published when the entry looked must still get its batch submitted; see the entry comment). + const auto pollService = [&](bool sleeping) { + const auto generation = Graphics::Recorder::WriteGeneration(); + const bool changed = generation != seenGeneration || recheck; + const bool labelDue = LabelFlushDue(); + const bool reap = sleeping && Graphics::Recorder::PendingCompletionLabels() != 0; + if (!changed && !labelDue && !reap) return false; + if (changed && unlocked && labelShortcut) { + // The table first, without the mutex: a hit ends the wait. A miss is not final (the + // producer may be between its generation bump and its table entry or snapshot), so + // the generation is consumed only below, under a successful try_lock, where the + // table, the open batch and the snapshot are consistent (a try_lock never waits, + // so this costs no mutex wait; a failed one leaves the generation for the next check). + std::uint32_t producer = 0; + if (const auto value = Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++outcomes.fromRecorderPolling; + ++outcomes.fromRecorderUnlocked; + if (producer == queue) { + ++outcomes.fromRecorderSameQueue; + epochPoint.bump = false; + } + return true; + } + } + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::try_to_lock); + if (!gpuLock.owns_lock()) return false; + const auto localDevice = device.load(); + if (localDevice == nullptr) { + seenGeneration = generation; + recheck = false; + return false; + } + bool submitted = false; + if (changed) { + seenGeneration = generation; + recheck = false; + if (labelShortcut) { + std::uint32_t producer = 0; + if (const auto value = localDevice->PendingLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++outcomes.fromRecorderPolling; + if (producer == queue) { + ++outcomes.fromRecorderSameQueue; + epochPoint.bump = false; + } + return true; + } + } + if (localDevice->OpenWriteOverlaps(awaited, awaitedBytes)) { + localDevice->SubmitRecorded(false); + submitted = true; + } + } + if (labelDue && !submitted && Graphics::Recorder::PendingLabelSince().has_value()) { + localDevice->SubmitRecorded(false); + submitted = true; + } + if (submitted) ++outcomes.pollSubmits; + if (reap) { + localDevice->ReapRecorded(); + ++outcomes.pollReaps; + } + return false; + }; + // A failed entry try is retried once at once, not only after the spin's first 64 pauses: + // the mutex may have freed during the entry's own lookup, and the submit a blocking entry + // made the instant it got the mutex should not wait for the loop's first service. + if (entryTryFailed && pollService(false)) return; + // The producer is usually another queue or the CPU: spin briefly with pause (the value is + // often written within microseconds; the graphics queue's waits average under a millisecond + // and it is the frame-critical thread, so it keeps its core longer than the compute queues, + // which mostly wait tens of milliseconds for it), then poll every 200 us so waiting queues + // do not burn cores. While spinning, the clock, the failure flag and the progress counters + // are read every 64 pauses only; the timeout accounting below stays on the sleeping path. + // Debug aid: APS5_NO_PAUSE_SPIN=1 yields 4000 times before sleeping as before. + static const bool pauseSpin = std::getenv("APS5_NO_PAUSE_SPIN") == nullptr; + const auto spinLimit = queue == 0 ? std::chrono::microseconds(1500) : std::chrono::microseconds(100); + const auto spinStart = std::chrono::steady_clock::now(); + bool spinning = pauseSpin; + std::uint32_t polls = 0; + while (!Pm4::WaitSatisfiedUnchecked(packet)) { + ++polls; + if (spinning) { + _mm_pause(); + if ((polls & 63u) != 0) continue; + if (std::chrono::steady_clock::now() - spinStart > spinLimit) spinning = false; + } else if (pauseSpin || polls >= 4000) { + PollSleep(); + } else { + std::this_thread::yield(); + } + CheckFailure(); + if (pollService(!spinning)) return; + if (const auto done = packetsDone.load(); done != lastDone || packetsInFlight.load() != 0) { + lastDone = done; + start = std::chrono::steady_clock::now(); + } + // Some producers (skipped shaders, interrupt-driven CPU work) are not modeled; rather than deadlock + // the queue, give up on a wait after a while and report it once. + if (!warned && std::chrono::steady_clock::now() - start > std::chrono::milliseconds(WaitTimeoutMs())) { + warned = true; + ++outcomes.timedOut; + static std::set reported; + static std::uint64_t timeouts = 0; + if (++timeouts % 20 == 0) std::fprintf(stderr, "[gpu] %llu GPU waits have timed out\n", static_cast(timeouts)); + if (!reported.insert(packet[2] | (static_cast(packet[3]) << 32u)).second) return; + std::fprintf(stderr, "[gpu] queue 0x%x WAIT_REG_MEM at 0x%llx timed out after %dms (function %u ref 0x%x mask 0x%x value 0x%x)\n", queue, + static_cast(packet[2] | (static_cast(packet[3]) << 32u)), WaitTimeoutMs(), packet[1] & 7u, packet[4], packet[5], + *reinterpret_cast(packet[2] | (static_cast(packet[3]) << 32u))); + const auto address = packet[2] | (static_cast(packet[3]) << 32u); + for (const auto& record : writeHistory()) { + if (record.length != 0 && record.target <= address && address < record.target + std::max(record.length, 4)) + std::fprintf(stderr, "[gpu] earlier write by queue 0x%x opcode 0x%x at 0x%llx+0x%llx\n", record.queue, record.opcode, static_cast(record.target), static_cast(record.length)); + } + context.Each([](const std::string& line) { std::fprintf(stderr, "[gpu] preceding packet %s\n", line.c_str()); }); + } + if (warned) return; + } + ++outcomes.polled; } void execute(const Submission& submission) { - includeSubmission(submission, true); if (submission.suspend) { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.Suspend"); - std::lock_guard gpuLock(gpuMutex); - timing.Mark("gpu_mutex_wait"); - if (device != nullptr) device->WaitIdle(); - timing.Mark("device_idle_wait"); + // A suspend point only marks where the system may suspend the title (the system, not the + // title, waits for the GPU there); this drained the device under the mutex once per + // frame, which was every 'idle' fence wait of queue 0 in the [recorder] line (~10 ms + // each, the frame's whole batch list) although nothing reads results here. Now the + // frame's batches are only submitted, this worker's queued labels first; the register + // reset that follows needs no idle GPU (recorded work keeps what it uses). + // Debug aid: APS5_SUSPEND_DRAIN=1 drains as before. + static const bool suspendDrain = std::getenv("APS5_SUSPEND_DRAIN") != nullptr; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + ++suspendPoints; + if (const auto localDevice = device.load()) { + recordDeferredLabels(localDevice.get(), submission.queue); + if (suspendDrain) localDevice->WaitIdle(); + else localDevice->SubmitRecorded(submission.queue == 0); + } resetGraphics = true; return; } - if (submission.queue == 0 && resetGraphics) { - queues.erase(0); - resetGraphics = false; + QueueState* state = nullptr; + { + std::lock_guard lock(mutex); + if (submission.queue == 0 && resetGraphics) { + queues.erase(0); + resetGraphics = false; + } + state = &queues[submission.queue]; } - auto& queue = queues[submission.queue]; + auto& queue = *state; + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + if (traceGpu) std::fprintf(stderr, "[gpu] %.1f execute serial=%llu queue=0x%x dwords=%zu\n", TraceMs(), static_cast(submission.serial), submission.queue, submission.commands.size()); + // Debug aid: APS5_DUMP_QUEUE= prints the packets of that queue's first 40 submissions. + static const long dumpQueue = [] { const char* text = std::getenv("APS5_DUMP_QUEUE"); return text ? std::strtol(text, nullptr, 16) : -1L; }(); + if (static_cast(submission.queue) == dumpQueue) { + // Only this queue's thread gets here. + static int dumped = 0; + if (dumped++ < 40) { + std::string text = "[queue] submission " + std::to_string(submission.serial) + ":\n"; + for (std::size_t cursor = 0; cursor < submission.commands.size();) { + const auto header = submission.commands[cursor]; + const auto count = Pm4::PacketWords(header); + char line[200]; + int length = std::snprintf(line, sizeof(line), "[queue] %s", Pm4::Name(header).c_str()); + for (std::size_t i = 1; i < count && i < 10 && length < 180; ++i) length += std::snprintf(line + length, sizeof(line) - length, " %08x", submission.commands[cursor + i]); + text += line; + text += "\n"; + cursor += count; + } + std::fputs(text.c_str(), stderr); + } + } + PacketHistory recent{submission.commands}; + // APS5_PROFILE_DRAW: per-queue time spent handling each packet type, reported every 10 s, so + // the packets a frame's submission spends its time on are visible. + static const bool profilePackets = std::getenv("APS5_PROFILE_DRAW") != nullptr; + struct PacketProfile { + std::map> byOpcode; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + std::uint64_t submissions = 0; + // Time in flushBetweenPackets (deferred label submits, reaps and, on queue 0, the mutex + // wait for them), kept apart from the packets so their buckets show their own cost. + double flushMs = 0; + }; + thread_local PacketProfile packetProfile; + ++packetProfile.submissions; + // A submission orders every CPU write the game made before submitting it (the collect memo + // of this worker starts over here); inside it only the ordering points below do, unless + // APS5_PACKET_EPOCH=1 restores an epoch per packet. + bumpEpoch(&EpochBumps::submissions); for (std::size_t cursor = 0; cursor < submission.commands.size();) { - if (frameTiming == nullptr) includeSubmission(submission, false); + if (PacketEpoch()) bumpEpoch(&EpochBumps::packets); + CheckFailure(); const auto header = submission.commands[cursor]; - const auto count = static_cast((header >> 16u) & 0x3fffu) + 2; + if (Pm4::FillerPacket(header)) { ++cursor; continue; } + const auto count = Pm4::PacketWords(header); const auto packet = std::span(submission.commands).subspan(cursor, count); const auto opcode = (header >> 8u) & 0xffu; - { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.Packet"); - CheckFailure(); - timing.Mark("failure_check"); - if (opcode == 0x37 || opcode == 0x40 || opcode == 0x50 || opcode == 0x42 || opcode == 0x46 || opcode == 0x58 || header == FlipPacketHeader) { - std::lock_guard gpuLock(gpuMutex); - timing.Mark("gpu_mutex_wait"); - const auto eventType = opcode == 0x46 ? packet[1] & 0x3fu : 0u; - const auto memoryTransfer = opcode == 0x37 || opcode == 0x40 || opcode == 0x50; - const auto waitDraws = memoryTransfer || opcode == 0x42 || (opcode == 0x46 && (eventType == 0x07 || eventType == 0x0f || eventType == 0x10)); - const auto gpuCacheBarrier = opcode == 0x58 && Pm4::UsesGpuCacheBarrier(packet); - if (device != nullptr) { - if (gpuCacheBarrier) device->AcquireGpuMemory(); - else if (waitDraws) device->WaitDraws(); - else { - const auto scope = header == FlipPacketHeader ? "Driver.FlipWait" : opcode == 0x58 ? "Driver.AcquireMemoryWait" : "Driver.CacheEventWait"; - PerformanceTimer waitTiming(scope); - device->WaitIdle(); + // Names this packet for the flush hook's sync attribution ([hooksync]); the flip is 0xffff. + GuestMemory::SetCurrentPacket(header == FlipPacketHeader ? 0xffffu : opcode, submission.queue); + // Pending labels and full batches go to the GPU before this packet's own work starts + // (see flushBetweenPackets); labels themselves only check the deadline, so a label + // group shares one submission. Timed on its own, before the packet's timer starts. + const auto flushStart = profilePackets ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + flushBetweenPackets(submission.queue, header, opcode == 0x49 || opcode == 0x37); + struct PacketTimer { + bool enabled; std::uint32_t key; std::uint32_t queue; PacketProfile& profile; std::chrono::steady_clock::time_point start; + ~PacketTimer() { + if (!enabled) return; + const auto now = std::chrono::steady_clock::now(); + auto& entry = profile.byOpcode[key]; + ++entry.first; + entry.second += std::chrono::duration(now - start).count(); + if (now - profile.lastReport < std::chrono::seconds(10)) return; + profile.lastReport = now; + std::vector>> hot(profile.byOpcode.begin(), profile.byOpcode.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.second > b.second.second; }); + std::string report; + for (std::size_t i = 0; i < hot.size() && i < 10; ++i) { + char text[96]; + std::snprintf(text, sizeof(text), " %s x%llu %.0fms", hot[i].first == 0xffffu ? "flip" : Pm4::Name(hot[i].first << 8u).c_str(), static_cast(hot[i].second.first), hot[i].second.second); + report += text; + } + auto& waits = waitOutcomes(); + auto& epochs = epochBumps(); + std::fprintf(stderr, "[packets] queue 0x%x %llu submissions, time by packet (10 s):%s, flush %.0fms; waits satisfied: at entry %llu, from recorder %llu (%llu while polling, same queue %llu, %llu without the GPU mutex), polled %llu (%llu entered without the GPU mutex, %llu entry tries failed), timed out %llu; poll submits %llu, poll reaps %llu; epoch bumps: submissions %llu, waits %llu, drains %llu, reaps %llu, packets %llu\n", queue, static_cast(profile.submissions), report.c_str(), profile.flushMs, static_cast(waits.atEntry), static_cast(waits.fromRecorder + waits.fromRecorderPolling), static_cast(waits.fromRecorderPolling), static_cast(waits.fromRecorderSameQueue), static_cast(waits.fromRecorderUnlocked), static_cast(waits.polled), static_cast(waits.entriesUnlocked), static_cast(waits.entryTriesFailed), static_cast(waits.timedOut), static_cast(waits.pollSubmits), static_cast(waits.pollReaps), static_cast(epochs.submissions), static_cast(epochs.waits), static_cast(epochs.drains), static_cast(epochs.reaps), static_cast(epochs.packets)); + waits = WaitOutcomes{}; + epochs = EpochBumps{}; + profile.byOpcode.clear(); + profile.submissions = 0; + profile.flushMs = 0; + } + } packetTimer{profilePackets, header == FlipPacketHeader ? 0xffffu : opcode, submission.queue, packetProfile, std::chrono::steady_clock::now()}; + if (profilePackets) packetProfile.flushMs += std::chrono::duration(packetTimer.start - flushStart).count(); + // Label writes are recorded on the GPU behind the work they signal whenever possible, so + // the CPU never waits for them. Otherwise packets that write guest memory in order with + // GPU work (labels, copies, constant RAM dumps), draws and flips drain the recorded batches + // first; reads (indirect arguments, register lists, waits) go through the flush hook. + // Debug aid: APS5_DRAIN_ALL=1 drains before every memory packet as before. + static const bool drainAll = std::getenv("APS5_DRAIN_ALL") != nullptr; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + bool wroteOnGpu = false; + // A label the CPU writes while the recorder is idle needs no drain: nothing the GPU was + // given is still running (draws complete synchronously), so the write is already ordered + // after every earlier packet. That was 70% of all drains at the movie stage. + bool orderedAlready = false; + // Labels that store nothing (RELEASE_MEM without data select or destination: + // interrupt-only) need no drain because Pm4::Execute is a no-op for them. The label + // counters and the [sync] report live at namespace scope (see reportSync). + if (!drainAll && (opcode == 0x49 || opcode == 0x37)) { + if (const auto label = Pm4::DecodeLabelWrite(packet)) { + const auto bytes = label->Bytes(); + if (DeferLabels() && bytes.size() <= DeferredLabel::Capacity && bytes.size() % 4 == 0 && label->address % 4 == 0) { + // Queued on this worker, no mutex: recorded with the group by + // flushBetweenPackets before the next packet that needs it (or by the + // deadline), still ahead of every later packet of this queue. + auto& deferred = deferredLabels(); + if (deferred.labels.empty()) deferred.since = std::chrono::steady_clock::now(); + auto& entry = deferred.labels.emplace_back(); + entry.address = label->address; + entry.size = bytes.size(); + std::memcpy(entry.bytes.data(), bytes.data(), bytes.size()); + ++queuedLabels; + wroteOnGpu = true; + } else { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + // Earlier labels of this queue go first (queue order). The stamp is taken + // under the mutex, right before the label is noted, so it orders after every + // submission the game made before this point. Without a device (reason 4) + // the packet drains nothing and Pm4::Execute stores it, as before. + recordDeferredLabels(localDevice.get(), submission.queue); + const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, ++eventSerial, submission.queue) : 4; + wroteOnGpu = reason == 0 || reason == 5 || reason == 6; + if (reason == 1) { + // Idle recorder: stored here, still under the mutex, as + // recordDeferredLabels does. Stored later by Pm4::Execute outside it, + // another worker could record work over the range first and the flush + // hook would then sync for the store. + GuestMemory::Write(label->address, bytes, 4); + wroteOnGpu = true; + } + countLabelOutcome(reason); + ++immediateLabels; + } + } else if (opcode == 0x49 ? ((packet[2] >> 29u) == 0 || (packet[3] | (static_cast(packet[4]) << 32u)) == 0) : (packet[2] | (static_cast(packet[3]) << 32u)) == 0) { + orderedAlready = true; + ++noOpLabels; + } else { + ++labelFallbacks[4]; + } + } + // The other stores the driver makes for the title (COPY_DATA and DMA_DATA to memory, + // DUMP_CONST_RAM) used to drain the device and store on the CPU. Their bytes are known + // before the GPU runs them (immediate, constant RAM, or a source read through the flush + // hook, which waits only for recorded work writing the source), so the store is recorded + // on the GPU exactly like a label (VulkanDevice::WriteLabelOnGpu: host import, + // barriers, label table, pending-write note): a destination that recorded work also + // writes is then ordered by the queue and no CPU wait happens. An idle recorder keeps + // the CPU store (nothing to order after; done under the mutex so no work can be recorded + // over the range first). A store WriteLabelOnGpu cannot take (more than 64 KiB, not + // 4-byte aligned, no device) or that does not decode drains and stores as before. + // Kill switch: APS5_CPU_STORES=1 keeps the drain and CPU store for every such packet. + static const bool cpuStores = std::getenv("APS5_CPU_STORES") != nullptr; + if (!drainAll && !cpuStores && (opcode == 0x40 || opcode == 0x50 || opcode == 0x83)) { + constexpr std::size_t gpuStoreLimit = 65536; + bool drained = true; + if (const auto store = Pm4::ResolveStore(packet, queue, gpuStoreLimit)) { + const auto bytes = store->Bytes(); + if (bytes.empty()) { + // Nothing to store: Pm4::Execute is a no-op for it too. + orderedAlready = true; + drained = false; + } else { + // No lock site of its own: these stores count under [lock] 'label'. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + // Results still on the GPU for the range would be stored over (or skipped + // around, as a block MarkWritten stamps) the store later: written back + // first, like a buffer fill (a GPU-direct write-back is recorded before the + // store and ordered by its ALL_COMMANDS -> TRANSFER barrier). The CPU and + // drained paths get this through the flush hook of GuestMemory::Write. + Graphics::StorageTexture::FlushPending(store->address, bytes.size(), nullptr, "packet store"); + // Earlier labels of this queue go first (queue order); the stamp is taken + // under the mutex like a label's. + recordDeferredLabels(localDevice.get(), submission.queue); + const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(store->address, bytes, ++eventSerial, submission.queue) : 4; + if (reason == 0 || reason == 5 || reason == 6) { + if (reason == 0) ++storesOnGpu; + else ++storesBehindCompletions; + wroteOnGpu = true; + drained = false; + } else if (reason == 1) { + GuestMemory::Write(store->address, bytes, 1); + ++storesOnCpu; + wroteOnGpu = true; + drained = false; } } - timing.Mark(gpuCacheBarrier ? "gpu_cache_barrier" : waitDraws ? "draw_wait" : "device_idle_wait"); } - if (header == FlipPacketHeader) { - CheckFailure(); - timing.Mark("flip_prepare"); - } else if (opcode == 0x15) { - dispatch(queue, packet, submission); - } else if (opcode == 0x16) { - std::array direct; - { - std::lock_guard gpuLock(gpuMutex); - const GuestMemory::MemoryAccessScope memoryScope(device.get(), [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - if (context) static_cast(context)->ResolveMemory(address, bytes, writable); - }); - direct = Pm4::ResolveDispatch(packet, queue); + if (drained) ++storesDrained; + } + // Draws no longer drain: a recorded draw follows earlier recorded work in queue order, and + // a synchronous draw's batch submits the recorder first (CommandBatch); the guest memory a + // draw's preparation reads goes through the flush hook. Debug aid: APS5_DRAW_DRAIN=1 restores. + static const bool drawDrain = std::getenv("APS5_DRAW_DRAIN") != nullptr; + const bool drawPacket = opcode == 0x2d || opcode == 0x35; + // Flips no longer drain either: the frame's batches are submitted and the presenter's blit + // follows them on the same queue (see Driver::Present). Debug aid: APS5_SYNC_FLIP=1 restores. + static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; + const bool drains = drainAll ? ((Pm4::AccessesMemory(header) && opcode != 0x16) || opcode == 0x42 || opcode == 0x46 || opcode == 0x58 || header == FlipPacketHeader) + : (!wroteOnGpu && !orderedAlready && (opcode == 0x49 || opcode == 0x37 || opcode == 0x40 || opcode == 0x50 || opcode == 0x83 || (drawPacket && drawDrain) || (header == FlipPacketHeader && syncFlip))); + if (drains) { + // The GPU wait happens without the mutex: the batches are submitted under it, the + // timeline value is waited for outside (the other workers keep recording), then the + // completions of the batches up to that serial run under it again, in order, before + // the packet's own CPU store. Work another thread submits meanwhile is not waited + // for: a RELEASE_MEM only orders after earlier work, and a CPU store into a range + // such work writes is caught by the flush hook. vkDeviceWaitIdle is not needed: the + // presenter waits its own render fence and every other submission is synchronous. + // The device is held by this shared_ptr across the wait and released under the + // mutex (a replacement on another thread must not destroy it outside). + // Debug aids: APS5_NO_UNLOCKED_DRAIN=1 and APS5_DRAIN_ALL=1 drain under the mutex. + static const bool unlockedDrain = std::getenv("APS5_NO_UNLOCKED_DRAIN") == nullptr && !drainAll; + // A drain is an ordering point for the collect memo (see GuestMemory::BumpCollectEpoch). + bumpEpoch(&EpochBumps::drains); + std::shared_ptr draining; + std::uint64_t epoch = 0; + { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (profile) { + ++drainCounts[header == FlipPacketHeader ? 0xffffu : opcode]; + ++drainTotal; + if (std::chrono::steady_clock::now() - lastSyncReport > std::chrono::seconds(10)) { + lastSyncReport = std::chrono::steady_clock::now(); + reportSync(); + } } - dispatch(queue, direct, submission); - } else if (opcode == 0x35 || opcode == 0x2d) { - draw(queue, packet, submission); - } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { - std::lock_guard gpuLock(gpuMutex); - const GuestMemory::MemoryAccessScope memoryScope(device.get(), [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - if (context) static_cast(context)->ResolveMemory(address, bytes, writable); - }); - Pm4::Execute(packet, queue); - timing.Mark("pm4_execute"); + draining = device.load(); + // A drain is the ordering point: this worker's queued labels (an undecodable + // label packet skips the record in flushBetweenPackets) go in first, so the + // packet's own CPU store lands after them in queue order. + recordDeferredLabels(draining.get(), submission.queue); + if (draining != nullptr) { + if (unlockedDrain && draining->CanWaitUnlocked()) epoch = draining->SubmitAndEpoch(); + else draining->WaitIdle(); + } + if (epoch == 0) draining.reset(); + } + if (epoch != 0) { + ++unlockedDrains; + try { + draining->WaitRecorded(epoch); + } catch (...) { + // A lost device: if Present replaced it meanwhile this copy is the last + // owner, and its recorder must still be torn down under the mutex. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + draining.reset(); + throw; + } + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + draining->ReapRecorded(epoch); + draining.reset(); } } + traceLabel(packet, submission.queue); + // Waits do not count as progress; every other packet does, including while it runs. + const bool waitPacket = opcode == 0x3c || opcode == 0x93; + struct Progress { + Driver& driver; + bool counted; + ~Progress() { + if (!counted) return; + --driver.packetsInFlight; + ++driver.packetsDone; + } + } progress{*this, !waitPacket}; + if (!waitPacket) ++packetsInFlight; + recent.Record(cursor); if (header == FlipPacketHeader) { - frameTiming->SetFlip(submission.serial, cursor, submission.received, FrameTiming::Clock::now()); - const auto completedFrame = std::exchange(frameTiming, nullptr); - submission.flips.at(cursor)->GpuReady(completedFrame); + CheckFailure(); + if (!drains) { + // Frame N's recorded batches go to the GPU now; the presenter's own submission + // follows them in queue order, so no wait is needed here and GpuReady returns at + // once (the title paces on flipPendingNum, which the presenter drops after showing + // the frame). + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + // This worker's queued labels belong to the frame: recorded first (queue order), + // then everything goes out (an empty open batch submits nothing) unless the + // record submitted already (a second call would only scan the fences again). + if (!recordLabelsForPacket(localDevice.get(), submission.queue) && localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0); + } + // Each flip carries a fresh frame timing record; the per-stage frame statistics upstream + // collects are not gathered on this path. + auto frame = std::make_shared(++frameSerial); + const auto now = FrameTiming::Clock::now(); + frame->IncludeSubmission(submission.serial, now, now, now, true); + frame->SetFlip(submission.serial, cursor, now, now); + submission.flips.at(cursor)->GpuReady(frame); + } else if (opcode == 0x15) { + timed(&WorkerProfile::dispatchMs, [&] { tolerate("dispatch", [&] { dispatch(queue, packet, submission); }); }); + Graphics::Recorder::CountRecordedWork(); + } else if (opcode == 0x16) { + timed(&WorkerProfile::dispatchMs, [&] { tolerate("indirect dispatch", [&] { dispatchIndirect(queue, packet, submission); }); }); + Graphics::Recorder::CountRecordedWork(); + } else if (opcode == 0x3c || opcode == 0x93) { + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + const auto waitStart = std::chrono::steady_clock::now(); + timed(&WorkerProfile::waitMs, [&] { waitMemory(packet, submission.queue, recent, submission.received); }); + if (traceGpu && std::chrono::steady_clock::now() - waitStart > std::chrono::milliseconds(200)) { + // List the rest of the submission to show what the stalled queue would have done next. + for (std::size_t next = cursor + count, shown = 0; next < submission.commands.size() && shown < 48; ++shown) { + const auto nextHeader = submission.commands[next]; + const auto nextCount = Pm4::PacketWords(nextHeader); + const auto nextPacket = std::span(submission.commands).subspan(next, nextCount); + std::fprintf(stderr, "[gpu] then %s", Pm4::Name(nextHeader).c_str()); + for (std::size_t i = 1; i < nextPacket.size() && i < 7; ++i) std::fprintf(stderr, " %08x", nextPacket[i]); + std::fprintf(stderr, "\n"); + next += nextCount; + } + } + } else if (opcode == 0x35 || opcode == 0x2d) { + timed(&WorkerProfile::drawMs, [&] { tolerate("draw", [&] { + static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; + const auto color = (static_cast(readRegister(queue.context, 0x390)) << 40u) | (static_cast(readRegister(queue.context, 0x318)) << 8u); + try { + draw(queue, packet, submission); + if (traceDraws) std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x ok\n",static_cast(color), readRegister(queue.context, 0x8e)); + } catch (const std::exception& error) { + if (traceDraws) std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x failed: %.160s\n",static_cast(color), readRegister(queue.context, 0x8e), error.what()); + // Name the color target so skipped draws can be matched against the scanout buffer. + char suffix[48]; + std::snprintf(suffix, sizeof(suffix), " [color target 0x%llx]", static_cast(color)); + // Debug aid: the first failing draw per color target and reason saves its register banks to + // draw__.regs. + static std::set> dumpedTargets; + const std::string reason = std::string(error.what()).substr(0, 48); + if (dumpedTargets.insert({color, reason}).second) { + char name[64]; + std::snprintf(name, sizeof(name), "draw_%llx_%08x.regs", static_cast(color), static_cast(std::hash{}(reason))); + if (std::FILE* file = std::fopen(name, "w")) { + std::fprintf(file, "# %s\n", error.what()); + recent.Each([&](const std::string& line) { std::fprintf(file, "# packet %s\n", line.c_str()); }); + for (const auto& [offset, value] : queue.context) std::fprintf(file, "context %x %08x\n", offset, value); + for (const auto& [offset, value] : queue.userConfig) std::fprintf(file, "uconfig %x %08x\n", offset, value); + for (const auto& [offset, value] : queue.shader) std::fprintf(file, "shader %x %08x\n", offset, value); + std::fclose(file); + } + } + throw std::runtime_error(error.what() + std::string(suffix)); + } + }); }); + } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { + if (!wroteOnGpu) Pm4::Execute(packet, queue); } + if (drawPacket) Graphics::Recorder::CountRecordedWork(); cursor += count; } + // The submission's last labels and dispatches go to the GPU before it counts as completed + // (Driver::WaitIdle and the game threads polling the labels must not depend on a later + // packet of some queue). Checked lock-free first: an empty open batch needs no submit. A + // compute worker submits without reaping (it must not wait under the mutex for queue 0). + if (!deferredLabels().labels.empty() || Graphics::Recorder::PendingLabelSince().has_value() || Graphics::Recorder::RecordedWorkSinceSubmit() != 0) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + // This worker's deferred labels are recorded first (they belong to this submission). + recordDeferredLabels(localDevice.get(), submission.queue); + if (localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0); + } } - void run() noexcept { + // An idle worker retires labels waiting in completion actions (see Recorder::AfterCompletions): + // without it a label whose only consumer is a game thread, submitted last before the game waits + // for it, would never land once every worker sleeps. Non-blocking (a fence status check under + // the mutex, tried only); the caller polls it every millisecond while such labels exist. + void reapCompletionLabels() { + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::try_to_lock); + if (!gpuLock.owns_lock()) return; + // Every idle worker polls this; once another worker reaped the labels there is nothing to + // do, and the epoch bump below would only inflate the "reaps" and [guestmem] epoch counts. + if (Graphics::Recorder::PendingCompletionLabels() == 0) return; + // The write-backs the reap runs collect outside any submission: a fresh epoch, so no walk + // of this worker's last packet is reused for them. + bumpEpoch(&EpochBumps::reaps); + if (const auto localDevice = device.load()) localDevice->ReapRecorded(); + } + + void run(std::uint32_t id) noexcept { + OnWorkerThread() = true; + // Named from the start, so a worker's fence waits before its first WAIT_REG_MEM land in its + // own row of the [recorder] "fence waits by thread" report and "untagged" is exactly the + // presenter and the game threads. + GuestMemory::TagGpuLockThread(id); + if (id == 0) StartWorkerSampler(); Submission submission; try { for (;;) { + submission = Submission{}; + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.Worker"); - submission = Submission{}; - timing.Mark("submission_release"); std::unique_lock lock(mutex); - timing.Mark("queue_mutex_wait"); - changed.wait(lock, [&] { return failure || stopping || !pending.empty(); }); - timing.Mark("wait_for_submission"); + auto& pending = workers.at(id).pending; + if (traceGpu && pending.empty()) std::fprintf(stderr, "[gpu] %.1f idle queue=0x%x\n", TraceMs(), id); + const auto ready = [&] { return failure || stopping || !pending.empty(); }; + // While completion labels are pending, the idle wait wakes every millisecond to + // reap them (the driver mutex is dropped for that: it is never held with the GPU + // mutex); otherwise it sleeps until work arrives. + while (!ready()) { + if (Graphics::Recorder::PendingCompletionLabels() == 0) { + changed.wait(lock, ready); + break; + } + if (changed.wait_for(lock, std::chrono::milliseconds(1), ready)) break; + lock.unlock(); + reapCompletionLabels(); + lock.lock(); + } rethrowFailure(); if (pending.empty()) { break; } submission = std::move(pending.front()); pending.pop_front(); - submission.dequeued = FrameTiming::Clock::now(); } execute(submission); + if (traceGpu) std::fprintf(stderr, "[gpu] %.1f done serial=%llu queue=0x%x\n", TraceMs(), static_cast(submission.serial), id); { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.SubmissionCompletion"); - std::lock_guard gpuLock(gpuMutex); - if (device) device->WaitIdle(); - } - { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.Completion"); std::lock_guard lock(mutex); - timing.Mark("mutex_wait"); rethrowFailure(); - completed = submission.serial; + markCompleted(submission.serial); } changed.notify_all(); } - std::lock_guard gpuLock(gpuMutex); - device.reset(); } catch (...) { const auto error = std::current_exception(); + try { + std::rethrow_exception(error); + } catch (const std::exception& reason) { + std::fprintf(stderr, "[gpu] worker failed: %s\n", reason.what()); + } catch (...) { + std::fprintf(stderr, "[gpu] worker failed with a non-standard exception\n"); + } for (const auto& [offset, flip] : submission.flips) flip->Fail(error); ReportFailure(error); { - std::lock_guard gpuLock(gpuMutex); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); device.reset(); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 52e4ccc09d..0789c853b5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -1,12 +1,18 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include +#include +#include +#include #include +#include +#include #include #include #include #include +#include #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX @@ -22,6 +28,471 @@ namespace { void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("AGC driver: ") + reason); } + +// A code address relative to its module, for symbolizing debug traces with nm. +unsigned long long ModuleOffset(const void* address) { +#ifdef _WIN32 + HMODULE module = nullptr; + GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, static_cast(address), &module); + return reinterpret_cast(address) - reinterpret_cast(module); +#else + return reinterpret_cast(address); +#endif +} + +#ifdef _WIN32 +bool readableProtection(DWORD protection) { + return protection == PAGE_READONLY || protection == PAGE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_READ || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY; +} + +bool writableProtection(DWORD protection) { + return protection == PAGE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY; +} +#endif + +} + +namespace { +std::atomic forgetCalls{0}; +std::atomic forgetBytes{0}; +std::atomic collectMemoHits{0}; +std::atomic collectEpochBumps{0}; +// Walks that reported at least one written page (the ones the probe pass used to double), and the +// tracker mutex acquisitions that had to wait (APS5_PROFILE_DRAW; see lockTracker). +std::atomic collectDirty{0}; +std::atomic trackerWaits{0}; +std::atomic trackerAcquisitions{0}; +std::uintptr_t PagesBase(); +std::size_t PagesSize(); +std::uintptr_t ImagePagesBase(); +std::size_t ImagePagesSize(); +} + +// Profile (APS5_PROFILE_DRAW): calls, bytes and time of the guest memory copies and comparisons the +// driver makes, printed every 10 s as [guestmem]. +struct MemoryCounter { + std::atomic calls{0}; + std::atomic bytes{0}; + std::atomic nanoseconds{0}; +}; + +struct MemoryProfile { + MemoryCounter read; + MemoryCounter compare; + MemoryCounter write; + MemoryCounter changedWrite; + MemoryCounter collect; + MemoryCounter verify; + MemoryCounter query; + std::atomic lastReport{0}; + // Sampled callers of Read and Write (module offsets), to name what touches guest memory most. + std::mutex callersMutex; + std::array, 16> callers{}; + std::array, 16> writeCallers{}; +}; + + +MemoryProfile& Profile() { + static MemoryProfile profile; + return profile; +} + +void CountCaller(std::array, 16>& callers, std::uint32_t& sampled, std::uint32_t every, const void* returnAddress) { + if (++sampled % every != 0) return; + auto& state = Profile(); + const auto offset = ModuleOffset(returnAddress); + std::lock_guard lock(state.callersMutex); + for (auto& [caller, count] : callers) { + if (caller == offset || count == 0) { + caller = offset; + count += every; + return; + } + } +} + +// Attribution tags (see the header): the packet a queue worker executes and the read site the +// driver code named. Plain thread-locals; a store per packet costs nothing measurable. +thread_local PacketTag currentPacket{NoPacket, 0xffffffffu}; +thread_local ReadSite currentReadSite = ReadSite::Unknown; +// Sampled reads by site (every 256th Read/ReadCommitted, like the callers), for the [guestmem] line. +std::atomic readSiteSamples[static_cast(ReadSite::Count)] = {}; + +void SetCurrentPacket(std::uint32_t opcode, std::uint32_t queue) { + currentPacket = {opcode, queue}; +} + +PacketTag CurrentPacket() { + return currentPacket; +} + +const char* ReadSiteName(ReadSite site) { + static const char* const names[static_cast(ReadSite::Count)] = {"unknown", "capture", "dispatch-cache", "texture-compare", "texture-read", "buffer-upload", "index-buffer", "vertex-buffer", "registers", "indirect-args", "wait", "label", "scanout", "store"}; + const auto index = static_cast(site); + return index < static_cast(ReadSite::Count) ? names[index] : "?"; +} + +ReadSite SetReadSite(ReadSite site) { + return std::exchange(currentReadSite, site); +} + +ReadSite CurrentReadSite() { + return currentReadSite; +} + +std::size_t CaptureCallerOffsets(std::span frames, unsigned skip) { + if (frames.empty()) return 0; +#ifdef _WIN32 + // Walks the x64 unwind tables, so it needs no frame pointers (unlike __builtin_return_address(n) + // for n > 0); frame 0 of the capture is this function's caller. + void* raw[16]; + const auto wanted = static_cast(std::min(frames.size(), std::size(raw))); + const auto captured = RtlCaptureStackBackTrace(1 + skip, wanted, raw, nullptr); + // The driver's module is resolved once (this function lives in it): a per-frame + // GetModuleHandleEx would take the loader lock several times per sync under the GpuMutex. + static const auto module = [] { + HMODULE handle = nullptr; + GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(&CaptureCallerOffsets), &handle); + std::uintptr_t size = 0; + if (handle != nullptr) { + // The image size comes from the mapped PE headers (no psapi dependency). + const auto* base = reinterpret_cast(handle); + const auto* dos = reinterpret_cast(base); + const auto* nt = reinterpret_cast(base + dos->e_lfanew); + size = nt->OptionalHeader.SizeOfImage; + } + return std::pair(reinterpret_cast(handle), size); + }(); + for (std::size_t i = 0; i < captured; ++i) { + const auto address = reinterpret_cast(raw[i]); + frames[i] = module.first != 0 && address >= module.first && address - module.first < module.second ? address - module.first : 0; + } + return captured; +#else + static_cast(skip); + frames[0] = ModuleOffset(__builtin_return_address(0)); + return 1; +#endif +} + +void CountReadCaller(const void* returnAddress) { + thread_local std::uint32_t sampled = 0; + // Sampled at the same rate as the callers, in step with them (CountCaller advances the counter). + if ((sampled + 1) % 256 == 0) readSiteSamples[static_cast(currentReadSite)].fetch_add(256, std::memory_order_relaxed); + CountCaller(Profile().callers, sampled, 256, returnAddress); +} + +void CountWriteCaller(const void* returnAddress) { + thread_local std::uint32_t sampled = 0; + CountCaller(Profile().writeCallers, sampled, 16, returnAddress); +} + +class TimedAccess { +public: + TimedAccess(MemoryCounter& counter, std::uint64_t bytes) : counter(counter), bytes(bytes), start(std::chrono::steady_clock::now()) {} + ~TimedAccess() { + const auto now = std::chrono::steady_clock::now(); + counter.calls += 1; + counter.bytes += bytes; + counter.nanoseconds += static_cast(std::chrono::duration_cast(now - start).count()); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& state = Profile(); + const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); + auto last = state.lastReport.load(); + if (nowMs - last < 10000 || !state.lastReport.compare_exchange_strong(last, nowMs)) return; + const auto line = [](const char* name, const MemoryCounter& c) { + const double seconds = c.nanoseconds / 1e9; + std::fprintf(stderr, " %s %llu calls %.0f MiB %.1f s", name, static_cast(c.calls.load()), c.bytes / 1048576.0, seconds); + }; + std::fprintf(stderr, "[guestmem]"); + line("read", state.read); + line("compare", state.compare); + line("write", state.write); + line("write-changed", state.changedWrite); + line("collect", state.collect); + line("verify", state.verify); + line("query", state.query); + std::fprintf(stderr, " collect-memo hits %llu, collect epochs %llu", static_cast(collectMemoHits.load()), static_cast(collectEpochBumps.load())); + std::fprintf(stderr, " collect-dirty %llu tracker waits %llu / %llu", static_cast(collectDirty.load()), static_cast(trackerWaits.load()), static_cast(trackerAcquisitions.load())); + std::fprintf(stderr, " | arena 0x%llx+0x%llx image 0x%llx+0x%llx forgets %llu (%.0f MiB)", static_cast(PagesBase()), static_cast(PagesSize()), static_cast(ImagePagesBase()), static_cast(ImagePagesSize()), static_cast(forgetCalls.load()), forgetBytes.load() / 1048576.0); + { + std::lock_guard lock(state.callersMutex); + std::fprintf(stderr, " | read callers:"); + for (const auto& [caller, count] : state.callers) { + if (count != 0) std::fprintf(stderr, " +0x%llx=%llu", caller, static_cast(count)); + } + std::fprintf(stderr, " | write callers:"); + for (const auto& [caller, count] : state.writeCallers) { + if (count != 0) std::fprintf(stderr, " +0x%llx=%llu", caller, static_cast(count)); + } + } + std::fprintf(stderr, " | read sites:"); + for (std::size_t site = 0; site < static_cast(ReadSite::Count); ++site) { + const auto count = readSiteSamples[site].load(std::memory_order_relaxed); + if (count != 0) std::fprintf(stderr, " %s=%llu", ReadSiteName(static_cast(site)), static_cast(count)); + } + std::fprintf(stderr, "\n"); + } +private: + MemoryCounter& counter; + std::uint64_t bytes; + std::chrono::steady_clock::time_point start; +}; + +namespace { + +// Page-state cache. VirtualQuery finds the extent of a run of equally mapped pages by walking page +// tables, which costs tens of milliseconds inside the game's committed gigabyte heap, and GPU workers +// verify ranges thousands of times per second. Committed pages of the guest arena are therefore +// remembered, one byte of access flags per 4 KiB page (the table is committed lazily, so only the +// pages ever verified cost memory). libc reports every range whose mapping or protection changed +// (registry mutations and guest heap decommits) through the invalidator, which forgets those pages. +// Only committed pages are cached; reserved pages are queried each time, so a commit made outside the +// registry (the guest heap grows that way) is seen at once. +// A second table covers the main guest image (the PE at 0x7ff6b...): its protections change only +// through registry mutations (mprotect replaces the registered range and invalidates through the same +// callback), and address-based builds verify every image section on each build, so without the table +// each of those verifications was a VirtualQuery. APS5_NO_IMAGE_PAGE_CACHE=1 leaves the image uncached. +constexpr std::size_t PageBytes = 4096; +constexpr std::uint8_t PageReadable = 1; +constexpr std::uint8_t PageWritable = 2; + +struct PageSpan { + std::uintptr_t base = 0; + std::size_t size = 0; + // Per page: access flags, zero while unknown. + std::uint8_t* pages = nullptr; + + // Reserves and commits the table; untouched table pages cost nothing until a guest page is recorded. + bool allocate() { + if (size == 0) return false; + const auto count = (size / PageBytes + 8) & ~static_cast(7); +#ifdef _WIN32 + pages = static_cast(VirtualAlloc(nullptr, count, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + pages = static_cast(std::calloc(count, 1)); +#endif + if (pages == nullptr) size = 0; + return pages != nullptr; + } + + bool covers(std::uintptr_t address) const { + return size != 0 && address >= base && address - base < size; + } + + std::uint8_t load(std::uintptr_t address) const { + return std::atomic_ref(pages[(address - base) / PageBytes]).load(std::memory_order_relaxed); + } + + void store(std::uintptr_t address, std::uint8_t value) { + std::atomic_ref(pages[(address - base) / PageBytes]).store(value, std::memory_order_relaxed); + } + + // End of the run of pages equal to `value` that starts at `address`, at most `limit`. + std::uintptr_t runEnd(std::uintptr_t address, std::uint8_t value, std::uintptr_t limit) const { + auto index = (address - base) / PageBytes; + const auto stop = std::min((std::min(limit, base + size) - base + PageBytes - 1) / PageBytes, size / PageBytes); + const std::uint64_t wide = 0x0101010101010101ull * value; + while (index < stop) { + if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref(*reinterpret_cast(pages + index)).load(std::memory_order_relaxed) == wide) { + index += 8; + continue; + } + if (std::atomic_ref(pages[index]).load(std::memory_order_relaxed) != value) break; + ++index; + } + return std::min(limit, base + index * PageBytes); + } + + void forget(std::uintptr_t address, std::size_t bytes) { + if (size == 0 || bytes == 0 || address >= base + size || address + bytes <= base) return; + const auto first = (std::max(address, base) - base) / PageBytes; + const auto last = (std::min(address + bytes, base + size) - 1 - base) / PageBytes; + for (auto page = first; page <= last; ++page) std::atomic_ref(pages[page]).store(0, std::memory_order_relaxed); + } +}; + +struct PageStates { + std::once_flag once; + PageSpan arena; + PageSpan image; + + void initialize(); + + PageSpan* spanOf(std::uintptr_t address) { + if (arena.covers(address)) return &arena; + if (image.covers(address)) return ℑ + return nullptr; + } + + void forget(std::uintptr_t address, std::size_t bytes) { + arena.forget(address, bytes); + image.forget(address, bytes); + } +}; + +PageStates& Pages() { + static PageStates pages; + return pages; +} + +std::uintptr_t PagesBase() { + return Pages().arena.base; +} + +std::size_t PagesSize() { + return Pages().arena.size; +} + +std::uintptr_t ImagePagesBase() { + return Pages().image.base; +} + +std::size_t ImagePagesSize() { + return Pages().image.size; +} + +void ForgetPages(std::uintptr_t address, std::size_t bytes) { + forgetCalls.fetch_add(1, std::memory_order_relaxed); + forgetBytes.fetch_add(bytes, std::memory_order_relaxed); + Pages().forget(address, bytes); +} + +void PageStates::initialize() { + std::call_once(once, [&] { + GuestArena::GuestArenaRange_nid_postfix(&arena.base, &arena.size); + const bool arenaCached = arena.allocate(); + bool imageCached = false; +#ifdef _WIN32 + // The image extent is the run of regions sharing the module's allocation base, as the registry + // walks it when it registers the main image. + static const bool noImageCache = std::getenv("APS5_NO_IMAGE_PAGE_CACHE") != nullptr; + if (const auto module = noImageCache ? nullptr : GetModuleHandleW(nullptr)) { + const auto start = reinterpret_cast(module); + auto cursor = start; + for (;;) { + MEMORY_BASIC_INFORMATION memory{}; + if (VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) != sizeof(memory) || memory.AllocationBase != module || memory.RegionSize == 0 || memory.RegionSize > std::numeric_limits::max() - cursor) break; + cursor += memory.RegionSize; + } + image.base = start; + image.size = cursor - start; + imageCached = image.allocate(); + } +#endif + if (arenaCached || imageCached) GuestAllocations::GuestAllocationsSetInvalidator_nid_postfix(&ForgetPages); + }); +} + +struct PageRun { + std::uintptr_t begin; + std::uintptr_t end; + bool readable; + bool writable; +}; + +// Calls `emit` with consecutive runs of uniform accessibility covering [address, address + bytes) in +// order, stopping early when it returns false. Returns false when the address space cannot be queried. +template +bool describePages(std::uintptr_t address, std::size_t bytes, Emit&& emit) { + auto& pages = Pages(); + pages.initialize(); + const auto end = address + bytes; + auto cursor = address; + while (cursor < end) { + if (const auto* span = pages.spanOf(cursor)) { + const auto value = span->load(cursor); + if ((value & 3u) != 0) { + const auto next = span->runEnd(cursor, value, end); + if (!emit(PageRun{cursor, next, true, (value & PageWritable) != 0})) return true; + cursor = next; + continue; + } + } +#ifdef _WIN32 + const TimedAccess timed(Profile().query, 0); + MEMORY_BASIC_INFORMATION memory{}; + const auto queryStart = std::chrono::steady_clock::now(); + // A mutation that changes these pages while they are being queried bumps the generation + // before it invalidates, so a generation change across the query and the store below means + // the stored pages may describe the old state and are forgotten again. + const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + if (VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) != sizeof(memory)) return false; + // Debug aid: APS5_TRACE_QUERY prints every 16th page query with its cost. + static const bool traceQuery = std::getenv("APS5_TRACE_QUERY") != nullptr; + if (traceQuery) { + static std::atomic queries{0}; + if (queries.fetch_add(1) % 16 == 0) { + const auto us = std::chrono::duration(std::chrono::steady_clock::now() - queryStart).count(); + std::fprintf(stderr, "[query] 0x%llx range 0x%zx: base 0x%llx size 0x%llx state 0x%lx protect 0x%lx type 0x%lx %.0f us gen %llu from +0x%llx\n", static_cast(cursor), bytes, reinterpret_cast(memory.BaseAddress), static_cast(memory.RegionSize), memory.State, memory.Protect, memory.Type, us, static_cast(GuestAllocations::GuestAllocationsGeneration_nid_postfix()), ModuleOffset(__builtin_return_address(0))); + } + } + const auto base = reinterpret_cast(memory.BaseAddress); + if (memory.RegionSize > std::numeric_limits::max() - base || base + memory.RegionSize <= cursor) return false; + const auto regionEnd = base + memory.RegionSize; + const auto protection = memory.Protect & 0xffu; + const bool committed = memory.State == MEM_COMMIT && (memory.Protect & (PAGE_GUARD | PAGE_NOACCESS)) == 0; + const bool readable = committed && readableProtection(protection); + const bool writable = readable && writableProtection(protection); + for (PageSpan* span : {&pages.arena, &pages.image}) { + if (!readable || span->size == 0 || regionEnd <= span->base || base >= span->base + span->size) continue; + const std::uint8_t value = PageReadable | (writable ? PageWritable : 0u); + const auto first = std::max(base, span->base); + const auto last = std::min(regionEnd, span->base + span->size); + for (auto at = first; at < last; at += PageBytes) span->store(at, value); + if (GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation) span->forget(first, static_cast(last - first)); + } + const auto next = std::min(end, regionEnd); + if (!emit(PageRun{cursor, next, readable, writable})) return true; + cursor = next; +#else + std::ifstream maps("/proc/self/maps"); + if (!maps.is_open()) return false; + std::string line; + bool found = false; + while (std::getline(maps, line)) { + std::istringstream fields(line); + std::uintptr_t first = 0; + std::uintptr_t last = 0; + char separator = 0; + std::string permissions; + if (!(fields >> std::hex >> first >> separator >> last >> permissions) || separator != '-' || first >= last || permissions.size() < 2) return false; + if (last <= cursor || first > cursor) continue; + const auto next = std::min(end, last); + if (!emit(PageRun{cursor, next, permissions[0] == 'r', permissions[0] == 'r' && permissions[1] == 'w'})) return true; + cursor = next; + found = true; + break; + } + if (!found) { + static_cast(emit(PageRun{cursor, end, false, false})); + return true; + } +#endif + } + return true; +} + +// Checks that [address, address + bytes) is mapped with read (and, if asked, write) access. Returns +// an empty string when it is, otherwise why it is not. +std::string verify(std::uintptr_t address, std::size_t bytes, bool writable) { + const TimedAccess timed(Profile().verify, bytes); + std::string reason; + const bool queried = describePages(address, bytes, [&](const PageRun& run) { + if (!run.readable) { + char text[128]; + std::snprintf(text, sizeof(text), "guest memory is not readable at 0x%llx (range 0x%llx+0x%zx)", static_cast(run.begin), static_cast(address), bytes); + reason = text; + } else if (writable && !run.writable) { + reason = "guest memory has no write permission"; + } + return reason.empty(); + }); + if (!queried && reason.empty()) reason = "cannot query guest memory"; + return reason; +} + } void CheckRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable) { @@ -29,60 +500,594 @@ void CheckRange(const void* pointer, std::size_t bytes, std::size_t alignment, b const auto address = reinterpret_cast(pointer); require(address != 0 && address % alignment == 0, "null or misaligned address"); require(bytes <= std::numeric_limits::max() - address, "address range overflow"); - MemoryAccessScope::Resolve(address, bytes, writable); - GuestMemoryTracking::GuestMemoryTrackingResolve_nid_postfix(address, bytes, writable); - auto cursor = address; - const auto end = address + bytes; + const auto reason = verify(address, bytes, writable); + if (!reason.empty()) { + // Debug aid: APS5_TRACE_UNREADABLE names the code that checked an inaccessible range. + static const bool trace = std::getenv("APS5_TRACE_UNREADABLE") != nullptr; + if (trace) std::fprintf(stderr, "[unreadable] %s from +0x%llx\n", reason.c_str(), ModuleOffset(__builtin_return_address(0))); + require(false, reason.c_str()); + } +} + +bool Accessible(const void* pointer, std::size_t bytes, bool writable) { + const auto address = reinterpret_cast(pointer); + return address != 0 && bytes <= std::numeric_limits::max() - address && verify(address, bytes, writable).empty(); +} + +Commitment DescribeCommitted(std::uint64_t address, std::size_t bytes, bool writable) { + require(bytes <= std::numeric_limits::max() - address, "address range overflow"); + const TimedAccess timed(Profile().verify, bytes); + Commitment result; + const bool queried = describePages(static_cast(address), bytes, [&](const PageRun& run) { + if (run.readable && (!writable || run.writable)) { + if (!result.ranges.empty() && result.ranges.back().second == run.begin) result.ranges.back().second = run.end; + else result.ranges.emplace_back(run.begin, run.end); + } + return true; + }); + require(queried, "cannot query guest memory"); + result.whole = bytes == 0 || (result.ranges.size() == 1 && result.ranges.front().first == address && result.ranges.front().second == address + bytes); + return result; +} + +std::vector> CommittedRanges(std::uint64_t address, std::size_t bytes, bool writable) { + return DescribeCommitted(address, bytes, writable).ranges; +} + +namespace { + +constexpr std::size_t WriteBlockBytes = 65536; + +struct WriteTracker { + std::mutex mutex; + bool initialized = false; + bool watched = false; + std::uintptr_t base = 0; + std::size_t size = 0; + // Generation of the last collected write per 64 KiB block of the arena. + std::vector blocks; + // Atomic only so a per-thread memo hit can read it without the mutex (every mutation and every + // stamp still happen under it): a hit must return the current value, including MarkWritten + // bumps, or an image refreshed after a stamp would keep failing UnchangedSince until the next + // walk of its range and re-upload at every use of the epoch. + std::atomic generation{1}; + std::vector pages; + // Collect memo: page-rounded ranges walked during the current epoch (see BumpCollectEpoch). A + // collect fully inside one skips GetWriteWatch, whose cost is proportional to the range's pages, + // and the same surfaces are collected several times per epoch (each storage mip binding, a sampled + // view plus its storage refresh, a write-back followed by a refresh, and every dispatch of the + // epoch binding the same targets). Entries carry the walking thread's epoch, so no entry matches + // another thread: the ring consulted first is per thread (ThreadCollectMemo, no mutex, no + // eviction by other threads' walks); this shared ring (8 workers with epochs of several packets + // hold a few hundred distinct ranges) serves APS5_SHARED_COLLECT_MEMO=1. + struct Memo { + std::uintptr_t begin = 0; + std::uintptr_t end = 0; + std::uint64_t epoch = 0; + }; + std::array memo{}; + std::size_t nextMemo = 0; + + void initialize() { + if (initialized) return; + initialized = true; + GuestArena::GuestArenaRange_nid_postfix(&base, &size); + watched = size != 0 && GuestArena::GuestArenaWriteWatched_nid_postfix(); + if (!watched) return; + blocks.assign(size / WriteBlockBytes + 1, 0); + pages.resize(1u << 16); + } +}; + +WriteTracker& Tracker() { + static WriteTracker tracker; + return tracker; +} + +// Takes the tracker mutex; under APS5_PROFILE_DRAW an acquisition that found it held is counted +// ('tracker waits N / M' in [guestmem]), which says whether walking outside the lock would pay. +std::unique_lock lockTracker(WriteTracker& tracker) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return std::unique_lock(tracker.mutex); + std::unique_lock lock(tracker.mutex, std::try_to_lock); + if (!lock.owns_lock()) { + trackerWaits.fetch_add(1, std::memory_order_relaxed); + lock.lock(); + } + trackerAcquisitions.fetch_add(1, std::memory_order_relaxed); + return lock; +} + +// Collect epochs are per thread and globally unique: every bump takes a fresh value from one counter, +// so a memo entry (stamped with the epoch of its walk) can only match the thread that made it, and +// only until that thread's next ordering point (the queue workers bump at theirs, see the header). A +// thread that never bumped is given a new epoch for every collect: its walks are never reused (the +// presenter, game threads inside the flush hook), as nothing orders the CPU's writes for it. +std::atomic nextCollectEpoch{1}; +thread_local std::uint64_t threadCollectEpoch = 0; + +std::uint64_t currentCollectEpoch() { + if (threadCollectEpoch != 0) return threadCollectEpoch; + return nextCollectEpoch.fetch_add(1, std::memory_order_relaxed) + 1; +} + +bool collectMemoEnabled() { + static const bool disabled = std::getenv("APS5_NO_COLLECT_MEMO") != nullptr; + return !disabled; +} + +// The per-thread memo ring (see WriteTracker::Memo): sized to a packet's working set (a dispatch +// collects each of its few dozen image surfaces in stage A and again in stage B). Entries of a +// thread that never bumps its epoch never match (each collect gets a fresh epoch), as intended. +// APS5_SHARED_COLLECT_MEMO=1 uses the tracker's shared ring under its mutex instead. +struct ThreadCollectMemo { + std::array entries{}; + std::size_t next = 0; +}; +thread_local ThreadCollectMemo threadCollectMemo; + +bool sharedCollectMemo() { + static const bool shared = std::getenv("APS5_SHARED_COLLECT_MEMO") != nullptr; + return shared; +} + +std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoized) { + auto& tracker = Tracker(); #ifdef _WIN32 - while (cursor < end) { - MEMORY_BASIC_INFORMATION memory{}; - require(VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) == sizeof(memory), "cannot query guest memory"); - require(memory.State == MEM_COMMIT && (memory.Protect & (PAGE_GUARD | PAGE_NOACCESS)) == 0, "guest memory is not readable"); - const auto protection = memory.Protect & 0xffu; - require(protection == PAGE_READONLY || protection == PAGE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_READ || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY, "guest memory has no read permission"); - const auto base = reinterpret_cast(memory.BaseAddress); - require(memory.RegionSize <= std::numeric_limits::max() - base && base + memory.RegionSize > cursor, "invalid guest memory mapping"); - require(!writable || protection == PAGE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY, "guest memory has no write permission"); - cursor = std::min(end, base + memory.RegionSize); + // Whole pages, so a page shared with the next range is collected with either. + constexpr std::uint64_t page = 4096; + const auto first = address & ~(page - 1); + const auto stop = (address + bytes + page - 1) & ~(page - 1); + // One epoch for the lookup and the entry made after the walk: an unbumped thread's fresh epoch + // must not differ between them. + const auto epoch = currentCollectEpoch(); + const bool useMemo = memoized && collectMemoEnabled() && bytes != 0; + if (useMemo && !sharedCollectMemo()) { + // An entry exists only for a completed walk of an in-arena range, so the tracker is + // initialized and watched; nothing below the lock needs asking. + for (const auto& entry : threadCollectMemo.entries) { + if (entry.epoch == epoch && entry.begin <= first && stop <= entry.end) { + // The current generation, not the memoized one: blocks stamped since (MarkWritten, an + // overlapping collect) were written before this caller reads, and an older value would + // make every later UnchangedSince fail until the epoch ends. + collectMemoHits.fetch_add(1, std::memory_order_relaxed); + return tracker.generation.load(std::memory_order_relaxed); + } + } } +#endif + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return 0; +#ifdef _WIN32 + auto cursor = first; + if (useMemo && sharedCollectMemo()) { + for (const auto& entry : tracker.memo) { + if (entry.epoch == epoch && entry.begin <= cursor && stop <= entry.end) { + collectMemoHits.fetch_add(1, std::memory_order_relaxed); + return tracker.generation; + } + } + } + const TimedAccess timed(Profile().collect, bytes); + ++tracker.generation; + // One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing + // after the walk passed a page is reported by the next walk, and a clean page is not touched by the + // reset, so a probe pass first (which doubled the walk of every dirty range) buys nothing. + // APS5_NO_SINGLE_PASS_COLLECT=1 restores the probe pass followed by a resetting pass when dirty. + static const bool singlePass = std::getenv("APS5_NO_SINGLE_PASS_COLLECT") == nullptr; + bool dirty = false; + while (cursor < stop) { + ULONG_PTR count = tracker.pages.size(); + DWORD granularity = 0; + if (GetWriteWatch(singlePass ? WRITE_WATCH_FLAG_RESET : 0, reinterpret_cast(cursor), static_cast(stop - cursor), tracker.pages.data(), &count, &granularity) != 0) { + // Uncommitted pages inside the range make the call fail; such ranges are compared instead. + return 0; + } + if (count != 0) dirty = true; + for (ULONG_PTR i = 0; i < count; ++i) tracker.blocks[(reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes] = tracker.generation; + if (count < tracker.pages.size()) break; + cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); + } + if (dirty) collectDirty.fetch_add(1, std::memory_order_relaxed); + if (dirty && !singlePass) { + // The resetting pass reports pages again, so a write racing the first pass is stamped too. + cursor = address & ~(page - 1); + while (cursor < stop) { + ULONG_PTR count = tracker.pages.size(); + DWORD granularity = 0; + if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, reinterpret_cast(cursor), static_cast(stop - cursor), tracker.pages.data(), &count, &granularity) != 0) return 0; + for (ULONG_PTR i = 0; i < count; ++i) tracker.blocks[(reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes] = tracker.generation; + if (count < tracker.pages.size()) break; + cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); + } + } + // Only a completed walk is remembered; a failed one (uncommitted pages) returned 0 above. + if (collectMemoEnabled()) { + if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch}; + else threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch}; + } + return tracker.generation; #else - std::ifstream maps("/proc/self/maps"); - require(maps.is_open(), "cannot query guest memory maps"); - std::string line; - while (cursor < end && std::getline(maps, line)) { - std::istringstream fields(line); - std::uintptr_t first = 0; - std::uintptr_t last = 0; - char separator = 0; - std::string permissions; - require(static_cast(fields >> std::hex >> first >> separator >> last >> permissions) && separator == '-' && first < last && !permissions.empty(), "invalid guest memory map entry"); - if (last <= cursor) continue; - require(first <= cursor && permissions[0] == 'r', "guest memory is not readable"); - require(!writable || (permissions.size() > 1 && permissions[1] == 'w'), "guest memory has no write permission"); - cursor = std::min(end, last); - } - require(cursor == end, "guest address range is not mapped"); + static_cast(memoized); + static_cast(threadCollectMemo); + return 0; #endif } +} + +std::uint64_t CollectWrites(std::uint64_t address, std::size_t bytes) { + return collectWrites(address, bytes, true); +} + +std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes) { + return collectWrites(address, bytes, false); +} + +void BumpCollectEpoch() { + threadCollectEpoch = nextCollectEpoch.fetch_add(1, std::memory_order_relaxed) + 1; + collectEpochBumps.fetch_add(1, std::memory_order_relaxed); +} + +std::uint64_t CollectEpochBumps() { + return collectEpochBumps.load(std::memory_order_relaxed); +} + +bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched || generation == 0 || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return false; + const auto first = (address - tracker.base) / WriteBlockBytes; + const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + for (auto block = first; block <= last; ++block) { + if (tracker.blocks[block] > generation) return false; + } + return true; +} + +void MarkWritten(std::uint64_t address, std::size_t bytes) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return; + const auto first = (address - tracker.base) / WriteBlockBytes; + const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + ++tracker.generation; + for (auto block = first; block <= last; ++block) tracker.blocks[block] = tracker.generation; +} + +namespace { + +// Per-thread GpuMutex wait accounting (APS5_PROFILE_DRAW): a queue worker tags its thread, other +// threads (the presenter) report untagged. +constexpr std::size_t GpuLockSiteCount = static_cast(GpuLockSite::Count); +constexpr const char* GpuLockSiteNames[GpuLockSiteCount] = {"other", "dispatch", "indirect", "draw", "hook", "wait", "label", "flush", "present", "fill", "try"}; + +struct GpuLockStats { + std::uint32_t tag = 0xffffffffu; + std::uint64_t acquisitions = 0; + std::uint64_t waits = 0; + double waitedMs = 0; + // The same split by the site tagged before the acquisition (TagGpuLockSite); the tag of the + // next lock() is kept here and consumed by it. + std::array siteAcquisitions{}; + std::array siteWaits{}; + std::array siteWaitedMs{}; + GpuLockSite nextSite = GpuLockSite::Other; + // Holds: the outermost (depth 0 -> 1) acquisition of this thread until the matching unlock, + // charged to that acquisition's site (a nested hook lock inside a dispatch's hold extends the + // dispatch's hold). The waits above say who waits; these say whom they wait for. + std::array siteHolds{}; + std::array siteHeldMs{}; + std::array siteMaxHeldMs{}; + std::size_t holdSite = 0; + std::chrono::steady_clock::time_point holdStart; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +GpuLockStats& LockStats() { + thread_local GpuLockStats stats; + return stats; +} + +bool GpuLockProfiled() { + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr && std::getenv("APS5_NO_LOCK_PROFILE") == nullptr; + return profiled; +} + +// Hold timing costs two clock reads per outermost acquisition (~20000 per 10 s on queue 0, well +// under a millisecond); APS5_NO_HOLD_PROFILE=1 keeps only the wait timing. +bool GpuHoldProfiled() { + static const bool profiled = GpuLockProfiled() && std::getenv("APS5_NO_HOLD_PROFILE") == nullptr; + return profiled; +} + +void BeginHold(std::size_t site) { + auto& stats = LockStats(); + stats.holdSite = site; + stats.holdStart = std::chrono::steady_clock::now(); +} + +void EndHold() { + auto& stats = LockStats(); + const auto heldMs = std::chrono::duration(std::chrono::steady_clock::now() - stats.holdStart).count(); + ++stats.siteHolds[stats.holdSite]; + stats.siteHeldMs[stats.holdSite] += heldMs; + stats.siteMaxHeldMs[stats.holdSite] = std::max(stats.siteMaxHeldMs[stats.holdSite], heldMs); +} + +} + +// The owner token is a thread-local's address: unique per live thread and free to read. +namespace { +const void* ThreadToken() { + thread_local char token; + return &token; +} +} + +void GpuMutexType::acquired() { + owner.store(ThreadToken(), std::memory_order_relaxed); + ++depth; +} + +bool GpuMutexType::try_lock() { + if (!GpuLockProfiled()) { + if (!mutex.try_lock()) return false; + acquired(); + return true; + } + // The tag is consumed whether or not the try succeeds (a failed try must not name the next + // acquisition); an untagged try's hold is charged to 'try' (a boundary reap, an idle worker). + auto& stats = LockStats(); + const auto tagged = std::exchange(stats.nextSite, GpuLockSite::Other); + if (!mutex.try_lock()) return false; + acquired(); + if (depth == 1 && GpuHoldProfiled()) BeginHold(static_cast(tagged == GpuLockSite::Other ? GpuLockSite::Try : tagged)); + return true; +} + +namespace { +std::atomic gpuUnlockHook{nullptr}; +} + +void SetGpuUnlockHook(void (*hook)()) { + gpuUnlockHook.store(hook, std::memory_order_release); +} + +void GpuMutexType::unlock() { + const bool outermost = --depth == 0; + if (outermost) { + owner.store(nullptr, std::memory_order_relaxed); + // Still the holder here: the hold ends when the mutex is given up, not before. + if (GpuHoldProfiled()) EndHold(); + } + mutex.unlock(); + // After the release, so what the hook does (destroying a batch's kept objects) is neither part + // of the hold nor waited for by the next holder. + if (outermost) { + if (auto* hook = gpuUnlockHook.load(std::memory_order_acquire); hook != nullptr) hook(); + } +} + +bool GpuMutexType::HeldByThisThread() const { + return owner.load(std::memory_order_relaxed) == ThreadToken(); +} + +std::uint32_t GpuMutexType::DepthOnThisThread() const { + // `depth` is written by the holder only, so it is read only when that is this thread. + return HeldByThisThread() ? depth : 0; +} + +void AssertGpuLockHeld(const char* where) { + static const bool enabled = std::getenv("APS5_ASSERT_GPU_LOCK") != nullptr; + if (!enabled || GpuMutex().HeldByThisThread()) return; + std::fprintf(stderr, "[lock] ASSERT: %s reached without GuestMemory::GpuMutex on thread tagged 0x%x\n", where, GpuLockThreadTag()); + std::abort(); +} + +void GpuMutexType::lock() { + if (!GpuLockProfiled()) { + mutex.lock(); + acquired(); + return; + } + auto& stats = LockStats(); + ++stats.acquisitions; + const auto site = static_cast(std::exchange(stats.nextSite, GpuLockSite::Other)); + ++stats.siteAcquisitions[site]; + // An uncontended (or recursive) acquisition costs no clock reads for the wait; only a wait is + // timed. The hold timer (outermost acquisitions only) starts once the mutex is held. + if (!mutex.try_lock()) { + const auto start = std::chrono::steady_clock::now(); + mutex.lock(); + acquired(); + const auto now = std::chrono::steady_clock::now(); + if (depth == 1 && GpuHoldProfiled()) { + stats.holdSite = site; + stats.holdStart = now; + } + const auto waitedMs = std::chrono::duration(now - start).count(); + stats.waitedMs += waitedMs; + ++stats.waits; + stats.siteWaitedMs[site] += waitedMs; + ++stats.siteWaits[site]; + if (now - stats.lastReport < std::chrono::seconds(10)) return; + // The line is printed from inside a contended acquisition, so a thread that rarely waits + // reports a longer span than 10 s: the real interval is printed. + const auto interval = std::chrono::duration(now - stats.lastReport).count(); + stats.lastReport = now; + std::string sites; + for (std::size_t i = 0; i < GpuLockSiteCount; ++i) { + if (stats.siteAcquisitions[i] == 0) continue; + char text[96]; + std::snprintf(text, sizeof(text), " %s %llu/%llu %.0f ms", GpuLockSiteNames[i], static_cast(stats.siteAcquisitions[i]), static_cast(stats.siteWaits[i]), stats.siteWaitedMs[i]); + sites += text; + } + // Holds by site, longest total first: the sites whose holds the other threads wait behind. + std::string holds; + if (GpuHoldProfiled()) { + std::array order{}; + for (std::size_t i = 0; i < GpuLockSiteCount; ++i) order[i] = i; + std::sort(order.begin(), order.end(), [&](std::size_t a, std::size_t b) { return stats.siteHeldMs[a] > stats.siteHeldMs[b]; }); + double heldMs = 0; + for (std::size_t i = 0; i < GpuLockSiteCount; ++i) heldMs += stats.siteHeldMs[i]; + char text[112]; + std::snprintf(text, sizeof(text), "; held %.0f ms in total, by site (holds, held, max):", heldMs); + holds += text; + for (const auto i : order) { + if (stats.siteHolds[i] == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu %.0f ms (max %.1f)", GpuLockSiteNames[i], static_cast(stats.siteHolds[i]), stats.siteHeldMs[i], stats.siteMaxHeldMs[i]); + holds += text; + } + } + if (stats.tag == 0xffffffffu) std::fprintf(stderr, "[lock] untagged thread waited %.0f ms for the GPU mutex in %llu of %llu acquisitions (%.0f s); by site (acquisitions/waits, waited):%s%s\n", stats.waitedMs, static_cast(stats.waits), static_cast(stats.acquisitions), interval, sites.c_str(), holds.c_str()); + else std::fprintf(stderr, "[lock] queue 0x%x waited %.0f ms for the GPU mutex in %llu of %llu acquisitions (%.0f s); by site (acquisitions/waits, waited):%s%s\n", stats.tag, stats.waitedMs, static_cast(stats.waits), static_cast(stats.acquisitions), interval, sites.c_str(), holds.c_str()); + stats.waitedMs = 0; + stats.waits = 0; + stats.acquisitions = 0; + stats.siteAcquisitions.fill(0); + stats.siteWaits.fill(0); + stats.siteWaitedMs.fill(0); + stats.siteHolds.fill(0); + stats.siteHeldMs.fill(0); + stats.siteMaxHeldMs.fill(0); + return; + } + acquired(); + if (depth == 1 && GpuHoldProfiled()) BeginHold(site); +} + +void TagGpuLockSite(GpuLockSite site) { + // Off the profiled path the tag is never read, so the thread-local is not touched either. + if (!GpuLockProfiled()) return; + LockStats().nextSite = site; +} + +std::uint32_t GpuLockThreadTag() { + return LockStats().tag; +} + +GpuMutexType& GpuMutex() { + static GpuMutexType mutex; + return mutex; +} + +void TagGpuLockThread(std::uint32_t queue) { + LockStats().tag = queue; +} + +unsigned long long CodeOffset(const void* address) { + return ModuleOffset(address); +} + +namespace { +std::atomic flushHook{nullptr}; +} + +void SetFlushHook(void (*hook)(std::uint64_t, std::size_t)) { + flushHook.store(hook, std::memory_order_release); +} + +void FlushGpuWrites(std::uint64_t address, std::size_t bytes) { + if (const auto hook = flushHook.load(std::memory_order_acquire)) hook(address, bytes); +} + void Read(std::uint64_t address, std::span destination, std::size_t alignment) { - PerformanceTimer timing("GuestMemory.Read"); if (destination.empty()) return; + FlushGpuWrites(address, destination.size()); + const TimedAccess timed(Profile().read, destination.size()); + CountReadCaller(__builtin_return_address(0)); const auto* source = reinterpret_cast(address); - CheckRange(source, destination.size(), alignment); - timing.Mark("range_check"); + try { + CheckRange(source, destination.size(), alignment); + } catch (const std::runtime_error&) { + static const bool trace = std::getenv("APS5_TRACE_UNREADABLE") != nullptr; + if (trace) std::fprintf(stderr, "[unreadable] read from +0x%llx\n", ModuleOffset(__builtin_return_address(0))); + throw; + } std::memcpy(destination.data(), source, destination.size()); - timing.Mark("copy", destination.size()); +} + +void ReadCommitted(std::uint64_t address, std::span destination) { + if (destination.empty()) return; + FlushGpuWrites(address, destination.size()); + const TimedAccess timed(Profile().read, destination.size()); + CountReadCaller(__builtin_return_address(0)); + if (Accessible(reinterpret_cast(address), destination.size())) { + std::memcpy(destination.data(), reinterpret_cast(address), destination.size()); + return; + } + std::memset(destination.data(), 0, destination.size()); + for (const auto& [begin, end] : CommittedRanges(address, destination.size())) std::memcpy(destination.data() + (begin - address), reinterpret_cast(begin), static_cast(end - begin)); +} + +bool EqualsCommitted(std::uint64_t address, std::span bytes) { + if (bytes.empty()) return true; + FlushGpuWrites(address, bytes.size()); + const TimedAccess timed(Profile().compare, bytes.size()); + if (Accessible(reinterpret_cast(address), bytes.size())) return std::memcmp(reinterpret_cast(address), bytes.data(), bytes.size()) == 0; + for (const auto& [begin, end] : CommittedRanges(address, bytes.size())) { + if (std::memcmp(reinterpret_cast(begin), bytes.data() + (begin - address), static_cast(end - begin)) != 0) return false; + } + return true; +} + +void WriteChangedCommitted(std::uint64_t address, std::span current, std::span original) { + require(current.size() == original.size(), "write-back snapshot sizes differ"); + if (Accessible(reinterpret_cast(address), current.size(), true)) { + WriteChanged(address, current, original); + return; + } + for (const auto& [begin, end] : CommittedRanges(address, current.size(), true)) { + const auto offset = static_cast(begin - address); + const auto length = static_cast(end - begin); + WriteChanged(begin, current.subspan(offset, length), original.subspan(offset, length)); + } +} + +void WriteChanged(std::uint64_t address, std::span current, std::span original) { + require(current.size() == original.size(), "write-back snapshot sizes differ"); + if (current.empty()) return; + // A store: the hook sync it makes orders this write after a pending GPU write, whatever a read + // site above it was doing (the [hooksync] line counts stores apart). + const ReadSiteScope site(ReadSite::Store); + FlushGpuWrites(address, current.size()); + const TimedAccess timed(Profile().changedWrite, current.size()); + auto* destination = reinterpret_cast(address); + CheckRange(destination, current.size(), 1, true); + constexpr std::size_t block = 256; + const auto size = current.size(); + const auto differs = [&](std::size_t at) { + const auto length = std::min(block, size - at); + return std::memcmp(current.data() + at, original.data() + at, length) != 0; + }; + std::size_t firstChanged = size; + std::size_t lastChanged = 0; + for (std::size_t at = 0; at < size; at += block) { + if (!differs(at)) continue; + std::memcpy(destination + at, current.data() + at, std::min(block, size - at)); + firstChanged = std::min(firstChanged, at); + lastChanged = std::min(at + block, size); + } + // Stamped like a GPU write: a collect memoized for this packet would not see the page fault. + if (firstChanged < lastChanged) MarkWritten(address + firstChanged, lastChanged - firstChanged); } void Write(std::uint64_t address, std::span source, std::size_t alignment) { - PerformanceTimer timing("GuestMemory.Write"); if (source.empty()) return; + // A store, as in WriteChanged. + const ReadSiteScope site(ReadSite::Store); + FlushGpuWrites(address, source.size()); + const TimedAccess timed(Profile().write, source.size()); + CountWriteCaller(__builtin_return_address(0)); auto* destination = reinterpret_cast(address); CheckRange(destination, source.size(), alignment, true); - timing.Mark("range_check"); std::memcpy(destination, source.data(), source.size()); - timing.Mark("copy", source.size()); + // Stamped like a GPU write: a collect memoized for this packet would not see the page fault. + MarkWritten(address, source.size()); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index af771512db..6e23d0166c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -2,14 +2,23 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/General.hpp" #include +#include +#include #include #include #include #include +#include namespace AgcDriver::Pm4 { namespace { +std::string ToHex(std::uint32_t value) { + char buffer[16]; + std::snprintf(buffer, sizeof(buffer), "%x", value); + return buffer; +} + void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(reason); } @@ -25,6 +34,13 @@ std::uint32_t registerOffset(std::uint32_t value) { return offset; } +// Debug aid: APS5_TRACE_CONTEXT_STATE logs context-state operations and the context registers +// written while a context is pushed. Read once: getenv scans the environment under a lock. +bool TraceContextState() { + static const bool value = std::getenv("APS5_TRACE_CONTEXT_STATE") != nullptr; + return value; +} + Registers& registersFor(QueueState& queue, std::uint32_t opcode) { if (opcode == 0x69 || opcode == 0x9f) return queue.context; if (opcode == 0x76 || opcode == 0x63) return queue.shader; @@ -32,7 +48,10 @@ Registers& registersFor(QueueState& queue, std::uint32_t opcode) { } void writeRegister(QueueState& queue, std::uint32_t opcode, std::uint32_t offset, std::uint32_t value) { + if ((opcode == 0x69 || opcode == 0x9f) && (offset == 0x8e || offset == 0x8f || offset == 0x318 || offset == 0x31b || offset == 0x31c || offset == 0x31d || offset == 0x390 || offset == 0x3b0 || offset == 0x3b8)) + APS5_LOG_OUT_DEBUG("CONTEXT WRITE opcode=0x%x offset=0x%x value=0x%x", opcode, offset, value); registersFor(queue, opcode).insert_or_assign(offset, value); + if (TraceContextState() && (opcode == 0x69 || opcode == 0x9f) && ((offset >= 0x318 && offset < 0x318 + 8 * 0xf && (offset - 0x318) % 0xf == 0) || offset == 0x8e)) std::fprintf(stderr, "[context] write %x = %08x (0x%x)\n", offset, value, opcode); if ((opcode == 0x64 || opcode == 0x79 || opcode == 0x7a) && offset == 0x243) queue.indexType = value & 3u; } @@ -48,9 +67,9 @@ std::uint32_t dmaDestination(std::span packet) { return ((packet[1] >> 20u) & 3u) | ((packet[6] >> 25u) & 4u) | ((packet[6] >> 26u) & 8u); } -void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t bytes, bool immediate) { - if (bytes == 0) return; - GuestMemory::CheckRange(reinterpret_cast(destination), bytes, 1, true); +// The bytes a COPY_DATA or DMA_DATA stores: an immediate is a repeating 32-bit pattern, a memory +// source is read through the checked path (it waits for recorded GPU work that writes it). +std::vector copySource(std::uint64_t source, std::size_t bytes, bool immediate) { std::vector data(bytes); if (immediate) { const auto value = static_cast(source); @@ -58,12 +77,19 @@ void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t byt } else { GuestMemory::Read(source, data); } - GuestMemory::Write(destination, data); + return data; +} + +void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t bytes, bool immediate) { + if (bytes == 0) return; + GuestMemory::CheckRange(reinterpret_cast(destination), bytes, 1, true); + GuestMemory::Write(destination, copySource(source, bytes, immediate)); } } std::string Name(std::uint32_t header) { + if (FillerPacket(header)) return "PM4_FILLER"; const auto opcode = (header >> 8u) & 0xffu; if (opcode == 0x10 && (header & 0xfcu) != 0) { switch ((header >> 2u) & 0x3fu) { @@ -107,11 +133,13 @@ std::string_view UnsupportedReason(std::uint32_t header) { case 0x20: return "GPU query predication is not implemented"; case 0x22: return "conditional command execution and conditional flip reservation are not implemented"; case 0x33: case 0x3f: return "nested command buffers, branching and nested flip reservation are not implemented"; - case 0x39: case 0x3c: case 0x59: case 0x93: + case 0x3c: case 0x93: return {}; + case 0x39: case 0x59: return "cooperative command-queue waits are not implemented"; case 0x84: case 0x85: case 0x86: case 0x88: return "separate CE/DE execution and counter synchronization are not implemented"; - case 0x43: case 0x47: case 0x48: case 0x49: + case 0x49: return {}; + case 0x43: case 0x47: case 0x48: return "guest cache actions, GPU events and interrupt delivery are not implemented"; case 0x8e: return "GPU LOD statistics are not implemented; synthetic results are forbidden"; case 0x28: case 0x41: case 0x68: case 0x78: @@ -121,8 +149,13 @@ std::string_view UnsupportedReason(std::uint32_t header) { } void Validate(std::span packet, std::uint32_t queue) { - require(packet.size() >= 2, "truncated PM4 header or payload"); + require(!packet.empty(), "truncated PM4 header"); const auto header = packet[0]; + if (FillerPacket(header)) { + require(packet.size() == 1, "invalid PM4 filler size"); + return; + } + require(packet.size() >= 2, "truncated PM4 header or payload"); require((header & 0xc0000000u) == 0xc0000000u, "unsupported PM4 packet type"); require(packet.size() == ((header >> 16u) & 0x3fffu) + 2u, "invalid PM4 packet size"); const auto opcode = (header >> 8u) & 0xffu; @@ -153,7 +186,13 @@ void Validate(std::span packet, std::uint32_t queue) { } return; } - require((header & 0xffu) == 0 || (opcode == 0x11 && (header & 0xffu) == 2), "PM4 header flags are not implemented"); + // Header bit 1 selects the compute shader type and bit 2 (register writes) resets the filter CAM; + // neither changes what the packet writes. Predication (bit 0) is not implemented. + const auto flags = header & 0xffu; + const bool registerWrite = opcode == 0x69 || opcode == 0x76 || opcode == 0x79 || opcode == 0x7a; + auto allowedFlags = opcode == 0x11 ? 2u : registerWrite ? 6u : opcode == 0x3c || opcode == 0x93 ? 2u : 0u; + if (IsTagMarker(packet)) allowedFlags |= 1u; + if ((flags & ~allowedFlags) != 0) throw std::runtime_error("PM4 header flags 0x" + ToHex(flags) + " are not implemented"); switch (opcode) { case 0x11: size(4); @@ -175,10 +214,10 @@ void Validate(std::span packet, std::uint32_t queue) { require(packet[3] <= packet[1], "index count exceeds maximum index size"); require((packet[4] & ~0x20u) == 0, "unsupported indexed draw flags"); break; - case 0x15: size(5); require((packet[4] & ~0x8000u) == 0x41u, "dispatch modifiers are not implemented"); break; + case 0x15: size(5); if ((packet[4] & ~0x8020u) != 0x41u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break; case 0x16: require(packet.size() == 3 || packet.size() == 4, "invalid indirect dispatch size"); - require((packet.back() & ~0x8000u) == 0x41u, "indirect dispatch modifiers are not implemented"); + require((packet.back() & ~0x8020u) == 0x41u, "indirect dispatch modifiers are not implemented"); break; case 0x42: size(2); require(packet[1] == 0, "unsupported PFP_SYNC_ME payload"); break; case 0x46: { @@ -222,6 +261,7 @@ void Validate(std::span packet, std::uint32_t queue) { require((packet[1] & 3u) == 0 && packet[3] == 0x80000000u && packet[4] <= 0x3fffu, "unsupported indirect-register address or control fields"); break; case 0x69: case 0x76: case 0x79: case 0x7a: { + if (IsTagMarker(packet)) break; if (opcode != 0x76) graphics(); require(packet.size() >= 3, "register packet has no values"); if (opcode == 0x7a) require((packet[1] & 0xf0000000u) == 0 || (packet.size() == 3 && packet[1] == 0x20000243u), "indexed register bank selection is not implemented"); @@ -229,6 +269,20 @@ void Validate(std::span packet, std::uint32_t queue) { require(packet.size() - 2 <= 0x10000u - offset, "register range overflow"); break; } + case 0x3c: case 0x93: { + size(opcode == 0x3c ? 7 : 9); + require((packet[1] & 0x10u) != 0, "register-space WAIT_REG_MEM is not implemented"); + require((packet[1] & 7u) <= 6u, "invalid WAIT_REG_MEM compare function"); + require((packet[2] & (opcode == 0x3c ? 3u : 7u)) == 0, "misaligned WAIT_REG_MEM address"); + break; + } + case 0x49: { + size(8); + const auto dataSelect = packet[2] >> 29u; + require(dataSelect <= 3, "GDS RELEASE_MEM data is not implemented"); + require(dataSelect == 0 || address(packet[3], packet[4]) == 0 || (packet[3] & (dataSelect == 1 ? 3u : 7u)) == 0, "misaligned RELEASE_MEM destination"); + break; + } case 0x81: graphics(); require(packet[1] <= 0xbffcu && (packet[1] & 3u) == 0 && packet.size() - 2 <= 0x3000u - packet[1] / 4u, "constant RAM write range overflow or misalignment"); @@ -266,6 +320,111 @@ void Validate(std::span packet, std::uint32_t queue) { } } +bool IsTagMarker(std::span packet) { + // libSceAgc brackets waits and draws with predicated SET_UCONFIG_REG writes of a tag value to + // 0x342; they carry no register state. + return packet.size() >= 3 && ((packet[0] >> 8u) & 0xffu) == 0x79u && (packet[0] & 1u) != 0 && packet[1] == 0x342u; +} + +namespace { + +bool waitCompares(std::span packet, bool wide, std::uint64_t value) { + const std::uint64_t reference = wide ? address(packet[4], packet[5]) : packet[4]; + const std::uint64_t mask = wide ? address(packet[6], packet[7]) : packet[5]; + const auto masked = value & mask; + switch (packet[1] & 7u) { + case 0: return true; + case 1: return masked < reference; + case 2: return masked <= reference; + case 3: return masked == reference; + case 4: return masked != reference; + case 5: return masked >= reference; + case 6: return masked > reference; + default: return false; + } +} + +} + +bool WaitSatisfied(std::span packet) { + const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; + const auto source = address(packet[2], packet[3]); + std::uint64_t value = 0; + // Named for the [hooksync] attribution (the read goes through the flush hook). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Wait); + GuestMemory::Read(source, std::as_writable_bytes(std::span(&value, 1)).first(wide ? 8 : 4), wide ? 8 : 4); + return waitCompares(packet, wide, value); +} + +bool WaitSatisfiedUnchecked(std::span packet) { + const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; + const auto source = address(packet[2], packet[3]); + const std::uint64_t value = wide ? *reinterpret_cast(source) : *reinterpret_cast(source); + return waitCompares(packet, wide, value); +} + +bool WaitComparesValue(std::span packet, std::uint64_t value) { + const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; + return waitCompares(packet, wide, value); +} + +std::optional DecodeLabelWrite(std::span packet) { + const auto opcode = (packet[0] >> 8u) & 0xffu; + if (opcode == 0x49 && packet.size() >= 7) { + const auto dataSelect = packet[2] >> 29u; + const auto destination = address(packet[3], packet[4]); + if (dataSelect == 0 || dataSelect > 3) return std::nullopt; + if (destination == 0) return std::nullopt; + std::uint64_t value = address(packet[5], packet[6]); + if (dataSelect == 3) value = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() / 10); + LabelWrite label{destination, {}}; + label.inlineSize = dataSelect == 1 ? 4 : 8; + std::memcpy(label.inlineBytes.data(), &value, label.inlineSize); + return label; + } + if (opcode == 0x37 && packet.size() > 4) { + const auto destination = address(packet[2], packet[3]); + if (destination == 0) return std::nullopt; + // WR_ONE_ADDR stores every value at the same address: the last one remains. + const auto words = (packet[1] & 0x10000u) != 0 ? packet.last(1) : packet.subspan(4); + return LabelWrite{destination, std::as_bytes(words)}; + } + return std::nullopt; +} + +std::optional ResolveStore(std::span packet, const QueueState& queue, std::size_t limit) { + // The same decode as Execute's cases below (a packet Validate accepted): a store that is not + // resolved here still runs there. The size and alignment tests precede any memory read: a store + // the GPU path would reject anyway must not read (and sync for) its source here and again there. + const auto fits = [limit](std::uint64_t destination, std::size_t bytes) { return bytes <= limit && destination % 4 == 0 && bytes % 4 == 0; }; + switch ((packet[0] >> 8u) & 0xffu) { + case 0x40: { + if (packet.size() < 6) return std::nullopt; + const auto source = ((packet[1] & 0xfu) << 1u) | ((packet[1] >> 30u) & 1u); + const std::size_t bytes = (packet[1] & 0x10000u) != 0 ? 8 : 4; + const auto destination = address(packet[4], packet[5]); + if (!fits(destination, bytes)) return std::nullopt; + return StoreWrite{destination, {}, copySource(address(packet[2], packet[3]), bytes, source >= 10)}; + } + case 0x50: { + if (packet.size() < 7) return std::nullopt; + const std::size_t bytes = packet[6] & 0x3ffffffu; + const auto destination = address(packet[4], packet[5]); + if (!fits(destination, bytes)) return std::nullopt; + return StoreWrite{destination, {}, copySource(address(packet[2], packet[3]), bytes, dmaSource(packet) == 2)}; + } + case 0x83: { + if (packet.size() < 5 || packet[1] / 4 > queue.constantRam.size() || packet[2] > queue.constantRam.size() - packet[1] / 4) return std::nullopt; + const auto words = std::span(queue.constantRam).subspan(packet[1] / 4, packet[2]); + if (words.size_bytes() > limit) return std::nullopt; + // A view of the queue's constant RAM: valid until the next CONST_RAM write on this queue, + // which is a later packet of the same worker. + return StoreWrite{address(packet[3], packet[4]), std::as_bytes(words), {}}; + } + default: return std::nullopt; + } +} + bool UsesGpuCacheBarrier(std::span packet) { require(!packet.empty() && ((packet[0] >> 8u) & 0xffu) == 0x58, "cache barrier requires ACQUIRE_MEM"); Validate(packet, 0); @@ -274,24 +433,30 @@ bool UsesGpuCacheBarrier(std::span packet) { bool AccessesMemory(std::uint32_t header) { switch ((header >> 8u) & 0xffu) { - case 0x16: case 0x2d: case 0x35: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true; + case 0x3c: case 0x93: case 0x49: case 0x16: case 0x2d: case 0x35: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true; default: return false; } } -std::array ResolveDispatch(std::span packet, const QueueState& queue) { - std::uint64_t source = 0; - if (packet.size() == 4) source = address(packet[1], packet[2]); - else { - require(queue.dispatchIndirectBase != 0, "indirect dispatch base has not been set"); - require(packet[1] <= std::numeric_limits::max() - queue.dispatchIndirectBase, "indirect dispatch address overflow"); - source = queue.dispatchIndirectBase + packet[1]; - } - std::array result{0xc0031500u, 0, 0, 0, packet.back()}; - GuestMemory::Read(source, std::as_writable_bytes(std::span(result).subspan(1, 3)), 4); +std::uint64_t DispatchArgumentAddress(std::span packet, const QueueState& queue) { + if (packet.size() == 4) return address(packet[1], packet[2]); + require(queue.dispatchIndirectBase != 0, "indirect dispatch base has not been set"); + require(packet[1] <= std::numeric_limits::max() - queue.dispatchIndirectBase, "indirect dispatch address overflow"); + return queue.dispatchIndirectBase + packet[1]; +} + +std::array ReadDispatchArguments(std::uint64_t arguments, std::uint32_t initiator) { + std::array result{0xc0031500u, 0, 0, 0, initiator}; + // Named for the [hooksync] attribution (the read goes through the flush hook). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::IndirectArguments); + GuestMemory::Read(arguments, std::as_writable_bytes(std::span(result).subspan(1, 3)), 4); return result; } +std::array ResolveDispatch(std::span packet, const QueueState& queue) { + return ReadDispatchArguments(DispatchArgumentAddress(packet, queue), packet.back()); +} + DrawParameters ResolveDraw(std::span packet, const QueueState& queue) { Validate(packet, 0); if (((packet[0] >> 8u) & 0xffu) == 0x2d) { @@ -311,6 +476,7 @@ DrawParameters ResolveDraw(std::span packet, const QueueSta const auto bytes = static_cast(packet[3]) * indexSize; require(bytes <= std::numeric_limits::max(), "index range size overflow"); GuestMemory::CheckRange(reinterpret_cast(address), static_cast(bytes), indexSize); + APS5_LOG_OUT_DEBUG("ResolveDraw context targetMask=0x%x shaderMask=0x%x indexCount=%u indexType=%u instances=%u", queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u, packet[3], queue.indexType, queue.instanceCount); return {address, packet[3], indexSize, queue.instanceCount, packet[4]}; } @@ -327,6 +493,8 @@ void Execute(std::span packet, QueueState& queue) { queue.markers.pop_back(); return; case 0x1a: + if (TraceContextState()) std::fprintf(stderr, "[context] op %u: %zu registers, saved %d, cb0 %x info %x mask %x\n", packet[1], queue.context.size(), queue.savedContext.has_value() ? 1 : 0, queue.context.contains(0x318) ? queue.context.at(0x318) : 0u, queue.context.contains(0x31c) ? queue.context.at(0x31c) : 0u, queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u); + APS5_LOG_OUT_DEBUG("CONTEXT_STATE operation=%u targetMaskBefore=0x%x shaderMaskBefore=0x%x", packet[1], queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u); switch (packet[1]) { case 0: queue.ClearContext(); break; case 1: case 3: @@ -340,6 +508,7 @@ void Execute(std::span packet, QueueState& queue) { queue.savedContext.reset(); break; } + APS5_LOG_OUT_DEBUG("CONTEXT_STATE done operation=%u targetMaskAfter=0x%x shaderMaskAfter=0x%x", packet[1], queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u); return; default: throw std::runtime_error("custom packet requires driver execution"); } @@ -347,6 +516,7 @@ void Execute(std::span packet, QueueState& queue) { ((packet[0] & 2u) == 0 ? queue.drawIndirectBase : queue.dispatchIndirectBase) = address(packet[2], packet[3]); return; case 0x12: + APS5_LOG_OUT_DEBUG("CLEAR_STATE targetMaskBefore=0x%x shaderMaskBefore=0x%x", queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u); queue.ClearContext(); return; case 0x13: queue.indexBufferSize = packet[1]; return; case 0x26: queue.indexBase = address(packet[1], packet[2]); return; @@ -354,12 +524,47 @@ void Execute(std::span packet, QueueState& queue) { case 0x2f: queue.instanceCount = packet[1]; return; case 0x63: case 0x64: case 0x9f: { std::vector pairs(static_cast(packet[4]) * 2); + // Named for the [hooksync] attribution (the read goes through the flush hook). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Registers); GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(pairs)), 4); for (std::size_t i = 0; i < pairs.size(); i += 2) registerOffset(pairs[i]); for (std::size_t i = 0; i < pairs.size(); i += 2) writeRegister(queue, opcode, registerOffset(pairs[i]), pairs[i + 1]); + if (queue.savedContext.has_value() && TraceContextState()) { + std::fprintf(stderr, "[context] indirect 0x%x:", opcode); + for (std::size_t i = 0; i < pairs.size(); i += 2) std::fprintf(stderr, " %x", registerOffset(pairs[i])); + std::fprintf(stderr, "\n"); + } + return; + } + case 0x3c: case 0x93: { + // The waited-on value is written by the CPU or another queue; poll it like the CP would. + const auto start = std::chrono::steady_clock::now(); + bool warned = false; + while (!WaitSatisfied(packet)) { + std::this_thread::sleep_for(std::chrono::microseconds(50)); + if (!warned && std::chrono::steady_clock::now() - start > std::chrono::seconds(5)) { + warned = true; + std::fprintf(stderr, "[gpu] WAIT_REG_MEM at 0x%llx still waiting after 5s (function %u ref 0x%x mask 0x%x)\n", + static_cast(address(packet[2], packet[3])), packet[1] & 7u, packet[4], packet[5]); + } + } + return; + } + case 0x49: { + // Earlier work has drained by the time the driver reaches this packet, so the end-of-pipe + // write can happen immediately. Interrupt delivery is not modeled. + const auto dataSelect = packet[2] >> 29u; + const auto destination = address(packet[3], packet[4]); + if (dataSelect == 0 || destination == 0) return; + std::uint64_t value = address(packet[5], packet[6]); + if (dataSelect == 3) { + value = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() / 10); + } + GuestMemory::Write(destination, std::as_bytes(std::span(&value, 1)).first(dataSelect == 1 ? 4 : 8), dataSelect == 1 ? 4 : 8); return; } case 0x69: case 0x76: case 0x79: case 0x7a: { + if (IsTagMarker(packet)) return; const auto offset = registerOffset(packet[1]); for (std::size_t i = 2; i < packet.size(); ++i) writeRegister(queue, opcode, offset + static_cast(i - 2), packet[i]); return; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp index fedb9a12bc..aa7cd9bc9c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp" #include "prx/libSceAgcDriver/Execution/include/AspectFit.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" namespace AgcDriver { @@ -77,17 +78,56 @@ void PresentationScaler::RecordUpload(VkCommandBuffer commands, VkBuffer uploadB pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); } +void PresentationScaler::RecordBlitInto(VkCommandBuffer commands, VkImage image, VkImageLayout layout, VkFilter filter) { + Graphics::Require(sourceImage != VK_NULL_HANDLE && image != VK_NULL_HANDLE, "presentation image is unavailable"); + const auto pipelineBarrier = context.Function("vkCmdPipelineBarrier"); + VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = sourceImage; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + VkImageBlit blit{}; + blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.srcOffsets[1] = {static_cast(sourceWidth), static_cast(sourceHeight), 1}; + blit.dstSubresource = blit.srcSubresource; + blit.dstOffsets[1] = blit.srcOffsets[1]; + context.Function("vkCmdBlitImage")(commands, image, layout, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, filter); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); +} + +void PresentationScaler::RecordReadback(VkCommandBuffer commands, VkBuffer destination) { + Graphics::Require(sourceImage != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "presentation readback is unavailable"); + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {sourceWidth, sourceHeight, 1u}; + context.Function("vkCmdCopyImageToBuffer")(commands, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination, 1, ©); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); +} + void PresentationScaler::RecordBlit(VkCommandBuffer commands, VkImage destinationImage, std::uint32_t destinationWidth, std::uint32_t destinationHeight) { Graphics::Require(sourceImage != VK_NULL_HANDLE, "presentation source image has not been created"); - const auto rect = ComputeContainRect_nid_postfix(sourceWidth, sourceHeight, destinationWidth, destinationHeight); + RecordBlitFrom(context, commands, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, sourceWidth, sourceHeight, VK_FILTER_LINEAR, destinationImage, destinationWidth, destinationHeight); +} + +void PresentationScaler::RecordBlitFrom(const Graphics::Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout layout, std::uint32_t width, std::uint32_t height, VkFilter filter, VkImage destinationImage, std::uint32_t destinationWidth, std::uint32_t destinationHeight) { + Graphics::Require(image != VK_NULL_HANDLE && width != 0 && height != 0, "presentation blit source is unavailable"); + const auto rect = ComputeContainRect_nid_postfix(width, height, destinationWidth, destinationHeight); VkImageBlit blit{}; blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; blit.srcOffsets[0] = {0, 0, 0}; - blit.srcOffsets[1] = {static_cast(sourceWidth), static_cast(sourceHeight), 1}; + blit.srcOffsets[1] = {static_cast(width), static_cast(height), 1}; blit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; blit.dstOffsets[0] = {rect.x, rect.y, 0}; blit.dstOffsets[1] = {rect.x + static_cast(rect.width), rect.y + static_cast(rect.height), 1}; - context.Function("vkCmdBlitImage")(commands, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destinationImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_LINEAR); + context.Function("vkCmdBlitImage")(commands, image, layout, destinationImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, filter); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index 50d913bc16..03853d0796 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -3,68 +3,128 @@ #include "Optimization/RequestMemoryView.hpp" #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" +#include +#include +#include +#include #include -#include #include #include -#include namespace AgcDriver { +namespace { -ShaderMemory::ShaderMemory(std::span initial) { - const ShaderRecompiler::RequestMemoryView validated(initial); - for (const auto& region : initial) { - regions.emplace(region.guestAddress, std::vector(region.bytes.begin(), region.bytes.end())); +struct CaptureProfile { + std::atomic captures{0}; + std::atomic reads{0}; + std::atomic pages{0}; + // The whole CaptureResources call: the plan lookup and the materialization are one call. + std::atomic captureNanoseconds{0}; +}; + +CaptureProfile& CaptureTotals() { + static CaptureProfile profile; + return profile; +} + +} + +ShaderMemory::ShaderMemory(std::span regions) { + const ShaderRecompiler::RequestMemoryView validated(regions); + for (const auto& region : regions) { + initial.emplace(region.guestAddress, region.bytes); } } +ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) { + const auto found = pages.find(base); + if (found != pages.end()) return found->second; + auto& page = pages[base]; + ++CaptureTotals().pages; + // A page is either mapped whole or read word by word where the guest mapped less than a page. + if (GuestMemory::Accessible(reinterpret_cast(base), PageBytes)) { + GuestMemory::Read(base, std::as_writable_bytes(std::span(page.words)), sizeof(std::uint32_t)); + page.valid.set(); + } + return page; +} + bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* value) { auto& self = *static_cast(context); if (address % sizeof(*value) != 0 || address > std::numeric_limits::max() - sizeof(*value)) { throw std::runtime_error("AGC driver: invalid shader memory read address"); } - const auto next = self.regions.upper_bound(address); - if (next != self.regions.begin()) { - const auto previous = std::prev(next); - const auto offset = address - previous->first; - if (offset < previous->second.size()) { - if (previous->second.size() - offset < sizeof(*value)) { - throw std::runtime_error("AGC driver: shader memory read crosses a snapshot boundary"); + ++CaptureTotals().reads; + if (!self.initial.empty()) { + const auto next = self.initial.upper_bound(address); + if (next != self.initial.begin()) { + const auto previous = std::prev(next); + const auto offset = address - previous->first; + if (offset < previous->second.size()) { + if (previous->second.size() - offset < sizeof(*value)) throw std::runtime_error("AGC driver: shader memory read crosses a snapshot boundary"); + std::memcpy(value, previous->second.data() + offset, sizeof(*value)); + return true; } - std::memcpy(value, previous->second.data() + offset, sizeof(*value)); - return true; } + if (next != self.initial.end() && next->first - address < sizeof(*value)) throw std::runtime_error("AGC driver: shader memory read overlaps a snapshot boundary"); } - if (next != self.regions.end() && next->first - address < sizeof(*value)) { - throw std::runtime_error("AGC driver: shader memory read overlaps a snapshot boundary"); + auto& page = self.page(address & ~static_cast(PageBytes - 1)); + const auto index = static_cast((address % PageBytes) / sizeof(*value)); + if (!page.valid.test(index)) { + std::uint32_t word = 0; + GuestMemory::Read(address, std::as_writable_bytes(std::span(&word, 1)), alignof(std::uint32_t)); + page.words[index] = word; + page.valid.set(index); } - std::vector bytes(sizeof(*value)); - GuestMemory::Read(address, bytes, alignof(std::uint32_t)); - std::memcpy(value, bytes.data(), sizeof(*value)); - self.regions.emplace(address, std::move(bytes)); + page.read.set(index); + *value = page.words[index]; return true; } -void ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request) { - const auto plan = ShaderRecompiler::GetResourcePlan(request); - constexpr ShaderRecompiler::ResourceMaterializer materializer; +std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request) { + // The capture's word and page reads (through `read`) are attributed to it ([hooksync], [guestmem]). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Capture); + const auto started = std::chrono::steady_clock::now(); ShaderRecompiler::SrtRuntime runtime; runtime.userData = request.context.userData; runtime.shaderBase = request.shader.codeAddress; runtime.userContext = this; runtime.readMemory = &read; runtime.readSpecializationMemory = &read; - ShaderRecompiler::ResourceSnapshot snapshot; - ShaderRecompiler::ResourceSpecialization specialization; - materializer.Materialize(*plan, runtime, snapshot, specialization); + auto capture = ShaderRecompiler::CaptureResources(request, runtime); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& totals = CaptureTotals(); + if (profile) { + const auto done = std::chrono::steady_clock::now(); + totals.captureNanoseconds += static_cast(std::chrono::duration_cast(done - started).count()); + std::uint64_t initialBytes = 0; + for (const auto& [address, bytes] : initial) initialBytes += bytes.size(); + if (++totals.captures % 500 == 0) std::fprintf(stderr, "[capture] %llu captures: %llu word reads, %llu pages fetched, capture (plan lookup + materialize) %.1f s, %llu KiB registered code this capture\n", static_cast(totals.captures.load()), static_cast(totals.reads.load()), static_cast(totals.pages.load()), totals.captureNanoseconds.load() / 1e9, static_cast(initialBytes / 1024)); + } + return capture; } std::vector ShaderMemory::Regions() const { std::vector result; - result.reserve(regions.size()); - for (const auto& [address, bytes] : regions) { - result.push_back({address, bytes}); + result.reserve(initial.size() + pages.size()); + auto next = initial.begin(); + // Both maps are ordered by address and never overlap, so a merge keeps the result sorted. + for (const auto& [base, page] : pages) { + while (next != initial.end() && next->first < base) { + result.push_back({next->first, next->second}); + ++next; + } + for (std::size_t index = 0; index < PageWords;) { + if (!page.read.test(index)) { + ++index; + continue; + } + const auto first = index; + while (index < PageWords && page.read.test(index)) ++index; + result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, index - first))}); + } } + for (; next != initial.end(); ++next) result.push_back({next->first, next->second}); return result; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 6f28b55256..651bd0bac3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1,30 +1,38 @@ #include "BdaAbi.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp" -#include "prx/libSceAgcDriver/Execution/include/SwapchainState.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureCache.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineCache.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/General.hpp" +#ifdef _WIN32 +#include +#endif +#include +#include #include #include #include #include +#include #include #include #include +#include #include #include +#include +#include #include #include @@ -41,6 +49,29 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("Vulkan presentation: ") + reason); } +// A compute dispatch's Vulkan objects, shared by every dispatch of the same compiled variant and +// kept by the recorder until the batches using them completed. +struct ComputePipelineObjects { + VkDevice device = VK_NULL_HANDLE; + PFN_vkDestroyShaderModule destroyModule = nullptr; + PFN_vkDestroyPipelineLayout destroyLayout = nullptr; + PFN_vkDestroyPipeline destroyPipeline = nullptr; + VkShaderModule module = VK_NULL_HANDLE; + VkPipelineLayout layout = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; + ~ComputePipelineObjects() { + if (pipeline != VK_NULL_HANDLE) destroyPipeline(device, pipeline, nullptr); + if (layout != VK_NULL_HANDLE) destroyLayout(device, layout, nullptr); + if (module != VK_NULL_HANDLE) destroyModule(device, module, nullptr); + } +}; + +// The ShaderResources content cache dispatches share with recorded draws: Graphics::ResourceCache, +// one process-wide instance (see SharedResourceCache) that the State references so this file keeps +// its Find/Insert/Remove/Clear calls; the device clears it at teardown before its descriptor caches +// go. APS5_NO_RESOURCE_CACHE=1 builds every dispatch's resources as before. +using ResourceCache = Graphics::ResourceCache; + } struct VulkanDevice::State { @@ -55,7 +86,6 @@ struct VulkanDevice::State { void* window = nullptr; VkSurfaceKHR surface = VK_NULL_HANDLE; VkSwapchainKHR swapchain = VK_NULL_HANDLE; - SwapchainState swapchainState; VkExtent2D extent{}; std::vector images; VkFence acquireFence = VK_NULL_HANDLE; @@ -80,22 +110,45 @@ struct VulkanDevice::State { bool meshShader = false; bool fragmentShaderBarycentric = false; bool depthClipControl = false; + bool depthClamp = false; + VkDeviceSize hostImportAlignment = 0; bool depthRangeUnrestricted = false; bool samplerAnisotropy = false; bool textureCompressionBC = false; + // VK_KHR_timeline_semaphore enabled: the recorder's unlocked waits are available. + bool timelineSemaphores = false; std::unique_ptr detiler; + std::unique_ptr recorder; + std::map> computePipelines; std::unique_ptr colorTransfer; std::shared_ptr bufferPool; - std::shared_ptr descriptorCache; - std::shared_ptr samplerCache; std::unique_ptr textureCache; std::unique_ptr pipelineCache; - std::unique_ptr drawQueue; - std::unique_ptr renderCache; - std::unique_ptr graphicsPipelines; + std::unique_ptr descriptorCache; + std::unique_ptr samplerCache; + Graphics::ResourceCache& resourceCache = Graphics::SharedResourceCache(); + // Recorded dispatches that write a copied buffer (their results reach guest memory by a CPU + // write-back when the batch is reaped), listed until that write-back ran. An indirect dispatch + // whose arguments one of them writes must read them on the CPU (DispatchIndirect); GPU-direct + // writes into host imports need no entry. Under GuestMemory::GpuMutex; shared with the completion + // actions so they never outlive it. + std::shared_ptr>> copiedWriters = std::make_shared>>(); VkPhysicalDeviceMeshShaderPropertiesEXT meshLimits{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_PROPERTIES_EXT}; std::unique_ptr scaler; - std::unique_ptr rgbaScaler; + // The swapchain image AcquireImage took for the next present(), consumed by that present. + bool imageAcquired = false; + std::uint32_t acquiredIndex = 0; + // The presentation in flight between present() and QueuePresent(): its swapchain image, and the + // resident image it blits from, alive until the render fence has been waited for. + bool presentPending = false; + std::uint32_t presentIndex = 0; + std::shared_ptr presentKept; + // APS5_DUMP_FRAMES: the presented frame is read back into this buffer by the same submission and + // written as a BMP by the writer thread after the render fence. + std::unique_ptr dumpBuffer; + bool dumpRecorded = false; + int dumpIndex = 0; + std::thread dumpWriter; template TFunction InstanceFunction(const char* name) const { @@ -116,7 +169,9 @@ struct VulkanDevice::State { } void Upload(std::span pixels) { + APS5_LOG_OUT("Upload pixels=%zu uploadSize=%llu buffer=%p memory=%p mapping=%p", pixels.size(), static_cast(uploadSize), reinterpret_cast(uploadBuffer), reinterpret_cast(uploadMemory), uploadMapping); if (uploadSize < pixels.size()) { + APS5_LOG_OUT("Upload reallocating oldSize=%llu newSize=%zu", static_cast(uploadSize), pixels.size()); if (uploadMapping) DeviceFunction("vkUnmapMemory")(device, uploadMemory); uploadMapping = nullptr; if (uploadBuffer) DeviceFunction("vkDestroyBuffer")(device, uploadBuffer, nullptr); @@ -140,6 +195,7 @@ struct VulkanDevice::State { } } require(memoryType < memoryProperties.memoryTypeCount, "coherent host upload memory is unavailable"); + APS5_LOG_OUT("Upload requirements size=%llu alignment=%llu typeBits=0x%x memoryType=%u", static_cast(requirements.size), static_cast(requirements.alignment), requirements.memoryTypeBits, memoryType); VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = memoryType; @@ -147,8 +203,10 @@ struct VulkanDevice::State { check(DeviceFunction("vkBindBufferMemory")(device, uploadBuffer, uploadMemory, 0), "vkBindBufferMemory display upload"); check(DeviceFunction("vkMapMemory")(device, uploadMemory, 0, pixels.size(), 0, &uploadMapping), "vkMapMemory display upload"); uploadSize = pixels.size(); + APS5_LOG_OUT("Upload allocation complete buffer=%p memory=%p mapping=%p size=%llu", reinterpret_cast(uploadBuffer), reinterpret_cast(uploadMemory), uploadMapping, static_cast(uploadSize)); } std::memcpy(uploadMapping, pixels.data(), pixels.size()); + APS5_LOG_OUT("Upload memcpy complete bytes=%zu", pixels.size()); } void DestroyRetiredSwapchains() { @@ -165,24 +223,29 @@ struct VulkanDevice::State { } ~State() { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + if (dumpWriter.joinable()) dumpWriter.join(); if (device != VK_NULL_HANDLE) { - if (drawQueue) drawQueue->Wait(); - if (renderCache) renderCache->Flush(); const auto idle = reinterpret_cast(deviceProc(device, "vkDeviceWaitIdle"))(device); if (idle != VK_SUCCESS && idle != VK_ERROR_DEVICE_LOST) std::terminate(); - drawQueue.reset(); - graphicsPipelines.reset(); - renderCache.reset(); + // A resident image kept for an unfinished presentation goes before the caches it came from. + presentKept.reset(); + dumpBuffer.reset(); + recorder.reset(); + // Cached graphics pipelines (with their framebuffers, modules, render passes and layouts) + // belong to this device and must be destroyed while it lives. + Graphics::ClearCachedPipelines(device); + computePipelines.clear(); + // Every ShaderResources (kept by the recorder or the resource cache) is gone now, so the + // sets and samplers they borrowed can go. + resourceCache.Clear(); + descriptorCache.reset(); + samplerCache.reset(); textureCache.reset(); detiler.reset(); colorTransfer.reset(); scaler.reset(); - rgbaScaler.reset(); pipelineCache.reset(); bufferPool.reset(); - descriptorCache.reset(); - samplerCache.reset(); const auto destroyFence = reinterpret_cast(deviceProc(device, "vkDestroyFence")); if (acquireFence) destroyFence(device, acquireFence, nullptr); if (renderFence) destroyFence(device, renderFence, nullptr); @@ -213,6 +276,7 @@ struct VulkanDevice::State { }; VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_unique()) { + APS5_LOG_OUT("VulkanDevice constructor window=%p", static_cast(window)); #ifdef _WIN32 state->library = SDL_LoadObject("vulkan-1.dll"); #else @@ -221,6 +285,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u if (state->library == nullptr) { throw std::runtime_error(std::string("Vulkan loader: ") + SDL_GetError()); } + APS5_LOG_OUT("Vulkan loader loaded library=%p", state->library); state->instanceProc = reinterpret_cast(SDL_LoadFunction(state->library, "vkGetInstanceProcAddr")); if (state->instanceProc == nullptr) { throw std::runtime_error("Vulkan loader: vkGetInstanceProcAddr missing"); @@ -232,6 +297,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u create.pApplicationInfo = &application; std::vector instanceExtensions; if (window != nullptr) { + APS5_LOG_OUT("Presentation window context=%p extent=%ux%u extensions=%zu", window->context, window->width, window->height, window->extensions.size()); require(window->context && window->createSurface && window->getDrawableSize && window->width && window->height, "invalid window descriptor"); instanceExtensions.assign(window->extensions.begin(), window->extensions.end()); std::uint32_t availableCount = 0; @@ -249,15 +315,18 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u create.ppEnabledExtensionNames = instanceExtensions.data(); } check(state->InstanceFunction("vkCreateInstance")(&create, nullptr, &state->instance), "vkCreateInstance"); + APS5_LOG_OUT("Vulkan instance created instance=%p", reinterpret_cast(state->instance)); if (window != nullptr) { state->surface = window->createSurface(window->context, state->instance); require(state->surface != VK_NULL_HANDLE, "window returned a null surface"); state->window = window->context; + APS5_LOG_OUT("Vulkan surface created surface=%p window=%p", reinterpret_cast(state->surface), state->window); } state->deviceProc = state->InstanceFunction("vkGetDeviceProcAddr"); const auto enumerate = state->InstanceFunction("vkEnumeratePhysicalDevices"); std::uint32_t count = 0; check(enumerate(state->instance, &count, nullptr), "vkEnumeratePhysicalDevices"); + APS5_LOG_OUT("Physical device count=%u", count); std::vector devices(count); check(enumerate(state->instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); devices.resize(count); @@ -267,6 +336,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u for (auto physical : devices) { VkPhysicalDeviceProperties properties{}; state->InstanceFunction("vkGetPhysicalDeviceProperties")(physical, &properties); + APS5_LOG_OUT("Physical device candidate=%p name=%s api=0x%x", reinterpret_cast(physical), properties.deviceName, properties.apiVersion); if (properties.apiVersion < VK_API_VERSION_1_1) { continue; } @@ -305,6 +375,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u if (selected == VK_NULL_HANDLE) { throw std::runtime_error(window ? "Vulkan: no Vulkan 1.1 device with graphics, compute and swapchain presentation" : "Vulkan: no Vulkan 1.1 graphics and compute queue"); } + APS5_LOG_OUT("Physical device selected physical=%p queueFamily=%u", reinterpret_cast(selected), family); VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; properties.pNext = &state->subgroup; state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); @@ -315,7 +386,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_BALLOT_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformBallot); if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_SHUFFLE_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformShuffle); } + APS5_LOG_OUT("Selected GPU name=%s vendor=0x%x device=0x%x subgroup=%u", state->properties.deviceName, state->properties.vendorID, state->properties.deviceID, state->subgroup.subgroupSize); state->InstanceFunction("vkGetPhysicalDeviceMemoryProperties")(selected, &state->memoryProperties); + for (std::uint32_t i = 0; i < state->memoryProperties.memoryHeapCount; ++i) APS5_LOG_OUT("Memory heap %u size=%.1f MiB flags=0x%x", i, state->memoryProperties.memoryHeaps[i].size / 1048576.0, state->memoryProperties.memoryHeaps[i].flags); + for (std::uint32_t i = 0; i < state->memoryProperties.memoryTypeCount; ++i) APS5_LOG_OUT("Memory type %u heap=%u flags=0x%x", i, state->memoryProperties.memoryTypes[i].heapIndex, state->memoryProperties.memoryTypes[i].propertyFlags); std::uint32_t extensionCount = 0; const auto enumerateDeviceExtensions = state->InstanceFunction("vkEnumerateDeviceExtensionProperties"); check(enumerateDeviceExtensions(selected, nullptr, &extensionCount, nullptr), "vkEnumerateDeviceExtensionProperties"); @@ -366,6 +440,26 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->spirvExtensions.push_back("SPV_KHR_physical_storage_buffer"); state->depthRangeUnrestricted = hasExtension(VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME); if (state->depthRangeUnrestricted) deviceExtensions.push_back(VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME); + // Guest dispatches cover whole thread groups past the edge of small images; robust image access + // drops those writes instead of faulting the device. + const bool imageRobustness = hasExtension(VK_EXT_IMAGE_ROBUSTNESS_EXTENSION_NAME); + if (imageRobustness) deviceExtensions.push_back(VK_EXT_IMAGE_ROBUSTNESS_EXTENSION_NAME); + // Guest memory is host memory: importing it lets address-based shaders use it in place instead of + // copying every registered allocation per draw. + if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) && std::getenv("APS5_NO_HOST_IMPORT") == nullptr) { + VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); + state->hostImportAlignment = hostProperties.minImportedHostPointerAlignment; +#ifdef _WIN32 + // Imported pages are locked; locking is bounded by the process working-set minimum. + if (const char* value = std::getenv("APS5_WORKING_SET_MIB")) { + const SIZE_T bytes = static_cast(std::strtoull(value, nullptr, 10)) << 20u; + if (!SetProcessWorkingSetSizeEx(GetCurrentProcess(), bytes, bytes * 2, QUOTA_LIMITS_HARDWS_MIN_DISABLE | QUOTA_LIMITS_HARDWS_MAX_DISABLE)) std::fprintf(stderr, "[gpu] SetProcessWorkingSetSizeEx failed: %lu\n", GetLastError()); + } +#endif + deviceExtensions.push_back(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); + } VkPhysicalDeviceDepthClipControlFeaturesEXT depthClipFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_CLIP_CONTROL_FEATURES_EXT}; if (hasExtension(VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME)) { VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &depthClipFeatures}; @@ -378,6 +472,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->capabilities.push_back(5283); state->spirvExtensions.push_back("SPV_EXT_mesh_shader"); } + APS5_LOG_OUT("Vulkan features tessellationAvailable=%u mesh=%u depthClip=%u depthRangeUnrestricted=%u", static_cast(state->tessellationShader), static_cast(state->meshShader), static_cast(state->depthClipControl), static_cast(state->depthRangeUnrestricted)); const float priority = 1.0f; VkDeviceQueueCreateInfo queueInfo{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; queueInfo.queueFamilyIndex = family; @@ -399,6 +494,20 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u require(available.samplerAnisotropy && available.textureCompressionBC, "device lacks sampler anisotropy or BC texture compression support required for texture sampling"); enabled.samplerAnisotropy = VK_TRUE; enabled.textureCompressionBC = VK_TRUE; + // Guest shaders routinely read past descriptor ranges; robust access turns that into zeros + // instead of a GPU fault that loses the device. + enabled.robustBufferAccess = available.robustBufferAccess; + // PA_CL_CLIP_CNTL near/far clip disable maps to depth clamping. + enabled.depthClamp = available.depthClamp; + state->depthClamp = enabled.depthClamp == VK_TRUE; + // Recompiled storage-image access declares no format (the guest descriptor decides it). + enabled.shaderStorageImageWriteWithoutFormat = available.shaderStorageImageWriteWithoutFormat; + enabled.shaderStorageImageReadWithoutFormat = available.shaderStorageImageReadWithoutFormat; + // Gathers with non-constant offsets (ImageGatherExtended). + enabled.shaderImageGatherExtended = available.shaderImageGatherExtended; + if (enabled.shaderImageGatherExtended) state->capabilities.push_back(spv::CapabilityImageGatherExtended); + if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat); + if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat); state->samplerAnisotropy = true; state->textureCompressionBC = true; deviceInfo.pEnabledFeatures = &enabled; @@ -417,24 +526,49 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u barycentricFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &barycentricFeatures; } + VkPhysicalDeviceImageRobustnessFeaturesEXT imageRobustnessFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_ROBUSTNESS_FEATURES_EXT, nullptr, VK_TRUE}; + if (imageRobustness) { + imageRobustnessFeatures.pNext = byteFeatures.pNext; + byteFeatures.pNext = &imageRobustnessFeatures; + } + // Timeline semaphores let a queue worker wait for recorded batches without holding the GPU mutex + // (see Recorder::WaitSerial). The instance is 1.1, so the KHR extension is used even on 1.2+ + // devices. Debug aid: APS5_NO_TIMELINE=1 leaves it off (drains wait under the mutex as before). + VkPhysicalDeviceTimelineSemaphoreFeaturesKHR timelineFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR}; + if (hasExtension(VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME) && std::getenv("APS5_NO_TIMELINE") == nullptr) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &timelineFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + state->timelineSemaphores = timelineFeatures.timelineSemaphore == VK_TRUE; + } + timelineFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR}; + timelineFeatures.timelineSemaphore = VK_TRUE; + if (state->timelineSemaphores) { + deviceExtensions.push_back(VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME); + deviceInfo.enabledExtensionCount = static_cast(deviceExtensions.size()); + deviceInfo.ppEnabledExtensionNames = deviceExtensions.data(); + timelineFeatures.pNext = byteFeatures.pNext; + byteFeatures.pNext = &timelineFeatures; + } else { + std::fprintf(stderr, "[gpu] timeline semaphores unavailable or disabled; drains wait under the GPU mutex\n"); + } bdaFeatures.pNext = &byteFeatures; deviceInfo.pNext = &bdaFeatures; check(state->InstanceFunction("vkCreateDevice")(selected, &deviceInfo, nullptr, &state->device), "vkCreateDevice"); state->DeviceFunction("vkGetDeviceQueue")(state->device, family, 0, &state->queue); + APS5_LOG_OUT("Vulkan device ready device=%p queue=%p family=%u", reinterpret_cast(state->device), reinterpret_cast(state->queue), family); VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; poolInfo.queueFamilyIndex = family; check(state->DeviceFunction("vkCreateCommandPool")(state->device, &poolInfo, nullptr, &state->pool), "vkCreateCommandPool"); state->bufferPool = std::make_shared(graphicsContext()); - state->descriptorCache = std::make_shared(); - state->samplerCache = std::make_shared(); state->pipelineCache = std::make_unique(graphicsContext()); state->detiler = std::make_unique(graphicsContext()); - state->drawQueue = std::make_unique(); - state->renderCache = std::make_unique(graphicsContext()); - state->graphicsPipelines = std::make_unique(graphicsContext()); state->textureCache = std::make_unique(graphicsContext()); state->colorTransfer = std::make_unique(graphicsContext()); + state->descriptorCache = std::make_unique(graphicsContext()); + state->samplerCache = std::make_unique(); + state->recorder = std::make_unique(graphicsContext(), state->timelineSemaphores); + state->recorder->Activate(); if (window != nullptr) { require(window->getDrawableSize != nullptr, "missing window drawable size query"); std::uint32_t drawableWidth = 0; @@ -444,7 +578,14 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u VkSurfaceCapabilitiesKHR surface{}; check(state->InstanceFunction("vkGetPhysicalDeviceSurfaceCapabilitiesKHR")(selected, state->surface, &surface), "vkGetPhysicalDeviceSurfaceCapabilitiesKHR"); state->extent = {drawableWidth, drawableHeight}; - require(surface.currentExtent.width == std::numeric_limits::max() || (surface.currentExtent.width == drawableWidth && surface.currentExtent.height == drawableHeight), "window extent differs from the real drawable size"); + APS5_LOG_OUT("Surface capabilities drawable=%ux%u min=%ux%u max=%ux%u minImages=%u maxImages=%u usage=0x%x", drawableWidth, drawableHeight, surface.minImageExtent.width, surface.minImageExtent.height, surface.maxImageExtent.width, surface.maxImageExtent.height, surface.minImageCount, surface.maxImageCount, surface.supportedUsageFlags); + // A surface that reports its extent (a fullscreen transition can make it differ from the + // drawable size for a moment) wins: the swapchain must match the surface. + if (surface.currentExtent.width != std::numeric_limits::max() && surface.currentExtent.width != 0 && surface.currentExtent.height != 0) { + drawableWidth = surface.currentExtent.width; + drawableHeight = surface.currentExtent.height; + state->extent = {drawableWidth, drawableHeight}; + } require(drawableWidth >= surface.minImageExtent.width && drawableWidth <= surface.maxImageExtent.width && drawableHeight >= surface.minImageExtent.height && drawableHeight <= surface.maxImageExtent.height, "unsupported output extent"); require((surface.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) != 0, "surface does not support transfer destination images"); require((surface.supportedCompositeAlpha & VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR) != 0, "opaque composition is unavailable"); @@ -474,6 +615,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u check(getImages(state->device, state->swapchain, &imageCount, nullptr), "vkGetSwapchainImagesKHR"); state->images.resize(imageCount); check(getImages(state->device, state->swapchain, &imageCount, state->images.data()), "vkGetSwapchainImagesKHR"); + APS5_LOG_OUT("Swapchain created swapchain=%p extent=%ux%u images=%u", reinterpret_cast(state->swapchain), state->extent.width, state->extent.height, imageCount); VkFenceCreateInfo fence{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; for (auto* destination : {&state->acquireFence, &state->renderFence}) { check(state->DeviceFunction("vkCreateFence")(state->device, &fence, nullptr, destination), "vkCreateFence"); @@ -486,25 +628,237 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u allocation.commandBufferCount = 1; check(state->DeviceFunction("vkAllocateCommandBuffers")(state->device, &allocation, &state->clearCommands), "vkAllocateCommandBuffers"); state->scaler = std::make_unique(graphicsContext(), VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM); - state->rgbaScaler = std::make_unique(graphicsContext(), VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM); } } VulkanDevice::~VulkanDevice() = default; void VulkanDevice::WaitIdle() { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - PerformanceTimer timing("Vulkan.WaitIdle"); - state->drawQueue->Wait(); - timing.Mark("draw_wait"); + APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle begin"); + if (state->recorder) { + // Announced with this call's return address: the [recorder] site table then names the + // driver site that drains (suspend point, label fallback, fill fallback, device replacement). + if (!state->recorder->Idle()) Graphics::Recorder::CountSync(0, __builtin_return_address(0)); + state->recorder->Sync(); + } check(state->DeviceFunction("vkDeviceWaitIdle")(state->device), "vkDeviceWaitIdle"); - timing.Mark("device_wait"); + APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle complete"); } -void VulkanDevice::WaitDraws() { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - PerformanceTimer timing("Vulkan.WaitDraws"); - state->drawQueue->Wait(); +namespace { + +bool OpportunisticReap() { + static const bool enabled = std::getenv("APS5_NO_OPPORTUNISTIC_REAP") == nullptr; + return enabled; +} + +} + +void VulkanDevice::SubmitRecorded(bool reapFirst) { + if (!state->recorder) return; + // Finished batches are retired first (a non-blocking fence status check on the front of the + // in-flight list), so PendingWriteOverlaps/HasCompletions scans stay short and a label recorded + // after a completing batch already finished can go to the GPU instead of behind a completion. + // Not for cross-queue submitters: a retired batch's write-back may sync a later batch under the + // mutex (see the header), which a poller or compute worker must never do for queue 0. + if (reapFirst && OpportunisticReap()) state->recorder->Reap(); + state->recorder->Submit(); +} + +bool VulkanDevice::CanWaitUnlocked() const { + return state->recorder && state->recorder->HasTimeline(); +} + +std::uint64_t VulkanDevice::SubmitAndEpoch() { + if (!state->recorder) return 0; + if (OpportunisticReap()) state->recorder->Reap(); + return state->recorder->SubmitAndEpoch(); +} + +void VulkanDevice::WaitRecorded(std::uint64_t serial) { + if (state->recorder) state->recorder->WaitSerial(serial); +} + +void VulkanDevice::ReapRecorded(std::uint64_t serial) { + if (!state->recorder) return; + // The drain's site in the [recorder] table is this call's caller (Driver::execute's drain, + // PrepareDispatch's presync), not this wrapper; FinishUpTo consumes the announcement. + Graphics::Recorder::AnnounceSyncSite(__builtin_return_address(0)); + state->recorder->FinishUpTo(serial); +} + +void VulkanDevice::ReapRecorded() { + if (state->recorder) state->recorder->Reap(); +} + +std::optional VulkanDevice::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { + if (!state->recorder) return std::nullopt; + return state->recorder->PendingLabel(address, bytes, afterStamp, queue); +} + +bool VulkanDevice::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const { + return state->recorder && state->recorder->OpenWriteOverlaps(address, bytes); +} + +int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) { + if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4; + auto& recorder = *state->recorder; + // Batches that already finished are retired before the checks: their completions ran, so they + // neither keep the recorder busy nor force this label behind a completion. Only for the + // graphics worker: a reap runs completions (copied write-backs, whose guest stores can re-enter + // the flush hook and sync a later batch) under the mutex, which a compute worker must never do + // against queue 0; its label goes behind a completion instead (reason 5, still no wait). + // Debug aid: APS5_LABEL_REAP_ALL_QUEUES=1 reaps on every queue as before. + static const bool reapAllQueues = std::getenv("APS5_LABEL_REAP_ALL_QUEUES") != nullptr; + if (reapFirst && (queue == 0 || reapAllQueues) && OpportunisticReap()) recorder.Reap(); + if (recorder.Idle()) return 1; + static const bool drain = std::getenv("APS5_DRAIN_COMPLETION_LABELS") != nullptr; + const auto context = graphicsContext(); + const auto* import = Graphics::HostImportFor(context, address, bytes.size()); + if (import == nullptr) { + // Memory the GPU has no view of: the store is a completion action of the batch (it runs after + // the recorded work, in order, like a label behind write-backs) instead of a device drain. + if (drain) return 3; + recorder.AfterCompletions(address, bytes, stamp, queue, false); + return 6; + } + // The store on the GPU, into the open batch: everything recorded so far completes before it, + // and it is visible to the host after. Both barriers stay per store: the first orders + // consecutive stores to one address (WAW), the second makes each store host-visible on its own, + // before the rest of the batch completes. + const auto recordStore = [&] { + const auto commands = recorder.Commands(); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + state->DeviceFunction("vkCmdUpdateBuffer")(commands, import->buffer, address - import->base, bytes.size(), bytes.data()); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT); + }; + if (recorder.HasCompletions()) { + // A copied buffer's write-back is still to run: the store must land after it. It is stored + // by a completion action of the batch (a CPU memcpy when the batch is reaped) instead of + // draining the device here. The same bytes are also recorded on the GPU into the open batch + // first, so GPU work recorded after this point that reads the range through the host import + // (a shader binding constants a DUMP_CONST_RAM dumped, a dispatch consuming a COPY_DATA or + // DMA_DATA fill) sees them in order; the completion memcpy stores identical bytes again + // after any write-back that overwrote them, so either order of the two stores is correct. + // CPU readers are covered by the pending-write note (the flush hook syncs) and waits by the + // label table. Debug aid: APS5_DRAIN_COMPLETION_LABELS=1 drains as before; try it first when + // a title misbehaves with this path. + if (drain) return 2; + recordStore(); + GuestMemory::MarkWritten(address, bytes.size()); + // Commands() opened the batch the store went into, so the completion is appended to that + // batch and runs at its reap, after the write-backs of every batch before it. + recorder.AfterCompletions(address, bytes, stamp, queue, true); + return 5; + } + recordStore(); + // The table entry goes in before the pending-write note: the note bumps the write generation a + // poller watches, and a poller that sees the bump then finds the label without the GPU mutex. + recorder.NoteLabel(address, bytes, stamp, queue); + recorder.NotePendingWrite(address, bytes.size()); + GuestMemory::MarkWritten(address, bytes.size()); + // No submit per label: the batch goes out at the queue worker's next non-label packet, at a + // wait on its range, after APS5_LABEL_FLUSH_US, or at the submission's end (Driver.cpp). + // Debug aid: APS5_LABEL_SUBMIT_NOW=1 submits every label at once, as before. + static const bool submitNow = std::getenv("APS5_LABEL_SUBMIT_NOW") != nullptr; + if (submitNow) recorder.Submit(); + return 0; +} + +bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern) { + if (!state->recorder || bytes == 0 || bytes % 16 != 0 || address % 16 != 0) return false; + auto& recorder = *state->recorder; + // Finished batches are retired first (one fence status check, as WriteLabelOnGpu does): their + // completion labels landed and their copied writers are delisted, so a batch the idle GPU already + // finished does not force a wait below. Before the import lookup: a completion may refresh the + // import table. Debug aid: APS5_NO_OPPORTUNISTIC_REAP=1 skips it. + if (OpportunisticReap()) recorder.Reap(); + const auto context = graphicsContext(); + const Graphics::HostImport* import = Graphics::HostImportFor(context, address, bytes); + if (import == nullptr) return false; + // Earlier recorded work that stores into the range must land before the fill, or it would + // overwrite the fill later. Stores the GPU makes (direct dispatch writes, GPU labels, earlier + // fills, GPU-direct write-backs) are ordered by the fill's own barrier below (ALL_COMMANDS -> + // TRANSFER), so only the stores the CPU makes when a batch is reaped need a wait: a recorded + // dispatch whose copied buffer overlaps the range (state->copiedWriters, listed until its + // write-back ran; per object over all its written V# ranges, as DispatchIndirect tests it) and a + // completion-deferred label (Recorder::AfterCompletions) over the range. The label table has no + // range query and a fill spans MiBs, so a small range is looked up per dword (a GPU label there + // counts too: a harmless extra wait) and a larger one waits whenever any completion label is + // pending at all. Batches recorded after the last such store keep running (SyncThrough). + // Debug aid: APS5_FILL_SYNC=1 waits for every recorded store over the range, as before. + static const bool alwaysSync = std::getenv("APS5_FILL_SYNC") != nullptr; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // APS5_PROFILE_DRAW: the fills' outcomes, cumulative, every 10 s (under the GpuMutex like the + // rest) on a [fill-sync] line; Driver.cpp's [fill] line counts the fills and their bytes. + // `labelSynced` counts syncs taken because a completion label was pending: over the range for + // a small fill, anywhere at all for a larger one. + static std::uint64_t fills = 0, synced = 0, labelSynced = 0, ordered = 0; + static double syncedMs = 0; + static auto lastReport = std::chrono::steady_clock::now(); + ++fills; + if (recorder.PendingWriteOverlaps(address, bytes)) { + // Recorded draws with copied writes keep their own registry (Graphics::DrawCopiedWriters): + // their CPU write-back would land after a fill recorded on the GPU, so they count too. + bool sync = alwaysSync || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }) + || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }); + if (!sync && Graphics::Recorder::PendingCompletionLabels() != 0) { + // A label the CPU will store after a completion must not be overwritten by the fill: + // only a label inside the filled range matters, whatever the range's size. + sync = recorder.PendingLabelIn(address, bytes); + if (sync) ++labelSynced; + } + if (sync) { + ++synced; + const auto syncStart = std::chrono::steady_clock::now(); + Graphics::Recorder::CountSync(4); + recorder.SyncThrough(address, bytes); + syncedMs += std::chrono::duration(std::chrono::steady_clock::now() - syncStart).count(); + // The sync ran completions, whose write-backs can refresh the import table and retire + // the import looked up above; the caller stores the fill itself when it is gone. + import = Graphics::HostImportFor(context, address, bytes); + if (import == nullptr) return false; + } else { + ++ordered; + } + } + if (profile) { + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport > std::chrono::seconds(10)) { + lastReport = now; + std::fprintf(stderr, "[fill-sync] fills: %llu synced (%llu with a completion label pending) waited %.0f ms, %llu ordered by barrier, %llu with no pending write\n", static_cast(synced), static_cast(labelSynced), syncedMs, static_cast(ordered), static_cast(fills - synced - ordered)); + } + } + const auto commands = recorder.Commands(); + // Every earlier recorded read or write of the range precedes the fill (WAR by the execution + // dependency, WAW by the writes made available), whichever stage made it. + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + const auto offset = address - import->base; + if (pattern[0] == pattern[1] && pattern[1] == pattern[2] && pattern[2] == pattern[3]) { + state->DeviceFunction("vkCmdFillBuffer")(commands, import->buffer, offset, bytes, pattern[0]); + } else { + // A 16-byte pattern is seeded once and doubled in place until the range is covered. + auto seed = std::make_shared(context, 16, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(seed->Bytes().data(), pattern.data(), 16); + recorder.Keep(seed); + const VkBufferCopy first{0, offset, 16}; + state->DeviceFunction("vkCmdCopyBuffer")(commands, seed->Handle(), import->buffer, 1, &first); + for (std::size_t done = 16; done < bytes;) { + const auto chunk = std::min(done, bytes - done); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + const VkBufferCopy copy{offset, offset + done, chunk}; + state->DeviceFunction("vkCmdCopyBuffer")(commands, import->buffer, import->buffer, 1, ©); + done += chunk; + } + } + // The filled bytes are visible to everything recorded after: shaders, transfers, the host, and + // an indirect dispatch reading its group counts in place (DispatchIndirect adds its own + // ALL_COMMANDS -> DRAW_INDIRECT barrier as well). A later GPU label store over the range is + // ordered behind the fill by its ALL_COMMANDS -> TRANSFER barrier (WriteLabelOnGpu). + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT); + recorder.NotePendingWrite(address, bytes); + GuestMemory::MarkWritten(address, bytes); + return true; } void* VulkanDevice::Window() const { @@ -512,24 +866,27 @@ void* VulkanDevice::Window() const { } void VulkanDevice::Resize(std::uint32_t width, std::uint32_t height) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + APS5_LOG_OUT_DEBUG("Resize requested=%ux%u current=%ux%u", width, height, state->extent.width, state->extent.height); require(state->swapchain != VK_NULL_HANDLE, "cannot resize an unavailable swapchain"); if (width == 0 || height == 0) { state->extent = {0, 0}; return; } + if (state->extent.width == width && state->extent.height == height) return; + WaitIdle(); VkSurfaceCapabilitiesKHR surface{}; check(state->InstanceFunction("vkGetPhysicalDeviceSurfaceCapabilitiesKHR")(state->physical, state->surface, &surface), "vkGetPhysicalDeviceSurfaceCapabilitiesKHR resize"); + // The window's drawable size and the surface's extent can disagree for a moment while the window + // is being resized; the swapchain must match the surface, and the next present catches up. if (surface.currentExtent.width != std::numeric_limits::max()) { width = surface.currentExtent.width; height = surface.currentExtent.height; + if (width == 0 || height == 0) { + state->extent = {0, 0}; + return; + } + if (state->extent.width == width && state->extent.height == height) return; } - if (width == 0 || height == 0) { - state->extent = {0, 0}; - return; - } - if (!state->swapchainState.NeedsRecreation() && state->extent.width == width && state->extent.height == height) return; - WaitIdle(); require(width >= surface.minImageExtent.width && width <= surface.maxImageExtent.width && height >= surface.minImageExtent.height && height <= surface.maxImageExtent.height, "unsupported resized output extent"); require((surface.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) != 0 && (surface.supportedCompositeAlpha & VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR) != 0 && (surface.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) != 0, "resized surface capabilities are unsupported"); VkSwapchainCreateInfoKHR create{VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR}; @@ -558,60 +915,200 @@ void VulkanDevice::Resize(std::uint32_t width, std::uint32_t height) { check(getImages(state->device, replacement, &count, state->images.data()), "vkGetSwapchainImagesKHR resize"); state->images.resize(count); state->rendered.assign(count, VK_NULL_HANDLE); - state->swapchainState.Recreated(); + APS5_LOG_OUT_DEBUG("Resize complete swapchain=%p extent=%ux%u images=%u", reinterpret_cast(state->swapchain), state->extent.width, state->extent.height, count); } bool VulkanDevice::Presentable() const { return state->extent.width != 0 && state->extent.height != 0; } -void VulkanDevice::PresentClear(std::uint32_t width, std::uint32_t height, bool opaque) { - present(width, height, opaque, {}); +bool VulkanDevice::PresentClear(std::uint32_t width, std::uint32_t height, bool opaque) { + APS5_LOG_OUT_DEBUG("PresentClear width=%u height=%u opaque=%u", width, height, static_cast(opaque)); + return present(width, height, opaque, {}); } void VulkanDevice::PresentPixels(std::uint32_t width, std::uint32_t height, std::span pixels) { + APS5_LOG_OUT("PresentPixels width=%u height=%u pixels=%zu expected=%llu", width, height, pixels.size(), static_cast(width) * height * 4u); require(width != 0 && height != 0 && width <= 16384 && height <= 16384, "invalid display image extent"); require(pixels.size() == static_cast(width) * height * 4, "invalid display pixel buffer size"); - present(width, height, true, pixels); -} - -void VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { - present(buffer.width, buffer.height, true, {}, &buffer); -} - -void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - PerformanceTimer timing("Vulkan.Present"); - state->drawQueue->Wait(); - timing.Mark("draw_wait"); - require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); - require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); - std::shared_ptr resident; - if (display != nullptr) { - const auto bytes = DisplayBufferSize(*display); - resident = state->renderCache->Find(display->address); - if (resident) { - const auto& color = resident->Description(); - if (color.extent.width != width || color.extent.height != height || color.bytes != bytes || color.tileMode != Graphics::ColorTileMode::RenderTarget) resident.reset(); - } - if (!resident) { - ResolveMemory(display->address, bytes, false); - state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget); + // Debug aid: APS5_DUMP_FRAMES= saves the first n presented frames as frame_.bmp. + static const int dumpLimit = [] { + const char* value = std::getenv("APS5_DUMP_FRAMES"); + return value ? std::atoi(value) : 0; + }(); + static int dumped = 0; + if (dumped < dumpLimit) { + char name[32]; + std::snprintf(name, sizeof(name), "frame_%03d.bmp", dumped++); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t imageBytes = width * height * 4; + const std::uint32_t header[13] = {0, 0, 54, 40, width, static_cast(-static_cast(height)), 1u | (32u << 16u), 0, imageBytes, 2835, 2835, 0, 0}; + const std::uint16_t magic = 0x4d42; + const std::uint32_t fileBytes = 54 + imageBytes; + std::fwrite(&magic, 2, 1, file); + std::fwrite(&fileBytes, 4, 1, file); + std::fwrite(header + 1, 4, 12, file); + std::fwrite(pixels.data(), 1, pixels.size(), file); + std::fclose(file); } } - auto* scaler = resident && display->pixelFormat == 0x8000000022000000ull ? state->rgbaScaler.get() : state->scaler.get(); - if (!pixels.empty()) state->Upload(pixels); - timing.Mark("pixel_upload"); + // Tests and tools present pixels synchronously; the game path splits the steps (see Driver::Present). + if (!present(width, height, true, pixels)) return; + FinishPresent(); + QueuePresent(); +} + +namespace { + +// APS5_DUMP_SCALE= (default 4) keeps every n-th pixel in each direction, so long runs of 4K +// frames stay small. +std::uint32_t DumpScale() { + static const std::uint32_t scale = [] { + const char* value = std::getenv("APS5_DUMP_SCALE"); + const int parsed = value ? std::atoi(value) : 4; + return static_cast(parsed > 0 ? parsed : 1); + }(); + return scale; +} + +// frame_.bmp: every DumpScale()-th pixel of a BGRA8 image, 32-bit top-down. +void WriteFrameBmp(int index, std::uint32_t fullWidth, std::uint32_t fullHeight, std::span full) { + const auto scale = DumpScale(); + const std::uint32_t width = (fullWidth + scale - 1) / scale; + const std::uint32_t height = (fullHeight + scale - 1) / scale; + std::vector pixels(static_cast(width) * height * 4); + for (std::uint32_t y = 0; y < height; ++y) { + for (std::uint32_t x = 0; x < width; ++x) std::memcpy(pixels.data() + (static_cast(y) * width + x) * 4, full.data() + (static_cast(y) * scale * fullWidth + x * scale) * 4, 4); + } + char name[32]; + std::snprintf(name, sizeof(name), "frame_%03d.bmp", index); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t imageBytes = width * height * 4; + const std::uint32_t header[13] = {0, 0, 54, 40, width, static_cast(-static_cast(height)), 1u | (32u << 16u), 0, imageBytes, 2835, 2835, 0, 0}; + const std::uint16_t magic = 0x4d42; + const std::uint32_t fileBytes = 54 + imageBytes; + std::fwrite(&magic, 2, 1, file); + std::fwrite(&fileBytes, 4, 1, file); + std::fwrite(header + 1, 4, 12, file); + std::fwrite(pixels.data(), 1, pixels.size(), file); + std::fclose(file); + } +} + +// Whether a resident image of the display buffer can be blitted to the swapchain as it is: one 2D +// level of the display's extent in a normalized 4-byte color format (a blit converts components, +// so an integer or float reinterpretation would present wrong values). `filter` gets the best +// filter the format allows. +bool ResidentPresentable(const Graphics::Context& context, const Graphics::StorageTexture& image, const DisplayBuffer& buffer, VkFilter& filter) { + const auto& descriptor = image.Descriptor(); + if (descriptor.width != buffer.width || descriptor.height != buffer.height || descriptor.mipCount != 1 || image.ImageLayers() != 1 || image.ImageDepth() != 1) return false; + if (image.GuestBytes() != DisplayBufferSize(buffer)) return false; + const auto format = Graphics::StorageFormatForGuest(context, descriptor.format); + switch (format) { + case VK_FORMAT_R8G8B8A8_UNORM: case VK_FORMAT_B8G8R8A8_UNORM: case VK_FORMAT_A8B8G8R8_UNORM_PACK32: + case VK_FORMAT_A2B10G10R10_UNORM_PACK32: case VK_FORMAT_A2R10G10B10_UNORM_PACK32: + break; + default: return false; + } + VkFormatProperties properties{}; + context.formatProperties(context.physical, format, &properties); + if ((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT) == 0) return false; + filter = (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT) != 0 ? VK_FILTER_LINEAR : VK_FILTER_NEAREST; + return true; +} + +} + +bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // The display buffer is usually a resident render target whose results are still on the GPU: it + // is blitted from that image, which skips the write-back retile, the 33 MB read of guest memory, + // the upload and the detile dispatch. The deferred write-back stays pending as for any storage + // image. Debug aid: APS5_NO_RESIDENT_PRESENT=1 always goes through guest memory. + static const bool noResidentPresent = std::getenv("APS5_NO_RESIDENT_PRESENT") != nullptr; + static std::uint64_t residentPresents = 0, refreshedPresents = 0, notPending = 0, unsuitable = 0, gpuDumps = 0; + static auto lastReport = std::chrono::steady_clock::now(); + const auto bytes = DisplayBufferSize(buffer); + std::shared_ptr resident; + VkFilter filter = VK_FILTER_LINEAR; + if (!noResidentPresent) { + resident = Graphics::StorageTexture::FindPending(buffer.address, bytes); + if (resident == nullptr) { + ++notPending; + } else if (!ResidentPresentable(graphicsContext(), *resident, buffer, filter)) { + resident.reset(); + ++unsuitable; + } else { + // 64 KiB blocks the CPU wrote since the image last matched guest memory would be shown + // stale (the write-back keeps the CPU's bytes for them): Refresh merges them into the + // image first, as a sampled texture of the image would (see cachedTexture). The uncached + // walk: the collect memo is per worker packet, so on this thread a memoized answer could + // miss a CPU write that landed after a worker's walk of the same range. + GuestMemory::CollectWritesUncached(buffer.address, bytes); + if (!GuestMemory::UnchangedSince(buffer.address, bytes, resident->Generation())) { + resident->Refresh(); + ++refreshedPresents; + } + ++residentPresents; + } + } + // Debug aid: APS5_DUMP_FRAMES= also covers the GPU display path: the presented frame is read + // back by the same submission and written after its fence (see writeFrameDump). + // APS5_NO_GPU_DUMP=1 decodes the tiled guest buffer on the CPU instead, as before. + static const int dumpLimit = [] { + const char* value = std::getenv("APS5_DUMP_FRAMES"); + return value ? std::atoi(value) : 0; + }(); + static const bool cpuDump = std::getenv("APS5_NO_GPU_DUMP") != nullptr; + static int dumped = 0; + bool dumpFrame = false; + if (dumped < dumpLimit) { + if (cpuDump) { + const auto full = ReadDisplayBuffer(buffer); + WriteFrameBmp(dumped++, buffer.width, buffer.height, full); + } else { + state->dumpIndex = dumped++; + dumpFrame = true; + ++gpuDumps; + } + } + if (profile && std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) { + lastReport = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[flip] %llu presents from the resident image (%llu refreshed first), through guest memory: %llu not pending, %llu unsuitable; %llu GPU frame dumps\n", static_cast(residentPresents), static_cast(refreshedPresents), static_cast(notPending), static_cast(unsuitable), static_cast(gpuDumps)); + } + if (!present(buffer.width, buffer.height, true, {}, &buffer, resident, filter, dumpFrame)) { + // A dropped frame (swapchain out of date) keeps the dump numbering contiguous. + if (dumpFrame) --dumped; + return false; + } + return true; +} + +bool VulkanDevice::AcquireImage() { + PerformanceTimer timing("Vulkan.Acquire"); + require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); + require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); + require(!state->presentPending, "the previous presentation was not finished"); + require(!state->imageAcquired, "the previously acquired image was not presented"); auto wait = state->DeviceFunction("vkWaitForFences"); auto reset = state->DeviceFunction("vkResetFences"); const std::array fences{state->acquireFence, state->renderFence}; check(reset(state->device, static_cast(fences.size()), fences.data()), "vkResetFences"); + APS5_LOG_CHARS_OUT_DEBUG("present fences reset"); std::uint32_t index = 0; timing.Mark("fence_reset"); - if (!state->swapchainState.ProcessResult(state->DeviceFunction("vkAcquireNextImageKHR")(state->device, state->swapchain, 5'000'000'000ULL, VK_NULL_HANDLE, state->acquireFence, &index), "vkAcquireNextImageKHR")) return; + // An out-of-date swapchain (the window changed) drops this frame; the next Resize recreates it. + const auto acquired = state->DeviceFunction("vkAcquireNextImageKHR")(state->device, state->swapchain, 5'000'000'000ULL, VK_NULL_HANDLE, state->acquireFence, &index); + if (acquired == VK_ERROR_OUT_OF_DATE_KHR) { + state->extent = {0, 0}; + return false; + } + if (acquired != VK_SUBOPTIMAL_KHR) check(acquired, "vkAcquireNextImageKHR"); timing.Mark("acquire_image"); + APS5_LOG_OUT_DEBUG("vkAcquireNextImageKHR index=%u imageCount=%zu", index, state->images.size()); check(wait(state->device, 1, &state->acquireFence, VK_TRUE, std::numeric_limits::max()), "vkWaitForFences acquire"); timing.Mark("acquire_fence_wait"); + APS5_LOG_OUT_DEBUG("Acquire fence complete index=%u", index); require(index < state->images.size() && index < state->rendered.size(), "acquired image index is out of range"); auto& rendered = state->rendered[index]; if (rendered != VK_NULL_HANDLE) { @@ -620,12 +1117,41 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu VkSemaphoreCreateInfo semaphore{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; check(state->DeviceFunction("vkCreateSemaphore")(state->device, &semaphore, nullptr, &rendered), "vkCreateSemaphore presentation"); } - auto commands = state->clearCommands; timing.Mark("retired_swapchains"); + state->acquiredIndex = index; + state->imageAcquired = true; + return true; +} + +bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display, const std::shared_ptr& resident, VkFilter residentFilter, bool dumpFrame) { + PerformanceTimer timing("Vulkan.Present"); + APS5_LOG_OUT_DEBUG("present begin width=%u height=%u opaque=%u pixels=%zu", width, height, static_cast(opaque), pixels.size()); + require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); + require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); + require(!state->presentPending, "the previous presentation was not finished"); + // The game path acquired the image before taking GpuMutex (Driver::Present); tests, tools and + // APS5_SYNC_FLIP=1 acquire here. + if (!state->imageAcquired && !AcquireImage()) return false; + state->imageAcquired = false; + const auto index = state->acquiredIndex; + require(index < state->images.size() && index < state->rendered.size(), "acquired image index is out of range"); + auto& rendered = state->rendered[index]; + if (display != nullptr && resident == nullptr) { + static_cast(DisplayBufferSize(*display)); + state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget); + } + if (!pixels.empty()) state->Upload(pixels); + // The frame's recorded work (and a refresh of the resident image) must reach the queue before the + // presentation's own submission, which reads the image in queue order. + SubmitRecorded(); + timing.Mark("pixel_upload"); + APS5_LOG_OUT_DEBUG("present source=%s bytes=%zu", pixels.empty() ? "clear" : "pixels", pixels.size()); + auto commands = state->clearCommands; check(state->DeviceFunction("vkResetCommandBuffer")(commands, 0), "vkResetCommandBuffer"); VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; check(state->DeviceFunction("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); + APS5_LOG_OUT_DEBUG("Presentation command buffer begin commands=%p image=%p", reinterpret_cast(commands), reinterpret_cast(state->images[index])); VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; @@ -637,18 +1163,41 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu auto pipelineBarrier = state->DeviceFunction("vkCmdPipelineBarrier"); pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); if (pixels.empty() && display == nullptr) { + APS5_LOG_OUT_DEBUG("Recording swapchain clear opaque=%u image=%p", static_cast(opaque), reinterpret_cast(barrier.image)); VkClearColorValue clear{}; clear.float32[3] = opaque ? 1.0f : 0.0f; state->DeviceFunction("vkCmdClearColorImage")(commands, barrier.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &clear, 1, &barrier.subresourceRange); } else { - require(scaler != nullptr, "presentation scaler is unavailable"); - scaler->EnsureSourceImage(width, height); - if (resident) { - resident->Transition(commands, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); - scaler->RecordImage(commands, resident->Target().Image()); + APS5_LOG_OUT_DEBUG("Recording swapchain scaled blit width=%u height=%u bytes=%zu buffer=%p image=%p", width, height, pixels.size(), reinterpret_cast(state->uploadBuffer), reinterpret_cast(barrier.image)); + require(state->scaler != nullptr, "presentation scaler is unavailable"); + state->scaler->EnsureSourceImage(width, height); + // A resident image is blitted straight to the swapchain; a frame dump goes through the + // scaler's BGRA8 image so the readback has one format. + const bool direct = resident != nullptr && !dumpFrame; + VkImageMemoryBarrier residentBarrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + if (resident != nullptr) { + // The draws and dispatches that produced the image were submitted before this command + // buffer; their writes become visible to the blit here. The layout stays GENERAL, the + // one the storage image is tracked in. + residentBarrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + residentBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + residentBarrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + residentBarrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; + residentBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + residentBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + residentBarrier.image = resident->Image(); + residentBarrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; + pipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &residentBarrier); + if (!direct) state->scaler->RecordBlitInto(commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, residentFilter); } else { - if (display != nullptr) state->colorTransfer->Detile(commands, display->pixelFormat == 0x8000000022000000ull); - scaler->RecordUpload(commands, display != nullptr ? state->colorTransfer->LinearBuffer() : state->uploadBuffer); + if (display != nullptr) { + // Bit 56 selects the 10-bit A2B10G10R10 variant; R8G8B8A8-ordered formats need red and blue + // swapped for the B8G8R8A8 swapchain. + constexpr std::uint64_t tenBitFormat = 0x0100000000000000ull; + const bool tenBit = (display->pixelFormat & tenBitFormat) != 0; + state->colorTransfer->Detile(commands, (display->pixelFormat & ~tenBitFormat) == 0x8000000022000000ull, tenBit); + } + state->scaler->RecordUpload(commands, display != nullptr ? state->colorTransfer->LinearBuffer() : state->uploadBuffer); } VkClearColorValue letterbox{}; letterbox.float32[3] = 1.0f; @@ -663,7 +1212,20 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu letterboxBarrier.image = barrier.image; letterboxBarrier.subresourceRange = barrier.subresourceRange; pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &letterboxBarrier); - scaler->RecordBlit(commands, barrier.image, state->extent.width, state->extent.height); + if (direct) PresentationScaler::RecordBlitFrom(graphicsContext(), commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, width, height, residentFilter, barrier.image, state->extent.width, state->extent.height); + else state->scaler->RecordBlit(commands, barrier.image, state->extent.width, state->extent.height); + if (resident != nullptr) { + // Later batches write the image again (the next frame into this buffer, a refresh): they + // start after the blit has read it. + residentBarrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + residentBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &residentBarrier); + } + if (dumpFrame) { + const auto dumpBytes = static_cast(state->scaler->SourceWidth()) * state->scaler->SourceHeight() * 4; + if (!state->dumpBuffer || state->dumpBuffer->Bytes().size() != dumpBytes) state->dumpBuffer = std::make_unique(graphicsContext(), dumpBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + state->scaler->RecordReadback(commands, state->dumpBuffer->Handle()); + } } barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.dstAccessMask = 0; @@ -671,6 +1233,7 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu barrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); check(state->DeviceFunction("vkEndCommandBuffer")(commands), "vkEndCommandBuffer"); + APS5_LOG_CHARS_OUT_DEBUG("Presentation command buffer recorded"); VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; submit.commandBufferCount = 1; submit.pCommandBuffers = &commands; @@ -679,16 +1242,64 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu timing.Mark("command_record_scale"); check(state->DeviceFunction("vkQueueSubmit")(state->queue, 1, &submit, state->renderFence), "vkQueueSubmit clear"); timing.Mark("queue_submit"); + APS5_LOG_CHARS_OUT_DEBUG("Presentation vkQueueSubmit OK"); + // The image stays alive until FinishPresent has waited for the blit (storage-cache eviction only + // drops the cache's reference). + state->presentKept = resident; + state->presentIndex = index; + state->presentPending = true; + // Only a submitted readback is written after the fence (a failed submit tears the device down). + state->dumpRecorded = dumpFrame; + return true; +} + +void VulkanDevice::FinishPresent() { + if (!state->presentPending) return; + PerformanceTimer timing("Vulkan.PresentWait"); + const auto wait = state->DeviceFunction("vkWaitForFences"); check(wait(state->device, 1, &state->renderFence, VK_TRUE, std::numeric_limits::max()), "vkWaitForFences clear"); timing.Mark("render_fence_wait"); + APS5_LOG_CHARS_OUT_DEBUG("Presentation render fence complete"); + if (state->dumpRecorded) { + state->dumpRecorded = false; + writeFrameDump(); + timing.Mark("frame_dump"); + } +} + +void VulkanDevice::writeFrameDump() { + require(state->dumpBuffer != nullptr && state->scaler != nullptr, "frame dump was not recorded"); + // The readback buffer is reused by the next dump, so its bytes are copied out before the writer + // thread (one at a time) downscales and writes them. + std::vector full(state->dumpBuffer->Bytes().begin(), state->dumpBuffer->Bytes().end()); + if (state->dumpWriter.joinable()) state->dumpWriter.join(); + const auto index = state->dumpIndex; + const auto width = state->scaler->SourceWidth(); + const auto height = state->scaler->SourceHeight(); + static const auto start = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, width, height, std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + state->dumpWriter = std::thread([index, width, height, pixels = std::move(full)] { WriteFrameBmp(index, width, height, pixels); }); +} + +void VulkanDevice::QueuePresent() { + if (!state->presentPending) return; + PerformanceTimer timing("Vulkan.QueuePresent"); + state->presentPending = false; + // The finished submission was the last reader of the resident image. + state->presentKept.reset(); + const auto index = state->presentIndex; + require(index < state->rendered.size(), "presented image index is out of range"); VkPresentInfoKHR present{VK_STRUCTURE_TYPE_PRESENT_INFO_KHR}; present.waitSemaphoreCount = 1; - present.pWaitSemaphores = &rendered; + present.pWaitSemaphores = &state->rendered[index]; present.swapchainCount = 1; present.pSwapchains = &state->swapchain; present.pImageIndices = &index; - state->swapchainState.ProcessResult(state->DeviceFunction("vkQueuePresentKHR")(state->queue, &present), "vkQueuePresentKHR"); + const auto presented = state->DeviceFunction("vkQueuePresentKHR")(state->queue, &present); + if (presented == VK_ERROR_OUT_OF_DATE_KHR || presented == VK_SUBOPTIMAL_KHR) state->extent = {0, 0}; + else check(presented, "vkQueuePresentKHR"); timing.Mark("queue_present"); + APS5_LOG_OUT_DEBUG("vkQueuePresentKHR queued imageIndex=%u", index); } ShaderRecompiler::SpirvTarget VulkanDevice::Target() const { @@ -703,7 +1314,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::Target() const { } Graphics::Context VulkanDevice::graphicsContext() const { - return Graphics::Context{ + auto context = Graphics::Context{ state->device, state->physical, state->queue, @@ -728,40 +1339,160 @@ Graphics::Context VulkanDevice::graphicsContext() const { state->bufferPool, state->textureCache.get(), state->pipelineCache ? state->pipelineCache->Handle() : VK_NULL_HANDLE, - state->renderCache.get(), - state->drawQueue.get(), - state->graphicsPipelines.get(), - state->descriptorCache, - state->samplerCache + state->depthClamp, + state->hostImportAlignment }; -} - -void VulkanDevice::ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - state->drawQueue->Resolve(address, bytes); - state->renderCache->Resolve(address, bytes, writable); + context.recorder = state->recorder.get(); + context.descriptorCache = state->descriptorCache.get(); + context.samplerCache = state->samplerCache.get(); + return context; } void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { - EnqueueDraw(graphics, draw, shaders, snapshots); - WaitIdle(); -} - -void VulkanDevice::EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - const GuestMemory::MemoryAccessScope memoryScope(this, [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - static_cast(context)->ResolveMemory(address, bytes, writable); - }); + APS5_LOG_OUT("VulkanDevice::Draw indices=%u instances=%u indexSize=%u address=0x%llx shaders=%zu colorTarget=%u", draw.indexCount, draw.instanceCount, draw.indexSize, static_cast(draw.indexAddress), shaders.size(), static_cast(graphics.hasColorTarget)); const auto context = graphicsContext(); Graphics::Draw(context, graphics, draw, shaders, snapshots); + APS5_LOG_CHARS_OUT("VulkanDevice::Draw complete"); } -void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); +namespace { + +// The resource cache serves dispatches unless disabled; Revalidate proves a hit by texture identity, +// which needs the texture caches: without them every lookup makes a fresh object, so a hit could +// never validate and would only upload everything twice. +bool ResourceCacheEnabled() { + static const bool noResourceCache = std::getenv("APS5_NO_RESOURCE_CACHE") != nullptr; + static const bool noTextureCache = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + return !noResourceCache && !noTextureCache; +} + +// The content key of a compute stage, naming the device: the cache is process-wide and the driver +// replaces the headless device with the windowed one while workers may still use the old one, so +// the key names the device that built the entry (its descriptor set and pooled buffers belong to it). +ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device) { + auto key = Graphics::ShaderResources::ContentKey(shader); + const auto deviceHandle = reinterpret_cast(device); + key.push_back(static_cast(deviceHandle)); + key.push_back(static_cast(deviceHandle >> 32u)); + return key; +} + +} + +struct PreparedDispatch { + // Stage A done; stage B (Complete) runs in the dispatch under the mutex. Null when the resource + // cache held the key's object at prepare time: the dispatch looks it up and revalidates it. + std::shared_ptr resources; + ResourceCache::Key key; + // APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch. + double prepareMs = 0; + // The stage-A pre-sync (see PrepareDispatch): the newest recorder serial waited for without the + // mutex, whose batches the dispatch reaps under it before stage B (0: nothing waited for). + std::uint64_t presyncSerial = 0; +}; + +namespace { + +// APS5_PROFILE_DRAW: the pre-syncs made and the time they waited (unlocked), as [presync] every 10 s. +struct PresyncCounters { + std::atomic checked{0}; + std::atomic presyncs{0}; + std::atomic waitedUs{0}; + std::atomic lastReport{0}; +}; + +PresyncCounters& Presyncs() { + static PresyncCounters counters; + return counters; +} + +} + +std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots) { + // APS5_LOCKED_BUILD=1: the whole build under the mutex, as before the split. + static const bool lockedBuild = std::getenv("APS5_LOCKED_BUILD") != nullptr; + if (lockedBuild || shader.spirv.size() < 5 || shader.spirv[0] != 0x07230203u) return nullptr; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto start = std::chrono::steady_clock::now(); + const Graphics::CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &shader, 0}; + const auto context = graphicsContext(); + auto prepared = std::make_shared(); + // A cached build (the map locks itself) is revalidated under the mutex by the dispatch, which + // looks it up again by the key made here; the object stays local, only its surfaces matter + // below. Anything inserted between here and the dispatch is simply replaced by this build. + std::shared_ptr cached; + if (ResourceCacheEnabled() && shader.variantId != 0) { + prepared->key = DispatchContentKey(compute, context.device); + cached = state->resourceCache.Find(prepared->key); + } + if (cached == nullptr) prepared->resources = std::make_shared(context, compute, snapshots, true); + // The pre-sync. Stage B's image lookups that read guest memory on the CPU (PresyncSurfaces) + // wait, through the flush hook and under the mutex, for the recorded work writing those + // surfaces: every other worker queues behind that wait. So the wait is made here instead, + // without the mutex: the lock-free pending-write snapshot says whether any recorded batch + // writes a surface; if so the mutex is taken only long enough to learn the newest such batch + // (submitting the open one when it is that batch), the timeline wait runs unlocked, and the + // dispatch reaps those batches at its own lock so the hook then finds nothing pending. Work + // noted in between falls back to the locked wait as before. A cached object's Revalidate + // repeats the same lookups when its stamps fail, so it gets the same pre-sync. Needs timeline + // semaphores. APS5_NO_PRESYNC=1 disables it. + static const bool noPresync = std::getenv("APS5_NO_PRESYNC") != nullptr; + if (!noPresync && CanWaitUnlocked()) { + auto& counters = Presyncs(); + counters.checked.fetch_add(1, std::memory_order_relaxed); + const auto surfaces = cached != nullptr ? cached->PresyncSurfaces() : prepared->resources->PresyncSurfaces(); + bool overlaps = false; + for (const auto& [address, bytes] : surfaces) overlaps = overlaps || Graphics::Recorder::SnapshotWriteOverlaps(address, static_cast(bytes)); + if (overlaps) { + std::uint64_t serial = 0; + { + // No reap here (SubmitAndEpoch's would run completions, which a compute worker must + // not do on queue 0's behalf, see SubmitRecorded): the dispatch reaps after the wait. + std::lock_guard lock(GuestMemory::GpuMutex()); + if (state->recorder) { + bool open = false; + for (const auto& [address, bytes] : surfaces) { + const auto info = state->recorder->DescribePendingWrite(address, static_cast(bytes)); + if (!info.has_value()) continue; + open = open || info->open; + serial = std::max(serial, info->serial); + } + if (open) serial = state->recorder->SubmitAndEpoch(); + } + } + if (serial != 0) { + const auto waitStart = std::chrono::steady_clock::now(); + WaitRecorded(serial); + prepared->presyncSerial = serial; + counters.presyncs.fetch_add(1, std::memory_order_relaxed); + counters.waitedUs.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), std::memory_order_relaxed); + } + } + if (profile) { + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(); + if (nowMs - last >= 10000 && counters.lastReport.compare_exchange_strong(last, nowMs)) std::fprintf(stderr, "[presync] %llu dispatches checked, %llu pre-syncs waited %.0f ms (cumulative)\n", static_cast(counters.checked.load()), static_cast(counters.presyncs.load()), counters.waitedUs.load() / 1000.0); + } + } + if (profile) prepared->prepareMs = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + return prepared; +} + +void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { + static_cast(dispatch(shader, x, y, z, 0, snapshots, programAddress, std::move(prepared))); +} + +VulkanDevice::IndirectOutcome VulkanDevice::DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { + return dispatch(shader, 0, 0, 0, arguments, snapshots, programAddress, std::move(prepared)); +} + +VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { PerformanceTimer timing("Vulkan.Dispatch"); - const GuestMemory::MemoryAccessScope memoryScope(this, [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - static_cast(context)->ResolveMemory(address, bytes, writable); - }); + // Group counts for the trace lines; an indirect dispatch does not know them. + char groupsText[40]; + if (arguments != 0) std::snprintf(groupsText, sizeof(groupsText), "indirect"); + else std::snprintf(groupsText, sizeof(groupsText), "%ux%ux%u", x, y, z); + APS5_LOG_OUT_DEBUG("Dispatch groups=%s spirvWords=%zu bindings=%zu pushConstants=%zu", groupsText, shader.spirv.size(), shader.bindings.size(), shader.pushConstants.size()); if (shader.spirv.size() < 5 || shader.spirv[0] != 0x07230203u) { throw std::runtime_error("Vulkan dispatch: invalid SPIR-V"); } @@ -773,99 +1504,343 @@ void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std const auto pushBytes = Graphics::AssemblePushConstants(shaders); const auto context = graphicsContext(); const auto* limit = state->properties.limits.maxComputeWorkGroupCount; - if (x > limit[0] || y > limit[1] || z > limit[2]) { + if (arguments == 0 && (x > limit[0] || y > limit[1] || z > limit[2])) { throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits"); } - state->drawQueue->Wait(); - timing.Mark("draw_wait"); - const auto destroyModule = state->DeviceFunction("vkDestroyShaderModule"); - const auto destroyLayout = state->DeviceFunction("vkDestroyPipelineLayout"); - const auto destroyPipeline = state->DeviceFunction("vkDestroyPipeline"); - const auto destroyFence = state->DeviceFunction("vkDestroyFence"); - const auto freeCommands = state->DeviceFunction("vkFreeCommandBuffers"); - const auto wait = state->DeviceFunction("vkWaitForFences"); - const auto submit = state->DeviceFunction("vkQueueSubmit"); - VkShaderModule module = VK_NULL_HANDLE; - VkPipelineLayout layout = VK_NULL_HANDLE; - VkPipeline pipeline = VK_NULL_HANDLE; - VkFence fence = VK_NULL_HANDLE; - VkCommandBuffer commands = VK_NULL_HANDLE; - const auto cleanup = [&] { - if (commands != VK_NULL_HANDLE) freeCommands(state->device, state->pool, 1, &commands); - if (fence != VK_NULL_HANDLE) destroyFence(state->device, fence, nullptr); - if (pipeline != VK_NULL_HANDLE) destroyPipeline(state->device, pipeline, nullptr); - if (layout != VK_NULL_HANDLE) destroyLayout(state->device, layout, nullptr); - if (module != VK_NULL_HANDLE) destroyModule(state->device, module, nullptr); + // Pipelines are shared by dispatches of one compiled variant; descriptor set layouts built from the + // same bindings are compatible, so the pipeline layout of the first dispatch serves them all. + const std::uint64_t pipelineKey = shader.variantId != 0 ? (shader.variantId << 1u) | (pushStages != 0 ? 1u : 0u) : 0u; + std::shared_ptr objects; + if (pipelineKey != 0) { + if (const auto found = state->computePipelines.find(pipelineKey); found != state->computePipelines.end()) objects = found->second; + } + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // Debug aid: APS5_SYNC_DISPATCH=1 waits for every dispatch, as before batching. + static const bool syncEachDispatch = std::getenv("APS5_SYNC_DISPATCH") != nullptr; + auto phaseStart = std::chrono::steady_clock::now(); + const auto dispatchStart = phaseStart; + std::string phases; + // Totals per phase across dispatches, reported every 1000 dispatches under APS5_PROFILE_DRAW. + static std::map phaseTotals; + static std::uint64_t profiledDispatches = 0; + // The same phases for indirect dispatches alone, every 10 s on an [indirect] line: what the + // 'indirect' GpuMutex hold ([lock] line) spends its time on inside this function (the label + // record before it is timed in the driver, [labels] line), with the longest hold seen and the + // copied-writer lists the indirect decision scans. All under the mutex, like the rest. + struct IndirectHold { + std::map phaseMs; + std::uint64_t count = 0; + double totalMs = 0; + double maxMs = 0; + std::uint64_t writersScanned = 0; + std::uint64_t drawWritersScanned = 0; + std::size_t maxWriters = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; - try { + static IndirectHold indirectHold; + const auto phase = [&](const char* name) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - phaseStart).count(); + char text[64]; + std::snprintf(text, sizeof(text), " %s=%.0fms", name, ms); + phases += text; + phaseTotals[name] += ms; + if (arguments != 0) indirectHold.phaseMs[name] += ms; + phaseStart = now; + // Slow phases are reported as they finish, so a dispatch that never completes shows where it is. + if (ms > 1000) std::fprintf(stderr, "[dispatch] %s took %.0f ms (%zu words)\n", name, ms, shader.spirv.size()); + }; + if (profile && ++profiledDispatches % 1000 == 0) { + std::string report; + for (const auto& [name, ms] : phaseTotals) report += " " + name + "=" + std::to_string(static_cast(ms / 1000)) + "s"; + std::fprintf(stderr, "[dispatch] %llu dispatches, phase totals:%s\n", static_cast(profiledDispatches), report.c_str()); + } + // A dispatch whose compiled content repeats an earlier one reuses that build's descriptor set + // when it is still valid (see ResourceCache); push constants still come from this dispatch. + static std::uint64_t cacheHits = 0, cacheMisses = 0, cacheInvalidated = 0; + static auto cacheReport = std::chrono::steady_clock::now(); + std::shared_ptr resources; + ResourceCache::Key contentKey; + const bool cacheable = ResourceCacheEnabled() && shader.variantId != 0; + // The batches the pre-sync (PrepareDispatch) waited for are reaped first: their completions + // (CPU write-backs) land before stage B or a cached object's Revalidate reads, and the flush + // hook then finds nothing pending over their surfaces. + if (prepared != nullptr && prepared->presyncSerial != 0) { + // Timed alone: the phase entry exists only when a reap ran, and measures just the reap. + phaseStart = std::chrono::steady_clock::now(); + ReapRecorded(prepared->presyncSerial); + phase("reap"); + } + // The 'resources' phase below, split (APS5_PROFILE_DRAW) into what runs under the mutex: the + // sub-phases are extra rows named "resources: ..." in the [dispatch] totals and the [indirect] + // line, so the hold's biggest part (stage B's image lookups, the whole build of an address-based + // shader) is named, and "resources: hook waits" says how much of the build was the nested flush + // hook waiting for recorded work (a locked wait: it overlaps the other rows). + const auto subPhase = [&](const char* name, double ms) { + if (!profile) return; + phaseTotals[name] += ms; + if (arguments != 0) indirectHold.phaseMs[name] += ms; + }; + const auto subPhaseStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto waitedBefore = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + if (prepared != nullptr && prepared->resources != nullptr) { + // Stage A ran without the mutex (PrepareDispatch); stage B completes the build here. + resources = std::move(prepared->resources); + contentKey = std::move(prepared->key); + // The build's own sub-phase totals before and after: the differences are stage B's parts + // (an address-based build also runs its stage A here, under the mutex: 'A locked'). + const auto before = profile ? resources->Timing() : Graphics::ShaderResources::BuildTiming{}; + resources->Complete(); + if (profile) { + const auto& after = resources->Timing(); + const auto images = after.bindingsMs - before.bindingsMs; + const auto upload = after.uploadMs - before.uploadMs; + const auto descriptors = after.descriptorsMs - before.descriptorsMs; + const auto stageA = after.prepareMs - before.prepareMs; + subPhase("resources: B images", images); + subPhase("resources: B upload", upload); + subPhase("resources: B descriptors", descriptors); + subPhase("resources: B bda+other", std::max(0.0, after.completeMs - before.completeMs - images - upload - descriptors)); + subPhase("resources: A locked (bda)", stageA); + char text[64]; + std::snprintf(text, sizeof(text), " resourcesA=%.0fms", prepared->prepareMs); + phases += text; + phaseTotals["resources A (unlocked)"] += prepared->prepareMs; + } + if (cacheable) { + ++cacheMisses; + if (resources->Reusable()) state->resourceCache.Insert(contentKey, resources); + } + } else if (cacheable) { + // PrepareDispatch made the key already when it found the cached object. + contentKey = prepared != nullptr && !prepared->key.empty() ? std::move(prepared->key) : DispatchContentKey(shaders[0], context.device); + if (auto cached = state->resourceCache.Find(contentKey)) { + if (cached->Revalidate(shaders[0])) { + resources = std::move(cached); + ++cacheHits; + } else { + state->resourceCache.Remove(contentKey); + ++cacheInvalidated; + } + if (profile) subPhase("resources: revalidate", std::chrono::duration(std::chrono::steady_clock::now() - subPhaseStart).count()); + } + } + if (resources == nullptr) { + const auto buildStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + resources = std::make_shared(context, shaders[0], snapshots); + if (profile) subPhase("resources: full build (locked)", std::chrono::duration(std::chrono::steady_clock::now() - buildStart).count()); + if (cacheable) { + ++cacheMisses; + if (resources->Reusable()) state->resourceCache.Insert(contentKey, resources); + } + } + if (profile) subPhase("resources: hook waits", Graphics::Recorder::ThreadWaitedMs() - waitedBefore); + if (profile && std::chrono::steady_clock::now() - cacheReport > std::chrono::seconds(10)) { + cacheReport = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw)\n", static_cast(cacheHits), static_cast(cacheMisses), static_cast(cacheInvalidated), state->resourceCache.Size()); + } + timing.Mark("shader_resources"); + phase("resources"); + if (objects == nullptr) { + objects = std::make_shared(); + objects->device = state->device; + objects->destroyModule = state->DeviceFunction("vkDestroyShaderModule"); + objects->destroyLayout = state->DeviceFunction("vkDestroyPipelineLayout"); + objects->destroyPipeline = state->DeviceFunction("vkDestroyPipeline"); VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; moduleInfo.codeSize = shader.spirv.size() * sizeof(std::uint32_t); moduleInfo.pCode = shader.spirv.data(); - check(state->DeviceFunction("vkCreateShaderModule")(state->device, &moduleInfo, nullptr, &module), "vkCreateShaderModule"); + check(state->DeviceFunction("vkCreateShaderModule")(state->device, &moduleInfo, nullptr, &objects->module), "vkCreateShaderModule"); timing.Mark("validate_shader_module"); - Graphics::ShaderResources resources(context, shaders[0], snapshots); - timing.Mark("shader_resources"); - const auto setLayout = resources.Layout(); + const auto setLayout = resources->Layout(); const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes}; VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; layoutInfo.setLayoutCount = 1; layoutInfo.pSetLayouts = &setLayout; layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; - check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout"); + check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout"); VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; pipelineInfo.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; - pipelineInfo.stage.module = module; + pipelineInfo.stage.module = objects->module; pipelineInfo.stage.pName = "main"; - pipelineInfo.layout = layout; - check(state->DeviceFunction("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateComputePipelines"); + pipelineInfo.layout = objects->layout; + if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words\n", shader.spirv.size()); + check(state->DeviceFunction("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &objects->pipeline), "vkCreateComputePipelines"); timing.Mark("pipeline_create"); - VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; - allocation.commandPool = state->pool; - allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - allocation.commandBufferCount = 1; - check(state->DeviceFunction("vkAllocateCommandBuffers")(state->device, &allocation, &commands), "vkAllocateCommandBuffers"); - VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; - begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - check(state->DeviceFunction("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); - VkMemoryBarrier upload{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - upload.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; - upload.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; - state->DeviceFunction("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &upload, 0, nullptr, 0, nullptr); - state->DeviceFunction("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); - resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, layout); - if (pushStages != 0) { - state->DeviceFunction("vkCmdPushConstants")(commands, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes, pushBytes.data()); - } - state->DeviceFunction("vkCmdDispatch")(commands, x, y, z); - VkMemoryBarrier download{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - download.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - download.dstAccessMask = VK_ACCESS_HOST_READ_BIT; - state->DeviceFunction("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &download, 0, nullptr, 0, nullptr); - check(state->DeviceFunction("vkEndCommandBuffer")(commands), "vkEndCommandBuffer"); - VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; - check(state->DeviceFunction("vkCreateFence")(state->device, &fenceInfo, nullptr, &fence), "vkCreateFence"); - VkSubmitInfo submission{VK_STRUCTURE_TYPE_SUBMIT_INFO}; - submission.commandBufferCount = 1; - submission.pCommandBuffers = &commands; - timing.Mark("command_record"); - check(submit(state->queue, 1, &submission, fence), "vkQueueSubmit"); - timing.Mark("queue_submit"); - const auto result = wait(state->device, 1, &fence, VK_TRUE, std::numeric_limits::max()); - timing.Mark("fence_wait"); - if (result != VK_SUCCESS && result != VK_ERROR_DEVICE_LOST) { - const auto idle = state->DeviceFunction("vkDeviceWaitIdle")(state->device); - check(idle, "vkDeviceWaitIdle after fence failure"); - } - check(result, "vkWaitForFences"); - resources.WriteBack(); - timing.Mark("resources_writeback"); - } catch (...) { - cleanup(); - throw; + if (pipelineKey != 0) state->computePipelines[pipelineKey] = objects; + phase("pipeline"); } - cleanup(); + // Recorded into the device's open batch: the CPU moves on to the next command while the GPU + // works; the batch is waited for where the guest expects results (labels, flips, CPU reads). + auto& recorder = *state->recorder; + IndirectOutcome outcome{0, 0}; + const Graphics::HostImport* argumentImport = nullptr; + if (arguments != 0) { + // Decided here, after the resource build (which may retire imports and wait for recorded + // work) and before anything of this dispatch is recorded, so the CPU fallback's sync cannot + // split the dispatch across two batches and the import cannot be dropped before the record. + // A label over the argument dwords (a completion-deferred one lands by a CPU store the GPU + // read would miss; a GPU one is ordered but rare enough to share the fallback). Only those + // dwords: a global "any completion label pending" test would send most indirect dispatches + // to the CPU, as ~5 such labels are pending per frame. Stamp 0 matches every live entry. + const auto labelPending = [&] { + std::uint32_t labelQueue = 0; + for (std::uint64_t dword = arguments; dword < arguments + 12; dword += 4) { + if (recorder.PendingLabel(dword, 4, 0, labelQueue).has_value()) return true; + } + return false; + }; + if (Graphics::StorageTexture::FlushPending(arguments, 12, nullptr, "indirect dispatch arguments")) { + // As the flush hook does: the stores were only recorded and the CPU is about to read them. + outcome.cpuReason = 1; + Graphics::Recorder::CountSync(2); + recorder.Sync(); + } else if (labelPending() || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); }) + || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); })) { + outcome.cpuReason = 2; + } else if ((argumentImport = Graphics::HostImportFor(context, arguments, 12)) == nullptr) { + // Valid until the next refreshImports, which only runs under GuestMemory::GpuMutex (held + // here, by every Dispatch caller) and Keeps a retired VkBuffer while the recorder is busy. + outcome.cpuReason = 3; + } + if (profile) { + indirectHold.writersScanned += state->copiedWriters->size(); + indirectHold.drawWritersScanned += Graphics::DrawCopiedWriters()->size(); + indirectHold.maxWriters = std::max(indirectHold.maxWriters, state->copiedWriters->size()); + } + phase("decide"); + if (outcome.cpuReason != 0) { + // The read waits for the producing batch through the flush hook, as the driver's resolve + // did for every indirect dispatch before; the counts then go through the direct checks. + const auto readStart = std::chrono::steady_clock::now(); + std::array groups{}; + GuestMemory::Read(arguments, std::as_writable_bytes(std::span(groups)), 4); + outcome.argumentReadMs = std::chrono::duration(std::chrono::steady_clock::now() - readStart).count(); + x = groups[0]; + y = groups[1]; + z = groups[2]; + // Timed before `arguments` is cleared, so the read counts as an indirect phase. + phase("argument read"); + arguments = 0; + std::snprintf(groupsText, sizeof(groupsText), "%ux%ux%u", x, y, z); + if (x > limit[0] || y > limit[1] || z > limit[2]) throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits"); + } + } + // From here on the phases of a CPU-resolved indirect dispatch are charged as direct ones + // (`arguments` is 0); the indirect hold total below still covers the whole call. + const bool indirect = arguments != 0 || outcome.cpuReason != 0; + const auto commands = recorder.Commands(); + recorder.Keep(objects); + recorder.Keep(resources); + if (argumentImport != nullptr) { + // The group counts were stored by earlier recorded work (a dispatch in place, a fill) or the + // host; the indirect read follows all of it. + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT); + } + // Results of earlier recorded work are visible to this dispatch, its own to everything after. + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + state->DeviceFunction("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, objects->pipeline); + resources->Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, objects->layout); + if (pushStages != 0) { + state->DeviceFunction("vkCmdPushConstants")(commands, objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes, pushBytes.data()); + } + const auto gpuTiming = recorder.BeginGpuTiming(programAddress != 0 ? programAddress : shader.variantId); + if (argumentImport != nullptr) state->DeviceFunction("vkCmdDispatchIndirect")(commands, argumentImport->buffer, arguments - argumentImport->base); + else state->DeviceFunction("vkCmdDispatch")(commands, x, y, z); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT); + recorder.EndGpuTiming(gpuTiming); + resources->MarkGpuWrites(recorder); + // Only copied written buffers (and BDA fault checks) need work once the GPU is done; without them + // the batch can signal its labels from the GPU. + if (resources->NeedsCompletion()) { + // Listed for DispatchIndirect until the write-back ran (a non-reusable object, so once). + auto writers = state->copiedWriters; + recorder.OnComplete([resources, writers] { + // Delisted before the write-back: one that fails must not keep indirect dispatches on the CPU. + writers->erase(std::remove(writers->begin(), writers->end(), resources), writers->end()); + resources->WriteBackBuffers(); + }); + // Listed after the registration: a throw there leaves nothing that would pin the CPU path forever. + writers->push_back(resources); + } + timing.Mark("command_record"); + phase("record"); + APS5_LOG_OUT_DEBUG("Dispatch recorded groups=%s", groupsText); + // Address-based shaders pin guest allocations until their write-back, which the completion runs + // when the batch finished (deferred lease release, see Graphics::SyncLeaseWork): a guest thread + // that needs a leased allocation syncs the recorder itself through the registry's pin waiter. + // APS5_SYNC_LEASE_DISPATCH=1 completes such dispatches at once, as before. + if (syncEachDispatch || (resources->HoldsLease() && Graphics::SyncLeaseWork())) { + Graphics::Recorder::CountSync(3); + const auto syncStart = std::chrono::steady_clock::now(); + recorder.Sync(); + phase("sync"); + if (resources->HoldsLease()) Graphics::CountLeaseOutcome(true, 0); + if (profile) { + // APS5_PROFILE_DRAW: the [recorder] line's "address-based" syncs by program, every 10 s, + // so the programs still waiting here are known (under the GpuMutex, like the rest). + struct Waits { std::uint64_t count = 0; double ms = 0; }; + static std::map byProgram; + static std::uint64_t syncs = 0; + static double waitedMs = 0; + static auto lastReport = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - syncStart).count(); + auto& waits = byProgram[programAddress != 0 ? programAddress : shader.variantId]; + ++waits.count; + waits.ms += ms; + ++syncs; + waitedMs += ms; + if (now - lastReport > std::chrono::seconds(10)) { + lastReport = now; + std::vector> hot(byProgram.begin(), byProgram.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.ms > b.second.ms; }); + std::fprintf(stderr, "[address-sync] %llu address-based dispatch syncs waited %.1f s in total; by program (10 s):", static_cast(syncs), waitedMs / 1000); + for (std::size_t i = 0; i < hot.size() && i < 8; ++i) std::fprintf(stderr, " 0x%llx x%llu %.0fms", static_cast(hot[i].first), static_cast(hot[i].second.count), hot[i].second.ms); + std::fprintf(stderr, "\n"); + byProgram.clear(); + } + } + } else if (resources->HoldsLease()) { + // The lease is in the open batch (kept above, nothing submitted since): the pin waiter + // finishes the recorder up to that batch's serial. Prints the [address-sync] leases line. + Graphics::CountLeaseOutcome(false, recorder.Submissions() + 1); + } + // Debug aid: APS5_TRACE_DISPATCH_IO prints every dispatch's resources (after write-back when synced). + static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr; + if (traceIo) std::fprintf(stderr, "[dispatch-io] %s:%s\n", groupsText, resources->Describe().c_str()); + if (profile) { + const auto now = std::chrono::steady_clock::now(); + const auto totalMs = std::chrono::duration(now - dispatchStart).count(); + if (totalMs > 100) std::fprintf(stderr, "[dispatch] %s %zu words:%s\n", groupsText, shader.spirv.size(), phases.c_str()); + if (indirect) { + auto& hold = indirectHold; + ++hold.count; + hold.totalMs += totalMs; + hold.maxMs = std::max(hold.maxMs, totalMs); + if (now - hold.lastReport > std::chrono::seconds(10)) { + hold.lastReport = now; + std::string report; + for (const auto& [name, ms] : hold.phaseMs) { + char text[80]; + std::snprintf(text, sizeof(text), " %s %.0f ms", name.c_str(), ms); + report += text; + } + std::fprintf(stderr, "[indirect] %llu indirect dispatches spent %.0f ms inside the device call (10 s; max %.1f ms), by phase (the 'resources: ...' rows split 'resources'; 'hook waits' overlaps them):%s; copied-writer lists scanned per decision: dispatch avg %.1f (max %zu), draw avg %.1f\n", static_cast(hold.count), hold.totalMs, hold.maxMs, report.c_str(), hold.count != 0 ? static_cast(hold.writersScanned) / hold.count : 0.0, hold.maxWriters, hold.count != 0 ? static_cast(hold.drawWritersScanned) / hold.count : 0.0); + hold.phaseMs.clear(); + hold.count = 0; + hold.totalMs = 0; + hold.maxMs = 0; + hold.writersScanned = 0; + hold.drawWritersScanned = 0; + hold.maxWriters = 0; + } + } + } + return outcome; } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp new file mode 100644 index 0000000000..7e4d7a3815 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp @@ -0,0 +1,181 @@ +#include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp" +#include +#include +#include +#ifdef _WIN32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +namespace AgcDriver { + +#ifdef _WIN32 +namespace { + +// Debug aid: APS5_SAMPLE_WORKER= samples the calling thread every millisecond and, every 20 s, +// writes "module+offset self inclusive" lines to for offline symbolization with nm. +struct Sampler { + HANDLE target = nullptr; + std::string path; + std::mutex mutex; + std::map> counts; + std::uint64_t samples = 0; + + void sample() { + CONTEXT context{}; + context.ContextFlags = CONTEXT_FULL; + if (SuspendThread(target) == static_cast(-1)) return; + const bool captured = GetThreadContext(target, &context) != 0; + std::vector frames; + if (captured) { + // Unwinding reads the target's stack while it is suspended; frames without unwind data + // (JIT/guest code) end the walk. + for (int depth = 0; depth < 12 && context.Rip != 0; ++depth) { + frames.push_back(context.Rip); + DWORD64 imageBase = 0; + auto* entry = RtlLookupFunctionEntry(context.Rip, &imageBase, nullptr); + if (entry == nullptr) break; + PVOID handler = nullptr; + DWORD64 establisher = 0; + RtlVirtualUnwind(UNW_FLAG_NHANDLER, imageBase, context.Rip, entry, &context, &handler, &establisher, nullptr); + } + } + ResumeThread(target); + std::lock_guard lock(mutex); + ++samples; + for (std::size_t i = 0; i < frames.size(); ++i) { + auto& count = counts[frames[i]]; + if (i == 0) ++count.first; + ++count.second; + } + } + + void write() { + std::lock_guard lock(mutex); + std::FILE* file = std::fopen(path.c_str(), "w"); + if (file == nullptr) return; + std::fprintf(file, "# %llu samples\n", static_cast(samples)); + for (const auto& [address, count] : counts) { + HMODULE module = nullptr; + char name[MAX_PATH] = "?"; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(address), &module)) GetModuleFileNameA(module, name, sizeof(name)); + const char* base = std::strrchr(name, '\\'); + std::fprintf(file, "%s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast(address - reinterpret_cast(module)), static_cast(count.first), static_cast(count.second)); + } + std::fclose(file); + } +}; + +// Debug aid: APS5_SAMPLE_THREADS= samples every thread in the process every 2 ms and, every +// 20 s, writes each busy thread's hottest frames ("module+offset self inclusive") to . +struct ProcessSampler { + std::string path; + DWORD self = 0; + std::map threads; + std::uint64_t rounds = 0; + + std::chrono::steady_clock::time_point enumerated{}; + + // The system-wide thread snapshot is slow (tens of ms); the thread list is refreshed every 5 s and + // the rounds in between only sample the known threads. + void enumerate() { + const HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (snapshot == INVALID_HANDLE_VALUE) return; + THREADENTRY32 entry{}; + entry.dwSize = sizeof(entry); + const DWORD process = GetCurrentProcessId(); + for (BOOL more = Thread32First(snapshot, &entry); more; more = Thread32Next(snapshot, &entry)) { + if (entry.th32OwnerProcessID != process || entry.th32ThreadID == self) continue; + auto& thread = threads[entry.th32ThreadID]; + if (thread.target == nullptr) thread.target = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, entry.th32ThreadID); + } + CloseHandle(snapshot); + enumerated = std::chrono::steady_clock::now(); + } + + void sample() { + if (threads.empty() || std::chrono::steady_clock::now() - enumerated > std::chrono::seconds(5)) enumerate(); + for (auto& [id, thread] : threads) { + if (thread.target != nullptr) thread.sample(); + } + ++rounds; + } + + void write() { + std::FILE* file = std::fopen(path.c_str(), "w"); + if (file == nullptr) return; + std::fprintf(file, "# %llu rounds\n", static_cast(rounds)); + for (auto& [id, thread] : threads) { + std::lock_guard lock(thread.mutex); + std::vector>> hot(thread.counts.begin(), thread.counts.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.second > b.second.second; }); + std::uint64_t leaves = 0; + for (const auto& [address, count] : hot) leaves += count.first; + if (leaves == 0) continue; + std::fprintf(file, "thread %lu samples %llu\n", static_cast(id), static_cast(thread.samples)); + for (std::size_t i = 0; i < hot.size() && i < 40; ++i) { + const auto address = hot[i].first; + HMODULE module = nullptr; + char name[MAX_PATH] = "?"; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(address), &module)) GetModuleFileNameA(module, name, sizeof(name)); + const char* base = std::strrchr(name, '\\'); + std::fprintf(file, " %s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast(address - reinterpret_cast(module)), static_cast(hot[i].second.first), static_cast(hot[i].second.second)); + } + } + std::fclose(file); + } +}; + +void StartProcessSampler() { + const char* path = std::getenv("APS5_SAMPLE_THREADS"); + if (path == nullptr) return; + std::thread([path = std::string(path)] { + auto* sampler = new ProcessSampler(); + sampler->path = path; + sampler->self = GetCurrentThreadId(); + auto flushed = std::chrono::steady_clock::now(); + for (;;) { + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + sampler->sample(); + if (std::chrono::steady_clock::now() - flushed > std::chrono::seconds(20)) { + sampler->write(); + flushed = std::chrono::steady_clock::now(); + } + } + }).detach(); +} + +} + +void StartWorkerSampler() { + StartProcessSampler(); + const char* path = std::getenv("APS5_SAMPLE_WORKER"); + if (path == nullptr) return; + auto* sampler = new Sampler(); + sampler->path = path; + DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), &sampler->target, THREAD_ALL_ACCESS, FALSE, 0); + std::thread([sampler] { + auto flushed = std::chrono::steady_clock::now(); + for (;;) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + sampler->sample(); + if (std::chrono::steady_clock::now() - flushed > std::chrono::seconds(20)) { + sampler->write(); + flushed = std::chrono::steady_clock::now(); + } + } + }).detach(); +} +#else +void StartWorkerSampler() {} +#endif + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp index 0163b21034..6e525bf902 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp @@ -12,9 +12,20 @@ public: VkDescriptorBufferInfo Table() const; VkDescriptorBufferInfo Fault() const; void CheckFault() const; + // APS5_PROFILE_DRAW: page tables served from the per-device cache and built anew (cumulative), + // and how many recent tables the cache holds right now. + struct TableCacheStats { + std::uint64_t hits = 0; + std::uint64_t misses = 0; + std::size_t held = 0; + }; + static TableCacheStats TableCacheCounters(); private: - std::unique_ptr table; + // The page table is read-only to the shader, so consecutive builds mapping the same ranges to the + // same device addresses share one buffer while one of them is alive (see the table cache in + // BdaResources.cpp, which refers to it weakly). + std::shared_ptr table; std::unique_ptr fault; std::size_t tableBytes = 0; }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 5e78a981ca..efc1bb176f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -2,10 +2,11 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_BUFFERPOOL_HPP #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" -#include +#include #include #include #include +#include namespace AgcDriver::Graphics { @@ -15,46 +16,80 @@ struct BufferAllocation { void* mapping; VkDeviceAddress address; VkDeviceSize allocationBytes; + // Size the VkBuffer was created with (see BufferPool::Capacity), not the size a user asked for. std::size_t bytes; VkBufferUsageFlags usage; VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; }; +// Released buffer allocations kept for reuse, since creating, binding and mapping one costs tens of +// microseconds and a build makes several. Small requests are served by size class (the next power of +// two), so the many differently sized data and copied-region buffers of consecutive builds hit; +// requests of a MiB and more keep their exact size (the multi-MiB registered-range snapshots repeat +// exactly, and rounding them would waste pinned host memory). Retained allocations are evicted least +// recently used under a byte budget and a slot count, so a burst of small buffers between two large +// builds no longer sweeps the large ones out. Reuse is safe because a Buffer is only released once +// the GPU work using it completed (kept until the batch fence, or after a CommandBatch wait). +// APS5_NO_BUFFER_CLASSES=1 matches exact sizes only, as before; APS5_BUFFER_POOL_SLOTS=64 restores +// the old slot count (the bound applies to each tier below, so twice that many slots in all). +// +// The size classes and the exact-size allocations are retained in two tiers with a budget each: +// the multi-MiB texture scratch buffers that come back from a batch (GPU-direct uploads and +// write-backs of storage images) filled the one budget, so every one returned evicted hundreds of +// the 256-byte copied-region buffers to fit, and those then missed on every build (81869 misses +// and 80850 evictions per run, each a Vulkan create or destroy under the pool mutex, 3.5 s per +// run). APS5_BUFFER_POOL_SHARED=1 keeps one tier as before. class BufferPool { public: explicit BufferPool(const Context& context); ~BufferPool(); BufferPool(const BufferPool&) = delete; BufferPool& operator=(const BufferPool&) = delete; + // The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large. + static std::size_t Capacity(std::size_t bytes); std::optional Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); void Put(const BufferAllocation& allocation) noexcept; private: - static constexpr std::size_t capacity = 1024; - static constexpr std::size_t bucketCount = 256; - static constexpr std::size_t none = capacity; struct Slot { - std::optional allocation; - std::size_t next = none; - std::size_t older = none; - std::size_t newer = none; + BufferAllocation allocation; + std::uint64_t lastUse; }; - static std::size_t bucket(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) noexcept; - BufferAllocation remove(std::size_t index) noexcept; + // One retention tier: its slots, their bytes, the byte budget they are evicted under and its + // counters (APS5_PROFILE_DRAW, reported every 10 s from Take). + struct Tier { + std::vector free; + VkDeviceSize retainedBytes = 0; + VkDeviceSize budget = 0; + std::uint64_t hits = 0; + std::uint64_t misses = 0; + std::uint64_t evictions = 0; + }; + // The tier a buffer of `capacity` is retained in (the large one for everything when shared). + Tier& tierFor(std::size_t capacity); void destroy(const BufferAllocation& allocation) noexcept; + // Moves the tier's least recently used slot to `evicted`; the caller destroys those after + // releasing the mutex, so builds taking buffers on other threads do not wait behind the + // Vulkan destroy calls. Nothing changes when the vector cannot grow. + void evictOldest(Tier& tier, std::vector& evicted); + // The retained-slot bound of each tier (APS5_BUFFER_POOL_SLOTS, default `defaultSlots`), read once. + static std::size_t MaxSlots(); VkDevice device; PFN_vkUnmapMemory unmap; PFN_vkDestroyBuffer destroyBuffer; PFN_vkFreeMemory freeMemory; std::mutex mutex; - std::array slots; - std::array buckets; - std::array freeSlots; - std::size_t freeCount = capacity; - std::size_t oldest = none; - std::size_t newest = none; - VkDeviceSize retainedBytes = 0; + // Not `small`/`large`: (via ) defines `small` as a macro. + Tier smallTier; + Tier largeTier; + std::uint64_t clock = 0; static constexpr VkDeviceSize budget = 512ull * 1024 * 1024; + // The small tier's own budget (512 slots of at most half a MiB each): pinned host memory the + // large tier's budget does not count. + static constexpr VkDeviceSize smallBudget = 64ull * 1024 * 1024; + // Requests of this size and more keep their exact size and go to the large tier. + static constexpr std::size_t classLimit = std::size_t{1} << 20u; + static constexpr std::size_t defaultSlots = 512; }; std::shared_ptr GetBufferPool(const Context& context); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp index 511e41751f..860584f7bf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace AgcDriver::Graphics { @@ -16,10 +17,11 @@ ColorTileMode DecodeColorTileMode(std::uint32_t attrib3); class ColorTargetLayout { public: - ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode); + ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement = 4); std::size_t Bytes() const { return bytes; } - std::size_t LinearBytes() const { return static_cast(width) * height * 4u; } + std::size_t LinearBytes() const { return static_cast(width) * height * elementBytes; } std::size_t Alignment() const { return mode == ColorTileMode::Linear ? 256u : 65536u; } + std::uint32_t BlocksPerRow() const { return pitch / blockWidth; } std::size_t Offset(std::uint32_t x, std::uint32_t y) const; void Detile(std::span source, std::span destination) const; void Tile(std::span source, std::span destination) const; @@ -31,6 +33,12 @@ private: std::uint32_t pitch; ColorTileMode mode; std::size_t bytes; + std::uint32_t elementBytes; + std::uint32_t blockWidth = 1; + std::uint32_t blockHeight = 1; + // The XOR swizzle is linear over GF(2), so a block offset is xOffsets[x] ^ yOffsets[y]. + std::vector xOffsets; + std::vector yOffsets; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index d5c2ec5432..a1c7ab64d0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -19,6 +19,7 @@ class TextureCache; class RenderCache; class DrawQueue; class GraphicsPipelineCache; +class Recorder; class DescriptorCache; class SamplerCache; @@ -59,11 +60,18 @@ struct Context { mutable std::shared_ptr bufferPool; TextureCache* textureCache = nullptr; VkPipelineCache pipelineCache = VK_NULL_HANDLE; + bool depthClamp = false; + // Nonzero when VK_EXT_external_memory_host is enabled: the required host pointer alignment. + VkDeviceSize hostImportAlignment = 0; RenderCache* renderCache = nullptr; DrawQueue* drawQueue = nullptr; GraphicsPipelineCache* graphicsPipelines = nullptr; - mutable std::shared_ptr descriptorCache; - mutable std::shared_ptr samplerCache; + // Batches GPU work across guest commands (see Recorder); null before the device finished setup. + Recorder* recorder = nullptr; + // Per-device caches of the immutable descriptor objects a ShaderResources build needs (set + // layouts, descriptor pools, samplers); null (tests) means every build makes and destroys its own. + DescriptorCache* descriptorCache = nullptr; + SamplerCache* samplerCache = nullptr; template TFunction Function(const char* name) const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp new file mode 100644 index 0000000000..80961435d5 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp @@ -0,0 +1,53 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DCCMETADATA_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DCCMETADATA_HPP + +#ifndef VK_NO_PROTOTYPES +#define VK_NO_PROTOTYPES +#endif +#include +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include +#include +#include + +namespace AgcDriver::Graphics { + +struct Context; + +// Delta color compression keeps one key byte per 256 bytes of a color surface. Surfaces are written +// uncompressed here, so the keys that matter are the fast-clear codes a title writes into the metadata +// (the surface then reads as a constant whatever its texels hold) and "uncompressed", which the driver +// stores after it writes a surface so later reads see the texels. +enum class DccKeys { Uncompressed, Clear0000, Clear0001, Clear1110, Clear1111, ClearRegister, Mixed, Unreadable }; + +const char* DccKeysName(DccKeys keys); +// The keys covering a surface of `surfaceBytes`, when they all agree. +DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes); +bool IsDccClear(DccKeys keys); +// Stores "uncompressed" keys over the surface's metadata on the CPU (a guest memory write: it waits +// for recorded GPU work that writes the keys first). +void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes); +// The same after a write-back recorded under GuestMemory::GpuMutex: when the keys are in host-imported +// memory and a recorder is active, the store is a fill recorded into the open batch (ordered behind +// the title's key-writing kernels like the write-back itself, nothing waits on the CPU) and the range +// reads as uncompressed from ReadDccKeys while that batch is pending; otherwise the CPU store above. +// APS5_CPU_DCC_KEYS=1 always stores on the CPU. +void MarkDccUncompressed(const Context& context, std::uint64_t metaAddress, std::uint64_t surfaceBytes); +// Fills `bytes` with the texel a 0000/0001/1110/1111 clear code stands for ("1" is 1.0 or the integer +// maximum; the alpha channel is the last one in memory when alphaOnMsb, else the first, and 3-channel +// formats have none). False when the format has no encoding here. +bool FillDccClear(VkFormat format, DccKeys keys, bool alphaOnMsb, std::span bytes); + +// Where a color target's metadata puts alpha: the last channel in memory unless the component swap is +// reversed (single-channel formats: only with the alternate reversed swap). +bool DccAlphaOnMsb(VkFormat format, std::uint32_t componentSwap); + +// The keys of a texture's DCC surface when they fast-clear it to a value encodable here, else +// Uncompressed (the texels are read as stored; other keys are reported once). +DccKeys TextureClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes); +// A surface's texels as a read sees them: the guest bytes, or the clear value of fast-cleared keys. +void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp index f535b67fc5..e3016a2aec 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp @@ -3,11 +3,28 @@ #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include +#include namespace AgcDriver::Graphics { void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); +// Recorded draws whose written guest buffers were copied (their results reach guest memory by a CPU +// write-back when their batch completes), listed until that write-back ran. It is the draw +// counterpart of VulkanDevice's State::copiedWriters for dispatches, which VulkanDevice.cpp consults +// in two places: DispatchIndirect (an indirect dispatch whose arguments such a writer produced reads +// them on the CPU, behind the write-back, instead of from the host import on the GPU) and the fill +// HLE (a vkCmdFillBuffer over a range a listed write-back will land on must wait for it, or the +// write-back overwrites the fill). Draws are listed here only under APS5_RECORD_COPIED_DRAWS=1, +// which is safe only once BOTH of those checks also test this list, i.e. each extends its +// `std::any_of(state->copiedWriters ...)` with +// `|| std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(); })`; +// cleaner still, State::copiedWriters can become this list (one registry for both producers), after +// which the switch can turn into APS5_NO_RECORD_COPIED_DRAWS. Until then such draws are recorded and +// waited for at once. Read and written under GuestMemory::GpuMutex, like the device's list. +std::shared_ptr>> DrawCopiedWriters(); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp index 72cbe172e7..63c5724fbe 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp @@ -13,18 +13,15 @@ public: GpuColorTransfer(const GpuColorTransfer&) = delete; GpuColorTransfer& operator=(const GpuColorTransfer&) = delete; void Upload(std::uint64_t address, std::uint32_t width, std::uint32_t height, ColorTileMode mode); - void Detile(VkCommandBuffer commands, bool swapRedBlue = false); + void Detile(VkCommandBuffer commands, bool swapRedBlue = false, bool tenBit = false); void Tile(VkCommandBuffer commands); void WriteBack(std::uint64_t address); - void WriteBackTracked(std::uint64_t address); - bool MatchesGuest(std::uint64_t address); VkBuffer LinearBuffer() const; RenderTarget& Target(const ColorTarget& color, bool blending); private: - void writeBack(std::uint64_t address, bool tracked); void prepare(std::uint32_t width, std::uint32_t height, ColorTileMode mode); - void convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue); + void convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit = false); void release() noexcept; Context context; VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE; @@ -32,10 +29,10 @@ private: VkDescriptorSet descriptorSet = VK_NULL_HANDLE; VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + // Guest bytes in host memory, and the device-local copies the conversion shader works on. std::unique_ptr tiled; - std::unique_ptr linear; - std::unique_ptr readback; - std::unique_ptr upload; + std::unique_ptr tiledDevice; + std::unique_ptr linear; std::uint32_t width = 0; std::uint32_t height = 0; ColorTileMode mode = ColorTileMode::Linear; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp deleted file mode 100644 index 17299f660b..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_GRAPHICSPIPELINECACHE_HPP -#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_GRAPHICSPIPELINECACHE_HPP - -#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include -#include -#include - -namespace AgcDriver::Graphics { - -class GraphicsPipelineCache { -public: - explicit GraphicsPipelineCache(const Context& context) : context(context) {} - std::shared_ptr Get(const State& state, const std::shared_ptr& target, const ShaderResources& resources, std::span shaders); - -private: - struct Entry { - std::string key; - std::shared_ptr target; - std::shared_ptr pipeline; - }; - Context context; - std::list entries; - std::map::iterator> lookup; -}; - -} - -#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index cb5f0011ff..06484200d7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -5,6 +5,8 @@ #include "BdaAbi.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include +#include +#include #include namespace AgcDriver::Graphics { @@ -14,17 +16,118 @@ struct GuestMemorySnapshot { std::span bytes; }; +// A guest allocation imported with VK_EXT_external_memory_host, usable by the GPU in place. +struct HostImport { + std::uint64_t base; + std::uint64_t bytes; + VkBuffer buffer; + VkDeviceMemory memory; + VkDeviceAddress address; + // Identity for the life of this import (see HostImportSerial); 0 until first asked for. + std::uint64_t serial = 0; +}; + +// The host import of the registered allocation containing [address, address + bytes), made on demand +// (alignment and budget permitting), or null. Bytes at `address` are at `address - import->base` in +// the import's buffer. +const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes); +// Whether an existing import covers [address, address + bytes), without reconciling the imports +// with the registry or making one (HostImportFor may take a registry lease): a hint for choices +// made outside the device lock (a sampled texture's path, a dispatch's pre-sync); the path taken +// re-checks with HostImportFor when it binds the import. +bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes); + +// A persistent device copy of one registered range of the main guest image (which cannot be host +// imported); see GuestBufferMemory.cpp. +struct ImageMirror; +class Recorder; + +// Identity of the import serving [address, address + bytes): a serial unique for the life of one +// import, 0 when no import serves the range. An equal serial later means the same VkBuffer still +// backs the range (a re-import gets a new serial), which is what a descriptor set written against it +// needs to stay valid (see the ShaderResources cache). With `reconcile`, imports whose registered +// range changed are dropped first, as an upload does; without it (right after an upload, before the +// work using the import is recorded) the imports are left as they are. +std::uint64_t HostImportSerial(const Context& context, std::uint64_t address, std::size_t bytes, bool reconcile); + +// Identity of the read-only image mirror serving [address, address + bytes) (the counterpart of +// HostImportSerial for main-image ranges, which no import serves): a serial unique for the life of +// one mirror with its top bit set, 0 when no read-only mirror whose registered range still exists +// covers the range. A read-only mirror's bytes and VkBuffer never change, so a descriptor set written +// against it stays valid while the serial is unchanged. +std::uint64_t ImageMirrorSerial(const Context& context, std::uint64_t address, std::size_t bytes); + +// Deferred lease release (see GuestBufferMemory.cpp). An address-based build leases every readable +// registered allocation until its write-back, which runs when its batch completed; the guest's +// unmap/mprotect/free of a leased allocation waits for that through the registry's pin waiter, which +// submits the recorder and finishes its batches up to the newest one recorded with a lease. SyncLeaseWork +// says whether the build's work must instead be synced as soon as it is recorded +// (APS5_SYNC_LEASE_DISPATCH=1, the behaviour before deferral). The driver and the draw path count +// their choice with CountLeaseOutcome right after the lease-holding resources were kept by the open +// batch: `batchSerial` is that batch's serial (Recorder::Submissions() + 1 under the device lock), 0 +// when synced, and the waiter targets the newest of them. Under APS5_PROFILE_DRAW CountLeaseOutcome +// prints the [address-sync] leases line every 10 s from LeaseCounters (cumulative). +bool SyncLeaseWork(); +void CountLeaseOutcome(bool synced, std::uint64_t batchSerial); +struct LeaseStats { + std::uint64_t deferred = 0; + std::uint64_t synced = 0; + // Pin-contention waits made by guest threads (the waiter calls), how many of them finished the + // newest lease batch, how many had to drain the whole recorder instead (no lease batch noted), + // and their total time. + std::uint64_t contentionWaits = 0; + std::uint64_t contentionSyncs = 0; + std::uint64_t contentionDrains = 0; + double contentionMs = 0; +}; +LeaseStats LeaseCounters(); + class GuestBufferMemory { public: explicit GuestBufferMemory(const Context& context); void AcquireRegistered(); + // A guest range bound through a descriptor. Both read live guest memory at upload and bind the + // same way (a storage buffer); AddWritable also notes the range in Writes(), so it gets the + // write-back's reference copy, a write-back, the recorder's pending-write note and the + // direct-write marks. AddReadable is for an element the shader is proved never to store to + // (DescriptorBinding::bufferWritten): the CPU never has to wait for it. void AddWritable(std::uint64_t address, std::size_t bytes); + void AddReadable(std::uint64_t address, std::size_t bytes); void AddSnapshot(const GuestMemorySnapshot& snapshot); + // Upload is the two stages below back to back. UploadPrepare needs no device lock: it merges the + // regions, binds the image mirrors and host imports that already serve them (an import pointer is + // re-checked against the registry epoch later) and copies read-only regions. UploadFinish runs + // under GuestMemory::GpuMutex: it reconciles and makes imports (retiring one hands its buffer to + // the recorder), refreshes writable mirrors and flushes pending results (both may wait for + // recorded work) and copies the regions a descriptor may write, so the window between that copy + // and the recorder's pending-write note stays closed. void Upload(bool addressable); + void UploadPrepare(bool addressable); + void UploadFinish(bool addressable); VkDescriptorBufferInfo Descriptor(std::uint64_t address, std::size_t bytes) const; std::vector AddressRanges() const; void WriteBack(); bool WritesOverlap(std::uint64_t address, std::size_t bytes) const; + // Guest ranges the shader may write through descriptors, as [begin, end). + const std::vector>& Writes() const { return writes; } + // Whether any written range lives in a copied buffer, so a write-back must run once the GPU is done. + bool HasCopiedWrites() const; + // Reports writes into host-imported memory (made by the GPU in place, or copied back into it by + // RecordCopyBacks) to the write tracking now. + void MarkDirectWrites() const; + // Records, into the recorder's open batch, the copy of every written sub-range of a region the + // GPU copied out of a host import (see Region::gpuCopy) back into the import: what the shader + // wrote lands in guest memory by the GPU, ordered after the recorded work, so the region needs + // no CPU write-back (HasCopiedWrites no longer counts it) and the caller notes the ranges as + // pending writes and marks them as MarkDirectWrites does. Called right after the work using the + // regions was recorded (ShaderResources::MarkGpuWrites), under GuestMemory::GpuMutex. A use that + // never calls it (a synchronous draw) stores the staging bytes from the CPU in WriteBack. + void RecordCopyBacks(Recorder& recorder); + // Whether registered allocations are pinned until write-back (address-based shaders). + bool HoldsLease() const { return !lease.empty(); } + // Every uploaded region as [begin, end) when all of them are served by host imports (nothing was + // copied, so the upload can serve a later identical build), else nothing. + std::optional>> DirectRegions() const; private: struct Region { @@ -32,14 +135,66 @@ private: std::uint64_t end; bool writable; std::vector snapshot; - std::unique_ptr buffer; + // Shared so a recorded GPU copy into it (see gpuCopy) can keep it in the batch itself, before + // the caller keeps the whole resources: a throw between the two would otherwise return it to + // the pool under an unsubmitted copy command. + std::shared_ptr buffer; + // Guest bytes as uploaded; write-back only stores bytes the GPU changed. + std::vector uploaded {}; + // Registered allocation that is imported: its bytes are read from live guest memory, not a snapshot. + bool hostBacked = false; + // Set when the region is served by an imported allocation; nothing is copied or written back. + const HostImport* direct = nullptr; + // Set when the region covers uncommitted pages: only the `backed` parts (possibly none) are guest + // memory; the rest reads as zeros and is never stored. + bool sparse = false; + std::vector> backed {}; + // Set when the region is served by an image mirror: nothing is copied; writable mirrors are + // written back by comparing with the mirror's shadow. Kept alive here for recorded work. + std::shared_ptr mirror {}; + // The mirror is a descriptor sub-range one found by UploadPrepare (not a lease mirror), which + // UploadFinish confirms still has its Range before the region binds it. + bool subrangeMirror = false; + // Set by UploadPrepare when UploadFinish still has device-lock work for the region: a copy a + // descriptor writes, a writable mirror's refresh, an import to reconcile or make, or the + // pending-results flush of an import it took. + bool pending = false; + // The region lies inside a host import but is misaligned for binding in place: `buffer` is + // filled by a vkCmdCopyBuffer from the import recorded into the batch (no CPU read, so no + // flush-hook wait), from `copySource` (the import's VkBuffer, alive until the batch + // completed, with the guest address of its first byte). `copiedBack` once RecordCopyBacks + // recorded the written sub-ranges' copies back into the import. + bool gpuCopy = false; + VkBuffer copySource = VK_NULL_HANDLE; + std::uint64_t copySourceBase = 0; + bool copiedBack = false; }; void validate(std::uint64_t address, std::size_t bytes) const; + // The region of a descriptor-bound range (AddWritable/AddReadable), committed pages only. + void addDescriptorRegion(std::uint64_t address, std::size_t bytes); + // Gives a region a buffer of its own with its bytes (guest memory for host-backed and writable + // ranges, plus the write-back's reference copy for a range a descriptor writes; else its snapshot). + void copyRegion(Region& region, bool addressable); + // Whether a region inside a host import that cannot be bound in place is copied by the GPU + // instead of the CPU (see Region::gpuCopy): live guest bytes, not sparse, at most the size + // APS5_GPU_COPY_MAX_KIB allows, and APS5_CPU_COPIES unset. + bool gpuCopyEligible(const Region& region) const; + // Records the import-to-buffer copies of the given gpuCopy regions into the open batch, with + // the barriers that order them after earlier recorded writes and before the shaders reading them. + void recordGpuCopies(std::span copies, bool addressable); Context context; GuestAllocations::Lease lease; + // Import registry epoch when `direct` pointers were taken at acquire time; they are reused while + // no import was destroyed since. + std::uint64_t importsEpoch = 0; std::vector regions; + // Whether `regions` is in ascending address order (true right after AcquireRegistered, whose + // regions follow the registry's order), so AddSnapshot can search instead of scanning. + bool regionsSorted = false; std::vector> writes; + // UploadPrepare ran (regions are frozen); `uploaded` once UploadFinish ran. + bool prepared = false; bool uploaded = false; bool committed = false; }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp index 7dfb9855d2..7d53930438 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp @@ -12,14 +12,32 @@ enum class TextureTileMode { kStandard256B, kStandard4KB, kStandard64KB, - RenderTarget64KB + // 64 KiB XOR swizzles; their address equations live in TextureSwizzleEquations.hpp. + kZ64KBX, + kS64KBX, + kD64KBX, + kR64KBX, + // Upstream's name for SW_64KB_R_X (tile mode 0x1b). + RenderTarget64KB = kR64KBX }; +// The hardware SW_MODE of an XOR swizzle tile mode, or 0 for the modes addressed without an equation table. +constexpr std::uint32_t XorSwizzleMode(TextureTileMode mode) { + switch (mode) { + case TextureTileMode::kZ64KBX: return 24u; + case TextureTileMode::kS64KBX: return 25u; + case TextureTileMode::kD64KBX: return 26u; + case TextureTileMode::kR64KBX: return 27u; + default: return 0u; + } +} + enum class TextureDimension { k1D, k2D, k2DArray, - kCube + kCube, + k3D }; struct GuestTextureResource { @@ -37,6 +55,11 @@ struct GuestTextureResource { std::uint8_t dstSelY; std::uint8_t dstSelZ; std::uint8_t dstSelW; + // Last mip level the view exposes; the surface itself holds mipCount levels. + std::uint32_t lastLevel = 0; + // DCC metadata of a compressed surface, or 0 (see DccMetadata.hpp). + std::uint64_t dccAddress = 0; + bool dccAlphaOnMsb = false; }; GuestTextureResource DecodeTextureResource(std::span words); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp index 87171cff33..a0722e1488 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp @@ -2,31 +2,86 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINE_HPP #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include namespace AgcDriver::Graphics { +struct VertexInputLayout; + +// The attachments of one render pass instance. Work recorded with it references the handle until +// the batch completes, so a recorded draw keeps the object (Recorder::Keep) like its pipeline. +class Framebuffer { +public: + Framebuffer(const Context& context, VkRenderPass renderPass, std::span targets, VkExtent2D extent); + ~Framebuffer(); + Framebuffer(const Framebuffer&) = delete; + Framebuffer& operator=(const Framebuffer&) = delete; + VkFramebuffer Handle() const { return framebuffer; } + // Forgets the handle without destroying it (the device it belongs to is already gone). + void Abandon() noexcept; + +private: + Context context; + VkFramebuffer framebuffer = VK_NULL_HANDLE; +}; + +// The shader modules, layout, render pass and VkPipeline of one draw configuration. Viewport and +// scissor are dynamic state set at Begin, so pipelines are shared by draws that differ only there +// (see CachedPipeline). class Pipeline { public: - Pipeline(const Context& context, const State& state, const RenderTarget* target, const ShaderResources& resources, std::span shaders); + // `state` carries the blend states with the outputs the pixel shader lacks already masked and + // `vertexInput` the layout of the draw's vertex descriptors; the shaders were validated by the + // caller (ValidateShaders). + Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders); ~Pipeline(); Pipeline(const Pipeline&) = delete; Pipeline& operator=(const Pipeline&) = delete; VkPipelineLayout Layout() const; - void Begin(VkCommandBuffer commands, VkExtent2D extent) const; + // The framebuffer of these attachment views. A resident target's view (`owners[i]` set) is stable + // while its StorageTexture lives, so framebuffers made of resident views are kept in the pipeline + // and reused when every owner is still the same object; any other set is built per call. + std::shared_ptr AcquireFramebuffer(std::span targets, std::span> owners, VkExtent2D extent); + // Begins the render pass on the framebuffer, binds the pipeline and sets viewport and scissor. + void Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, VkExtent2D extent, const VkViewport& viewport, const VkRect2D& scissor) const; void PushConstants(VkCommandBuffer commands, std::span shaders) const; + // Forgets the Vulkan objects without destroying them: for entries of a device that is already gone. + void Abandon() noexcept; private: + struct CachedFramebuffer { + std::vector views; + std::vector> owners; + VkExtent2D extent; + std::shared_ptr framebuffer; + }; void release() noexcept; Context context; std::vector _modules; VkPipelineLayout layout = VK_NULL_HANDLE; VkRenderPass renderPass = VK_NULL_HANDLE; - VkFramebuffer framebuffer = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + std::size_t attachments = 0; + std::vector framebuffers; }; +// The pipeline for this configuration, shared across draws and frames: keyed by the stages' variant +// ids (the rect-list tessellation pair by the vertex and fragment ids that generate it), the vertex +// input layout, the descriptor set layout key, push constant stages and every pipeline state field. +// A configuration whose stage has no variant id gets a private pipeline. Entries are LRU-bounded and +// only evicted once no recorded draw holds them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per +// draw as before. +std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders); +// Destroys the cached pipelines of a device; to be called before the device goes away. Without it, +// entries of a gone device are recognised by their buffer pool (made and reset with the device, so +// it tells device instances apart when the loader reuses a VkDevice handle) and forgotten unused. +void ClearCachedPipelines(VkDevice device); +// Device limit checks of the viewport, which is dynamic state and so no longer checked by Pipeline. +void ValidateViewport(const Context& context, const VkViewport& viewport); + void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment); -void ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric); +// Returns the color attachment locations the pixel shader writes. +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp new file mode 100644 index 0000000000..0f37b4497a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -0,0 +1,262 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RECORDER_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RECORDER_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +// Accumulates GPU work across guest commands so the CPU does not wait for each one. Dispatches and the +// copies that feed them record into one open batch; Submit sends it to the queue without waiting and +// Sync waits for every batch, then runs its completion actions (write-backs) in order. Objects handed +// to Keep live until the batch that recorded them completed. Guest memory ranges the recorded work +// will write are noted, so a CPU access to such a range (the GuestMemory flush hook) syncs first. +// Every call happens under GuestMemory::GpuMutex, except WaitSerial and the static lock-free readers +// below. The noted ranges are also published as an immutable snapshot (rebuilt under the mutex at +// every note and after a batch's completions ran) that the flush hook reads without the mutex, so +// accesses overlapping nothing never wait behind device work. +// +// Submissions are numbered (serials) and, when the device has timeline semaphores, every Submit +// signals a timeline semaphore with its serial: a thread can then wait for "everything submitted +// up to serial S" WITHOUT the mutex (WaitSerial), because the monotonic value cannot be reset or +// recycled the way the pooled batch fences are, and afterwards run the completions under the mutex +// (FinishUpTo). Without timeline semaphores callers use the locked Sync as before. +class Recorder { +public: + // `timelineSemaphores`: the device enabled VK_KHR_timeline_semaphore (WaitSerial is usable). + explicit Recorder(const Context& context, bool timelineSemaphores = false); + ~Recorder(); + Recorder(const Recorder&) = delete; + Recorder& operator=(const Recorder&) = delete; + + // The open batch's command buffer, starting a batch when none is open. + VkCommandBuffer Commands(); + bool Recording() const { return open != nullptr; } + bool Idle() const { return open == nullptr && inFlight.empty(); } + // Whether recorded work still has completion actions (write-backs the CPU must see) to run. + bool HasCompletions() const; + // The object lives at least until the batch open now completed; it is then destroyed AFTER the + // finishing thread released GuestMemory::GpuMutex, on a release thread of its own (never under + // a hold, see Recorder.cpp ReleaseDeferredKeeps; the finishing thread destroys it itself when + // the thread's queue is full, APS5_RELEASE_QUEUE_MAX batches, or with APS5_RELEASE_ON_UNLOCK=1; + // APS5_RELEASE_UNDER_LOCK=1 destroys it in the reap as before), so its destructor must need + // neither the mutex nor the recorder nor a particular thread. ~Recorder joins the release + // thread and waits for every release in progress before the device goes. + void Keep(std::shared_ptr object); + void OnComplete(std::function action); + void NotePendingWrite(std::uint64_t address, std::size_t bytes); + // Notes several [begin, end) ranges and publishes the snapshot once (a dispatch writes many buffers). + void NotePendingWrites(std::span> ranges); + bool PendingWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + // Whether the OPEN (unsubmitted) batch writes the range: a wait on such a range must submit it. + bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + // What SyncThrough(address, bytes) would wait for, for the [hooksync] attribution: the newest + // batch whose noted range overlaps the access (the open batch counts first, with the serial it + // will get), whether that batch's fence has already signaled (its GPU work is done, so the + // access's bytes are already final; never for the open batch), that noted range, and how many + // batches the wait finishes. Nullopt: no overlap. + struct PendingWriteInfo { + std::uint64_t serial; + bool open; + bool signaled; + std::uint64_t rangeBegin; + std::uint64_t rangeEnd; + std::size_t batchesToFinish; + }; + std::optional DescribePendingWrite(std::uint64_t address, std::size_t bytes) const; + // Ends and submits the open batch without waiting. + void Submit(); + // Submits the open batch and returns the serial of the newest submitted batch (0 when nothing was + // ever submitted); WaitSerial(serial) then covers all recorded work. + std::uint64_t SubmitAndEpoch(); + bool HasTimeline() const { return timeline != VK_NULL_HANDLE; } + // Waits until every batch up to `serial` completed on the GPU. Called WITHOUT GuestMemory::GpuMutex: + // it touches only the immutable timeline semaphore (the caller keeps the device alive). Requires + // HasTimeline(); the debug switch APS5_NO_TIMELINE makes the device create the recorder without one. + void WaitSerial(std::uint64_t serial); + // Under the mutex: finishes (completions, release) the in-flight batches up to `serial`, from the + // front only; tolerates batches another thread finished meanwhile. Later batches stay in flight. + void FinishUpTo(std::uint64_t serial); + // Submits and waits for every batch, running completions in order. + void Sync(); + // Waits only for the batches up to the newest one that writes the range (submitting the open + // batch when it is that one); later batches stay in flight. Fences of one queue signal in + // submission order, so completions still run in order. Debug aid: APS5_NO_SYNC_THROUGH=1 syncs all. + // `waitUnlocked` (the flush hook): when the caller's GpuMutex acquisition is the outermost one + // on this thread and the device has a timeline, the GPU wait runs WITHOUT the mutex (released + // and retaken here; the caller must hold nothing else that orders after it) and only the + // completions run under it, so other queues do not queue behind a CPU read's wait. Inside a + // batch's completion action no wait is made at all: every batch still in flight was recorded + // after the completing one, and its store is that batch's in-order write (see Recorder.cpp). + void SyncThrough(std::uint64_t address, std::size_t bytes, bool waitUnlocked = false); + // Completes batches that already finished; returns whether nothing is in flight. + bool Reap(); + std::uint64_t Submissions() const { return submissions; } + + // Pending-label table. A GPU label (RELEASE_MEM/WRITE_DATA recorded as a store into the batch) + // is noted per 4-byte dword with the value it will store and a record-order stamp (the driver's + // event serial, taken when the label is recorded). A WAIT_REG_MEM whose submission was received + // (stamped) BEFORE the label was recorded may take the value from the table instead of waiting + // for the GPU: any CPU store the game ordered before that submit call precedes the label in + // program order exactly as on hardware, and later recorded work follows the label in queue + // order. Entries recorded before the submission (the previous frame's label at the same address) + // are never trusted. Entries leave the table with their batch (a memory bound only). + void NoteLabel(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue); + // The 4- or 8-byte value the table holds for `address` when every dword is present with a stamp + // newer than `afterStamp`; `queue` receives the recording queue of the first dword. + std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; + std::size_t PendingLabels() const { return labels.size(); } + // Whether any tracked label dword lies inside [address, address + bytes): a range query for + // large ranges (a fill of megabytes), where a per-dword lookup would not do. + bool PendingLabelIn(std::uint64_t address, std::size_t bytes) const; + // PendingLabel of the active recorder WITHOUT GuestMemory::GpuMutex: the table has a small mutex + // of its own (every mutation holds both), so a WAIT_REG_MEM consults it without queueing behind + // device work. Nothing is done under the table mutex but the lookup (it never takes the GPU mutex). + static std::optional LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue); + // Whether the lock-free pending-write snapshot (open, in-flight and finishing batches) overlaps + // the range: false means no recorded work writes it, so a wait on it has nothing to submit. + static bool SnapshotWriteOverlaps(std::uint64_t address, std::size_t bytes); + // A label behind pending completions (CPU write-backs the label must not precede): stored to + // guest memory by a completion action appended to the open batch, or to the newest in-flight + // batch when none is open (no empty submission), so it runs after every earlier write-back in + // submission order. The range is noted as a pending write of that batch so CPU reads through the + // flush hook still sync, and the label enters the table like a GPU one. + // `storedOnGpu`: the same bytes were also recorded into the open batch (a host import), so the + // completion re-stores them only when a CPU write-back noted by NoteWrittenBack overlapped the + // range since; the plain store of a label the GPU has no view of always runs. An unconditional + // completion store would land on memory the game may have reused by then (a stale label value + // over a fresh command buffer). Debug aid: APS5_LABEL_STORE_ALWAYS=1 stores unconditionally. + void AfterCompletions(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu); + // A CPU store of GPU results into guest memory (GuestBufferMemory::WriteBack): recorded so a + // completion label store can tell whether its bytes were overwritten. + static void NoteWrittenBack(std::uint64_t address, std::size_t bytes); + + // Lock-free readers for the queue workers' packet loops and WAIT_REG_MEM polls (values of the + // active recorder; a torn-down recorder never publishes): + // when the open batch received its first label (nullopt: none pending), so the label's + // submission can be bounded (APS5_LABEL_FLUSH_US) without taking the mutex to look; + static std::optional PendingLabelSince(); + // bumped by every note of a pending write into the open batch, so a poller learns that the + // producer may have recorded the label it waits for; + static std::uint64_t WriteGeneration(); + // labels waiting in completion actions (the CPU must reap their batch for them to land); + static std::uint64_t PendingCompletionLabels(); + // dispatches and draws recorded since the last submit (the driver counts them: CountRecordedWork), + // so a batch is submitted after a bounded amount of work (APS5_BATCH_CAP). + static std::uint64_t RecordedWorkSinceSubmit(); + static void CountRecordedWork(); + + // The recorder of the current device, for code that only has guest addresses (the flush hook) + // and for helpers that must not recycle resources the recorded work still uses. + static Recorder* Active(); + void Activate(); + // APS5_PROFILE_DRAW: syncs by source (0 device idle, 1 CPU access to pending writes, 2 CPU read + // after a recorded store, 3 address-based dispatch, 4 other). Contract: the call announces the + // source for the Sync/SyncThrough this thread makes NEXT, which consumes it (a sync without a + // preceding CountSync counts as "other"); call it right before the sync and never without one, + // or the announcement attributes an unrelated later wait. `site` names the call site for the + // [recorder] "top sync sites" table (a return address, printed as a module offset like the + // [guestmem] callers): a caller that syncs on behalf of others (VulkanDevice::WaitIdle) passes + // its own return address, so the table names the driver site, not the wrapper; without it the + // return address of the Sync/SyncThrough call itself is taken, so an unannounced sync still + // names its real caller. + static void CountSync(int source, const void* site = nullptr); + // Names only the site for the sync or drain this thread makes next (its source stays as + // announced, or as the drain counts it): for a wrapper (VulkanDevice::ReapRecorded) whose + // FinishUpTo counts the drain itself, so a CountSync there would count it twice. + static void AnnounceSyncSite(const void* site); + // APS5_PROFILE_DRAW: the calling thread's fence and timeline waits so far, in milliseconds (0 + // when not profiling): a caller reads it around a span of its own work to learn how much of + // that span waited for the GPU (a resource build's nested flush-hook waits, a draw's). + static double ThreadWaitedMs(); + // APS5_PROFILE_GPU=1: GPU time of recorded work by key (a guest program address), from timestamp + // queries around each timed range; the totals per key are reported every 10 s. Begin returns the + // range index to pass to End, or NoTiming when timing is off or the batch's queries are used up. + static constexpr std::uint32_t NoTiming = 0xffffffffu; + std::uint32_t BeginGpuTiming(std::uint64_t key); + void EndGpuTiming(std::uint32_t index); + +private: + struct Batch { + VkCommandBuffer commands = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + std::vector> kept; + std::vector> completions; + std::vector> writes; + // Submission number (1-based): identifies a batch after its allocation may have been reused. + std::uint64_t serial = 0; + bool submitted = false; + VkQueryPool queries = VK_NULL_HANDLE; + std::vector timedKeys; + // Dword addresses this batch noted in the label table (removed when it finishes). + std::vector labelDwords; + // Labels stored by this batch's completion actions (see AfterCompletions); subtracted from + // the lock-free pending count when the batch finishes, whether or not its completions ran. + std::uint32_t completionLabelCount = 0; + }; + struct LabelEntry { + std::uint32_t value; + std::uint32_t queue; + std::uint64_t stamp; + const Batch* batch; + }; + void readGpuTiming(Batch& batch); + + // The unlocked wait of SyncThrough(waitUnlocked): the target batch (submitting the open one when + // it is the target), the timeline wait with the mutex released, then the completions up to it. + // Returns false when the locked path must run instead (no timeline, nested acquisition, off). + bool syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int source, const void* site); + // `source` is the CountSync source the wait is attributed to. + void finish(std::unique_ptr batch, bool wait, int source); + void release(Batch& batch) noexcept; + static bool overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end); + // Rebuilds the lock-free snapshot of pending writes from open, inFlight and finishing. + void publishPendingWrites() const; + // Appends one range to the open batch; returns whether the snapshot must be rebuilt for it. + bool noteWrite(std::uint64_t address, std::size_t bytes); + // Appends one range to `batch` (open or in flight) and publishes the snapshot if needed. + void noteWriteOn(Batch& batch, std::uint64_t address, std::size_t bytes); + void noteLabelOn(Batch& batch, std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue); + // Whether a write-back noted after `sequence` overlapped [begin, end); true when the ring no + // longer reaches back to `sequence` (conservative: the store runs as before). + bool writtenBackSince(std::uint64_t sequence, std::uint64_t begin, std::uint64_t end); + std::mutex writtenBackMutex; + std::deque> writtenBack; + std::uint64_t writtenBackSequence = 0; + // PendingLabel without the table mutex (the caller holds it, or the GPU mutex). + std::optional lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; + + Context context; + VkSemaphore timeline = VK_NULL_HANDLE; + // Identity for a thread that released the mutex around a wait (syncThroughUnlocked): the + // recorder may have been torn down meanwhile, so it is looked up by this id, not by pointer. + std::uint64_t id = 0; + // Mutated only under both GuestMemory::GpuMutex and the table mutex (Recorder.cpp), so a holder + // of either reads it consistently. + std::unordered_map labels; + std::unique_ptr open; + std::deque> inFlight; + // Batches popped from inFlight whose completions have not run yet: their writes stay in the + // snapshot (a rebuild from inside a completion must not drop them) until finish returns. + std::vector finishing; + std::uint64_t submissions = 0; + // Command buffers and fences of completed batches, reused by later ones (hundreds of batches per + // frame would otherwise allocate and free their objects each time). + std::vector> spare; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp index c205f5bb4d..4d4f226f2f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp @@ -26,13 +26,44 @@ private: VkDeviceMemory memory = VK_NULL_HANDLE; void* mapping = nullptr; std::size_t size; + // The VkBuffer's own size, `size` rounded up to its pool class (see BufferPool::Capacity). + std::size_t capacity; VkDeviceSize allocationBytes = 0; VkBufferUsageFlags usage; VkMemoryPropertyFlags properties; - bool reusable = false; std::shared_ptr cache; }; +// Device-local scratch memory for GPU-side layout conversion. The detiler reads and writes scattered +// elements, which crawls across PCIe, so guest bytes move between host and device buffers with DMA +// copies and are only swizzled in video memory. +class DeviceBuffer { +public: + DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage); + ~DeviceBuffer(); + DeviceBuffer(const DeviceBuffer&) = delete; + DeviceBuffer& operator=(const DeviceBuffer&) = delete; + VkBuffer Handle() const; + std::size_t Size() const; + +private: + void release() noexcept; + Context context; + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + std::size_t size; + std::size_t capacity; + VkDeviceSize allocationBytes = 0; + VkBufferUsageFlags usage; + std::shared_ptr cache; +}; + +// Records a whole-range buffer copy. +void CopyBuffer(const Context& context, VkCommandBuffer commands, VkBuffer source, VkDeviceSize sourceOffset, VkBuffer destination, VkDeviceSize destinationOffset, VkDeviceSize bytes); + +// Records a global memory barrier. +void RecordMemoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess); + class RenderTarget { public: RenderTarget(const Context& context, const ColorTarget& target, bool blending); @@ -60,8 +91,6 @@ public: void SubmitAndWait(); void Submit(); void Wait(); - bool IsComplete(); - void Reset(); private: void release() noexcept; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp index 2e190d67c6..e560d391ac 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp @@ -4,8 +4,11 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp" #include +#include #include +#include #include +#include namespace AgcDriver::Graphics { @@ -21,18 +24,36 @@ public: private: void release() noexcept; - VkDevice device; - PFN_vkDestroySampler destroySampler; + Context context; VkSampler sampler = VK_NULL_HANDLE; }; +// One VkSampler per distinct S# (its 4 words plus the shader's depth-compare use): samplers are +// immutable and DecodeSamplerResource is a pure function of the words, so equal keys mean equal +// samplers and nothing ever invalidates an entry. Holders keep a shared_ptr, so an entry evicted from +// the LRU-capped cache lives on while a descriptor set in flight references it. One cache per device. +// APS5_NO_SAMPLER_CACHE=1 creates a sampler per binding as before. class SamplerCache { public: - std::shared_ptr Get(const Context& context, std::span words, const GuestSamplerResource& descriptor); + explicit SamplerCache(std::size_t capacity = 1024); + SamplerCache(const SamplerCache&) = delete; + SamplerCache& operator=(const SamplerCache&) = delete; + std::shared_ptr Get(const Context& context, std::span words, bool compareEnable); + // APS5_PROFILE_DRAW counters: lookups served by an existing sampler, and samplers created. + std::uint64_t Hits() const { return hits; } + std::uint64_t Misses() const { return misses; } private: + struct Entry { + std::shared_ptr sampler; + std::uint64_t lastUse; + }; std::mutex mutex; - std::map, std::shared_ptr> entries; + std::map, Entry> entries; + std::uint64_t clock = 0; + std::size_t capacity; + std::uint64_t hits = 0; + std::uint64_t misses = 0; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 7736330957..25b3629aa7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -5,36 +5,166 @@ #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp" #include "Recompiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" #include +#include +#include +#include #include +#include +#include +#include #include namespace AgcDriver::Graphics { +class Recorder; + +// The cached storage image of a surface (render targets use it as their resident image); brought up +// to date with guest memory before it is returned. +std::shared_ptr CachedStorageSurface(const Context& context, const GuestTextureResource& resource); + +// Defined in Texture.cpp beside the pending-results registry, for the fast Revalidate below: whether +// a storage image other than `except` has results pending in [address, address + bytes). +bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const StorageTexture* except); + +// Per-device descriptor objects shared by ShaderResources builds: set layouts by their binding list +// (immutable; kept until the device is torn down, which Vulkan allows even for pipeline layouts made +// from them) and a chain of descriptor pools that sets are freed back to, so a build creates and +// destroys no layout or pool of its own. A set lives exactly as long as its ShaderResources, which +// its batch keeps until the GPU completed. The cache locks itself: stage A of a dispatch build +// (ShaderResources::buildPrepare, from VulkanDevice::PrepareDispatch) takes layouts and sets from it +// without GuestMemory::GpuMutex, while other threads free and update sets of the same pools under +// that lock (Vulkan synchronizes the pool for allocate/free and the set for update, so that is +// legal). APS5_NO_LAYOUT_CACHE=1 / APS5_NO_POOL_CACHE=1 restore the per-build objects. +class DescriptorCache { +public: + explicit DescriptorCache(const Context& context); + ~DescriptorCache(); + DescriptorCache(const DescriptorCache&) = delete; + DescriptorCache& operator=(const DescriptorCache&) = delete; + + // The layout for `key` (binding, type, count, stage flags per binding, as ShaderResources builds + // it from `bindings`), created on first use. + VkDescriptorSetLayout Layout(std::span key, std::span bindings); + struct SetAllocation { + VkDescriptorSet set = VK_NULL_HANDLE; + VkDescriptorPool pool = VK_NULL_HANDLE; + }; + // A set of `layout` needing `sizes` descriptors from the pool chain, opening a pool when no pool + // has room; a null set when the needs exceed what one chain pool holds (the caller then makes a + // dedicated pool, as before). + SetAllocation Allocate(VkDescriptorSetLayout layout, std::span sizes); + void Free(const SetAllocation& allocation) noexcept; + // APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened. + struct Stats { + std::uint64_t layoutHits = 0; + std::uint64_t layoutMisses = 0; + std::uint64_t sets = 0; + std::uint64_t pools = 0; + }; + Stats Counters() const; + +private: + Context context; + PFN_vkDestroyDescriptorSetLayout destroyLayout; + PFN_vkDestroyDescriptorPool destroyPool; + PFN_vkFreeDescriptorSets freeSets; + mutable std::mutex mutex; + std::map, VkDescriptorSetLayout> layouts; + std::vector pools; + Stats stats; +}; + class ShaderResources { public: ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes); ShaderResources(const Context& context, std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes, std::span snapshots = {}); ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots = {}); + // Two-stage build for dispatches (see build): with `deferred` the constructor runs stage A only, + // which needs no device lock, and Complete() runs stage B under GuestMemory::GpuMutex; until + // then no other member may be used. `compute` and `snapshots` must outlive Complete(). An + // address-based shader (BDA tables) builds entirely in Complete(): its lease acquisition + // reconciles imports and refreshes mirrors, which needs the lock. + ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred); + void Complete(); + bool Completed() const { return completed; } ~ShaderResources(); ShaderResources(const ShaderResources&) = delete; ShaderResources& operator=(const ShaderResources&) = delete; VkDescriptorSetLayout Layout() const; void Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout) const; void WriteBack(); + // Deferred completion: MarkGpuWrites registers the results the recorded work leaves on the GPU + // (storage images stay there; buffer ranges are noted so CPU reads wait); WriteBackBuffers runs + // once the work completed. + void MarkGpuWrites(Recorder& recorder); + void WriteBackBuffers(); + // Whether WriteBackBuffers has anything the CPU must see (copied written buffers, BDA faults). + bool NeedsCompletion() const { return bda != nullptr || guestMemory.HasCopiedWrites(); } + // Whether a written buffer was copied (its results reach guest memory by the CPU write-back). + bool HasCopiedWrites() const { return guestMemory.HasCopiedWrites(); } + bool HoldsLease() const { return guestMemory.HoldsLease(); } bool WritesOverlap(std::uint64_t address, std::size_t bytes) const { return guestMemory.WritesOverlap(address, bytes); } const std::vector& LayoutKey() const { return layoutKey; } + // Debug aid: each bound guest resource with the fraction of sampled bytes that are nonzero. + std::string Describe() const; + + // Reuse across dispatches of identical content (see the resource cache in VulkanDevice): the + // key is everything a compute build consumes from its compiled shader (variant, and per binding + // kind, role, slot, count, guest descriptor words, written images, depth-compare samplers); push + // constants are not part of it because every dispatch pushes its own. Reusable says whether this + // object can serve a later dispatch with the same key: nothing needs completion work and every + // guest buffer is read in place through a host import. Revalidate brings a reusable object up to + // date for another use, or says it cannot be reused any more (a buffer's import went away, or a + // texture or storage image the set references is no longer the one its memory maps to). + static std::vector ContentKey(const CompiledShader& shader); + bool Reusable() const { return reusable; } + // `shaders` are the stages the object was built from, in build order (a recorded draw's vertex + // and fragment stages, or one compute stage): their bindings are walked like the build did. + bool Revalidate(std::span shaders); + bool Revalidate(const CompiledShader& shader) { return Revalidate(std::span(&shader, 1)); } + // APS5_PROFILE_DRAW: the build's sub-phase times in milliseconds (zero when not profiling), so + // the draw path can total them separately from dispatches. bindingsMs covers the binding plan + // (stage A) and the image lookups (stage B); prepareMs/completeMs are the two stages' totals. + struct BuildTiming { + double bindingsMs = 0; + double uploadMs = 0; + double descriptorsMs = 0; + double prepareMs = 0; + double completeMs = 0; + }; + const BuildTiming& Timing() const { return timing; } + // The image surfaces (address, bytes) whose stage-B lookup reads guest memory on the CPU and so + // would wait, under the device lock, for recorded work writing them: a surface without a host + // import, or a sampled one that stays a CPU snapshot (block compressed, or the storage-view path + // is off). VulkanDevice::PrepareDispatch waits for that work before the lock (the pre-sync). + // Usable right after the deferred constructor, for address-based builds too (it decodes the + // compiled shader's bindings itself when stage A has not run), and on a completed object from + // the resource cache (its Revalidate repeats the same lookups), without the device lock. + std::vector> PresyncSurfaces() const; private: + struct DescribedRange { + const char* kind; + std::uint64_t address; + std::uint64_t bytes; + std::uint32_t width = 0; + std::uint32_t height = 0; + std::uint32_t format = 0; + int tileMode = 0; + std::uint64_t dccAddress = 0; + }; + std::vector describedRanges; struct Allocation { std::uint64_t address; std::size_t size; bool guest; std::unique_ptr buffer; ShaderRecompiler::DescriptorRole role = ShaderRecompiler::DescriptorRole::ShaderData; + // Guest buffers: whether the shader may store to the range (see addGuestBuffer). + bool written = true; }; struct Binding { @@ -43,13 +173,92 @@ private: std::vector imageAllocations; }; + // A host-imported buffer region the set reads in place, with its import's identity at build time. + struct DirectRegion { + std::uint64_t begin; + std::uint64_t end; + std::uint64_t serial; + }; + + // The build is two stages. A (buildPrepare, no device lock): the binding plan with its layout + // entries, samplers, data buffers, the upload's prepare stage (imports already serving the + // regions, read-only copies), the set layout and the descriptor set. B (buildComplete, under + // GuestMemory::GpuMutex): the texture and storage image lookups (they refresh, upload and flush + // through the recorder), the rest of the upload, the BDA objects, the descriptor writes and the + // reusability record. build runs both. void build(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); - std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); + void buildPrepare(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); + void buildComplete(); + // APS5_PROFILE_DRAW: closes the current sub-phase of the build into the [resources] totals. + double phase(const char* name); + // `written` is the element's DescriptorBinding::bufferWritten: a read-only element binds the + // same way but is left out of the write set (no write-back, no pending-write note). + std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written); std::size_t addDataBuffer(std::span words); - void addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags, std::vector& bindings); + // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache + // locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding). + void addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags); + // Stage A: one record per planned image element (ImageRecord), in plan order: the decoded + // descriptor and its surface size (computed once for the build), the write watch walked so stage + // B's collects are memo hits (APS5_NO_PRECOLLECT=1 skips the pass entirely), and for a sampled + // element the cache entry it maps to, taken now so stage B needs only the fastRevalidate + // predicate under the lock (APS5_NO_STAGE_A_IMAGES=1 leaves every element to the full lookup). + // Whether it ran. + bool precollectImages(); + struct ImageRecord { + bool sampled = false; + // The descriptor decoded and its surface described; false leaves the element to stage B's + // own decode (which reports the error). + bool decoded = false; + std::array words{}; + GuestTextureResource resource{}; + std::uint64_t guestBytes = 0; + VkComponentMapping components{}; + // Sampled elements: the keys read in stage A and the collect made BEFORE stage B's checks + // (the generation the entry moves to when they pass). + DccKeys keys = DccKeys::Uncompressed; + std::uint64_t generation = 0; + // The cache entry's object, storage source, keys and generation as of stage A (texture null: + // no entry, the full lookup makes one). + std::shared_ptr texture; + std::shared_ptr source; + DccKeys entryKeys = DccKeys::Uncompressed; + std::uint64_t entryGeneration = 0; + }; + std::vector imageRecords; + // The next record resolveImageBinding consumes (records follow the deferredImages order). + std::size_t nextImageRecord = 0; + // Stage B: the record's texture when the fastRevalidate predicate proves it current under the + // lock and the cache still holds it; null sends the element to cachedTexture. + std::shared_ptr fastTexture(const ImageRecord& record); + void resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item); + void forgetDeferredInputs(); void release() noexcept; void prepareAddressBindings(std::span shaders, std::span snapshots); VkDescriptorBufferInfo descriptor(const Allocation& allocation) const; + void noteReusable(); + void reportDescriptorCaches() const; + // What a sampled texture was proved current against when the build (or the last full Revalidate) + // looked it up, so the next Revalidate can repeat the proof from write stamps and the DCC keys + // alone instead of repeating the lookup (see fastRevalidate). One record per textures[] entry; an + // element without a record always takes the full lookup. Storage images need no record: their + // proof comes from the image itself (its own surface and generation). + struct ValidatedSurface { + GuestTextureResource resource{}; + std::uint64_t bytes = 0; + // Keys read at the lookup (a snapshot texture holds its clear texels while they are unchanged). + DccKeys keys = DccKeys::Uncompressed; + // Snapshot textures: write generation the content was known current at, always from a + // collect issued before the checks that proved it. + std::uint64_t generation = 0; + // The collect of the check in progress, moved into `generation` once every element passed. + std::uint64_t collected = 0; + // Storage-sourced textures: the image the view follows (kept alive by the texture). + const StorageTexture* source = nullptr; + bool valid = false; + }; + void captureValidation(); + bool fastRevalidate(); Context context; std::vector layoutKey; GuestBufferMemory guestMemory; @@ -57,13 +266,93 @@ private: bool usesBda = false; bool usesFaultBuffer = false; VkDescriptorSetLayout _layout = VK_NULL_HANDLE; + // Whether the layout is this object's own (no cache) and destroyed with it. + bool ownsLayout = false; VkDescriptorSet _set = VK_NULL_HANDLE; - std::unique_ptr descriptors; + // Dedicated pool of this object's set (no cache, or a set too large for a cache pool). + VkDescriptorPool pool = VK_NULL_HANDLE; + // The cache pool the set was allocated from, freed back to it on release. + VkDescriptorPool cachePool = VK_NULL_HANDLE; std::vector allocations; + // Guest buffer elements bound read-only: each use of this object skips that many pending-write + // notes (counted in MarkGpuWrites for the [buffers] line). + std::size_t readOnlyBuffers = 0; std::vector> textures; + std::vector> storageTextures; + std::vector storageMips; + std::vector storageWritten; std::vector> samplers; + bool reusable = false; + std::vector directRegions; + std::vector validatedTextures; + BuildTiming timing; + // Build state carried from stage A to stage B: the bindings in plan order, the image bindings + // still to look up (index into `bindings`; the DescriptorBinding lives in the compiled shader), + // the descriptor counts the set was sized for, and the compute stage of a deferred build. + std::vector bindings; + struct DeferredImages { + const ShaderRecompiler::DescriptorBinding* binding; + std::size_t index; + }; + std::vector deferredImages; + std::uint64_t storageBuffers = 0; + std::uint32_t plannedSampledImages = 0; + std::uint32_t plannedStorageImages = 0; + // The compute constructor's shader and captured regions: the caller's objects, valid only until + // the build (Complete() for a deferred one) is done, and reset then (forgetDeferredInputs). + CompiledShader deferredCompute{}; + std::span deferredSnapshots; + // The deferred build is address-based: everything runs in Complete(). + bool lockedBuild = false; + // Stage A runs without the device lock (a deferred, non-address-based build). + bool unlockedPrepare = false; + bool completed = false; + std::chrono::steady_clock::time_point phaseStart; }; +// Built ShaderResources by the content the build consumed (see ShaderResources::ContentKey), so a +// dispatch or recorded draw repeating an earlier one binds the earlier descriptor set instead of +// building one (import lookups, sampler and descriptor objects, guest buffer uploads). Only reusable +// objects (nothing to complete, every buffer read in place) are kept, and an entry serves a later +// use only after ShaderResources::Revalidate brought it up to date. Entries hold their object alive; +// the recorder keeps it as well while its batch runs. One LRU bound covers dispatch and draw entries +// together (APS5_RESOURCE_CACHE_ENTRIES, default 1024); the map has its own mutex and locks itself: +// VulkanDevice::PrepareDispatch probes it without GuestMemory::GpuMutex (a hit there is revalidated +// under the lock by the dispatch; inserts and Revalidate stay under it). Compute keys start with the +// stage, draw keys with a marker word, so the two never meet, and both name the device handle: the driver replaces +// the headless device with the windowed one while workers may still hold the old one, so a replaced +// device's entries (whose descriptor sets and pooled buffers belong to it) are unreachable by key +// and dropped by that device's Clear(). APS5_NO_RESOURCE_CACHE=1 (dispatches) and +// APS5_NO_DRAW_RESOURCE_CACHE=1 (draws) build every object as before; the device clears the cache at +// teardown before its descriptor caches go. +class ResourceCache { +public: + using Key = std::vector; + std::shared_ptr Find(const Key& key); + void Insert(const Key& key, std::shared_ptr resources); + void Remove(const Key& key); + void Clear(); + std::size_t Size() const; + +private: + void erase(const Key& key); + void noteMiss(const Key& key); + struct KeyHash { + std::size_t operator()(const Key& key) const noexcept { + std::uint64_t hash = 14695981039346656037ull; + for (const auto word : key) hash = (hash ^ word) * 1099511628211ull; + return static_cast(hash); + } + }; + mutable std::mutex mutex; + std::list>> entries; + std::unordered_map index; +}; + +// The process-wide cache the device and the draw path share (keys name the device; a device clears +// it when it goes). Never destroyed: entries belong to the device, not to static teardown. +ResourceCache& SharedResourceCache(); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp index 590dd23f6c..b20568e1f2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" +#include #include #include #include @@ -34,17 +35,25 @@ struct ColorTarget { std::size_t bytes; std::uint8_t componentMapping; ColorTileMode tileMode = ColorTileMode::Linear; + std::uint32_t elementBytes = 4; + // DCC metadata of a compressed target (CB_COLOR_INFO DCC_ENABLE), or 0 (see DccMetadata.hpp). + std::uint64_t dccAddress = 0; + bool dccAlphaOnMsb = false; }; struct State { ShaderStages stages; + // MRT slot 0; `colors`/`blends` hold every written slot, attachment i being slot i. ColorTarget color; + std::vector colors; + std::vector blends; bool hasColorTarget; bool rectList = false; VkExtent2D renderExtent; VkPrimitiveTopology topology; VkViewport viewport; bool negativeOneToOne; + bool depthClamp = false; VkRect2D scissor; VkCullModeFlags cullMode; VkFrontFace frontFace; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index aca05fbf38..b1d9f9353d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -2,35 +2,174 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURE_HPP #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include +#include +#include +#include namespace AgcDriver::Graphics { class ResidentColor; class CommandBatch; +class StorageTexture; + +// The Vulkan format storage images of a guest format use (sRGB formats store as their UNORM form). +// Throws when the format has no storage form; the query is cached per format (a lookup cost a +// vkGetPhysicalDeviceFormatProperties call before). +VkFormat StorageFormatForGuest(const Context& context, std::uint32_t guestFormat); +// Whether StorageFormatForGuest would succeed, without throwing (unknown guest formats included). +bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat); + +// A sampled texture's own VkImage with its memory, shared with the recorder while a recorded upload +// still writes it (see the snapshot constructor), so the texture may go before the batch completes. +struct OwnedImage { + OwnedImage(const Context& context, VkImage image, VkDeviceMemory memory) : context(context), image(image), memory(memory) {} + OwnedImage(const OwnedImage&) = delete; + OwnedImage& operator=(const OwnedImage&) = delete; + ~OwnedImage() { + if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); + if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); + } + Context context; + VkImage image; + VkDeviceMemory memory; +}; class Texture { public: Texture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, VkComponentMapping components, std::span snapshot); Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components); + // A view of a storage image's own VkImage: the sampled texture follows the image's content, so a + // compute pass writing it and the next pass sampling it share one image and copy nothing. + // CanCopyFrom says whether the two descriptors address the same surface compatibly. + Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components); + static bool CanCopyFrom(const StorageTexture& source, const GuestTextureResource& descriptor); ~Texture(); Texture(const Texture&) = delete; Texture& operator=(const Texture&) = delete; VkImageView View() const; + // The layout the image is kept in while sampled. + VkImageLayout Layout() const { return layout; } VkDeviceSize AllocationBytes() const { return allocationBytes; } + // Whether this texture is a view of a storage image (no snapshot of its own). + bool ViewsStorageImage() const { return storageSource != nullptr; } private: void release() noexcept; + // Held by value: cached textures outlive the Context of the draw that created them. Context context; + // The snapshot constructor's image (null for a view of a storage image); `owned` frees it. + VkImage image = VK_NULL_HANDLE; + std::shared_ptr owned; + VkImageView view = VK_NULL_HANDLE; + VkImageLayout layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkDeviceSize allocationBytes = 0; + std::shared_ptr source; + std::shared_ptr storageSource; + std::unique_ptr upload; +}; + +// A guest texture a shader writes through a storage image. It is uploaded like a sampled texture; +// after the GPU work completes its results are stored to guest memory (retiled, changed bytes only), +// either at once (WriteBack) or deferred: MarkDirty keeps them on the GPU until something reads that +// memory (FlushPending, through the GuestMemory flush hook), the image is refreshed after a CPU write, +// or it leaves the cache. Dispatches reusing the image meanwhile skip the round trip entirely. +class StorageTexture : public std::enable_shared_from_this { +public: + StorageTexture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, std::uint32_t mipLevel); + ~StorageTexture(); + StorageTexture(const StorageTexture&) = delete; + StorageTexture& operator=(const StorageTexture&) = delete; + + VkImageView View() const; + // The image holds every mip of the surface; one storage view per written mip is made on demand, + // so successive mip writes of a chain share one image and one write-back. + VkImageView View(std::uint32_t mip); + // Render targets live in the same images: draws attach mip 0 through a view of the color + // buffer's format and mark the image dirty like a storage write. + bool Attachable() const { return attachable; } + VkImageView AttachmentView(VkFormat format); + void WriteBack(); + // Deferred write-back (APS5_EAGER_WRITEBACK=1 stores at once instead). + void MarkDirty(); + void Flush(); + // Stores every pending image overlapping the range, except `except`; returns whether any was. + // Stores into host-imported memory are recorded (not waited for): a CPU reader syncs afterwards. + static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access"); + static void FlushAllPending(const char* reason); + // The pending image whose surface starts at `address` and covers at least `bytes` (a mip chain + // contains a descriptor of its first mips), if any; the caller decides whether the geometry fits + // (Texture::CanCopyFrom, ResidentPresentable). + static std::shared_ptr FindPending(std::uint64_t address, std::uint64_t bytes); + VkImage Image() const { return image; } + const GuestTextureResource& Descriptor() const { return descriptor; } + std::uint32_t ImageLayers() const { return geometry.imageLayers; } + std::uint32_t ImageDepth() const { return geometry.imageDepth; } + // Content version: advances when the image is re-uploaded or a shader wrote it. Together with + // Generation (the write generation guest memory was last known to match the content at) it + // validates textures copied from this image. + std::uint64_t Version() const { return version; } + std::uint64_t Generation() const { return generation; } + // Brings the image up to date with guest memory before another use; returns whether its content + // was still current (nothing uploaded). + // Keeps the image current with guest memory (see GuestMemory::CollectWrites). + bool Refresh(); + std::uint64_t GuestBytes() const; + +private: + std::vector CopyRegions() const; + void upload(); + // Stores the image to guest memory; with `skipWrittenSince` set, 64 KiB blocks the CPU wrote since + // that write generation keep the CPU's bytes. + void writeBack(std::uint64_t skipWrittenSince); + // Pending images whose surface shares a 64 KiB write-stamp block with this one without + // overlapping it, and whose memory is unchanged since their generation (see writeBack), with + // the generation seen at the check. + struct Adjacent { + std::shared_ptr texture; + std::uint64_t generation; + }; + std::vector adjacentPendingUnchanged() const; + // Advances the entries still at their seen generation to `now`, the generation this write-back's + // stamps of blocks firstBlock..lastBlock (64 KiB) are below, unless the neighbour's other blocks + // were stamped since the check. + static void advanceAdjacent(const std::vector& adjacent, std::uint64_t now, std::uint64_t firstBlock, std::uint64_t lastBlock); + bool overlaps(std::uint64_t address, std::size_t bytes) const; + VkImageView createView(std::uint32_t mip) const; + void release() noexcept; + + Context context; + TextureDetiler& detiler; + GuestTextureResource descriptor; + std::vector mips; + std::uint32_t arrayLayers = 1; + std::uint64_t guestBytes = 0; + std::uint64_t sliceLinearBytes = 0; + SurfaceGeometry geometry; + std::vector original; + // DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value. + DccKeys uploadedKeys = DccKeys::Uncompressed; + // Write generation `original` is known current at. + std::uint64_t generation = 0; VkImage image = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; VkImageView view = VK_NULL_HANDLE; - VkDeviceSize allocationBytes = 0; - std::shared_ptr source; - std::unique_ptr upload; + std::uint32_t defaultMip = 0; + std::map extraViews; + bool attachable = false; + std::map attachmentViews; + VkFormat storageFormat = VK_FORMAT_UNDEFINED; + // Results are on the GPU only (guarded by the pending-write registry lock). + bool dirty = false; + std::uint64_t version = 0; + // `original` holds the guest bytes; false after a GPU-side clear, which never read them. + bool originalValid = true; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp index f6782186e4..9436f2597f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTUREDETILER_HPP #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTUREDETILER_HPP +#include #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" @@ -17,11 +18,14 @@ namespace AgcDriver::Graphics { TextureDetiler(const TextureDetiler&) = delete; TextureDetiler& operator=(const TextureDetiler&) = delete; - void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, std::uint32_t arrayLayer); + // Detiles `source` (tiled) into `destination` (linear), or with `retile` writes linear `source` + // into tiled `destination`; offsets always refer to the respective buffers. + void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false); + // Recycles the descriptor sets of the previous batch; call before recording a new command batch. void BeginBatch(); private: - VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes); + VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick); void release() noexcept; VkDescriptorSet allocateSet(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp new file mode 100644 index 0000000000..99dc592e8b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp @@ -0,0 +1,63 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURESWIZZLEEQUATIONS_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURESWIZZLEEQUATIONS_HPP + +#include +#include + +namespace AgcDriver::Graphics { + +// Address equations of the 64 KiB XOR swizzle modes (SW_64KB_Z_X, S_X, D_X, R_X) for the PS5 GPU, a +// Navi1x-class part with 16 pipes. Generated from AMD addrlib's GFX10 non-RB+ swizzle pattern tables +// (the 32 bpp R_X entry reproduces the scanout layout in ColorTargetLayout). +// Each entry gives, for byte-address bit 0..15 inside a block, the element-coordinate bits XORed into it: +// bits 0-11 select x bits, bits 12-23 select y bits and bits 24-31 select slice (or depth) bits. +struct TextureSwizzleEquation { + std::uint32_t swizzleMode; + std::uint32_t elementBytes; + std::array bits; +}; + +inline constexpr TextureSwizzleEquation kTextureSwizzleEquations[] = { + {24u, 1u, {0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x00000004u, 0x00004000u, 0x00000008u, 0x00010000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00040000u, 0x00000040u, 0x00080000u, 0x00000080u}}, + {24u, 2u, {0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x00000004u, 0x00004000u, 0x00000008u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00010000u, 0x00000040u, 0x00040000u, 0x00000080u}}, + {24u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x00000004u, 0x00004000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00008000u, 0x00000010u, 0x00040000u, 0x00000040u}}, + {24u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x00000004u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00004000u, 0x00000008u, 0x00010000u, 0x00000040u}}, + {24u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00004000u, 0x00000004u, 0x00008000u, 0x00000010u}}, + {25u, 1u, {0x00000001u, 0x00000002u, 0x00000004u, 0x00000008u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00008000u, 0x08010080u, 0x04080010u, 0x02020040u, 0x01040020u, 0x00040000u, 0x00000040u, 0x00080000u, 0x00000080u}}, + {25u, 2u, {0x00000000u, 0x00000001u, 0x00000002u, 0x00000004u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000008u, 0x08008080u, 0x04040010u, 0x02010040u, 0x01020020u, 0x00020000u, 0x00000040u, 0x00040000u, 0x00000080u}}, + {25u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000002u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000004u, 0x08008040u, 0x04040008u, 0x02010020u, 0x01020010u, 0x00020000u, 0x00000020u, 0x00040000u, 0x00000040u}}, + {25u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00002000u, 0x00000002u, 0x00000004u, 0x08004040u, 0x04020008u, 0x02008020u, 0x01010010u, 0x00010000u, 0x00000020u, 0x00020000u, 0x00000040u}}, + {25u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001000u, 0x00002000u, 0x00000001u, 0x00000002u, 0x08004020u, 0x04020004u, 0x02008010u, 0x01010008u, 0x00010000u, 0x00000010u, 0x00020000u, 0x00000020u}}, + {26u, 1u, {0x00000001u, 0x00000002u, 0x00000004u, 0x00002000u, 0x00001000u, 0x00004000u, 0x00000008u, 0x00008000u, 0x08010080u, 0x04080010u, 0x02020040u, 0x01040020u, 0x00040000u, 0x00000040u, 0x00080000u, 0x00000080u}}, + {26u, 2u, {0x00000000u, 0x00000001u, 0x00000002u, 0x00000004u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000008u, 0x08008080u, 0x04040010u, 0x02010040u, 0x01020020u, 0x00020000u, 0x00000040u, 0x00040000u, 0x00000080u}}, + {26u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000002u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000004u, 0x08008040u, 0x04040008u, 0x02010020u, 0x01020010u, 0x00020000u, 0x00000020u, 0x00040000u, 0x00000040u}}, + {26u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00000004u, 0x00002000u, 0x08004040u, 0x04020008u, 0x02008020u, 0x01010010u, 0x00010000u, 0x00000020u, 0x00020000u, 0x00000040u}}, + {26u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x08004020u, 0x04020004u, 0x02008010u, 0x01010008u, 0x00010000u, 0x00000010u, 0x00020000u, 0x00000020u}}, + {27u, 1u, {0x00000001u, 0x00000002u, 0x00000004u, 0x00002000u, 0x00001000u, 0x00004000u, 0x00000008u, 0x00010000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00040000u, 0x00000040u, 0x00080000u, 0x00000080u}}, + {27u, 2u, {0x00000000u, 0x00000001u, 0x00000002u, 0x00000004u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000008u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00010000u, 0x00000040u, 0x00040000u, 0x00000080u}}, + {27u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000002u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000004u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00008000u, 0x00000010u, 0x00040000u, 0x00000040u}}, + {27u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00000004u, 0x00002000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00004000u, 0x00000008u, 0x00010000u, 0x00000040u}}, + {27u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00004000u, 0x00000004u, 0x00008000u, 0x00000010u}}, + // Thick (3D) standard swizzles SW_4KB_S and SW_64KB_S use the S3 patterns, keyed 0x100 | SW_MODE. + {0x105u, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x105u, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x105u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x00004000u, 0x00000004u, 0x04000000u, 0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x105u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x105u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x109u, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000010u, 0x10000000u, 0x00010000u, 0x00000020u}}, + {0x109u, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000008u, 0x10000000u, 0x00010000u, 0x00000010u}}, + {0x109u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x00004000u, 0x00000004u, 0x04000000u, 0x00008000u, 0x00000008u, 0x08000000u, 0x00010000u, 0x00000010u}}, + {0x109u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000010u}}, + {0x109u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000008u}}, +}; + +inline const TextureSwizzleEquation* FindTextureSwizzleEquation(std::uint32_t swizzleMode, std::uint32_t elementBytes) { + for (const auto& equation : kTextureSwizzleEquations) { + if (equation.swizzleMode == swizzleMode && equation.elementBytes == elementBytes) return &equation; + } + return nullptr; +} + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp index fbfe21b381..3372142915 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp @@ -2,6 +2,7 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURETILING_HPP #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include #include #include @@ -23,6 +24,43 @@ struct TileMipLayout { std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount); std::uint64_t ComputeSurfaceSize(const std::vector& mips, std::uint32_t arrayLayers); +// Mip chain of an uncompressed surface given its element size (color targets have no texture format). +std::vector ComputeElementMipLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount); + +// Thick (3D) SW_4KB_S / SW_64KB_S blocks span several depth slices: {width, height, depth} in elements. +std::array ThickBlockExtent(TextureTileMode tileMode, std::uint32_t bytesPerElement); + +// Single-mip 3D surface. Each depth slice z is detiled from slab z / blockDepth (slabBytes apart) with the +// swizzle's slice input set to z, into linear slices sliceLinearBytes apart. +struct ThickLayout { + TileMipLayout mip; + std::uint32_t depth; + std::uint32_t blockDepth; + std::uint64_t slabBytes; + std::uint64_t sliceLinearBytes; + std::uint64_t guestBytes; +}; +ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth); + +// Guest surface as the texture upload/write-back loops walk it: array layers of thin surfaces, or the +// depth slices of a volume (which Vulkan holds as one layer with extent depth). +struct SurfaceGeometry { + std::vector mips; + std::uint32_t layers = 1; + std::uint32_t imageLayers = 1; + std::uint32_t imageDepth = 1; + std::uint64_t guestBytes = 0; + std::uint64_t sliceLinearBytes = 0; + bool thick = false; + std::uint32_t blockDepth = 1; + std::uint64_t layerBytes = 0; + + std::uint64_t GuestLayerOffset(std::uint32_t layer) const { return thick ? static_cast(layer / blockDepth) * layerBytes : static_cast(layer) * layerBytes; } + std::uint64_t LinearLayerOffset(std::uint32_t layer) const { return static_cast(layer) * sliceLinearBytes; } + std::uint32_t CopyLayer(std::uint32_t layer) const { return imageDepth > 1 ? 0u : layer; } + std::int32_t CopyDepth(std::uint32_t layer) const { return imageDepth > 1 ? static_cast(layer) : 0; } +}; +SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp index 30d6f32389..d9ae43c660 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp @@ -22,6 +22,8 @@ void main() { tiledIndex = ((y / 128u) * parameters.blocksPerRow + x / 128u) * 16384u + offset / 4u; } uint pixel = (parameters.mode & 1u) != 0u ? linear[linearIndex] : tiled[tiledIndex]; + // Mode 8: A2B10G10R10 source (red in the low bits) presented as 8-bit RGBA. + if ((parameters.mode & 8u) != 0u) pixel = ((pixel >> 2u) & 0xffu) | (((pixel >> 12u) & 0xffu) << 8u) | (((pixel >> 22u) & 0xffu) << 16u) | 0xff000000u; if ((parameters.mode & 2u) != 0u) pixel = (pixel & 0xff00ff00u) | ((pixel & 0xffu) << 16u) | ((pixel >> 16u) & 0xffu); if ((parameters.mode & 1u) != 0u) tiled[tiledIndex] = pixel; else linear[linearIndex] = pixel; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp index 2c8d49d46c..912df386d2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp @@ -3,6 +3,29 @@ layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; layout(constant_id = 1) const uint BLOCK_BYTES = 256u; layout(constant_id = 2) const uint TILE_FAMILY = 0u; +// 0: tiled input -> linear output (detile); 1: linear input -> tiled output (retile). +layout(constant_id = 3) const uint RETILE = 0u; +// TILE_FAMILY 2: XOR swizzle given per byte-address bit as masks of x (bits 0-11), y (12-23) and +// slice (24-31) element-coordinate bits. +layout(constant_id = 4) const uint EQ0 = 0u; +layout(constant_id = 5) const uint EQ1 = 0u; +layout(constant_id = 6) const uint EQ2 = 0u; +layout(constant_id = 7) const uint EQ3 = 0u; +layout(constant_id = 8) const uint EQ4 = 0u; +layout(constant_id = 9) const uint EQ5 = 0u; +layout(constant_id = 10) const uint EQ6 = 0u; +layout(constant_id = 11) const uint EQ7 = 0u; +layout(constant_id = 12) const uint EQ8 = 0u; +layout(constant_id = 13) const uint EQ9 = 0u; +layout(constant_id = 14) const uint EQ10 = 0u; +layout(constant_id = 15) const uint EQ11 = 0u; +layout(constant_id = 16) const uint EQ12 = 0u; +layout(constant_id = 17) const uint EQ13 = 0u; +layout(constant_id = 18) const uint EQ14 = 0u; +layout(constant_id = 19) const uint EQ15 = 0u; +// Nonzero: block extent in elements (thick 3D blocks are not the thin 2D shape derived below). +layout(constant_id = 20) const uint BLOCK_WIDTH = 0u; +layout(constant_id = 21) const uint BLOCK_HEIGHT = 0u; layout(local_size_x = 8, local_size_y = 8) in; @@ -20,9 +43,23 @@ layout(push_constant) uniform Push { uint tailX; uint tailY; uint elementBytes; - uint arrayLayer; + uint slice; } params; +uint equationBit(uint mask, uvec2 p) { + uint selected = (p.x & (mask & 0xfffu)) ^ ((p.y << 12) & (mask & 0xfff000u)) ^ ((params.slice << 24) & (mask & 0xff000000u)); + return bitCount(selected) & 1u; +} + +uint equationOffset(uvec2 p) { + const uint eq[16] = uint[16](EQ0, EQ1, EQ2, EQ3, EQ4, EQ5, EQ6, EQ7, EQ8, EQ9, EQ10, EQ11, EQ12, EQ13, EQ14, EQ15); + uint offset = 0u; + for (uint bit = 0u; bit < 16u; ++bit) { + offset |= equationBit(eq[bit], p) << bit; + } + return offset; +} + uint standardOffset(uint x, uint y) { switch (ELEMENT_BYTES) { case 1u: @@ -67,76 +104,9 @@ uint standard64Extra(uint x, uint y) { } -uint renderTargetOffset(uint x, uint y) { - uint offset = 0u; - switch (ELEMENT_BYTES) { - case 1u: - offset ^= (y << 2u) & 0x0008u; - offset ^= (y << 4u) & 0x0010u; - offset ^= (y << 3u) & 0x00a0u; - offset ^= (y << 5u) & 0x0f00u; - offset ^= (y << 6u) & 0x1000u; - offset ^= (y << 7u) & 0x4000u; - offset ^= x & 0x0007u; - offset ^= (x << 3u) & 0x0040u; - offset ^= (x << 5u) & 0x0300u; - offset ^= (x << 4u) & 0x0400u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 7u) & 0x2000u; - offset ^= (x << 8u) & 0x8000u; - break; - case 2u: - offset ^= (y << 4u) & 0x0070u; - offset ^= (y << 5u) & 0x0f00u; - offset ^= (y << 8u) & 0x5000u; - offset ^= (x << 1u) & 0x000eu; - offset ^= (x << 4u) & 0x0480u; - offset ^= (x << 5u) & 0x0300u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 7u) & 0x2000u; - offset ^= (x << 8u) & 0x8000u; - break; - case 4u: - offset ^= (y << 4u) & 0x0070u; - offset ^= (y << 5u) & 0x0f00u; - offset ^= (y << 9u) & 0x1000u; - offset ^= (y << 8u) & 0x4000u; - offset ^= (x << 2u) & 0x000cu; - offset ^= (x << 5u) & 0x0380u; - offset ^= (x << 4u) & 0x0400u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 9u) & 0xa000u; - break; - case 8u: - offset ^= (y << 4u) & 0x0010u; - offset ^= (y << 6u) & 0x0080u; - offset ^= (y << 5u) & 0x0f00u; - offset ^= (y << 10u) & 0x5000u; - offset ^= (x << 3u) & 0x0008u; - offset ^= (x << 4u) & 0x0460u; - offset ^= (x << 5u) & 0x0300u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 10u) & 0x2000u; - offset ^= (x << 9u) & 0x8000u; - break; - case 16u: - offset ^= (x << 4u) & 0x0410u; - offset ^= (x << 5u) & 0x0340u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 11u) & 0xa000u; - offset ^= (y << 5u) & 0x0f20u; - offset ^= (y << 6u) & 0x0080u; - offset ^= (y << 10u) & 0x1000u; - offset ^= (y << 11u) & 0x4000u; - break; - } - uint z = params.arrayLayer; - return offset ^ ((z & 8u) << 5u) ^ ((z & 4u) << 7u) ^ ((z & 2u) << 9u) ^ ((z & 1u) << 11u); -} - uint blockOffset(uvec2 p) { if (TILE_FAMILY == 2u) { - return renderTargetOffset(p.x, p.y); + return equationOffset(p); } uint offset = standardOffset(p.x, p.y); if (BLOCK_BYTES > 4096u) { @@ -146,6 +116,9 @@ uint blockOffset(uvec2 p) { } uvec2 blockExtent() { + if (BLOCK_WIDTH != 0u) { + return uvec2(BLOCK_WIDTH, BLOCK_HEIGHT); + } uint width; if (BLOCK_BYTES <= 256u) { width = ELEMENT_BYTES <= 2u ? 16u : (ELEMENT_BYTES <= 8u ? 8u : 4u); @@ -197,6 +170,10 @@ void main() { uint blockIndex = block.y * params.blocksPerRow + block.x; uint tiledOffset = blockIndex * BLOCK_BYTES + blockOffset(swizzle); - copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset); + if (RETILE != 0u) { + copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset); + } else { + copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset); + } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index 94793d3332..f9978ce2c0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -1,11 +1,90 @@ #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" +#include #include +#include +#include #include #include +#include +#include #include namespace AgcDriver::Graphics { +namespace { + +// The page tables built per device, reused by a build whose ranges (guest ranges, device +// addresses, permissions) equal one of them: consecutive address-based builds map the same registered +// set through the same imports and mirrors, so the ~1200-entry table repeats. The shader only reads +// the table (its lookup keeps its state in function variables), and a batch in flight keeps its +// BdaResources, so sharing the buffer is safe. The cache holds the tables weakly: they never outlive +// the builds that own them (a static owner would destroy the buffer after its device, at exit or on +// device replacement, and would fail the tests' leak check), and with deferred lease release the +// previous builds are still in flight when the next one asks, which is what makes the hit rate. +// One table was enough while every build repeated the last one exactly; now the misaligned +// descriptor sub-ranges the GPU copies out of imports sit in pool staging buffers whose device +// addresses cycle through the few buffers the batches in flight leave free, and the three +// address-based programs of one queue interleave with the lease draws, so a table differs from the +// previous one and comes back a few builds later (69% misses in the wave-8 run, 4% before the GPU +// copies). Hence several recent tables, most recently used first, matched by a word hash before +// the byte compare. APS5_BDA_TABLE_CACHE_ENTRIES sets how many (default 8; 1 is the previous +// behaviour); APS5_NO_BDA_TABLE_CACHE=1 builds every table. +struct TableEntry { + std::uint64_t hash; + std::vector ranges; + std::weak_ptr buffer; +}; + +struct TableCache { + std::mutex mutex; + VkDevice device = VK_NULL_HANDLE; + std::list entries; + std::uint64_t hits = 0; + std::uint64_t misses = 0; +}; + +TableCache& Tables() { + static TableCache cache; + return cache; +} + +bool tableCacheEnabled() { + static const bool disabled = std::getenv("APS5_NO_BDA_TABLE_CACHE") != nullptr; + return !disabled; +} + +std::size_t tableCacheEntries() { + static const std::size_t entries = [] { + const char* value = std::getenv("APS5_BDA_TABLE_CACHE_ENTRIES"); + const auto parsed = value != nullptr ? std::strtoull(value, nullptr, 10) : 8ull; + return static_cast(std::clamp(parsed, 1, 64)); + }(); + return entries; +} + +bool sameRanges(const std::vector& left, const std::vector& right) { + return left.size() == right.size() && (left.empty() || std::memcmp(left.data(), right.data(), left.size() * sizeof(left.front())) == 0); +} + +// A 64-bit mix of the table's words: a table is ~1200 ranges of four words, and comparing the bytes +// against every held table would cost as much as building it. +std::uint64_t hashRanges(const std::vector& ranges) { + static_assert(sizeof(ShaderRecompiler::BdaAbi::Range) % sizeof(std::uint64_t) == 0); + std::uint64_t hash = 0x9e3779b97f4a7c15ull ^ ranges.size(); + for (const auto& range : ranges) { + std::uint64_t words[sizeof(range) / sizeof(std::uint64_t)]; + std::memcpy(words, &range, sizeof(range)); + for (const auto word : words) { + hash ^= word; + hash *= 0xff51afd7ed558ccdull; + hash ^= hash >> 33u; + } + } + return hash; +} + +} + BdaResources::BdaResources(const Context& context) { static_assert(std::endian::native == std::endian::little); Require(sizeof(ShaderRecompiler::BdaAbi::Fault) <= context.limits.maxStorageBufferRange, "BDA fault buffer exceeds storage buffer range limit"); @@ -14,15 +93,67 @@ BdaResources::BdaResources(const Context& context) { } BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memory) : BdaResources(context) { - const auto ranges = memory.AddressRanges(); + auto ranges = memory.AddressRanges(); Require(ranges.size() <= std::numeric_limits::max(), "BDA table range count overflow"); Require(ranges.size() <= (std::numeric_limits::max() - sizeof(ShaderRecompiler::BdaAbi::Header)) / sizeof(ShaderRecompiler::BdaAbi::Range), "BDA table size overflow"); tableBytes = sizeof(ShaderRecompiler::BdaAbi::Header) + ranges.size() * sizeof(ShaderRecompiler::BdaAbi::Range); Require(tableBytes <= context.limits.maxStorageBufferRange && sizeof(ShaderRecompiler::BdaAbi::Fault) <= context.limits.maxStorageBufferRange, "BDA descriptors exceed storage buffer range limit"); - table = std::make_unique(context, tableBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + auto& cache = Tables(); + const auto hash = tableCacheEnabled() ? hashRanges(ranges) : 0; + if (tableCacheEnabled()) { + std::lock_guard lock(cache.mutex); + // Another device's tables are not this one's (their buffers are dead or foreign). + if (cache.device != context.device) { + cache.entries.clear(); + cache.device = context.device; + } + for (auto it = cache.entries.begin(); it != cache.entries.end();) { + // Compare the hash before taking a strong reference: locking a non-matching entry's + // buffer could make this thread its last owner (a concurrent reap releasing it) and + // run the pool release under the cache mutex; expired() takes no reference. + if (it->hash != hash) { + if (it->buffer.expired()) { + it = cache.entries.erase(it); + } else { + ++it; + } + continue; + } + auto shared = it->buffer.lock(); + if (shared == nullptr) { + // Every build that owned the table is gone, and its buffer with it. + it = cache.entries.erase(it); + continue; + } + if (sameRanges(it->ranges, ranges)) { + ++cache.hits; + table = std::move(shared); + cache.entries.splice(cache.entries.begin(), cache.entries, it); + return; + } + ++it; + } + } + table = std::make_shared(context, tableBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); const ShaderRecompiler::BdaAbi::Header header{ShaderRecompiler::BdaAbi::Version, static_cast(ranges.size()), sizeof(ShaderRecompiler::BdaAbi::Range), 0}; std::memcpy(table->Bytes().data(), &header, sizeof(header)); if (!ranges.empty()) std::memcpy(table->Bytes().data() + sizeof(header), ranges.data(), ranges.size() * sizeof(ranges.front())); + if (tableCacheEnabled()) { + std::lock_guard lock(cache.mutex); + ++cache.misses; + if (cache.device != context.device) { + cache.entries.clear(); + cache.device = context.device; + } + cache.entries.push_front({hash, std::move(ranges), table}); + while (cache.entries.size() > tableCacheEntries()) cache.entries.pop_back(); + } +} + +BdaResources::TableCacheStats BdaResources::TableCacheCounters() { + auto& cache = Tables(); + std::lock_guard lock(cache.mutex); + return {cache.hits, cache.misses, cache.entries.size()}; } VkDescriptorBufferInfo BdaResources::Table() const { @@ -43,6 +174,12 @@ void BdaResources::CheckFault() const { } Require(report.state == ShaderRecompiler::BdaAbi::FaultState::Ready && report.reserved == 0, "incomplete or invalid BDA fault record"); Require(report.reason != ShaderRecompiler::BdaAbi::FaultReason::InvalidRectangle, "rect-list requires finite nondegenerate axis-aligned positions with equal positive W"); + if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::LoopLimit) { + // APS5_LOOP_GUARD: the shader left a loop that ran past the guard; the dispatch result is kept. + std::fprintf(stderr, "[gpu] loop guard: the loop exit at pc 0x%x ran past %u evaluations\n", report.instruction, report.bytes); + std::memset(fault->Bytes().data(), 0, fault->Bytes().size()); + return; + } std::ostringstream message; message << "BDA access failed: address=0x" << std::hex << report.address << " instruction=0x" << report.instruction << std::dec << " bytes=" << report.bytes << " stage=" << report.stage << " reason=" << static_cast(report.reason); throw std::runtime_error(message.str()); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 6c4477baff..b3896cc464 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -1,74 +1,118 @@ #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include +#include +#include +#include +#include namespace AgcDriver::Graphics { +namespace { + +// APS5_BUFFER_POOL_SHARED=1: one tier for every size, as before the split. +bool sharedTiers() { + static const bool shared = std::getenv("APS5_BUFFER_POOL_SHARED") != nullptr; + return shared; +} + +} + BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")) { - buckets.fill(none); - for (std::size_t index = 0; index < capacity; ++index) freeSlots[index] = index; + smallTier.budget = smallBudget; + largeTier.budget = budget; } BufferPool::~BufferPool() { - for (const auto& slot : slots) { - if (slot.allocation) destroy(*slot.allocation); - } -} - -std::size_t BufferPool::bucket(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) noexcept { - auto hash = bytes ^ (bytes >> 16u); - hash = hash * 16777619u ^ usage; - hash = hash * 16777619u ^ properties; - return (hash ^ (hash >> 8u) ^ (hash >> 16u)) % bucketCount; -} - -BufferAllocation BufferPool::remove(std::size_t index) noexcept { - auto& slot = slots[index]; - const auto allocation = *slot.allocation; - auto* link = &buckets[bucket(allocation.bytes, allocation.usage, allocation.properties)]; - while (*link != index) link = &slots[*link].next; - *link = slot.next; - if (slot.older != none) slots[slot.older].newer = slot.newer; - else oldest = slot.newer; - if (slot.newer != none) slots[slot.newer].older = slot.older; - else newest = slot.older; - retainedBytes -= allocation.allocationBytes; - slot = Slot{}; - freeSlots[freeCount++] = index; - return allocation; + for (const auto& slot : smallTier.free) destroy(slot.allocation); + for (const auto& slot : largeTier.free) destroy(slot.allocation); } void BufferPool::destroy(const BufferAllocation& allocation) noexcept { + // Device-local allocations (see DeviceBuffer) are never mapped. if (allocation.mapping != nullptr) unmap(device, allocation.memory); destroyBuffer(device, allocation.buffer, nullptr); freeMemory(device, allocation.memory, nullptr); } +std::size_t BufferPool::Capacity(std::size_t bytes) { + static const bool exact = std::getenv("APS5_NO_BUFFER_CLASSES") != nullptr; + constexpr std::size_t smallest = 256; + if (exact || bytes >= classLimit) return bytes; + return std::max(smallest, std::bit_ceil(bytes)); +} + +BufferPool::Tier& BufferPool::tierFor(std::size_t capacity) { + return !sharedTiers() && capacity < classLimit ? smallTier : largeTier; +} + std::optional BufferPool::Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto capacity = Capacity(bytes); std::lock_guard lock(mutex); - for (auto index = buckets[bucket(bytes, usage, properties)]; index != none; index = slots[index].next) { - const auto& allocation = *slots[index].allocation; - if (allocation.bytes == bytes && allocation.usage == usage && allocation.properties == properties) return remove(index); + ++clock; + if (profile) { + static auto lastReport = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport > std::chrono::seconds(10)) { + lastReport = now; + std::fprintf(stderr, "[bufferpool] small: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB); large: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB)%s\n", static_cast(smallTier.hits), static_cast(smallTier.misses), static_cast(smallTier.evictions), smallTier.free.size(), smallTier.retainedBytes / 1048576.0, static_cast(largeTier.hits), static_cast(largeTier.misses), static_cast(largeTier.evictions), largeTier.free.size(), largeTier.retainedBytes / 1048576.0, sharedTiers() ? " (shared)" : ""); + } } + auto& tier = tierFor(capacity); + for (auto it = tier.free.begin(); it != tier.free.end(); ++it) { + const auto& allocation = it->allocation; + if (allocation.bytes != capacity || allocation.usage != usage || allocation.properties != properties) continue; + auto result = allocation; + tier.retainedBytes -= allocation.allocationBytes; + *it = std::move(tier.free.back()); + tier.free.pop_back(); + ++tier.hits; + return result; + } + ++tier.misses; return std::nullopt; } +void BufferPool::evictOldest(Tier& tier, std::vector& evicted) { + const auto oldest = std::min_element(tier.free.begin(), tier.free.end(), [](const Slot& left, const Slot& right) { return left.lastUse < right.lastUse; }); + // First: a throw here leaves the slot retained and counted. + evicted.push_back(oldest->allocation); + tier.retainedBytes -= oldest->allocation.allocationBytes; + *oldest = std::move(tier.free.back()); + tier.free.pop_back(); + ++tier.evictions; +} + +std::size_t BufferPool::MaxSlots() { + // APS5_BUFFER_POOL_SLOTS= bounds the retained allocations; 64 is the capacity the pool had + // before least-recently-used retention, for A/B runs. + static const std::size_t slots = [] { + const char* value = std::getenv("APS5_BUFFER_POOL_SLOTS"); + const auto parsed = value != nullptr ? std::strtoull(value, nullptr, 10) : 0; + return parsed != 0 ? static_cast(parsed) : defaultSlots; + }(); + return slots; +} + void BufferPool::Put(const BufferAllocation& allocation) noexcept { - std::lock_guard lock(mutex); - if (allocation.allocationBytes > budget) { + // Evicted allocations are destroyed after the mutex is released (see evictOldest). The vector + // may throw on growth; a Put that cannot retain simply destroys, as noexcept requires. + std::vector evicted; + try { + std::lock_guard lock(mutex); + auto& tier = tierFor(allocation.bytes); + if (allocation.allocationBytes > tier.budget) { + evicted.push_back(allocation); + } else { + const auto maxSlots = MaxSlots(); + while (!tier.free.empty() && (tier.retainedBytes + allocation.allocationBytes > tier.budget || tier.free.size() >= maxSlots)) evictOldest(tier, evicted); + tier.free.push_back({allocation, ++clock}); + tier.retainedBytes += allocation.allocationBytes; + } + } catch (...) { destroy(allocation); - return; } - while (retainedBytes > budget - allocation.allocationBytes || freeCount == 0) destroy(remove(oldest)); - const auto index = freeSlots[--freeCount]; - auto& slot = slots[index]; - auto& head = buckets[bucket(allocation.bytes, allocation.usage, allocation.properties)]; - slot.allocation = allocation; - slot.next = head; - slot.older = newest; - if (newest != none) slots[newest].newer = index; - else oldest = index; - newest = index; - head = index; - retainedBytes += allocation.allocationBytes; + for (const auto& gone : evicted) destroy(gone); } std::shared_ptr GetBufferPool(const Context& context) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp index 563a39d2e3..f9d4c41e9d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp @@ -1,4 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" +#include #include #include #include @@ -10,9 +12,8 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(reason); } -std::uint32_t blockOffset(std::uint32_t x, std::uint32_t y) { - return ((y << 4u) & 0x0070u) ^ ((y << 5u) & 0x0f00u) ^ ((y << 9u) & 0x1000u) ^ ((y << 8u) & 0x4000u) - ^ ((x << 2u) & 0x000cu) ^ ((x << 5u) & 0x0380u) ^ ((x << 4u) & 0x0400u) ^ ((x << 6u) & 0x0800u) ^ ((x << 9u) & 0xa000u); +std::uint32_t parity(std::uint32_t value) { + return static_cast(std::popcount(value)) & 1u; } } @@ -26,28 +27,48 @@ ColorTileMode DecodeColorTileMode(std::uint32_t attrib3) { return static_cast(mode); } -ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode) : width(width), height(height), pitch(width), mode(mode), bytes(0) { +ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement) : width(width), height(height), pitch(width), mode(mode), bytes(0), elementBytes(bytesPerElement) { require(width != 0 && height != 0 && width <= 16384 && height <= 16384, "AGC graphics: invalid color surface extent"); + require(std::has_single_bit(bytesPerElement) && bytesPerElement <= 16u, "AGC graphics: unsupported color element size"); std::uint32_t paddedHeight = height; switch (mode) { case ColorTileMode::Linear: - require(width % 64u == 0, "AGC graphics: linear surface pitch requires a width aligned to 256 bytes"); + require(width * bytesPerElement % 256u == 0, "AGC graphics: linear surface pitch requires a width aligned to 256 bytes"); break; - case ColorTileMode::RenderTarget: - pitch = (width + 127u) & ~127u; - paddedHeight = (height + 127u) & ~127u; + case ColorTileMode::RenderTarget: { + // SW_64KB_R_X: 64 KiB blocks of 2^(16 - log2(bpe)) elements, wider than tall for odd powers. + const auto log2Elements = 16u - static_cast(std::countr_zero(bytesPerElement)); + blockWidth = 1u << ((log2Elements + 1u) / 2u); + blockHeight = 1u << (log2Elements / 2u); + pitch = (width + blockWidth - 1u) / blockWidth * blockWidth; + paddedHeight = (height + blockHeight - 1u) / blockHeight * blockHeight; + const auto* equation = FindTextureSwizzleEquation(27u, bytesPerElement); + require(equation != nullptr, "AGC graphics: no SW_64KB_R_X equation for the color element size"); + xOffsets.resize(blockWidth); + yOffsets.resize(blockHeight); + for (std::uint32_t x = 0; x < blockWidth; ++x) { + std::uint32_t offset = 0; + for (std::uint32_t bit = 0; bit < 16u; ++bit) offset |= parity(x & equation->bits[bit] & 0xfffu) << bit; + xOffsets[x] = offset; + } + for (std::uint32_t y = 0; y < blockHeight; ++y) { + std::uint32_t offset = 0; + for (std::uint32_t bit = 0; bit < 16u; ++bit) offset |= parity((y << 12u) & equation->bits[bit] & 0xfff000u) << bit; + yOffsets[y] = offset; + } break; + } default: throw std::runtime_error("AGC graphics: unsupported color tile mode"); } - const auto size = static_cast(pitch) * paddedHeight * 4u; + const auto size = static_cast(pitch) * paddedHeight * bytesPerElement; require(size <= std::numeric_limits::max(), "AGC graphics: color surface size overflow"); bytes = static_cast(size); } std::size_t ColorTargetLayout::offset(std::uint32_t x, std::uint32_t y) const { - if (mode == ColorTileMode::Linear) return (static_cast(y) * pitch + x) * 4u; - const auto block = static_cast(y / 128u) * (pitch / 128u) + x / 128u; - return block * 65536u + blockOffset(x, y); + if (mode == ColorTileMode::Linear) return (static_cast(y) * pitch + x) * elementBytes; + const auto block = static_cast(y / blockHeight) * (pitch / blockWidth) + x / blockWidth; + return block * 65536u + (xOffsets[x % blockWidth] ^ yOffsets[y % blockHeight]); } std::size_t ColorTargetLayout::Offset(std::uint32_t x, std::uint32_t y) const { @@ -58,18 +79,16 @@ std::size_t ColorTargetLayout::Offset(std::uint32_t x, std::uint32_t y) const { void ColorTargetLayout::Detile(std::span source, std::span destination) const { require(source.size() == Bytes() && destination.size() == LinearBytes(), "AGC graphics: color detile buffer size mismatch"); for (std::uint32_t y = 0; y < height; ++y) { - for (std::uint32_t x = 0; x < width; ++x) { - std::memcpy(destination.data() + (static_cast(y) * width + x) * 4u, source.data() + offset(x, y), 4); - } + auto* row = destination.data() + static_cast(y) * width * elementBytes; + for (std::uint32_t x = 0; x < width; ++x) std::memcpy(row + static_cast(x) * elementBytes, source.data() + offset(x, y), elementBytes); } } void ColorTargetLayout::Tile(std::span source, std::span destination) const { require(source.size() == LinearBytes() && destination.size() == Bytes(), "AGC graphics: color tile buffer size mismatch"); for (std::uint32_t y = 0; y < height; ++y) { - for (std::uint32_t x = 0; x < width; ++x) { - std::memcpy(destination.data() + offset(x, y), source.data() + (static_cast(y) * width + x) * 4u, 4); - } + const auto* row = source.data() + static_cast(y) * width * elementBytes; + for (std::uint32_t x = 0; x < width; ++x) std::memcpy(destination.data() + offset(x, y), row + static_cast(x) * elementBytes, elementBytes); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp index 0b3003e573..9d9d7d3465 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp @@ -7,7 +7,7 @@ namespace AgcDriver::Graphics { void ReadColorTarget(const ColorTarget& target, std::span destination) { PerformanceTimer timing("ColorTarget.Read"); - const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode); + const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes); Require(target.bytes == layout.Bytes() && destination.size() == layout.LinearBytes(), "color target transfer size mismatch"); if (target.tileMode == ColorTileMode::Linear) { GuestMemory::Read(target.address, destination, layout.Alignment()); @@ -23,7 +23,7 @@ void ReadColorTarget(const ColorTarget& target, std::span destination void WriteColorTarget(const ColorTarget& target, std::span source) { PerformanceTimer timing("ColorTarget.Write"); - const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode); + const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes); Require(target.bytes == layout.Bytes() && source.size() == layout.LinearBytes(), "color target transfer size mismatch"); if (target.tileMode == ColorTileMode::Linear) { GuestMemory::Write(target.address, source, layout.Alignment()); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp new file mode 100644 index 0000000000..4693139ac0 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp @@ -0,0 +1,433 @@ +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { +namespace { + +constexpr std::uint64_t KeyBytes = 256; + +// Whether every key in [keys, keys + count) equals `first`. Eight keys per compare: the scan runs +// on every texture and storage validation, and an all-0xff (uncompressed) surface's keys are read +// to the end each time (132 KiB for a 4K RGBA8 target). APS5_NO_DCC_WORD_SCAN=1 restores the byte loop. +bool AllKeysEqual(const std::uint8_t* keys, std::size_t count, std::uint8_t first) { + static const bool byteScan = std::getenv("APS5_NO_DCC_WORD_SCAN") != nullptr; + if (byteScan) return std::all_of(keys, keys + count, [&](std::uint8_t key) { return key == first; }); + const std::uint64_t pattern = 0x0101010101010101ull * first; + std::size_t i = 0; + for (; i + 8 <= count; i += 8) { + std::uint64_t word; + std::memcpy(&word, keys + i, sizeof(word)); + if (word != pattern) return false; + } + for (; i < count; ++i) { + if (keys[i] != first) return false; + } + return true; +} + +// APS5_PROFILE_DRAW: key scans, their bytes and time, printed as [dcc] every 10 s, with the hits of +// the GPU-store memo below and the "uncompressed" key stores by where they were made. Flush syncs +// stay 0 (no key read syncs the recorder); the field is kept so the line's shape does not change. +struct ScanProfile { + std::atomic scans{0}; + std::atomic bytes{0}; + std::atomic nanoseconds{0}; + std::atomic memoHits{0}; + std::atomic flushSyncs{0}; + std::atomic gpuStores{0}; + // GPU stores whose range had a 4-byte unaligned head or tail (copied instead of filled). + std::atomic gpuSplitStores{0}; + std::atomic cpuStores{0}; + std::atomic lastReport{0}; +}; + +ScanProfile& Scans() { + static ScanProfile profile; + return profile; +} + +bool ScanProfileEnabled() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + +void ReportScans(std::chrono::steady_clock::time_point now) { + auto& profile = Scans(); + const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); + auto last = profile.lastReport.load(); + // The first event starts the period rather than reporting an empty one. + if (last == 0) { + profile.lastReport.compare_exchange_strong(last, nowMs); + return; + } + if (nowMs - last < 10000 || !profile.lastReport.compare_exchange_strong(last, nowMs)) return; + std::fprintf(stderr, "[dcc] %llu scans, %.1f MiB scanned, %llu memo hits, %llu flush syncs, %.1f ms; uncompressed keys stored: %llu on the GPU (%llu with an unaligned head or tail), %llu on the CPU\n", static_cast(profile.scans.load()), profile.bytes.load() / 1048576.0, static_cast(profile.memoHits.load()), static_cast(profile.flushSyncs.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.gpuStores.load()), static_cast(profile.gpuSplitStores.load()), static_cast(profile.cpuStores.load())); +} + +void CountScan(std::size_t bytes, std::chrono::steady_clock::time_point start) { + auto& profile = Scans(); + const auto now = std::chrono::steady_clock::now(); + profile.scans.fetch_add(1, std::memory_order_relaxed); + profile.bytes.fetch_add(bytes, std::memory_order_relaxed); + profile.nanoseconds.fetch_add(static_cast(std::chrono::duration_cast(now - start).count()), std::memory_order_relaxed); + ReportScans(now); +} + +void CountStore(std::atomic& counter) { + if (!ScanProfileEnabled()) return; + counter.fetch_add(1, std::memory_order_relaxed); + ReportScans(std::chrono::steady_clock::now()); +} + +// "Uncompressed" keys the driver stored on the GPU (MarkDccUncompressed's recorded fill) whose batch +// may not have run yet: the bytes ReadDccKeys scans would still show the keys from before the store +// (a fast-clear code), and a storage image whose write-back made the store would be re-uploaded as +// cleared. While recorded work that writes the range is still running, the range reads as +// uncompressed from here; the bytes are trusted again only once the newest recorded writer of the +// range has signaled its fence (batches of the recorder's queue finish in order, so the fill and +// every writer up to that one have landed and the bytes are final, whichever batch wrote last). +// Dropping an entry on a later batch's serial alone would not do: until the fill's batch runs the +// bytes PREDATE the fill (the previous clear code), and a later writer that is not a same-code clear +// (a decompress kernel, a partial key write, a false overlap from an unrelated V#) would make the +// next Refresh compare that stale code against the uploaded keys. The lookup runs with or without +// GuestMemory::GpuMutex (a resource build's stage A reads keys unlocked, its stage B re-reads them +// under the lock): unlocked it can only ask the lock-free snapshot whether the range is written at +// all. Entries belong to one recorder: when the active recorder changes (device replacement; a new +// recorder can reuse the old one's address) they are all dropped. Its mutex is a leaf (nothing is +// taken under it). +struct GpuKeyStore { + std::uint64_t begin; + std::uint64_t end; +}; + +struct KeyStoreMemo { + std::mutex mutex; + std::vector entries; + const Recorder* recorder = nullptr; +}; + +KeyStoreMemo& Memo() { + static KeyStoreMemo memo; + return memo; +} + +// Under memo.mutex: the active recorder, with the entries of any other recorder dropped. +Recorder* MemoRecorder(KeyStoreMemo& memo) { + auto* active = Recorder::Active(); + if (active != memo.recorder) { + memo.entries.clear(); + memo.recorder = active; + } + return active; +} + +void ForgetStores(std::uint64_t begin, std::uint64_t end) { + auto& memo = Memo(); + std::lock_guard lock(memo.mutex); + std::erase_if(memo.entries, [&](const GpuKeyStore& store) { return begin < store.end && store.begin < end; }); +} + +// Uncompressed when a GPU store still pending covers the whole range, else nullopt (the bytes decide). +std::optional MemoizedKeys(std::uint64_t begin, std::uint64_t end) { + auto& memo = Memo(); + std::lock_guard lock(memo.mutex); + auto* recorder = MemoRecorder(memo); + for (auto it = memo.entries.begin(); it != memo.entries.end(); ++it) { + if (it->begin > begin || it->end < end) continue; + const auto bytes = static_cast(it->end - it->begin); + bool pending = false; + if (recorder != nullptr) { + if (GuestMemory::GpuMutex().HeldByThisThread()) { + // The open batch is never signaled (also with APS5_NO_SYNC_THROUGH, where the info + // carries the open flag with the newest in-flight batch's fence state). + const auto info = recorder->DescribePendingWrite(it->begin, bytes); + pending = info.has_value() && !info->signaled; + } else { + pending = Recorder::SnapshotWriteOverlaps(it->begin, bytes); + } + } + if (!pending) { + memo.entries.erase(it); + return std::nullopt; + } + if (ScanProfileEnabled()) Scans().memoHits.fetch_add(1, std::memory_order_relaxed); + return DccKeys::Uncompressed; + } + return std::nullopt; +} + +// The 0xff keys as a fill recorded into the active recorder's open batch, into the range's host +// import (the WriteLabelOnGpu / FillBuffer pattern): ordered behind every earlier recorded read or +// write of the range (the title's DCC clear or decompress kernel storing keys through a V#, which the +// CPU store had to wait for) and visible to the work recorded after it and to the host. vkCmdFillBuffer +// needs a 4-byte aligned offset and size; an unaligned head or tail is copied from a kept 4-byte +// buffer of 0xff instead. Under GuestMemory::GpuMutex (it records). False when the keys cannot be +// stored this way (memory the GPU has no view of): the caller stores them on the CPU. +bool StoreUncompressedOnGpu(const Context& context, Recorder& recorder, std::uint64_t begin, std::size_t count) { + const auto* import = HostImportFor(context, begin, count); + if (import == nullptr) return false; + const VkDeviceSize first = begin - import->base; + const VkDeviceSize last = first + count; + const VkDeviceSize fillBegin = std::min((first + 3) & ~VkDeviceSize{3}, last); + const VkDeviceSize fillEnd = std::max(last & ~VkDeviceSize{3}, fillBegin); + // Everything that can throw (the seed's allocation, the batch's command buffer) comes before the + // first recorded command: a fill left in the batch without its note and memo entry would be an + // unordered write of the keys. + std::shared_ptr seed; + if (fillBegin > first || last > fillEnd) { + seed = std::make_shared(context, 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memset(seed->Bytes().data(), 0xff, seed->Bytes().size()); + recorder.Keep(seed); + } + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + if (fillEnd > fillBegin) context.Function("vkCmdFillBuffer")(commands, import->buffer, fillBegin, fillEnd - fillBegin, 0xffffffffu); + if (seed != nullptr) { + VkBufferCopy copies[2]; + std::uint32_t copyCount = 0; + if (fillBegin > first) copies[copyCount++] = {0, first, fillBegin - first}; + if (last > fillEnd) copies[copyCount++] = {0, fillEnd, last - fillEnd}; + context.Function("vkCmdCopyBuffer")(commands, seed->Handle(), import->buffer, copyCount, copies); + CountStore(Scans().gpuSplitStores); + } + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT); + recorder.NotePendingWrite(begin, count); + GuestMemory::MarkWritten(begin, count); + // After the note: an unlocked memo lookup drops an entry the snapshot does not cover yet. The + // entry is kept only for the active recorder (the one the flush hook and the lookups consult). + { + auto& memo = Memo(); + std::lock_guard lock(memo.mutex); + const auto end = begin + count; + std::erase_if(memo.entries, [&](const GpuKeyStore& store) { return begin < store.end && store.begin < end; }); + if (MemoRecorder(memo) == &recorder) memo.entries.push_back({begin, end}); + } + CountStore(Scans().gpuStores); + return true; +} + +void StoreUncompressedOnCpu(std::uint64_t begin, std::size_t count) { + // The write waits for recorded work over the range (the flush hook), so the bytes are current after. + ForgetStores(begin, begin + count); + const std::vector uncompressed(count, std::byte{0xff}); + GuestMemory::Write(begin, uncompressed); + CountStore(Scans().cpuStores); +} + +// Whether the keys already read as uncompressed: scans the same bytes as ReadDccKeys, so it counts +// as a scan too. +bool AlreadyUncompressed(std::uint64_t metaAddress, std::size_t count) { + const auto* keys = reinterpret_cast(metaAddress); + const auto start = ScanProfileEnabled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const bool uncompressed = AllKeysEqual(keys, count, 0xff); + if (ScanProfileEnabled()) CountScan(count, start); + return uncompressed; +} + +bool CpuKeysOnly() { + static const bool cpuKeys = std::getenv("APS5_CPU_DCC_KEYS") != nullptr; + return cpuKeys; +} + +enum class Kind { Unorm, Snorm, Uint, Sint, Float }; + +struct Layout { + std::uint32_t channels; + std::uint32_t bits[4]; + Kind kind; +}; + +bool LayoutFor(VkFormat format, Layout& layout) { + switch (format) { + case VK_FORMAT_R8_UNORM: case VK_FORMAT_R8_SRGB: layout = {1, {8}, Kind::Unorm}; return true; + case VK_FORMAT_R8_UINT: layout = {1, {8}, Kind::Uint}; return true; + case VK_FORMAT_R16_UNORM: layout = {1, {16}, Kind::Unorm}; return true; + case VK_FORMAT_R16_SNORM: layout = {1, {16}, Kind::Snorm}; return true; + case VK_FORMAT_R16_UINT: layout = {1, {16}, Kind::Uint}; return true; + case VK_FORMAT_R16_SINT: layout = {1, {16}, Kind::Sint}; return true; + case VK_FORMAT_R16_SFLOAT: layout = {1, {16}, Kind::Float}; return true; + case VK_FORMAT_R8G8_UNORM: case VK_FORMAT_R8G8_SRGB: layout = {2, {8, 8}, Kind::Unorm}; return true; + case VK_FORMAT_R8G8_SNORM: layout = {2, {8, 8}, Kind::Snorm}; return true; + case VK_FORMAT_R8G8_UINT: layout = {2, {8, 8}, Kind::Uint}; return true; + case VK_FORMAT_R8G8_SINT: layout = {2, {8, 8}, Kind::Sint}; return true; + case VK_FORMAT_R32_UINT: layout = {1, {32}, Kind::Uint}; return true; + case VK_FORMAT_R32_SINT: layout = {1, {32}, Kind::Sint}; return true; + case VK_FORMAT_R32_SFLOAT: layout = {1, {32}, Kind::Float}; return true; + case VK_FORMAT_R16G16_UNORM: layout = {2, {16, 16}, Kind::Unorm}; return true; + case VK_FORMAT_R16G16_SNORM: layout = {2, {16, 16}, Kind::Snorm}; return true; + case VK_FORMAT_R16G16_UINT: layout = {2, {16, 16}, Kind::Uint}; return true; + case VK_FORMAT_R16G16_SINT: layout = {2, {16, 16}, Kind::Sint}; return true; + case VK_FORMAT_R16G16_SFLOAT: layout = {2, {16, 16}, Kind::Float}; return true; + case VK_FORMAT_A2B10G10R10_UNORM_PACK32: layout = {4, {10, 10, 10, 2}, Kind::Unorm}; return true; + case VK_FORMAT_A2B10G10R10_UINT_PACK32: layout = {4, {10, 10, 10, 2}, Kind::Uint}; return true; + case VK_FORMAT_R8G8B8A8_UNORM: case VK_FORMAT_R8G8B8A8_SRGB: case VK_FORMAT_B8G8R8A8_UNORM: case VK_FORMAT_B8G8R8A8_SRGB: layout = {4, {8, 8, 8, 8}, Kind::Unorm}; return true; + case VK_FORMAT_R8G8B8A8_SNORM: layout = {4, {8, 8, 8, 8}, Kind::Snorm}; return true; + case VK_FORMAT_R8G8B8A8_UINT: layout = {4, {8, 8, 8, 8}, Kind::Uint}; return true; + case VK_FORMAT_R8G8B8A8_SINT: layout = {4, {8, 8, 8, 8}, Kind::Sint}; return true; + case VK_FORMAT_R32G32_UINT: layout = {2, {32, 32}, Kind::Uint}; return true; + case VK_FORMAT_R32G32_SINT: layout = {2, {32, 32}, Kind::Sint}; return true; + case VK_FORMAT_R32G32_SFLOAT: layout = {2, {32, 32}, Kind::Float}; return true; + case VK_FORMAT_R16G16B16A16_UNORM: layout = {4, {16, 16, 16, 16}, Kind::Unorm}; return true; + case VK_FORMAT_R16G16B16A16_SNORM: layout = {4, {16, 16, 16, 16}, Kind::Snorm}; return true; + case VK_FORMAT_R16G16B16A16_UINT: layout = {4, {16, 16, 16, 16}, Kind::Uint}; return true; + case VK_FORMAT_R16G16B16A16_SINT: layout = {4, {16, 16, 16, 16}, Kind::Sint}; return true; + case VK_FORMAT_R16G16B16A16_SFLOAT: layout = {4, {16, 16, 16, 16}, Kind::Float}; return true; + case VK_FORMAT_R32G32B32A32_UINT: layout = {4, {32, 32, 32, 32}, Kind::Uint}; return true; + case VK_FORMAT_R32G32B32A32_SINT: layout = {4, {32, 32, 32, 32}, Kind::Sint}; return true; + case VK_FORMAT_R32G32B32A32_SFLOAT: layout = {4, {32, 32, 32, 32}, Kind::Float}; return true; + default: return false; + } +} + +// "1" in a channel of this kind and width. +std::uint64_t One(Kind kind, std::uint32_t bits) { + switch (kind) { + case Kind::Unorm: case Kind::Uint: return bits >= 64 ? ~0ull : (1ull << bits) - 1u; + case Kind::Snorm: case Kind::Sint: return (1ull << (bits - 1u)) - 1u; + case Kind::Float: return bits == 16 ? 0x3c00u : 0x3f800000u; + } + return 0; +} + +} + +const char* DccKeysName(DccKeys keys) { + switch (keys) { + case DccKeys::Uncompressed: return "uncompressed"; + case DccKeys::Clear0000: return "0000"; + case DccKeys::Clear0001: return "0001"; + case DccKeys::Clear1110: return "1110"; + case DccKeys::Clear1111: return "1111"; + case DccKeys::ClearRegister: return "register"; + case DccKeys::Mixed: return "mixed"; + case DccKeys::Unreadable: return "unreadable"; + } + return "?"; +} + +DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + const auto count = static_cast(surfaceBytes / KeyBytes); + if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count)) return DccKeys::Unreadable; + // A driver store still pending on the GPU is not in the bytes yet. + if (const auto memoized = MemoizedKeys(metaAddress, metaAddress + count)) return *memoized; + const auto* keys = reinterpret_cast(metaAddress); + const auto first = keys[0]; + const auto start = ScanProfileEnabled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const bool uniform = AllKeysEqual(keys + 1, count - 1, first); + if (ScanProfileEnabled()) CountScan(count, start); + if (!uniform) return DccKeys::Mixed; + switch (first) { + case 0x00: return DccKeys::Clear0000; + case 0x40: return DccKeys::Clear0001; + case 0x80: return DccKeys::Clear1110; + case 0xc0: return DccKeys::Clear1111; + case 0x20: return DccKeys::ClearRegister; + case 0xff: return DccKeys::Uncompressed; + default: return DccKeys::Mixed; + } +} + +bool IsDccClear(DccKeys keys) { + return keys == DccKeys::Clear0000 || keys == DccKeys::Clear0001 || keys == DccKeys::Clear1110 || keys == DccKeys::Clear1111 || keys == DccKeys::ClearRegister; +} + +void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + const auto count = static_cast(surfaceBytes / KeyBytes); + if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count, true)) return; + // The bytes are trusted only when no recorded work writes them (lock-free, with or without + // GuestMemory::GpuMutex); otherwise the store runs and waits through the flush hook. A pending + // GPU store of the driver's own is no reason to skip: a title kernel recorded into the same + // batch after it may write clear keys, and the memo cannot order inside a batch. + if (!Recorder::SnapshotWriteOverlaps(metaAddress, count) && AlreadyUncompressed(metaAddress, count)) return; + StoreUncompressedOnCpu(metaAddress, count); +} + +void MarkDccUncompressed(const Context& context, std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + const auto count = static_cast(surfaceBytes / KeyBytes); + if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count, true)) return; + auto* recorder = CpuKeysOnly() || !GuestMemory::GpuMutex().HeldByThisThread() ? nullptr : Recorder::Active(); + // The bytes are scanned only when no recorded work writes them: a pending clear kernel's keys are + // not in memory yet, so the bytes could read as uncompressed while the kernel will store a clear + // code, and the range is stored without looking (the fill lands after the kernel). That includes + // a pending GPU store of the driver's own (the memo): a title kernel recorded into the same batch + // after it may write clear keys, and nothing orders inside a batch, so a fresh fill is recorded + // (idempotent, no wait) rather than trusting the earlier one. + const bool pending = recorder != nullptr && recorder->PendingWriteOverlaps(metaAddress, count); + if (!pending && AlreadyUncompressed(metaAddress, count)) return; + if (recorder != nullptr && StoreUncompressedOnGpu(context, *recorder, metaAddress, count)) return; + StoreUncompressedOnCpu(metaAddress, count); +} + +bool FillDccClear(VkFormat format, DccKeys keys, bool alphaOnMsb, std::span bytes) { + if (keys == DccKeys::Clear0000) { + std::fill(bytes.begin(), bytes.end(), std::byte{0}); + return true; + } + Layout layout{}; + if (!IsDccClear(keys) || keys == DccKeys::ClearRegister || !LayoutFor(format, layout)) return false; + const bool color = keys == DccKeys::Clear1110 || keys == DccKeys::Clear1111; + const bool alpha = keys == DccKeys::Clear0001 || keys == DccKeys::Clear1111; + const int alphaChannel = layout.channels == 3 ? -1 : alphaOnMsb ? static_cast(layout.channels) - 1 : 0; + std::uint32_t elementBits = 0; + for (std::uint32_t channel = 0; channel < layout.channels; ++channel) elementBits += layout.bits[channel]; + std::vector element(elementBits / 8u); + std::uint32_t bit = 0; + for (std::uint32_t channel = 0; channel < layout.channels; ++channel) { + const bool set = static_cast(channel) == alphaChannel ? alpha : color; + const auto value = set ? One(layout.kind, layout.bits[channel]) : 0u; + for (std::uint32_t i = 0; i < layout.bits[channel]; ++i, ++bit) { + if (((value >> i) & 1u) != 0) element[bit / 8u] |= static_cast(1u << (bit % 8u)); + } + } + for (std::size_t offset = 0; offset + element.size() <= bytes.size(); offset += element.size()) std::memcpy(bytes.data() + offset, element.data(), element.size()); + return true; +} + +bool DccAlphaOnMsb(VkFormat format, std::uint32_t componentSwap) { + Layout layout{}; + const auto channels = LayoutFor(format, layout) ? layout.channels : 4u; + constexpr std::uint32_t standardReversed = 2; + constexpr std::uint32_t alternateReversed = 3; + if (channels == 1) return componentSwap == alternateReversed; + return componentSwap != standardReversed && componentSwap != alternateReversed; +} + +DccKeys TextureClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes) { + if (resource.dccAddress == 0) return DccKeys::Uncompressed; + const auto keys = ReadDccKeys(resource.dccAddress, guestBytes); + if (keys == DccKeys::Uncompressed) return keys; + std::byte probe[16]{}; + if (!IsDccClear(keys) || !FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min(sizeof(probe), BytesPerElement(resource.format))))) { + static std::mutex reportedMutex; + static std::set> reported; + std::lock_guard lock(reportedMutex); + if (reported.size() < 32 && reported.insert({resource.baseAddress, static_cast(keys)}).second) std::fprintf(stderr, "[gpu] texture 0x%llx (format %u) has %s DCC keys at 0x%llx; its texels are read as stored\n", static_cast(resource.baseAddress), resource.format, DccKeysName(keys), static_cast(resource.dccAddress)); + return DccKeys::Uncompressed; + } + return keys; +} + +void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes) { + // Named for the [hooksync] attribution: the read goes through the flush hook. + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureRead); + if (keys == DccKeys::Uncompressed) GuestMemory::ReadCommitted(resource.baseAddress, bytes); + else FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, bytes); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index b0a7ba2d60..f73afaee7e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -2,30 +2,491 @@ #include "prx/libSceAgcDriver/Graphics/include/ColorTargetTransfer.hpp" #include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libc/include/General.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp" +#include "Optimization/include/Optimization/ShaderStageInputInfo.hpp" +#include +#include +#include +#include +#include +#include +#include #include #include +#include #include +#include +#include +#include namespace AgcDriver::Graphics { + namespace { -struct DrawStorage { - std::unique_ptr indices; - std::vector> vertices; - std::shared_ptr color; - std::shared_ptr pipeline; +// A guest texture format code with the same Vulkan format and texel size as a color buffer. +std::uint32_t GuestFormatFor(VkFormat format, std::uint32_t elementBytes) { + static std::mutex mutex; + static std::map, std::uint32_t> known; + std::lock_guard lock(mutex); + if (const auto found = known.find({format, elementBytes}); found != known.end()) return found->second; + for (std::uint32_t code = 0; code < 256; ++code) { + try { + if (!IsBlockCompressed(code) && ResolveTextureFormat(code) == format && BytesPerElement(code) == elementBytes) { + known.emplace(std::make_pair(format, elementBytes), code); + return code; + } + } catch (const std::exception&) { + } + } + throw std::runtime_error("AGC graphics: no guest texture format matches the color buffer format " + std::to_string(static_cast(format))); +} + +// The color buffer as a single-mip 2D surface descriptor (tile mode SW_64KB_R_X). +GuestTextureResource SurfaceForTarget(const ColorTarget& color) { + Require(color.tileMode == ColorTileMode::RenderTarget, "only 64 KiB tiled color targets are resident"); + GuestTextureResource surface{}; + surface.baseAddress = color.address; + surface.width = color.extent.width; + surface.height = color.extent.height; + surface.depthOrLastArray = 0; + surface.baseArray = 0; + surface.mipCount = 1; + surface.baseLevel = 0; + surface.lastLevel = 0; + surface.tileMode = TextureTileMode::kR64KBX; + surface.dimension = TextureDimension::k2D; + surface.format = GuestFormatFor(color.format, color.elementBytes); + surface.dstSelX = 4; + surface.dstSelY = 5; + surface.dstSelZ = 6; + surface.dstSelW = 7; + surface.dccAddress = color.dccAddress; + surface.dccAlphaOnMsb = color.dccAlphaOnMsb; + return surface; +} + +} + +namespace { + +void imageBarrier(const Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { + VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.srcAccessMask = sourceAccess; + barrier.dstAccessMask = destinationAccess; + barrier.oldLayout = oldLayout; + barrier.newLayout = newLayout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 0, nullptr, 0, nullptr, 1, &barrier); +} + +void memoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { + const VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, sourceAccess, destinationAccess}; + context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + +// The color surface as one untailed SW_64KB_R_X mip for the GPU detiler, with tightly packed linear rows. +TileMipLayout ColorTargetMip(const ColorTarget& color, const ColorTargetLayout& layout) { + TileMipLayout mip{}; + mip.width = color.extent.width; + mip.height = color.extent.height; + mip.blocksPerRow = layout.BlocksPerRow(); + mip.pitchBytes = color.extent.width * color.elementBytes; + mip.tiledSize = layout.Bytes(); + mip.linearSize = layout.LinearBytes(); + return mip; +} + +// APS5_PROFILE_DRAW: per-draw phase timers in microseconds (a recorded draw's phases are far below +// the millisecond the old print rounded to), totalled over 10 s in the [draws] line. +enum DrawPhase : std::size_t { PhaseValidate, PhaseVertex, PhaseSetup, PhaseReadTarget, PhasePrepare, PhaseLookup, PhaseResources, PhasePipeline, PhaseRecord, PhaseKeep, PhaseSync, PhaseWriteBack, PhaseDescribe, PhaseCount }; +constexpr std::array DrawPhaseNames{"validate", "vertex", "setup", "readTarget", "prepare", "lookup", "resources", "pipeline", "record", "keep", "sync", "writeBack", "describe"}; + +// Why a draw did not go into the recorder without a wait (counted in the [draws] line): its targets +// are not all resident, no recorder is active, a switch (APS5_SYNC_DRAWS, APS5_DUMP_TARGETS, +// APS5_SYNC_COMPLETION_DRAWS) forced it, or its completion work writes copied guest buffers or +// releases an address-based build's lease, which the CPU must not outrun (see `recorded` in Draw). +enum SyncReason : std::size_t { SyncNone, SyncNotResident, SyncNoRecorder, SyncDisabled, SyncCopiedWrites, SyncLease, SyncCount }; +constexpr std::array SyncReasonNames{"none", "non-resident target", "no recorder", "disabled", "copied writes", "lease"}; + +// What became of one draw, for the totals. +struct DrawOutcome { + // In the recorder; `waited` when the recorder was synced right after (copied writes or a lease). + bool recorded = false; + bool waited = false; + bool completion = false; + SyncReason reason = SyncNone; + // Shader validation memo (see CachedFragmentOutputs). + bool validateMemoized = false; + bool validateHit = false; + // A build that acquired the allocation registry lease (BDA), whose cost sits outside the + // build's own sub-phases. + bool addressBased = false; + // Resident target lookups, and those that took a millisecond or more. Whether a slow one + // re-uploaded the image or just walked its pages under contention is not visible from here + // (StorageTexture::Version advances on every MarkDirty as well as on an upload, and Generation + // is stamped afresh by every write-watch walk): the [texture] line's "reused" and "direct + // uploads" counters and APS5_TRACE_UPLOAD name the actual uploads. + std::uint64_t targetLookups = 0; + std::uint64_t slowLookups = 0; + double slowLookupUs = 0; + // The flush hook's fence waits made inside this draw (index and vertex reads, the target + // lookups' flushes, the build's texture lookups): nested in the draw's GpuMutex hold, so they + // are the part of the hold that is waiting rather than working. The draw's own syncs (a + // recorded-then-waited draw's recorder Sync, a synchronous draw's SubmitAndWait) are kept apart + // in ownSyncUs: those wait by design and would otherwise count as nested hook waits. + double hookWaitUs = 0; + double ownSyncUs = 0; }; +struct DrawProfile { + std::mutex mutex; + std::array totalsUs{}; + // The longest single draw's time per phase, and the longest draw: the [lock] line's 'draw' + // hold max (tens of ms against an average well under a millisecond) needs a phase name. + std::array maxUs{}; + double maxDrawUs = 0; + double hookWaitUs = 0; + double maxHookWaitUs = 0; + double ownSyncUs = 0; + // The resources build's sub-phases (ShaderResources::Timing) of draws only; dispatches report + // theirs in the [resources] line. `other` is the rest of the build (prepareAddressBindings: + // the registry lease and snapshots of an address-based build, the BDA table, the layout + // lookup), and how much of it address-based builds account for. + double bindingsUs = 0; + double uploadUs = 0; + double descriptorsUs = 0; + double otherUs = 0; + double addressOtherUs = 0; + std::uint64_t addressBuilds = 0; + std::uint64_t draws = 0; + std::uint64_t recorded = 0; + std::uint64_t waited = 0; + // Recorded draws whose write-back (BDA fault check) runs as a completion action instead of + // making the draw synchronous (see APS5_SYNC_COMPLETION_DRAWS). + std::uint64_t completion = 0; + std::array reasons{}; + // The CPU inside Graphics::Draw, split by outcome: a synchronous draw costs ten times a recorded + // one, so one average would only show the synchronous population. + double recordedUs = 0; + double waitedUs = 0; + double synchronousUs = 0; + std::uint64_t targetLookups = 0; + std::uint64_t slowLookups = 0; + double slowLookupUs = 0; + // Resource cache outcomes: hits, misses (built and inserted when reusable), entries that failed + // Revalidate, and draws that could not use the cache (synchronous, no variant id, or disabled). + // Which key words the misses differ in is the cache's own "[rescache] miss churn" line + // (ResourceCache::noteMiss compares each miss with the last key of the same variants). + std::uint64_t cacheHits = 0; + std::uint64_t cacheMisses = 0; + std::uint64_t cacheInvalidated = 0; + std::uint64_t uncacheable = 0; + // Shader validation memo (see CachedFragmentOutputs). + std::uint64_t validateHits = 0; + std::uint64_t validateMisses = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +DrawProfile& Profile() { + static DrawProfile profile; + return profile; +} + +// Adds one draw's phases to the totals and prints the [draws] and [rescache] lines every 10 s. +void reportDraw(const std::array& us, const ShaderResources::BuildTiming* built, const DrawOutcome& outcome) { + auto& profile = Profile(); + std::lock_guard lock(profile.mutex); + double drawUs = 0; + for (std::size_t i = 0; i < PhaseCount; ++i) { + profile.totalsUs[i] += us[i]; + profile.maxUs[i] = std::max(profile.maxUs[i], us[i]); + drawUs += us[i]; + } + profile.maxDrawUs = std::max(profile.maxDrawUs, drawUs); + profile.hookWaitUs += outcome.hookWaitUs; + profile.maxHookWaitUs = std::max(profile.maxHookWaitUs, outcome.hookWaitUs); + profile.ownSyncUs += outcome.ownSyncUs; + if (built != nullptr) { + profile.bindingsUs += built->bindingsMs * 1000.0; + profile.uploadUs += built->uploadMs * 1000.0; + profile.descriptorsUs += built->descriptorsMs * 1000.0; + const auto other = std::max(0.0, us[PhaseResources] - (built->bindingsMs + built->uploadMs + built->descriptorsMs) * 1000.0); + profile.otherUs += other; + if (outcome.addressBased) { + profile.addressOtherUs += other; + ++profile.addressBuilds; + } + } + ++profile.draws; + if (outcome.recorded) ++(outcome.waited ? profile.waited : profile.recorded); + if (outcome.completion) ++profile.completion; + ++profile.reasons[outcome.reason]; + (outcome.recorded ? (outcome.waited ? profile.waitedUs : profile.recordedUs) : profile.synchronousUs) += drawUs; + profile.targetLookups += outcome.targetLookups; + profile.slowLookups += outcome.slowLookups; + profile.slowLookupUs += outcome.slowLookupUs; + if (outcome.validateMemoized) ++(outcome.validateHit ? profile.validateHits : profile.validateMisses); + const auto now = std::chrono::steady_clock::now(); + if (now - profile.lastReport < std::chrono::seconds(10)) return; + profile.lastReport = now; + const auto synchronous = profile.draws - profile.recorded - profile.waited; + const auto average = [](double total, std::uint64_t count) { return count != 0 ? total / static_cast(count) : 0.0; }; + char line[1536]; + int n = std::snprintf(line, sizeof(line), "[draws] %llu draws over 10 s (%llu recorded avg %.0f us, of them %llu with completion; %llu recorded then waited avg %.0f us; %llu synchronous avg %.0f us; waited or synchronous because:", static_cast(profile.draws), static_cast(profile.recorded), average(profile.recordedUs, profile.recorded), static_cast(profile.completion), static_cast(profile.waited), average(profile.waitedUs, profile.waited), static_cast(synchronous), average(profile.synchronousUs, synchronous)); + const auto room = [&] { return n > 0 && static_cast(n) < sizeof(line); }; + for (std::size_t i = SyncNone + 1; i < SyncCount && room(); ++i) { + if (profile.reasons[i] != 0) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %llu", SyncReasonNames[i], static_cast(profile.reasons[i])); + } + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "):"); + for (std::size_t i = 0; i < PhaseCount && room(); ++i) { + if (profile.totalsUs[i] <= 0) continue; + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s=%.1fms", DrawPhaseNames[i], profile.totalsUs[i] / 1000.0); + if (i == PhaseResources && room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " (bindings %.1f, upload %.1f, descriptors %.1f, other %.1f of which %.1f in %llu address-based builds)", profile.bindingsUs / 1000.0, profile.uploadUs / 1000.0, profile.descriptorsUs / 1000.0, profile.otherUs / 1000.0, profile.addressOtherUs / 1000.0, static_cast(profile.addressBuilds)); + if (i == PhaseReadTarget && room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " (%llu resident lookups, %llu of them >= 1 ms = %.1f)", static_cast(profile.targetLookups), static_cast(profile.slowLookups), profile.slowLookupUs / 1000.0); + } + // The longest draw and the longest single phase of any draw (which phase a long 'draw' hold + // was), plus the hook waits nested inside the draws. + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; longest draw %.1f ms, longest phase of any draw:", profile.maxDrawUs / 1000.0); + for (std::size_t i = 0; i < PhaseCount && room(); ++i) { + if (profile.maxUs[i] < 500.0) continue; + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %.1f", DrawPhaseNames[i], profile.maxUs[i] / 1000.0); + } + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; hook waits inside draws %.1f ms (max %.1f; the draws' own syncs, %.1f ms, not counted)", profile.hookWaitUs / 1000.0, profile.maxHookWaitUs / 1000.0, profile.ownSyncUs / 1000.0); + std::fprintf(stderr, "%s\n", line); + std::fprintf(stderr, "[rescache] draws: %llu hits, %llu misses, %llu invalidated, %llu uncacheable; validation memo %llu hits / %llu misses (which key words the misses differ in: the miss churn line)\n", static_cast(profile.cacheHits), static_cast(profile.cacheMisses), static_cast(profile.cacheInvalidated), static_cast(profile.uncacheable), static_cast(profile.validateHits), static_cast(profile.validateMisses)); + profile.totalsUs.fill(0); + profile.maxUs.fill(0); + profile.maxDrawUs = profile.hookWaitUs = profile.maxHookWaitUs = profile.ownSyncUs = 0; + profile.bindingsUs = profile.uploadUs = profile.descriptorsUs = profile.otherUs = profile.addressOtherUs = 0; + profile.addressBuilds = 0; + profile.draws = profile.recorded = profile.waited = profile.completion = 0; + profile.reasons.fill(0); + profile.recordedUs = profile.waitedUs = profile.synchronousUs = 0; + profile.targetLookups = profile.slowLookups = 0; + profile.slowLookupUs = 0; + profile.cacheHits = profile.cacheMisses = profile.cacheInvalidated = profile.uncacheable = 0; + profile.validateHits = profile.validateMisses = 0; +} + +void countCache(std::uint64_t DrawProfile::*counter) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& stats = Profile(); + std::lock_guard lock(stats.mutex); + ++(stats.*counter); +} + +// ValidateShaders decodes every SPIR-V instruction of every stage on every draw. Its outcome depends +// only on the stages' compiled variants (a variant id names identical SPIR-V and binding layout), +// their push constant placement and vertex attribute shapes, and the state fields it checks, so the +// fragment output set is remembered by exactly those. Nothing here is keyed by pointer: the results +// a draw hands in live in a per-draw vector. A stage without a variant id is validated as before, +// except the rect-list control and evaluation stages, which are generated from the vertex and +// fragment results the key already names (the pipeline key treats them the same way). `memoized` +// says whether the memo applied, `hit` whether it answered. APS5_NO_VALIDATE_CACHE=1 validates +// every draw. +std::set CachedFragmentOutputs(const Context& context, std::span shaders, const State& state, bool& memoized, bool& hit) { + using Stage = ShaderRecompiler::ShaderStage; + static const bool disabled = std::getenv("APS5_NO_VALIDATE_CACHE") != nullptr; + memoized = false; + hit = false; + std::vector key; + const auto add = [&](auto value) { key.push_back(static_cast(value)); }; + bool keyed = !disabled; + if (keyed) { + key.reserve(40 + shaders.size() * 12); + add(shaders.size()); + for (const auto& shader : shaders) { + Require(shader.program != nullptr, "missing compiled shader"); + const auto& program = *shader.program; + const bool generated = state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation); + if (!generated && program.variantId == 0) { + keyed = false; + break; + } + add(shader.stage); + add(generated ? std::uint64_t{0} : program.variantId); + add(shader.pushConstantOffset); + add(program.pushConstants.size()); + add(program.bdaAbiVersion); + add(program.vertexAttributes.size()); + for (const auto& attribute : program.vertexAttributes) { + add(attribute.location); + add(attribute.components); + add(attribute.fetchIndex); + // The data format bits of the V# decide the attribute's signature. + add((attribute.resource.fields[3] >> 12u) & 0x7fu); + } + } + } + if (keyed) { + add(state.stages.path); + add(state.rectList); + add(state.topology); + add(state.cullMode); + add(state.colors.size()); + add(context.subgroup.subgroupSize); + add(context.subgroup.supportedStages); + add(context.subgroup.supportedOperations); + add(context.fragmentShaderBarycentric); + add(state.stages.mesh.has_value()); + if (state.stages.mesh) { + const auto& mesh = *state.stages.mesh; + add(mesh.inputPrimitive); + add(mesh.primitivesPerGroup); + add(mesh.verticesPerGroup); + add(mesh.maxVertices); + add(mesh.maxPrimitives); + add(mesh.threadsPerGroup); + add(mesh.ldsSizeDwords); + add(mesh.provokingVertex); + } + add(state.stages.tessellation.has_value()); + if (state.stages.tessellation) { + const auto& tessellation = *state.stages.tessellation; + add(tessellation.inputControlPoints); + add(tessellation.outputControlPoints); + add(tessellation.domain); + add(tessellation.partitioning); + add(tessellation.outputTopology); + } + } + static std::mutex mutex; + static std::map, std::set> memo; + if (keyed) { + memoized = true; + std::lock_guard lock(mutex); + if (const auto found = memo.find(key); found != memo.end()) { + hit = true; + return found->second; + } + } + auto outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric); + if (keyed) { + std::lock_guard lock(mutex); + // A handful of configurations recur; a runaway key space is dropped wholesale. + if (memo.size() >= 1024) memo.clear(); + memo.emplace(std::move(key), outputs); + } + return outputs; +} + +// The resource cache key of a recorded draw: a marker no compute key starts with (those begin with +// the stage), the device handle (the cache is process-wide and the driver replaces the headless +// device with the windowed one while workers may still use the old one: an entry's descriptor set +// and pooled buffers belong to the device that built it) and every stage's content key +// (ResourceCache::noteMiss walks this layout to attribute misses, so it changes together with it). +// With `ranges`, also the render target and the index buffer the build's alias checks compared the +// guest buffers against; without it the checks are repeated for a hit (CheckBufferAliases), since +// neither range is part of the descriptor set (the target is attached, the index buffer copied per +// draw), so a target or index ring that moves per frame does not miss on every draw. That guards a +// workload with such rings; in the profiled menu stage every draw was DRAW_INDEX_AUTO (index range +// 0) onto one fixed target, so its misses come from the stages' descriptor words themselves. +ResourceCache::Key DrawResourceKey(const Context& context, std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::uint64_t indexBytes, bool ranges) { + ResourceCache::Key key{0xffffffffu}; + const auto append64 = [&](std::uint64_t value) { + key.push_back(static_cast(value)); + key.push_back(static_cast(value >> 32u)); + }; + append64(reinterpret_cast(context.device)); + key.push_back(static_cast(shaders.size())); + for (const auto& shader : shaders) { + const auto part = ShaderResources::ContentKey(shader); + key.push_back(static_cast(part.size())); + key.insert(key.end(), part.begin(), part.end()); + } + if (ranges) { + append64(target.address); + append64(target.bytes); + append64(indexAddress); + append64(indexBytes); + } + return key; +} + +// The alias checks the build makes for every guest buffer descriptor (ShaderResources:: +// addGuestBuffer), repeated for a cached build whose own checks compared the buffers against +// another draw's render target and index buffer. The same conditions and messages, so a draw that +// would have failed its build fails its hit. +void CheckBufferAliases(std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::uint64_t indexBytes) { + const auto overlap = [](std::uint64_t first, std::uint64_t firstSize, std::uint64_t second, std::uint64_t secondSize) { return first < second + secondSize && second < first + firstSize; }; + for (const auto& shader : shaders) { + for (const auto& binding : shader.program->bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; + const auto& words = binding.guestDescriptor; + for (std::size_t offset = 0; offset + 4 <= words.size(); offset += 4) { + const ShaderRecompiler::ShaderBufferResource descriptor{{words[offset], words[offset + 1], words[offset + 2], words[offset + 3]}}; + const auto address = descriptor.Base48(); + const auto size = descriptor.GetSize(); + Require(!overlap(address, size, target.address, target.bytes), "shader buffer aliases the render target"); + Require(!overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer"); + } + } + } +} + +} + +std::shared_ptr>> DrawCopiedWriters() { + // Never destroyed: a completion action of a batch still in flight at static teardown may erase from it. + static auto* const writers = new std::shared_ptr>>(std::make_shared>>()); + return *writers; } void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { PerformanceTimer timing("Graphics.Draw"); + // APS5_PROFILE_DRAW prints the time of each phase of the draw (microseconds) and the [draws] totals. + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto phaseStart = std::chrono::steady_clock::now(); + std::array us{}; + const auto phase = [&](DrawPhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + us[which] += std::chrono::duration(now - phaseStart).count(); + phaseStart = now; + }; + // APS5_TRACE_DRAWS (together with APS5_PROFILE_DRAW; alone it only enables Driver.cpp's own + // [draw] lines) additionally prints one line per draw with its phases and, for synchronous + // draws, their inputs: per-draw string building at ~1400 draws per 10 s is itself a cost the + // totals must not carry. + static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; + DrawOutcome outcome; + const ShaderResources::BuildTiming* built = nullptr; + // The thread's GPU waits so far: the difference at the report, less the draw's own syncs + // (`ownWaitedMs`, sampled around them below), is what this draw waited for through the flush + // hook (nested in the draw's hold, see DrawOutcome::hookWaitUs). + const auto waitedBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; + double ownWaitedMs = 0; + // One line per draw with the phases that took time (tracing), and the 10 s totals. + const auto report = [&](const char* suffix) { + if (!profile) return; + outcome.hookWaitUs = std::max(0.0, Recorder::ThreadWaitedMs() - waitedBefore - ownWaitedMs) * 1000.0; + outcome.ownSyncUs = ownWaitedMs * 1000.0; + if (traceDraws) { + char line[512]; + int n = std::snprintf(line, sizeof(line), "[draw] %ux%u %zu targets%s:", state.renderExtent.width, state.renderExtent.height, state.colors.size(), suffix); + for (std::size_t i = 0; i < PhaseCount && n > 0 && static_cast(n) < sizeof(line); ++i) { + if (us[i] <= 0) continue; + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s=%.0fus", DrawPhaseNames[i], us[i]); + } + std::fprintf(stderr, "%s\n", line); + } + reportDraw(us, built, outcome); + }; + APS5_LOG_OUT_DEBUG("Draw indices=%u instances=%u indexSize=%u flags=%u indexAddress=0x%llx", draw.indexCount, draw.instanceCount, draw.indexSize, draw.flags, static_cast(draw.indexAddress)); + APS5_LOG_OUT_DEBUG("State colorTarget=%u render=%ux%u colorAddress=0x%llx colorBytes=%llu colorExtent=%ux%u", state.hasColorTarget ? 1u : 0u, state.renderExtent.width, state.renderExtent.height, static_cast(state.color.address), static_cast(state.color.bytes), state.color.extent.width, state.color.extent.height); + APS5_LOG_OUT_DEBUG("Viewport x=%f y=%f w=%f h=%f minDepth=%f maxDepth=%f", state.viewport.x, state.viewport.y, state.viewport.width, state.viewport.height, state.viewport.minDepth, state.viewport.maxDepth); + APS5_LOG_OUT_DEBUG("Scissor x=%d y=%d w=%u h=%u topology=%u cullMode=0x%x frontFace=%u", state.scissor.offset.x, state.scissor.offset.y, state.scissor.extent.width, state.scissor.extent.height, static_cast(state.topology), static_cast(state.cullMode), static_cast(state.frontFace)); Require(draw.indexed ? draw.flags == 0 : (draw.flags & ~0x20u) == 0, "draw modifiers are unsupported"); if (draw.indexed) { Require(draw.indexSize == 2 || draw.indexSize == 4, "only uint16 and uint32 index buffers are supported"); @@ -38,15 +499,21 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& } Require(draw.indexCount != 0 && draw.instanceCount != 0, "zero-count indexed draws are unsupported"); const auto indexBytes = static_cast(draw.indexCount) * draw.indexSize; + APS5_LOG_OUT_DEBUG("Index buffer bytes=%llu", static_cast(indexBytes)); Require(indexBytes <= std::numeric_limits::max(), "index buffer size overflow"); if (draw.indexed) GuestMemory::CheckRange(reinterpret_cast(draw.indexAddress), static_cast(indexBytes), draw.indexSize); + APS5_LOG_CHARS_OUT_DEBUG("Index buffer range OK"); Require(!draw.indexed || !state.hasColorTarget || draw.indexAddress + indexBytes <= state.color.address || state.color.address + state.color.bytes <= draw.indexAddress, "index buffer aliases the render target"); if (state.rectList) Require(draw.indexCount % 3 == 0, "incomplete rect-list primitive"); - ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric); + APS5_LOG_CHARS_OUT_DEBUG("ValidateShaders"); + const auto fragmentOutputs = CachedFragmentOutputs(context, shaders, state, outcome.validateMemoized, outcome.validateHit); + APS5_LOG_CHARS_OUT_DEBUG("ValidateShaders OK"); const auto shaderStages = PipelineStages(shaders); + APS5_LOG_OUT_DEBUG("PipelineStages=0x%x", static_cast(shaderStages)); std::uint32_t meshGroups = 0; if (state.stages.mesh) { Require(draw.firstVertex == 0 && draw.firstInstance == 0, "mesh draw offsets are unsupported"); + APS5_LOG_CHARS_OUT_DEBUG("Mesh path"); Require(context.meshShader, "device does not support mesh shaders"); const auto& mesh = *state.stages.mesh; const auto inputSize = mesh.inputPrimitive == 1 ? 1u : mesh.inputPrimitive == 2 ? 2u : 3u; @@ -54,13 +521,15 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& const auto step = mesh.inputPrimitive == 6 ? 1u : inputSize; const auto primitives = (draw.indexCount - inputSize) / step + 1u; meshGroups = (primitives - 1u) / mesh.primitivesPerGroup + 1u; + APS5_LOG_OUT_DEBUG("Mesh primitives=%u groups=%u", primitives, meshGroups); Require(meshGroups <= context.meshLimits.maxMeshWorkGroupCount[0] && draw.instanceCount <= context.meshLimits.maxMeshWorkGroupCount[1] && static_cast(meshGroups) * draw.instanceCount <= context.meshLimits.maxMeshWorkGroupTotalCount, "mesh draw exceeds workgroup count limits"); } if (state.stages.tessellation) Require(draw.indexCount % state.stages.tessellation->inputControlPoints == 0, "incomplete tessellation patch"); + // Viewport and scissor are dynamic pipeline state, so their limits are checked here per draw. + ValidateViewport(context, state.viewport); timing.Mark("validate"); - Require(context.renderCache != nullptr && context.drawQueue != nullptr && context.graphicsPipelines != nullptr, "device graphics execution caches are unavailable"); - auto storage = std::make_shared(); - auto& indices = storage->indices; + phase(PhaseValidate); + std::unique_ptr indices; std::uint32_t maxIndex = draw.indexed ? 0u : draw.firstVertex + draw.indexCount - 1u; if (draw.indexed) { indices = std::make_unique(context, static_cast(indexBytes), VK_BUFFER_USAGE_INDEX_BUFFER_BIT); @@ -78,10 +547,12 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& maxIndex = std::max(maxIndex, index); } } + APS5_LOG_CHARS_OUT_DEBUG("Index validation OK"); timing.Mark("index_upload"); const auto& attributes = shaders.front().program->vertexAttributes; - static_cast(BuildVertexInputLayout(context, attributes)); - auto& vertexBuffers = storage->vertices; + // Validates the vertex descriptors; the layout also keys and builds the pipeline below. + const auto vertexInput = BuildVertexInputLayout(context, attributes); + std::vector> vertexBuffers; std::vector vertexHandles; std::vector vertexOffsets(attributes.size(), 0); for (const auto& attribute : attributes) { @@ -95,28 +566,261 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& vertexHandles.push_back(buffer->Handle()); vertexBuffers.push_back(std::move(buffer)); } - timing.Mark("vertex_upload"); - auto resources = std::make_shared(context, shaders, state.color, draw.indexAddress, static_cast(indexBytes), snapshots); - timing.Mark("shader_resources"); - if (state.hasColorTarget) { - const ColorTargetLayout colorLayout(state.color.extent.width, state.color.extent.height, state.color.tileMode); - Require(state.color.bytes == colorLayout.Bytes(), "color target transfer size mismatch"); - storage->color = context.renderCache->Get(state.color, state.blend.blendEnable != 0); + phase(PhaseVertex); + // One binding per color attachment. Tiled targets are detiled and retiled on the GPU in video memory; + // only changed guest blocks are stored back. + struct TargetBinding { + ColorTarget color; + bool gpuTiling = false; + // Resident target: the surface's cached storage image is attached directly and marked dirty + // afterwards, so nothing is copied in or out per draw. + std::shared_ptr resident; + // Linear pixels converted on the CPU, for targets the GPU detiler does not handle. + std::unique_ptr transfer; + // Guest bytes in host memory, and their device-local copies for the detiler. + std::unique_ptr tiled; + std::unique_ptr tiledDevice; + std::unique_ptr linearDevice; + std::vector original; + TileMipLayout mip{}; + std::unique_ptr target; + // APS5_DUMP_TARGETS: the rendered linear pixels, read back for target_
_.raw. + std::unique_ptr dump; + }; + // Debug aid: APS5_DUMP_TARGETS= saves the first n renders of every color target as a raw file + // (u32 width, u32 height, u32 VkFormat, then tightly packed rows). + static const int dumpLimit = [] { const char* text = std::getenv("APS5_DUMP_TARGETS"); return text ? std::atoi(text) : 0; }(); + static std::mutex dumpMutex; + static std::map dumped; + std::vector targets(state.colors.size()); + std::vector targetViews; + for (std::size_t index = 0; index < state.colors.size(); ++index) { + auto& binding = targets[index]; + binding.color = state.colors[index]; + const auto& color = binding.color; + binding.gpuTiling = color.tileMode == ColorTileMode::RenderTarget && context.detiler != nullptr; + APS5_LOG_OUT_DEBUG("Creating color target %zu address=0x%llx bytes=%llu extent=%ux%u", index, static_cast(color.address), static_cast(color.bytes), color.extent.width, color.extent.height); + const ColorTargetLayout colorLayout(color.extent.width, color.extent.height, color.tileMode, color.elementBytes); + constexpr VkBufferUsageFlags copies = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + phase(PhaseSetup); + // Debug aid: APS5_NO_RESIDENT_TARGETS=1 copies every target in and out again. + static const bool residentTargets = std::getenv("APS5_NO_RESIDENT_TARGETS") == nullptr; + if (binding.gpuTiling && residentTargets) { + // The lookup refreshes the image on every draw (StorageTexture::Refresh: FlushPending, + // CollectWrites over the target's pages, the DCC key scan of TextureClearKeys, then + // UnchangedSince). The page walk is skipped while the worker's collect epoch lasts + // (GuestMemory::BumpCollectEpoch: ordering points of the queue, or every packet under + // APS5_PACKET_EPOCH=1); the key scan runs on every lookup regardless. Lookups of a + // millisecond or more are counted apart; the [texture] line says how many uploaded. + const auto lookupStart = std::chrono::steady_clock::now(); + try { + auto resident = CachedStorageSurface(context, SurfaceForTarget(color)); + Require(resident->Attachable(), "storage format cannot be a color attachment"); + Require(resident->GuestBytes() == colorLayout.Bytes(), "resident image layout differs from the color layout"); + binding.resident = std::move(resident); + } catch (const std::exception& error) { + static std::mutex reportMutex; + static std::set reported; + std::lock_guard lock(reportMutex); + if (reported.insert(color.address).second) std::fprintf(stderr, "[gpu] color target 0x%llx stays non-resident: %s\n", static_cast(color.address), error.what()); + } + if (profile) { + const auto lookupUs = std::chrono::duration(std::chrono::steady_clock::now() - lookupStart).count(); + ++outcome.targetLookups; + if (lookupUs >= 1000.0) { + ++outcome.slowLookups; + outcome.slowLookupUs += lookupUs; + } + } + } + if (binding.resident != nullptr) { + phase(PhaseReadTarget); + targetViews.push_back(binding.resident->AttachmentView(color.format)); + continue; + } + if (binding.gpuTiling) { + binding.mip = ColorTargetMip(color, colorLayout); + binding.tiled = std::make_unique(context, colorLayout.Bytes(), copies); + binding.tiledDevice = std::make_unique(context, colorLayout.Bytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + binding.linearDevice = std::make_unique(context, colorLayout.LinearBytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + binding.original.resize(colorLayout.Bytes()); + GuestMemory::Read(color.address, binding.original, colorLayout.Alignment()); + std::memcpy(binding.tiled->Bytes().data(), binding.original.data(), binding.original.size()); + } else { + binding.transfer = std::make_unique(context, colorLayout.LinearBytes(), copies); + ReadColorTarget(color, binding.transfer->Bytes()); + } + // A fast-cleared DCC target holds its clear value wherever the draw does not write. + if (color.dccAddress != 0) { + const auto keys = ReadDccKeys(color.dccAddress, colorLayout.Bytes()); + if (IsDccClear(keys)) { + const auto pixels = binding.gpuTiling ? binding.tiled->Bytes() : binding.transfer->Bytes(); + if (!FillDccClear(color.format, keys, color.dccAlphaOnMsb, pixels)) { + static std::set> reported; + if (reported.size() < 32 && reported.insert({color.address, static_cast(keys)}).second) std::fprintf(stderr, "[gpu] color target 0x%llx (VkFormat %d) has %s DCC keys; its stored texels are used\n", static_cast(color.address), static_cast(color.format), DccKeysName(keys)); + if (binding.gpuTiling) std::memcpy(pixels.data(), binding.original.data(), binding.original.size()); + else ReadColorTarget(color, pixels); + } + } + } + phase(PhaseReadTarget); + binding.target = std::make_unique(context, color, state.blends[index].blendEnable != 0); + targetViews.push_back(binding.target->View()); } - timing.Mark("render_target_cache"); - storage->pipeline = context.graphicsPipelines->Get(state, storage->color, *resources, shaders); - auto& pipeline = *storage->pipeline; - timing.Mark("pipeline_cache"); - const auto commands = context.drawQueue->Begin(context); + phase(PhasePrepare); + // A draw whose targets are all resident is recorded into the device's open batch like a dispatch: + // the CPU does not wait for it, and the drain before it goes away (queue order and the barriers + // below keep it behind earlier recorded work); see `recorded` below for the buffers that still + // need a synchronous draw. Other draws keep their own synchronous batch. Debug aid: + // APS5_SYNC_DRAWS=1 makes every draw synchronous. + static const bool recordDraws = std::getenv("APS5_SYNC_DRAWS") == nullptr; + auto* recorder = Recorder::Active(); + const bool recordable = recordDraws && recorder != nullptr && dumpLimit == 0 && std::all_of(targets.begin(), targets.end(), [](const TargetBinding& binding) { return binding.resident != nullptr; }); + APS5_LOG_CHARS_OUT_DEBUG("Creating ShaderResources"); + // A recordable draw whose stages' compiled content repeats an earlier one binds that build's + // descriptor set when it is still valid (see ResourceCache; the dispatch path does the same). + // Only recordable draws take part: a synchronous draw writes its resources back, which a shared + // object must never do. Revalidate proves a hit by texture identity, which needs the texture + // caches. Push constants still come from this draw's stages. APS5_NO_DRAW_RESOURCE_CACHE=1 + // builds every draw's resources as before. + static const bool noDrawResourceCache = std::getenv("APS5_NO_DRAW_RESOURCE_CACHE") != nullptr; + static const bool noTextureCache = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + // The render target and index ranges stay out of the key (see DrawResourceKey); a hit repeats + // the alias checks instead. Debug aid: APS5_NO_DRAW_KEY_TRIM=1 keys them as before. + static const bool trimKey = std::getenv("APS5_NO_DRAW_KEY_TRIM") == nullptr; + const bool cacheable = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->variantId != 0; }); + std::shared_ptr resources; + ResourceCache::Key contentKey; + if (cacheable) { + contentKey = DrawResourceKey(context, shaders, state.color, draw.indexAddress, indexBytes, !trimKey); + if (auto cached = SharedResourceCache().Find(contentKey)) { + if (cached->Revalidate(shaders)) { + if (trimKey) CheckBufferAliases(shaders, state.color, draw.indexAddress, indexBytes); + resources = std::move(cached); + countCache(&DrawProfile::cacheHits); + } else { + SharedResourceCache().Remove(contentKey); + countCache(&DrawProfile::cacheInvalidated); + } + } + } else { + countCache(&DrawProfile::uncacheable); + } + phase(PhaseLookup); + if (resources == nullptr) { + resources = std::make_shared(context, shaders, state.color, draw.indexAddress, static_cast(indexBytes), snapshots); + built = &resources->Timing(); + outcome.addressBased = resources->HoldsLease(); + if (cacheable) countCache(&DrawProfile::cacheMisses); + } + phase(PhaseResources); + APS5_LOG_CHARS_OUT_DEBUG("ShaderResources created"); + APS5_LOG_CHARS_OUT_DEBUG("Creating Pipeline"); + // Vulkan leaves attachments a pixel shader has no output for undefined, so their writes are masked + // (shaders that only store to images or buffers keep their targets as they were). The state is + // copied only when a mask must change. + std::optional masked; + for (std::size_t index = 0; index < state.blends.size(); ++index) { + if (fragmentOutputs.contains(static_cast(index)) || state.blends[index].colorWriteMask == 0) continue; + if (!masked.has_value()) masked = state; + masked->blends[index].colorWriteMask = 0; + } + const State& pipelineState = masked.has_value() ? *masked : state; + auto pipeline = CachedPipeline(context, pipelineState, vertexInput, *resources, shaders); + // Resident targets' views are stable while their storage image lives, so the framebuffer is + // reused with the pipeline; a per-draw RenderTarget gets a framebuffer of its own. + std::vector> owners; + owners.reserve(targets.size()); + for (const auto& binding : targets) owners.push_back(binding.resident); + auto framebuffer = pipeline->AcquireFramebuffer(targetViews, owners, state.renderExtent); + phase(PhasePipeline); + APS5_LOG_CHARS_OUT_DEBUG("Pipeline created"); + APS5_LOG_CHARS_OUT_DEBUG("Creating CommandBatch"); + // A draw whose only completion work is a BDA fault check (the rect-list stages carry a fault + // buffer, so every rect-list draw needed one) is recorded too, with the check as a completion + // action like a dispatch's write-back: its own batch would first wait for everything recorded + // before it. A draw that writes a copied buffer (at the menu stage: 0x48-byte constant buffers + // inside an import but not at its storage offset alignment, so GuestBufferMemory copies them + // and every guest buffer counts as writable) is recorded and then waited for at once: the + // write-back runs before the next packet, without the separate command batch and fence a + // synchronous draw takes. The copied write-back is listed in DrawCopiedWriters, which lets such + // draws run without the wait; both consumers of the device's list in VulkanDevice.cpp + // (DispatchIndirect's argument check and the fill HLE's range check, see Draw.hpp) consult this + // list too. A draw holding an address-based build's lease (which pins guest allocations until + // the write-back releases it) runs free like the dispatch path's: the completion releases the + // lease, and a guest thread that needs a leased allocation syncs the recorder through the + // registry's pin waiter (see SyncLeaseWork); APS5_SYNC_LEASE_DISPATCH=1 waits for such draws + // at once as before. Debug aids: + // APS5_SYNC_COMPLETION_DRAWS=1 keeps every draw with completion work synchronous; + // APS5_NO_RECORDER_SYNC_DRAWS=1 gives the waited-for draws their own batch as before; + // APS5_NO_RECORD_COPIED_DRAWS=1 waits for copied-write draws at once instead of listing them. + static const bool syncCompletionDraws = std::getenv("APS5_SYNC_COMPLETION_DRAWS") != nullptr; + static const bool recorderSyncDraws = std::getenv("APS5_NO_RECORDER_SYNC_DRAWS") == nullptr; + static const bool recordCopiedDraws = std::getenv("APS5_NO_RECORD_COPIED_DRAWS") == nullptr; + const bool completion = resources->NeedsCompletion(); + const bool copiedWrites = completion && resources->HasCopiedWrites(); + // A lease only forces the wait when deferred release is off. + const bool lease = resources->HoldsLease() && SyncLeaseWork(); + outcome.completion = completion; + outcome.recorded = recordable; + // Whether the copied write-back is listed in DrawCopiedWriters instead of waited for. + bool listed = false; + if (!recordable) { + outcome.reason = !recordDraws || dumpLimit != 0 ? SyncDisabled : recorder == nullptr ? SyncNoRecorder : SyncNotResident; + } else if (completion && (syncCompletionDraws || copiedWrites || lease)) { + outcome.reason = syncCompletionDraws ? SyncDisabled : lease ? SyncLease : SyncCopiedWrites; + if (syncCompletionDraws) outcome.recorded = false; + else if (copiedWrites && !lease && recordCopiedDraws) listed = true; + else if (recorderSyncDraws) outcome.waited = true; + else outcome.recorded = false; + } + const bool recorded = outcome.recorded; + // A build this recorded draw can share with later identical ones goes into the cache (a cache + // hit is reusable by construction, so `recorded` holds for it; one with completion work is + // never reusable). + if (cacheable && built != nullptr && recorded && resources->Reusable()) SharedResourceCache().Insert(contentKey, resources); + std::optional batch; + if (!recorded) { + // The color-target detiles below take their descriptor sets from a fresh batch. + if (context.detiler != nullptr) context.detiler->BeginBatch(); + batch.emplace(context); + } + const auto commands = recorded ? recorder->Commands() : batch->Handle(); + APS5_LOG_CHARS_OUT_DEBUG("CommandBatch created"); VkMemoryBarrier upload{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; upload.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; upload.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | shaderStages, 0, 1, &upload, 0, nullptr, 0, nullptr); - if (storage->color) storage->color->Begin(commands); - pipeline.Begin(commands, state.renderExtent); - resources->Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline.Layout()); - pipeline.PushConstants(commands, shaders); + APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded"); + for (auto& binding : targets) { + if (binding.resident != nullptr) { + // Earlier recorded work (dispatches, the previous draw) wrote the image in the general layout. + imageBarrier(context, commands, binding.resident->Image(), VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); + continue; + } + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {binding.color.extent.width, binding.color.extent.height, 1}; + const VkBuffer linear = binding.gpuTiling ? binding.linearDevice->Handle() : binding.transfer->Handle(); + if (binding.gpuTiling) { + CopyBuffer(context, commands, binding.tiled->Handle(), 0, binding.tiledDevice->Handle(), 0, binding.original.size()); + memoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + context.detiler->Dispatch(commands, TextureTileMode::kR64KBX, binding.color.elementBytes, binding.tiledDevice->Handle(), 0, binding.linearDevice->Handle(), 0, binding.mip); + memoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + } + imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdCopyBufferToImage")(commands, linear, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); + imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); + } + APS5_LOG_OUT_DEBUG("Beginning pipeline renderExtent=%ux%u", state.renderExtent.width, state.renderExtent.height); + pipeline->Begin(commands, *framebuffer, state.renderExtent, state.viewport, state.scissor); + APS5_LOG_CHARS_OUT_DEBUG("Pipeline Begin OK"); + resources->Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->Layout()); + APS5_LOG_CHARS_OUT_DEBUG("Resources bound"); + pipeline->PushConstants(commands, shaders); + APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded"); if (state.stages.mesh) { + APS5_LOG_OUT_DEBUG("vkCmdDrawMeshTasksEXT groups=%u instances=%u", meshGroups, draw.instanceCount); context.Function("vkCmdDrawMeshTasksEXT")(commands, meshGroups, draw.instanceCount, 1); } else { if (!vertexHandles.empty()) context.Function("vkCmdBindVertexBuffers")(commands, 0, static_cast(vertexHandles.size()), vertexHandles.data(), vertexOffsets.data()); @@ -127,14 +831,170 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& context.Function("vkCmdDraw")(commands, draw.indexCount, draw.instanceCount, draw.firstVertex, draw.firstInstance); } } + APS5_LOG_CHARS_OUT_DEBUG("Draw recorded"); context.Function("vkCmdEndRenderPass")(commands); + APS5_LOG_CHARS_OUT_DEBUG("Render pass ended"); + for (auto& binding : targets) { + if (binding.resident != nullptr) { + imageBarrier(context, commands, binding.resident->Image(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + continue; + } + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {binding.color.extent.width, binding.color.extent.height, 1}; + imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + const VkBuffer linear = binding.gpuTiling ? binding.linearDevice->Handle() : binding.transfer->Handle(); + VkBufferMemoryBarrier reuse{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + reuse.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + reuse.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + reuse.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + reuse.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + reuse.buffer = linear; + reuse.size = VK_WHOLE_SIZE; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &reuse, 0, nullptr); + context.Function("vkCmdCopyImageToBuffer")(commands, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear, 1, ©); + if (binding.gpuTiling) { + memoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + if (dumpLimit > 0) { + std::lock_guard lock(dumpMutex); + if (dumped[binding.color.address] < dumpLimit) { + binding.dump = std::make_unique(context, binding.linearDevice->Size(), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + CopyBuffer(context, commands, binding.linearDevice->Handle(), 0, binding.dump->Handle(), 0, binding.linearDevice->Size()); + } + } + context.detiler->Dispatch(commands, TextureTileMode::kR64KBX, binding.color.elementBytes, binding.linearDevice->Handle(), 0, binding.tiledDevice->Handle(), 0, binding.mip, true); + memoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, binding.tiledDevice->Handle(), 0, binding.tiled->Handle(), 0, binding.original.size()); + } + } VkMemoryBarrier download{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; download.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT; download.dstAccessMask = VK_ACCESS_HOST_READ_BIT; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | shaderStages, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &download, 0, nullptr, 0, nullptr); - timing.Mark("command_record"); - context.drawQueue->Enqueue(std::move(resources), std::move(storage)); - timing.Mark("enqueue"); + APS5_LOG_CHARS_OUT_DEBUG("Download barrier recorded"); + APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait"); + phase(PhaseRecord); + if (recorded) { + // Everything the recorded draw uses lives until its batch completes. + struct Kept { + std::shared_ptr resources; + std::shared_ptr pipeline; + std::shared_ptr framebuffer; + std::unique_ptr indices; + std::vector> vertexBuffers; + std::vector> targets; + }; + auto kept = std::make_shared(); + kept->resources = resources; + kept->pipeline = pipeline; + kept->framebuffer = framebuffer; + kept->indices = std::move(indices); + kept->vertexBuffers = std::move(vertexBuffers); + for (auto& binding : targets) kept->targets.push_back(binding.resident); + recorder->Keep(std::move(kept)); + // Counted for the [address-sync] line: a lease released by the completion (the pin waiter + // finishes the recorder up to the open batch, which holds the kept resources and gets the + // next serial), or waited for below (any wait releases it at once). + if (resources->HoldsLease()) CountLeaseOutcome(outcome.waited, outcome.waited ? 0 : recorder->Submissions() + 1); + // Every use of a (possibly shared) resources object registers its GPU writes anew. + resources->MarkGpuWrites(*recorder); + // The write-back (fault check, copied buffers) runs when the batch completed; a fault is + // reported by the recorder ("deferred write-back failed") instead of thrown out of the draw. + if (listed) { + // As VulkanDevice::dispatch lists its copied writers: delisted before the write-back + // (one that fails must not keep indirect dispatches on the CPU), listed after the + // registration (a throw there leaves nothing behind). + auto writers = DrawCopiedWriters(); + recorder->OnComplete([resources, writers] { + writers->erase(std::remove(writers->begin(), writers->end(), resources), writers->end()); + resources->WriteBackBuffers(); + }); + writers->push_back(resources); + } else if (completion) { + recorder->OnComplete([resources] { resources->WriteBackBuffers(); }); + } + for (auto& binding : targets) binding.resident->MarkDirty(); + phase(PhaseKeep); + if (outcome.waited) { + // The wait the dispatch path makes for such work (source 3, "address-based"): the + // write-back runs before the next packet, so the CPU never reads copied results or + // touches a leased allocation before they landed. + Recorder::CountSync(3); + const auto ownBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; + recorder->Sync(); + if (profile) ownWaitedMs += Recorder::ThreadWaitedMs() - ownBefore; + phase(PhaseSync); + } + report(outcome.waited ? (outcome.reason == SyncLease ? " recorded then waited (lease)" : " recorded then waited (copied writes)") : completion ? " recorded with completion" : " recorded"); + return; + } + { + // The recorder sync SubmitAndWait makes is the draw's own wait, not a nested hook wait. + const auto ownBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; + batch->SubmitAndWait(); + if (profile) ownWaitedMs += Recorder::ThreadWaitedMs() - ownBefore; + } + phase(PhaseSync); + APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait OK"); + for (const auto& binding : targets) GuestMemory::CheckRange(reinterpret_cast(binding.color.address), binding.color.bytes, 256, true); + for (auto& binding : targets) { + if (!binding.dump) continue; + std::lock_guard lock(dumpMutex); + char name[64]; + std::snprintf(name, sizeof(name), "target_%llx_%d.raw", static_cast(binding.color.address), dumped[binding.color.address]++); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t header[3] = {binding.color.extent.width, binding.color.extent.height, static_cast(binding.color.format)}; + std::fwrite(header, sizeof(header), 1, file); + std::fwrite(binding.dump->Bytes().data(), 1, binding.dump->Bytes().size(), file); + std::fclose(file); + } + } + APS5_LOG_CHARS_OUT_DEBUG("Shader resources WriteBack"); + resources->WriteBack(); + APS5_LOG_CHARS_OUT_DEBUG("Shader resources WriteBack OK"); + static const bool skipTargetWrite = std::getenv("APS5_NO_TARGET_WRITEBACK") != nullptr; + for (const auto& binding : targets) { + if (skipTargetWrite) break; + if (binding.resident != nullptr) { + // The results stay on the GPU until something reads the target's memory. + binding.resident->MarkDirty(); + continue; + } + if (binding.gpuTiling) GuestMemory::WriteChanged(binding.color.address, binding.tiled->Bytes(), binding.original); + else WriteColorTarget(binding.color, binding.transfer->Bytes()); + // The stored texels are the whole target now, so later reads must see them rather than a fast clear. + MarkDccUncompressed(binding.color.dccAddress, ColorTargetLayout(binding.color.extent.width, binding.color.extent.height, binding.color.tileMode, binding.color.elementBytes).Bytes()); + } + phase(PhaseWriteBack); + if (profile && traceDraws) { + // A synchronous draw's inputs and target sample are described as a rendering debug aid; + // the ~0.5 ms that takes (Describe samples every bound range and reads its DCC keys) is + // shown as its own phase and kept out of the plain profile. + std::size_t nonzero = 0; + std::size_t sampled = 0; + if (!targets.empty() && targets.front().resident == nullptr) { + const auto bytes = targets.front().gpuTiling ? targets.front().tiled->Bytes() : targets.front().transfer->Bytes(); + sampled = bytes.size() / 64; + for (std::size_t i = 0; i < bytes.size(); i += 64) nonzero += bytes[i] != std::byte{0}; + } + std::fprintf(stderr, "[draw] inputs:%s\n", resources->Describe().c_str()); + if (targets.size() > 1) { + std::string list; + for (const auto& binding : targets) { + char entry[64]; + std::snprintf(entry, sizeof(entry), " 0x%llx(%d,%ux%u)", static_cast(binding.color.address), static_cast(binding.color.format), binding.color.extent.width, binding.color.extent.height); + list += entry; + } + std::fprintf(stderr, "[draw] targets:%s\n", list.c_str()); + } + phase(PhaseDescribe); + char suffix[160]; + std::snprintf(suffix, sizeof(suffix), " synchronous (%s), first 0x%llx format %d (%zu of %zu sampled bytes nonzero)", SyncReasonNames[outcome.reason], static_cast(state.color.address), static_cast(state.color.format), nonzero, sampled); + report(suffix); + } else { + report(" synchronous"); + } + APS5_LOG_CHARS_OUT_DEBUG("Draw finished"); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp index 1f01f27688..55c236b729 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp @@ -2,8 +2,6 @@ #include "prx/libSceAgcDriver/Graphics/shaders/ColorTransfer_spv.h" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include -#include -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" namespace AgcDriver::Graphics { @@ -72,11 +70,11 @@ void GpuColorTransfer::prepare(std::uint32_t newWidth, std::uint32_t newHeight, Require(layout.Bytes() <= context.limits.maxStorageBufferRange && layout.LinearBytes() <= context.limits.maxStorageBufferRange, "color transfer exceeds storage buffer limits"); Require((newWidth + 7u) / 8u <= context.limits.maxComputeWorkGroupCount[0] && (newHeight + 7u) / 8u <= context.limits.maxComputeWorkGroupCount[1], "color transfer exceeds workgroup limits"); if (tiled && width == newWidth && height == newHeight && mode == newMode) return; - auto newTiled = std::make_unique(context, layout.Bytes(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - auto newLinear = std::make_unique(context, layout.LinearBytes(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - auto newReadback = std::make_unique(context, layout.Bytes(), VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); - auto newUpload = std::make_unique(context, layout.Bytes(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); - const std::array buffers{{{newTiled->Handle(), 0, layout.Bytes()}, {newLinear->Handle(), 0, layout.LinearBytes()}}}; + constexpr VkBufferUsageFlags copies = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + auto newTiled = std::make_unique(context, layout.Bytes(), copies); + auto newTiledDevice = std::make_unique(context, layout.Bytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + auto newLinear = std::make_unique(context, layout.LinearBytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + const std::array buffers{{{newTiledDevice->Handle(), 0, layout.Bytes()}, {newLinear->Handle(), 0, layout.LinearBytes()}}}; std::array writes{}; for (std::uint32_t i = 0; i < writes.size(); ++i) { writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; @@ -88,31 +86,30 @@ void GpuColorTransfer::prepare(std::uint32_t newWidth, std::uint32_t newHeight, } context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); tiled = std::move(newTiled); + tiledDevice = std::move(newTiledDevice); linear = std::move(newLinear); - readback = std::move(newReadback); - upload = std::move(newUpload); width = newWidth; height = newHeight; mode = newMode; } void GpuColorTransfer::Upload(std::uint64_t address, std::uint32_t newWidth, std::uint32_t newHeight, ColorTileMode newMode) { - PerformanceTimer timing("ColorTransfer.Upload"); prepare(newWidth, newHeight, newMode); - timing.Mark("prepare"); const ColorTargetLayout layout(width, height, mode); - GuestMemory::Read(address, upload->Bytes(), layout.Alignment()); - timing.Mark("guest_read", layout.Bytes()); + GuestMemory::Read(address, tiled->Bytes(), layout.Alignment()); } -void GpuColorTransfer::convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue) { - Require(commands != VK_NULL_HANDLE && tiled && linear, "color transfer is not prepared"); +void GpuColorTransfer::convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit) { + Require(commands != VK_NULL_HANDLE && tiled && tiledDevice && linear, "color transfer is not prepared"); + // The guest bytes move to video memory with one DMA copy; the shader's scattered accesses stay local. + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + CopyBuffer(context, commands, tiled->Handle(), 0, tiledDevice->Handle(), 0, tiledDevice->Size()); VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - before.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + before.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; before.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; const auto barrier = context.Function("vkCmdPipelineBarrier"); - barrier(commands, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); - const std::array push{width, height, (width + 127u) / 128u, (toTiled ? 1u : 0u) | (swapRedBlue ? 2u : 0u) | (mode == ColorTileMode::RenderTarget ? 4u : 0u)}; + barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); + const std::array push{width, height, (width + 127u) / 128u, (toTiled ? 1u : 0u) | (swapRedBlue ? 2u : 0u) | (mode == ColorTileMode::RenderTarget ? 4u : 0u) | (tenBit ? 8u : 0u)}; context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &descriptorSet, 0, nullptr); context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); @@ -121,32 +118,24 @@ void GpuColorTransfer::convert(VkCommandBuffer commands, bool toTiled, bool swap after.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; after.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; barrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &after, 0, nullptr, 0, nullptr); + if (toTiled) { + CopyBuffer(context, commands, tiledDevice->Handle(), 0, tiled->Handle(), 0, tiledDevice->Size()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + } } -void GpuColorTransfer::Detile(VkCommandBuffer commands, bool swapRedBlue) { - Require(commands != VK_NULL_HANDLE && upload && tiled, "color upload is not prepared"); - VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - before.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; - before.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); - const VkBufferCopy copy{0, 0, upload->Bytes().size()}; - context.Function("vkCmdCopyBuffer")(commands, upload->Handle(), tiled->Handle(), 1, ©); - convert(commands, false, swapRedBlue); +void GpuColorTransfer::Detile(VkCommandBuffer commands, bool swapRedBlue, bool tenBit) { + convert(commands, false, swapRedBlue, tenBit); } void GpuColorTransfer::Tile(VkCommandBuffer commands) { convert(commands, true, false); - VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - before.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - before.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - const auto barrier = context.Function("vkCmdPipelineBarrier"); - barrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); - const VkBufferCopy copy{0, 0, readback->Bytes().size()}; - context.Function("vkCmdCopyBuffer")(commands, tiled->Handle(), readback->Handle(), 1, ©); - VkMemoryBarrier after{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - after.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - after.dstAccessMask = VK_ACCESS_HOST_READ_BIT; - barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &after, 0, nullptr, 0, nullptr); +} + +void GpuColorTransfer::WriteBack(std::uint64_t address) { + Require(tiled != nullptr, "color transfer is not prepared for writeback"); + const ColorTargetLayout layout(width, height, mode); + GuestMemory::Write(address, tiled->Bytes(), layout.Alignment()); } VkBuffer GpuColorTransfer::LinearBuffer() const { @@ -154,13 +143,6 @@ VkBuffer GpuColorTransfer::LinearBuffer() const { return linear->Handle(); } -bool GpuColorTransfer::MatchesGuest(std::uint64_t address) { - Require(readback != nullptr, "color readback is unavailable"); - const auto bytes = readback->Bytes(); - GuestMemory::CheckRange(reinterpret_cast(address), bytes.size(), 1); - return std::memcmp(reinterpret_cast(address), bytes.data(), bytes.size()) == 0; -} - RenderTarget& GpuColorTransfer::Target(const ColorTarget& color, bool blending) { if (!target || targetExtent.width != color.extent.width || targetExtent.height != color.extent.height || targetFormat != color.format || targetBlending != blending) { auto replacement = std::make_unique(context, color, blending); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GraphicsPipelineCache.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GraphicsPipelineCache.cpp deleted file mode 100644 index 7414c171c2..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GraphicsPipelineCache.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" -#include -#include -#include - -namespace AgcDriver::Graphics { -namespace { - -template -void append(std::string& key, const TValue& value) { - static_assert(std::is_trivially_copyable_v); - key.append(reinterpret_cast(&value), sizeof(value)); -} - -std::string makeKey(const Context& context, const State& state, const std::shared_ptr& target, const ShaderResources& resources, std::span shaders) { - PerformanceTimer timing("Graphics.PipelineKey"); - std::string key; - key.reserve(512); - append(key, target ? target->Target().View() : VK_NULL_HANDLE); - append(key, state.hasColorTarget); - append(key, state.rectList); - append(key, state.renderExtent.width); - append(key, state.renderExtent.height); - append(key, state.topology); - append(key, state.viewport.x); - append(key, state.viewport.y); - append(key, state.viewport.width); - append(key, state.viewport.height); - append(key, state.viewport.minDepth); - append(key, state.viewport.maxDepth); - append(key, state.negativeOneToOne); - append(key, state.scissor.offset.x); - append(key, state.scissor.offset.y); - append(key, state.scissor.extent.width); - append(key, state.scissor.extent.height); - append(key, state.cullMode); - append(key, state.frontFace); - append(key, state.blend.blendEnable); - append(key, state.blend.srcColorBlendFactor); - append(key, state.blend.dstColorBlendFactor); - append(key, state.blend.colorBlendOp); - append(key, state.blend.srcAlphaBlendFactor); - append(key, state.blend.dstAlphaBlendFactor); - append(key, state.blend.alphaBlendOp); - append(key, state.blend.colorWriteMask); - for (const auto value : state.blendConstants) append(key, value); - append(key, state.stages.mesh.has_value()); - append(key, state.stages.tessellation.has_value()); - if (state.stages.mesh) { - const auto& mesh = *state.stages.mesh; - append(key, mesh.inputPrimitive); - append(key, mesh.primitivesPerGroup); - append(key, mesh.verticesPerGroup); - append(key, mesh.maxVertices); - append(key, mesh.maxPrimitives); - append(key, mesh.threadsPerGroup); - append(key, mesh.ldsSizeDwords); - append(key, mesh.provokingVertex); - } - if (state.stages.tessellation) { - const auto& tessellation = *state.stages.tessellation; - append(key, tessellation.inputControlPoints); - append(key, tessellation.outputControlPoints); - append(key, tessellation.domain); - append(key, tessellation.partitioning); - append(key, tessellation.outputTopology); - } - timing.Mark("state"); - const auto input = BuildVertexInputLayout(context, shaders.front().program->vertexAttributes); - timing.Mark("vertex_layout"); - append(key, input.bindings.size()); - for (const auto& binding : input.bindings) { - append(key, binding.binding); - append(key, binding.stride); - append(key, binding.inputRate); - } - append(key, input.attributes.size()); - for (const auto& attribute : input.attributes) { - append(key, attribute.location); - append(key, attribute.binding); - append(key, attribute.format); - append(key, attribute.offset); - } - append(key, resources.LayoutKey().size()); - const auto layout = std::span(resources.LayoutKey()); - if (!layout.empty()) key.append(reinterpret_cast(layout.data()), layout.size_bytes()); - append(key, PushConstantStages(shaders)); - append(key, shaders.size()); - timing.Mark("resources"); - std::uint64_t shaderBytes = 0; - for (const auto& shader : shaders) { - append(key, shader.stage); - append(key, shader.program->spirv.size()); - const auto bytes = std::as_bytes(std::span(shader.program->spirv)); - if (!bytes.empty()) key.append(reinterpret_cast(bytes.data()), bytes.size()); - shaderBytes += bytes.size(); - } - timing.Mark("shader_code", shaderBytes); - return key; -} - -} - -std::shared_ptr GraphicsPipelineCache::Get(const State& state, const std::shared_ptr& target, const ShaderResources& resources, std::span shaders) { - PerformanceTimer timing("Graphics.PipelineCache"); - auto key = makeKey(context, state, target, resources, shaders); - timing.Mark("key"); - const auto found = lookup.find(key); - if (found != lookup.end()) { - const auto it = found->second; - auto pipeline = it->pipeline; - entries.splice(entries.end(), entries, it); - timing.Mark("hit"); - return pipeline; - } - timing.Mark("miss"); - auto pipeline = std::make_shared(context, state, target ? &target->Target() : nullptr, resources, shaders); - timing.Mark("create"); - entries.push_back({std::move(key), target, pipeline}); - try { - const auto it = std::prev(entries.end()); - Require(lookup.emplace(it->key, it).second, "duplicate graphics pipeline cache key"); - } catch (...) { - entries.pop_back(); - throw; - } - while (entries.size() > 128) { - const auto it = std::find_if(entries.begin(), entries.end(), [](const auto& entry) { return entry.pipeline.use_count() == 1; }); - if (it == entries.end()) break; - lookup.erase(it->key); - entries.erase(it); - } - return pipeline; -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index b2a6eeb035..3d0c250043 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -1,51 +1,969 @@ #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#ifdef _WIN32 +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include namespace AgcDriver::Graphics { +// Image mirrors. The main guest image (the PE at 0x7ff6b...) is MEM_IMAGE memory, which +// VK_EXT_external_memory_host refuses, and address-based shaders map every registered range, so each +// of their builds copied the whole image (~55 MiB: ~40 MiB of read-only sections snapshotted and copied +// again, ~15 MiB of writable ones copied twice). Instead one host-cached device buffer per registered +// image range persists here. Read-only ranges are filled once: their pages change only through an +// mprotect, which replaces the registered Range object, and a mirror whose Range is gone is rebuilt. +// Writable ranges keep a shadow of the guest bytes and copy the 64 KiB blocks that differ on each use. +// A mirror never holds the Range shared_ptr: a pinned range makes the guest's mprotect spin and fail. +struct ImageMirror { + std::uint64_t base = 0; + std::uint64_t bytes = 0; + bool writable = false; + std::weak_ptr range; + std::shared_ptr buffer; + // Writable ranges only: the guest bytes the mirror holds, compared on refresh and at write-back. + std::vector shadow; + // Identity for the life of this mirror (see ImageMirrorSerial); the top bit keeps mirror serials + // apart from host import serials. + std::uint64_t serial = 0; +}; + +namespace { + +// Imports persist across draws, keyed by allocation base, and are dropped when their allocation leaves +// the registered set; a dropped import is destroyed once the recorded work that may read it completed. +struct HostImports { + std::mutex mutex; + VkDevice device = VK_NULL_HANDLE; + std::map imports; + std::set failed; + // Registry generation the imports were last reconciled with. + std::uint64_t refreshedGeneration = 0; + // Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be + // reused while it is unchanged. + std::uint64_t epoch = 1; +}; + +HostImports& Imports() { + static HostImports imports; + return imports; +} + +void destroyImport(const Context& context, const HostImport& entry) { + context.Function("vkDestroyBuffer")(context.device, entry.buffer, nullptr); + context.Function("vkFreeMemory")(context.device, entry.memory, nullptr); +} + +// Frees a dropped import's Vulkan objects when it is released. Never copied: a copy would destroy the +// same handles twice (and a temporary would destroy them at once). +struct RetiredImport { + RetiredImport(const Context& context, const HostImport& entry) : context(context), entry(entry) {} + RetiredImport(const RetiredImport&) = delete; + RetiredImport& operator=(const RetiredImport&) = delete; + ~RetiredImport() { destroyImport(context, entry); } + Context context; + HostImport entry; +}; + +// Drops an import. Recorded batches (dispatches, recorded draws, GPU label writes) may still read it, +// so the objects live until the batch open now completed; with no batches in flight all GPU work that +// used it was synchronous and it is destroyed at once. +void retireImport(const Context& context, HostImports& state, std::map::iterator it) { + auto holder = std::make_shared(context, it->second); + if (auto* recorder = Recorder::Active(); recorder != nullptr && !recorder->Idle()) recorder->Keep(std::move(holder)); + ++state.epoch; + state.imports.erase(it); +} + +const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t base, std::uint64_t bytes) { + if (const auto found = state.imports.find(base); found != state.imports.end()) { + if (found->second.bytes == bytes) return &found->second; + retireImport(context, state, found); + } + const auto alignment = context.hostImportAlignment; + if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0 || state.failed.contains(base)) return nullptr; + // Pinned imports count against the driver's system memory budget; past it ordinary host + // allocations fail, so imports stop at APS5_HOST_IMPORT_MIB (default 6 GiB, which covers the + // registered memory of address-based shaders; past it they copy gigabytes per dispatch). + static const std::uint64_t budget = [] { + const char* value = std::getenv("APS5_HOST_IMPORT_MIB"); + return (value ? std::strtoull(value, nullptr, 10) : 6144ull) << 20u; + }(); + std::uint64_t live = 0; + for (const auto& [address, existing] : state.imports) live += existing.bytes; + if (live + bytes > budget) { + // A refused import turns every later use of the range into CPU copies, so say so. + static std::uint64_t refused = 0, refusedBytes = 0; + static auto lastReport = std::chrono::steady_clock::now() - std::chrono::seconds(60); + ++refused; + refusedBytes += bytes; + if (std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) { + lastReport = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[gpu] host import budget: %llu MiB live of %llu MiB (APS5_HOST_IMPORT_MIB); %llu imports (%llu MiB) refused so far, last 0x%llx+0x%llx\n", static_cast(live >> 20u), static_cast(budget >> 20u), static_cast(refused), static_cast(refusedBytes >> 20u), static_cast(base), static_cast(bytes)); + } + return nullptr; + } +#ifdef _WIN32 + // Drivers pin imported pages, so every page must be committed and accessible. + for (std::uint64_t cursor = base; cursor < base + bytes;) { + MEMORY_BASIC_INFORMATION info{}; + if (VirtualQuery(reinterpret_cast(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) { + state.failed.insert(base); + return nullptr; + } + cursor = reinterpret_cast(info.BaseAddress) + info.RegionSize; + } +#endif + HostImport entry{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0}; + const auto fail = [&](const char* step, VkResult result) -> const HostImport* { + if (entry.buffer) context.Function("vkDestroyBuffer")(context.device, entry.buffer, nullptr); + if (entry.memory) context.Function("vkFreeMemory")(context.device, entry.memory, nullptr); + state.failed.insert(base); + std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx failed at %s (%d); falling back to copies\n", static_cast(base), static_cast(bytes), step, static_cast(result)); +#ifdef _WIN32 + static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; + for (std::uint64_t cursor = base; trace && cursor < base + bytes;) { + MEMORY_BASIC_INFORMATION info{}; + if (VirtualQuery(reinterpret_cast(cursor), &info, sizeof(info)) == 0) break; + const auto regionEnd = reinterpret_cast(info.BaseAddress) + info.RegionSize; + std::fprintf(stderr, "[gpu] 0x%llx+0x%llx state 0x%lx protect 0x%lx type 0x%lx allocation 0x%llx\n", static_cast(cursor), static_cast(std::min(regionEnd, base + bytes) - cursor), info.State, info.Protect, info.Type, reinterpret_cast(info.AllocationBase)); + cursor = regionEnd; + } +#endif + return nullptr; + }; + const VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT}; + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; + info.size = bytes; + // INDIRECT_BUFFER: DISPATCH_INDIRECT group counts are read in place (VulkanDevice::DispatchIndirect). + info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &entry.buffer); result != VK_SUCCESS) return fail("vkCreateBuffer", result); + VkMemoryHostPointerPropertiesEXT pointer{VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT}; + if (const auto result = context.Function("vkGetMemoryHostPointerPropertiesEXT")(context.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, reinterpret_cast(base), &pointer); result != VK_SUCCESS) return fail("vkGetMemoryHostPointerPropertiesEXT", result); + VkMemoryRequirements requirements{}; + context.Function("vkGetBufferMemoryRequirements")(context.device, entry.buffer, &requirements); + const auto types = requirements.memoryTypeBits & pointer.memoryTypeBits; + if (types == 0) return fail("memory type selection", VK_ERROR_FORMAT_NOT_SUPPORTED); + const VkImportMemoryHostPointerInfoEXT import{VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, reinterpret_cast(base)}; + const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, &import, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &flags}; + allocation.allocationSize = bytes; + allocation.memoryTypeIndex = static_cast(std::countr_zero(types)); + if (const auto result = context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &entry.memory); result != VK_SUCCESS) return fail("vkAllocateMemory", result); + if (const auto result = context.Function("vkBindBufferMemory")(context.device, entry.buffer, entry.memory, 0); result != VK_SUCCESS) return fail("vkBindBufferMemory", result); + const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, entry.buffer}; + entry.address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); + if (entry.address == 0) return fail("vkGetBufferDeviceAddressKHR", VK_ERROR_UNKNOWN); + static std::uint64_t importedBytes = 0; + importedBytes += bytes; + static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; + std::uint64_t liveBytes = bytes; + for (const auto& [address, existing] : state.imports) liveBytes += existing.bytes; + if (trace) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx ok (%zu live, %.1f MiB live, %.1f MiB ever)\n", static_cast(base), static_cast(bytes), state.imports.size() + 1, liveBytes / 1048576.0, importedBytes / 1048576.0); + return &state.imports.emplace(base, entry).first->second; +} + +// The lease is in address order (it is built from the registry map), so a range is found by binary search. +const GuestAllocations::Range* leasedRangeAt(const GuestAllocations::Lease& lease, std::uint64_t base) { + const auto found = std::lower_bound(lease.begin(), lease.end(), base, [](const auto& range, std::uint64_t value) { return range->address < value; }); + return found != lease.end() && (*found)->address == base ? found->get() : nullptr; +} + +// The readable registered range containing [begin, end), or null. +const GuestAllocations::Range* containingRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end) { + const auto found = std::upper_bound(lease.begin(), lease.end(), begin, [](std::uint64_t value, const auto& range) { return value < range->address; }); + if (found == lease.begin()) return nullptr; + const auto& range = *std::prev(found); + return range->readable && begin >= range->address && end <= range->address + range->bytes ? range.get() : nullptr; +} + +// Drops imports whose registered range changed or disappeared: their pages may no longer back the +// guest addresses. Walks the imports only when the registry changed since the last walk. +void refreshImports(const Context& context, HostImports& state, const GuestAllocations::Lease& lease) { + if (state.device != context.device) { + state.imports.clear(); + state.failed.clear(); + state.device = context.device; + state.refreshedGeneration = 0; + ++state.epoch; + } + const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + if (generation == state.refreshedGeneration) return; + state.refreshedGeneration = generation; + for (auto it = state.imports.begin(); it != state.imports.end();) { + const auto* range = leasedRangeAt(lease, it->first); + if (range != nullptr && range->bytes == it->second.bytes) { + ++it; + continue; + } + const auto next = std::next(it); + retireImport(context, state, it); + it = next; + } + for (auto it = state.failed.begin(); it != state.failed.end();) it = leasedRangeAt(lease, *it) != nullptr ? std::next(it) : state.failed.erase(it); +} + +const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint64_t end) { + auto found = state.imports.upper_bound(begin); + if (found == state.imports.begin()) return nullptr; + --found; + const auto& entry = found->second; + return begin >= entry.base && end <= entry.base + entry.bytes ? &entry : nullptr; +} + +// Whether the imports need reconciling before a lookup: the registry changed since the last walk. +bool importsStale(const Context& context, const HostImports& state) { + return state.device != context.device || GuestAllocations::GuestAllocationsGeneration_nid_postfix() != state.refreshedGeneration; +} + +// The mirror registry: by range base. The map is protected by `mutex` (findMirror runs in a build's +// unlocked stage A, UploadPrepare); the mirrors' contents (fills, refreshes, sweeps) change under +// GuestMemory::GpuMutex only (every completion and every stage B holds it). APS5_NO_LEASE_MIRROR=1 +// restores copies. +struct ImageMirrors { + std::mutex mutex; + VkDevice device = VK_NULL_HANDLE; + std::map> entries; + std::set failed; + // APS5_PROFILE_DRAW counters, reported every 10 s as [buffers] image mirrors. + std::uint64_t builds = 0; + std::uint64_t subranges = 0; + std::uint64_t rebuilds = 0; + std::uint64_t refreshes = 0; + std::uint64_t blocksCompared = 0; + std::uint64_t blocksCopied = 0; + // Refreshes that had to wait for recorded work writing the range. + std::uint64_t syncs = 0; + std::uint64_t serials = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +ImageMirrors& Mirrors() { + static ImageMirrors mirrors; + return mirrors; +} + +bool mirrorsEnabled() { + static const bool disabled = std::getenv("APS5_NO_LEASE_MIRROR") != nullptr; + return !disabled; +} + +bool sameRange(const std::weak_ptr& mirrored, const std::shared_ptr& range) { + return !mirrored.owner_before(range) && !range.owner_before(mirrored); +} + +// One 64 KiB (or shorter, at a range's ends) block of a writable mirror to compare with guest memory. +struct RefreshBlock { + ImageMirror* mirror; + std::uint64_t address; + std::size_t length; +}; + +// Compares a block's guest bytes with the mirror's shadow and copies a changed block into the device +// buffer and the shadow. Blocks are distinct, so blocks of one refresh can be compared in parallel. +bool compareBlock(const RefreshBlock& block) { + const auto offset = static_cast(block.address - block.mirror->base); + const auto* guest = reinterpret_cast(block.address); + if (std::memcmp(block.mirror->shadow.data() + offset, guest, block.length) == 0) return false; + std::memcpy(block.mirror->buffer->Bytes().data() + offset, guest, block.length); + std::memcpy(block.mirror->shadow.data() + offset, guest, block.length); + return true; +} + +// Helper threads for the block compares. An address-based build refreshes every writable image range +// (~15 MiB in ~300 blocks) under the device lock, and that memcmp was most of its cost; the blocks +// are handed out by an atomic cursor to the helpers and the calling thread alike, and Run returns +// once every block was taken and finished. The threads are started on first use and never joined +// (they only wait on the condition variable). APS5_MIRROR_REFRESH_THREADS sets the helper count +// (default 3); APS5_NO_PARALLEL_MIRROR_REFRESH=1 compares on the calling thread only. +class RefreshPool { +public: + static RefreshPool& Get() { + // Never destroyed (as WorkerSampler's): the helpers block on its condition variable, which a + // static destructor at std::exit would tear down under them. + static RefreshPool* pool = new RefreshPool; + return *pool; + } + + // Compares `blocks` and returns how many were copied. + std::uint64_t Run(std::span blocks) { + // Small jobs are not worth waking the helpers (a sub-range refresh at Upload is a few blocks). + if (helpers == 0 || blocks.size() < 8) { + std::uint64_t copied = 0; + for (const auto& block : blocks) copied += compareBlock(block) ? 1 : 0; + return copied; + } + // Every caller holds the device lock today; the pool serializes its jobs regardless. + std::lock_guard running(runMutex); + { + std::lock_guard lock(mutex); + job = blocks; + next.store(0, std::memory_order_relaxed); + copiedBlocks.store(0, std::memory_order_relaxed); + finished = 0; + ++serial; + } + wake.notify_all(); + work(blocks); + std::unique_lock lock(mutex); + done.wait(lock, [&] { return finished == helpers; }); + job = {}; + return copiedBlocks.load(std::memory_order_relaxed); + } + +private: + RefreshPool() { + static const bool disabled = std::getenv("APS5_NO_PARALLEL_MIRROR_REFRESH") != nullptr; + const char* text = std::getenv("APS5_MIRROR_REFRESH_THREADS"); + const auto requested = text != nullptr ? std::atoi(text) : 3; + const auto wanted = disabled ? 0u : static_cast(std::clamp(requested, 0, 16)); + // Only started threads are counted: Run waits for `helpers` finishes, and a thread that could + // not be started would leave it waiting forever. + for (unsigned i = 0; i < wanted; ++i) { + try { + std::thread([this] { helper(); }).detach(); + ++helpers; + } catch (const std::system_error& error) { + std::fprintf(stderr, "[gpu] mirror refresh helper %u not started: %s\n", i, error.what()); + break; + } + } + } + + // Takes blocks until none is left. + void work(std::span blocks) { + std::uint64_t copied = 0; + for (auto index = next.fetch_add(1, std::memory_order_relaxed); index < blocks.size(); index = next.fetch_add(1, std::memory_order_relaxed)) copied += compareBlock(blocks[index]) ? 1 : 0; + copiedBlocks.fetch_add(copied, std::memory_order_relaxed); + } + + void helper() { + std::uint64_t seen = 0; + for (;;) { + std::span blocks; + { + std::unique_lock lock(mutex); + wake.wait(lock, [&] { return serial != seen; }); + seen = serial; + blocks = job; + } + work(blocks); + { + std::lock_guard lock(mutex); + ++finished; + } + done.notify_all(); + } + } + + unsigned helpers = 0; + std::mutex runMutex; + std::mutex mutex; + std::condition_variable wake; + std::condition_variable done; + std::span job; + std::uint64_t serial = 0; + unsigned finished = 0; + std::atomic next{0}; + std::atomic copiedBlocks{0}; +}; + +// The device-lock work before [address, address + bytes) of a writable mirror can be compared with +// guest memory: recorded GPU work that writes the range (a descriptor dispatch bound to a sub-range) +// completes first, so its results reach the guest and the shadow before the compare and no batch +// writes a block being replaced; GPU reads of the mirror by earlier recorded work (an address-based +// dispatch whose lease is released at completion) see the newer bytes, as they would on hardware. +// Returns whether there is anything to compare. +bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) { + if (!mirror.writable || bytes == 0) return false; + // The compare reads the pages directly; a mirror is only reused while its Range exists, but a + // lease of a build in flight can keep a replaced Range alive past a protection change. + Require(GuestMemory::Accessible(reinterpret_cast(address), static_cast(bytes)), "image mirror range became inaccessible"); + // Named for the [hooksync] attribution: the flush goes through the hook, which waits when + // recorded work notes a write inside the range (a descriptor written inside a writable exe + // range; the lease itself maps the range read-only). The access is the whole range. + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::MirrorRefresh); + GuestMemory::FlushGpuWrites(address, static_cast(bytes)); + auto& state = Mirrors(); + if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(address, static_cast(bytes))) { + Recorder::CountSync(4); + ++state.syncs; + recorder->Sync(); + } + ++state.refreshes; + return true; +} + +void appendBlocks(std::vector& blocks, ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) { + constexpr std::uint64_t block = 65536; + const auto end = address + bytes; + for (auto at = address; at < end;) { + const auto next = std::min(end, (at / block + 1) * block); + blocks.push_back({&mirror, at, static_cast(next - at)}); + at = next; + } +} + +// Compares the gathered blocks (of one or many mirrors) and counts them. +void compareBlocks(std::span blocks) { + if (blocks.empty()) return; + const auto copied = RefreshPool::Get().Run(blocks); + auto& state = Mirrors(); + state.blocksCompared += blocks.size(); + state.blocksCopied += copied; +} + +// Brings [address, address + bytes) of a writable mirror up to date with guest memory. +void refreshMirror(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) { + if (!prepareRefresh(mirror, address, bytes)) return; + std::vector blocks; + appendBlocks(blocks, mirror, address, bytes); + compareBlocks(blocks); +} + +// The mirror for a leased image range: the existing one, or a new one when none exists or the +// registered Range object changed (a protection change). Null when the range cannot be mirrored. An +// existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every +// mirror of the build (prepareRefresh ran); a new mirror is filled here. +std::shared_ptr acquireMirror(const Context& context, const std::shared_ptr& range, std::vector& blocks) { + auto& state = Mirrors(); + std::shared_ptr mirror; + { + std::lock_guard lock(state.mutex); + if (state.device != context.device) { + state.entries.clear(); + state.failed.clear(); + state.device = context.device; + } + if (state.failed.contains(range->address)) return nullptr; + const auto found = state.entries.find(range->address); + if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->bytes == range->bytes && found->second->writable == range->writable) mirror = found->second; + } + if (mirror != nullptr) { + if (prepareRefresh(*mirror, range->address, range->bytes)) appendBlocks(blocks, *mirror, range->address, range->bytes); + return mirror; + } + // The registered flags were taken from the pages; a range that disagrees is copied by this build. + if (!GuestMemory::Accessible(reinterpret_cast(range->address), range->bytes, range->writable)) return nullptr; + mirror = std::make_shared(); + mirror->base = range->address; + mirror->bytes = range->bytes; + mirror->writable = range->writable; + mirror->range = range; + try { + mirror->buffer = std::make_shared(context, range->bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + } catch (const std::runtime_error& error) { + std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast(range->address), static_cast(range->bytes), error.what()); + std::lock_guard lock(state.mutex); + state.failed.insert(range->address); + return nullptr; + } + // Read stores GPU results pending in the range first (write-backs of earlier copies of it); + // named for the [hooksync] attribution like the refresh's flush. + { + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::MirrorRefresh); + GuestMemory::Read(range->address, mirror->buffer->Bytes()); + } + if (range->writable) mirror->shadow.assign(mirror->buffer->Bytes().begin(), mirror->buffer->Bytes().end()); + std::lock_guard lock(state.mutex); + ++state.rebuilds; + mirror->serial = (1ull << 63u) | ++state.serials; + // A replaced mirror lives on in the regions of recorded work that bound it. + state.entries[range->address] = mirror; + return mirror; +} + +// The mirror containing [begin, end) whose registered range still exists, or null. A mirror whose +// Range was replaced is only rebuilt by the next address-based build; until then the range is copied. +std::shared_ptr findMirror(const Context& context, std::uint64_t begin, std::uint64_t end) { + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + if (state.device != context.device || state.entries.empty()) return nullptr; + auto found = state.entries.upper_bound(begin); + if (found == state.entries.begin()) return nullptr; + --found; + const auto& mirror = found->second; + if (begin < mirror->base || end > mirror->base + mirror->bytes || mirror->range.expired()) return nullptr; + return mirror; +} + +// Drops mirrors whose registered range is gone (a later build made a new one for the replacement). +void sweepMirrors() { + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + for (auto it = state.entries.begin(); it != state.entries.end();) it = it->second->range.expired() ? state.entries.erase(it) : std::next(it); +} + +void reportMirrors() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& state = Mirrors(); + const auto now = std::chrono::steady_clock::now(); + if (now - state.lastReport < std::chrono::seconds(10)) return; + state.lastReport = now; + std::size_t count = 0; + std::size_t writable = 0; + std::uint64_t bytes = 0; + { + std::lock_guard lock(state.mutex); + for (const auto& [base, mirror] : state.entries) { + ++count; + bytes += mirror->bytes; + if (mirror->writable) ++writable; + } + } + std::fprintf(stderr, "[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs\n", count, bytes / 1048576.0, writable, static_cast(state.builds), static_cast(state.subranges), static_cast(state.rebuilds), static_cast(state.refreshes), static_cast(state.blocksCompared), static_cast(state.blocksCopied), static_cast(state.syncs)); +} + +// Deferred lease release. The lease an address-based build takes (AcquireRegistered) is dropped by its +// write-back, which runs as a completion action when the batch that recorded the work finished; the +// work is no longer synced as soon as it is recorded (that sync was ~30000 waits per 200 s). A guest +// thread that unmaps, mprotects or frees a leased allocation finds it pinned in the registry and calls +// this waiter (with the registry lock released, see GuestAllocationsRequireUnpinned): under the device +// lock it submits the recorder and finishes its batches up to the newest one that was recorded with a +// lease (CountLeaseOutcome notes that serial); the completions release the leases, later batches stay +// in flight, and the registry rescans. A lease held by a build in progress belongs to a worker holding +// the device lock, so the waiter waits for that worker first; once it holds the lock every lease of +// recorded work is in a batch. A lease whose batch was not noted (the record threw between the keep +// and the note) is covered by draining the recorder when the noted serial is already finished. The +// waiter yields instead when its own thread holds the device lock (a driver thread mutating the +// registry mid-packet, as the plain spin did) and reports whether it finished any work. +struct LeaseState { + std::mutex mutex; + LeaseStats stats; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + // Under GuestMemory::GpuMutex: the recorder whose batches hold leases (serials restart with a + // replaced device's recorder), the serial of its newest batch recorded with a lease, and the + // newest one a waiter finished. + const Recorder* leaseRecorder = nullptr; + std::uint64_t newestLeaseSerial = 0; + std::uint64_t finishedLeaseSerial = 0; +}; + +LeaseState& Leases() { + static LeaseState state; + return state; +} + +// APS5_PROFILE_DRAW: AddSnapshot's consistency compares of a captured region against the region +// serving it, made and skipped (see AddSnapshot); printed in the [address-sync] line. Atomic: the +// unlocked stage A of descriptor builds adds snapshots too. +struct SnapshotStats { + std::atomic checked{0}; + std::atomic skipped{0}; +}; + +SnapshotStats& Snapshots() { + static SnapshotStats stats; + return stats; +} + +bool WaitForLeases() noexcept { + const auto start = std::chrono::steady_clock::now(); + bool synced = false; + bool drained = false; + if (GuestMemory::GpuMutex().HeldByThisThread()) { + std::this_thread::yield(); + } else { + try { + std::lock_guard lock(GuestMemory::GpuMutex()); + auto& state = Leases(); + auto* recorder = Recorder::Active(); + const auto target = state.newestLeaseSerial; + if (recorder != nullptr && recorder == state.leaseRecorder && target > state.finishedLeaseSerial) { + // The open batch may be the target; FinishUpTo waits front to back up to it only. + Recorder::CountSync(4); + recorder->Submit(); + recorder->FinishUpTo(target); + state.finishedLeaseSerial = target; + synced = true; + } else if (recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } else { + // Nothing recorded holds the lease: the holder is a lease-holding object that has + // yet to record (a test's, or a build whose worker gave up the lock); give it time. + std::this_thread::yield(); + } + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] lease wait failed: %s\n", error.what()); + } + } + auto& state = Leases(); + std::lock_guard lock(state.mutex); + ++state.stats.contentionWaits; + if (synced) ++state.stats.contentionSyncs; + if (drained) ++state.stats.contentionDrains; + state.stats.contentionMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + return synced; +} + +void ensurePinWaiter() { + static const bool registered = [] { + GuestAllocations::GuestAllocationsSetPinWaiter_nid_postfix(&WaitForLeases); + return true; + }(); + static_cast(registered); +} + +} + +bool SyncLeaseWork() { + static const bool sync = std::getenv("APS5_SYNC_LEASE_DISPATCH") != nullptr; + return sync; +} + +void CountLeaseOutcome(bool synced, std::uint64_t batchSerial) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& state = Leases(); + if (!synced) { + // Serials grow with the submissions, so the newest noted one covers every earlier lease batch + // of the same recorder. + GuestMemory::AssertGpuLockHeld("CountLeaseOutcome"); + if (const auto* recorder = Recorder::Active(); recorder != state.leaseRecorder) { + state.leaseRecorder = recorder; + state.newestLeaseSerial = 0; + state.finishedLeaseSerial = 0; + } + state.newestLeaseSerial = std::max(state.newestLeaseSerial, batchSerial); + } + std::lock_guard lock(state.mutex); + ++(synced ? state.stats.synced : state.stats.deferred); + if (!profile) return; + // APS5_PROFILE_DRAW: the leases (dispatches and draws) deferred and synced, the guest's + // pin-contention waits and the BDA table cache, every 10 s (cumulative counts, like the + // [recorder] line's), printed by whichever path counts a lease. + const auto now = std::chrono::steady_clock::now(); + if (now - state.lastReport < std::chrono::seconds(10)) return; + state.lastReport = now; + const auto& stats = state.stats; + const auto table = BdaResources::TableCacheCounters(); + const auto& snapshots = Snapshots(); + std::fprintf(stderr, "[address-sync] leases: %llu released at completion, %llu synced at once; %llu pin-contention waits by guest threads (%llu finished the lease batch, %llu drained the recorder) %.1f s; BDA table cache %llu hits / %llu misses (%zu tables held); snapshot compares: %llu skipped (live-backed region), %llu made\n", static_cast(stats.deferred), static_cast(stats.synced), static_cast(stats.contentionWaits), static_cast(stats.contentionSyncs), static_cast(stats.contentionDrains), stats.contentionMs / 1000, static_cast(table.hits), static_cast(table.misses), table.held, static_cast(snapshots.skipped.load(std::memory_order_relaxed)), static_cast(snapshots.checked.load(std::memory_order_relaxed))); +} + +LeaseStats LeaseCounters() { + auto& state = Leases(); + std::lock_guard lock(state.mutex); + return state.stats; +} + +const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { + if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; + auto& state = Imports(); + std::lock_guard lock(state.mutex); + // A hit is only valid while the registry has not changed since the imports were reconciled. + if (!importsStale(context, state)) { + if (const auto* entry = findImport(state, address, address + bytes)) return entry; + } + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + refreshImports(context, state, lease); + if (const auto* range = containingRange(lease, address, address + bytes)) return importAllocation(context, state, range->address, range->bytes); + return nullptr; +} + +bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes) { + if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return false; + auto& state = Imports(); + std::lock_guard lock(state.mutex); + // Imports of a replaced device are not this device's; a stale registry only means an import + // may be dropped at the next reconcile, which the binding path handles. + return state.device == context.device && findImport(state, address, address + bytes) != nullptr; +} + GuestBufferMemory::GuestBufferMemory(const Context& context) : context(context) {} bool GuestBufferMemory::WritesOverlap(std::uint64_t address, std::size_t bytes) const { return std::any_of(writes.begin(), writes.end(), [&](const auto& range) { return address < range.second && range.first < address + bytes; }); } +void GuestBufferMemory::AcquireRegistered() { + Require(!uploaded && regions.empty() && lease.empty(), "guest allocation lease must precede resource registration"); + ensurePinWaiter(); + // Pending GPU results in these ranges are stored when Upload prepares each region. + lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + const bool importable = context.hostImportAlignment != 0; + if (importable) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + refreshImports(context, state, lease); + // Taken before the pass: the imports it takes are the registry's ones (just reconciled, so + // none is retired by a make below), and a mirror's preparation may sync the recorder, whose + // completions can retire an import already taken; that moves the epoch, and Upload then + // re-checks every pointer instead of trusting it. + importsEpoch = state.epoch; + } + // One pass in the registry's (ascending) order, so the regions come out sorted for AddSnapshot. + // The import lock is taken per range: a mirror's preparation may wait for recorded work, which + // must not happen under it. + regions.reserve(lease.size()); + std::vector blocks; + bool mirrored = false; + for (const auto& range : lease) { + if (!range->readable) continue; + validate(range->address, range->bytes); + Region region{range->address, range->address + range->bytes, range->writable, {}, nullptr}; + const HostImport* entry = nullptr; + if (importable) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + entry = importAllocation(context, state, range->address, range->bytes); + } + if (entry != nullptr) { + region.hostBacked = true; + // Reused by Upload while no import was dropped since (see importsEpoch). + region.direct = entry; + regions.push_back(std::move(region)); + continue; + } + if (!range->releasable && mirrorsEnabled()) { + // Main image: served by a persistent mirror (see ImageMirror); the writable ones are + // compared with guest memory below, all at once. + region.mirror = acquireMirror(context, range, blocks); + if (region.mirror != nullptr) { + mirrored = true; + regions.push_back(std::move(region)); + continue; + } + } + // Read-only ranges are read from live guest memory at Upload: the lease pins them and the guest + // cannot write them, so that equals a snapshot taken now without the extra copy. Reserved heaps + // are registered whole while the guest commits pages on demand: committed pages only. + region.hostBacked = !range->writable; + auto committed = GuestMemory::DescribeCommitted(range->address, range->bytes, range->writable); + if (!committed.whole) { + region.sparse = true; + region.backed = std::move(committed.ranges); + } + regions.push_back(std::move(region)); + } + // APS5_NO_SORTED_SNAPSHOT_LOOKUP=1: AddSnapshot scans the regions as before instead of searching. + static const bool sortedLookup = std::getenv("APS5_NO_SORTED_SNAPSHOT_LOOKUP") == nullptr; + regionsSorted = sortedLookup; + compareBlocks(blocks); + if (mirrorsEnabled()) { + if (mirrored) ++Mirrors().builds; + sweepMirrors(); + reportMirrors(); + } +} + void GuestBufferMemory::validate(std::uint64_t address, std::size_t bytes) const { Require(!uploaded, "guest memory ownership is frozen for GPU execution"); Require(address != 0 && bytes != 0, "empty guest memory range"); Require(bytes <= std::numeric_limits::max() - address, "guest memory range overflow"); } -void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes) { +void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes) { validate(address, bytes); - GuestMemory::CheckRange(reinterpret_cast(address), bytes, 1, true); - regions.push_back({address, address + bytes, true, {}, nullptr}); + // `writable` here means the bytes come from live guest memory (not a snapshot), whether or not + // the shader stores to them; what is written back is decided by Writes() alone. + Region region{address, address + bytes, true, {}, nullptr}; + auto committed = GuestMemory::DescribeCommitted(address, bytes, true); + if (!committed.whole) { + // A GPU heap bound whole while the guest commits its pages on demand, or a descriptor left + // pointing at unmapped memory: only committed pages are copied in and stored back. Shaders + // must not touch the rest, which reads as zeros. + region.sparse = true; + region.backed = std::move(committed.ranges); + } + regions.push_back(std::move(region)); + regionsSorted = false; +} + +void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes) { + addDescriptorRegion(address, bytes); writes.emplace_back(address, address + bytes); } +void GuestBufferMemory::AddReadable(std::uint64_t address, std::size_t bytes) { + // Not in `writes`: no reference copy (copyRegion), no write-back, no pending-write note, no + // direct-write mark, and UploadPrepare copies it without the device lock. A written descriptor + // overlapping the range still covers it through its own Writes() entry. + addDescriptorRegion(address, bytes); +} + void GuestBufferMemory::AddSnapshot(const GuestMemorySnapshot& snapshot) { validate(snapshot.address, snapshot.bytes.size()); - for (const auto& region : regions) { - if (region.begin <= snapshot.address && snapshot.address + snapshot.bytes.size() <= region.end) { - const auto offset = static_cast(snapshot.address - region.begin); - const auto* source = region.writable ? reinterpret_cast(snapshot.address) : region.snapshot.data() + offset; - Require(std::memcmp(source, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "guest snapshot differs from registered memory"); - return; + const auto end = snapshot.address + snapshot.bytes.size(); + // A snapshot inside a region needs no copy: the region's bytes serve it (they are checked to + // agree). Right after AcquireRegistered the regions are sorted and non-overlapping (registered + // ranges), so the one that can contain the snapshot is found by search rather than by scanning + // the ~1200 of an address-based build for each captured region. + const Region* owner = nullptr; + if (regionsSorted) { + const auto found = std::upper_bound(regions.begin(), regions.end(), snapshot.address, [](std::uint64_t value, const Region& region) { return value < region.begin; }); + if (found != regions.begin() && end <= std::prev(found)->end) owner = &*std::prev(found); + } else { + for (const auto& region : regions) { + if (region.begin <= snapshot.address && end <= region.end) { + owner = ®ion; + break; + } } } - regions.push_back({snapshot.address, snapshot.address + snapshot.bytes.size(), false, {snapshot.bytes.begin(), snapshot.bytes.end()}, nullptr}); + if (owner != nullptr) { + // A region read live (an import or mirror bound in place, a copy of live bytes) serves the + // GPU whatever the snapshot holds: the compare only checked that the capture (made before + // the device lock) still agreed with memory the guest, or a dispatch writing a descriptor + // range around the captured words, may since have changed, and a difference failed the + // whole dispatch where hardware would have run with the bytes in memory. A snapshot-backed + // region uploads its own snapshot, so two snapshots of one range must agree. The live + // compare reads the pages directly (no flush hook), so it costs no sync, only the memcmp; + // it stays on because it is the one check that the words the recompiler baked into + // descriptors and specialization still equal what the GPU reads. APS5_NO_SNAPSHOT_CHECK=1 + // skips the compare of live-backed regions (the region serves the GPU whatever the snapshot + // held; only the diagnostic is lost). + static const bool checkLive = std::getenv("APS5_NO_SNAPSHOT_CHECK") == nullptr; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const bool live = owner->writable || owner->hostBacked || owner->mirror != nullptr; + if (live && !checkLive) { + if (profile) Snapshots().skipped.fetch_add(1, std::memory_order_relaxed); + return; + } + const auto offset = static_cast(snapshot.address - owner->begin); + const auto* source = live ? reinterpret_cast(snapshot.address) : owner->snapshot.data() + offset; + Require(std::memcmp(source, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "guest snapshot differs from registered memory"); + if (profile) Snapshots().checked.fetch_add(1, std::memory_order_relaxed); + return; + } + regions.push_back({snapshot.address, end, false, {snapshot.bytes.begin(), snapshot.bytes.end()}, nullptr}); + regionsSorted = false; } void GuestBufferMemory::Upload(bool addressable) { - Require(!uploaded, "guest memory was already uploaded"); - uploaded = true; + UploadPrepare(addressable); + UploadFinish(addressable); +} + +namespace { + +// APS5_PROFILE_DRAW: sub-phase totals of the uploads in microseconds, reported every 2000 uploads +// ([buffers] line); atomic because the prepare stage of several builds runs at once. +std::atomic importUs{0}; +std::atomic allocateUs{0}; +std::atomic readUs{0}; +std::atomic uploadsProfiled{0}; + +std::uint64_t microsecondsSince(std::chrono::steady_clock::time_point start) { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); +} + +// GPU-side copies. A descriptor sub-range inside a host import whose offset is not a multiple of +// minStorageBufferOffsetAlignment cannot be bound in place; ~31000 of them per run (about 34 bytes +// each) were copied through the CPU, and every such read went through the flush hook and waited +// for the producer batch ([hooksync] 'buffer-upload'). Instead the copy is a vkCmdCopyBuffer from +// the import into the region's buffer, recorded into the open batch after the producer, and the +// written sub-ranges are copied back into the import by the GPU (RecordCopyBacks), so neither +// direction touches guest memory from the CPU. APS5_CPU_COPIES=1 copies on the CPU as before; +// APS5_GPU_COPY_MAX_KIB (default 1024) bounds the region size the GPU copies (larger ones and +// regions outside imports keep the CPU path). +bool gpuCopiesEnabled() { + static const bool disabled = std::getenv("APS5_CPU_COPIES") != nullptr; + return !disabled; +} + +std::uint64_t gpuCopyLimit() { + static const std::uint64_t limit = [] { + const char* value = std::getenv("APS5_GPU_COPY_MAX_KIB"); + return (value != nullptr ? std::strtoull(value, nullptr, 10) : 1024ull) << 10u; + }(); + return limit; +} + +VkBufferUsageFlags gpuCopyUsage(bool addressable) { + return VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); +} + +// APS5_PROFILE_DRAW: the GPU copies, reported in the [buffers] uploads line (cumulative). +struct CopyStats { + std::atomic gpuCopies{0}; + std::atomic gpuCopyBytes{0}; + std::atomic gpuCopyBacks{0}; + std::atomic gpuCopyBackBytes{0}; + // Written gpuCopy regions of a use that recorded no copy-back (a synchronous draw): stored + // from the staging buffer by the CPU in WriteBack. + std::atomic stagingStores{0}; +}; + +CopyStats& Copies() { + static CopyStats stats; + return stats; +} + +} + +bool GuestBufferMemory::gpuCopyEligible(const Region& region) const { + if (!gpuCopiesEnabled() || region.sparse || !(region.hostBacked || region.writable)) return false; + return region.end - region.begin <= gpuCopyLimit(); +} + +void GuestBufferMemory::UploadPrepare(bool addressable) { + Require(!prepared && !uploaded, "guest memory was already uploaded"); + prepared = true; std::sort(regions.begin(), regions.end(), [](const Region& left, const Region& right) { return left.begin < right.begin; }); + // Merged regions stay sparse when either part covered uncommitted pages. + const auto mergeBacked = [](Region& into, const Region& from) { + if (!into.sparse && !from.sparse) return; + auto ranges = into.sparse ? into.backed : decltype(into.backed){{into.begin, into.end}}; + if (from.sparse) ranges.insert(ranges.end(), from.backed.begin(), from.backed.end()); + else ranges.emplace_back(from.begin, from.end); + std::sort(ranges.begin(), ranges.end()); + into.backed.clear(); + for (const auto& range : ranges) { + if (!into.backed.empty() && range.first <= into.backed.back().second) into.backed.back().second = std::max(into.backed.back().second, range.second); + else into.backed.push_back(range); + } + const auto end = std::max(into.end, from.end); + into.sparse = !(into.backed.size() == 1 && into.backed.front().first <= into.begin && into.backed.front().second >= end); + if (!into.sparse) into.backed.clear(); + }; std::vector merged; for (auto& region : regions) { if (!merged.empty() && region.begin < merged.back().end) { auto& previous = merged.back(); + if (previous.mirror != nullptr || region.mirror != nullptr) { + // A descriptor range inside a mirrored image range binds the mirror's bytes (registered + // ranges never overlap, so the other part is such a descriptor; it sorts first when it + // starts at the range's base). Writes into a read-only mirror are never stored. One + // crossing the mirror's end cannot be served by it: this build copies the range. + if (previous.mirror == nullptr) std::swap(previous, region); + Require(region.mirror == nullptr, "image mirrors overlap"); + if (region.begin >= previous.begin && region.end <= previous.end) continue; + previous.mirror = nullptr; + previous.hostBacked = true; + } + mergeBacked(previous, region); + // A range starting before the mirror (only possible after the swap) keeps its prefix; the + // earlier merged region ends at or before it, so the merged list stays sorted. + previous.begin = std::min(previous.begin, region.begin); + if (previous.hostBacked || region.hostBacked) { + // Live guest bytes back the merged range, so snapshots inside it add nothing. + previous.hostBacked = true; + previous.writable = previous.writable || region.writable; + previous.snapshot.clear(); + previous.end = std::max(previous.end, region.end); + continue; + } Require(previous.writable == region.writable, "writable guest memory overlaps an immutable snapshot"); if (!region.writable) { const auto offset = static_cast(region.begin - previous.begin); @@ -59,15 +977,358 @@ void GuestBufferMemory::Upload(bool addressable) { } } regions = std::move(merged); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto started = std::chrono::steady_clock::now(); for (auto& region : regions) { - const auto bytes = region.end - region.begin; - Require(bytes <= std::numeric_limits::max(), "guest GPU allocation size overflow"); - const auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); - region.buffer = std::make_unique(context, static_cast(bytes), usage); - if (region.writable) GuestMemory::Read(region.begin, region.buffer->Bytes()); - else std::memcpy(region.buffer->Bytes().data(), region.snapshot.data(), region.snapshot.size()); - region.snapshot.clear(); + Require(region.end - region.begin <= std::numeric_limits::max(), "guest GPU allocation size overflow"); + // Served by a lease mirror, brought up to date when it was acquired. + if (region.mirror != nullptr || region.direct != nullptr || !mirrorsEnabled()) continue; + // A descriptor range inside a mirrored image range binds the mirror's sub-range. Looked up + // before the imports: the image can never be imported, and the import fallback would acquire + // a registry lease for it on nearly every descriptor upload. + auto mirror = findMirror(context, region.begin, region.end); + if (mirror == nullptr || (region.begin - mirror->base) % context.limits.minStorageBufferOffsetAlignment != 0) continue; + // A read-only mirror's bytes never change while its Range exists; a writable one is + // refreshed by UploadFinish (the refresh may wait for recorded work). Either is confirmed + // there, under the lock, before it serves the region: the Range can go between the stages, + // and until then the region keeps its snapshot for the copy it would need instead. + region.pending = mirror->writable; + region.mirror = std::move(mirror); + region.subrangeMirror = true; } + // Imports already serving regions, taken under one hold of the import lock so every pointer + // belongs to the same registry epoch (UploadFinish trusts them while the epoch is unchanged). No + // reconcile and no new import here: retiring one hands its buffer to the recorder, and a region + // without an import waits for UploadFinish, which may make one. + if (context.hostImportAlignment != 0) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + const bool stale = importsStale(context, state); + for (auto& region : regions) { + if (region.mirror != nullptr) continue; + // An import found when the lease was acquired is reused while none was dropped since. + const HostImport* entry = region.direct != nullptr && state.epoch == importsEpoch ? region.direct : nullptr; + region.direct = nullptr; + if (stale) { + region.pending = true; + continue; + } + if (entry == nullptr) entry = findImport(state, region.begin, region.end); + if (entry == nullptr) { + region.pending = true; + continue; + } + // Misaligned inside an import: copied, whatever the registry does meanwhile. By the GPU + // when eligible (see Region::gpuCopy): UploadFinish records the copy from the import it + // confirms then (the pointer is provisional, as for an aligned region); the buffer the + // copy fills is made here, without the device lock, and serves the CPU fallback too. + if ((region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment != 0) { + if (!gpuCopyEligible(region)) continue; + const auto allocateStart = std::chrono::steady_clock::now(); + region.buffer = std::make_shared(context, static_cast(region.end - region.begin), gpuCopyUsage(addressable)); + if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); + region.direct = entry; + region.pending = true; + continue; + } + // Provisional until UploadFinish confirms it against the epoch; the snapshot stays for + // the copy the region falls back to when the import is gone and none can be made. + region.direct = entry; + // Storage results pending in the range are flushed by UploadFinish. + region.pending = true; + } + importsEpoch = state.epoch; + } + if (profile) importUs.fetch_add(microsecondsSince(started), std::memory_order_relaxed); + // Copies that need no device work: a range no descriptor writes needs no reference copy, and + // GuestMemory::Read stores pending GPU results first (the flush hook locks for itself). + for (auto& region : regions) { + if (region.pending || region.mirror != nullptr || region.direct != nullptr) continue; + if (region.writable && WritesOverlap(region.begin, static_cast(region.end - region.begin))) { + region.pending = true; + continue; + } + copyRegion(region, addressable); + } +} + +void GuestBufferMemory::UploadFinish(bool addressable) { + Require(prepared && !uploaded, "guest memory upload was not prepared"); + uploaded = true; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto started = std::chrono::steady_clock::now(); + // Registered ranges to import from. Address-based shaders hold their lease until write-back; + // descriptor-only uploads acquire one only when the registry changed or an import must be made + // (a lease copies every registered range's shared_ptr under the registry lock), so the guest can + // keep managing memory. + GuestAllocations::Lease acquired; + const auto importRanges = [&]() -> const GuestAllocations::Lease& { + if (lease.empty() && acquired.empty()) acquired = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + return lease.empty() ? acquired : lease; + }; + // Sub-range mirrors were found by UploadPrepare, possibly long before this lock was taken: one + // whose Range went meanwhile (the guest reprotected its image) is dropped as findMirror would + // have dropped it, and the region takes the import or copy path below instead; a confirmed one + // serves the region, so its snapshot goes. (Lease mirrors need no check: the lease pins their + // Range.) + for (auto& region : regions) { + if (region.mirror == nullptr || !region.subrangeMirror) continue; + if (region.mirror->range.expired()) { + region.mirror = nullptr; + region.subrangeMirror = false; + region.pending = true; + continue; + } + region.snapshot.clear(); + std::lock_guard lock(Mirrors().mutex); + ++Mirrors().subranges; + } + bool importsRefreshed = false; + // Regions the GPU copies out of their import (see Region::gpuCopy), recorded together below. + // No recorder (tests) keeps them on the CPU path. + auto* recorder = Recorder::Active(); + std::vector gpuCopies; + for (auto& region : regions) { + if (!region.pending) continue; + region.pending = false; + const auto bytes = region.end - region.begin; + if (region.mirror != nullptr) { + // A writable sub-range mirror: refreshed here, where waiting for recorded work is allowed. + // Read-only mirrors are never written back; a writable one is diffed for this sub-range + // at write-back. + const auto refreshStart = std::chrono::steady_clock::now(); + refreshMirror(*region.mirror, region.begin, bytes); + if (profile) readUs.fetch_add(microsecondsSince(refreshStart), std::memory_order_relaxed); + continue; + } + bool copyOnGpu = false; + if (context.hostImportAlignment != 0) { + const auto lookupStart = std::chrono::steady_clock::now(); + auto& state = Imports(); + std::lock_guard lock(state.mutex); + // Imports are made on demand for the registered range around each region. + if (!importsRefreshed) { + if (importsStale(context, state)) refreshImports(context, state, importRanges()); + importsRefreshed = true; + } + // An import taken by UploadPrepare (or at the lease) is still the registry's unless one + // was dropped since. + const HostImport* entry = region.direct != nullptr && state.epoch == importsEpoch ? region.direct : nullptr; + region.direct = nullptr; + if (entry == nullptr) entry = findImport(state, region.begin, region.end); + if (entry == nullptr) { + if (const auto* range = containingRange(importRanges(), region.begin, region.end)) entry = importAllocation(context, state, range->address, range->bytes); + } + if (entry != nullptr && (region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment == 0) { + region.direct = entry; + region.snapshot.clear(); + // A buffer UploadPrepare made for a GPU copy is not needed: the import serves the + // region in place (Descriptor prefers `direct`), and a kept buffer would count as copied. + region.buffer.reset(); + } else if (entry != nullptr && recorder != nullptr && gpuCopyEligible(region)) { + // Misaligned in the import: the GPU copies the sub-range out of it. The handle and + // base are taken now, under the import lock: a reconcile by the flush below could + // retire the entry (the batch opened below keeps a retired import's buffer alive). + copyOnGpu = true; + region.copySource = entry->buffer; + region.copySourceBase = entry->base; + } + if (profile) importUs.fetch_add(microsecondsSince(lookupStart), std::memory_order_relaxed); + } + if (region.direct != nullptr) { + // The GPU reads this range in place, so storage results pending in it must be stored first + // (outside the import lock: the store may import memory itself). The whole registered heaps + // an address-based shader maps are exempt (they hold every pending image, and such shaders + // do not read texture memory through pointers); APS5_BDA_FLUSH=1 flushes for them too. + static const bool bdaFlush = std::getenv("APS5_BDA_FLUSH") != nullptr; + if (!(addressable && region.hostBacked) || bdaFlush) StorageTexture::FlushPending(region.begin, static_cast(bytes), nullptr, "imported buffer region"); + continue; + } + if (copyOnGpu) { + // As for a direct region, storage results pending in the range reach the import before + // the copy reads it: the flush is recorded (GPU-direct into the import) ahead of the copy + // in the same batch. The batch is opened first so that an import the flush's own + // reconcile retires is handed to it (retireImport keeps nothing while the recorder is + // idle) and outlives the copy. + static_cast(recorder->Commands()); + StorageTexture::FlushPending(region.begin, static_cast(bytes), nullptr, "copied buffer region"); + region.gpuCopy = true; + gpuCopies.push_back(®ion); + continue; + } + copyRegion(region, addressable); + } + if (!gpuCopies.empty()) recordGpuCopies(gpuCopies, addressable); + if (!profile) return; + const auto uploads = uploadsProfiled.fetch_add(1, std::memory_order_relaxed) + 1; + if (uploads % 2000 == 0) { + const auto& copies = Copies(); + std::fprintf(stderr, "[buffers] %llu uploads: import lookup %.0f ms, buffer allocation %.0f ms, guest read %.0f ms; copies on the GPU: %llu regions (%.0f KiB) copied out of imports, %llu written sub-ranges (%.0f KiB) copied back, %llu staging stores by the CPU at write-back\n", static_cast(uploads), importUs.load(std::memory_order_relaxed) / 1000.0, allocateUs.load(std::memory_order_relaxed) / 1000.0, readUs.load(std::memory_order_relaxed) / 1000.0, static_cast(copies.gpuCopies.load(std::memory_order_relaxed)), copies.gpuCopyBytes.load(std::memory_order_relaxed) / 1024.0, static_cast(copies.gpuCopyBacks.load(std::memory_order_relaxed)), copies.gpuCopyBackBytes.load(std::memory_order_relaxed) / 1024.0, static_cast(copies.stagingStores.load(std::memory_order_relaxed))); + } + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + if (ms > 50) { + std::uint64_t total = 0; + std::uint64_t largest = 0; + std::uint64_t copied = 0; + for (const auto& region : regions) { + total += region.end - region.begin; + largest = std::max(largest, region.end - region.begin); + if (region.buffer != nullptr) copied += region.end - region.begin; + } + std::fprintf(stderr, "[resources] guest upload of %zu regions, %.1f MiB (largest %.1f MiB, copied %.1f MiB, addressable %d) took %.0f ms to finish\n", regions.size(), total / 1048576.0, largest / 1048576.0, copied / 1048576.0, addressable ? 1 : 0, ms); + } + // Debug aid: APS5_TRACE_BIGBUF lists every copied region of at least 1 MiB. + static const bool traceBig = std::getenv("APS5_TRACE_BIGBUF") != nullptr; + if (traceBig) { + for (const auto& region : regions) { + if (region.buffer != nullptr && region.end - region.begin >= (1u << 20u)) std::fprintf(stderr, "[bigbuf] upload 0x%llx+0x%llx writable=%d hostBacked=%d sparse=%d addressable=%d\n", static_cast(region.begin), static_cast(region.end - region.begin), region.writable ? 1 : 0, region.hostBacked ? 1 : 0, region.sparse ? 1 : 0, addressable ? 1 : 0); + } + } +} + +void GuestBufferMemory::copyRegion(Region& region, bool addressable) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto bytes = region.end - region.begin; + const auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); + if (profile) { + // Why the region is copied rather than bound in place, totalled every 1000 uploads (under a + // mutex: prepare stages of several builds copy at once). + static std::mutex reasonsMutex; + static std::uint64_t uploads = 0, noImport = 0, noImportBytes = 0, misaligned = 0, misalignedBytes = 0, snapshotOnly = 0, snapshotBytes = 0; + static std::map> outside; + bool inImport = false; + if (context.hostImportAlignment != 0) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + inImport = findImport(state, region.begin, region.end) != nullptr; + } + std::lock_guard lock(reasonsMutex); + if (!region.hostBacked && !region.writable) { ++snapshotOnly; snapshotBytes += bytes; } + else if (inImport) { ++misaligned; misalignedBytes += bytes; } + else { + ++noImport; + noImportBytes += bytes; + auto& bucket = outside[region.begin >> 28u]; + ++bucket.first; + bucket.second += bytes; + } + if (++uploads % 1000 == 0) { + // 'misaligned in imports' are the ones the GPU did not copy: over APS5_GPU_COPY_MAX_KIB, + // sparse, APS5_CPU_COPIES, no recorder, or an import gone by UploadFinish. + std::fprintf(stderr, "[buffers] %llu copied regions on the CPU: %llu snapshots (%.0f MiB), %llu misaligned in imports (%.0f MiB), %llu outside imports (%.0f MiB):", static_cast(uploads), static_cast(snapshotOnly), snapshotBytes / 1048576.0, static_cast(misaligned), misalignedBytes / 1048576.0, static_cast(noImport), noImportBytes / 1048576.0); + for (const auto& [granule, counts] : outside) std::fprintf(stderr, " 0x%llx0000000:%llu/%.0fMiB", static_cast(granule), static_cast(counts.first), counts.second / 1048576.0); + std::fprintf(stderr, "\n"); + } + } + const auto allocateStart = std::chrono::steady_clock::now(); + // A buffer made by UploadPrepare for a GPU copy that fell back here (its import went) is kept. + if (region.buffer == nullptr) region.buffer = std::make_shared(context, static_cast(bytes), usage); + const auto readStart = std::chrono::steady_clock::now(); + if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); + // Named for the [hooksync] attribution: the reads below go through the flush hook. + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::BufferUpload); + if (region.hostBacked || region.writable) { + // Only ranges a descriptor may write need the copy the write-back diffs against; the + // registered ranges of an address-based shader are read-only to it (AddressRanges). + const bool written = region.writable && WritesOverlap(region.begin, static_cast(bytes)); + if (!region.sparse) { + GuestMemory::Read(region.begin, region.buffer->Bytes()); + if (written) region.uploaded.assign(region.buffer->Bytes().begin(), region.buffer->Bytes().end()); + } else { + // `uploaded` holds the backed pieces back to back. + for (const auto& [first, last] : region.backed) { + const auto piece = region.buffer->Bytes().subspan(static_cast(first - region.begin), static_cast(last - first)); + GuestMemory::Read(first, piece); + if (written) region.uploaded.insert(region.uploaded.end(), piece.begin(), piece.end()); + } + } + } + else std::memcpy(region.buffer->Bytes().data(), region.snapshot.data(), region.snapshot.size()); + region.snapshot.clear(); + if (profile) readUs.fetch_add(microsecondsSince(readStart), std::memory_order_relaxed); +} + +void GuestBufferMemory::recordGpuCopies(std::span copies, bool addressable) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto* recorder = Recorder::Active(); + Require(recorder != nullptr, "GPU buffer copies need an active recorder"); + const auto commands = recorder->Commands(); + // Every earlier recorded store into the imports (dispatches in place, fills, GPU labels, + // GPU-direct write-backs and the flushes just recorded) and the host's own writes precede the + // copies. A CPU write made after the batch is submitted is the game's race, as on hardware. + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + for (auto* region : copies) { + const auto bytes = region->end - region->begin; + if (region->buffer == nullptr) { + // Not made by UploadPrepare (the registry was stale then): made here, under the lock. + const auto allocateStart = std::chrono::steady_clock::now(); + region->buffer = std::make_shared(context, static_cast(bytes), gpuCopyUsage(addressable)); + if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); + } + CopyBuffer(context, commands, region->copySource, region->begin - region->copySourceBase, region->buffer->Handle(), 0, bytes); + // Kept by the batch at record time, as every other recorded target is: the caller keeps + // the resources only after descriptor and pipeline work that may throw, and a released + // buffer would go back to the pool (reused or destroyed) under this copy command. The + // source is a live import or one retireImport already handed to this batch. + recorder->Keep(region->buffer); + region->snapshot.clear(); + if (profile) { + Copies().gpuCopies.fetch_add(1, std::memory_order_relaxed); + Copies().gpuCopyBytes.fetch_add(bytes, std::memory_order_relaxed); + } + } + // The copied bytes are visible to the shaders bound to the buffers, which may also store into + // them. Every stage: dispatches and draws share this path, and draws run mesh, task and + // tessellation shaders too (the batch's own barriers around dispatches are as wide). + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); +} + +void GuestBufferMemory::RecordCopyBacks(Recorder& recorder) { + if (!uploaded || committed) return; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // The written ranges, merged: a V# bound twice would otherwise copy the same bytes twice. + std::vector> merged; + bool recording = false; + VkCommandBuffer commands = VK_NULL_HANDLE; + for (auto& region : regions) { + if (!region.gpuCopy || region.copiedBack) continue; + if (merged.empty() && !writes.empty()) { + auto sorted = writes; + std::sort(sorted.begin(), sorted.end()); + for (const auto& range : sorted) { + if (!merged.empty() && range.first < merged.back().second) merged.back().second = std::max(merged.back().second, range.second); + else merged.push_back(range); + } + } + for (const auto& [begin, end] : merged) { + const auto from = std::max(begin, region.begin); + const auto to = std::min(end, region.end); + if (from >= to) continue; + if (!recording) { + commands = recorder.Commands(); + // The shader's stores into the buffer (whichever stage made them) precede the copy. + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + recording = true; + } + // Exactly the sub-range the shader may write, in place in the import: what the shader + // would have stored there itself had the range been bindable. + CopyBuffer(context, commands, region.buffer->Handle(), from - region.begin, region.copySource, from - region.copySourceBase, to - from); + recorder.Keep(region.buffer); + if (profile) { + Copies().gpuCopyBacks.fetch_add(1, std::memory_order_relaxed); + Copies().gpuCopyBackBytes.fetch_add(to - from, std::memory_order_relaxed); + } + } + // Only once its copies are recorded: a throw above leaves the region counted by + // HasCopiedWrites, so the caller still registers the CPU write-back that stores the + // staging bytes, instead of losing the shader's results. + region.copiedBack = true; + } + if (!recording) return; + // As a GPU label store or fill: visible to everything recorded after (shaders, transfers, an + // indirect dispatch's arguments) and to the host once the batch completed; the caller's + // pending-write note makes CPU readers wait for that. + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT); } VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std::size_t bytes) const { @@ -76,25 +1337,52 @@ VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std: const auto found = std::upper_bound(regions.begin(), regions.end(), address, [](std::uint64_t value, const Region& region) { return value < region.begin; }); Require(found != regions.begin(), "guest buffer has no GPU owner"); const auto& region = *std::prev(found); - Require(address >= region.begin && address + bytes <= region.end && region.buffer != nullptr, "guest buffer view exceeds its GPU owner"); - const auto offset = address - region.begin; + Require(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr), "guest buffer view exceeds its GPU owner"); + const auto base = region.direct != nullptr ? region.direct->base : region.mirror != nullptr ? region.mirror->base : region.begin; + const auto offset = address - base; Require(context.limits.minStorageBufferOffsetAlignment != 0 && offset % context.limits.minStorageBufferOffsetAlignment == 0, "guest buffer view violates storage buffer offset alignment"); Require(bytes <= context.limits.maxStorageBufferRange, "guest buffer view exceeds descriptor range limit"); - return {region.buffer->Handle(), offset, bytes}; + const auto handle = region.direct != nullptr ? region.direct->buffer : region.mirror != nullptr ? region.mirror->buffer->Handle() : region.buffer->Handle(); + return {handle, offset, bytes}; } std::vector GuestBufferMemory::AddressRanges() const { Require(uploaded && !committed, "guest GPU address ranges are not available"); std::vector result; for (const auto& region : regions) { - Require(region.buffer != nullptr, "incomplete guest GPU upload"); - const auto address = region.buffer->DeviceAddress(); + Require(region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr, "incomplete guest GPU upload"); + const auto address = region.direct != nullptr ? region.direct->address + (region.begin - region.direct->base) : region.mirror != nullptr ? region.mirror->buffer->DeviceAddress() + (region.begin - region.mirror->base) : region.buffer->DeviceAddress(); Require(region.end - region.begin <= std::numeric_limits::max() - address, "GPU address range overflow"); result.push_back({region.begin, region.end, address, ShaderRecompiler::BdaAbi::Read, 0}); } return result; } +bool GuestBufferMemory::HasCopiedWrites() const { + for (const auto& [begin, end] : writes) { + const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); + if (found == regions.begin()) continue; + const auto& region = *std::prev(found); + if (region.direct != nullptr) continue; + // A GPU copy whose written sub-ranges were copied back by the GPU lands like a direct + // write; until RecordCopyBacks ran it still counts (a use without it stores in WriteBack). + if (region.gpuCopy && region.copiedBack) continue; + // Writes into a read-only image mirror are never stored (its pages could not take them). + if (region.mirror != nullptr && !region.mirror->writable) continue; + return true; + } + return false; +} + +void GuestBufferMemory::MarkDirectWrites() const { + for (const auto& [begin, end] : writes) { + const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); + if (found == regions.begin()) continue; + const auto& region = *std::prev(found); + if (region.direct != nullptr || (region.gpuCopy && region.copiedBack)) GuestMemory::MarkWritten(begin, static_cast(end - begin)); + } +} + void GuestBufferMemory::WriteBack() { Require(uploaded && !committed, "guest memory cannot be committed twice or before upload"); std::sort(writes.begin(), writes.end()); @@ -103,18 +1391,109 @@ void GuestBufferMemory::WriteBack() { if (!merged.empty() && range.first < merged.back().second) merged.back().second = std::max(merged.back().second, range.second); else merged.push_back(range); } - std::vector> sources; + // The CPU keeps running while the GPU works, so storing whole ranges would roll back its writes to + // bytes the shader never touched. Only runs that differ from the uploaded copy are stored. for (const auto& [begin, end] : merged) { - GuestMemory::CheckRange(reinterpret_cast(begin), static_cast(end - begin), 1, true); const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); Require(found != regions.begin(), "write-back range has no GPU owner"); const auto& region = *std::prev(found); + // Every branch below but the direct and copied-back ones stores from the CPU. + if (region.direct == nullptr && !(region.gpuCopy && region.copiedBack) && !(region.mirror != nullptr && !region.mirror->writable)) Recorder::NoteWrittenBack(begin, static_cast(end - begin)); + if (region.direct != nullptr) { + // The GPU wrote imported guest memory directly; page write watching did not see it. + GuestMemory::MarkWritten(begin, static_cast(end - begin)); + continue; + } + if (region.mirror != nullptr) { + if (!region.mirror->writable) continue; + Require(end <= region.end, "write-back range exceeds its GPU owner"); + // The shadow is what the GPU started from; it then takes the mirror's bytes so the next + // refresh does not mistake the GPU's results for a guest change. + auto& mirror = *region.mirror; + const auto offset = static_cast(begin - mirror.base); + const auto length = static_cast(end - begin); + const auto current = mirror.buffer->Bytes().subspan(offset, length); + GuestMemory::WriteChanged(begin, current, std::span(mirror.shadow).subspan(offset, length)); + std::memcpy(mirror.shadow.data() + offset, current.data(), length); + continue; + } + if (region.gpuCopy) { + Require(region.buffer != nullptr && end <= region.end, "write-back range exceeds its GPU owner"); + if (region.copiedBack) { + // The GPU copied the written sub-range back into the import; page write watching + // did not see it. + GuestMemory::MarkWritten(begin, static_cast(end - begin)); + continue; + } + // No copy-back was recorded (a synchronous draw): the staging bytes of the written + // sub-range are stored whole, which is what the copy-back would have done. There is no + // reference copy to diff against (the copy in never passed through the CPU). + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + GuestMemory::Write(begin, region.buffer->Bytes().subspan(static_cast(begin - region.begin), static_cast(end - begin))); + if (profile) Copies().stagingStores.fetch_add(1, std::memory_order_relaxed); + continue; + } Require(region.buffer != nullptr && region.writable && end <= region.end, "write-back range exceeds its GPU owner"); - sources.push_back(region.buffer->Bytes().subspan(static_cast(begin - region.begin), static_cast(end - begin))); + static const bool traceBig = std::getenv("APS5_TRACE_BIGBUF") != nullptr; + if (traceBig && end - begin >= (1u << 20u)) std::fprintf(stderr, "[bigbuf] writeback 0x%llx+0x%llx sparse=%d\n", static_cast(begin), static_cast(end - begin), region.sparse ? 1 : 0); + if (!region.sparse) { + const auto first = static_cast(begin - region.begin); + const auto length = static_cast(end - begin); + GuestMemory::WriteChanged(begin, region.buffer->Bytes().subspan(first, length), std::span(region.uploaded).subspan(first, length)); + continue; + } + std::size_t packed = 0; + for (const auto& [first, last] : region.backed) { + const auto from = std::max(first, begin); + const auto to = std::min(last, end); + if (from < to) { + const auto length = static_cast(to - from); + GuestMemory::WriteChanged(from, region.buffer->Bytes().subspan(static_cast(from - region.begin), length), std::span(region.uploaded).subspan(packed + static_cast(from - first), length)); + } + packed += static_cast(last - first); + } } - for (std::size_t i = 0; i < merged.size(); ++i) GuestMemory::Write(merged[i].first, sources[i]); committed = true; lease.clear(); } +std::optional>> GuestBufferMemory::DirectRegions() const { + if (!uploaded || committed) return std::nullopt; + std::vector> result; + for (const auto& region : regions) { + // A read-only mirror is as fixed as an import: its bytes and VkBuffer never change while its + // Range exists (see ImageMirrorSerial). A writable one is refreshed and written back per build. + const bool fixedMirror = region.mirror != nullptr && !region.mirror->writable; + if (region.direct == nullptr && !fixedMirror) return std::nullopt; + result.emplace_back(region.begin, region.end); + } + return result; +} + +std::uint64_t ImageMirrorSerial(const Context& context, std::uint64_t address, std::size_t bytes) { + if (bytes == 0 || bytes > std::numeric_limits::max() - address || !mirrorsEnabled()) return 0; + // A replaced Range expires the mirror (findMirror), and a replacement mirror has a new serial. + const auto mirror = findMirror(context, address, address + bytes); + return mirror != nullptr && !mirror->writable ? mirror->serial : 0; +} + +std::uint64_t HostImportSerial(const Context& context, std::uint64_t address, std::size_t bytes, bool reconcile) { + if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return 0; + auto& state = Imports(); + std::lock_guard lock(state.mutex); + // An import whose registered range changed must not be reused: reconcile first, as an upload + // does (the walk only runs when the registry changed since the last one). + if (reconcile && GuestAllocations::GuestAllocationsGeneration_nid_postfix() != state.refreshedGeneration) static_cast(refreshImports(context, state, GuestAllocations::GuestAllocationsAcquire_nid_postfix())); + auto found = state.imports.upper_bound(address); + if (found == state.imports.begin()) return 0; + --found; + auto& entry = found->second; + if (address < entry.base || address + bytes > entry.base + entry.bytes) return 0; + // Serials are handed out on first use: a fresh import starts at 0, so one made after an earlier + // import of the same range was dropped can never repeat that import's serial. + static std::uint64_t serials = 0; + if (entry.serial == 0) entry.serial = ++serials; + return entry.serial; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp index 9528d9f998..a3f2890c8d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp @@ -75,14 +75,17 @@ GuestSamplerResource DecodeSamplerResource(std::span words) const auto borderColorType = (words[3] >> 30u) & 0x3u; Require(!forceUnormCoords, "guest sampler descriptor uses unnormalized coordinates which are not implemented"); - Require(anisoThreshold == 0, "guest sampler descriptor uses an anisotropy threshold override which is not implemented"); Require(!forceSrgb, "guest sampler descriptor forces sRGB decoding which is not implemented"); - Require(anisoBias == 0, "guest sampler descriptor uses an anisotropy bias which is not implemented"); - Require(!truncCoord, "guest sampler descriptor uses coordinate truncation which is not implemented"); + // TRUNC_COORD picks point-sampled texels by truncation instead of rounding, and the perf fields + // trade mip/depth precision for speed; Vulkan's nearest filtering already floors, so these only + // move texel selection by half a texel at most and are accepted as is. ANISO_THRESHOLD and + // ANISO_BIAS only tune how many anisotropic taps the hardware takes; the host filter picks its own. + static_cast(truncCoord); + static_cast(anisoThreshold); + static_cast(anisoBias); Require(!disableCubeWrap, "guest sampler descriptor disables seamless cube filtering which is not implemented"); Require(filterMode == 0, "guest sampler descriptor uses a reduction filter mode which is not implemented"); Require(!disableDegamma, "guest sampler descriptor disables degamma which is not implemented"); - Require(perfMip == 0 && perfZ == 0, "guest sampler descriptor uses performance counters which are not implemented"); Require(lodBiasSec == 0, "guest sampler descriptor uses a secondary LOD bias which is not implemented"); Require(!pointPreclamp, "guest sampler descriptor uses point preclamping which is not implemented"); Require(!anisoOverride, "guest sampler descriptor uses an anisotropy override which is not implemented"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp index b7e13b992d..3cd364eae5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp @@ -1,5 +1,7 @@ +#include #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include #include #include @@ -17,7 +19,10 @@ TextureTileMode resolveTileMode(std::uint32_t raw) { case 0x01: return TextureTileMode::kStandard256B; case 0x05: return TextureTileMode::kStandard4KB; case 0x09: return TextureTileMode::kStandard64KB; - case 0x1b: return TextureTileMode::RenderTarget64KB; + case 0x18: return TextureTileMode::kZ64KBX; + case 0x19: return TextureTileMode::kS64KBX; + case 0x1a: return TextureTileMode::kD64KBX; + case 0x1b: return TextureTileMode::kR64KBX; default: throw std::runtime_error("AGC graphics: guest texture descriptor uses an unsupported tile mode " + std::to_string(raw)); } } @@ -26,6 +31,7 @@ TextureDimension resolveDimension(std::uint32_t raw) { switch (raw) { case 8: return TextureDimension::k1D; case 9: return TextureDimension::k2D; + case 10: return TextureDimension::k3D; case 11: return TextureDimension::kCube; case 13: return TextureDimension::k2DArray; default: throw std::runtime_error("AGC graphics: guest texture descriptor uses an unsupported image type " + std::to_string(raw)); @@ -51,7 +57,7 @@ GuestTextureResource DecodeTextureResource(std::span words) const auto dstSelZ = (words[3] >> 6u) & 0x7u; const auto dstSelW = (words[3] >> 9u) & 0x7u; const auto baseLevel = (words[3] >> 12u) & 0xfu; - const auto lastLevel = (words[3] >> 16u) & 0xfu; + auto lastLevel = (words[3] >> 16u) & 0xfu; const auto tileModeRaw = (words[3] >> 20u) & 0x1fu; const auto bcSwizzle = (words[3] >> 25u) & 0x7u; const auto typeRaw = (words[3] >> 28u) & 0xfu; @@ -83,21 +89,54 @@ GuestTextureResource DecodeTextureResource(std::span words) requireValidDstSel(dstSelW); Require(minLod == 0, "guest texture descriptor uses a nonzero minimum LOD clamp which is not implemented"); - Require(minLodWarn == 0, "guest texture descriptor uses a minimum LOD warning threshold which is not implemented"); - Require(mipStatsCntId == 0 && !mipStatsCntEn, "guest texture descriptor uses mip statistics counters which are not implemented"); + // The LOD warning threshold and mip statistics counters are texture-streaming feedback: the GPU + // reports which mips were wanted. Nothing is reported back here, which only affects what the title + // chooses to stream, not what this access returns. + if (minLodWarn != 0 || mipStatsCntId != 0 || mipStatsCntEn) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] texture streaming feedback (LOD warning / mip statistics) is not reported\n"); + } + } Require(!cornerSample, "guest texture descriptor uses corner sampling which is not implemented"); Require(!prtDefColor, "guest texture descriptor uses a partially resident default color which is not implemented"); Require(arrayPitch == 0, "guest texture descriptor uses a nonzero array pitch which is not implemented"); Require(!msaaDepth, "guest texture descriptor uses MSAA which is not implemented"); - Require(maxUncompBlkSize == 0 && maxCompBlkSize == 0, "guest texture descriptor uses DCC block size overrides which are not implemented"); - Require(!metaPipeAligned && !writeCompress && !metaCompress && !dccAlphaPos && !dccColorTransf && metaAddr == 0, "guest texture descriptor uses metadata compression which is not implemented"); + // DCC block sizes only apply with metadata compression, which is rejected above. + static_cast(maxUncompBlkSize); + static_cast(maxCompBlkSize); + // Surfaces are always written uncompressed here (render targets and storage images bypass DCC), so + // only the fast-clear keys in DCC metadata change what a read returns (see DccMetadata.hpp). + static_cast(metaPipeAligned); + static_cast(writeCompress); + if (dccColorTransf) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] texture DCC color transform is ignored (word6=0x%08x word7=0x%08x)\n", words[6], words[7]); + } + } Require(bcSwizzle == 0, "guest texture descriptor uses a BC swizzle which is not implemented"); Require(baseLevel <= lastLevel, "guest texture descriptor has a base mip level past its last mip level"); - Require(lastLevel == maxMip, "guest texture descriptor must expose every mip level down to the last one"); - const auto tileMode = resolveTileMode(tileModeRaw); - const auto dimension = resolveDimension(typeRaw); + + const auto describe = [&] { return " (format " + std::to_string(format) + ", " + std::to_string(width) + "x" + std::to_string(height) + "x" + std::to_string(depth + 1) + ", type " + std::to_string(typeRaw) + ", tile " + std::to_string(tileModeRaw) + ", mips " + std::to_string(baseLevel) + ".." + std::to_string(lastLevel) + " of " + std::to_string(maxMip + 1) + ")"; }; + TextureTileMode tileMode; + TextureDimension dimension; + try { + tileMode = resolveTileMode(tileModeRaw); + dimension = resolveDimension(typeRaw); + } catch (const std::exception& error) { + throw std::runtime_error(std::string(error.what()) + describe()); + } + Require(baseLevel <= maxMip, "guest texture descriptor starts past the surface's last mip level" + describe()); + // Views may name levels past MAX_MIP (a 512x512 view through 1x1 over a 9-level surface); the + // hardware never addresses them, so the view ends at the surface's last level. + lastLevel = std::min(lastLevel, maxMip); + // XOR swizzles fold a pipe/bank XOR into the low address bits; only unmodified 64 KiB bases are modeled. + Require(XorSwizzleMode(tileMode) == 0 || (baseAddress & 0xffffu) == 0, "guest texture descriptor combines an XOR swizzle with a pipe/bank XOR base which is not implemented"); switch (dimension) { case TextureDimension::k1D: @@ -106,6 +145,10 @@ GuestTextureResource DecodeTextureResource(std::span words) case TextureDimension::k2D: Require(depth == 0 && baseArray == 0, "guest 2D texture descriptor has a nonzero depth or base array"); break; + case TextureDimension::k3D: + // word4 DEPTH holds the depth minus one; BASE_ARRAY has no meaning for volumes. + Require(baseArray == 0, "guest 3D texture descriptor has a nonzero base array"); + break; case TextureDimension::k2DArray: Require(baseArray <= depth, "guest 2D array texture descriptor has a base array past its last array slice"); break; @@ -124,6 +167,7 @@ GuestTextureResource DecodeTextureResource(std::span words) result.baseArray = baseArray; result.mipCount = maxMip + 1u; result.baseLevel = baseLevel; + result.lastLevel = lastLevel; result.tileMode = tileMode; result.dimension = dimension; result.format = format; @@ -131,6 +175,8 @@ GuestTextureResource DecodeTextureResource(std::span words) result.dstSelY = static_cast(dstSelY); result.dstSelZ = static_cast(dstSelZ); result.dstSelW = static_cast(dstSelW); + result.dccAddress = metaCompress ? metaAddr << 8u : 0u; + result.dccAlphaOnMsb = dccAlphaPos; return result; } @@ -138,9 +184,9 @@ bool MatchesGuestDimension(ShaderRecompiler::DescriptorImageShape shape, Texture switch (shape) { case ShaderRecompiler::DescriptorImageShape::Image1D: return dimension == TextureDimension::k1D; case ShaderRecompiler::DescriptorImageShape::Image2D: return dimension == TextureDimension::k2D; - case ShaderRecompiler::DescriptorImageShape::Image2DArray: return dimension == TextureDimension::k2DArray; + case ShaderRecompiler::DescriptorImageShape::Image2DArray: return dimension == TextureDimension::k2DArray || dimension == TextureDimension::kCube; case ShaderRecompiler::DescriptorImageShape::ImageCube: return dimension == TextureDimension::kCube; - case ShaderRecompiler::DescriptorImageShape::Image3D: return false; + case ShaderRecompiler::DescriptorImageShape::Image3D: return dimension == TextureDimension::k3D; } throw std::runtime_error("AGC graphics: MatchesGuestDimension encountered an unknown descriptor image shape"); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 3d55bc8784..517072b9a8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -1,16 +1,53 @@ #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include namespace AgcDriver::Graphics { -Pipeline::Pipeline(const Context& context, const State& state, const RenderTarget* target, const ShaderResources& resources, std::span shaders) : context(context), _modules(shaders.size()) { - Require((target != nullptr) == state.hasColorTarget, "render target does not match decoded color state"); - Require(state.renderExtent.width != 0 && state.renderExtent.height != 0 && state.renderExtent.width <= context.limits.maxFramebufferWidth && state.renderExtent.height <= context.limits.maxFramebufferHeight, "framebuffer extent exceeds device limits"); +Framebuffer::Framebuffer(const Context& context, VkRenderPass renderPass, std::span targets, VkExtent2D extent) : context(context) { + // Cached objects outlive their device's teardown; they must not keep its buffer pool alive past it. + this->context.bufferPool.reset(); + Require(extent.width != 0 && extent.height != 0 && extent.width <= context.limits.maxFramebufferWidth && extent.height <= context.limits.maxFramebufferHeight, "framebuffer extent exceeds device limits"); + VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; + framebufferInfo.renderPass = renderPass; + framebufferInfo.attachmentCount = static_cast(targets.size()); + framebufferInfo.pAttachments = targets.empty() ? nullptr : targets.data(); + framebufferInfo.width = extent.width; + framebufferInfo.height = extent.height; + framebufferInfo.layers = 1; + Check(context.Function("vkCreateFramebuffer")(context.device, &framebufferInfo, nullptr, &framebuffer), "vkCreateFramebuffer"); +} + +Framebuffer::~Framebuffer() { + if (framebuffer) context.Function("vkDestroyFramebuffer")(context.device, framebuffer, nullptr); +} + +void ValidateViewport(const Context& context, const VkViewport& viewport) { + Require(std::isfinite(viewport.minDepth) && std::isfinite(viewport.maxDepth), "non-finite viewport depth range"); + Require(context.depthRangeUnrestricted || (viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1), "viewport depth outside [0, 1] requires VK_EXT_depth_range_unrestricted"); + Require(std::isfinite(viewport.x) && std::isfinite(viewport.y) && std::isfinite(viewport.width) && std::isfinite(viewport.height), "viewport arithmetic overflow"); + Require(viewport.width <= context.limits.maxViewportDimensions[0] && std::abs(viewport.height) <= context.limits.maxViewportDimensions[1], "viewport dimensions exceed device limits"); + Require(viewport.x >= context.limits.viewportBoundsRange[0] && viewport.x + viewport.width <= context.limits.viewportBoundsRange[1], "viewport X exceeds device bounds"); + Require(std::min(viewport.y, viewport.y + viewport.height) >= context.limits.viewportBoundsRange[0] && std::max(viewport.y, viewport.y + viewport.height) <= context.limits.viewportBoundsRange[1], "viewport Y exceeds device bounds"); +} + +Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders) : context(context), _modules(shaders.size()), attachments(state.colors.size()) { + // A cached pipeline may outlive its device's teardown (see ClearCachedPipelines); it must not keep + // the buffer pool, which is reset with the device, alive past it. + this->context.bufferPool.reset(); + Require(state.blends.size() == state.colors.size(), "blend states do not match decoded color state"); + Require(state.colors.size() <= context.limits.maxColorAttachments, "color targets exceed device attachment limits"); Require(state.hasColorTarget || (context.limits.framebufferNoAttachmentsSampleCounts & VK_SAMPLE_COUNT_1_BIT) != 0, "device does not support single-sample rendering without attachments"); - ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric); Require(!state.negativeOneToOne || context.depthClipControl, "negative-one-to-one depth clipping requires VK_EXT_depth_clip_control with depthClipControl enabled"); if (state.rectList) Require(context.tessellationShader && context.limits.maxTessellationPatchSize >= 4, "rect-list requires tessellation with four output control points"); if (state.stages.tessellation) { @@ -23,13 +60,6 @@ Pipeline::Pipeline(const Context& context, const State& state, const RenderTarge Require(mesh.threadsPerGroup <= context.meshLimits.maxMeshWorkGroupInvocations && mesh.threadsPerGroup <= context.meshLimits.maxMeshWorkGroupSize[0], "mesh workgroup exceeds device limits"); Require(mesh.maxVertices <= context.meshLimits.maxMeshOutputVertices && mesh.maxPrimitives <= context.meshLimits.maxMeshOutputPrimitives && static_cast(mesh.ldsSizeDwords) * 4 <= context.meshLimits.maxMeshSharedMemorySize, "mesh output or LDS exceeds device limits"); } - const auto& viewport = state.viewport; - Require(std::isfinite(viewport.minDepth) && std::isfinite(viewport.maxDepth), "non-finite viewport depth range"); - Require(context.depthRangeUnrestricted || (viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1), "viewport depth outside [0, 1] requires VK_EXT_depth_range_unrestricted"); - Require(std::isfinite(viewport.x) && std::isfinite(viewport.y) && std::isfinite(viewport.width) && std::isfinite(viewport.height), "viewport arithmetic overflow"); - Require(viewport.width <= context.limits.maxViewportDimensions[0] && std::abs(viewport.height) <= context.limits.maxViewportDimensions[1], "viewport dimensions exceed device limits"); - Require(viewport.x >= context.limits.viewportBoundsRange[0] && viewport.x + viewport.width <= context.limits.viewportBoundsRange[1], "viewport X exceeds device bounds"); - Require(std::min(viewport.y, viewport.y + viewport.height) >= context.limits.viewportBoundsRange[0] && std::max(viewport.y, viewport.y + viewport.height) <= context.limits.viewportBoundsRange[1], "viewport Y exceeds device bounds"); const auto pushStages = PushConstantStages(shaders); Require(pushStages == 0 || context.limits.maxPushConstantsSize >= PipelinePushConstantBytes, "graphics push constant range exceeds device limit"); try { @@ -46,6 +76,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const RenderTarge stages[i].module = _modules[i]; stages[i].pName = "main"; } + // A descriptor set layout with the same bindings as this one is compatible with the pipeline + // layout, so later draws bind their own ShaderResources' set under it. const auto setLayout = resources.Layout(); const VkPushConstantRange push{pushStages, 0, PipelinePushConstantBytes}; VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; @@ -54,61 +86,61 @@ Pipeline::Pipeline(const Context& context, const State& state, const RenderTarge layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout graphics"); - VkAttachmentDescription color{}; - color.format = state.color.format; - color.samples = VK_SAMPLE_COUNT_1_BIT; - color.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; - color.storeOp = VK_ATTACHMENT_STORE_OP_STORE; - color.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; - color.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; - color.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - color.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - VkAttachmentReference reference{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + std::vector colors; + std::vector references; + for (std::uint32_t index = 0; index < state.colors.size(); ++index) { + VkAttachmentDescription color{}; + color.format = state.colors[index].format; + color.samples = VK_SAMPLE_COUNT_1_BIT; + color.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; + color.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + color.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + color.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + color.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + color.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + colors.push_back(color); + references.push_back({index, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}); + } VkSubpassDescription subpass{}; subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; - subpass.colorAttachmentCount = state.hasColorTarget ? 1 : 0; - subpass.pColorAttachments = state.hasColorTarget ? &reference : nullptr; + subpass.colorAttachmentCount = static_cast(references.size()); + subpass.pColorAttachments = references.empty() ? nullptr : references.data(); VkRenderPassCreateInfo passInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; - passInfo.attachmentCount = state.hasColorTarget ? 1 : 0; - passInfo.pAttachments = state.hasColorTarget ? &color : nullptr; + passInfo.attachmentCount = static_cast(colors.size()); + passInfo.pAttachments = colors.empty() ? nullptr : colors.data(); passInfo.subpassCount = 1; passInfo.pSubpasses = &subpass; Check(context.Function("vkCreateRenderPass")(context.device, &passInfo, nullptr, &renderPass), "vkCreateRenderPass"); - const auto view = target ? target->View() : VK_NULL_HANDLE; - VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; - framebufferInfo.renderPass = renderPass; - framebufferInfo.attachmentCount = state.hasColorTarget ? 1 : 0; - framebufferInfo.pAttachments = state.hasColorTarget ? &view : nullptr; - framebufferInfo.width = state.renderExtent.width; - framebufferInfo.height = state.renderExtent.height; - framebufferInfo.layers = 1; - Check(context.Function("vkCreateFramebuffer")(context.device, &framebufferInfo, nullptr, &framebuffer), "vkCreateFramebuffer"); VkPipelineVertexInputStateCreateInfo input{VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; - const auto vertexInput = BuildVertexInputLayout(context, shaders.front().program->vertexAttributes); input.vertexBindingDescriptionCount = static_cast(vertexInput.bindings.size()); input.pVertexBindingDescriptions = vertexInput.bindings.data(); input.vertexAttributeDescriptionCount = static_cast(vertexInput.attributes.size()); input.pVertexAttributeDescriptions = vertexInput.attributes.data(); VkPipelineInputAssemblyStateCreateInfo assembly{VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO}; assembly.topology = state.topology; + // Viewport and scissor are set per draw (Begin), so they do not multiply pipelines; the depth + // clip control stays baked in. VkPipelineViewportStateCreateInfo viewports{VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO}; VkPipelineViewportDepthClipControlCreateInfoEXT depthClip{VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT}; depthClip.negativeOneToOne = state.negativeOneToOne; if (state.negativeOneToOne) viewports.pNext = &depthClip; viewports.viewportCount = 1; - viewports.pViewports = &state.viewport; viewports.scissorCount = 1; - viewports.pScissors = &state.scissor; + const std::array dynamicStates{VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR}; + VkPipelineDynamicStateCreateInfo dynamic{VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO}; + dynamic.dynamicStateCount = static_cast(dynamicStates.size()); + dynamic.pDynamicStates = dynamicStates.data(); VkPipelineRasterizationStateCreateInfo raster{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO}; raster.polygonMode = VK_POLYGON_MODE_FILL; + raster.depthClampEnable = state.depthClamp && context.depthClamp ? VK_TRUE : VK_FALSE; raster.cullMode = state.cullMode; raster.frontFace = state.frontFace; raster.lineWidth = 1; VkPipelineMultisampleStateCreateInfo samples{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; samples.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; VkPipelineColorBlendStateCreateInfo blend{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; - blend.attachmentCount = state.hasColorTarget ? 1 : 0; - blend.pAttachments = state.hasColorTarget ? &state.blend : nullptr; + blend.attachmentCount = static_cast(state.blends.size()); + blend.pAttachments = state.blends.empty() ? nullptr : state.blends.data(); std::copy(state.blendConstants.begin(), state.blendConstants.end(), blend.blendConstants); VkGraphicsPipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO}; pipelineInfo.stageCount = static_cast(stages.size()); @@ -124,6 +156,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const RenderTarge pipelineInfo.pRasterizationState = &raster; pipelineInfo.pMultisampleState = &samples; pipelineInfo.pColorBlendState = &blend; + pipelineInfo.pDynamicState = &dynamic; pipelineInfo.layout = layout; pipelineInfo.renderPass = renderPass; Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines"); @@ -138,26 +171,83 @@ Pipeline::~Pipeline() { } void Pipeline::release() noexcept { + framebuffers.clear(); if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); - if (framebuffer) context.Function("vkDestroyFramebuffer")(context.device, framebuffer, nullptr); if (renderPass) context.Function("vkDestroyRenderPass")(context.device, renderPass, nullptr); if (layout) context.Function("vkDestroyPipelineLayout")(context.device, layout, nullptr); for (auto module : _modules) { if (module) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); } + pipeline = VK_NULL_HANDLE; + renderPass = VK_NULL_HANDLE; + layout = VK_NULL_HANDLE; + _modules.clear(); +} + +void Pipeline::Abandon() noexcept { + for (auto& entry : framebuffers) entry.framebuffer->Abandon(); + framebuffers.clear(); + pipeline = VK_NULL_HANDLE; + renderPass = VK_NULL_HANDLE; + layout = VK_NULL_HANDLE; + _modules.clear(); +} + +void Framebuffer::Abandon() noexcept { + framebuffer = VK_NULL_HANDLE; } VkPipelineLayout Pipeline::Layout() const { return layout; } -void Pipeline::Begin(VkCommandBuffer commands, VkExtent2D extent) const { +std::shared_ptr Pipeline::AcquireFramebuffer(std::span targets, std::span> owners, VkExtent2D extent) { + Require(targets.size() == attachments && targets.size() == owners.size(), "render targets do not match the pipeline's color attachments"); + const bool resident = std::all_of(owners.begin(), owners.end(), [](const auto& owner) { return owner != nullptr; }); + if (resident) { + // Entries whose views are gone can never match again and go as soon as no recorded draw holds + // them (Kept keeps its framebuffer until the batch completes). + std::erase_if(framebuffers, [](const CachedFramebuffer& entry) { + return entry.framebuffer.use_count() == 1 && std::any_of(entry.owners.begin(), entry.owners.end(), [](const auto& owner) { return owner.expired(); }); + }); + for (auto it = framebuffers.begin(); it != framebuffers.end(); ++it) { + if (it->extent.width != extent.width || it->extent.height != extent.height || !std::equal(it->views.begin(), it->views.end(), targets.begin(), targets.end())) continue; + // View handles are recycled once a StorageTexture is destroyed, so the owners must be the + // very objects the views were made for. + bool same = true; + for (std::size_t i = 0; i < owners.size() && same; ++i) same = it->owners[i].lock().get() == owners[i].get(); + if (!same) continue; + std::rotate(it, std::next(it), framebuffers.end()); + return framebuffers.back().framebuffer; + } + } + auto framebuffer = std::make_shared(context, renderPass, targets, extent); + if (!resident) return framebuffer; + // Beyond the bound the least recently used unreferenced entry goes. + constexpr std::size_t bound = 8; + while (framebuffers.size() >= bound) { + const auto victim = std::find_if(framebuffers.begin(), framebuffers.end(), [](const CachedFramebuffer& entry) { return entry.framebuffer.use_count() == 1; }); + if (victim == framebuffers.end()) break; + framebuffers.erase(victim); + } + CachedFramebuffer entry; + entry.views.assign(targets.begin(), targets.end()); + entry.owners.assign(owners.begin(), owners.end()); + entry.extent = extent; + entry.framebuffer = framebuffer; + framebuffers.push_back(std::move(entry)); + return framebuffer; +} + +void Pipeline::Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, VkExtent2D extent, const VkViewport& viewport, const VkRect2D& scissor) const { VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; begin.renderPass = renderPass; - begin.framebuffer = framebuffer; + begin.framebuffer = framebuffer.Handle(); begin.renderArea = {{0, 0}, extent}; context.Function("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE); context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + context.Function("vkCmdSetViewport")(commands, 0, 1, &viewport); + context.Function("vkCmdSetScissor")(commands, 0, 1, &scissor); } void Pipeline::PushConstants(VkCommandBuffer commands, std::span shaders) const { @@ -167,4 +257,211 @@ void Pipeline::PushConstants(VkCommandBuffer commands, std::span("vkCmdPushConstants")(commands, layout, stages, 0, PipelinePushConstantBytes, bytes.data()); } +namespace { + +template +void append(std::vector& key, const TValue& value) { + static_assert(std::is_trivially_copyable_v); + const auto bytes = std::as_bytes(std::span(&value, 1)); + key.insert(key.end(), bytes.begin(), bytes.end()); +} + +// Everything the Pipeline objects are built from, or empty when a stage's result has no variant id +// (the recompiler could not identify it, so nothing else may share its pipeline). The rect-list +// control and evaluation stages are generated from the vertex and fragment results, which the key +// already names, so they carry no id of their own. +std::vector pipelineKey(const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span shaders) { + using Stage = ShaderRecompiler::ShaderStage; + std::vector key; + append(key, context.device); + append(key, shaders.size()); + for (const auto& shader : shaders) { + Require(shader.program != nullptr, "missing compiled shader"); + const bool generated = state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation); + if (!generated && shader.program->variantId == 0) return {}; + append(key, shader.stage); + append(key, generated ? std::uint64_t{0} : shader.program->variantId); + // Where the stage's push constants sit in the block (AssemblePushConstants). + append(key, shader.pushConstantOffset); + } + append(key, PushConstantStages(shaders)); + append(key, input.bindings.size()); + for (const auto& binding : input.bindings) { + append(key, binding.binding); + append(key, binding.stride); + append(key, binding.inputRate); + } + append(key, input.attributes.size()); + for (const auto& attribute : input.attributes) { + append(key, attribute.location); + append(key, attribute.binding); + append(key, attribute.format); + append(key, attribute.offset); + } + append(key, resources.LayoutKey().size()); + for (const auto word : resources.LayoutKey()) append(key, word); + append(key, state.hasColorTarget); + append(key, state.rectList); + append(key, state.topology); + append(key, state.cullMode); + append(key, state.frontFace); + append(key, state.negativeOneToOne); + append(key, state.depthClamp && context.depthClamp); + append(key, state.blends.size()); + for (const auto& blend : state.blends) append(key, blend); + for (const auto value : state.blendConstants) append(key, value); + append(key, state.colors.size()); + for (const auto& color : state.colors) append(key, color.format); + append(key, state.stages.mesh.has_value()); + if (state.stages.mesh) { + const auto& mesh = *state.stages.mesh; + append(key, mesh.inputPrimitive); + append(key, mesh.primitivesPerGroup); + append(key, mesh.verticesPerGroup); + append(key, mesh.maxVertices); + append(key, mesh.maxPrimitives); + append(key, mesh.threadsPerGroup); + append(key, mesh.ldsSizeDwords); + append(key, mesh.provokingVertex); + } + append(key, state.stages.tessellation.has_value()); + if (state.stages.tessellation) { + const auto& tessellation = *state.stages.tessellation; + append(key, tessellation.inputControlPoints); + append(key, tessellation.outputControlPoints); + append(key, tessellation.domain); + append(key, tessellation.partitioning); + append(key, tessellation.outputTopology); + } + return key; +} + +std::uint64_t hashKey(const std::vector& key) { + std::uint64_t hash = 14695981039346656037ull; + for (const auto byte : key) { + hash ^= static_cast(byte); + hash *= 1099511628211ull; + } + return hash; +} + +struct PipelineStore { + struct Entry { + VkDevice device; + // The device's buffer pool at insertion: it is made and reset with the device, so it tells + // the device instance apart from a later one the loader gave the same handle value. + std::weak_ptr pool; + std::uint64_t hash; + std::vector key; + std::shared_ptr pipeline; + }; + std::mutex mutex; + // Least recently used first. + std::list entries; + std::unordered_map::iterator> index; + std::uint64_t hits = 0; + std::uint64_t misses = 0; + std::uint64_t uncached = 0; + std::uint64_t evicted = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +// Never destroyed: the pipelines belong to a device that may already be gone when statics die, and +// the device's teardown (ClearCachedPipelines) is the place to destroy them. +PipelineStore& Pipelines() { + static auto* store = new PipelineStore(); + return *store; +} + +// Whether the entry's objects belong to the device the context names. A context without a buffer +// pool (tests) is identified by the handle alone. +bool alive(const PipelineStore::Entry& entry, const Context& context) { + if (entry.device != context.device) return false; + return context.bufferPool == nullptr || entry.pool.lock() == context.bufferPool; +} + +// Drops an entry of a device that is gone: its objects went with the device, so they are forgotten, +// not destroyed. +std::list::iterator abandon(PipelineStore& store, std::list::iterator it) { + it->pipeline->Abandon(); + store.index.erase(it->hash); + return store.entries.erase(it); +} + +void reportPipelines(PipelineStore& store) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + if (now - store.lastReport < std::chrono::seconds(10)) return; + store.lastReport = now; + const auto lookups = store.hits + store.misses + store.uncached; + std::fprintf(stderr, "[pipecache] %llu lookups over 10 s: %llu hits (%.0f%%), %llu misses, %llu private (no variant id), %llu evicted, %zu cached\n", static_cast(lookups), static_cast(store.hits), lookups != 0 ? 100.0 * static_cast(store.hits) / static_cast(lookups) : 0.0, static_cast(store.misses), static_cast(store.uncached), static_cast(store.evicted), store.entries.size()); + store.hits = store.misses = store.uncached = store.evicted = 0; +} + +} + +std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders) { + static const bool disabled = std::getenv("APS5_NO_PIPELINE_CACHE") != nullptr; + if (disabled) return std::make_shared(context, state, vertexInput, resources, shaders); + auto& store = Pipelines(); + std::lock_guard lock(store.mutex); + reportPipelines(store); + const auto key = pipelineKey(context, state, vertexInput, resources, shaders); + if (key.empty()) { + ++store.uncached; + return std::make_shared(context, state, vertexInput, resources, shaders); + } + const auto hash = hashKey(key); + if (const auto found = store.index.find(hash); found != store.index.end()) { + const auto it = found->second; + if (it->key == key) { + // The key names the device handle, which the loader may reuse for a device created after + // this one was destroyed without ClearCachedPipelines: such an entry is a miss. + if (alive(*it, context)) { + ++store.hits; + store.entries.splice(store.entries.end(), store.entries, it); + return it->pipeline; + } + abandon(store, it); + } else { + // A different configuration with the same hash keeps the resident entry; this one stays private. + ++store.uncached; + return std::make_shared(context, state, vertexInput, resources, shaders); + } + } + ++store.misses; + // Entries of another device belong to one the driver replaced (it does so under the GpuMutex + // before any draw reaches the new device), whose objects went with it: forget them. + for (auto it = store.entries.begin(); it != store.entries.end();) { + it = alive(*it, context) ? std::next(it) : abandon(store, it); + } + auto pipeline = std::make_shared(context, state, vertexInput, resources, shaders); + store.entries.push_back({context.device, context.bufferPool, hash, key, pipeline}); + store.index[hash] = std::prev(store.entries.end()); + constexpr std::size_t bound = 256; + while (store.entries.size() > bound) { + // Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence). + const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; }); + if (victim == store.entries.end()) break; + store.index.erase(victim->hash); + store.entries.erase(victim); + ++store.evicted; + } + return pipeline; +} + +void ClearCachedPipelines(VkDevice device) { + auto& store = Pipelines(); + std::lock_guard lock(store.mutex); + for (auto it = store.entries.begin(); it != store.entries.end();) { + if (it->device != device) { + ++it; + continue; + } + store.index.erase(it->hash); + it = store.entries.erase(it); + } +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp new file mode 100644 index 0000000000..938c438b0c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -0,0 +1,1592 @@ +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/Pm4Opcodes.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +namespace { + +Recorder* activeRecorder = nullptr; + +bool DrawProfiled() { + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profiled; +} + +std::uint64_t syncCounts[5] = {}; +// Fence wait time by sync source (APS5_PROFILE_DRAW), and the source CountSync announced for the +// Sync/SyncThrough that follows it on this thread. +double syncWaitedMs[5] = {}; +thread_local int announcedSource = 4; +// The call site CountSync named for that sync (nullptr: none; the sync's own return address is +// taken then), and the fence-wait table per (source, site) it feeds (APS5_PROFILE_DRAW, under the +// GpuMutex like finish()): which caller's syncs wait, when the source alone does not say (source +// 0 is every WaitIdle and drain). `activeSyncSite` is the entry of the sync in progress on this +// thread (an index: a nested sync from a completion replaces and restores it), so finish() charges +// each batch's wait to it. +thread_local const void* announcedSite = nullptr; +struct SyncSiteWaits { + int source; + const void* site; + std::uint64_t syncs; + std::uint64_t batches; + double waitedMs; +}; +std::vector syncSites; +constexpr std::size_t NoSyncSite = static_cast(-1); +constexpr std::size_t SyncSiteLimit = 48; +thread_local std::size_t activeSyncSite = NoSyncSite; + +bool SyncSitesProfiled() { + // APS5_NO_SYNC_SITES=1 leaves only the per-source and per-thread counts. + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr && std::getenv("APS5_NO_SYNC_SITES") == nullptr; + return profiled; +} + +// Begins a sync's attribution: returns the previous active entry for the caller to restore, after +// counting the sync under (source, site). Beyond the table's limit every further site shares the +// last entry (a nullptr site), so the table stays small. +std::size_t BeginSyncSite(int source, const void* site) { + const auto previous = activeSyncSite; + if (!SyncSitesProfiled()) return previous; + auto it = std::find_if(syncSites.begin(), syncSites.end(), [&](const SyncSiteWaits& entry) { return entry.source == source && entry.site == site; }); + if (it == syncSites.end()) { + if (syncSites.size() >= SyncSiteLimit) { + it = std::find_if(syncSites.begin(), syncSites.end(), [&](const SyncSiteWaits& entry) { return entry.source == source && entry.site == nullptr; }); + if (it == syncSites.end()) it = syncSites.insert(syncSites.end(), SyncSiteWaits{source, nullptr, 0, 0, 0}); + } else { + it = syncSites.insert(syncSites.end(), SyncSiteWaits{source, site, 0, 0, 0}); + } + } + ++it->syncs; + activeSyncSite = static_cast(it - syncSites.begin()); + return previous; +} + +void CountSiteWait(double ms) { + if (activeSyncSite == NoSyncSite || activeSyncSite >= syncSites.size()) return; + auto& entry = syncSites[activeSyncSite]; + ++entry.batches; + entry.waitedMs += ms; +} + +// The sites by wait, longest first, as "source@+offset syncs/batches/wait" (offsets symbolize with +// nm against the driver's image like the [guestmem] callers; +0x0 is the overflow entry). +std::string SyncSiteReport() { + static const char* const names[5] = {"idle", "pending-write", "recorded-store", "address-based", "other"}; + std::vector order; + for (const auto& entry : syncSites) order.push_back(&entry); + std::sort(order.begin(), order.end(), [](const SyncSiteWaits* a, const SyncSiteWaits* b) { return a->waitedMs > b->waitedMs; }); + std::string report; + for (std::size_t i = 0; i < order.size() && i < 8; ++i) { + char text[96]; + std::snprintf(text, sizeof(text), " %s@+0x%llx %llu/%llu/%.1fs", names[order[i]->source], order[i]->site != nullptr ? GuestMemory::CodeOffset(order[i]->site) : 0ull, static_cast(order[i]->syncs), static_cast(order[i]->batches), order[i]->waitedMs / 1000); + report += text; + } + return report; +} +// Fence waits by thread and source (APS5_PROFILE_DRAW), keyed by the thread's GpuMutex queue tag: +// a sync waits for the GPU under the mutex, so which worker's syncs (and which kind) hold it against +// queue 0 has to be visible. Its own mutex: the table is read while the report line is printed. +struct ThreadSyncs { + std::uint32_t tag; + std::array counts; + std::array waitedMs; +}; +std::vector threadSyncs; +std::mutex threadSyncsMutex; + +// This thread's fence and timeline waits in total (Recorder::ThreadWaitedMs): a caller times a span +// of its own work and reads the difference to learn how much of it was waiting for the GPU. +thread_local double threadWaitedMs = 0; + +void CountThreadSync(int source, double ms) { + threadWaitedMs += ms; + const auto tag = GuestMemory::GpuLockThreadTag(); + std::lock_guard lock(threadSyncsMutex); + auto it = std::find_if(threadSyncs.begin(), threadSyncs.end(), [&](const ThreadSyncs& thread) { return thread.tag == tag; }); + if (it == threadSyncs.end()) it = threadSyncs.insert(threadSyncs.end(), ThreadSyncs{tag, {}, {}}); + ++it->counts[source]; + it->waitedMs[source] += ms; +} + +std::string ThreadSyncReport() { + static const char* const names[5] = {"idle", "pending-write", "recorded-store", "address-based", "other"}; + std::string report; + std::lock_guard lock(threadSyncsMutex); + for (const auto& thread : threadSyncs) { + char text[64]; + if (thread.tag == 0xffffffffu) std::snprintf(text, sizeof(text), " untagged:"); + else std::snprintf(text, sizeof(text), " queue 0x%x:", thread.tag); + report += text; + for (int source = 0; source < 5; ++source) { + if (thread.counts[source] == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu/%.1fs", names[source], static_cast(thread.counts[source]), thread.waitedMs[source] / 1000); + report += text; + } + } + return report; +} +// Unlocked timeline waits (WaitSerial): count and time, for the [recorder] line. +std::atomic unlockedWaits{0}, unlockedWaitedUs{0}; +// Threads inside a timeline wait with the GpuMutex released (WaitSerial, syncThroughUnlocked): a +// recorder's teardown, after its Sync() made every wait satisfiable, waits until they left the +// semaphore before destroying it. Counted per recorder id (a small fixed array; the waiter cannot +// touch a recorder that may be dying, so it is keyed by the id it copied out), so a replaced +// device's old recorder does not spin on the new device's in-flight waits. +constexpr std::size_t WaiterSlots = 16; +std::atomic unlockedWaiters[WaiterSlots]{}; +std::atomic& WaitersOf(std::uint64_t id) { return unlockedWaiters[id % WaiterSlots]; } +// Recorders alive, by id (own mutex, taken under the GpuMutex or with nothing held): a thread that +// released the GpuMutex around a wait learns whether its recorder still exists before touching it. +std::mutex liveRecordersMutex; +std::vector liveRecorders; +std::atomic nextRecorderId{1}; + +bool RecorderAlive(std::uint64_t id) { + std::lock_guard lock(liveRecordersMutex); + return std::find(liveRecorders.begin(), liveRecorders.end(), id) != liveRecorders.end(); +} + +// Completion actions in progress on this thread (finish()): a pending-write sync the flush hook +// makes from inside one (a copied buffer's write-back store) is skipped, see SyncThrough. +thread_local int completionDepth = 0; + +// APS5_COMPLETION_STORE_SYNC=1: a store made by a completion action waits for later batches that +// note its range, as before (the wait runs under whichever hold reaped the batch). +bool CompletionStoreSyncs() { + static const bool enabled = std::getenv("APS5_COMPLETION_STORE_SYNC") != nullptr; + return enabled; +} + +// APS5_HOOK_LOCKED_WAIT=1: the flush hook's pending-write wait stays under the GpuMutex as before. +bool HookLockedWait() { + static const bool locked = std::getenv("APS5_HOOK_LOCKED_WAIT") != nullptr; + return locked; +} + +// What the holds spend on the recorder (APS5_PROFILE_DRAW, all under the GpuMutex, printed at the +// end of the [recorder] line): completions and their time (of which releasing the kept objects), +// pending-write syncs from inside completions (skipped, or waited with the kill switch), reaps and +// the batches they retired, and the flush hook's unlocked waits (with those that found the +// recorder torn down when they retook the mutex, and those that had to wait locked). +struct HoldCounters { + std::uint64_t completions = 0; + double completionMs = 0; + double keptReleaseMs = 0; + std::uint64_t completionSyncsSkipped = 0; + std::uint64_t completionSyncsWaited = 0; + double completionSyncWaitMs = 0; + std::uint64_t reaps = 0; + std::uint64_t reapsWithWork = 0; + std::uint64_t reapBatches = 0; + double reapMs = 0; + std::uint64_t hookUnlockedWaits = 0; + // The GPU wait alone (timeline wait with the mutex released) and, separately, the relock's + // own wait for the GpuMutex (also counted by the [lock] line under 'hook'), so the two + // remaining costs, GPU latency and hold contention, stay apart. + double hookUnlockedWaitMs = 0; + double hookRelockMs = 0; + std::uint64_t hookUnlockedTornDown = 0; + // Genuine nested hooks (depth >= 2 outside a completion action) that had to wait locked. + std::uint64_t hookLockedWaits = 0; +}; +HoldCounters holdCounters; + +// Deferred release of a finished batch's kept objects (and its completion actions, whose captures +// hold some of the same objects): finish() moves them to this list of the finishing thread instead +// of destroying them under the mutex, and the GpuMutex unlock hook (ReleaseDeferredKeeps, set by +// the constructor) hands the list to the release thread once the thread gave up its outermost +// hold. Destroying a ShaderResources with its buffers, descriptor set, textures and pipelines cost +// ~1.5 ms per reap with work, inside whichever hold reaped ('kept objects released', 15.9 s per +// 200 s run), and most reaps are queue 0's own (ReapRecorded before its dispatches), so a release +// on the unlocking thread would still be the frame's time. The destructors touch only their own +// caches' mutexes and the device, never the recorder or GpuMutex, so they need no hold and no +// particular thread (every Vulkan object is externally synchronized per object, and its cache's +// mutex covers its pool); they need the device alive, which ~Recorder guarantees by draining its +// own thread's list, stopping and joining the release thread (it drains the queue before it +// leaves) and waiting for every release still in progress on another thread (`deferredPending`: +// counted up under the mutex, down after each batch's objects are gone) before the device goes. +// The queue is bounded (APS5_RELEASE_QUEUE_MAX batches, default 256): a thread whose hand-off +// would exceed it destroys its batches itself, as it did before the release thread, so memory +// cannot grow behind a release thread that falls behind. +// APS5_RELEASE_UNDER_LOCK=1 destroys them in finish() as before; APS5_RELEASE_ON_UNLOCK=1 destroys +// them on the unlocking thread (no release thread). +struct DeferredBatch { + std::vector> kept; + std::vector> completions; +}; +thread_local std::vector deferredBatches; +std::atomic deferredPending{0}; +// APS5_PROFILE_DRAW, for the [recorder] line: batches and objects released after an unlock and the +// time that took, on the release thread and inline (on the unlocking thread: the kill switch, a +// full queue, a stopping thread or a failed hand-off), the batches that went inline because the +// queue was full, and the longest queue seen (in batches). +std::atomic threadReleases{0}, threadObjects{0}, threadReleaseUs{0}; +std::atomic inlineReleases{0}, inlineObjects{0}, inlineReleaseUs{0}, inlineOverBound{0}; +std::atomic releaseQueueMax{0}; + +bool ReleaseUnderLock() { + static const bool locked = std::getenv("APS5_RELEASE_UNDER_LOCK") != nullptr; + return locked; +} + +bool ReleaseOnUnlock() { + static const bool onUnlock = std::getenv("APS5_RELEASE_ON_UNLOCK") != nullptr; + return onUnlock; +} + +std::size_t ReleaseQueueBound() { + static const std::size_t bound = [] { + const char* value = std::getenv("APS5_RELEASE_QUEUE_MAX"); + const auto parsed = value != nullptr ? std::strtoull(value, nullptr, 10) : 0ull; + return parsed != 0 ? static_cast(parsed) : std::size_t{256}; + }(); + return bound; +} + +// Destroys deferred batches, on the release thread (`onThread`) or on the thread that deferred +// them, and counts each batch off `deferredPending` only once its objects are gone: ~Recorder +// waits for that count. +void DestroyDeferred(std::vector releasing, bool onThread) { + if (releasing.empty()) return; + const bool profile = DrawProfiled(); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto count = releasing.size(); + std::uint64_t objects = 0; + for (auto& batch : releasing) { + objects += batch.kept.size(); + // The completions first (their captures hold some of the objects), then the objects. + batch.completions.clear(); + batch.kept.clear(); + deferredPending.fetch_sub(1, std::memory_order_acq_rel); + } + releasing.clear(); + if (profile) { + const auto us = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); + (onThread ? threadReleases : inlineReleases).fetch_add(count, std::memory_order_relaxed); + (onThread ? threadObjects : inlineObjects).fetch_add(objects, std::memory_order_relaxed); + (onThread ? threadReleaseUs : inlineReleaseUs).fetch_add(us, std::memory_order_relaxed); + } +} + +// The release thread's queue: one for the process. The thread is started at the first hand-off and +// stopped and joined by ~Recorder (JoinReleaseThread), after which the next hand-off starts it +// again (a device replacement makes a new recorder). The queue object is leaked so that a static +// destructor never races the thread: at process exit the thread is idle on its condition variable +// (every ~Recorder joined it) or, when no ~Recorder ran, mid-release like any worker thread. Nothing +// is done under `mutex` but the hand-off and the take; `joinMutex` serializes joiners and orders +// before `mutex`; neither is ever taken by a kept object's destructor. +struct ReleaseQueue { + std::mutex mutex; + std::condition_variable wake; + std::vector items; + std::thread thread; + bool started = false; + // Set by a joiner under `mutex` until the thread was joined: the thread leaves once the queue + // is empty, and hand-offs meanwhile destroy inline (nothing may be queued without a taker). + bool stop = false; + std::mutex joinMutex; +}; + +ReleaseQueue& ReleaseThreadQueue() { + static ReleaseQueue* const queue = new ReleaseQueue; + return *queue; +} + +void ReleaseThreadMain() { + auto& queue = ReleaseThreadQueue(); + std::unique_lock lock(queue.mutex); + for (;;) { + queue.wake.wait(lock, [&] { return !queue.items.empty() || queue.stop; }); + if (queue.items.empty()) return; + auto items = std::move(queue.items); + queue.items.clear(); + lock.unlock(); + DestroyDeferred(std::move(items), true); + lock.lock(); + } +} + +// Stops the release thread once it emptied the queue and joins it (~Recorder, before its device +// goes). Hand-offs made while the thread stops destroy inline, so nothing can sit in the queue +// without a taker; a later hand-off starts the thread again. +void JoinReleaseThread() { + auto& queue = ReleaseThreadQueue(); + std::lock_guard joining(queue.joinMutex); + std::thread worker; + { + std::lock_guard lock(queue.mutex); + if (!queue.started) return; + queue.stop = true; + worker = std::move(queue.thread); + } + queue.wake.notify_all(); + worker.join(); + std::lock_guard lock(queue.mutex); + queue.started = false; + queue.stop = false; +} + +// The unlock hook: hands what this thread deferred to the release thread, or destroys it here +// (APS5_RELEASE_ON_UNLOCK=1, the queue is full or stopping, or the hand-off cannot be made). +void ReleaseDeferredKeeps() { + if (deferredBatches.empty()) return; + // Taken off the thread's list first: a destructor that took and released the mutex (none is + // known to) would re-enter here and must find nothing. + auto releasing = std::move(deferredBatches); + deferredBatches.clear(); + if (ReleaseOnUnlock()) { + DestroyDeferred(std::move(releasing), false); + return; + } + auto& queue = ReleaseThreadQueue(); + const bool profile = DrawProfiled(); + bool handedOff = false; + bool overBound = false; + try { + std::lock_guard lock(queue.mutex); + if (queue.items.size() + releasing.size() > ReleaseQueueBound()) { + overBound = true; + } else if (!queue.stop) { + // The thread before the items: items queued with no thread to take them would keep + // `deferredPending` up and ~Recorder waiting forever. + if (!queue.started) { + queue.thread = std::thread(&ReleaseThreadMain); + queue.started = true; + } + // The allocation before any move: a failure here leaves `releasing` intact for the + // fallback below, and the moves cannot throw. + queue.items.reserve(queue.items.size() + releasing.size()); + for (auto& batch : releasing) queue.items.push_back(std::move(batch)); + handedOff = true; + if (profile) { + const auto queued = static_cast(queue.items.size()); + auto seen = releaseQueueMax.load(std::memory_order_relaxed); + while (queued > seen && !releaseQueueMax.compare_exchange_weak(seen, queued, std::memory_order_relaxed)) { + } + } + } + } catch (...) { + } + if (!handedOff) { + if (profile && overBound) inlineOverBound.fetch_add(releasing.size(), std::memory_order_relaxed); + DestroyDeferred(std::move(releasing), false); + return; + } + queue.wake.notify_one(); +} +// Lock-free state of the active recorder's open batch, read by the queue workers between packets +// and inside WAIT_REG_MEM polls (see the static readers in Recorder.hpp). Written under the mutex. +constexpr std::int64_t NoPendingLabel = std::numeric_limits::min(); +std::atomic pendingLabelSince{NoPendingLabel}; +std::atomic writeGeneration{0}; +std::atomic completionLabels{0}; +std::atomic completionStoresSkipped{0}; +std::atomic completionStoresRun{0}; +std::atomic workSinceSubmit{0}; +// The pending-label table's own mutex: every mutation of a recorder's `labels` holds it (under the +// GPU mutex), so a WAIT_REG_MEM looks a label up with this small lock alone (Recorder::LookupLabel) +// instead of queueing behind a dispatch. `labelTableOwner` is the active recorder while it lives, +// set and cleared under this mutex, so a lookup never touches a table being destroyed. Nothing +// under this mutex takes the GPU mutex (no lock-order cycle). +std::mutex labelTableMutex; +Recorder* labelTableOwner = nullptr; +// Flush hook statistics: calls, calls whose snapshot check overlapped (the GpuMutex was taken), +// targeted syncs and the batches they left in flight. +std::atomic hookCalls{0}, hookLocks{0}, targetedSyncs{0}, batchesLeftInFlight{0}; +// Snapshot maintenance (under the GpuMutex): notes whose range the snapshot already covered (no +// rebuild), rebuilds, and the time the rebuilds took (APS5_PROFILE_DRAW), so their cost is visible. +std::uint64_t snapshotCovered = 0, snapshotRebuilds = 0; +double snapshotRebuildMs = 0; + +using WriteRanges = std::vector>; +// The sorted, merged union of the guest ranges every unfinished batch will write. Rebuilt under +// GuestMemory::GpuMutex (the only writer) and read by the flush hook without it: the lock is the +// graphics worker's main stall, so a no-overlap access must not wait for another queue's device +// work. A range leaves the snapshot only after its batch's completions (CPU write-backs) ran, so a +// reader that sees no overlap either precedes the note (the queues are unordered then, as on the +// GPU) or follows the write-back. +std::atomic> pendingWrites; + +bool HookSnapshotEnabled() { + // Debug aid: APS5_NO_HOOK_SNAPSHOT=1 takes the GpuMutex on every access as before. + static const bool enabled = std::getenv("APS5_NO_HOOK_SNAPSHOT") == nullptr; + return enabled; +} + +bool SnapshotOverlaps(std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return false; + const auto snapshot = pendingWrites.load(std::memory_order_acquire); + if (snapshot == nullptr || snapshot->empty()) return false; + // Merged ranges are ordered by both bounds: the first one ending past the access decides. + const auto it = std::partition_point(snapshot->begin(), snapshot->end(), [&](const auto& range) { return range.second <= address; }); + return it != snapshot->end() && it->first < address + bytes; +} + +// Whether one merged range of the snapshot contains [address, end) entirely. +bool SnapshotCovers(std::uint64_t address, std::uint64_t end) { + const auto snapshot = pendingWrites.load(std::memory_order_acquire); + if (snapshot == nullptr || snapshot->empty()) return false; + const auto it = std::partition_point(snapshot->begin(), snapshot->end(), [&](const auto& range) { return range.second <= address; }); + return it != snapshot->end() && it->first <= address && end <= it->second; +} + +// Attribution of the pending-write syncs the hook makes (the [hooksync] line every 10 s, under +// APS5_PROFILE_DRAW; APS5_NO_HOOKSYNC_PROFILE=1 leaves only the plain counts). Every sync is charged +// to the packet the accessing thread executes and the read site it named (or, when it named none, +// its return addresses), together with the noted range it hit, the batch that noted it and the +// time waited. A small read (up to 64 KiB) has its bytes compared before and after the wait: a +// sync whose bytes come out unchanged although its target batch had not run yet was not needed +// for that read (the noted range was larger than what the GPU wrote, or the GPU wrote the same +// values), which separates false overlaps from real producer/consumer dependencies; see +// HookSyncOutcomes for the cases that prove nothing. All state lives under the GpuMutex, which +// the hook holds. +// Debug aid: APS5_NO_SYNC_THROUGH=1 makes every pending-write sync a full Sync (SyncThrough and +// the attribution's DescribePendingWrite read the same switch). +bool SyncThroughEnabled() { + static const bool enabled = std::getenv("APS5_NO_SYNC_THROUGH") == nullptr; + return enabled; +} + +// Six frames: whether the hook's constructor and FlushGpuWrites are inlined into the GuestMemory +// entry decides at which frame the driver caller sits, so enough are kept to see its own caller. +constexpr std::size_t HookSyncFrames = 6; + +struct HookSyncKey { + std::uint32_t queue; + std::uint32_t opcode; + GuestMemory::ReadSite site; + std::array frames; + bool operator<(const HookSyncKey& other) const { + return std::tie(queue, opcode, site, frames) < std::tie(other.queue, other.opcode, other.site, other.frames); + } +}; + +// Whether the bytes of a small access changed over the wait, split by the state of the target +// batch: only an unchanged access whose target had not run yet (open, or in flight with its fence +// unsignaled) shows the sync was not needed; against a signaled fence the copy made before the +// wait already held the GPU's values, so unchanged proves nothing. A store (ReadSite::Store) is +// counted apart: its sync orders a CPU write after the GPU's, whatever the bytes do. +struct HookSyncOutcomes { + std::uint64_t unchangedOpen = 0; + std::uint64_t unchangedPending = 0; + std::uint64_t unchangedSignaled = 0; + std::uint64_t changed = 0; + std::uint64_t stores = 0; + std::uint64_t unchecked = 0; +}; + +struct HookSyncTotals { + std::uint64_t count = 0; + HookSyncOutcomes outcomes; + std::uint64_t rangeBytes = 0; + std::uint64_t accessBytes = 0; + double waitedMs = 0; +}; + +// Size buckets for the noted ranges and the accesses: <=4K, <=64K, <=1M, <=16M, <=256M, larger. +constexpr std::size_t SizeBuckets = 6; +constexpr const char* SizeBucketNames[SizeBuckets] = {"<=4K", "<=64K", "<=1M", "<=16M", "<=256M", ">256M"}; + +std::size_t SizeBucket(std::uint64_t bytes) { + if (bytes <= 4096) return 0; + if (bytes <= 65536) return 1; + if (bytes <= (1u << 20u)) return 2; + if (bytes <= (16u << 20u)) return 3; + if (bytes <= (256u << 20u)) return 4; + return 5; +} + +struct HookSyncStats { + std::map byKey; + std::uint64_t count = 0, openTargets = 0, signaledTargets = 0, batchesFinished = 0; + HookSyncOutcomes outcomes; + double waitedMs = 0; + std::array rangeBuckets{}; + std::array accessBuckets{}; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +HookSyncStats& HookSyncs() { + static HookSyncStats stats; + return stats; +} + +bool HookSyncProfiled() { + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr && std::getenv("APS5_NO_HOOKSYNC_PROFILE") == nullptr; + return profiled; +} + +std::string PacketName(std::uint32_t opcode) { + if (opcode == GuestMemory::NoPacket) return "no-packet"; + if (opcode == 0xffffu) return "flip"; + // 0xfffd is reserved for the deferred-label stores (recordDeferredLabels), once its owner + // brackets them with SetCurrentPacket; until then they are charged to the following packet. + if (opcode == 0xfffdu) return "deferred-labels"; + for (const auto& entry : Pm4::Opcodes) { + if (entry.value == opcode) return std::string(entry.name); + } + char text[24]; + std::snprintf(text, sizeof(text), "op 0x%x", opcode); + return text; +} + +void ReportHookSyncs(HookSyncStats& stats) { + std::vector> hot; + hot.reserve(stats.byKey.size()); + for (const auto& [key, totals] : stats.byKey) hot.emplace_back(&key, &totals); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second->waitedMs > b.second->waitedMs; }); + std::string report; + char text[512]; + const auto count = [](std::uint64_t value) { return static_cast(value); }; + const auto& o = stats.outcomes; + std::snprintf(text, sizeof(text), "[hooksync] %llu pending-write syncs waited %.0f ms (10 s); read bytes after the wait: unchanged %llu (target open %llu, in flight unsignaled %llu, signaled %llu), changed %llu; stores %llu, unchecked %llu; targets: %llu open, %llu already signaled, %llu batches finished; noted range:", count(stats.count), stats.waitedMs, count(o.unchangedOpen + o.unchangedPending + o.unchangedSignaled), count(o.unchangedOpen), count(o.unchangedPending), count(o.unchangedSignaled), count(o.changed), count(o.stores), count(o.unchecked), count(stats.openTargets), count(stats.signaledTargets), count(stats.batchesFinished)); + report += text; + for (std::size_t i = 0; i < SizeBuckets; ++i) { + if (stats.rangeBuckets[i] == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu", SizeBucketNames[i], static_cast(stats.rangeBuckets[i])); + report += text; + } + report += "; access:"; + for (std::size_t i = 0; i < SizeBuckets; ++i) { + if (stats.accessBuckets[i] == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu", SizeBucketNames[i], static_cast(stats.accessBuckets[i])); + report += text; + } + report += "; top by wait (queue packet site frames: count/ms, unchanged open/unsignaled/signaled, changed, stores, avg range/access):"; + for (std::size_t i = 0; i < hot.size() && i < 10; ++i) { + const auto& key = *hot[i].first; + const auto& totals = *hot[i].second; + const auto& k = totals.outcomes; + std::snprintf(text, sizeof(text), " [0x%x %s %s +0x%llx/+0x%llx/+0x%llx/+0x%llx/+0x%llx/+0x%llx: %llu/%.0fms u%llu/%llu/%llu c%llu s%llu %.0fK/%.0fK]", key.queue, PacketName(key.opcode).c_str(), GuestMemory::ReadSiteName(key.site), key.frames[0], key.frames[1], key.frames[2], key.frames[3], key.frames[4], key.frames[5], count(totals.count), totals.waitedMs, count(k.unchangedOpen), count(k.unchangedPending), count(k.unchangedSignaled), count(k.changed), count(k.stores), totals.rangeBytes / 1024.0 / totals.count, totals.accessBytes / 1024.0 / totals.count); + report += text; + } + std::fprintf(stderr, "%s\n", report.c_str()); + stats = HookSyncStats{}; +} + +// Wraps one pending-write sync (constructed before it, under the GpuMutex): gathers what the sync +// waits for and, in the destructor, the time it took and whether the bytes changed. +class HookSyncScope { +public: + HookSyncScope(const Recorder& recorder, std::uint64_t address, std::size_t bytes) : enabled(HookSyncProfiled()), address(address), bytes(bytes) { + if (!enabled) return; + info = recorder.DescribePendingWrite(address, bytes); + const auto packet = GuestMemory::CurrentPacket(); + key = HookSyncKey{packet.queue, packet.opcode, GuestMemory::CurrentReadSite(), {}}; + // Frame 0 is the hook's caller (the GuestMemory entry, or the driver code when the entry + // was inlined); the driver code that made the access is one of the next frames. + GuestMemory::CaptureCallerOffsets(key.frames, 1); + // The access's bytes are copied raw: a Read here would re-enter the hook. The same access + // check the accessing code makes guards the copy; a range that fails it is left unchecked. + // A store's bytes are not compared (counted as a store instead). + if (key.site != GuestMemory::ReadSite::Store && bytes <= CompareLimit && GuestMemory::Accessible(reinterpret_cast(address), bytes)) { + before.resize(bytes); + std::memcpy(before.data(), reinterpret_cast(address), bytes); + } + start = std::chrono::steady_clock::now(); + } + ~HookSyncScope() { + if (!enabled) return; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - start).count(); + auto& stats = HookSyncs(); + auto& totals = stats.byKey[key]; + ++totals.count; + totals.waitedMs += ms; + totals.accessBytes += bytes; + ++stats.count; + stats.waitedMs += ms; + stats.accessBuckets[SizeBucket(bytes)] += 1; + if (info.has_value()) { + const auto rangeBytes = info->rangeEnd - info->rangeBegin; + totals.rangeBytes += rangeBytes; + stats.rangeBuckets[SizeBucket(rangeBytes)] += 1; + if (info->open) ++stats.openTargets; + if (info->signaled) ++stats.signaledTargets; + stats.batchesFinished += info->batchesToFinish; + } + // One outcome per sync, on both the key's and the interval's counters. + const auto outcome = [&]() -> std::uint64_t HookSyncOutcomes::* { + if (key.site == GuestMemory::ReadSite::Store) return &HookSyncOutcomes::stores; + if (before.empty() || !GuestMemory::Accessible(reinterpret_cast(address), bytes)) return &HookSyncOutcomes::unchecked; + if (std::memcmp(before.data(), reinterpret_cast(address), bytes) != 0) return &HookSyncOutcomes::changed; + // A target already signaled (or no target found: the batch is finishing) had run + // before the copy, so its unchanged bytes are not evidence of a false overlap. + if (!info.has_value() || info->signaled) return &HookSyncOutcomes::unchangedSignaled; + return info->open ? &HookSyncOutcomes::unchangedOpen : &HookSyncOutcomes::unchangedPending; + }(); + ++(totals.outcomes.*outcome); + ++(stats.outcomes.*outcome); + if (now - stats.lastReport > std::chrono::seconds(10)) ReportHookSyncs(stats); + } + HookSyncScope(const HookSyncScope&) = delete; + HookSyncScope& operator=(const HookSyncScope&) = delete; + +private: + static constexpr std::size_t CompareLimit = 65536; + bool enabled; + std::uint64_t address; + std::size_t bytes; + std::optional info; + HookSyncKey key{}; + std::vector before; + std::chrono::steady_clock::time_point start; +}; + +// GuestMemory flush hook: a CPU access to memory that recorded GPU work will write waits for that +// work first; then storage image results pending for the range are stored. The recorder is only +// dereferenced under the GpuMutex (it is destroyed with its device under that lock). +void FlushForAccess(std::uint64_t address, std::size_t bytes) { + hookCalls.fetch_add(1, std::memory_order_relaxed); + if (!HookSnapshotEnabled() || SnapshotOverlaps(address, bytes)) { + hookLocks.fetch_add(1, std::memory_order_relaxed); + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); + std::lock_guard gpu(GuestMemory::GpuMutex()); + if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(address, bytes)) { + const HookSyncScope attribution(*recorder, address, bytes); + Recorder::CountSync(1); + // The wait runs without the mutex when this acquisition is the outermost one (a stage-A + // read, a game thread); the recorder is not dereferenced after the call. + recorder->SyncThrough(address, bytes, true); + } + } + if (StorageTexture::FlushPending(address, bytes)) { + // Stores into imported memory were only recorded; the CPU is about to read them. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); + std::lock_guard gpu(GuestMemory::GpuMutex()); + if (auto* recorder = Recorder::Active(); recorder != nullptr) { + Recorder::CountSync(2); + recorder->Sync(); + } + } +} + +} + +Recorder::Recorder(const Context& context, bool timelineSemaphores) : context(context), id(nextRecorderId.fetch_add(1)) { + { + std::lock_guard lock(liveRecordersMutex); + liveRecorders.push_back(id); + } + // Idempotent: the same hook for every recorder (the deferred lists are per thread, not per recorder). + GuestMemory::SetGpuUnlockHook(&ReleaseDeferredKeeps); + if (!timelineSemaphores) return; + // The timeline starts at 0 and every Submit signals its serial (1, 2, ...): a value that only + // grows, so a thread waiting for it outside the mutex can never observe a reset or a reused + // handle (the batch fences are pooled and reset in release()). A failure here only disables the + // unlocked waits; the locked Sync path still works. + VkSemaphoreTypeCreateInfoKHR type{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO_KHR}; + type.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE_KHR; + type.initialValue = 0; + VkSemaphoreCreateInfo info{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &type}; + const auto result = context.Function("vkCreateSemaphore")(context.device, &info, nullptr, &timeline); + if (result != VK_SUCCESS) { + std::fprintf(stderr, "[gpu] timeline semaphore creation failed (Vulkan result %d); drains wait under the GPU mutex\n", static_cast(result)); + timeline = VK_NULL_HANDLE; + } +} + +Recorder::~Recorder() { + try { + Sync(); + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] recorder teardown: %s\n", error.what()); + } + if (activeRecorder == this) { + activeRecorder = nullptr; + pendingWrites.store(nullptr, std::memory_order_release); + pendingLabelSince.store(NoPendingLabel, std::memory_order_release); + completionLabels.store(0, std::memory_order_release); + workSinceSubmit.store(0, std::memory_order_release); + } + { + // A thread that released the mutex around a wait on this recorder re-checks the id when it + // retakes the mutex and touches nothing then. That check is only sound while the device + // (and so this destructor) is torn down under the GpuMutex, as the driver's drain and + // failure paths do; a worker dropping the last device reference after releasing its lock + // would run this without the mutex, concurrently with a relocked waiter. + std::lock_guard lock(liveRecordersMutex); + liveRecorders.erase(std::remove(liveRecorders.begin(), liveRecorders.end(), id), liveRecorders.end()); + } + // Sync() above completed every batch, so every timeline wait in progress on this recorder + // returns now; the semaphore is destroyed only once they all left it. + while (WaitersOf(id).load(std::memory_order_acquire) != 0) std::this_thread::yield(); + // The kept objects of every batch this thread finished (Sync() above included) belong to the + // device that is going away: destroyed now, here (no hand-off: nothing should stay in flight + // behind this thread); the release thread is then stopped and joined (it drains its queue + // first, holding no mutex this thread holds: the destructors take only their caches' own + // mutexes, which order after the GpuMutex), and the releases still in progress inline on + // other threads (an unlock hook that found the queue stopping or full) are waited for before + // the caller destroys the device. The count is global: a newer recorder's batches (device + // replacement) are waited for too, which only prolongs the spin. + if (!deferredBatches.empty()) { + auto own = std::move(deferredBatches); + deferredBatches.clear(); + DestroyDeferred(std::move(own), false); + } + JoinReleaseThread(); + while (deferredPending.load(std::memory_order_acquire) != 0) std::this_thread::yield(); + { + // Unlocked lookups stop before `labels` is destroyed (after this body). + std::lock_guard tableLock(labelTableMutex); + if (labelTableOwner == this) labelTableOwner = nullptr; + } + for (auto& [commands, fence] : spare) { + context.Function("vkFreeCommandBuffers")(context.device, context.pool, 1, &commands); + context.Function("vkDestroyFence")(context.device, fence, nullptr); + } + spare.clear(); + // Sync() above waited for every batch, so no submission still signals the timeline. + if (timeline != VK_NULL_HANDLE) context.Function("vkDestroySemaphore")(context.device, timeline, nullptr); + timeline = VK_NULL_HANDLE; +} + +std::optional Recorder::PendingLabelSince() { + const auto since = pendingLabelSince.load(std::memory_order_acquire); + if (since == NoPendingLabel) return std::nullopt; + return std::chrono::steady_clock::time_point(std::chrono::steady_clock::duration(since)); +} + +std::uint64_t Recorder::WriteGeneration() { + return writeGeneration.load(std::memory_order_acquire); +} + +std::uint64_t Recorder::PendingCompletionLabels() { + return completionLabels.load(std::memory_order_acquire); +} + +std::uint64_t Recorder::RecordedWorkSinceSubmit() { + return workSinceSubmit.load(std::memory_order_relaxed); +} + +void Recorder::CountRecordedWork() { + workSinceSubmit.fetch_add(1, std::memory_order_relaxed); +} + +Recorder* Recorder::Active() { + return activeRecorder; +} + +void Recorder::Activate() { + activeRecorder = this; + { + std::lock_guard tableLock(labelTableMutex); + labelTableOwner = this; + } + GuestMemory::SetFlushHook(&FlushForAccess); +} + +std::optional Recorder::LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) { + std::lock_guard tableLock(labelTableMutex); + if (labelTableOwner == nullptr) return std::nullopt; + return labelTableOwner->lookupLabel(address, bytes, afterStamp, queue); +} + +bool Recorder::SnapshotWriteOverlaps(std::uint64_t address, std::size_t bytes) { + return SnapshotOverlaps(address, bytes); +} + +void Recorder::CountSync(int source, const void* site) { + if (source < 0 || source >= 5) return; + ++syncCounts[source]; + announcedSource = source; + announcedSite = site; +} + +void Recorder::AnnounceSyncSite(const void* site) { + announcedSite = site; +} + +double Recorder::ThreadWaitedMs() { + return threadWaitedMs; +} + +VkCommandBuffer Recorder::Commands() { + GuestMemory::AssertGpuLockHeld("Recorder::Commands"); + if (open == nullptr) { + auto batch = std::make_unique(); + try { + if (!spare.empty()) { + // The pool resets command buffers on begin; the fence was reset when the batch completed. + std::tie(batch->commands, batch->fence) = spare.back(); + spare.pop_back(); + } else { + VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + allocation.commandPool = context.pool; + allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocation.commandBufferCount = 1; + Check(context.Function("vkAllocateCommandBuffers")(context.device, &allocation, &batch->commands), "vkAllocateCommandBuffers recorder"); + VkFenceCreateInfo info{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + Check(context.Function("vkCreateFence")(context.device, &info, nullptr, &batch->fence), "vkCreateFence recorder"); + } + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + Check(context.Function("vkBeginCommandBuffer")(batch->commands, &begin), "vkBeginCommandBuffer recorder"); + } catch (...) { + release(*batch); + throw; + } + open = std::move(batch); + } + return open->commands; +} + +namespace { + +constexpr std::uint32_t MaxTimedRanges = 512; + +bool GpuTimingEnabled() { + static const bool enabled = std::getenv("APS5_PROFILE_GPU") != nullptr; + return enabled; +} + +} + +std::uint32_t Recorder::BeginGpuTiming(std::uint64_t key) { + if (!GpuTimingEnabled()) return NoTiming; + const auto commands = Commands(); + if (open->queries == VK_NULL_HANDLE) { + VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; + info.queryType = VK_QUERY_TYPE_TIMESTAMP; + info.queryCount = MaxTimedRanges * 2; + if (context.Function("vkCreateQueryPool")(context.device, &info, nullptr, &open->queries) != VK_SUCCESS) { + open->queries = VK_NULL_HANDLE; + return NoTiming; + } + context.Function("vkCmdResetQueryPool")(commands, open->queries, 0, info.queryCount); + } + if (open->timedKeys.size() >= MaxTimedRanges) return NoTiming; + const auto index = static_cast(open->timedKeys.size()); + open->timedKeys.push_back(key); + // Both stamps wait for everything before them to complete, so the range is the timed work alone + // (a top-of-pipe stamp is not held back by the barrier that precedes the work). + context.Function("vkCmdWriteTimestamp")(commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, open->queries, index * 2); + return index; +} + +void Recorder::EndGpuTiming(std::uint32_t index) { + if (index == NoTiming || open == nullptr || open->queries == VK_NULL_HANDLE) return; + context.Function("vkCmdWriteTimestamp")(open->commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, open->queries, index * 2 + 1); +} + +void Recorder::readGpuTiming(Batch& batch) { + if (batch.queries == VK_NULL_HANDLE || batch.timedKeys.empty()) return; + std::vector stamps(batch.timedKeys.size() * 2); + const auto result = context.Function("vkGetQueryPoolResults")(context.device, batch.queries, 0, static_cast(stamps.size()), stamps.size() * sizeof(std::uint64_t), stamps.data(), sizeof(std::uint64_t), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); + if (result != VK_SUCCESS) return; + struct Totals { std::uint64_t count = 0; double ms = 0; }; + static std::map byKey; + static double totalMs = 0; + static std::uint64_t batches = 0; + static auto lastReport = std::chrono::steady_clock::now(); + for (std::size_t i = 0; i < batch.timedKeys.size(); ++i) { + const auto ns = static_cast(stamps[i * 2 + 1] - stamps[i * 2]) * context.limits.timestampPeriod; + auto& totals = byKey[batch.timedKeys[i]]; + ++totals.count; + totals.ms += ns / 1e6; + totalMs += ns / 1e6; + } + ++batches; + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::vector> hot(byKey.begin(), byKey.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.ms > b.second.ms; }); + std::fprintf(stderr, "[gputime] %.0f ms of GPU time in %llu batches over 10 s; by program:", totalMs, static_cast(batches)); + for (std::size_t i = 0; i < hot.size() && i < 12; ++i) std::fprintf(stderr, " 0x%llx x%llu %.0fms", static_cast(hot[i].first), static_cast(hot[i].second.count), hot[i].second.ms); + std::fprintf(stderr, "\n"); + byKey.clear(); + totalMs = 0; + batches = 0; +} + +void Recorder::Keep(std::shared_ptr object) { + Commands(); + open->kept.push_back(std::move(object)); +} + +void Recorder::OnComplete(std::function action) { + Commands(); + open->completions.push_back(std::move(action)); +} + +bool Recorder::noteWrite(std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return false; + Commands(); + const auto end = address + bytes; + open->writes.emplace_back(address, end); + // A poller waiting on this range learns that the open batch may now hold its producer. + if (activeRecorder == this) writeGeneration.fetch_add(1, std::memory_order_release); + // Consecutive dispatches write the same output buffers: a range the snapshot already contains + // leaves the published union unchanged, so the O(N log N) rebuild is skipped. + if (SnapshotCovers(address, end)) { + ++snapshotCovered; + return false; + } + return true; +} + +void Recorder::noteWriteOn(Batch& batch, std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return; + if (&batch == open.get()) { + NotePendingWrite(address, bytes); + return; + } + // An in-flight batch: its range joins the snapshot at once (the completion that stores it runs + // when the batch finishes, and the hook must sync for a CPU read until then). The generation + // moves as well, so a poller re-consults the label table for a completion label. + batch.writes.emplace_back(address, address + bytes); + if (activeRecorder == this) writeGeneration.fetch_add(1, std::memory_order_release); + if (!SnapshotCovers(address, address + bytes)) publishPendingWrites(); + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +void Recorder::NotePendingWrite(std::uint64_t address, std::size_t bytes) { + if (!noteWrite(address, bytes)) return; + publishPendingWrites(); + // The note precedes this thread's vkQueueSubmit and the label another queue polls for; the + // fence makes that order hold without relying on x86 store ordering. + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +void Recorder::NotePendingWrites(std::span> ranges) { + bool publish = false; + for (const auto& [begin, end] : ranges) { + if (end > begin && noteWrite(begin, static_cast(end - begin))) publish = true; + } + if (!publish) return; + publishPendingWrites(); + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +void Recorder::publishPendingWrites() const { + // Only the active recorder owns the snapshot: one torn down after its successor was activated, + // or one a worker still dispatches into after device.reset(), must not publish its ranges. + if (activeRecorder != this) return; + const auto started = DrawProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + auto merged = std::make_shared(); + if (open != nullptr) merged->insert(merged->end(), open->writes.begin(), open->writes.end()); + for (const auto& batch : inFlight) merged->insert(merged->end(), batch->writes.begin(), batch->writes.end()); + for (const auto* batch : finishing) merged->insert(merged->end(), batch->writes.begin(), batch->writes.end()); + std::sort(merged->begin(), merged->end()); + std::size_t out = 0; + for (const auto& [begin, end] : *merged) { + if (out != 0 && begin <= (*merged)[out - 1].second) (*merged)[out - 1].second = std::max((*merged)[out - 1].second, end); + else (*merged)[out++] = {begin, end}; + } + merged->resize(out); + pendingWrites.store(std::move(merged), std::memory_order_release); + ++snapshotRebuilds; + if (DrawProfiled()) snapshotRebuildMs += std::chrono::duration(std::chrono::steady_clock::now() - started).count(); +} + +bool Recorder::overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end) { + for (const auto& [begin, finish] : batch.writes) { + if (address < finish && begin < end) return true; + } + return false; +} + +bool Recorder::PendingWriteOverlaps(std::uint64_t address, std::size_t bytes) const { + if (bytes == 0) return false; + const auto end = address + bytes; + if (open != nullptr && overlaps(*open, address, end)) return true; + for (const auto& batch : inFlight) { + if (overlaps(*batch, address, end)) return true; + } + return false; +} + +bool Recorder::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const { + return bytes != 0 && open != nullptr && overlaps(*open, address, address + bytes); +} + +std::optional Recorder::DescribePendingWrite(std::uint64_t address, std::size_t bytes) const { + if (bytes == 0) return std::nullopt; + const auto end = address + bytes; + // The same choice SyncThrough makes: the open batch forces a full Sync, otherwise the newest + // overlapping in-flight batch is the target. Of that batch's ranges the first overlapping one + // is reported (a dispatch notes each written V# once, so it is the range the access hit). + const auto firstOverlap = [&](const Batch& batch) -> const std::pair* { + for (const auto& range : batch.writes) { + if (address < range.second && range.first < end) return ⦥ + } + return nullptr; + }; + // With SyncThrough disabled every hit is a full Sync: the open batch (if any) is submitted and + // everything in flight finishes, whichever batch noted the range. + const bool syncAll = !SyncThroughEnabled(); + const auto allBatches = inFlight.size() + (open != nullptr ? 1 : 0); + if (open != nullptr) { + if (const auto* range = firstOverlap(*open)) return PendingWriteInfo{submissions + 1, true, false, range->first, range->second, inFlight.size() + 1}; + } + std::size_t finished = inFlight.size(); + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it, --finished) { + const auto* range = firstOverlap(**it); + if (range == nullptr) continue; + // In flight means submitted, so the fence is live; signaled = the GPU already ran it. + const bool signaled = context.Function("vkGetFenceStatus")(context.device, (*it)->fence) == VK_SUCCESS; + if (syncAll) return PendingWriteInfo{submissions + (open != nullptr ? 1 : 0), open != nullptr, signaled, range->first, range->second, allBatches}; + return PendingWriteInfo{(*it)->serial, false, signaled, range->first, range->second, finished}; + } + return std::nullopt; +} + +bool Recorder::HasCompletions() const { + if (open != nullptr && !open->completions.empty()) return true; + for (const auto& batch : inFlight) { + if (!batch->completions.empty()) return true; + } + return false; +} + +void Recorder::noteLabelOn(Batch& batch, std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue) { + // One entry per dword; a label overlapping older entries (a 4-byte store inside an 8-byte one or + // the reverse) replaces exactly the dwords it stores, so a lookup composes what memory will hold. + std::lock_guard tableLock(labelTableMutex); + for (std::size_t offset = 0; offset + 4 <= bytes.size(); offset += 4) { + std::uint32_t value = 0; + std::memcpy(&value, bytes.data() + offset, 4); + const auto dword = address + offset; + labels.insert_or_assign(dword, LabelEntry{value, queue, stamp, &batch}); + batch.labelDwords.push_back(dword); + } +} + +void Recorder::NoteLabel(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue) { + Commands(); + noteLabelOn(*open, address, bytes, stamp, queue); + if (activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) { + pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release); + } +} + +std::optional Recorder::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { + // Under the GPU mutex; the table mutex is taken too so this one path serves both lock regimes. + std::lock_guard tableLock(labelTableMutex); + return lookupLabel(address, bytes, afterStamp, queue); +} + +bool Recorder::PendingLabelIn(std::uint64_t address, std::size_t bytes) const { + std::lock_guard tableLock(labelTableMutex); + if (labels.empty() || bytes == 0) return false; + const auto end = address + bytes; + for (const auto& [dword, entry] : labels) { + if (dword >= address && dword < end) return true; + } + return false; +} + +std::optional Recorder::lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { + if (labels.empty() || (bytes != 4 && bytes != 8) || address % 4 != 0) return std::nullopt; + std::uint64_t value = 0; + for (std::size_t offset = 0; offset < bytes; offset += 4) { + const auto found = labels.find(address + offset); + if (found == labels.end() || found->second.stamp <= afterStamp) return std::nullopt; + value |= static_cast(found->second.value) << (offset * 8u); + if (offset == 0) queue = found->second.queue; + } + return value; +} + +void Recorder::AfterCompletions(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu) { + Require(open != nullptr || !inFlight.empty(), "no batch to append a completion label to"); + Batch& batch = open != nullptr ? *open : *inFlight.back(); + std::vector copy(bytes.begin(), bytes.end()); + // The store bypasses the flush hook: it runs inside finish() under the mutex, and the hook would + // find this very range noted (or a later label to it) and sync re-entrantly. Stamped like every + // driver store so a collect memoized for the packet sees it. + static const bool always = std::getenv("APS5_LABEL_STORE_ALWAYS") != nullptr; + const auto sequence = [&] { + std::lock_guard ringLock(writtenBackMutex); + return writtenBackSequence; + }(); + batch.completions.push_back([this, address, copy = std::move(copy), sequence, storedOnGpu] { + if (storedOnGpu && !always && !writtenBackSince(sequence, address, address + copy.size())) { + completionStoresSkipped.fetch_add(1, std::memory_order_relaxed); + return; + } + completionStoresRun.fetch_add(1, std::memory_order_relaxed); + GuestMemory::CheckRange(reinterpret_cast(address), copy.size(), 4, true); + std::memcpy(reinterpret_cast(address), copy.data(), copy.size()); + GuestMemory::MarkWritten(address, copy.size()); + }); + // Counted per batch and counted down when the batch finishes, on every exit path of finish() + // (a label whose range became unmapped is reported there, and a lost device throws before the + // completions run): a label that will never land must not leave the workers reaping forever. + ++batch.completionLabelCount; + completionLabels.fetch_add(1, std::memory_order_acq_rel); + // The table entry before the write note: the note bumps the generation a poller watches, and a + // poller that sees the bump then finds the label without the GPU mutex. + noteLabelOn(batch, address, bytes, stamp, queue); + noteWriteOn(batch, address, bytes.size()); + if (&batch == open.get() && activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) { + pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release); + } +} + +void Recorder::NoteWrittenBack(std::uint64_t address, std::size_t bytes) { + Recorder* recorder = activeRecorder; + if (recorder == nullptr || bytes == 0) return; + std::lock_guard ringLock(recorder->writtenBackMutex); + recorder->writtenBack.push_back({++recorder->writtenBackSequence, address, address + bytes}); + while (recorder->writtenBack.size() > 16384) recorder->writtenBack.pop_front(); +} + +bool Recorder::writtenBackSince(std::uint64_t sequence, std::uint64_t begin, std::uint64_t end) { + std::lock_guard ringLock(writtenBackMutex); + if (writtenBack.empty() || writtenBack.back()[0] <= sequence) return false; + if (writtenBack.front()[0] > sequence + 1) return true; + for (auto it = writtenBack.rbegin(); it != writtenBack.rend() && (*it)[0] > sequence; ++it) { + if ((*it)[1] < end && begin < (*it)[2]) return true; + } + return false; +} + +void Recorder::Submit() { + // The work count is cleared even when nothing is open: the driver counts a dispatch after its + // call returns (outside the mutex), so a submit by another thread in between leaves a stale + // count behind, and the callers that act on it would otherwise take the mutex for nothing at + // every packet or submission end until a real batch is next submitted. + if (activeRecorder == this) workSinceSubmit.store(0, std::memory_order_relaxed); + if (open == nullptr) return; + GuestMemory::AssertGpuLockHeld("Recorder::Submit"); + auto batch = std::move(open); + Check(context.Function("vkEndCommandBuffer")(batch->commands), "vkEndCommandBuffer recorder"); + VkSubmitInfo submission{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + submission.commandBufferCount = 1; + submission.pCommandBuffers = &batch->commands; + // The timeline reaches this batch's serial when it completes (see WaitSerial). + const std::uint64_t serial = submissions + 1; + VkTimelineSemaphoreSubmitInfoKHR timelineInfo{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR}; + timelineInfo.signalSemaphoreValueCount = 1; + timelineInfo.pSignalSemaphoreValues = &serial; + if (timeline != VK_NULL_HANDLE) { + submission.pNext = &timelineInfo; + submission.signalSemaphoreCount = 1; + submission.pSignalSemaphores = &timeline; + } + Check(context.Function("vkQueueSubmit")(context.queue, 1, &submission, batch->fence), "vkQueueSubmit recorder"); + batch->submitted = true; + batch->serial = ++submissions; + inFlight.push_back(std::move(batch)); + if (activeRecorder == this) pendingLabelSince.store(NoPendingLabel, std::memory_order_release); +} + +std::uint64_t Recorder::SubmitAndEpoch() { + Submit(); + return inFlight.empty() ? 0 : submissions; +} + +namespace { + +// One thread's registration as a timeline waiter (see `unlockedWaiters`): constructed while the +// caller still holds the mutex (or the device), so a teardown that starts after the mutex is +// released sees it and keeps the semaphore until the wait returned. +struct UnlockedWaiter { + std::atomic& count; + explicit UnlockedWaiter(std::uint64_t recorderId) : count(WaitersOf(recorderId)) { count.fetch_add(1, std::memory_order_acq_rel); } + ~UnlockedWaiter() { count.fetch_sub(1, std::memory_order_acq_rel); } + UnlockedWaiter(const UnlockedWaiter&) = delete; + UnlockedWaiter& operator=(const UnlockedWaiter&) = delete; +}; + +// Restores the thread's active sync-site entry on every exit path (a Check throw on a lost device +// included), so the next sync on the thread is not attributed to this one's site. +struct RestoreSyncSite { + std::size_t site; + ~RestoreSyncSite() { activeSyncSite = site; } +}; + +// The timeline wait itself, on handles copied out of the recorder: the caller holds no mutex, and +// the recorder may be torn down meanwhile (its destructor waits for the registered waiters to +// leave after completing every batch, so the handles stay valid until this returns). +VkResult WaitTimeline(VkDevice device, VkSemaphore timeline, PFN_vkWaitSemaphoresKHR waitSemaphores, std::uint64_t serial) { + VkSemaphoreWaitInfoKHR wait{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO_KHR}; + wait.semaphoreCount = 1; + wait.pSemaphores = &timeline; + wait.pValues = &serial; + auto result = waitSemaphores(device, &wait, 5'000'000'000ull); + for (int waited = 5; result == VK_TIMEOUT; waited += 5) { + // As for the fence wait in finish(): a hung batch is reported every 5 s. + std::fprintf(stderr, "[gpu] recorded batch %llu still running on the GPU after %d s (timeline wait)\n", static_cast(serial), waited); + result = waitSemaphores(device, &wait, 5'000'000'000ull); + } + return result; +} + +} + +void Recorder::WaitSerial(std::uint64_t serial) { + if (timeline == VK_NULL_HANDLE || serial == 0) return; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const UnlockedWaiter waiter{id}; + const auto result = WaitTimeline(context.device, timeline, context.Function("vkWaitSemaphoresKHR"), serial); + if (profile) { + unlockedWaits.fetch_add(1, std::memory_order_relaxed); + unlockedWaitedUs.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()), std::memory_order_relaxed); + } + Check(result, "vkWaitSemaphoresKHR recorder"); +} + +void Recorder::FinishUpTo(std::uint64_t serial) { + // A drain: the wait already happened (unlocked), the fences are signaled, so the source-0 wait + // recorded here is the bookkeeping only (a caller that did not wait first, WaitForLeases, shows + // up in the site table by its own address). + ++syncCounts[0]; + announcedSource = 4; + const void* site = std::exchange(announcedSite, nullptr); + const auto previousSite = inFlight.empty() || inFlight.front()->serial > serial ? activeSyncSite : BeginSyncSite(0, site != nullptr ? site : __builtin_return_address(0)); + while (!inFlight.empty() && inFlight.front()->serial <= serial) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, 0); + } + activeSyncSite = previousSite; +} + +void Recorder::Sync() { + const auto source = std::exchange(announcedSource, 4); + const void* site = std::exchange(announcedSite, nullptr); + Submit(); + // Counted only when something is waited for: an idle recorder's Sync is free and would only + // swamp the site table with the callers that check nothing first. + const auto previousSite = inFlight.empty() ? activeSyncSite : BeginSyncSite(source, site != nullptr ? site : __builtin_return_address(0)); + while (!inFlight.empty()) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, source); + } + activeSyncSite = previousSite; +} + +void Recorder::SyncThrough(std::uint64_t address, std::size_t bytes, bool waitUnlocked) { + if (bytes == 0) return; + const auto end = address + bytes; + if (completionDepth != 0 && !CompletionStoreSyncs()) { + // A store made by a completion action (a copied buffer's write-back, GuestBufferMemory:: + // WriteBack) reached the flush hook, and a batch still in flight notes the range. Every + // batch in flight now was recorded after the completing one (finish runs front to back), + // so in program order this store precedes their writes: landing it before them is the + // order the hardware gives; waiting for them first (what the hook would do) lands the + // completing batch's bytes over the later batch's, which is wrong, and does so under the + // hold of whoever reaped (the [hooksync] 'store' entries, up to tens of ms per reap). The + // later batch's own ordering against this write-back is its recorder's business at record + // time (FillBuffer and DispatchIndirect consult copiedWriters; a later copied writer's + // write-back runs after this one in finish order; a label behind completions lands after). + // Known gap, not introduced here: a later batch writing the same range GPU-direct in place + // (a writable region bound in place, or a GPU copy-back) is not ordered against this + // write-back by the resource build; it went from deterministically stale to racy. + announcedSource = 4; + announcedSite = nullptr; + ++holdCounters.completionSyncsSkipped; + return; + } + if (completionDepth != 0) ++holdCounters.completionSyncsWaited; + const auto completionWaitStart = completionDepth != 0 && DrawProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + struct CompletionWait { + std::chrono::steady_clock::time_point start; + ~CompletionWait() { + if (start != std::chrono::steady_clock::time_point{}) holdCounters.completionSyncWaitMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + } + } completionWait{completionWaitStart}; + if (waitUnlocked) { + // The hook's wait without the mutex: its own acquisition is the outermost on this thread. + const auto source = announcedSource; + const void* site = announcedSite != nullptr ? announcedSite : __builtin_return_address(0); + if (syncThroughUnlocked(address, end, source, site)) return; + // A completion-store wait (kill switch above) is already counted as 'waited'. + if (completionDepth == 0) ++holdCounters.hookLockedWaits; + } + if (!SyncThroughEnabled() || (open != nullptr && overlaps(*open, address, end))) { + // The announced site (or this call's caller) names the sync, not this function. + if (announcedSite == nullptr) announcedSite = __builtin_return_address(0); + Sync(); + return; + } + const auto source = std::exchange(announcedSource, 4); + const void* site = std::exchange(announcedSite, nullptr); + std::uint64_t targetSerial = 0; + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) { + if (overlaps(**it, address, end)) { + targetSerial = (*it)->serial; + break; + } + } + if (targetSerial == 0) return; + targetedSyncs.fetch_add(1, std::memory_order_relaxed); + const RestoreSyncSite restoreSite{BeginSyncSite(source, site != nullptr ? site : __builtin_return_address(0))}; + // Batches are in flight in serial order. A completion's own guest access can sync re-entrantly + // and finish the target first (and a new batch may reuse its allocation, so the serial, not the + // pointer, identifies it); the loop stops then, and later batches stay in flight either way. + while (!inFlight.empty() && inFlight.front()->serial <= targetSerial) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, source); + } + batchesLeftInFlight.fetch_add(inFlight.size(), std::memory_order_relaxed); +} + +bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int source, const void* site) { + // Only for the outermost acquisition: a nested hook (inside a dispatch's hold) must not give + // up the caller's mutex, and a thread that waits here holds nothing the relock could invert + // (the hook took the mutex from a depth of 0 with the same locks held). + if (HookLockedWait() || timeline == VK_NULL_HANDLE || GuestMemory::GpuMutex().DepthOnThisThread() != 1) return false; + // The target as SyncThrough chooses it: the open batch (submitted here, so the GPU can reach + // it) makes everything the target, else the newest overlapping in-flight batch; with + // SyncThrough disabled everything is the target as well. + std::uint64_t targetSerial = 0; + if (!SyncThroughEnabled() || (open != nullptr && overlaps(*open, address, end))) { + Submit(); + targetSerial = submissions; + } else { + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) { + if (overlaps(**it, address, end)) { + targetSerial = (*it)->serial; + break; + } + } + } + announcedSource = 4; + announcedSite = nullptr; + if (targetSerial == 0) return true; + // Everything the wait needs is copied out: the recorder may be torn down while the mutex is + // released (its destructor completes every batch and waits for the semaphore to be left, see + // WaitTimeline), after which `this` must not be touched. The id re-check below relies on the + // teardown running under the GpuMutex (the driver's drain and failure paths reset the device + // under it; a worker dropping its last device reference outside its lock would not). + const auto myId = id; + const auto device = context.device; + const auto semaphore = timeline; + const auto waitSemaphores = context.Function("vkWaitSemaphoresKHR"); + // The site entry is taken under the mutex (the table is only touched under it; entries are + // appended, never removed, so the index stays valid across the release). + const RestoreSyncSite restoreSite{BeginSyncSite(source, site)}; + const bool profile = DrawProfiled(); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + auto waited = start; + auto& mutex = GuestMemory::GpuMutex(); + VkResult result = VK_SUCCESS; + { + // Registered before the release: a teardown can only start once the mutex is free. + const UnlockedWaiter waiter{myId}; + mutex.unlock(); + result = WaitTimeline(device, semaphore, waitSemaphores, targetSerial); + // Read before the relock: the GPU wait, not the wait for the mutex behind it. + if (profile) waited = std::chrono::steady_clock::now(); + } + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); + mutex.lock(); + if (profile) { + const auto ms = std::chrono::duration(waited - start).count(); + // The GPU wait is charged like a locked wait, so the per-source, per-thread and per-site + // totals of the [recorder] line keep their meaning; the unlocked count says how many were + // not held. The relock's wait for the mutex is kept apart (the [lock] line also counts it + // under 'hook' waits, for a cross-check). + syncWaitedMs[source] += ms; + CountThreadSync(source, ms); + CountSiteWait(ms); + ++holdCounters.hookUnlockedWaits; + holdCounters.hookUnlockedWaitMs += ms; + holdCounters.hookRelockMs += std::chrono::duration(std::chrono::steady_clock::now() - waited).count(); + } + if (!RecorderAlive(myId)) { + // Torn down meanwhile: its destructor's Sync() ran every completion, nothing is pending. + ++holdCounters.hookUnlockedTornDown; + Check(result, "vkWaitSemaphoresKHR recorder"); + return true; + } + Check(result, "vkWaitSemaphoresKHR recorder"); + // The completions up to the target, under the mutex again; the fences are signaled, so the + // waits inside finish() return at once (charged to the same site entry). Another thread may + // have finished some meanwhile. + targetedSyncs.fetch_add(1, std::memory_order_relaxed); + while (!inFlight.empty() && inFlight.front()->serial <= targetSerial) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, source); + } + batchesLeftInFlight.fetch_add(inFlight.size(), std::memory_order_relaxed); + return true; +} + +bool Recorder::Reap() { + const auto status = context.Function("vkGetFenceStatus"); + const bool profile = DrawProfiled(); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + std::uint64_t retired = 0; + while (!inFlight.empty()) { + if (status(context.device, inFlight.front()->fence) != VK_SUCCESS) break; + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), false, 4); + ++retired; + } + if (profile) { + ++holdCounters.reaps; + if (retired != 0) ++holdCounters.reapsWithWork; + holdCounters.reapBatches += retired; + holdCounters.reapMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + } + return inFlight.empty(); +} + +void Recorder::finish(std::unique_ptr batch, bool wait, int source) { + // The batch's writes stay published until its completions ran, on every exit path. + struct Finishing { + Recorder& recorder; + const Batch* batch; + ~Finishing() { + auto& list = recorder.finishing; + list.erase(std::remove(list.begin(), list.end(), batch), list.end()); + // The batch's completion labels have landed, or never will (fence failure below). + if (batch->completionLabelCount != 0) completionLabels.fetch_sub(batch->completionLabelCount, std::memory_order_acq_rel); + // This also runs while the fence-failure throw unwinds: an allocation failure in the + // rebuild keeps the previous snapshot, which is a superset (a stale entry only causes a + // false hit, never a miss) rather than terminating. + try { + recorder.publishPendingWrites(); + } catch (...) { + } + } + } finishingScope{*this, batch.get()}; + finishing.push_back(batch.get()); + if (wait) { + // APS5_PROFILE_DRAW: how long the CPU waits for recorded GPU work, reported every 10 s. + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static double waitedMs = 0; + static std::uint64_t waits = 0; + static auto lastReport = std::chrono::steady_clock::now(); + const auto waitStart = std::chrono::steady_clock::now(); + struct Report { + bool enabled; + int source; + std::chrono::steady_clock::time_point start; + const Recorder& recorder; + ~Report() { + if (!enabled) return; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - start).count(); + waitedMs += ms; + syncWaitedMs[source] += ms; + ++waits; + CountThreadSync(source, ms); + CountSiteWait(ms); + if (now - lastReport > std::chrono::seconds(10)) { + lastReport = now; + const auto& h = holdCounters; + std::fprintf(stderr, "[recorder] %llu syncs waited %.1f s for the GPU in total (sources, count/wait: idle %llu/%.1fs, pending write %llu/%.1fs, recorded store %llu/%.1fs, address-based %llu/%.1fs, other %llu/%.1fs); hook %llu calls, %llu locked; %llu targeted syncs left %llu batches in flight; snapshot %llu rebuilds %.0f ms, %llu notes covered; %llu unlocked timeline waits %.1f s; %llu submissions, %zu label entries, %llu completion labels pending; fence waits by thread (count/wait):%s; top sync sites (source@caller syncs/batches/wait):%s; under holds (cumulative): %llu completions ran %.0f ms (kept objects released %.0f ms), pending-write syncs from completions: %llu skipped, %llu waited %.0f ms; %llu reaps (%llu with work) retired %llu batches in %.0f ms; hook waits unlocked %llu / %.0f ms GPU + %.0f ms relock (%llu found the recorder torn down), locked %llu; deferred releases: on the release thread %llu batches (%llu objects) in %.0f ms, inline %llu batches (%llu objects) in %.0f ms (%llu batches over the queue bound of %zu), queue max %llu batches, %llu pending\n",static_cast(waits), waitedMs / 1000, static_cast(syncCounts[0]), syncWaitedMs[0] / 1000, static_cast(syncCounts[1]), syncWaitedMs[1] / 1000, static_cast(syncCounts[2]), syncWaitedMs[2] / 1000, static_cast(syncCounts[3]), syncWaitedMs[3] / 1000, static_cast(syncCounts[4]), syncWaitedMs[4] / 1000, static_cast(hookCalls.load()), static_cast(hookLocks.load()), static_cast(targetedSyncs.load()), static_cast(batchesLeftInFlight.load()), static_cast(snapshotRebuilds), snapshotRebuildMs, static_cast(snapshotCovered), static_cast(unlockedWaits.load()), unlockedWaitedUs.load() / 1e6, static_cast(recorder.submissions), recorder.labels.size(), static_cast(completionLabels.load()), ThreadSyncReport().c_str(), SyncSiteReport().c_str(), static_cast(h.completions), h.completionMs, h.keptReleaseMs, static_cast(h.completionSyncsSkipped), static_cast(h.completionSyncsWaited), h.completionSyncWaitMs, static_cast(h.reaps), static_cast(h.reapsWithWork), static_cast(h.reapBatches), h.reapMs, static_cast(h.hookUnlockedWaits), h.hookUnlockedWaitMs, h.hookRelockMs, static_cast(h.hookUnlockedTornDown), static_cast(h.hookLockedWaits), static_cast(threadReleases.load()), static_cast(threadObjects.load()), threadReleaseUs.load() / 1000.0, static_cast(inlineReleases.load()), static_cast(inlineObjects.load()), inlineReleaseUs.load() / 1000.0, static_cast(inlineOverBound.load()), ReleaseQueueBound(), static_cast(releaseQueueMax.load()), static_cast(deferredPending.load())); + std::fprintf(stderr, "[recorder] completion label stores: %llu run, %llu skipped (no CPU write-back overlapped them)\n", static_cast(completionStoresRun.load()), static_cast(completionStoresSkipped.load())); + } + } + } report{profile, source, waitStart, *this}; + const auto waitFences = context.Function("vkWaitForFences"); + auto result = waitFences(context.device, 1, &batch->fence, VK_TRUE, 5'000'000'000ull); + for (int waited = 5; result == VK_TIMEOUT; waited += 5) { + // A batch still running after 5 s is reported (every 5 s) so a GPU-side hang is visible. + std::fprintf(stderr, "[gpu] recorded batch still running on the GPU after %d s\n", waited); + result = waitFences(context.device, 1, &batch->fence, VK_TRUE, 5'000'000'000ull); + } + if (result != VK_SUCCESS && result != VK_ERROR_DEVICE_LOST) { + const auto idle = context.Function("vkDeviceWaitIdle")(context.device); + Check(idle, "vkDeviceWaitIdle after recorder fence failure"); + } + if (result != VK_SUCCESS) { + release(*batch); + Check(result, "vkWaitForFences recorder"); + } + } + readGpuTiming(*batch); + // Completions store GPU results to guest memory; a failing one is reported, the rest still run. + // Timed (APS5_PROFILE_DRAW) with the release of the kept objects: both run under whichever hold + // reaped or synced the batch. Inside them the flush hook makes no pending-write wait (SyncThrough). + const bool profile = DrawProfiled(); + const auto completionStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + { + struct InCompletion { + InCompletion() { ++completionDepth; } + ~InCompletion() { --completionDepth; } + } inCompletion; + for (auto& action : batch->completions) { + try { + action(); + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] deferred write-back failed: %s\n", error.what()); + } + } + } + if (profile) holdCounters.completions += batch->completions.size(); + const auto keptStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + // Deferred only from inside a hold: the unlock that releases the list is this thread's own, and + // a caller finishing batches without the mutex (a test) might never make one. + if (ReleaseUnderLock() || !GuestMemory::GpuMutex().HeldByThisThread()) { + batch->completions.clear(); + batch->kept.clear(); + } else { + // To this thread's unlock (see ReleaseDeferredKeeps); the completions go with the objects, + // their captures hold the same ones. The push comes first: a failed one (the list's growth) + // destroys the batch's objects here, under the mutex, counts nothing the destructor would + // wait for and must not escape (the rest of finish() releases the fence and command buffer). + try { + deferredBatches.push_back({std::move(batch->kept), std::move(batch->completions)}); + deferredPending.fetch_add(1, std::memory_order_acq_rel); + } catch (...) { + } + batch->completions.clear(); + batch->kept.clear(); + } + if (profile) { + const auto now = std::chrono::steady_clock::now(); + holdCounters.keptReleaseMs += std::chrono::duration(now - keptStart).count(); + holdCounters.completionMs += std::chrono::duration(now - completionStart).count(); + } + // The batch's label entries leave the table (a later label to the same dword already replaced + // its entry and belongs to another batch). Correctness never depended on this removal. + if (!batch->labelDwords.empty()) { + std::lock_guard tableLock(labelTableMutex); + for (const auto dword : batch->labelDwords) { + const auto found = labels.find(dword); + if (found != labels.end() && found->second.batch == batch.get()) labels.erase(found); + } + } + batch->labelDwords.clear(); + release(*batch); +} + +void Recorder::release(Batch& batch) noexcept { + if (batch.queries != VK_NULL_HANDLE) context.Function("vkDestroyQueryPool")(context.device, batch.queries, nullptr); + batch.queries = VK_NULL_HANDLE; + // A completed (or never submitted) batch's objects are kept for reuse: the fence is signaled or + // untouched, so resetting it cannot block, and the command buffer is no longer pending. + if (batch.commands != VK_NULL_HANDLE && batch.fence != VK_NULL_HANDLE && spare.size() < 64 && context.Function("vkResetFences")(context.device, 1, &batch.fence) == VK_SUCCESS) { + spare.emplace_back(batch.commands, batch.fence); + batch.commands = VK_NULL_HANDLE; + batch.fence = VK_NULL_HANDLE; + return; + } + if (batch.commands != VK_NULL_HANDLE) context.Function("vkFreeCommandBuffers")(context.device, context.pool, 1, &batch.commands); + if (batch.fence != VK_NULL_HANDLE) context.Function("vkDestroyFence")(context.device, batch.fence, nullptr); + batch.commands = VK_NULL_HANDLE; + batch.fence = VK_NULL_HANDLE; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/RegisteredGpuMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/RegisteredGpuMemory.cpp deleted file mode 100644 index 058dc9229b..0000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/RegisteredGpuMemory.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" -#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" - -namespace AgcDriver::Graphics { - -void GuestBufferMemory::AcquireRegistered() { - Require(!uploaded && regions.empty() && lease.empty(), "guest allocation lease must precede resource registration"); - lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); - regions.reserve(lease.size()); - for (const auto& range : lease) { - validate(range->address, range->bytes); - std::vector snapshot; - if (!range->writable) { - snapshot.resize(range->bytes); - GuestMemory::Read(range->address, snapshot); - } - regions.push_back({range->address, range->address + range->bytes, range->writable, std::move(snapshot), nullptr}); - } -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index b219865d9c..a8ad8e72dc 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -1,11 +1,13 @@ #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include namespace AgcDriver::Graphics { -Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), usage(usage), properties(properties) { +Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(usage), properties(properties) { Require(size != 0, "zero-sized GPU buffer"); const bool addressable = (usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0; Require(!addressable || context.bufferDeviceAddress, "buffer device address is not enabled"); @@ -16,12 +18,11 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag mapping = allocation->mapping; deviceAddress = allocation->address; allocationBytes = allocation->allocationBytes; - reusable = true; return; } try { VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; - info.size = size; + info.size = capacity; info.usage = usage; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Check(context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer"); @@ -32,12 +33,22 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag if (addressable) allocation.pNext = &flags; allocation.allocationSize = requirements.size; allocationBytes = requirements.size; - allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, properties); + // The CPU reads most of these buffers back (write-back, diffs), which is very slow from + // write-combined memory, so the default host properties prefer cached host memory. + constexpr VkMemoryPropertyFlags hostDefault = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + if (properties == hostDefault) { + try { + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, hostDefault | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + } catch (const std::runtime_error&) { + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, hostDefault); + } + } else { + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, properties); + } Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory buffer"); Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory"); initializeAddress(usage); - if ((properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) Check(context.Function("vkMapMemory")(context.device, memory, 0, VK_WHOLE_SIZE, 0, &mapping), "vkMapMemory"); - reusable = true; + Check(context.Function("vkMapMemory")(context.device, memory, 0, VK_WHOLE_SIZE, 0, &mapping), "vkMapMemory"); } catch (...) { release(); throw; @@ -49,8 +60,8 @@ Buffer::~Buffer() { } void Buffer::release() noexcept { - if (reusable && buffer && memory && cache) { - cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, size, usage, properties}); + if (mapping && buffer && memory && cache) { + cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, capacity, usage, properties}); return; } if (mapping) context.Function("vkUnmapMemory")(context.device, memory); @@ -63,18 +74,76 @@ VkBuffer Buffer::Handle() const { } std::span Buffer::Bytes() { - Require(mapping != nullptr, "GPU-only buffer has no CPU mapping"); return {static_cast(mapping), size}; } void Buffer::Invalidate() { - Require(mapping != nullptr, "cannot invalidate an unmapped GPU buffer"); VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; range.memory = memory; range.size = VK_WHOLE_SIZE; Check(context.Function("vkInvalidateMappedMemoryRanges")(context.device, 1, &range), "vkInvalidateMappedMemoryRanges"); } +DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(usage) { + Require(size != 0, "zero-sized device buffer"); + cache = GetBufferPool(context); + if (const auto allocation = cache->Take(size, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { + buffer = allocation->buffer; + memory = allocation->memory; + allocationBytes = allocation->allocationBytes; + return; + } + try { + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + info.size = capacity; + info.usage = usage; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + Check(context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer device"); + VkMemoryRequirements requirements{}; + context.Function("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + allocationBytes = requirements.size; + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory device buffer"); + Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory device"); + } catch (...) { + release(); + throw; + } +} + +DeviceBuffer::~DeviceBuffer() { + release(); +} + +void DeviceBuffer::release() noexcept { + if (buffer && memory && cache) { + cache->Put({buffer, memory, nullptr, 0, allocationBytes, capacity, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}); + return; + } + if (buffer) context.Function("vkDestroyBuffer")(context.device, buffer, nullptr); + if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); +} + +VkBuffer DeviceBuffer::Handle() const { + return buffer; +} + +std::size_t DeviceBuffer::Size() const { + return size; +} + +void CopyBuffer(const Context& context, VkCommandBuffer commands, VkBuffer source, VkDeviceSize sourceOffset, VkBuffer destination, VkDeviceSize destinationOffset, VkDeviceSize bytes) { + const VkBufferCopy region{sourceOffset, destinationOffset, bytes}; + context.Function("vkCmdCopyBuffer")(commands, source, destination, 1, ®ion); +} + +void RecordMemoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { + const VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, sourceAccess, destinationAccess}; + context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + RenderTarget::RenderTarget(const Context& context, const ColorTarget& target, bool blending) : context(context) { VkFormatProperties properties{}; context.formatProperties(context.physical, target.format, &properties); @@ -136,6 +205,8 @@ VkImageView RenderTarget::View() const { } CommandBatch::CommandBatch(const Context& context) : context(context) { + // Records into the device's one command pool and submits to its queue: device-lock work only. + GuestMemory::AssertGpuLockHeld("CommandBatch"); try { VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; allocation.commandPool = context.pool; @@ -147,6 +218,8 @@ CommandBatch::CommandBatch(const Context& context) : context(context) { VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; Check(context.Function("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); + // Work recorded so far goes first in queue order, so this batch sees its results. + if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->Recording()) recorder->Submit(); } catch (...) { release(); throw; @@ -176,16 +249,6 @@ void CommandBatch::SubmitAndWait() { Wait(); } -void CommandBatch::Reset() { - Require(submitted && !pending, "command batch must complete before reuse"); - Check(context.Function("vkResetFences")(context.device, 1, &fence), "vkResetFences graphics"); - Check(context.Function("vkResetCommandBuffer")(commands, 0), "vkResetCommandBuffer graphics"); - VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; - begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - Check(context.Function("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer graphics"); - submitted = false; -} - void CommandBatch::Submit() { PerformanceTimer timing("Graphics.Submit"); Require(!submitted, "command batch has already been submitted"); @@ -208,6 +271,8 @@ void CommandBatch::Wait() { timing.Mark("fence_wait"); if (result == VK_SUCCESS || result == VK_ERROR_DEVICE_LOST) pending = false; Check(result, "vkWaitForFences graphics"); + // Recorded batches preceded this one, so their completions (write-backs) can run now. + if (auto* recorder = Recorder::Active()) recorder->Reap(); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp index 5a0e4cf8f9..660b16bb12 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp @@ -1,8 +1,10 @@ #include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" +#include +#include namespace AgcDriver::Graphics { - Sampler::Sampler(const Context& context, const GuestSamplerResource& descriptor) : device(context.device), destroySampler(context.Function("vkDestroySampler")) { + Sampler::Sampler(const Context& context, const GuestSamplerResource& descriptor) : context(context) { Require(!descriptor.anisotropyEnable || context.samplerAnisotropy, "guest sampler descriptor requests anisotropic filtering which the device does not support"); Require(descriptor.maxAnisotropy <= context.limits.maxSamplerAnisotropy, "guest sampler descriptor requests an anisotropy ratio beyond the device limit"); Require(descriptor.lodBias >= -context.limits.maxSamplerLodBias && descriptor.lodBias <= context.limits.maxSamplerLodBias, "guest sampler descriptor requests a LOD bias beyond the device limit"); @@ -31,22 +33,36 @@ namespace AgcDriver::Graphics { } void Sampler::release() noexcept { - if (sampler) destroySampler(device, sampler, nullptr); + if (sampler) context.Function("vkDestroySampler")(context.device, sampler, nullptr); } VkSampler Sampler::Handle() const { return sampler; } - std::shared_ptr SamplerCache::Get(const Context& context, std::span words, const GuestSamplerResource& descriptor) { - Require(words.size() == 4, "sampler cache descriptor must contain four DWORDs"); - const std::array key{words[0], words[1], words[2], words[3], descriptor.compareEnable ? 1u : 0u}; + SamplerCache::SamplerCache(std::size_t capacity) : capacity(std::max(capacity, 1)) {} + + std::shared_ptr SamplerCache::Get(const Context& context, std::span words, bool compareEnable) { + Require(words.size() == 4, "guest sampler descriptor must contain 4 dwords"); + const std::array key{words[0], words[1], words[2], words[3], compareEnable ? 1u : 0u}; std::lock_guard lock(mutex); - const auto found = entries.find(key); - if (found != entries.end()) return found->second; - auto sampler = std::make_shared(context, descriptor); - if (entries.size() >= 256) entries.erase(entries.begin()); - entries.emplace(key, sampler); + ++clock; + if (const auto found = entries.find(key); found != entries.end()) { + ++hits; + found->second.lastUse = clock; + return found->second.sampler; + } + ++misses; + auto resource = DecodeSamplerResource(words); + resource.compareEnable = compareEnable; + auto sampler = std::make_shared(context, resource); + // The cap keeps live samplers well below the device's limit (NVIDIA: ~4000); a set in flight + // still holds the evicted sampler through its own shared_ptr. + while (entries.size() >= capacity) { + const auto oldest = std::min_element(entries.begin(), entries.end(), [](const auto& left, const auto& right) { return left.second.lastUse < right.second.lastUse; }); + entries.erase(oldest); + } + entries.emplace(key, Entry{sampler, clock}); return sampler; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp index 04951b202b..b5663629f1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp @@ -2,7 +2,10 @@ #include "SceShaders.hpp" #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include #include +#include +#include #include #include #include @@ -25,7 +28,9 @@ constexpr std::uint32_t spiShaderColFormat = 0x1C5; std::uint32_t read(const Registers& registers, std::uint32_t offset) { const auto it = registers.find(offset); if (it == registers.end()) { - throw std::runtime_error("AGC graphics: missing register at DWORD 0x" + std::to_string(offset)); + char text[64]; + std::snprintf(text, sizeof(text), "AGC graphics: missing register at DWORD 0x%x", offset); + throw std::runtime_error(text); } return it->second; } @@ -65,9 +70,12 @@ ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& if ((rsrc2 & 0x1u) != 0) { throw std::runtime_error("AGC graphics: COMPUTE_PGM_RSRC2.SCRATCH_EN is unsupported"); } + // Debug aid: APS5_LDS_SLACK= grows every dispatch's LDS allocation by that much, to tell + // whether a program depends on addresses past its declared allocation. + static const std::uint32_t ldsSlack = [] { const char* text = std::getenv("APS5_LDS_SLACK"); return text ? static_cast(std::strtoul(text, nullptr, 0)) : 0u; }(); return ShaderRecompiler::ShaderComputeStageInfo{ {numThreadX, numThreadY, numThreadZ}, - ((rsrc2 >> 15u) & 0x1FFu) * 128u, + std::min(((rsrc2 >> 15u) & 0x1FFu) * 128u + ldsSlack, 16384u), {((rsrc2 >> 7u) & 0x1u) != 0, ((rsrc2 >> 8u) & 0x1u) != 0, ((rsrc2 >> 9u) & 0x1u) != 0}, ((rsrc2 >> 10u) & 0x1u) != 0, ((rsrc2 >> 11u) & 0x3u) + 1u @@ -92,12 +100,8 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con interpolatorSettings[i] = read(context, spiPsInputCntl0 + i); } const auto shaderControl = read(context, dbShaderControl); - if (((shaderControl >> 9u) & 0x1u) != 0) { - throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.DUAL_EXPORT_ENABLE is unsupported"); - } - if (((shaderControl >> 11u) & 0x1u) != 0) { - throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.ALPHA_TO_MASK_DISABLE is unsupported"); - } + // Bits 9 and 11 are EXEC_ON_HIER_FAIL and ALPHA_TO_MASK_DISABLE on GFX10; neither changes what the + // recompiled pixel shader computes. if (((shaderControl >> 13u) & 0x3u) != 0) { throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.CONSERVATIVE_Z_EXPORT is unsupported"); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 45db4dda8c..637d3ce4f7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -1,13 +1,28 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" -#include "prx/libSceAgcDriver/Graphics/include/TextureCache.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include +#include +#include +#include +#include +#include +#include +#include #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libc/include/General.hpp" #include "Optimization/include/Optimization/ShaderStageInputInfo.hpp" #include #include +#include +#include +#include #include #include +#include +#include "prx/libc/include/GuestAllocations.hpp" namespace AgcDriver::Graphics { namespace { @@ -28,6 +43,446 @@ VkComponentSwizzle ComponentSwizzleFor(std::uint8_t dstSel) { } } +// Sampled textures are reused across draws and dispatches while their guest bytes are unchanged; a +// byte copy of the guest surface validates each reuse, so CPU or GPU writes to it force a re-upload. +// The key is everything a lookup matches: the device, the eight descriptor words and the swizzle. +struct TextureKey { + VkDevice device; + std::array words; + std::array components; + bool operator==(const TextureKey&) const = default; +}; + +std::uint64_t hashWords(std::uint64_t hash, const std::uint32_t* words, std::size_t count) { + for (std::size_t i = 0; i < count; ++i) hash = (hash ^ words[i]) * 1099511628211ull; + return hash; +} + +struct TextureKeyHash { + std::size_t operator()(const TextureKey& key) const noexcept { + auto hash = 14695981039346656037ull ^ reinterpret_cast(key.device); + hash = hashWords(hash, key.words.data(), key.words.size()); + return static_cast(hashWords(hash, key.components.data(), key.components.size())); + } +}; + +TextureKey MakeTextureKey(VkDevice device, std::span words, VkComponentMapping components) { + TextureKey key{device, {}, {static_cast(components.r), static_cast(components.g), static_cast(components.b), static_cast(components.a)}}; + std::copy(words.begin(), words.end(), key.words.begin()); + return key; +} + +struct CachedTexture { + TextureKey key; + std::uint64_t address; + std::vector bytes; + std::shared_ptr texture; + // A fast-cleared surface is cached as its clear texels; it stays valid while the keys are unchanged. + DccKeys keys = DccKeys::Uncompressed; + // Write generation the snapshot is known current at (see GuestMemory::CollectWrites). + std::uint64_t generation = 0; + // A texture viewing a storage image on the GPU has no snapshot; it stays valid while that image + // is still its surface's cached image and nothing wrote the guest memory since it matched. + std::shared_ptr source; + std::uint64_t sourceVersion = 0; + std::uint64_t accounted = 0; +}; + +// Entries in use order (front = most recent) with a hash index by key: a lookup is O(1) and the +// eviction takes the back. APS5_NO_TEXTURE_HASH=1 finds entries by scanning the list (the index is +// still kept), the linear lookup of before. +struct TextureCache { + std::mutex mutex; + std::list entries; + std::unordered_map::iterator, TextureKeyHash> index; + std::uint64_t bytes = 0; +}; + +TextureCache& Textures() { + static TextureCache cache; + return cache; +} + +bool TextureHashEnabled() { + static const bool disabled = std::getenv("APS5_NO_TEXTURE_HASH") != nullptr; + return !disabled; +} + +std::list::iterator findTexture(TextureCache& cache, const TextureKey& key) { + if (TextureHashEnabled()) { + const auto found = cache.index.find(key); + return found == cache.index.end() ? cache.entries.end() : found->second; + } + for (auto it = cache.entries.begin(); it != cache.entries.end(); ++it) { + if (it->key == key) return it; + } + return cache.entries.end(); +} + +void eraseTexture(TextureCache& cache, std::list::iterator it) { + cache.bytes -= it->accounted; + cache.index.erase(it->key); + cache.entries.erase(it); +} + +// Moves an entry to the front (most recently used). +void touchTexture(TextureCache& cache, std::list::iterator it) { + cache.entries.splice(cache.entries.begin(), cache.entries, it); +} + +// APS5_PROFILE_DRAW: what the sampled-texture and storage-image lookups did, printed as [textures] +// every 10 s (cumulative). Storage-sourced sampled textures need no CPU read; snapshots do, and a +// pending read is a snapshot compare or read made while recorded work still wrote the surface (the +// flush hook may find that work already signaled, so this bounds the hook syncs from above; the +// [hooksync] line attributes the actual waits). +struct TextureCounters { + std::atomic fromStorage{0}; + std::atomic snapshots{0}; + std::atomic pendingReads{0}; + std::atomic storageFallbacks{0}; + std::atomic records{0}; + std::atomic fastHits{0}; + std::atomic fastMisses{0}; + std::atomic storageHits{0}; + std::atomic storageCreated{0}; + std::atomic lastReport{0}; +}; + +TextureCounters& TextureCounts() { + static TextureCounters counters; + return counters; +} + +void reportTextureCounters() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& counters = TextureCounts(); + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(); + if (nowMs - last < 10000 || !counters.lastReport.compare_exchange_strong(last, nowMs)) return; + const auto count = [](const std::atomic& value) { return static_cast(value.load(std::memory_order_relaxed)); }; + std::fprintf(stderr, "[textures] sampled created: %llu from storage images, %llu snapshots (%llu snapshot reads over recorded writes inside cachedTexture, %llu storage-path fallbacks); stage-A records %llu: %llu fast hits, %llu full lookups; storage images %llu hits, %llu created\n", count(counters.fromStorage), count(counters.snapshots), count(counters.pendingReads), count(counters.storageFallbacks), count(counters.records), count(counters.fastHits), count(counters.fastMisses), count(counters.storageHits), count(counters.storageCreated)); +} + +// What the sampled-texture lookups on this thread proved their returned objects current against, +// taken by object into the ShaderResources being built or revalidated (captureValidation), which +// then repeats the proof from write stamps on its next Revalidate. The lookups run inside one build +// on one thread; records nobody takes (a build that threw, a resident target looked up outside a +// build, a thread that never captures) are dropped by the next capture or by the bound below. +struct LookupRecord { + const void* object; + GuestTextureResource resource; + std::uint64_t bytes; + DccKeys keys; + std::uint64_t generation; + const StorageTexture* source; +}; + +thread_local std::vector lookupLog; + +void logLookup(const LookupRecord& record) { + // The bound drops the oldest half, never everything: one build's lookups (a few dozen at most) + // are the newest records, and a capture must find them even when the bound trips mid-build. + constexpr std::size_t bound = 1024; + if (lookupLog.size() >= bound) lookupLog.erase(lookupLog.begin(), lookupLog.begin() + bound / 2); + lookupLog.push_back(record); +} + +std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes = 0); + +// Whether a sampled texture over `resource` can be a view of the surface's cached storage image +// instead of a CPU snapshot (see cachedTexture): the format has a storage form and is not block +// compressed, and the surface lives in host-imported memory, where the image uploads and refreshes +// GPU-direct (a snapshot of such a surface reads and compares its bytes on the CPU, waiting for +// the recorded work that wrote them). APS5_NO_SAMPLED_FROM_STORAGE=1 keeps every sampled texture a +// snapshot as before. +bool SampledFromStorageEligible(const Context& context, std::uint32_t format, std::uint64_t address, std::uint64_t guestBytes) { + static const bool disabled = std::getenv("APS5_NO_SAMPLED_FROM_STORAGE") != nullptr; + if (disabled || IsBlockCompressed(format) || !StorageFormatAvailable(context, format)) return false; + return HostImportCovers(context, address, static_cast(guestBytes)); +} + +bool SampledFromStorageEligible(const Context& context, const GuestTextureResource& resource, std::uint64_t guestBytes) { + return SampledFromStorageEligible(context, resource.format, resource.baseAddress, guestBytes); +} + +// Surfaces whose storage image could not be made (no writable committed pages, say): remembered +// so the sampled lookup does not throw and fall back on every use. A failure is keyed by the surface +// (address and size: the heap reuses addresses) and holds only while the guest mappings are what +// they were when it failed (the allocation generation): pages committed later, or another surface +// at the address, get a fresh attempt. +struct StorageFailures { + std::mutex mutex; + std::map, std::uint64_t> generations; +}; + +StorageFailures& StorageFailed() { + static StorageFailures failures; + return failures; +} + +// The cached storage image for a sampled descriptor, or null when it cannot be made (then the +// snapshot path serves the descriptor, as before). Every null return counts as a fallback. +std::shared_ptr sampledStorageSource(const Context& context, const GuestTextureResource& resource, std::uint64_t guestBytes) { + auto& counters = TextureCounts(); + auto& failures = StorageFailed(); + const auto surface = std::make_pair(resource.baseAddress, guestBytes); + const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + { + std::lock_guard lock(failures.mutex); + if (const auto it = failures.generations.find(surface); it != failures.generations.end()) { + if (it->second == generation) { + counters.storageFallbacks.fetch_add(1, std::memory_order_relaxed); + return nullptr; + } + failures.generations.erase(it); + } + } + try { + return cachedStorageTexture(context, {}, resource, 0, guestBytes); + } catch (const std::exception& error) { + counters.storageFallbacks.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(failures.mutex); + if (failures.generations.size() < 4096 && failures.generations.emplace(surface, generation).second) std::fprintf(stderr, "[textures] sampled texture 0x%llx (%ux%u format %u) keeps the snapshot path: %s\n", static_cast(resource.baseAddress), resource.width, resource.height, resource.format, error.what()); + return nullptr; + } +} + +// `guestBytes` is the surface size when the caller described the surface already (0: described here). +std::shared_ptr cachedTexture(const Context& context, std::span words, const GuestTextureResource& resource, VkComponentMapping components, std::uint64_t guestBytes = 0) { + static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + if (guestBytes == 0) guestBytes = DescribeSurface(resource).guestBytes; + auto& counters = TextureCounts(); + const auto address = resource.baseAddress; + const auto bytes = static_cast(guestBytes); + // A storage image whose results for this surface (or for a mip chain containing it) are still on + // the GPU supplies the texture by a view of it; anything else needs those results in guest + // memory first. + auto source = StorageTexture::FindPending(address, guestBytes); + if (source != nullptr && !Texture::CanCopyFrom(*source, resource)) source.reset(); + // Otherwise a surface in host-imported memory is viewed through its cached storage image (made + // here when there is none): its refresh after a CPU or GPU write is a GPU-direct detile from the + // import, recorded behind the producer, so no bytes are read or compared on the CPU and nothing + // waits for the producer. A fast-cleared surface (keys) stays a snapshot: its texels are not + // read either, and the image's own descriptor may not carry the DCC address. + std::optional keys; + if (source == nullptr && SampledFromStorageEligible(context, resource, guestBytes)) { + keys = TextureClearKeys(resource, guestBytes); + if (*keys == DccKeys::Uncompressed) source = sampledStorageSource(context, resource, guestBytes); + } + // Pages of the surface written since they were last collected are stamped now, before any + // UnchangedSince (here and on a cache hit) looks at them: the checks only compare stamped blocks, + // so a write the game made since the last walk is invisible until a collect stamps its page. The + // walk is memoized per packet, so a texture viewed from a storage image repeats it for free. + auto generation = GuestMemory::CollectWrites(address, bytes); + if (source != nullptr && !GuestMemory::UnchangedSince(address, bytes, source->Generation())) { + // The CPU wrote the memory while results were pending: Refresh merges them the usual way. + source->Refresh(); + } + // Results stored to guest memory just now are stamped newer than `generation`; a snapshot read + // after them is current at the generation of a second (memoized) collect. The store marks the + // surface's DCC keys uncompressed, so the keys are read after it. + if (source == nullptr && StorageTexture::FlushPending(address, bytes, nullptr, "sampled texture")) { + generation = GuestMemory::CollectWrites(address, bytes); + keys.reset(); + } + if (!keys.has_value()) keys = TextureClearKeys(resource, guestBytes); + if (disabled) { + if (source != nullptr) return std::make_shared(context, source, resource, components); + std::vector snapshot(bytes); + ReadTextureSurface(resource, *keys, snapshot); + return std::make_shared(context, *context.detiler, resource, components, snapshot); + } + auto& cache = Textures(); + const auto key = MakeTextureKey(context.device, words, components); + std::lock_guard lock(cache.mutex); + if (auto it = findTexture(cache, key); it != cache.entries.end()) { + if (it->source != nullptr) { + // The view follows the storage image, whatever the GPU wrote to it since; guest memory + // written meanwhile is taken in by refreshing the image. A fast clear the image cannot + // see (keys, and no pending results to prefer) ends the view: a snapshot holds the clear. + if ((source == nullptr || source == it->source) && (source != nullptr || *keys == DccKeys::Uncompressed)) { + if (!GuestMemory::UnchangedSince(address, bytes, it->source->Generation())) it->source->Refresh(); + touchTexture(cache, it); + logLookup({it->texture.get(), resource, guestBytes, *keys, 0, it->source.get()}); + reportTextureCounters(); + return it->texture; + } + } else if (source == nullptr && it->bytes.size() == guestBytes && it->keys == *keys) { + // Unwritten pages need no comparison; partially resident textures compare only committed + // pages. The compare goes through the flush hook: it waits for recorded work over the + // surface (counted, and named for the [hooksync] line). + const auto equalsCommitted = [&] { + if (Recorder::SnapshotWriteOverlaps(address, bytes)) counters.pendingReads.fetch_add(1, std::memory_order_relaxed); + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureCompare); + return GuestMemory::EqualsCommitted(address, it->bytes); + }; + if (*keys != DccKeys::Uncompressed || GuestMemory::UnchangedSince(address, it->bytes.size(), it->generation) || equalsCommitted()) { + it->generation = generation; + touchTexture(cache, it); + logLookup({it->texture.get(), resource, guestBytes, *keys, generation, nullptr}); + reportTextureCounters(); + return it->texture; + } + } + eraseTexture(cache, it); + } + CachedTexture entry{key, address, std::vector(source != nullptr ? 0u : bytes), nullptr, *keys, generation}; + entry.accounted = guestBytes; + if (source != nullptr) { + entry.source = source; + entry.sourceVersion = source->Version(); + entry.texture = std::make_shared(context, source, resource, components); + counters.fromStorage.fetch_add(1, std::memory_order_relaxed); + } else { + // Snapshot before the upload so a write racing with it is caught by the next comparison. + if (*keys == DccKeys::Uncompressed && Recorder::SnapshotWriteOverlaps(address, bytes)) counters.pendingReads.fetch_add(1, std::memory_order_relaxed); + ReadTextureSurface(resource, *keys, entry.bytes); + static const bool traceTextures = std::getenv("APS5_TRACE_TEXTURES") != nullptr; + if (traceTextures) { + std::size_t nonzero = 0; + for (std::size_t i = 0; i < entry.bytes.size(); i += 64) nonzero += entry.bytes[i] != std::byte{0}; + std::fprintf(stderr, "[texture] 0x%llx %ux%u format %u tile %d: %zu of %zu sampled bytes nonzero\n", static_cast(resource.baseAddress), resource.width, resource.height, resource.format, static_cast(resource.tileMode), nonzero, entry.bytes.size() / 64); + } + entry.texture = std::make_shared(context, *context.detiler, resource, components, entry.bytes); + counters.snapshots.fetch_add(1, std::memory_order_relaxed); + } + constexpr std::uint64_t budget = 2048ull << 20u; + while (!cache.entries.empty() && cache.bytes + entry.accounted > budget) eraseTexture(cache, std::prev(cache.entries.end())); + cache.bytes += entry.accounted; + auto texture = entry.texture; + logLookup({texture.get(), resource, guestBytes, *keys, generation, source.get()}); + cache.entries.push_front(std::move(entry)); + cache.index[key] = cache.entries.begin(); + reportTextureCounters(); + return texture; +} + +// Storage images stay on the GPU between dispatches: while guest memory still holds what an image was +// last uploaded from or written back as, its next use skips the upload and detile. +struct StorageKey { + VkDevice device; + std::array words; + bool operator==(const StorageKey&) const = default; +}; + +struct StorageKeyHash { + std::size_t operator()(const StorageKey& key) const noexcept { + return static_cast(hashWords(14695981039346656037ull ^ reinterpret_cast(key.device), key.words.data(), key.words.size())); + } +}; + +struct CachedStorageTexture { + StorageKey key; + std::uint32_t mip; + std::shared_ptr texture; +}; + +// As TextureCache: use order with a hash index by key, plus one by image for StorageImageCached. +struct StorageTextureCache { + std::mutex mutex; + std::list entries; + std::unordered_map::iterator, StorageKeyHash> index; + std::unordered_map::iterator> byImage; + std::uint64_t bytes = 0; +}; + +StorageTextureCache& StorageTextures() { + static StorageTextureCache cache; + return cache; +} + +std::list::iterator findStorage(StorageTextureCache& cache, const StorageKey& key) { + if (TextureHashEnabled()) { + const auto found = cache.index.find(key); + return found == cache.index.end() ? cache.entries.end() : found->second; + } + for (auto it = cache.entries.begin(); it != cache.entries.end(); ++it) { + if (it->key == key) return it; + } + return cache.entries.end(); +} + +std::list::iterator findStorageByImage(StorageTextureCache& cache, VkDevice device, const StorageTexture* image) { + if (TextureHashEnabled()) { + const auto found = cache.byImage.find(image); + return found == cache.byImage.end() || found->second->key.device != device ? cache.entries.end() : found->second; + } + for (auto it = cache.entries.begin(); it != cache.entries.end(); ++it) { + if (it->key.device == device && it->texture.get() == image) return it; + } + return cache.entries.end(); +} + +// Evicts an entry: its pending results go to guest memory first (the image may die with the entry). +void evictStorage(StorageTextureCache& cache, std::list::iterator it) { + it->texture->Flush(); + cache.bytes -= it->texture->GuestBytes(); + cache.index.erase(it->key); + cache.byImage.erase(it->texture.get()); + cache.entries.erase(it); +} + +// Storage images are shared by every descriptor of one surface (address, extent, layers, format, tile +// mode): the image holds the whole mip chain, and render targets in the same memory attach to it. +std::array SurfaceKey(const Context& context, const GuestTextureResource& resource) { + // Guest formats that store in the same Vulkan format share the image (views carry the difference). + return {static_cast(resource.baseAddress), static_cast(resource.baseAddress >> 32u), resource.width, resource.height, (resource.depthOrLastArray << 16u) | (resource.mipCount & 0xffffu), (static_cast(resource.tileMode) << 12u) | (static_cast(resource.dimension) << 20u), resource.baseArray, static_cast(StorageFormatForGuest(context, resource.format))}; +} + +// `guestBytes` is the surface size when the caller described the surface already (0: described here). +std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes) { + static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + if (disabled) return std::make_shared(context, *context.detiler, resource, mip); + static_cast(words); + auto& counters = TextureCounts(); + const StorageKey key{context.device, SurfaceKey(context, resource)}; + auto& cache = StorageTextures(); + std::lock_guard lock(cache.mutex); + if (auto it = findStorage(cache, key); it != cache.entries.end()) { + it->texture->Refresh(); + cache.entries.splice(cache.entries.begin(), cache.entries, it); + counters.storageHits.fetch_add(1, std::memory_order_relaxed); + return it->texture; + } + // Results other images hold over this memory reach it before the new image reads it: a hit's + // Refresh flushes them, the constructor's upload does not, and a GPU-direct upload reads the + // import buffer without the flush hook. The flush is recorded ahead of the upload in the batch. + if (guestBytes == 0) guestBytes = DescribeSurface(resource).guestBytes; + StorageTexture::FlushPending(resource.baseAddress, static_cast(guestBytes), nullptr, "storage image creation"); + CachedStorageTexture entry{key, mip, std::make_shared(context, *context.detiler, resource, mip)}; + constexpr std::uint64_t budget = 2048ull << 20u; + while (!cache.entries.empty() && cache.bytes + entry.texture->GuestBytes() > budget) evictStorage(cache, std::prev(cache.entries.end())); + cache.bytes += entry.texture->GuestBytes(); + auto texture = entry.texture; + cache.entries.push_front(std::move(entry)); + cache.index[key] = cache.entries.begin(); + cache.byImage[texture.get()] = cache.entries.begin(); + counters.storageCreated.fetch_add(1, std::memory_order_relaxed); + return texture; +} + +// Whether `image` is still the storage cache's image of its surface, i.e. what a lookup would return +// (an evicted image next to a newer one of the same memory must not be reused: two images of one +// surface would hold results). A use through the fast Revalidate counts as a use for the cache's +// eviction order, as a lookup would. +bool StorageImageCached(const Context& context, const StorageTexture* image) { + auto& cache = StorageTextures(); + std::lock_guard lock(cache.mutex); + const auto it = findStorageByImage(cache, context.device, image); + if (it == cache.entries.end()) return false; + cache.entries.splice(cache.entries.begin(), cache.entries, it); + return true; +} + +} + +std::shared_ptr CachedStorageSurface(const Context& context, const GuestTextureResource& resource) { + return cachedStorageTexture(context, {}, resource, 0); +} + +namespace { + const char* roleName(ShaderRecompiler::DescriptorRole role) { switch (role) { case ShaderRecompiler::DescriptorRole::GuestBuffers: return "GuestBuffers"; @@ -55,6 +510,21 @@ const char* kindName(ShaderRecompiler::DescriptorKind kind) { throw std::runtime_error("AGC graphics: unknown descriptor kind"); } +// Consecutive identical storage descriptors address successive mips of one texture (dynamic-mip +// storage writes), so the second and later ones share the first one's image lookup. +// APS5_NO_STORAGE_DEDUPE=1 looks each one up (and refreshes the image) separately as before. +bool StorageDedupeEnabled() { + static const bool disabled = std::getenv("APS5_NO_STORAGE_DEDUPE") != nullptr; + return !disabled; +} + +// Whether storage element `element` of `binding` repeats the previous element's eight words. +bool SameAsPreviousStorageElement(const ShaderRecompiler::DescriptorBinding& binding, std::uint32_t element) { + if (element == 0) return false; + const auto words = binding.guestDescriptor.begin() + static_cast(element) * 8u; + return std::equal(words, words + 8, words - 8); +} + } ShaderResources::ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes) : ShaderResources(context, std::array{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, static_cast(vertex.pushConstants.size())}}}, target, indexAddress, indexBytes) {} @@ -64,19 +534,130 @@ ShaderResources::ShaderResources(const Context& context, std::span snapshots) : context(context), guestMemory(context) { +ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots) : ShaderResources(context, compute, snapshots, false) {} + +namespace { + +// Whether a compute stage maps registered guest memory through BDA tables (see prepareAddressBindings). +bool UsesAddressTables(const CompiledShader& compute) { + Require(compute.program != nullptr, "missing compiled shader"); + return std::any_of(compute.program->bindings.begin(), compute.program->bindings.end(), [](const auto& binding) { return binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable; }); +} + +} + +ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred) : context(context), guestMemory(context), deferredCompute(compute), deferredSnapshots(snapshots) { Require(compute.stage == ShaderRecompiler::ShaderStage::Compute, "compute resources require a compute shader"); - prepareAddressBindings(std::span(&compute, 1), snapshots); - build(std::span(&compute, 1), nullptr, 0, 0); + const std::span shaders(&deferredCompute, 1); + if (!deferred) { + prepareAddressBindings(shaders, snapshots); + build(shaders, nullptr, 0, 0); + forgetDeferredInputs(); + return; + } + // Address-based shaders pin and mirror registered memory in prepareAddressBindings (reconciling + // imports retires buffers to the recorder, refreshing mirrors waits for recorded work): that is + // device-lock work, so their whole build waits for Complete(). Without tables the call only + // checks the bindings. + lockedBuild = UsesAddressTables(compute); + if (lockedBuild) return; + unlockedPrepare = true; + prepareAddressBindings(shaders, snapshots); + buildPrepare(shaders, nullptr, 0, 0); +} + +void ShaderResources::Complete() { + Require(!completed, "shader resources were already completed"); + const std::span shaders(&deferredCompute, 1); + if (lockedBuild) { + prepareAddressBindings(shaders, deferredSnapshots); + buildPrepare(shaders, nullptr, 0, 0); + } + buildComplete(); + forgetDeferredInputs(); +} + +void ShaderResources::forgetDeferredInputs() { + // The compiled shader and the captured regions belong to the dispatch, which returns while this + // object lives on in the resource cache and the recorder: nothing may reach for them after the + // build, so they are dropped rather than left dangling. + deferredCompute = {}; + deferredSnapshots = {}; } void ShaderResources::build(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { - PerformanceTimer timing("Graphics.ShaderResources"); + buildPrepare(shaders, target, indexAddress, indexBytes); + buildComplete(); +} + +namespace { + +// APS5_PROFILE_DRAW: the [resources] phase totals, under a mutex because stage A of several builds +// runs at once. +struct BuildProfile { + std::mutex mutex; + std::map phaseTotals; + std::uint64_t builds = 0; +}; + +BuildProfile& Builds() { + static BuildProfile profile; + return profile; +} + +bool BuildProfiled() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + +void addBuildPhase(const char* name, double ms) { + auto& profile = Builds(); + std::lock_guard lock(profile.mutex); + profile.phaseTotals[name] += ms; +} + +// APS5_PROFILE_DRAW: guest buffer elements bound by descriptor (addGuestBuffer) and how many the +// recompiler proved read-only, plus the pending-write notes their uses skipped (MarkGpuWrites), +// printed as [buffers] every 10 s from MarkGpuWrites. Cumulative. +struct BufferWriteCounters { + std::atomic elements{0}; + std::atomic readOnly{0}; + std::atomic notesSkipped{0}; + std::atomic lastReport{0}; +}; + +BufferWriteCounters& BufferWrites() { + static BufferWriteCounters counters; + return counters; +} + +} + +double ShaderResources::phase(const char* name) { + if (!BuildProfiled()) return 0.0; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - phaseStart).count(); + addBuildPhase(name, ms); + if (ms > 50) std::fprintf(stderr, "[resources] %s took %.0f ms (%zu textures, %zu storage images, %zu buffers, bda %d)\n", name, ms, textures.size(), storageTextures.size(), allocations.size(), usesBda ? 1 : 0); + phaseStart = now; + return ms; +} + +void ShaderResources::buildPrepare(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { + const auto stageStart = std::chrono::steady_clock::now(); + phaseStart = stageStart; + if (BuildProfiled()) { + auto& profile = Builds(); + std::lock_guard lock(profile.mutex); + if (++profile.builds % 1000 == 0) { + std::string report; + for (const auto& [name, ms] : profile.phaseTotals) report += " " + name + "=" + std::to_string(static_cast(ms)) + "ms"; + std::fprintf(stderr, "[resources] %llu builds, phase totals:%s\n", static_cast(profile.builds), report.c_str()); + } + } try { Require(!shaders.empty() && context.limits.maxBoundDescriptorSets >= 1, "shader descriptor set exceeds device limits"); - std::vector bindings; std::set occupied; - std::uint64_t storageBuffers = 0; for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); const VkShaderStageFlags flags = VulkanStage(shader.stage); @@ -88,7 +669,7 @@ void ShaderResources::build(std::span shaders, const Color const bool bufferRole = addressRole || binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers || binding.role == ShaderRecompiler::DescriptorRole::ShaderData || binding.role == ShaderRecompiler::DescriptorRole::FlattenedSrt; const bool imageRole = binding.role == ShaderRecompiler::DescriptorRole::GuestImages || binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers; if (imageRole) { - addImageBinding(binding, flags, bindings); + addImageBinding(binding, flags); continue; } Require(bufferRole, std::string("unsupported descriptor role ") + roleName(binding.role)); @@ -101,7 +682,12 @@ void ShaderResources::build(std::span shaders, const Color Binding item{{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, binding.count, flags, nullptr}, {}}; if (binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers) { Require(binding.guestDescriptor.size() == static_cast(binding.count) * 4, "guest buffer descriptor must contain four DWORDs per array element"); - for (std::uint32_t element = 0; element < binding.count; ++element) item.allocations.push_back(addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes)); + for (std::uint32_t element = 0; element < binding.count; ++element) { + // An element the recompiler did not classify (a producer without the + // vector) counts as written, like imageWritten below. + const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; + item.allocations.push_back(addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written)); + } } else if (addressRole) { item.allocations.push_back(allocations.size()); allocations.push_back({0, 0, false, nullptr, binding.role}); @@ -114,66 +700,662 @@ void ShaderResources::build(std::span shaders, const Color } } Require(storageBuffers <= context.limits.maxDescriptorSetStorageBuffers, "pipeline descriptors exceed device limits"); - timing.Mark("bindings"); - guestMemory.Upload(usesBda); - if (usesBda) bda = std::make_unique(context, guestMemory); - else if (usesFaultBuffer) bda = std::make_unique(context); - timing.Mark("memory_upload"); + timing.bindingsMs = phase("bindings"); + // For every build, locked ones included: their stage B then takes the fast path too, and the + // collects cost the same wherever they run. + if (precollectImages()) phase("precollect"); + guestMemory.UploadPrepare(usesBda); + timing.uploadMs = phase("guest memory upload"); std::vector description; for (const auto& binding : bindings) { description.push_back(binding.layout); layoutKey.insert(layoutKey.end(), {binding.layout.binding, static_cast(binding.layout.descriptorType), binding.layout.descriptorCount, binding.layout.stageFlags}); } - std::vector sizes; - if (storageBuffers != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, static_cast(storageBuffers)}); - if (!textures.empty()) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, static_cast(textures.size())}); - if (!samplers.empty()) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLER, static_cast(samplers.size())}); - if (!context.descriptorCache) context.descriptorCache = std::make_shared(); - descriptors = context.descriptorCache->Take(layoutKey); - if (!descriptors) descriptors = std::make_unique(context, layoutKey, description, sizes); - _layout = descriptors->Layout(); - _set = descriptors->Set(); - timing.Mark("descriptor_acquire"); - std::vector buffers; - std::vector images; - std::vector writes; - buffers.reserve(allocations.size()); - images.reserve(textures.size() + samplers.size()); - writes.reserve(bindings.size()); - for (const auto& binding : bindings) { - const auto bufferOffset = buffers.size(); - const auto imageOffset = images.size(); - VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; - write.dstSet = _set; - write.dstBinding = binding.layout.binding; - write.descriptorCount = binding.layout.descriptorCount; - write.descriptorType = binding.layout.descriptorType; - switch (binding.layout.descriptorType) { - case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: - for (const auto index : binding.allocations) buffers.push_back(descriptor(allocations[index])); - write.pBufferInfo = buffers.data() + bufferOffset; - break; - case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: - for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, textures[index]->View(), VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL}); - write.pImageInfo = images.data() + imageOffset; - break; - case VK_DESCRIPTOR_TYPE_SAMPLER: - for (const auto index : binding.imageAllocations) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); - write.pImageInfo = images.data() + imageOffset; - break; - default: throw std::runtime_error("AGC graphics: ShaderResources encountered an unknown descriptor type while writing the descriptor set"); + static const bool noLayoutCache = std::getenv("APS5_NO_LAYOUT_CACHE") != nullptr; + static const bool noPoolCache = std::getenv("APS5_NO_POOL_CACHE") != nullptr; + if (context.descriptorCache != nullptr && !noLayoutCache) { + _layout = context.descriptorCache->Layout(layoutKey, description); + } else { + VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + info.bindingCount = static_cast(description.size()); + info.pBindings = description.data(); + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &info, nullptr, &_layout), "vkCreateDescriptorSetLayout"); + ownsLayout = true; + } + if (!bindings.empty()) { + // The set is sized from the plan: every image element becomes exactly one descriptor + // when stage B looks it up. + std::vector sizes; + if (storageBuffers != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, static_cast(storageBuffers)}); + if (plannedSampledImages != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, plannedSampledImages}); + if (plannedStorageImages != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, plannedStorageImages}); + if (!samplers.empty()) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLER, static_cast(samplers.size())}); + if (context.descriptorCache != nullptr && !noPoolCache) { + const auto allocated = context.descriptorCache->Allocate(_layout, sizes); + _set = allocated.set; + cachePool = allocated.pool; } - writes.push_back(write); + if (_set == VK_NULL_HANDLE) { + VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + poolInfo.maxSets = 1; + poolInfo.poolSizeCount = static_cast(sizes.size()); + poolInfo.pPoolSizes = sizes.data(); + Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); + VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + allocation.descriptorPool = pool; + allocation.descriptorSetCount = 1; + allocation.pSetLayouts = &_layout; + Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &_set), "vkAllocateDescriptorSets"); + } + } + timing.descriptorsMs = phase("descriptors"); + if (BuildProfiled()) { + timing.prepareMs = std::chrono::duration(std::chrono::steady_clock::now() - stageStart).count(); + addBuildPhase("stage A", timing.prepareMs); } - if (!writes.empty()) context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); - timing.Mark("descriptor_update"); } catch (...) { release(); throw; } } -std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { +void ShaderResources::buildComplete() { + const auto stageStart = std::chrono::steady_clock::now(); + phaseStart = stageStart; + try { + // The lookups run in plan order: Revalidate walks the bindings the same way, and consecutive + // storage elements of one mip chain share the previous element's image. + for (const auto& deferred : deferredImages) resolveImageBinding(*deferred.binding, bindings[deferred.index]); + deferredImages.clear(); + // The records served their purpose: each holds the cache entry's objects as of stage A, + // which would otherwise keep a replaced texture or an evicted storage image (and its device + // memory) alive, outside the caches' budgets, for as long as this object is cached. + imageRecords.clear(); + imageRecords.shrink_to_fit(); + nextImageRecord = 0; + Require(textures.size() == plannedSampledImages && storageTextures.size() == plannedStorageImages, "image lookups disagree with the descriptor plan"); + timing.bindingsMs += phase("images"); + guestMemory.UploadFinish(usesBda); + timing.uploadMs += phase("guest memory upload"); + if (usesBda) bda = std::make_unique(context, guestMemory); + else if (usesFaultBuffer) bda = std::make_unique(context); + phase("bda"); + if (_set != VK_NULL_HANDLE) { + // One update call for the whole set: the info arrays are sized up front so every write's + // pointer into them stays valid until the call. + std::size_t bufferCount = 0; + std::size_t imageCount = 0; + for (const auto& binding : bindings) { + bufferCount += binding.allocations.size(); + imageCount += binding.imageAllocations.size(); + } + std::vector buffers; + std::vector images; + buffers.reserve(bufferCount); + images.reserve(imageCount); + std::vector writes; + writes.reserve(bindings.size()); + for (const auto& binding : bindings) { + VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = _set; + write.dstBinding = binding.layout.binding; + write.descriptorCount = binding.layout.descriptorCount; + write.descriptorType = binding.layout.descriptorType; + switch (binding.layout.descriptorType) { + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + write.pBufferInfo = buffers.data() + buffers.size(); + for (const auto index : binding.allocations) buffers.push_back(descriptor(allocations[index])); + break; + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: + write.pImageInfo = images.data() + images.size(); + for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, textures[index]->View(), textures[index]->Layout()}); + break; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: + write.pImageInfo = images.data() + images.size(); + for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, storageTextures[index]->View(storageMips[index]), VK_IMAGE_LAYOUT_GENERAL}); + break; + case VK_DESCRIPTOR_TYPE_SAMPLER: + write.pImageInfo = images.data() + images.size(); + for (const auto index : binding.imageAllocations) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); + break; + default: throw std::runtime_error("AGC graphics: ShaderResources encountered an unknown descriptor type while writing the descriptor set"); + } + writes.push_back(write); + } + context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + } + timing.descriptorsMs += phase("descriptors"); + noteReusable(); + completed = true; + if (BuildProfiled()) { + timing.completeMs = std::chrono::duration(std::chrono::steady_clock::now() - stageStart).count(); + addBuildPhase("stage B", timing.completeMs); + reportDescriptorCaches(); + } + } catch (...) { + release(); + throw; + } +} + +// Whether this build can serve later dispatches of the same content (see ContentKey): nothing to do +// once the GPU completed (no BDA, no copied written buffers, no lease) and every guest buffer bound +// in place through a host import whose identity is recorded for Revalidate. +void ShaderResources::noteReusable() { + // Taken whether or not the object turns out reusable: the lookups' records are this build's. + captureValidation(); + reusable = false; + directRegions.clear(); + if (NeedsCompletion() || HoldsLease()) return; + const auto regions = guestMemory.DirectRegions(); + if (!regions.has_value()) return; + for (const auto& [begin, end] : *regions) { + // No reconcile here: the upload just took these imports, and the set is about to be recorded + // against them. + auto serial = HostImportSerial(context, begin, static_cast(end - begin), false); + // Read-only image mirrors (exe ranges) are as stable as imports; their serials have the top + // bit set, so the two spaces never collide. + if (serial == 0) serial = ImageMirrorSerial(context, begin, static_cast(end - begin)); + if (serial == 0) return; + directRegions.push_back({begin, end, serial}); + } + reusable = true; +} + +// APS5_PROFILE_DRAW: the per-device descriptor caches' counters, every 10 s. +void ShaderResources::reportDescriptorCaches() const { + static std::mutex reportMutex; + static auto lastReport = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(reportMutex); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + const auto descriptors = context.descriptorCache != nullptr ? context.descriptorCache->Counters() : DescriptorCache::Stats{}; + const auto samplerHits = context.samplerCache != nullptr ? context.samplerCache->Hits() : 0; + const auto samplerMisses = context.samplerCache != nullptr ? context.samplerCache->Misses() : 0; + std::fprintf(stderr, "[descriptors] layouts %llu hits / %llu created, sets %llu from %llu pools, samplers %llu hits / %llu created\n", static_cast(descriptors.layoutHits), static_cast(descriptors.layoutMisses), static_cast(descriptors.sets), static_cast(descriptors.pools), static_cast(samplerHits), static_cast(samplerMisses)); +} + +std::vector ShaderResources::ContentKey(const CompiledShader& shader) { + Require(shader.program != nullptr, "missing compiled shader"); + const auto& program = *shader.program; + std::vector key; + key.reserve(8 + program.bindings.size() * 12); + key.push_back(static_cast(shader.stage)); + key.push_back(static_cast(program.variantId)); + key.push_back(static_cast(program.variantId >> 32u)); + key.push_back(static_cast(program.bindings.size())); + const auto packBits = [&](const std::vector& bits) { + key.push_back(static_cast(bits.size())); + std::uint32_t word = 0; + for (std::size_t i = 0; i < bits.size(); ++i) { + if (bits[i]) word |= 1u << (i % 32u); + if (i % 32u == 31u || i + 1 == bits.size()) { + key.push_back(word); + word = 0; + } + } + }; + for (const auto& binding : program.bindings) { + key.insert(key.end(), {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(binding.guestDescriptor.size())}); + key.insert(key.end(), binding.guestDescriptor.begin(), binding.guestDescriptor.end()); + packBits(binding.imageWritten); + packBits(binding.samplerDepthCompare); + // Read-only elements are bound without a write set: an object built for one written set + // must not serve a build with another (the variant implies it, this makes it explicit). + packBits(binding.bufferWritten); + } + return key; +} + +namespace { + +// APS5_PROFILE_DRAW: Revalidate outcomes and time, printed as [rescache] every 10 s next to the +// device's hit/miss line: failed = the object could not be reused (whichever image path it took); +// of the reused ones, fast = every image proved current from stamps, full = the lookups were repeated. +struct RevalidateProfile { + std::atomic calls{0}; + std::atomic nanoseconds{0}; + std::atomic fast{0}; + std::atomic full{0}; + std::atomic failed{0}; + std::atomic lastReport{0}; +}; + +RevalidateProfile& Revalidations() { + static RevalidateProfile profile; + return profile; +} + +void countRevalidate(bool fast, bool ok, std::chrono::steady_clock::time_point start) { + auto& profile = Revalidations(); + const auto now = std::chrono::steady_clock::now(); + profile.calls.fetch_add(1, std::memory_order_relaxed); + profile.nanoseconds.fetch_add(static_cast(std::chrono::duration_cast(now - start).count()), std::memory_order_relaxed); + (!ok ? profile.failed : fast ? profile.fast : profile.full).fetch_add(1, std::memory_order_relaxed); + const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); + auto last = profile.lastReport.load(); + if (nowMs - last < 10000 || !profile.lastReport.compare_exchange_strong(last, nowMs)) return; + std::fprintf(stderr, "[rescache] revalidate %llu calls %.1f ms: fast %llu, full %llu, failed %llu\n", static_cast(profile.calls.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.fast.load()), static_cast(profile.full.load()), static_cast(profile.failed.load())); +} + +} + +// Turns the lookups' records into this object's per-texture validation records (see +// ValidatedSurface); the last record of an object is the freshest. +void ShaderResources::captureValidation() { + const auto record = [](const void* object) -> const LookupRecord* { + for (auto it = lookupLog.rbegin(); it != lookupLog.rend(); ++it) { + if (it->object == object) return &*it; + } + return nullptr; + }; + validatedTextures.assign(textures.size(), {}); + for (std::size_t i = 0; i < textures.size(); ++i) { + if (const auto* found = record(textures[i].get())) validatedTextures[i] = {found->resource, found->bytes, found->keys, found->generation, 0, found->source, true}; + } + lookupLog.clear(); +} + +// Proves every texture and storage image still current without repeating its lookup: exactly the +// "unchanged" branches of cachedTexture and StorageTexture::Refresh, evaluated from write stamps, +// the pending-results registry and the DCC keys. Per element: collect first (stamps come only from +// collects, and the generation a record moves to must predate the checks, so a write landing between +// them is stamped newer: the rule of Driver's dispatch cache), then no other image may have results +// pending over the memory (a lookup would flush them into it), a storage image must still be the +// cache's image of its surface, the memory must be unchanged since the object's content matched it, +// and the DCC keys must be what the content was made under whenever the surface has any (a fast +// clear touches only the keys). Anything else, including a failed collect (memory outside the arena, +// uncommitted pages), is left to the full walk. +bool ShaderResources::fastRevalidate() { + if (validatedTextures.size() != textures.size()) return false; + for (std::size_t i = 0; i < textures.size(); ++i) { + auto& surface = validatedTextures[i]; + if (!surface.valid) return false; + const auto address = surface.resource.baseAddress; + const auto bytes = static_cast(surface.bytes); + surface.collected = GuestMemory::CollectWrites(address, bytes); + if (surface.collected == 0) return false; + if (PendingStorageOverlaps(address, bytes, surface.source)) return false; + if (surface.source != nullptr) { + // The view follows the image: it needs the image current with guest memory, as the + // lookup's Refresh would make it. + if (!StorageImageCached(context, surface.source) || !GuestMemory::UnchangedSince(address, bytes, surface.source->Generation())) return false; + } else if (!GuestMemory::UnchangedSince(address, bytes, surface.generation)) { + return false; + } + if (surface.resource.dccAddress != 0 && TextureClearKeys(surface.resource, surface.bytes) != surface.keys) return false; + // A view made under fast-clear keys stays one only while its image still has results pending + // over the surface (the lookup prefers them to the clear); once they are flushed the clear + // the image cannot see wins and the lookup makes a snapshot (cachedTexture's hit rule). + if (surface.source != nullptr && surface.keys != DccKeys::Uncompressed && StorageTexture::FindPending(address, surface.bytes).get() != surface.source) return false; + } + for (std::size_t i = 0; i < storageTextures.size(); ++i) { + // Consecutive elements of one mip chain share the image (see addStorageImageBinding). + if (i != 0 && storageTextures[i] == storageTextures[i - 1]) continue; + const auto* image = storageTextures[i].get(); + if (image == nullptr) return false; + // StorageTexture::Refresh checks the image's own surface, not the element's descriptor (one + // image serves every descriptor of its memory, with or without a DCC address), and compares + // its keys with the keys it was uploaded under, which only the image knows (they move to + // Uncompressed on every write-back; a fast clear after it puts the key bytes back to what a + // record saw while the image holds last frame's results), so a surface with keys is left to + // the full walk. Without a DCC address both sides of that compare are Uncompressed whatever + // happened, and the memory checks below are Refresh's. + const auto& own = image->Descriptor(); + if (own.dccAddress != 0) return false; + const auto address = own.baseAddress; + const auto bytes = static_cast(image->GuestBytes()); + if (GuestMemory::CollectWrites(address, bytes) == 0) return false; + if (PendingStorageOverlaps(address, bytes, image)) return false; + if (!StorageImageCached(context, image)) return false; + if (!GuestMemory::UnchangedSince(address, bytes, image->Generation())) return false; + } + // Every check passed: snapshot textures are current at the collects issued before them. Storage + // images keep their own generation: nothing was stamped above it, so the blocks a later write + // stamps are the same set either way. + for (auto& surface : validatedTextures) { + if (surface.source == nullptr) surface.generation = surface.collected; + } + return true; +} + +bool ShaderResources::Revalidate(std::span shaders) { + if (!reusable || shaders.empty()) return false; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // APS5_NO_FAST_REVALIDATE=1 always repeats the lookups. + static const bool noFast = std::getenv("APS5_NO_FAST_REVALIDATE") != nullptr; + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto finish = [&](bool fast, bool ok) { + if (profile) countRevalidate(fast, ok, start); + return ok; + }; + // (1) Textures and storage images, from stamps when every element allows it (fastRevalidate), + // else by the same lookups as the build (which refresh or replace them as guest memory changed): + // they must hand back the very objects the set's views belong to. The build appended them stage + // by stage, binding by binding, so the walk repeats that order. + const bool fast = !noFast && fastRevalidate(); + if (!fast) { + lookupLog.clear(); + std::size_t textureIndex = 0; + std::size_t storageIndex = 0; + for (const auto& shader : shaders) { + if (shader.program == nullptr) return finish(false, false); + for (const auto& binding : shader.program->bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; + if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { + const auto elementWords = binding.count != 0 ? binding.guestDescriptor.size() / binding.count : 0; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + const auto resource = DecodeTextureResource(words); + const VkComponentMapping components{ComponentSwizzleFor(resource.dstSelX), ComponentSwizzleFor(resource.dstSelY), ComponentSwizzleFor(resource.dstSelZ), ComponentSwizzleFor(resource.dstSelW)}; + if (textureIndex >= textures.size() || cachedTexture(context, words, resource, components) != textures[textureIndex]) return finish(false, false); + ++textureIndex; + } + } else if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + if (storageIndex >= storageTextures.size()) return finish(false, false); + std::shared_ptr expected; + if (SameAsPreviousStorageElement(binding, element) && StorageDedupeEnabled()) expected = storageTextures[storageIndex - 1]; + else expected = cachedStorageTexture(context, words, DecodeTextureResource(words), storageMips[storageIndex]); + if (expected != storageTextures[storageIndex]) return finish(false, false); + ++storageIndex; + } + } + } + } + if (textureIndex != textures.size() || storageIndex != storageTextures.size()) return finish(false, false); + // The walk proved every object current again: the records move to what it proved, or one + // spurious stamp (a label sharing a 64 KiB block with a surface's edge) would keep this + // object on the full walk for good. + captureValidation(); + } + // (2) The imported buffers the set reads in place: results of storage images pending in them go + // to guest memory first (as an upload does), then each import must still be the one the set was + // written against. This comes last because the lookups and flushes above can reconcile imports + // themselves; nothing else touches them between here and the dispatch being recorded. + for (const auto& region : directRegions) StorageTexture::FlushPending(region.begin, static_cast(region.end - region.begin), nullptr, "imported buffer region"); + for (const auto& region : directRegions) { + auto serial = HostImportSerial(context, region.begin, static_cast(region.end - region.begin), true); + if (serial == 0) serial = ImageMirrorSerial(context, region.begin, static_cast(region.end - region.begin)); + if (serial != region.serial) return finish(fast, false); + } + return finish(fast, true); +} + +namespace { + +// APS5_PROFILE_DRAW: what makes content keys miss. On a miss the key is compared with the last key +// seen for the same shader variant(s) and the first differing word is charged to its binding's role +// (and to its index within the V# or T# element), printed as [rescache] every 10 s: it says whether +// V# bases of ring allocations, SRT words or image descriptors churn. +struct ChurnProfile { + std::mutex mutex; + std::unordered_map lastByVariant; + std::array roles{}; + std::array bufferWords{}; + std::array imageWords{}; + std::uint64_t firstSeen = 0; + std::uint64_t sameKey = 0; + std::uint64_t layout = 0; + std::uint64_t drawWords = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +ChurnProfile& Churn() { + static ChurnProfile profile; + return profile; +} + +// Charges word `diff` of `key` to the binding of the ContentKey that starts at `at` (the layout of +// ShaderResources::ContentKey), moving `at` past that key; false when `diff` lies beyond it. +bool chargeShaderKey(ChurnProfile& profile, const ResourceCache::Key& key, std::size_t& at, std::size_t diff) { + if (at + 4 > key.size() || diff < at + 4) { + ++profile.layout; + return true; + } + const auto bindings = key[at + 3]; + at += 4; + for (std::uint32_t binding = 0; binding < bindings; ++binding) { + if (at + 8 > key.size() || diff < at + 8) { + ++profile.layout; + return true; + } + const auto kind = key[at]; + const auto role = key[at + 1]; + const auto descriptorWords = key[at + 7]; + at += 8; + if (diff < at + descriptorWords) { + if (role < profile.roles.size()) ++profile.roles[role]; + const auto index = diff - at; + if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestBuffers)) ++profile.bufferWords[index % 4]; + else if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestImages) && (kind == static_cast(ShaderRecompiler::DescriptorKind::SampledImage) || kind == static_cast(ShaderRecompiler::DescriptorKind::StorageImage))) ++profile.imageWords[index % 8]; + return true; + } + at += descriptorWords; + // imageWritten, samplerDepthCompare and bufferWritten: a count word, then one word per 32 bits. + for (int flags = 0; flags < 3; ++flags) { + if (at >= key.size()) { + ++profile.layout; + return true; + } + const auto words = 1 + (static_cast(key[at]) + 31) / 32; + if (diff < at + words) { + ++profile.layout; + return true; + } + at += words; + } + } + return false; +} + +} + +std::shared_ptr ResourceCache::Find(const Key& key) { + std::lock_guard lock(mutex); + const auto found = index.find(key); + if (found == index.end()) { + noteMiss(key); + return nullptr; + } + entries.splice(entries.begin(), entries, found->second); + return found->second->second; +} + +void ResourceCache::noteMiss(const Key& key) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile || key.size() < 4) return; + // The variant(s) the key belongs to: a compute key starts with the stage and its variant, a draw + // key with a marker, the device and its stages' keys (Draw.cpp DrawResourceKey). + const bool draw = key[0] == 0xffffffffu; + std::uint64_t variants = 0; + if (draw) { + std::size_t at = 4; + for (std::uint32_t stage = 0; stage < key[3] && at + 3 < key.size(); ++stage) { + variants = variants * 1000003ull ^ (key[at + 2] | (static_cast(key[at + 3]) << 32u)); + at += 1 + key[at]; + } + } else { + variants = key[1] | (static_cast(key[2]) << 32u); + } + auto& churn = Churn(); + std::lock_guard lock(churn.mutex); + const auto found = churn.lastByVariant.find(variants); + if (found == churn.lastByVariant.end()) { + ++churn.firstSeen; + churn.lastByVariant.emplace(variants, key); + } else { + const auto& previous = found->second; + const auto common = std::min(key.size(), previous.size()); + std::size_t diff = 0; + while (diff < common && key[diff] == previous[diff]) ++diff; + if (diff == common && key.size() == previous.size()) { + ++churn.sameKey; + } else if (draw) { + std::size_t at = 4; + bool charged = false; + for (std::uint32_t stage = 0; stage < key[3] && at < key.size() && !charged; ++stage) { + ++at; + charged = chargeShaderKey(churn, key, at, diff); + } + if (!charged) ++churn.drawWords; + } else { + std::size_t at = 0; + if (!chargeShaderKey(churn, key, at, diff)) ++churn.layout; + } + found->second = key; + } + const auto now = std::chrono::steady_clock::now(); + if (now - churn.lastReport < std::chrono::seconds(10)) return; + churn.lastReport = now; + std::string line = "[rescache] miss churn, first differing word by role:"; + char item[256]; + for (std::size_t role = 0; role < churn.roles.size(); ++role) { + if (churn.roles[role] == 0) continue; + std::snprintf(item, sizeof(item), " %s %llu", roleName(static_cast(role)), static_cast(churn.roles[role])); + line += item; + if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestBuffers)) { + std::snprintf(item, sizeof(item), " (V# word %llu/%llu/%llu/%llu)", static_cast(churn.bufferWords[0]), static_cast(churn.bufferWords[1]), static_cast(churn.bufferWords[2]), static_cast(churn.bufferWords[3])); + line += item; + } else if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestImages)) { + std::snprintf(item, sizeof(item), " (T# word %llu/%llu/%llu/%llu/%llu/%llu/%llu/%llu)", static_cast(churn.imageWords[0]), static_cast(churn.imageWords[1]), static_cast(churn.imageWords[2]), static_cast(churn.imageWords[3]), static_cast(churn.imageWords[4]), static_cast(churn.imageWords[5]), static_cast(churn.imageWords[6]), static_cast(churn.imageWords[7])); + line += item; + } + } + std::snprintf(item, sizeof(item), "; layout %llu, draw target/index %llu, same key %llu (not inserted or evicted), first seen %llu", static_cast(churn.layout), static_cast(churn.drawWords), static_cast(churn.sameKey), static_cast(churn.firstSeen)); + line += item; + std::fprintf(stderr, "%s\n", line.c_str()); +} + +void ResourceCache::Insert(const Key& key, std::shared_ptr resources) { + std::lock_guard lock(mutex); + erase(key); + entries.emplace_front(key, std::move(resources)); + index.emplace(key, entries.begin()); + // Entries pin their textures and storage images past the texture caches' budgets, so the bound + // stays modest: APS5_RESOURCE_CACHE_ENTRIES (default 1024) covers several frames of distinct + // dispatch and draw content. + static const std::size_t capacity = [] { + const char* value = std::getenv("APS5_RESOURCE_CACHE_ENTRIES"); + const auto parsed = value ? std::strtoull(value, nullptr, 10) : 1024ull; + return static_cast(parsed != 0 ? parsed : 1024ull); + }(); + while (entries.size() > capacity) { + index.erase(entries.back().first); + entries.pop_back(); + } +} + +void ResourceCache::Remove(const Key& key) { + std::lock_guard lock(mutex); + erase(key); +} + +void ResourceCache::Clear() { + std::lock_guard lock(mutex); + index.clear(); + entries.clear(); +} + +std::size_t ResourceCache::Size() const { + std::lock_guard lock(mutex); + return entries.size(); +} + +void ResourceCache::erase(const Key& key) { + const auto found = index.find(key); + if (found == index.end()) return; + entries.erase(found->second); + index.erase(found); +} + +ResourceCache& SharedResourceCache() { + static auto* cache = new ResourceCache(); + return *cache; +} + +DescriptorCache::DescriptorCache(const Context& context) : context(context), destroyLayout(context.Function("vkDestroyDescriptorSetLayout")), destroyPool(context.Function("vkDestroyDescriptorPool")), freeSets(context.Function("vkFreeDescriptorSets")) {} + +DescriptorCache::~DescriptorCache() { + for (const auto pool : pools) destroyPool(context.device, pool, nullptr); + for (const auto& [key, layout] : layouts) destroyLayout(context.device, layout, nullptr); +} + +VkDescriptorSetLayout DescriptorCache::Layout(std::span key, std::span bindings) { + std::lock_guard lock(mutex); + std::vector keyCopy(key.begin(), key.end()); + if (const auto found = layouts.find(keyCopy); found != layouts.end()) { + ++stats.layoutHits; + return found->second; + } + ++stats.layoutMisses; + VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + info.bindingCount = static_cast(bindings.size()); + info.pBindings = bindings.data(); + VkDescriptorSetLayout layout = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &info, nullptr, &layout), "vkCreateDescriptorSetLayout"); + layouts.emplace(std::move(keyCopy), layout); + return layout; +} + +namespace { +// What one chain pool holds; a set needing more of any type gets a dedicated pool. +constexpr std::uint32_t ChainPoolSets = 1024; +constexpr std::array ChainPoolSizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_SAMPLER, 512}}}; +} + +DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout layout, std::span sizes) { + for (const auto& size : sizes) { + const auto capacity = std::find_if(ChainPoolSizes.begin(), ChainPoolSizes.end(), [&](const auto& item) { return item.type == size.type; }); + if (capacity == ChainPoolSizes.end() || size.descriptorCount > capacity->descriptorCount) return {}; + } + std::lock_guard lock(mutex); + const auto allocate = context.Function("vkAllocateDescriptorSets"); + VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + allocation.descriptorSetCount = 1; + allocation.pSetLayouts = &layout; + // Newest pool first: it has the most room; a full or fragmented pool is left for its sets to + // drain and tried again later. + for (auto it = pools.rbegin(); it != pools.rend(); ++it) { + allocation.descriptorPool = *it; + VkDescriptorSet set = VK_NULL_HANDLE; + const auto result = allocate(context.device, &allocation, &set); + if (result == VK_SUCCESS) { + ++stats.sets; + return {set, *it}; + } + if (result != VK_ERROR_OUT_OF_POOL_MEMORY && result != VK_ERROR_FRAGMENTED_POOL) Check(result, "vkAllocateDescriptorSets"); + } + VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + poolInfo.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; + poolInfo.maxSets = ChainPoolSets; + poolInfo.poolSizeCount = static_cast(ChainPoolSizes.size()); + poolInfo.pPoolSizes = ChainPoolSizes.data(); + VkDescriptorPool pool = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); + pools.push_back(pool); + ++stats.pools; + allocation.descriptorPool = pool; + VkDescriptorSet set = VK_NULL_HANDLE; + Check(allocate(context.device, &allocation, &set), "vkAllocateDescriptorSets"); + ++stats.sets; + return {set, pool}; +} + +void DescriptorCache::Free(const SetAllocation& allocation) noexcept { + if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return; + std::lock_guard lock(mutex); + static_cast(freeSets(context.device, allocation.pool, 1, &allocation.set)); +} + +DescriptorCache::Stats DescriptorCache::Counters() const { + std::lock_guard lock(mutex); + return stats; +} + +std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written) { Require(words.size() == 4, "buffer descriptor must contain four DWORDs"); Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set"); const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; @@ -185,14 +1367,86 @@ std::size_t ShaderResources::addGuestBuffer(std::span words Require(byteSize <= context.limits.maxStorageBufferRange, "shader buffer exceeds descriptor range limit"); Require(byteSize <= std::numeric_limits::max(), "shader buffer size exceeds host address space"); const auto size = static_cast(byteSize); - GuestMemory::CheckRange(reinterpret_cast(address), size, 1, true); Require(target == nullptr || !overlap(address, size, target->address, target->bytes), "shader buffer aliases the render target"); + // APS5_ALL_BUFFERS_WRITTEN=1: every element is noted as written, as before bufferWritten existed. + static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr; + written = written || allWritten; + // Unconditional (also for read-only elements): Draw.cpp's CheckBufferAliases repeats this check + // on every resource-cache hit, and a draw must fail its hit exactly when it fails its build. Require(!overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer"); - guestMemory.AddWritable(address, size); - allocations.push_back({address, size, true, nullptr}); + if (written) guestMemory.AddWritable(address, size); + else { + guestMemory.AddReadable(address, size); + ++readOnlyBuffers; + } + if (BuildProfiled()) { + auto& counters = BufferWrites(); + counters.elements.fetch_add(1, std::memory_order_relaxed); + if (!written) counters.readOnly.fetch_add(1, std::memory_order_relaxed); + } + allocations.push_back({address, size, true, nullptr, ShaderRecompiler::DescriptorRole::ShaderData, written}); return allocations.size() - 1; } +std::string ShaderResources::Describe() const { + const auto sample = [](std::uint64_t address, std::uint64_t bytes) { + // Heaps bound whole can include uncommitted pages; those ranges are not sampled. + if (!GuestMemory::Accessible(reinterpret_cast(address), static_cast(bytes))) return -1.0; + std::size_t nonzero = 0; + std::size_t samples = 0; + const auto* data = reinterpret_cast(address); + const auto words = bytes / 4; + const auto step = std::max(1, words / 4096); + for (std::uint64_t i = 0; i < words; i += step, ++samples) nonzero += data[i] != 0; + return samples == 0 ? 0.0 : static_cast(nonzero) / samples; + }; + std::string text; + char line[160]; + for (const auto& range : describedRanges) { + std::snprintf(line, sizeof(line), " %s 0x%llx+0x%llx(%ux%u f%u t%d) nz=%.2f", range.kind, static_cast(range.address), static_cast(range.bytes), range.width, range.height, range.format, range.tileMode, sample(range.address, range.bytes)); + text += line; + if (range.dccAddress != 0) { + const auto keys = ReadDccKeys(range.dccAddress, range.bytes); + std::snprintf(line, sizeof(line), " dcc=%s@0x%llx", DccKeysName(keys), static_cast(range.dccAddress)); + text += line; + if (keys == DccKeys::Mixed) { + // Where the keys change: first key, how many leading keys match it, and the next key. + const auto* bytes = reinterpret_cast(range.dccAddress); + const auto count = static_cast(range.bytes / 256u); + std::size_t run = 1; + while (run < count && bytes[run] == bytes[0]) ++run; + std::snprintf(line, sizeof(line), "(%02x x%zu then %02x of %zu)", bytes[0], run, run < count ? bytes[run] : 0u, count); + text += line; + } + } + } + // APS5_TRACE_DISPATCH_IO=2 also lists the words of small buffers (constants). + static const bool words = [] { const char* value = std::getenv("APS5_TRACE_DISPATCH_IO"); return value != nullptr && value[0] == '2'; }(); + const auto appendWords = [&](const std::uint32_t* data, std::size_t bytes) { + if (!words || bytes > 0x200) return; + text += " ["; + for (std::size_t i = 0; i < bytes / 4; ++i) { + char word[12]; + std::snprintf(word, sizeof(word), "%s%08x", i == 0 ? "" : " ", data[i]); + text += word; + } + text += "]"; + }; + for (const auto& allocation : allocations) { + if (allocation.guest) { + std::snprintf(line, sizeof(line), " buffer%s 0x%llx+0x%zx nz=%.2f", allocation.written ? "" : "(ro)", static_cast(allocation.address), allocation.size, sample(allocation.address, allocation.size)); + text += line; + if (GuestMemory::Accessible(reinterpret_cast(allocation.address), allocation.size)) appendWords(reinterpret_cast(allocation.address), allocation.size); + } else if (allocation.buffer) { + const auto bytes = allocation.buffer->Bytes(); + std::snprintf(line, sizeof(line), " data+0x%zx nz=%.2f", allocation.size, sample(reinterpret_cast(bytes.data()), allocation.size)); + text += line; + appendWords(reinterpret_cast(bytes.data()), allocation.size); + } + } + return text; +} + std::size_t ShaderResources::addDataBuffer(std::span words) { const auto size = words.size() * sizeof(std::uint32_t); Require(size <= context.limits.maxStorageBufferRange, "shader data buffer exceeds descriptor range limit"); @@ -202,8 +1456,19 @@ std::size_t ShaderResources::addDataBuffer(std::span words) return allocations.size() - 1; } -void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags, std::vector& bindings) { +void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags) { Require(binding.count != 0, "empty descriptor binding"); + if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { + // Storage images are guest textures the shader writes (looked up in stage B). + Require(binding.role == ShaderRecompiler::DescriptorRole::GuestImages, "storage image binding has a non-image role"); + Require(binding.guestDescriptor.size() == static_cast(binding.count) * 8u, "guest storage image descriptors must contain 8 dwords each"); + Require(context.detiler != nullptr, "device texture detiler is unavailable"); + Require(binding.count <= context.limits.maxPerStageDescriptorStorageImages, "shader storage-image descriptors exceed per-stage limits"); + bindings.push_back({{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, binding.count, flags, nullptr}, {}, {}}); + plannedStorageImages += binding.count; + deferredImages.push_back({&binding, bindings.size() - 1}); + return; + } const bool sampledImage = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; const bool samplerKind = binding.kind == ShaderRecompiler::DescriptorKind::Sampler; Require(sampledImage || samplerKind, std::string("unsupported descriptor kind ") + kindName(binding.kind) + " for role " + roleName(binding.role)); @@ -219,31 +1484,243 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& Require(context.detiler != nullptr, "device texture detiler is unavailable"); Require(context.textureCache != nullptr, "device texture cache is unavailable"); Require(binding.count <= context.limits.maxPerStageDescriptorSampledImages, "shader sampled-image descriptors exceed per-stage limits"); - for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); - const auto resource = DecodeTextureResource(words); - Require(MatchesGuestDimension(*binding.imageShape, resource.dimension), "guest texture dimension disagrees with the shader's declared image shape"); - const VkComponentMapping components{ComponentSwizzleFor(resource.dstSelX), ComponentSwizzleFor(resource.dstSelY), ComponentSwizzleFor(resource.dstSelZ), ComponentSwizzleFor(resource.dstSelW)}; - textures.push_back(context.textureCache->Get(words, resource, components)); - item.imageAllocations.push_back(textures.size() - 1); - } - Require(textures.size() <= context.limits.maxDescriptorSetSampledImages, "pipeline sampled-image descriptors exceed device limits"); + plannedSampledImages += binding.count; + Require(plannedSampledImages <= context.limits.maxDescriptorSetSampledImages, "pipeline sampled-image descriptors exceed device limits"); } else { Require(elementWords == 4, "guest sampler descriptor must contain 4 dwords"); Require(binding.count <= context.limits.maxPerStageDescriptorSamplers, "shader sampler descriptors exceed per-stage limits"); Require(binding.samplerDepthCompare.size() == binding.count, "guest sampler binding is missing depth comparison metadata"); for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); - auto resource = DecodeSamplerResource(words); - resource.compareEnable = binding.samplerDepthCompare.at(element); - if (!context.samplerCache) context.samplerCache = std::make_shared(); - samplers.push_back(context.samplerCache->Get(context, words, resource)); + const bool compareEnable = binding.samplerDepthCompare.at(element); + static const bool noSamplerCache = std::getenv("APS5_NO_SAMPLER_CACHE") != nullptr; + if (context.samplerCache != nullptr && !noSamplerCache) { + samplers.push_back(context.samplerCache->Get(context, words, compareEnable)); + } else { + auto resource = DecodeSamplerResource(words); + resource.compareEnable = compareEnable; + samplers.push_back(std::make_shared(context, resource)); + } item.imageAllocations.push_back(samplers.size() - 1); } Require(samplers.size() <= context.limits.maxDescriptorSetSamplers, "pipeline sampler descriptors exceed device limits"); } bindings.push_back(std::move(item)); + if (sampledImage) deferredImages.push_back({&binding, bindings.size() - 1}); +} + +namespace { + +// Calls `visit(binding, element, words)` for every sampled and storage image element of a compiled +// shader's bindings, in plan order (the order addImageBinding and resolveImageBinding walk). +template +void forEachImageElement(const ShaderRecompiler::RecompileResult& program, Visit&& visit) { + for (const auto& binding : program.bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.count == 0) continue; + if (binding.kind != ShaderRecompiler::DescriptorKind::SampledImage && binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) continue; + const auto elementWords = binding.guestDescriptor.size() / binding.count; + for (std::uint32_t element = 0; element < binding.count; ++element) visit(binding, element, std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords)); + } +} + +} + +bool ShaderResources::precollectImages() { + // The image lookups of stage B (cachedTexture, StorageTexture::Refresh) each start with a + // GuestMemory::CollectWrites of their surface, the GetWriteWatch walk that is most of the + // lookups' time under the lock. The walk is memoized per thread epoch, and the worker thread + // passes no ordering point between the stages (see GuestMemory::BumpCollectEpoch), so walking + // every surface here makes stage B's collects memo hits: the walk leaves the lock, the lookups' + // checks stay where they were. A memo miss (the ring overflowed, an uncommitted page) just walks + // under the lock as before; a descriptor stage B rejects is left to it. APS5_NO_PRECOLLECT=1 + // disables the pass. + // The pass also leaves one ImageRecord per element: the decode and the surface description are + // made once for the build, and a sampled element's cache entry is taken here (a hash lookup + // under the cache's own mutex) so that stage B, under the device lock, runs only the + // fastRevalidate predicate on it (see fastTexture). The collect comes before the entry is read, + // as the predicate's rule demands (the generation the entry moves to must predate the checks). + static const bool disabled = std::getenv("APS5_NO_PRECOLLECT") != nullptr; + static const bool noRecords = std::getenv("APS5_NO_STAGE_A_IMAGES") != nullptr; + if (disabled || deferredImages.empty()) return false; + imageRecords.clear(); + nextImageRecord = 0; + auto& counters = TextureCounts(); + for (const auto& deferred : deferredImages) { + const auto& binding = *deferred.binding; + const auto elementWords = binding.guestDescriptor.size() / binding.count; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + ImageRecord record; + record.sampled = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; + try { + record.resource = DecodeTextureResource(words); + record.guestBytes = DescribeSurface(record.resource).guestBytes; + record.generation = GuestMemory::CollectWrites(record.resource.baseAddress, static_cast(record.guestBytes)); + record.decoded = true; + if (record.sampled && !noRecords && words.size() == 8) { + std::copy(words.begin(), words.end(), record.words.begin()); + record.components = {ComponentSwizzleFor(record.resource.dstSelX), ComponentSwizzleFor(record.resource.dstSelY), ComponentSwizzleFor(record.resource.dstSelZ), ComponentSwizzleFor(record.resource.dstSelW)}; + record.keys = TextureClearKeys(record.resource, record.guestBytes); + auto& cache = Textures(); + std::lock_guard lock(cache.mutex); + if (const auto it = findTexture(cache, MakeTextureKey(context.device, words, record.components)); it != cache.entries.end()) { + record.texture = it->texture; + record.source = it->source; + record.entryKeys = it->keys; + record.entryGeneration = it->generation; + counters.records.fetch_add(1, std::memory_order_relaxed); + } + } + } catch (const std::exception&) { + record.decoded = false; + } + imageRecords.push_back(std::move(record)); + } + } + return true; +} + +std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) { + if (record.texture == nullptr) return nullptr; + // Exactly the "unchanged" branches of cachedTexture, from write stamps, the pending-results + // registry and the DCC keys, the way fastRevalidate proves a built object current: the collect + // first (a memo hit: stage A walked the range), then no other image may have results pending + // over the memory (a lookup would flush them into it, or view them instead), a storage-sourced + // view needs its image to be the cache's image of the surface and current with guest memory, + // a snapshot needs the memory unchanged since its content matched, and the keys must be what + // the content was made under whenever the surface has any (re-read under the lock: a flush + // between the stages marks them uncompressed). + const auto address = record.resource.baseAddress; + const auto bytes = static_cast(record.guestBytes); + if (GuestMemory::CollectWrites(address, bytes) == 0) return nullptr; + if (PendingStorageOverlaps(address, bytes, record.source.get())) return nullptr; + auto keys = record.keys; + if (record.resource.dccAddress != 0) keys = TextureClearKeys(record.resource, record.guestBytes); + // The keys the entry was made under (a view of a pending render target was made under its clear + // keys and stays valid while they are unchanged); a change is the full lookup's to judge. + if (keys != record.entryKeys) return nullptr; + if (record.source != nullptr) { + if (!StorageImageCached(context, record.source.get()) || !GuestMemory::UnchangedSince(address, bytes, record.source->Generation())) return nullptr; + // Under fast-clear keys the view holds only while its image's results are still pending over + // the surface; flushed, the clear the image cannot see makes the lookup take a snapshot. + if (keys != DccKeys::Uncompressed && StorageTexture::FindPending(address, record.guestBytes) != record.source) return nullptr; + } else if (keys == DccKeys::Uncompressed && !GuestMemory::UnchangedSince(address, bytes, record.entryGeneration)) { + return nullptr; + } + // The entry must still be the cache's, holding this object (an evicted object is not reused: the + // next lookup would make another), and it takes the stage-A generation like a hit would. + auto& cache = Textures(); + std::lock_guard lock(cache.mutex); + const auto it = findTexture(cache, MakeTextureKey(context.device, record.words, record.components)); + if (it == cache.entries.end() || it->texture != record.texture) return nullptr; + if (it->source == nullptr) it->generation = record.generation; + touchTexture(cache, it); + logLookup({record.texture.get(), record.resource, record.guestBytes, keys, record.source != nullptr ? 0 : record.generation, record.source.get()}); + return record.texture; +} + +void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item) { + auto& counters = TextureCounts(); + // The element's stage-A record, when the pass ran (records follow the plan order exactly). + const auto nextRecord = [&]() -> const ImageRecord* { return nextImageRecord < imageRecords.size() ? &imageRecords[nextImageRecord++] : nullptr; }; + if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { + const auto elementWords = binding.guestDescriptor.size() / binding.count; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + const auto* record = nextRecord(); + const auto resource = record != nullptr && record->decoded ? record->resource : DecodeTextureResource(words); + Require(MatchesGuestDimension(*binding.imageShape, resource.dimension), "guest texture dimension disagrees with the shader's declared image shape"); + const VkComponentMapping components{ComponentSwizzleFor(resource.dstSelX), ComponentSwizzleFor(resource.dstSelY), ComponentSwizzleFor(resource.dstSelZ), ComponentSwizzleFor(resource.dstSelW)}; + const auto guestBytes = record != nullptr && record->decoded ? record->guestBytes : DescribeSurface(resource).guestBytes; + std::shared_ptr texture; + if (record != nullptr && record->texture != nullptr) { + texture = fastTexture(*record); + (texture != nullptr ? counters.fastHits : counters.fastMisses).fetch_add(1, std::memory_order_relaxed); + } + if (texture == nullptr) texture = cachedTexture(context, words, resource, components, guestBytes); + textures.push_back(std::move(texture)); + describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress}); + item.imageAllocations.push_back(textures.size() - 1); + } + // The line is due even when every element took the fast path (cachedTexture reports too). + reportTextureCounters(); + return; + } + // Consecutive identical storage descriptors address successive mips of one texture (dynamic-mip + // storage writes). + std::uint32_t mipOffset = 0; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + const bool sameAsPrevious = SameAsPreviousStorageElement(binding, element); + if (sameAsPrevious) ++mipOffset; + else mipOffset = 0; + const auto* record = nextRecord(); + const auto resource = record != nullptr && record->decoded ? record->resource : DecodeTextureResource(words); + if (binding.imageShape.has_value()) Require(MatchesGuestDimension(*binding.imageShape, resource.dimension), "guest storage texture dimension disagrees with the shader's declared image shape"); + const auto mip = std::min(resource.baseLevel + mipOffset, resource.mipCount - 1u); + const auto guestBytes = record != nullptr && record->decoded ? record->guestBytes : DescribeSurface(resource).guestBytes; + // The same surface as the previous element: its image was just looked up and refreshed. + if (sameAsPrevious && StorageDedupeEnabled()) storageTextures.push_back(storageTextures.back()); + else storageTextures.push_back(cachedStorageTexture(context, words, resource, mip, guestBytes)); + storageMips.push_back(mip); + // Images the shader only reads have nothing to store back. + storageWritten.push_back(element >= binding.imageWritten.size() || binding.imageWritten[element]); + describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress}); + item.imageAllocations.push_back(storageTextures.size() - 1); + } +} + +std::vector> ShaderResources::PresyncSurfaces() const { + std::vector> surfaces; + // A surface's lookup reads guest memory on the CPU unless it is served GPU-direct from a host + // import: a storage image of imported memory uploads and refreshes from the import, and a + // sampled texture of one views that image (SampledFromStorageEligible), so neither waits for + // the producer. + const auto consider = [&](std::uint32_t format, std::uint64_t address, std::uint64_t guestBytes, bool sampled) { + const bool gpuDirect = sampled ? SampledFromStorageEligible(context, format, address, guestBytes) : HostImportCovers(context, address, static_cast(guestBytes)); + if (!gpuDirect) surfaces.emplace_back(address, guestBytes); + }; + if (completed) { + // A cached object (the resource cache): Revalidate repeats the lookups of the objects the + // build made, which never change afterwards (a lookup returning another object fails the + // revalidation instead), so the surfaces come from the build's own record of them. A + // sampled texture viewing a storage image refreshes like that image; a snapshot texture + // over an eligible surface is replaced by a view without reading. Storage images are + // checked on their own surface (a mip chain can exceed the element's descriptor). Read + // without the device lock: these members are fixed once the build completed. + std::size_t textureIndex = 0; + for (const auto& range : describedRanges) { + if (std::strcmp(range.kind, "texture") == 0 && textureIndex < textures.size()) { + const auto& texture = textures[textureIndex++]; + if (texture->ViewsStorageImage()) consider(range.format, range.address, range.bytes, false); + else consider(range.format, range.address, range.bytes, true); + } + } + for (std::size_t index = 0; index < storageTextures.size(); ++index) { + if (index != 0 && storageTextures[index] == storageTextures[index - 1]) continue; + const auto& image = *storageTextures[index]; + consider(image.Descriptor().format, image.Descriptor().baseAddress, image.GuestBytes(), false); + } + return surfaces; + } + if (!imageRecords.empty()) { + for (const auto& record : imageRecords) { + if (record.decoded) consider(record.resource.format, record.resource.baseAddress, record.guestBytes, record.sampled); + } + return surfaces; + } + // Stage A has not run (an address-based build, or the pass is off): decoded from the shader. + if (deferredCompute.program == nullptr) return surfaces; + forEachImageElement(*deferredCompute.program, [&](const ShaderRecompiler::DescriptorBinding& binding, std::uint32_t, std::span words) { + try { + const auto resource = DecodeTextureResource(words); + consider(resource.format, resource.baseAddress, DescribeSurface(resource).guestBytes, binding.kind == ShaderRecompiler::DescriptorKind::SampledImage); + } catch (const std::exception&) { + // Stage B reports the bad descriptor. + } + }); + return surfaces; } ShaderResources::~ShaderResources() { @@ -251,9 +1728,14 @@ ShaderResources::~ShaderResources() { } void ShaderResources::release() noexcept { - if (descriptors) context.descriptorCache->Put(std::move(descriptors)); - _layout = VK_NULL_HANDLE; + // A set from the cache's pool chain goes back to it; a dedicated pool dies with its set. + if (cachePool && context.descriptorCache != nullptr) context.descriptorCache->Free({_set, cachePool}); + if (pool) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + if (_layout && ownsLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, _layout, nullptr); + cachePool = VK_NULL_HANDLE; + pool = VK_NULL_HANDLE; _set = VK_NULL_HANDLE; + _layout = VK_NULL_HANDLE; } VkDescriptorSetLayout ShaderResources::Layout() const { @@ -265,8 +1747,51 @@ void ShaderResources::Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoi context.Function("vkCmdBindDescriptorSets")(commands, bindPoint, layout, 0, 1, &_set, 0, nullptr); } +namespace { +// Debug aid: APS5_GPU_NO_WRITEBACK=1 keeps GPU results out of guest memory. +bool SkipWriteBack() { + static const bool skip = std::getenv("APS5_GPU_NO_WRITEBACK") != nullptr; + return skip; +} +} + void ShaderResources::WriteBack() { + WriteBackBuffers(); + if (SkipWriteBack()) return; + for (std::size_t index = 0; index < storageTextures.size(); ++index) { + if (storageWritten[index]) storageTextures[index]->MarkDirty(); + } +} + +void ShaderResources::MarkGpuWrites(Recorder& recorder) { + if (SkipWriteBack()) return; + for (std::size_t index = 0; index < storageTextures.size(); ++index) { + if (storageWritten[index]) storageTextures[index]->MarkDirty(); + } + // Written sub-ranges of buffers the GPU copied out of a host import go back into it by the GPU, + // recorded here after the work: those regions then need no CPU write-back (HasCopiedWrites), + // and the note and mark below cover them like direct writes. + guestMemory.RecordCopyBacks(recorder); + // Only the written elements' ranges (AddWritable): a read-only element is neither noted here + // nor marked as a direct write, so CPU reads of its memory never wait for this work. + recorder.NotePendingWrites(guestMemory.Writes()); + guestMemory.MarkDirectWrites(); + if (!BuildProfiled()) return; + auto& counters = BufferWrites(); + // Recorded uses only (dispatches and recorded draws): a synchronous draw writes back in + // WriteBack() and publishes no pending writes, so it has no notes to skip. + counters.notesSkipped.fetch_add(readOnlyBuffers, std::memory_order_relaxed); + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(); + if (nowMs - last < 10000 || !counters.lastReport.compare_exchange_strong(last, nowMs)) return; + const auto elements = counters.elements.load(); + const auto readOnly = counters.readOnly.load(); + std::fprintf(stderr, "[buffers] descriptor elements bound: %llu total, %llu written, %llu read-only; %llu pending-write notes skipped\n", static_cast(elements), static_cast(elements - readOnly), static_cast(readOnly), static_cast(counters.notesSkipped.load())); +} + +void ShaderResources::WriteBackBuffers() { if (bda) bda->CheckFault(); + if (SkipWriteBack()) return; guestMemory.WriteBack(); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index e12e886ab3..1eb231c1aa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -1,7 +1,6 @@ #include "BdaAbi.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include #include #include @@ -9,8 +8,6 @@ #include #include #include -#include -#include namespace AgcDriver::Graphics { namespace { @@ -205,13 +202,26 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi mesh && capability == spv::CapabilityMeshShadingEXT; + // Enabled unconditionally or by the device setup in VulkanDevice. + const bool isFeatureCapability = + capability == spv::CapabilityImageGatherExtended || + capability == spv::CapabilityImageQuery || + capability == spv::CapabilityStorageImageWriteWithoutFormat || + capability == spv::CapabilityStorageImageReadWithoutFormat || + capability == spv::CapabilityInt64 || + capability == spv::CapabilityInt16 || + capability == spv::CapabilityFloat16 || + capability == spv::CapabilityStorageBuffer8BitAccess || + capability == spv::CapabilityPhysicalStorageBufferAddresses; + Require( isBaseCapability || isBarycentricCapability || isSubgroupCapability || isBdaCapability || isTessellationCapability || - isMeshCapability, + isMeshCapability || + isFeatureCapability, std::string("SPIR-V requires unsupported device capability ") + std::to_string(static_cast(capability))); @@ -242,11 +252,10 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi case spv::OpSpecConstantOp: throw std::runtime_error("AGC graphics: unsupported SPIR-V extension, grouped decoration or specialization constant"); case spv::OpMemoryModel: { - const auto addressingModel = shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && !rectListControl - ? spv::AddressingModelPhysicalStorageBuffer64 - : spv::AddressingModelLogical; + // Address-based shaders use PhysicalStorageBuffer64; everything else (including the + // generated rect-list stages) is Logical. Require(count == 3, "invalid SPIR-V OpMemoryModel word count: " + std::to_string(count)); - Require(instruction[1] == addressingModel, "unsupported SPIR-V addressing model: " + std::to_string(instruction[1])); + Require(instruction[1] == spv::AddressingModelPhysicalStorageBuffer64 || instruction[1] == spv::AddressingModelLogical, "unsupported SPIR-V addressing model: " + std::to_string(instruction[1])); Require(instruction[2] == spv::MemoryModelGLSL450, "unsupported SPIR-V memory model: " + std::to_string(instruction[2])); ++memoryModels; break; @@ -417,7 +426,6 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto binding = std::find_if(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return item.descriptorSet == key.first && item.binding == key.second; }); Require(binding != shader.bindings.end(), "SPIR-V resource is absent from recompiler binding metadata"); Require(binding->kind == ShaderRecompiler::DescriptorKind::Sampler || binding->kind == ShaderRecompiler::DescriptorKind::SampledImage || binding->kind == ShaderRecompiler::DescriptorKind::StorageImage, "SPIR-V descriptor type disagrees with recompiler binding metadata"); - Require(binding->kind != ShaderRecompiler::DescriptorKind::StorageImage, "storage image resources are not implemented"); } else { Require(variable.storage == spv::StorageClassStorageBuffer && decoration.set && decoration.binding, "unsupported or unbound shader resource"); const auto key = std::make_pair(*decoration.set, *decoration.binding); @@ -466,77 +474,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi } -namespace { - -struct ValidatedInterface { - std::map inputs; - std::map outputs; -}; - -std::shared_ptr inspectCached(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric) { - PerformanceTimer timing("Graphics.ShaderValidation"); - Require(compiled.program != nullptr, "missing compiled shader"); - const auto& shader = *compiled.program; - std::string key; - const auto append = [&](const TValue& value) { - static_assert(std::is_trivially_copyable_v); - key.append(reinterpret_cast(&value), sizeof(value)); - }; - append(compiled.stage); - append(state.rectList); - append(state.stages.mesh.has_value()); - if (state.stages.mesh) { - append(state.stages.mesh->threadsPerGroup); - append(state.stages.mesh->maxVertices); - append(state.stages.mesh->maxPrimitives); - } - append(state.stages.tessellation.has_value()); - if (state.stages.tessellation) { - append(state.stages.tessellation->inputControlPoints); - append(state.stages.tessellation->outputControlPoints); - } - append(subgroup.supportedStages); - append(subgroup.supportedOperations); - append(fragmentShaderBarycentric); - append(shader.bdaAbiVersion); - append(shader.pushConstants.empty()); - append(shader.bindings.size()); - for (const auto& binding : shader.bindings) { - append(binding.kind); - append(binding.role); - append(binding.descriptorSet); - append(binding.binding); - append(binding.count); - append(binding.readOnly); - append(binding.guestDescriptor.empty()); - } - append(shader.vertexAttributes.size()); - for (const auto& attribute : shader.vertexAttributes) { - append(attribute.location); - append(attribute.components); - append(attribute.resource.fields[3]); - } - append(shader.spirv.size()); - const auto code = std::as_bytes(std::span(shader.spirv)); - if (!code.empty()) key.append(reinterpret_cast(code.data()), code.size()); - timing.Mark("key", key.size()); - static thread_local std::map> cache; - const auto found = cache.find(key); - if (found != cache.end()) { - timing.Mark("hit"); - return found->second; - } - auto module = Inspect(compiled, state, subgroup, fragmentShaderBarycentric); - auto result = std::make_shared(ValidatedInterface{std::move(module.inputs), std::move(module.outputs)}); - if (cache.size() >= 128) cache.clear(); - cache.emplace(std::move(key), result); - timing.Mark("inspect"); - return result; -} - -} - -void ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric) { +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric) { using Stage = ShaderRecompiler::ShaderStage; const bool tessellation = state.stages.path == ShaderPath::Tessellation; const bool mesh = state.stages.path == ShaderPath::Geometry; @@ -546,22 +484,30 @@ void ValidateShaders(std::span shaders, const State& state Require(shaders.size() == (tessellation || state.rectList ? 4u : 2u), "incorrect graphics stage count"); const std::array tessStages{Stage::Local, Stage::TessellationControl, Stage::TessellationEvaluation, Stage::Fragment}; static_cast(AssemblePushConstants(shaders)); - std::shared_ptr previous; + Module previous; for (std::size_t i = 0; i < shaders.size(); ++i) { const auto expected = state.rectList ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : i == 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex; Require(shaders[i].program != nullptr, "missing compiled shader"); Require(shaders[i].stage == expected, "graphics stage order disagrees"); for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == 0, "graphics resource uses a descriptor set other than zero"); - const auto current = inspectCached(shaders[i], state, subgroup, fragmentShaderBarycentric); + const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric); if (i != 0) { - for (const auto& [location, signature] : current->inputs) { - const auto output = previous->outputs.find(location); - Require(output != previous->outputs.end() && output->second == signature, "graphics interfaces disagree at location " + std::to_string(location)); + for (const auto& [location, signature] : current.inputs) { + const auto output = previous.outputs.find(location); + Require(output != previous.outputs.end() && output->second == signature, "graphics interfaces disagree at location " + std::to_string(location)); } } previous = current; } - Require(previous->outputs.size() == 1 && previous->outputs.contains(0) && previous->outputs.at(0) == "vertex:f32x4", "fragment shader must export one float4 color at location zero"); + // One float4 color per MRT attachment. A pixel shader may also export no color at all when it + // writes storage images or buffers instead; its attachments are then left untouched. + const auto attachments = std::max(state.colors.size(), 1u); + std::set locations; + for (const auto& [location, signature] : previous.outputs) { + Require(location < attachments && signature == "vertex:f32x4", "fragment shader must export float4 colors at locations below the attachment count"); + locations.insert(location); + } + return locations; } void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index c759c5dfbe..5b71e9fae4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -1,4 +1,7 @@ +#include #include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/General.hpp" #include @@ -27,7 +30,12 @@ float readFloat(const Registers& registers, std::uint32_t offset) { } void zero(const Registers& registers, std::uint32_t offset, std::uint32_t mask, const char* name, const char* bank = "context") { - Require((read(registers, offset, bank) & mask) == 0, std::string(name) + " is unsupported"); + const auto value = read(registers, offset, bank); + if ((value & mask) != 0) { + char detail[64]; + std::snprintf(detail, sizeof(detail), " (register 0x%x = 0x%08x)", offset, value); + throw std::runtime_error(std::string(name) + " is unsupported" + detail); + } } VkBlendFactor blendFactor(std::uint32_t value) { @@ -62,6 +70,82 @@ VkBlendOp blendOp(std::uint32_t value) { } } +struct DecodedColorFormat { + VkFormat format; + std::uint32_t elementBytes; +}; + +// CB_COLOR_INFO FORMAT / NUMBER_TYPE / COMP_SWAP to a Vulkan attachment format. AMD formats list +// components from the least significant bits, as Vulkan's non-packed formats do. +DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, std::uint32_t swap) { + constexpr std::uint32_t unorm = 0, sint = 5, srgb = 6, floating = 7; + constexpr std::uint32_t uint = 4; + const auto fail = [&]() -> DecodedColorFormat { + throw std::runtime_error("AGC graphics: unsupported color format " + std::to_string(format) + " number type " + std::to_string(number) + " component swap " + std::to_string(swap)); + }; + const bool alternate = swap == 1; + if (swap > 1) return fail(); + switch (format) { + case 1: + if (swap != 0) return fail(); + if (number == unorm) return {VK_FORMAT_R8_UNORM, 1}; + if (number == uint) return {VK_FORMAT_R8_UINT, 1}; + return fail(); + case 2: + if (swap != 0) return fail(); + if (number == unorm) return {VK_FORMAT_R16_UNORM, 2}; + if (number == floating) return {VK_FORMAT_R16_SFLOAT, 2}; + if (number == uint) return {VK_FORMAT_R16_UINT, 2}; + return fail(); + case 3: + if (swap != 0) return fail(); + if (number == unorm) return {VK_FORMAT_R8G8_UNORM, 2}; + return fail(); + case 4: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R32_SFLOAT, 4}; + if (number == uint) return {VK_FORMAT_R32_UINT, 4}; + if (number == sint) return {VK_FORMAT_R32_SINT, 4}; + return fail(); + case 5: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R16G16_SFLOAT, 4}; + if (number == unorm) return {VK_FORMAT_R16G16_UNORM, 4}; + if (number == uint) return {VK_FORMAT_R16G16_UINT, 4}; + return fail(); + case 6: + // COLOR_10_11_11: red in the low 11 bits, the Vulkan B10G11R11 packing. + if (swap != 0 || number != floating) return fail(); + return {VK_FORMAT_B10G11R11_UFLOAT_PACK32, 4}; + case 9: + // COLOR_2_10_10_10 keeps red in the low bits, the Vulkan A2B10G10R10 packing. + if (number != unorm) return fail(); + return {alternate ? VK_FORMAT_A2R10G10B10_UNORM_PACK32 : VK_FORMAT_A2B10G10R10_UNORM_PACK32, 4}; + case 10: + if (number == unorm) return {alternate ? VK_FORMAT_B8G8R8A8_UNORM : VK_FORMAT_R8G8B8A8_UNORM, 4}; + if (number == srgb) return {alternate ? VK_FORMAT_B8G8R8A8_SRGB : VK_FORMAT_R8G8B8A8_SRGB, 4}; + return fail(); + case 11: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R32G32_SFLOAT, 8}; + if (number == uint) return {VK_FORMAT_R32G32_UINT, 8}; + return fail(); + case 12: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R16G16B16A16_SFLOAT, 8}; + if (number == unorm) return {VK_FORMAT_R16G16B16A16_UNORM, 8}; + if (number == uint) return {VK_FORMAT_R16G16B16A16_UINT, 8}; + return fail(); + case 14: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R32G32B32A32_SFLOAT, 16}; + if (number == uint) return {VK_FORMAT_R32G32B32A32_UINT, 16}; + return fail(); + default: + return fail(); + } +} + void intersect(VkRect2D& result, const Registers& registers, std::uint32_t offset, bool screen) { const auto tl = read(registers, offset); const auto br = read(registers, offset + 1); @@ -95,6 +179,7 @@ ShaderStages DecodeShaderStages(const QueueState& queue) { validate(!tessellation || !geometry, "combined tessellation and geometry is unsupported by the reference path"); const auto path = tessellation ? ShaderPath::Tessellation : geometry ? ShaderPath::Geometry : ShaderPath::Vertex; ShaderStages result{path, value, (value & 0x00400000u) != 0 ? 32u : 64u, (read(queue.context, 0x1b6) & 0x8000u) != 0 ? 32u : 64u, {}, {}}; + APS5_LOG_OUT_DEBUG("DecodeShaderStages value=0x%x primitive=%u path=%u vertexWave=%u", value, primitive, static_cast(result.path), result.vertexWaveSize); if (path == ShaderPath::Vertex) { validate((value & 0x2000u) != 0, "legacy vertex routing without PRIMGEN_EN is unsupported"); validate((value & ~0x02402010u) == 0, "unsupported vertex routing, scheduling or wave-ID state"); @@ -132,6 +217,7 @@ State DecodeState(const QueueState& queue) { State result{}; result.stages = DecodeShaderStages(queue); const auto primitive = read(queue.userConfig, 0x242, "user-config"); + APS5_LOG_OUT_DEBUG("DecodeState primitive=%u path=%u vertexWave=%u", primitive, static_cast(result.stages.path), result.stages.vertexWaveSize); switch (primitive) { case 1: Require(result.stages.mesh.has_value(), "point-list vertex rendering requires point-size output support"); result.topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; break; case 2: result.topology = VK_PRIMITIVE_TOPOLOGY_LINE_LIST; break; @@ -147,10 +233,42 @@ State DecodeState(const QueueState& queue) { case 6: result.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; break; default: throw std::runtime_error("AGC graphics: unsupported primitive type " + std::to_string(primitive)); } + APS5_LOG_OUT_DEBUG("Topology=%u", static_cast(result.topology)); zero(queue.userConfig, 0x24b, ~0u, "primitive restart (GE_MULTI_PRIM_IB_RESET_EN)", "user-config"); - zero(cx, 0x207, ~0u, "clip distances, layer, viewport or auxiliary vertex exports"); - zero(cx, 0x200, ~0x007007f0u, "depth, stencil or conditional color writes"); - zero(cx, 0x203, ~0x00009870u, "depth export, shader coverage or ordered fragment execution"); + // Render target index, viewport index and the misc export vector that carries them are accepted but + // not routed: color targets are single-layer, so layered draws land in layer 0. + constexpr std::uint32_t layerExports = (1u << 18u) | (1u << 19u) | (1u << 21u) | (1u << 24u); + if ((read(cx, 0x207, "context") & layerExports) != 0) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] layer/viewport index vertex exports are ignored (PA_CL_VS_OUT_CNTL=0x%08x)\n", read(cx, 0x207, "context")); + } + } + zero(cx, 0x207, ~layerExports, "clip distances, layer, viewport or auxiliary vertex exports"); + // DB_DEPTH_CONTROL: depth and stencil tests whose function is ALWAYS and that write no depth cannot + // change color output, so they are accepted without a depth target (stencil writes are dropped). + { + const auto depthControl = read(cx, 0x200, "context"); + const bool stencil = (depthControl & 1u) != 0; + const bool depth = (depthControl & 2u) != 0; + const bool depthWrite = (depthControl & 4u) != 0; + const bool passThroughDepth = !depth || (!depthWrite && ((depthControl >> 4u) & 7u) == 7u); + const bool passThroughStencil = !stencil || (((depthControl >> 8u) & 7u) == 7u && ((depthControl & 0x80u) == 0 || ((depthControl >> 20u) & 7u) == 7u)); + if ((stencil || depth) && passThroughDepth && passThroughStencil) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] always-pass depth/stencil state is rendered without a depth target (DB_DEPTH_CONTROL=0x%08x)\n", depthControl); + } + } else { + zero(cx, 0x200, ~0x007007f0u, "depth, stencil or conditional color writes"); + } + Require((depthControl & 0xc0000008u) == 0, "depth bounds or depth-conditional color writes are unsupported"); + } + // EXEC_ON_HIER_FAIL / EXEC_ON_NOOP / EXEC_IF_OVERLAPPED (bits 9, 10, 17) only force the pixel shader + // to run, which it always does here. + zero(cx, 0x203, ~(0x00009870u | 0x00020600u), "depth export, shader coverage or ordered fragment execution"); zero(cx, 0x2dc, ~0x0001ff00u, "alpha-to-coverage"); zero(cx, 0x2f8, ~0u, "multisampling or coverage conversion"); zero(cx, 0x292, ~2u, "scan conversion mode"); @@ -168,53 +286,120 @@ State DecodeState(const QueueState& queue) { message << "AGC graphics: PA_CL_VTE_CNTL=0x" << std::hex << viewportControl << ": expected 0x43f for homogeneous positions and all viewport transforms; pre-divided coordinates, reciprocal W or disabled transforms are unsupported"; throw std::runtime_error(message.str()); } - zero(cx, 0x204, ~0x80000u, "unsupported PA_CL_CLIP_CNTL flags"); + // Bits 26/27 (ZCLIP_NEAR/FAR_DISABLE) become depth clamping; bit 19 selects the [0, 1] clip space. + zero(cx, 0x204, ~(0x80000u | 0x0c000000u), "unsupported PA_CL_CLIP_CNTL flags"); + result.depthClamp = (read(cx, 0x204) & 0x0c000000u) != 0; result.negativeOneToOne = (read(cx, 0x204) & 0x80000u) == 0; const auto raster = read(cx, 0x205); - Require((raster & ~0x7u) == 0 || (raster & ~0x7u) == 0x240u, "polygon mode, depth bias, provoking vertex or nonstandard rasterization is unsupported"); + // Bits 5-10 give the front/back polygon type (2 = filled triangles), which POLY_MODE (bit 3) turns on + // explicitly; KEEP_TOGETHER_ENABLE (bit 24) only affects primitive distribution across the chip. + const auto rasterMode = raster & ~0x7u & ~(1u << 24u); + Require(rasterMode == 0 || rasterMode == 0x240u || rasterMode == 0x248u, "polygon mode, depth bias, provoking vertex or nonstandard rasterization is unsupported"); result.cullMode = ((raster & 1u) != 0 ? VK_CULL_MODE_FRONT_BIT : 0u) | ((raster & 2u) != 0 ? VK_CULL_MODE_BACK_BIT : 0u); if (result.rectList) result.cullMode = VK_CULL_MODE_NONE; result.frontFace = (raster & 4u) != 0 ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE; - const auto targetMask = read(cx, 0x8e); + APS5_LOG_OUT_DEBUG("Raster=0x%x cullMode=0x%x frontFace=%u negativeOneToOne=%u", raster, static_cast(result.cullMode), static_cast(result.frontFace), result.negativeOneToOne ? 1u : 0u); const auto shaderMask = read(cx, 0x8f); - if ((targetMask & ~0xfu) != 0 || (shaderMask & ~0xfu) != 0) { - std::ostringstream message; - message << "AGC graphics: only color target zero is supported: CB_TARGET_MASK=0x" << std::hex << targetMask << ", CB_SHADER_MASK=0x" << shaderMask; - throw std::runtime_error(message.str()); + // Channels of targets the pixel shader does not export are never written, so the target mask only + // matters where the shader exports. + const auto targetMask = read(cx, 0x8e) & shaderMask; + APS5_LOG_OUT_DEBUG("CB_TARGET_MASK=0x%x CB_SHADER_MASK=0x%x", targetMask, shaderMask); + // MRT slots 0..n-1 become attachments 0..n-1; a slot between written slots must be written too. + std::uint32_t slotCount = 0; + for (std::uint32_t slot = 0; slot < 8; ++slot) { + if (((targetMask >> (4u * slot)) & 0xfu) != 0) slotCount = slot + 1; } - result.hasColorTarget = targetMask != 0; - Require(!result.hasColorTarget || shaderMask == 0xfu, "partial shader color exports are unsupported"); - Require(read(cx, 0x202) == 0xcc0010u, "only normal color rendering with copy ROP is supported"); + for (std::uint32_t slot = 0; slot < slotCount; ++slot) { + if (((targetMask >> (4u * slot)) & 0xfu) == 0) { + std::ostringstream message; + message << "AGC graphics: color targets with gaps are unsupported: CB_TARGET_MASK=0x" << std::hex << targetMask << ", CB_SHADER_MASK=0x" << shaderMask; + throw std::runtime_error(message.str()); + } + } + result.hasColorTarget = slotCount != 0; + APS5_LOG_OUT_DEBUG("hasColorTarget=%u slots=%u", result.hasColorTarget ? 1u : 0u, slotCount); + + // CB_COLOR_CONTROL mode 0 disables color writes, which only matters when a target is written. + Require(read(cx, 0x202) == 0xcc0010u || (!result.hasColorTarget && (read(cx, 0x202) & ~0x70u) == 0xcc0000u), "only normal color rendering with copy ROP is supported"); zero(cx, 0x1c4, ~0u, "depth or sample-mask export"); const auto exportFormat = read(cx, 0x1c5); - Require(exportFormat == 4 || exportFormat == 9, "only FP16_ABGR or 32_ABGR color export is supported"); - Require(read(cx, 0x1c3) == 4, "additional position exports are unsupported"); - if (result.hasColorTarget) { - const auto info = read(cx, 0x31c); + APS5_LOG_OUT_DEBUG("Export format=%u", exportFormat); + // SPI_SHADER_POS_FORMAT: POS0 must be a 4-component position; later vectors carry the misc/clip + // exports that PA_CL_VS_OUT_CNTL validation above already limits to ignored layer/viewport data. + Require((read(cx, 0x1c3) & 0xfu) == 4, "additional position exports are unsupported"); + for (std::uint32_t slot = 0; slot < slotCount; ++slot) { + // Export formats only matter for the targets the draw writes. + const auto slotExport = (exportFormat >> (4u * slot)) & 0xfu; + Require(slotExport == 4 || slotExport == 9, "only FP16_ABGR or 32_ABGR color export is supported"); + const auto stride = slot * 0xfu; + ColorTarget color{}; + const auto info = read(cx, 0x31c + stride); const auto number = (info >> 8u) & 7u; const auto swap = (info >> 11u) & 3u; - Require(((info >> 2u) & 0x1fu) == 10 && (number == 0 || number == 6) && swap <= 1, "unsupported color format or component order"); - Require((info & ~0x00029f7cu) == 0, "color compression, DCC, endian conversion, nonstandard rounding or color optimization is unsupported"); - Require((info & 0x8000u) != 0, "unclamped normalized color is unsupported"); - zero(cx, 0x31b, ~0u, "color mip or array view"); - zero(cx, 0x31d, ~0u, "color samples, fragments or destination alpha override"); - const auto attrib2 = read(cx, 0x3b0); - Require((attrib2 >> 28u) == 0, "mipmapped render targets are unsupported"); - const auto attrib3 = read(cx, 0x3b8); - result.color.tileMode = DecodeColorTileMode(attrib3); - result.color.extent = {((attrib2 >> 14u) & 0x3fffu) + 1u, (attrib2 & 0x3fffu) + 1u}; - const ColorTargetLayout colorLayout(result.color.extent.width, result.color.extent.height, result.color.tileMode); - const auto high = read(cx, 0x390); + APS5_LOG_OUT_DEBUG("Color %u info=0x%x number=%u swap=%u", slot, info, number, swap); + const auto format = (info >> 2u) & 0x1fu; + const auto decoded = DecodeColorFormat(format, number, swap); + // ROUND_MODE (bit 18) only affects unorm rounding. With DCC_ENABLE (bit 28) the target is written + // uncompressed and only its fast-clear keys matter (see DccMetadata.hpp). + Require((info & ~(0x00029f7cu | 0x00040000u | 0x10000000u)) == 0, "color compression, DCC, endian conversion, nonstandard rounding or color optimization is unsupported"); + Require((info & 0x8000u) != 0 || number == 7, "unclamped normalized color is unsupported"); + // CB_COLOR_VIEW: MIP_LEVEL (bits 24-27) selects the rendered mip; array slices are not modeled. + const auto view = read(cx, 0x31b + stride); + Require((view & ~0x0f000000u) == 0, "color array views are unsupported"); + const auto viewMip = (view >> 24u) & 0xfu; + zero(cx, 0x31d + stride, ~0u, "color samples, fragments or destination alpha override"); + const auto attrib2 = read(cx, 0x3b0 + slot); + const auto maxMip = attrib2 >> 28u; + Require(viewMip <= maxMip, "color view mip exceeds the surface"); + const auto attrib3 = read(cx, 0x3b8 + slot); + color.tileMode = DecodeColorTileMode(attrib3); + color.extent = {((attrib2 >> 14u) & 0x3fffu) + 1u, (attrib2 & 0x3fffu) + 1u}; + color.elementBytes = decoded.elementBytes; + std::uint64_t mipOffset = 0; + if (maxMip != 0) { + // A mipmapped surface is addressed like a texture; the view renders into one mip of it. + const auto mips = ComputeElementMipLayout(color.tileMode == ColorTileMode::Linear ? TextureTileMode::kLinear : TextureTileMode::kR64KBX, color.elementBytes, color.extent.width, color.extent.height, maxMip + 1u); + const auto& mip = mips.at(viewMip); + Require(!mip.tail, "rendering into a packed mip tail is unsupported"); + mipOffset = mip.tiledOffset; + color.extent = {mip.width, mip.height}; + } + const ColorTargetLayout colorLayout(color.extent.width, color.extent.height, color.tileMode, color.elementBytes); + const auto high = read(cx, 0x390 + slot); Require((high & ~0xffu) == 0, "invalid color address extension"); - result.color.address = (static_cast(high) << 40u) | (static_cast(read(cx, 0x318)) << 8u); - result.color.bytes = colorLayout.Bytes(); - GuestMemory::CheckGpuRange(reinterpret_cast(result.color.address), result.color.bytes, colorLayout.Alignment(), true); - result.color.format = swap == 0 ? (number == 0 ? VK_FORMAT_R8G8B8A8_UNORM : VK_FORMAT_R8G8B8A8_SRGB) : (number == 0 ? VK_FORMAT_B8G8R8A8_UNORM : VK_FORMAT_B8G8R8A8_SRGB); - result.color.componentMapping = 0xe4u; - result.renderExtent = result.color.extent; + color.address = ((static_cast(high) << 40u) | (static_cast(read(cx, 0x318 + stride)) << 8u)) + mipOffset; + color.bytes = colorLayout.Bytes(); + GuestMemory::CheckRange(reinterpret_cast(color.address), color.bytes, colorLayout.Alignment(), true); + color.format = decoded.format; + color.componentMapping = 0xe4u; + if ((info & 0x10000000u) != 0) { + if (maxMip == 0) { + const auto dccHigh = cx.find(0x3a8 + slot); + color.dccAddress = ((dccHigh == cx.end() ? 0ull : static_cast(dccHigh->second & 0xffu)) << 40u) | (static_cast(read(cx, 0x325 + stride)) << 8u); + color.dccAlphaOnMsb = DccAlphaOnMsb(color.format, swap); + } else { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] DCC keys of mipmapped color targets are ignored\n"); + } + } + } + APS5_LOG_OUT_DEBUG("Color %u address=0x%llx extent=%ux%u bytes=%llu VkFormat=%u", slot, static_cast(color.address), color.extent.width, color.extent.height, static_cast(color.bytes), static_cast(color.format)); + if (slot == 0) { + result.renderExtent = color.extent; + } else { + result.renderExtent = {std::min(result.renderExtent.width, color.extent.width), std::min(result.renderExtent.height, color.extent.height)}; + } + result.colors.push_back(color); + } + if (result.hasColorTarget) { + result.color = result.colors.front(); } else { const auto screenBottomRight = read(cx, 0xd); + APS5_LOG_OUT_DEBUG("No color target, screen BR register=0x%x", screenBottomRight); result.renderExtent = {screenBottomRight & 0xffffu, screenBottomRight >> 16u}; + APS5_LOG_OUT_DEBUG("Render extent from screen=%ux%u", result.renderExtent.width, result.renderExtent.height); Require(result.renderExtent.width != 0 && result.renderExtent.height != 0, "empty framebuffer extent for a draw without color writes"); } const auto xs = readFloat(cx, 0x10f); @@ -225,6 +410,7 @@ State DecodeState(const QueueState& queue) { const auto zo = readFloat(cx, 0x114); const auto minDepth = result.negativeOneToOne ? zo - zs : zo; const auto maxDepth = zo + zs; + APS5_LOG_OUT_DEBUG("Viewport transform scale=(%f,%f,%f) offset=(%f,%f,%f) depth=(%f,%f)", xs, ys, zs, xo, yo, zo, minDepth, maxDepth); if (!(xs > 0 && ys != 0 && std::isfinite(minDepth) && std::isfinite(maxDepth))) { std::ostringstream message; message << "AGC graphics: unsupported viewport transform: scale=(" << xs << ", " << ys << ", " << zs << "), offset=(" << xo << ", " << yo << ", " << zo << "), depth=(" << minDepth << ", " << maxDepth << "), negativeOneToOne=" << result.negativeOneToOne; @@ -232,28 +418,35 @@ State DecodeState(const QueueState& queue) { } Require(readFloat(cx, 0xb4) <= readFloat(cx, 0xb5), "inverted viewport depth clamp bounds"); result.viewport = {xo - xs, yo - ys, 2 * xs, 2 * ys, minDepth, maxDepth}; + APS5_LOG_OUT_DEBUG("Viewport x=%f y=%f w=%f h=%f minDepth=%f maxDepth=%f", result.viewport.x, result.viewport.y, result.viewport.width, result.viewport.height, result.viewport.minDepth, result.viewport.maxDepth); result.scissor = {{0, 0}, result.renderExtent}; intersect(result.scissor, cx, 0xc, true); intersect(result.scissor, cx, 0x81, false); intersect(result.scissor, cx, 0x90, false); if ((read(cx, 0x292) & 2u) != 0) intersect(result.scissor, cx, 0x94, false); - if (result.hasColorTarget) { - const auto blend = read(cx, 0x1e0); + APS5_LOG_OUT_DEBUG("Scissor offset=(%d,%d) extent=%ux%u", result.scissor.offset.x, result.scissor.offset.y, result.scissor.extent.width, result.scissor.extent.height); + for (std::uint32_t slot = 0; slot < result.colors.size(); ++slot) { + const auto blend = read(cx, 0x1e0 + slot); + APS5_LOG_OUT_DEBUG("Blend %u control=0x%x", slot, blend); Require((blend & 0x0000e000u) == 0, "reserved blend control bits"); - result.blend.colorWriteMask = targetMask; - result.blend.blendEnable = (blend >> 30u) & 1u; - if (result.blend.blendEnable) { - Require((read(cx, 0x31c) & 0x10000u) == 0, "blend bypass conflicts with enabled blending"); - result.blend.srcColorBlendFactor = blendFactor(blend & 0x1fu); - result.blend.dstColorBlendFactor = blendFactor((blend >> 8u) & 0x1fu); - result.blend.colorBlendOp = blendOp((blend >> 5u) & 7u); + VkPipelineColorBlendAttachmentState state{}; + state.colorWriteMask = (targetMask >> (4u * slot)) & 0xfu; + state.blendEnable = (blend >> 30u) & 1u; + if (state.blendEnable) { + Require((read(cx, 0x31c + slot * 0xfu) & 0x10000u) == 0, "blend bypass conflicts with enabled blending"); + state.srcColorBlendFactor = blendFactor(blend & 0x1fu); + state.dstColorBlendFactor = blendFactor((blend >> 8u) & 0x1fu); + state.colorBlendOp = blendOp((blend >> 5u) & 7u); const auto alpha = (blend & 0x20000000u) != 0 ? blend >> 16u : blend; - result.blend.srcAlphaBlendFactor = blendFactor(alpha & 0x1fu); - result.blend.dstAlphaBlendFactor = blendFactor((alpha >> 8u) & 0x1fu); - result.blend.alphaBlendOp = blendOp((alpha >> 5u) & 7u); + state.srcAlphaBlendFactor = blendFactor(alpha & 0x1fu); + state.dstAlphaBlendFactor = blendFactor((alpha >> 8u) & 0x1fu); + state.alphaBlendOp = blendOp((alpha >> 5u) & 7u); for (std::uint32_t i = 0; i < 4; ++i) result.blendConstants[i] = readFloat(cx, 0x105 + i); } + result.blends.push_back(state); } + if (!result.blends.empty()) result.blend = result.blends.front(); + APS5_LOG_OUT_DEBUG("DecodeState done colorTarget=%u render=%ux%u topology=%u", result.hasColorTarget ? 1u : 0u, result.renderExtent.width, result.renderExtent.height, static_cast(result.topology)); return result; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 35bf09e0d3..890e9dbcfb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1,68 +1,103 @@ #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/General.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +// For the declaration of PendingStorageOverlaps, defined below. +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include +#include +#include #include #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include namespace AgcDriver::Graphics { namespace { +// APS5_PROFILE_DRAW: accumulate texture setup phases and report every 200 textures. +struct TextureProfile { + double allocate = 0, read = 0, gpu = 0, view = 0; + std::uint64_t count = 0; + std::uint64_t fromStorage = 0; + double storageCreate = 0, storageWriteBack = 0, storageAlloc = 0, storageHostCopy = 0, storageGpu = 0, storageStore = 0; + std::uint64_t storageCount = 0; + std::uint64_t storageBytes = 0; + std::uint64_t storageReused = 0; + std::uint64_t storageDirectUploads = 0; + std::uint64_t storageDirectWriteBacks = 0; + // Sampled-texture uploads recorded into the open batch instead of waited for. + std::uint64_t recordedUploads = 0; +}; -std::uint32_t FullArrayLayers(const GuestTextureResource& descriptor) { - switch (descriptor.dimension) { - case TextureDimension::k1D: - case TextureDimension::k2D: return 1u; - case TextureDimension::k2DArray: - case TextureDimension::kCube: return descriptor.depthOrLastArray + 1u; - } - throw std::runtime_error("AGC graphics: Texture encountered an unknown guest texture dimension"); +TextureProfile& Profile() { + static TextureProfile profile; + return profile; } +struct PhaseTimer { + std::chrono::steady_clock::time_point last = std::chrono::steady_clock::now(); + double lap() { + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - last).count(); + last = now; + return ms; + } +}; + + VkImageType ImageTypeFor(TextureDimension dimension) { - return dimension == TextureDimension::k1D ? VK_IMAGE_TYPE_1D : VK_IMAGE_TYPE_2D; + return dimension == TextureDimension::k1D ? VK_IMAGE_TYPE_1D : dimension == TextureDimension::k3D ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D; } -VkImageViewType ViewTypeFor(TextureDimension dimension, std::uint32_t viewLayerCount) { +VkImageViewType ViewTypeFor(TextureDimension dimension, [[maybe_unused]] std::uint32_t viewLayerCount) { switch (dimension) { case TextureDimension::k1D: return VK_IMAGE_VIEW_TYPE_1D; case TextureDimension::k2D: return VK_IMAGE_VIEW_TYPE_2D; case TextureDimension::k2DArray: return VK_IMAGE_VIEW_TYPE_2D_ARRAY; - case TextureDimension::kCube: return viewLayerCount == 6u ? VK_IMAGE_VIEW_TYPE_CUBE : VK_IMAGE_VIEW_TYPE_CUBE_ARRAY; + // Shaders address cube maps as 2D arrays of faces. + case TextureDimension::kCube: return VK_IMAGE_VIEW_TYPE_2D_ARRAY; + case TextureDimension::k3D: return VK_IMAGE_VIEW_TYPE_3D; } throw std::runtime_error("AGC graphics: Texture encountered an unknown guest texture dimension"); } -std::uint64_t SliceLinearBytes(const std::vector& mips) { - std::uint64_t bytes = 0; - for (const auto& mip : mips) bytes = std::max(bytes, mip.linearOffset + mip.linearSize); - return bytes; -} - } Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, VkComponentMapping components, std::span snapshot) : context(context) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + PhaseTimer timer; try { const auto vkFormat = ResolveTextureFormat(descriptor.format); if (IsBlockCompressed(descriptor.format)) { Require(context.textureCompressionBC, "device does not support BC compressed textures"); } - const auto mips = ComputeMipLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, descriptor.mipCount); - const auto arrayLayers = FullArrayLayers(descriptor); + const auto geometry = DescribeSurface(descriptor); + const auto& mips = geometry.mips; + const auto arrayLayers = geometry.layers; const auto elementBytes = BytesPerElement(descriptor.format); + APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u layers=%u dim=%d tile=%d format=%u vk=%d element=%u", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, arrayLayers, + static_cast(descriptor.dimension), static_cast(descriptor.tileMode), descriptor.format, static_cast(vkFormat), elementBytes); + for (const auto& mip : mips) APS5_LOG_OUT(" mip %ux%u tiled=0x%llx+0x%llx linear=0x%llx+0x%llx blocksPerRow=%u pitch=%u tail=%d", mip.width, mip.height, static_cast(mip.tiledOffset), static_cast(mip.tiledSize), static_cast(mip.linearOffset), static_cast(mip.linearSize), mip.blocksPerRow, mip.pitchBytes, mip.tail ? 1 : 0); - const auto guestBytes = ComputeSurfaceSize(mips, arrayLayers); - const auto guestSliceBytes = guestBytes / arrayLayers; + const auto guestBytes = geometry.guestBytes; Require(snapshot.size() == guestBytes, "texture snapshot size mismatch"); - const auto sliceLinearBytes = SliceLinearBytes(mips); + const auto sliceLinearBytes = geometry.sliceLinearBytes; Require(arrayLayers == 0 || sliceLinearBytes <= UINT64_MAX / arrayLayers, "detiled texture buffer size overflows"); const auto linearBytes = sliceLinearBytes * arrayLayers; @@ -70,15 +105,16 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex imageInfo.flags = descriptor.dimension == TextureDimension::kCube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u; imageInfo.imageType = ImageTypeFor(descriptor.dimension); imageInfo.format = vkFormat; - imageInfo.extent = {descriptor.width, descriptor.height, 1u}; + imageInfo.extent = {descriptor.width, descriptor.height, geometry.imageDepth}; imageInfo.mipLevels = descriptor.mipCount; - imageInfo.arrayLayers = arrayLayers; + imageInfo.arrayLayers = geometry.imageLayers; imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; Check(context.Function("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage"); + owned = std::make_shared(context, image, VK_NULL_HANDLE); VkMemoryRequirements requirements{}; context.Function("vkGetImageMemoryRequirements")(context.device, image, &requirements); @@ -86,45 +122,61 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory texture"); - Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory"); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory texture"); + Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory"); { - Buffer staging(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); - std::memcpy(staging.Bytes().data(), snapshot.data(), snapshot.size()); - Buffer linear(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + // Debug aid: APS5_DUMP_TEXTURE= saves the detiled first mip + // of those sampled textures on their first 8 uploads as texture_
_.raw (u32 + // width, height, VkFormat, then tightly packed rows). + static const std::string dumpList = [] { const char* text = std::getenv("APS5_DUMP_TEXTURE"); return text ? std::string(text) : std::string(); }(); + bool dumpWanted = false; + if (!dumpList.empty()) { + char address[32]; + std::snprintf(address, sizeof(address), "%llx", static_cast(descriptor.baseAddress)); + char extent[32]; + std::snprintf(extent, sizeof(extent), "%ux%u", descriptor.width, descriptor.height); + // Entries may also be extents ("3840x2160"), since heap addresses change between runs. + dumpWanted = dumpList.find(address) != std::string::npos || dumpList.find(extent) != std::string::npos; + } + auto staging = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(staging->Bytes().data(), snapshot.data(), snapshot.size()); + auto tiled = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + if (profile) Profile().allocate += timer.lap(); + if (profile) Profile().read += timer.lap(); detiler.BeginBatch(); - if (context.drawQueue) context.drawQueue->Flush(); - CommandBatch batch(context); - const auto commands = batch.Handle(); + // The upload is recorded into the open batch, like a storage image's: the work that samples + // the texture is recorded after it, and a batch of its own (SubmitAndWait) was a full GPU + // drain under the device lock for every new texture. The buffers and the image live with + // the batch (the cache may drop the texture before it completes). The dump aid needs the + // linear bytes on the CPU, so it keeps the waiting batch, as does a build without a + // recorder (tests). APS5_SYNC_TEXTURE_UPLOAD=1 restores the waiting batch for every upload. + static const bool syncUploads = std::getenv("APS5_SYNC_TEXTURE_UPLOAD") != nullptr; + auto* recorder = dumpWanted || syncUploads ? nullptr : Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + if (recorder != nullptr) { + commands = recorder->Commands(); + recorder->Keep(staging); + recorder->Keep(tiled); + recorder->Keep(linear); + recorder->Keep(owned); + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } - VkBufferMemoryBarrier stagingReadBarrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; - stagingReadBarrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; - stagingReadBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; - stagingReadBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - stagingReadBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - stagingReadBarrier.buffer = staging.Handle(); - stagingReadBarrier.offset = 0; - stagingReadBarrier.size = VK_WHOLE_SIZE; - - VkBufferMemoryBarrier linearWriteBarrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; - linearWriteBarrier.srcAccessMask = 0; - linearWriteBarrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - linearWriteBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - linearWriteBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - linearWriteBarrier.buffer = linear.Handle(); - linearWriteBarrier.offset = 0; - linearWriteBarrier.size = VK_WHOLE_SIZE; - - const VkBufferMemoryBarrier preBarriers[] = {stagingReadBarrier, linearWriteBarrier}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 2, preBarriers, 0, nullptr); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, staging->Handle(), 0, tiled->Handle(), 0, guestBytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { - const auto guestLayerOffset = static_cast(layer) * guestSliceBytes; - const auto linearLayerOffset = static_cast(layer) * sliceLinearBytes; + const auto guestLayerOffset = geometry.GuestLayerOffset(layer); + const auto linearLayerOffset = geometry.LinearLayerOffset(layer); for (const auto& mip : mips) { - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, staging.Handle(), guestLayerOffset + mip.tiledOffset, linear.Handle(), linearLayerOffset + mip.linearOffset, mip, layer); + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, tiled->Handle(), guestLayerOffset + mip.tiledOffset, linear->Handle(), linearLayerOffset + mip.linearOffset, mip, false, layer, geometry.thick); } } @@ -133,7 +185,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex linearReadBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; linearReadBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; linearReadBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - linearReadBarrier.buffer = linear.Handle(); + linearReadBarrier.buffer = linear->Handle(); linearReadBarrier.offset = 0; linearReadBarrier.size = VK_WHOLE_SIZE; @@ -145,7 +197,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex toTransferDst.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toTransferDst.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toTransferDst.image = image; - toTransferDst.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, arrayLayers}; + toTransferDst.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearReadBarrier, 1, &toTransferDst); std::vector regions; @@ -156,15 +208,22 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex const auto& mip = mips[level]; VkBufferImageCopy region{}; region.bufferOffset = linearLayerOffset + mip.linearOffset; - region.bufferRowLength = mip.pitchBytes / elementBytes * BlockWidth(descriptor.format); + region.bufferRowLength = mip.pitchBytes / BytesPerElement(descriptor.format) * BlockWidth(descriptor.format); region.bufferImageHeight = 0; - region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, layer, 1}; - region.imageOffset = {0, 0, 0}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, geometry.CopyLayer(layer), 1}; + region.imageOffset = {0, 0, geometry.CopyDepth(layer)}; region.imageExtent = {std::max(descriptor.width >> level, 1u), std::max(descriptor.height >> level, 1u), 1u}; regions.push_back(region); } } - context.Function("vkCmdCopyBufferToImage")(commands, linear.Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + context.Function("vkCmdCopyBufferToImage")(commands, linear->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + std::unique_ptr dump; + if (dumpWanted) { + dump = std::make_unique(context, static_cast(mips[0].linearSize), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, linear->Handle(), mips[0].linearOffset, dump->Handle(), 0, mips[0].linearSize); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + } VkImageMemoryBarrier toShaderRead{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; toShaderRead.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; @@ -177,11 +236,30 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex toShaderRead.subresourceRange = toTransferDst.subresourceRange; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toShaderRead); - batch.SubmitAndWait(); + if (batch) batch->SubmitAndWait(); + else ++Profile().recordedUploads; + if (profile) Profile().gpu += timer.lap(); + if (dump) { + static std::mutex dumpMutex; + static std::map dumped; + std::lock_guard lock(dumpMutex); + auto& count = dumped[descriptor.baseAddress]; + if (count < 8) { + char name[64]; + std::snprintf(name, sizeof(name), "texture_%llx_%d.raw", static_cast(descriptor.baseAddress), count++); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t header[3] = {mips[0].pitchBytes / static_cast(elementBytes), mips[0].height, static_cast(vkFormat)}; + std::fwrite(header, sizeof(header), 1, file); + std::fwrite(dump->Bytes().data(), 1, dump->Bytes().size(), file); + std::fclose(file); + std::fprintf(stderr, "[texture] dumped %s (%ux%u VkFormat %d, tile %d)\n", name, header[0], header[1], static_cast(vkFormat), static_cast(descriptor.tileMode)); + } + } + } } - const auto viewLevelCount = descriptor.mipCount - descriptor.baseLevel; - const auto viewLayerCount = arrayLayers - descriptor.baseArray; + const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; + const auto viewLayerCount = geometry.imageLayers - descriptor.baseArray; if (descriptor.dimension == TextureDimension::kCube) { Require(viewLayerCount % 6u == 0, "guest cube texture view does not contain a multiple of 6 array slices"); } @@ -193,6 +271,53 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex viewInfo.components = components; viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, descriptor.baseLevel, viewLevelCount, descriptor.baseArray, viewLayerCount}; Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView"); + if (profile) { + auto& totals = Profile(); + totals.view += timer.lap(); + if (++totals.count % 200 == 0) std::fprintf(stderr, "[texture] %llu textures (%llu copied from storage images, %llu uploads recorded): allocate+image %.0f ms, guest read %.0f ms, detile+copy %.0f ms, view+buffer release %.0f ms\n", static_cast(totals.count), static_cast(totals.fromStorage), static_cast(totals.recordedUploads), totals.allocate, totals.read, totals.gpu, totals.view); + } + } catch (...) { + release(); + throw; + } +} + +bool Texture::CanCopyFrom(const StorageTexture& source, const GuestTextureResource& descriptor) { + const auto& from = source.Descriptor(); + if (IsBlockCompressed(descriptor.format) || IsBlockCompressed(from.format)) return false; + // Same memory, same layout, same texel size: the GPU copy reinterprets the texels exactly as a + // guest read through the sampled descriptor would. + return descriptor.baseAddress == from.baseAddress && descriptor.width == from.width && descriptor.height == from.height && descriptor.dimension == from.dimension && descriptor.tileMode == from.tileMode && descriptor.mipCount <= from.mipCount && descriptor.depthOrLastArray == from.depthOrLastArray && BytesPerElement(descriptor.format) == BytesPerElement(from.format) && BlockWidth(descriptor.format) == BlockWidth(from.format); +} + +Texture::Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components) : context(context), storageSource(source) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + PhaseTimer timer; + try { + Require(source != nullptr && CanCopyFrom(*source, descriptor), "storage image does not match the sampled texture"); + const auto vkFormat = ResolveTextureFormat(descriptor.format); + const auto geometry = DescribeSurface(descriptor); + APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u viewed from storage image (vk=%d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, static_cast(vkFormat)); + // Storage images stay in the general layout; the view samples them there. + layout = VK_IMAGE_LAYOUT_GENERAL; + const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; + const auto viewLayerCount = std::min(geometry.imageLayers, source->ImageLayers()) - descriptor.baseArray; + if (descriptor.dimension == TextureDimension::kCube) { + Require(viewLayerCount % 6u == 0, "guest cube texture view does not contain a multiple of 6 array slices"); + } + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = source->Image(); + viewInfo.viewType = ViewTypeFor(descriptor.dimension, viewLayerCount); + viewInfo.format = vkFormat; + viewInfo.components = components; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, descriptor.baseLevel, viewLevelCount, descriptor.baseArray, viewLayerCount}; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView storage view"); + if (profile) { + auto& totals = Profile(); + totals.view += timer.lap(); + ++totals.fromStorage; + if (++totals.count % 200 == 0) std::fprintf(stderr, "[texture] %llu textures (%llu viewed from storage images): allocate+image %.0f ms, guest read %.0f ms, detile+copy %.0f ms, view+buffer release %.0f ms\n", static_cast(totals.count), static_cast(totals.fromStorage), totals.allocate, totals.read, totals.gpu, totals.view); + } } catch (...) { release(); throw; @@ -206,12 +331,894 @@ Texture::~Texture() { void Texture::release() noexcept { upload.reset(); if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); - if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); - if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); + view = VK_NULL_HANDLE; + // The image and its memory go with the last holder: this texture, or the batch still uploading it. + owned.reset(); + image = VK_NULL_HANDLE; } VkImageView Texture::View() const { return view; } + +namespace { + +VkBufferMemoryBarrier WholeBufferBarrier(VkBuffer buffer, VkAccessFlags from, VkAccessFlags to) { + VkBufferMemoryBarrier barrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + barrier.srcAccessMask = from; + barrier.dstAccessMask = to; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = 0; + barrier.size = VK_WHOLE_SIZE; + return barrier; +} + +// Storage images cannot use sRGB formats; the shader works on the raw encoded values either way. +// The answer is a property of the physical device, so it is computed once per format (every storage +// image lookup asks: a live vkGetPhysicalDeviceFormatProperties call each time was measurable) and +// remembered as VK_FORMAT_UNDEFINED when the format has no storage form. +VkFormat StorageFormatOrUndefined(const Context& context, VkFormat format) { + struct Table { + std::mutex mutex; + std::unordered_map formats; + }; + static Table table; + // The key names the physical device: the headless and the windowed device share one GPU, but a + // second GPU would answer differently. + const auto key = (static_cast(reinterpret_cast(context.physical)) << 20u) ^ static_cast(format); + { + std::lock_guard lock(table.mutex); + if (const auto found = table.formats.find(key); found != table.formats.end()) return found->second; + } + const auto supports = [&](VkFormat candidate) { + VkFormatProperties properties{}; + context.formatProperties(context.physical, candidate, &properties); + return (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; + }; + VkFormat storage = VK_FORMAT_UNDEFINED; + if (supports(format)) { + storage = format; + } else { + VkFormat linear = VK_FORMAT_UNDEFINED; + switch (format) { + case VK_FORMAT_R8G8B8A8_SRGB: linear = VK_FORMAT_R8G8B8A8_UNORM; break; + case VK_FORMAT_B8G8R8A8_SRGB: linear = VK_FORMAT_B8G8R8A8_UNORM; break; + case VK_FORMAT_A8B8G8R8_SRGB_PACK32: linear = VK_FORMAT_A8B8G8R8_UNORM_PACK32; break; + case VK_FORMAT_R8_SRGB: linear = VK_FORMAT_R8_UNORM; break; + case VK_FORMAT_R8G8_SRGB: linear = VK_FORMAT_R8G8_UNORM; break; + default: break; + } + if (linear != VK_FORMAT_UNDEFINED && supports(linear)) storage = linear; + } + std::lock_guard lock(table.mutex); + table.formats.emplace(key, storage); + return storage; +} + +VkFormat StorageFormatFor(const Context& context, VkFormat format) { + const auto storage = StorageFormatOrUndefined(context, format); + Require(storage != VK_FORMAT_UNDEFINED, "guest storage texture format " + std::to_string(format) + " cannot be used as a storage image"); + return storage; +} + +} + +bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat) { + try { + return StorageFormatOrUndefined(context, ResolveTextureFormat(guestFormat)) != VK_FORMAT_UNDEFINED; + } catch (const std::exception&) { + // An unknown guest format: the sampled texture path reports it when it gets there. + return false; + } +} + +// Integer formats take integer clear values; the DCC clear codes are only mapped for the others. +bool IntegerFormat(VkFormat format) { + switch (format) { + case VK_FORMAT_R8_UINT: case VK_FORMAT_R8_SINT: case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8_SINT: + case VK_FORMAT_R8G8B8A8_UINT: case VK_FORMAT_R8G8B8A8_SINT: case VK_FORMAT_B8G8R8A8_UINT: case VK_FORMAT_B8G8R8A8_SINT: + case VK_FORMAT_A8B8G8R8_UINT_PACK32: case VK_FORMAT_A8B8G8R8_SINT_PACK32: case VK_FORMAT_A2R10G10B10_UINT_PACK32: case VK_FORMAT_A2B10G10R10_UINT_PACK32: + case VK_FORMAT_R16_UINT: case VK_FORMAT_R16_SINT: case VK_FORMAT_R16G16_UINT: case VK_FORMAT_R16G16_SINT: + case VK_FORMAT_R16G16B16A16_UINT: case VK_FORMAT_R16G16B16A16_SINT: case VK_FORMAT_R32_UINT: case VK_FORMAT_R32_SINT: + case VK_FORMAT_R32G32_UINT: case VK_FORMAT_R32G32_SINT: case VK_FORMAT_R32G32B32_UINT: case VK_FORMAT_R32G32B32_SINT: + case VK_FORMAT_R32G32B32A32_UINT: case VK_FORMAT_R32G32B32A32_SINT: case VK_FORMAT_R64_UINT: case VK_FORMAT_R64_SINT: + return true; + default: return false; + } +} + +// The clear value a DCC clear code stands for, for non-integer formats. +bool ClearColorFor(VkFormat format, DccKeys keys, VkClearColorValue& clear) { + if (IntegerFormat(format)) return false; + switch (keys) { + case DccKeys::Clear0000: clear.float32[0] = clear.float32[1] = clear.float32[2] = clear.float32[3] = 0.0f; return true; + case DccKeys::Clear0001: clear.float32[0] = clear.float32[1] = clear.float32[2] = 0.0f; clear.float32[3] = 1.0f; return true; + case DccKeys::Clear1110: clear.float32[0] = clear.float32[1] = clear.float32[2] = 1.0f; clear.float32[3] = 0.0f; return true; + case DccKeys::Clear1111: clear.float32[0] = clear.float32[1] = clear.float32[2] = clear.float32[3] = 1.0f; return true; + default: return false; + } +} + +VkFormat StorageFormatForGuest(const Context& context, std::uint32_t guestFormat) { + return StorageFormatFor(context, ResolveTextureFormat(guestFormat)); +} + +StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, std::uint32_t mipLevel) : context(context), detiler(detiler), descriptor(descriptor) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + PhaseTimer timer; + try { + Require(!IsBlockCompressed(descriptor.format), "block-compressed textures cannot be storage images"); + Require(mipLevel < descriptor.mipCount, "storage texture mip level is outside the texture"); + const auto vkFormat = StorageFormatFor(context, ResolveTextureFormat(descriptor.format)); + storageFormat = vkFormat; + APS5_LOG_OUT("StorageTexture address=0x%llx %ux%u mips=%u mip=%u layers=%u base=%u dim=%d tile=%d format=%u vk=%d", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, mipLevel, + descriptor.depthOrLastArray, descriptor.baseArray, static_cast(descriptor.dimension), static_cast(descriptor.tileMode), descriptor.format, static_cast(vkFormat)); + geometry = DescribeSurface(descriptor); + mips = geometry.mips; + arrayLayers = geometry.layers; + const auto elementBytes = BytesPerElement(descriptor.format); + guestBytes = geometry.guestBytes; + // Storage images in heaps the guest commits on demand only read and store committed pages. + Require(!GuestMemory::CommittedRanges(descriptor.baseAddress, static_cast(guestBytes), true).empty(), "storage texture has no committed guest pages"); + sliceLinearBytes = geometry.sliceLinearBytes; + const auto linearBytes = sliceLinearBytes * arrayLayers; + + VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + // Sampled views of other same-size formats (sRGB, reinterpretations) read the image directly. + imageInfo.flags = (descriptor.dimension == TextureDimension::kCube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u) | VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; + imageInfo.imageType = ImageTypeFor(descriptor.dimension); + imageInfo.format = vkFormat; + imageInfo.extent = {descriptor.width, descriptor.height, geometry.imageDepth}; + imageInfo.mipLevels = descriptor.mipCount; + imageInfo.arrayLayers = geometry.imageLayers; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + { + VkFormatProperties properties{}; + context.formatProperties(context.physical, vkFormat, &properties); + attachable = descriptor.dimension == TextureDimension::k2D && (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) != 0; + if (attachable) imageInfo.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + } + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + Check(context.Function("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage storage"); + VkMemoryRequirements requirements{}; + context.Function("vkGetImageMemoryRequirements")(context.device, image, &requirements); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory storage texture"); + Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage"); + upload(); + defaultMip = mipLevel; + view = createView(mipLevel); + if (profile) { + auto& totals = Profile(); + totals.storageCreate += timer.lap(); + totals.storageBytes += guestBytes; + if (++totals.storageCount % 50 == 0) std::fprintf(stderr, "[texture] %llu storage images (%.0f MiB): create %.0f ms, write-back %.0f ms (alloc %.0f, host copy %.0f, gpu %.0f, store %.0f), %llu reused, %llu direct uploads, %llu direct write-backs\n", static_cast(totals.storageCount), totals.storageBytes / 1048576.0, totals.storageCreate, totals.storageWriteBack, totals.storageAlloc, totals.storageHostCopy, totals.storageGpu, totals.storageStore, static_cast(totals.storageReused), static_cast(totals.storageDirectUploads), static_cast(totals.storageDirectWriteBacks)); + } + } catch (...) { + release(); + throw; + } +} + +namespace { + +// Storage images whose results have not reached guest memory yet. +struct PendingWrites { + std::mutex mutex; + std::vector textures; + // Images taken out of `textures` by a FlushPending still storing them (see adjacentPendingUnchanged). + std::vector flushing; +}; + +PendingWrites& Pending() { + static PendingWrites pending; + return pending; +} + +// The image being validated by Refresh: its own pending results are what the next dispatch wants, +// so the comparison with guest memory must not flush them. +thread_local const StorageTexture* refreshing = nullptr; + +void FlushHook(std::uint64_t address, std::size_t bytes) { + StorageTexture::FlushPending(address, bytes); +} + +// Write stamps are per 64 KiB block (GuestMemory::UnchangedSince), so a write-back's own MarkWritten +// stamps the blocks its surface shares with an adjacent surface (video planes packed back to back: +// the luma plane's last block is the chroma plane's first). A pending image of that adjacent surface +// would take the stamp for a CPU write since its generation and, at its own write-back, keep the +// guest bytes of the shared block in place of its results (a stale band at the plane's start, zero +// chroma on a fresh buffer). So a write-back first finds the adjacent pending images unchanged since +// their generation and, once its stamps are made, advances them past the stamps: nothing of theirs +// changed. APS5_NO_ADJACENT_GENERATION=1 leaves them at their generation, as before. +bool AdjacentGenerationEnabled() { + static const bool disabled = std::getenv("APS5_NO_ADJACENT_GENERATION") != nullptr; + return !disabled; +} + +} + +VkImageView StorageTexture::createView(std::uint32_t mip) const { + Require(mip < descriptor.mipCount, "storage texture mip level is outside the texture"); + const auto viewLayerCount = geometry.imageLayers - descriptor.baseArray; + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = image; + // Storage views address one mip; cube faces are written as array layers. + viewInfo.viewType = descriptor.dimension == TextureDimension::k1D ? VK_IMAGE_VIEW_TYPE_1D : descriptor.dimension == TextureDimension::k2D ? VK_IMAGE_VIEW_TYPE_2D : descriptor.dimension == TextureDimension::k3D ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; + viewInfo.format = storageFormat; + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, mip, 1u, descriptor.baseArray, viewLayerCount}; + VkImageView created = VK_NULL_HANDLE; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &created), "vkCreateImageView storage"); + return created; +} + +VkImageView StorageTexture::AttachmentView(VkFormat format) { + Require(attachable, "storage image cannot be a color attachment"); + const auto found = attachmentViews.find(format); + if (found != attachmentViews.end()) return found->second; + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = image; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = format; + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1u, descriptor.baseArray, 1u}; + VkImageView created = VK_NULL_HANDLE; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &created), "vkCreateImageView attachment"); + attachmentViews.emplace(format, created); + return created; +} + +VkImageView StorageTexture::View(std::uint32_t mip) { + if (mip == defaultMip) return view; + const auto found = extraViews.find(mip); + if (found != extraViews.end()) return found->second; + const auto created = createView(mip); + extraViews.emplace(mip, created); + return created; +} + +bool StorageTexture::Refresh() { + // Results of other images pending in this memory must reach it first; this image's own pending + // results stay on the GPU, where the next dispatch wants them. + FlushPending(descriptor.baseAddress, static_cast(guestBytes), this, "storage refresh"); + // `original` holds the guest bytes the image was last uploaded from or written back as; while the + // guest memory and the DCC keys still match, the image content is current. Pages nobody wrote + // since `generation` need no comparison. + const auto current = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + bool unchanged = false; + { + struct Exempt { + const StorageTexture* previous; + ~Exempt() { refreshing = previous; } + } exempt{refreshing}; + refreshing = this; + const auto equalsOriginal = [&] { + // Named for the [hooksync] attribution: the compare goes through the flush hook. + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureCompare); + return GuestMemory::EqualsCommitted(descriptor.baseAddress, original); + }; + unchanged = TextureClearKeys(descriptor, guestBytes) == uploadedKeys && (GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast(guestBytes), generation) || (originalValid && equalsOriginal())); + } + if (unchanged) { + ++Profile().storageReused; + generation = current; + return true; + } + // Debug aid: APS5_TRACE_UPLOAD names why a storage image is uploaded again. + static const bool traceUpload = std::getenv("APS5_TRACE_UPLOAD") != nullptr; + if (traceUpload) { + const auto keys = TextureClearKeys(descriptor, guestBytes); + std::fprintf(stderr, "[upload] 0x%llx+0x%llx %ux%u mips %u: %s (keys %s -> %s, originalValid %d, dirty %d)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), descriptor.width, descriptor.height, descriptor.mipCount, keys != uploadedKeys ? "DCC keys changed" : "guest memory changed", DccKeysName(uploadedKeys), DccKeysName(keys), originalValid ? 1 : 0, dirty ? 1 : 0); + } + bool pendingResults = false; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + if (dirty) { + dirty = false; + std::erase(pending.textures, this); + pendingResults = true; + } + } + if (pendingResults) { + // The CPU wrote this memory while GPU results were pending: as with an immediate write-back + // followed by the CPU write, its blocks win and the results land everywhere else. + static std::atomic reports{0}; + if (reports.fetch_add(1) < 8) std::fprintf(stderr, "[gpu] storage image 0x%llx: guest memory changed while GPU results were pending; keeping the CPU's blocks\n", static_cast(descriptor.baseAddress)); + writeBack(generation); + } + upload(); + return false; +} + +void StorageTexture::upload() { + const auto elementBytes = BytesPerElement(descriptor.format); + const auto linearBytes = sliceLinearBytes * arrayLayers; + original.resize(static_cast(guestBytes)); + generation = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + uploadedKeys = TextureClearKeys(descriptor, guestBytes); + VkClearColorValue clearValue{}; + if (IsDccClear(uploadedKeys) && ClearColorFor(storageFormat, uploadedKeys, clearValue)) { + // A fast-cleared surface is cleared on the GPU; its texel memory is neither read nor filled. + originalValid = false; + CommandBatch batch(context); + const auto commands = batch.Handle(); + VkImageMemoryBarrier toClear{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toClear.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + toClear.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toClear.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toClear.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toClear.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.image = image; + toClear.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toClear); + context.Function("vkCmdClearColorImage")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &clearValue, 1, &toClear.subresourceRange); + VkImageMemoryBarrier toGeneral = toClear; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + batch.SubmitAndWait(); + ++version; + return; + } + if (const auto* import = uploadedKeys == DccKeys::Uncompressed ? HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) : nullptr) { + // The surface lives in host-imported memory: the detiler reads it in place, no guest bytes + // are copied, and write tracking alone validates the image (a change re-runs this). + originalValid = false; + auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + detiler.BeginBatch(); + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + if (recorder != nullptr) { + commands = recorder->Commands(); + recorder->Keep(linear); + // The image itself must outlive the recorded copy: the cache may evict it right after. + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } + const auto importOffset = descriptor.baseAddress - import->base; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (const auto& mip : mips) { + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick); + } + } + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = image; + toTransfer.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); + const auto regions = CopyRegions(); + context.Function("vkCmdCopyBufferToImage")(commands, linear->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + VkImageMemoryBarrier toGeneral = toTransfer; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + if (batch) batch->SubmitAndWait(); + ++Profile().storageDirectUploads; + ++version; + return; + } + originalValid = true; + GuestMemory::ReadCommitted(descriptor.baseAddress, original); + { + Buffer staging(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + if (uploadedKeys == DccKeys::Uncompressed) std::memcpy(staging.Bytes().data(), original.data(), original.size()); + else ReadTextureSurface(descriptor, uploadedKeys, staging.Bytes().first(original.size())); + DeviceBuffer tiled(context, original.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + DeviceBuffer linear(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + detiler.BeginBatch(); + CommandBatch batch(context); + const auto commands = batch.Handle(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, staging.Handle(), 0, tiled.Handle(), 0, original.size()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (const auto& mip : mips) { + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick); + } + } + const auto linearRead = WholeBufferBarrier(linear.Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = image; + toTransfer.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); + const auto regions = CopyRegions(); + context.Function("vkCmdCopyBufferToImage")(commands, linear.Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + VkImageMemoryBarrier toGeneral = toTransfer; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + APS5_LOG_CHARS_OUT("StorageTexture upload submit"); + batch.SubmitAndWait(); + APS5_LOG_CHARS_OUT("StorageTexture upload done"); + } + ++version; +} + +std::vector StorageTexture::CopyRegions() const { + std::vector regions; + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) { + const auto& mip = mips[level]; + VkBufferImageCopy region{}; + region.bufferOffset = layer * sliceLinearBytes + mip.linearOffset; + region.bufferRowLength = mip.pitchBytes / BytesPerElement(descriptor.format) * BlockWidth(descriptor.format); + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, geometry.CopyLayer(layer), 1}; + region.imageOffset = {0, 0, geometry.CopyDepth(layer)}; + region.imageExtent = {std::max(descriptor.width >> level, 1u), std::max(descriptor.height >> level, 1u), 1u}; + regions.push_back(region); + } + } + return regions; +} + +bool StorageTexture::overlaps(std::uint64_t address, std::size_t bytes) const { + return address < descriptor.baseAddress + guestBytes && descriptor.baseAddress < address + bytes; +} + +void StorageTexture::MarkDirty() { + static const bool eager = std::getenv("APS5_EAGER_WRITEBACK") != nullptr || std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + if (eager) { + WriteBack(); + return; + } + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + // The active recorder installs a hook that also waits for recorded work; without one (tests), + // pending storage results alone are flushed. + if (Recorder::Active() == nullptr) { + static const bool hooked = [] { + GuestMemory::SetFlushHook(&FlushHook); + return true; + }(); + static_cast(hooked); + } + ++version; + if (dirty) return; + dirty = true; + pending.textures.push_back(this); +} + +void StorageTexture::Flush() { + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + if (!dirty) return; + dirty = false; + std::erase(pending.textures, this); + } + std::lock_guard gpu(GuestMemory::GpuMutex()); + writeBack(generation); +} + +bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason) { + // The images stay alive across the scan: their last owner may be a batch's kept list, which + // another thread releases outside the GPU mutex once the batch completed. + std::vector> flush; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (auto it = pending.textures.begin(); it != pending.textures.end();) { + auto* texture = *it; + if (texture != except && texture != refreshing && texture->overlaps(address, bytes)) { + texture->dirty = false; + if (auto alive = texture->weak_from_this().lock()) flush.push_back(std::move(alive)); + it = pending.textures.erase(it); + } else { + ++it; + } + } + // Still pending for the adjacency rule of writeBack until each one is stored. + for (const auto& texture : flush) pending.flushing.push_back(texture.get()); + } + if (flush.empty()) return false; + struct Unregister { + const std::vector>& flush; + ~Unregister() { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (const auto& texture : flush) std::erase(pending.flushing, texture.get()); + } + } unregister{flush}; + // Debug aid: APS5_TRACE_FLUSH names what forces pending results to guest memory. + static const bool trace = std::getenv("APS5_TRACE_FLUSH") != nullptr; + if (trace) { + for (const auto& texture : flush) std::fprintf(stderr, "[flush] image 0x%llx+0x%llx for %s 0x%llx+0x%zx\n", static_cast(texture->descriptor.baseAddress), static_cast(texture->guestBytes), reason, static_cast(address), bytes); + } + std::lock_guard gpu(GuestMemory::GpuMutex()); + std::exception_ptr failure; + for (const auto& texture : flush) { + try { + texture->writeBack(texture->generation); + } catch (...) { + if (!failure) failure = std::current_exception(); + } + } + if (failure) std::rethrow_exception(failure); + return true; +} + +void StorageTexture::FlushAllPending(const char* reason) { + static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason)); +} + +std::shared_ptr StorageTexture::FindPending(std::uint64_t address, std::uint64_t bytes) { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (auto* texture : pending.textures) { + // Containment, not equality: a descriptor of a chain's first mips (its own guestBytes are + // shorter) is served by the chain's image; CanCopyFrom then checks the geometry. + if (texture->descriptor.baseAddress == address && texture->guestBytes >= bytes) return texture->weak_from_this().lock(); + } + return nullptr; +} + +bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const StorageTexture* except) { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (const auto* texture : pending.textures) { + // A free function (declared in ShaderResources.hpp): the overlap is computed from the public + // surface description rather than the private helper. + const auto begin = texture->Descriptor().baseAddress; + if (texture != except && address < begin + texture->GuestBytes() && begin < address + bytes) return true; + } + return false; +} + +void StorageTexture::WriteBack() { + writeBack(generation); +} + +// See AdjacentGenerationEnabled above. +std::vector StorageTexture::adjacentPendingUnchanged() const { + std::vector adjacent; + if (!AdjacentGenerationEnabled() || guestBytes == 0) return adjacent; + constexpr std::uint64_t block = 65536; + const auto firstBlock = descriptor.baseAddress / block; + const auto lastBlock = (descriptor.baseAddress + guestBytes - 1) / block; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + const auto consider = [&](StorageTexture* texture) { + if (texture == this || texture->guestBytes == 0 || texture->overlaps(descriptor.baseAddress, static_cast(guestBytes))) return; + const auto begin = texture->descriptor.baseAddress; + if (begin / block > lastBlock || (begin + texture->guestBytes - 1) / block < firstBlock) return; + if (auto alive = texture->weak_from_this().lock()) adjacent.push_back({std::move(alive), texture->generation}); + }; + for (auto* texture : pending.textures) consider(texture); + for (auto* texture : pending.flushing) consider(texture); + } + // Only an image whose memory has not changed since its generation can be advanced: the collect + // stamps the CPU's writes so far, and a later one lands at a newer generation either way. + std::erase_if(adjacent, [](const Adjacent& entry) { + const auto& texture = entry.texture; + GuestMemory::CollectWritesUncached(texture->descriptor.baseAddress, static_cast(texture->guestBytes)); + return !GuestMemory::UnchangedSince(texture->descriptor.baseAddress, static_cast(texture->guestBytes), texture->generation); + }); + return adjacent; +} + +void StorageTexture::advanceAdjacent(const std::vector& adjacent, std::uint64_t now, std::uint64_t firstBlock, std::uint64_t lastBlock) { + static const bool trace = std::getenv("APS5_TRACE_FLUSH") != nullptr; + constexpr std::uint64_t block = 65536; + for (const auto& [texture, seen] : adjacent) { + // An image stored or refreshed meanwhile set its own generation (an old one on purpose when + // it kept blocks for the CPU): only the value seen at the check is advanced. + if (texture->generation != seen || now <= seen) continue; + // A CPU write to the neighbour landing since the check may have been stamped by another + // thread's walk of its range at a value in (seen, now]; the advance would hide it. Its blocks + // outside this write-back's span carry no stamp of ours, so they are re-checked against the + // seen generation (a stamp scan, no walk). A write inside the shared block itself in that + // window is indistinguishable from this write-back's stamp, as it is for the block rule. + const auto begin = texture->descriptor.baseAddress; + const auto end = begin + texture->guestBytes; + const auto beforeEnd = std::min(end, firstBlock * block); + const auto afterBegin = std::max(begin, (lastBlock + 1) * block); + if (begin < beforeEnd && !GuestMemory::UnchangedSince(begin, static_cast(beforeEnd - begin), seen)) continue; + if (afterBegin < end && !GuestMemory::UnchangedSince(afterBegin, static_cast(end - afterBegin), seen)) continue; + if (trace) std::fprintf(stderr, "[flush] adjacent pending image 0x%llx+0x%llx advanced past a write-back's stamps (generation %llu -> %llu)\n", static_cast(begin), static_cast(texture->guestBytes), static_cast(seen), static_cast(now)); + texture->generation = now; + } +} + +void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + PhaseTimer timer; + struct Account { + bool enabled; + PhaseTimer& timer; + ~Account() { if (enabled) Profile().storageWriteBack += timer.lap(); } + } account{profile, timer}; + const auto elementBytes = BytesPerElement(descriptor.format); + // 64 KiB blocks the CPU wrote since the image was last in sync keep the CPU's bytes: the game may + // have reused the memory for something else entirely (see skipWrittenSince). Zero means no + // tracking, and everything is stored. + std::vector> keep; + bool skippedAny = false; + std::size_t skipped = 0; + { + const auto begin = descriptor.baseAddress; + const auto end = begin + guestBytes; + if (skipWrittenSince != 0) { + constexpr std::uint64_t block = 65536; + // The keep-blocks decision must see every CPU write up to now, so it bypasses the + // per-packet collect memo (a Refresh collect earlier in the same packet would satisfy it). + GuestMemory::CollectWritesUncached(begin, static_cast(guestBytes)); + for (auto at = begin & ~(block - 1); at < end; at += block) { + const auto from = std::max(at, begin); + const auto to = std::min(at + block, end); + if (from >= to) continue; + if (!GuestMemory::UnchangedSince(from, static_cast(to - from), skipWrittenSince)) { + skippedAny = true; + ++skipped; + continue; + } + if (!keep.empty() && keep.back().second == from) keep.back().second = to; + else keep.emplace_back(from, to); + } + } else { + keep.emplace_back(begin, end); + } + } + // Debug aid: APS5_TRACE_FLUSH also names the blocks a write-back leaves to the CPU. + static const bool traceKept = std::getenv("APS5_TRACE_FLUSH") != nullptr; + if (traceKept && skippedAny) { + const auto first = keep.empty() ? descriptor.baseAddress + guestBytes : keep.front().first; + std::fprintf(stderr, "[flush] image 0x%llx+0x%llx keeps %zu CPU-written 64 KiB blocks (first stored byte at +0x%llx, %zu ranges stored, generation %llu)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), skipped, static_cast(first - descriptor.baseAddress), keep.size(), static_cast(skipWrittenSince)); + } + // Taken before this write-back stamps anything (see advanceAdjacent), with the 64 KiB block span + // its stamps cover. + const auto adjacent = adjacentPendingUnchanged(); + const auto firstBlock = descriptor.baseAddress / 65536; + const auto lastBlock = guestBytes == 0 ? firstBlock : (descriptor.baseAddress + guestBytes - 1) / 65536; + if (const auto* import = HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes))) { + // The surface lives in host-imported memory: the retiler writes into device scratch and the + // untouched blocks are copied into the imported bytes in place, recorded behind the work that + // produced the image; nothing crosses to the CPU. + auto linear = std::make_shared(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + auto tiledScratch = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + detiler.BeginBatch(); + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + if (recorder != nullptr) { + commands = recorder->Commands(); + recorder->Keep(linear); + recorder->Keep(tiledScratch); + // The image itself must outlive the recorded retile: the cache may evict it right after. + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } + const auto importOffset = descriptor.baseAddress - import->base; + VkImageMemoryBarrier toSource{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toSource.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toSource.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toSource.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.image = image; + toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + const auto regions = CopyRegions(); + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (const auto& mip : mips) { + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiledScratch->Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick); + } + } + { + const auto scratchDone = WholeBufferBarrier(tiledScratch->Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &scratchDone, 0, nullptr); + std::vector copies; + for (const auto& [from, to] : keep) copies.push_back({from - descriptor.baseAddress, importOffset + (from - descriptor.baseAddress), to - from}); + if (!copies.empty()) context.Function("vkCmdCopyBuffer")(commands, tiledScratch->Handle(), import->buffer, static_cast(copies.size()), copies.data()); + } + VkImageMemoryBarrier backToGeneral = toSource; + backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + backToGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + backToGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + backToGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral); + if (batch) batch->SubmitAndWait(); + else recorder->NotePendingWrite(descriptor.baseAddress, static_cast(guestBytes)); + if (profile) Profile().storageGpu += timer.lap(); + // The guest bytes now differ from `original`; other caches of the range see the write. With + // every block stored the image is current at a fresh generation; with blocks kept for the CPU + // the image is stale there, so the old generation stays and the next use re-uploads. + originalValid = false; + // The keys go the same way as the texels: a fill recorded behind the retile when the + // metadata is host-imported (no CPU wait for the title's key-writing kernels), else a CPU + // store (APS5_CPU_DCC_KEYS=1 keeps the CPU store; see DccMetadata.hpp). + MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + uploadedKeys = DccKeys::Uncompressed; + for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); + if (!skippedAny || !adjacent.empty()) { + // The walk covers this surface's pages only: an adjacent image's later CPU write is + // stamped newer than this value when its own range is collected. No CPU wrote the pages + // here (MarkWritten made the only stamps), so the memoized collect is exact. + const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + if (!skippedAny) generation = now; + advanceAdjacent(adjacent, now, firstBlock, lastBlock); + } + ++Profile().storageDirectWriteBacks; + return; + } + if (!originalValid) { + // The image was cleared on the GPU without reading the guest bytes; the store below compares + // against them, so read them now. + GuestMemory::ReadCommitted(descriptor.baseAddress, original); + originalValid = true; + } + DeviceBuffer linear(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + DeviceBuffer tiled(context, original.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + Buffer host(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (profile) Profile().storageAlloc += timer.lap(); + // Start from the uploaded bytes so padding and untouched texels keep their guest values. + std::memcpy(host.Bytes().data(), original.data(), original.size()); + if (profile) Profile().storageHostCopy += timer.lap(); + detiler.BeginBatch(); + CommandBatch batch(context); + const auto commands = batch.Handle(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, host.Handle(), 0, tiled.Handle(), 0, original.size()); + VkImageMemoryBarrier toSource{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toSource.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toSource.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toSource.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.image = image; + toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + const auto regions = CopyRegions(); + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear.Handle(), static_cast(regions.size()), regions.data()); + const VkBufferMemoryBarrier toShader[] = {WholeBufferBarrier(linear.Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT), WholeBufferBarrier(tiled.Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT)}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 2, toShader, 0, nullptr); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (const auto& mip : mips) { + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick); + } + } + // Debug aid: APS5_DUMP_STORAGE= saves that storage image's first mip after each of + // its first 8 write-backs as storage_
_.raw (u32 width, height, VkFormat, then rows). + static const std::uint64_t dumpAddress = [] { const char* text = std::getenv("APS5_DUMP_STORAGE"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); + static int dumps = 0; + std::unique_ptr dump; + if (dumpAddress != 0 && descriptor.baseAddress == dumpAddress && dumps < 8) { + dump = std::make_unique(context, static_cast(mips[0].linearSize), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, linear.Handle(), mips[0].linearOffset, dump->Handle(), 0, mips[0].linearSize); + } + const auto toCopy = WholeBufferBarrier(tiled.Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkImageMemoryBarrier backToGeneral = toSource; + backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + backToGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + backToGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + backToGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 1, &toCopy, 1, &backToGeneral); + CopyBuffer(context, commands, tiled.Handle(), 0, host.Handle(), 0, original.size()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + APS5_LOG_CHARS_OUT("StorageTexture writeback submit"); + batch.SubmitAndWait(); + APS5_LOG_CHARS_OUT("StorageTexture writeback done"); + if (profile) Profile().storageGpu += timer.lap(); + if (dump) { + char name[64]; + std::snprintf(name, sizeof(name), "storage_%llx_%d.raw", static_cast(descriptor.baseAddress), dumps++); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t header[3] = {mips[0].pitchBytes / static_cast(elementBytes), mips[0].height, static_cast(storageFormat)}; + std::fwrite(header, sizeof(header), 1, file); + std::fwrite(dump->Bytes().data(), 1, dump->Bytes().size(), file); + std::fclose(file); + } + } + // Store only blocks that changed so concurrent CPU writes to untouched texels survive. + const auto current = host.Bytes(); + if (skipWrittenSince == 0) { + GuestMemory::WriteChangedCommitted(descriptor.baseAddress, current, original); + } else { + constexpr std::uint64_t block = 65536; + const auto begin = descriptor.baseAddress; + const auto end = begin + guestBytes; + for (auto at = begin & ~(block - 1); at < end; at += block) { + const auto from = std::max(at, begin); + const auto to = std::min(at + block, end); + if (from >= to || !GuestMemory::UnchangedSince(from, static_cast(to - from), skipWrittenSince)) continue; + const auto offset = static_cast(from - begin); + const auto length = static_cast(to - from); + GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span(original).subspan(offset, length)); + } + } + std::memcpy(original.data(), current.data(), original.size()); + // The texels now hold the whole image, so later reads must see them rather than a fast clear + // (the keys may be host-imported although the texels were not: then a recorded fill, else a CPU store). + MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + uploadedKeys = DccKeys::Uncompressed; + // The store above is the only write to these pages, so `original` is current at a fresh + // generation, unless blocks were kept for the CPU: then the image is stale there and the old + // generation stays so the next use re-uploads. + if (skippedAny) originalValid = false; + if (!skippedAny || !adjacent.empty()) { + // As in the GPU-direct path, but the store's memcpy dirtied this surface's pages in the write + // watch: a memoized collect would leave them for the next walk to stamp newer than this value + // (the shared edge block included, undoing the advance), so the walk is made here and + // consumes them. + const auto now = GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); + if (!skippedAny) generation = now; + advanceAdjacent(adjacent, now, firstBlock, lastBlock); + } + if (profile) Profile().storageStore += timer.lap(); +} + +StorageTexture::~StorageTexture() { + // Cache eviction flushes first; anything still pending here is being torn down with the device. + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + if (dirty) { + dirty = false; + std::erase(pending.textures, this); + static std::atomic reports{0}; + if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx destroyed with GPU results pending\n", static_cast(descriptor.baseAddress)); + } + } + release(); +} + +void StorageTexture::release() noexcept { + for (const auto& [mip, extra] : extraViews) context.Function("vkDestroyImageView")(context.device, extra, nullptr); + extraViews.clear(); + for (const auto& [format, attachment] : attachmentViews) context.Function("vkDestroyImageView")(context.device, attachment, nullptr); + attachmentViews.clear(); + if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); + if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); + if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); +} + +std::uint64_t StorageTexture::GuestBytes() const { + return guestBytes; +} + +VkImageView StorageTexture::View() const { + return view; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp index 4091d2860b..28e9700838 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp @@ -1,8 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureCache.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include #include #include @@ -29,26 +27,9 @@ std::shared_ptr TextureCache::Get(std::span words, trim(); std::array key; std::copy(words.begin(), words.end(), key.begin()); - auto source = context.renderCache ? context.renderCache->Find(resource.baseAddress) : nullptr; - if (source) { - const auto& color = source->Description(); - const auto compatibleTiling = (color.tileMode == ColorTileMode::RenderTarget && resource.tileMode == TextureTileMode::RenderTarget64KB) || (color.tileMode == ColorTileMode::Linear && resource.tileMode == TextureTileMode::kLinear); - if (!compatibleTiling || resource.width != color.extent.width || resource.height != color.extent.height || resource.dimension != TextureDimension::k2D || resource.mipCount != 1 || resource.baseLevel != 0 || resource.baseArray != 0 || IsBlockCompressed(resource.format) || BytesPerElement(resource.format) != 4) source.reset(); - } for (auto it = entries.begin(); it != entries.end(); ++it) { if (it->descriptor != key) continue; - if (source || it->generation != 0) { - if (source && it->source.lock() == source && it->generation == source->Generation()) { - auto result = it->texture; - entries.splice(entries.end(), entries, it); - return result; - } - retainedBytes -= it->snapshot.size() + it->texture->AllocationBytes(); - entries.erase(it); - break; - } - GuestMemory::CheckRange(reinterpret_cast(resource.baseAddress), it->snapshot.size(), 1); - if (std::memcmp(reinterpret_cast(resource.baseAddress), it->snapshot.data(), it->snapshot.size()) == 0) { + if (GuestMemory::EqualsCommitted(resource.baseAddress, it->snapshot)) { auto result = it->texture; entries.splice(entries.end(), entries, it); return result; @@ -57,19 +38,12 @@ std::shared_ptr TextureCache::Get(std::span words, entries.erase(it); break; } - if (source) { - auto texture = std::make_shared(context, source, resource, components); - entries.push_back({key, {}, texture, source, source->Generation()}); - retainedBytes += texture->AllocationBytes(); - trim(); - return texture; - } const auto mips = ComputeMipLayout(resource.tileMode, resource.format, resource.width, resource.height, resource.mipCount); const auto layers = resource.dimension == TextureDimension::k2DArray || resource.dimension == TextureDimension::kCube ? resource.depthOrLastArray + 1u : 1u; const auto bytes = ComputeSurfaceSize(mips, layers); Require(bytes != 0 && bytes <= std::numeric_limits::max(), "texture cache surface size overflow"); std::vector snapshot(static_cast(bytes)); - GuestMemory::Read(resource.baseAddress, snapshot, 1); + GuestMemory::ReadCommitted(resource.baseAddress, snapshot); auto texture = std::make_shared(context, *context.detiler, resource, components, snapshot); const auto retained = snapshot.size() + texture->AllocationBytes(); entries.push_back({key, std::move(snapshot), texture}); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index 0b65f2b3c4..b6bcab80d2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -1,4 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetile_spv.h" #include #include @@ -22,7 +24,7 @@ struct Push { std::uint32_t tailX; std::uint32_t tailY; std::uint32_t elementBytes; - std::uint32_t arrayLayer; + std::uint32_t slice; }; std::uint32_t BlockBytesFor(TextureTileMode tileMode) { @@ -30,14 +32,17 @@ std::uint32_t BlockBytesFor(TextureTileMode tileMode) { case TextureTileMode::kLinear: return 0u; case TextureTileMode::kStandard256B: return 256u; case TextureTileMode::kStandard4KB: return 4096u; - case TextureTileMode::RenderTarget64KB: - case TextureTileMode::kStandard64KB: return 65536u; + case TextureTileMode::kStandard64KB: + case TextureTileMode::kZ64KBX: + case TextureTileMode::kS64KBX: + case TextureTileMode::kD64KBX: + case TextureTileMode::kR64KBX: return 65536u; } throw std::runtime_error("AGC graphics: TextureDetiler encountered an unknown tile mode"); } -std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes) { - return (static_cast(tileMode) << 8) | elementBytes; +std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { + return (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; } } @@ -87,19 +92,38 @@ void TextureDetiler::release() noexcept { if (descriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr); } -VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes) { +VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { Require(std::has_single_bit(elementBytes) && elementBytes <= 16u, "unsupported element size for texture detiling"); - const auto key = PipelineKey(tileMode, elementBytes); + const auto key = PipelineKey(tileMode, elementBytes, retile, thick); for (const auto& entry : pipelines) { if (entry.first == key) return entry.second; } - const std::uint32_t values[3] = {elementBytes, BlockBytesFor(tileMode), tileMode == TextureTileMode::kLinear ? 0u : (tileMode == TextureTileMode::RenderTarget64KB ? 2u : 1u)}; - const VkSpecializationMapEntry entries[3] = {{0, 0, 4}, {1, 4, 4}, {2, 8, 4}}; + // Constants 0-3 select element size, block size, addressing family and direction; 4-19 carry the + // per-bit XOR equation for the equation family (2); 20-21 override the block extent in elements. + std::array values{elementBytes, BlockBytesFor(tileMode), tileMode == TextureTileMode::kLinear ? 0u : 1u, retile ? 1u : 0u}; + if (thick) { + const auto thickMode = tileMode == TextureTileMode::kStandard4KB ? 0x105u : tileMode == TextureTileMode::kStandard64KB ? 0x109u : 0u; + Require(thickMode != 0, "3D textures are only detiled from SW_4KB_S or SW_64KB_S"); + const auto* equation = FindTextureSwizzleEquation(thickMode, elementBytes); + Require(equation != nullptr, "no thick swizzle equation for the element size"); + values[2] = 2u; + std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4); + const auto extent = ThickBlockExtent(tileMode, elementBytes); + values[20] = extent[0]; + values[21] = extent[1]; + } else if (const auto mode = XorSwizzleMode(tileMode); mode != 0) { + const auto* equation = FindTextureSwizzleEquation(mode, elementBytes); + Require(equation != nullptr, "no swizzle equation for tile mode " + std::to_string(mode) + " at " + std::to_string(elementBytes) + " bytes per element"); + values[2] = 2u; + std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4); + } + std::array entries{}; + for (std::uint32_t index = 0; index < entries.size(); ++index) entries[index] = {index, index * 4u, 4}; VkSpecializationInfo specialization{}; - specialization.mapEntryCount = 3; - specialization.pMapEntries = entries; + specialization.mapEntryCount = static_cast(entries.size()); + specialization.pMapEntries = entries.data(); specialization.dataSize = sizeof(values); - specialization.pData = values; + specialization.pData = values.data(); VkPipelineShaderStageCreateInfo stage{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; stage.module = module; @@ -114,20 +138,20 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem return result; } -void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, std::uint32_t arrayLayer) { +void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile, std::uint32_t slice, bool thick) { Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); Require(source != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "texture detiling requires source and destination buffers"); Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); - const auto target = pipeline(tileMode, elementBytes); + const auto target = pipeline(tileMode, elementBytes, retile, thick); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment; const auto destinationDescriptorOffset = destinationOffset - destinationOffset % alignment; const auto sourceBase = sourceOffset - sourceDescriptorOffset; const auto destinationBase = destinationOffset - destinationDescriptorOffset; Require(sourceBase <= UINT32_MAX && destinationBase <= UINT32_MAX, "texture detiling buffer offset exceeds addressable range"); - const auto sourceRange = (sourceBase + layout.tiledSize + 3) / 4 * 4; - const auto destinationRange = (destinationBase + layout.linearSize + 3) / 4 * 4; + const auto sourceRange = (sourceBase + (retile ? layout.linearSize : layout.tiledSize) + 3) / 4 * 4; + const auto destinationRange = (destinationBase + (retile ? layout.tiledSize : layout.linearSize) + 3) / 4 * 4; Require(sourceRange <= context.limits.maxStorageBufferRange && destinationRange <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits"); const auto set = allocateSet(); const VkDescriptorBufferInfo sourceInfo{source, sourceDescriptorOffset, sourceRange}; @@ -159,7 +183,7 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode push.tailX = layout.tailX; push.tailY = layout.tailY; push.elementBytes = elementBytes; - push.arrayLayer = arrayLayer; + push.slice = slice; context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); const auto groupsX = (layout.width + 7u) / 8u; const auto groupsY = (layout.height + 7u) / 8u; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp index e9da522f70..ed3b905e4f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp @@ -1,8 +1,14 @@ +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" namespace AgcDriver::Graphics { void TextureDetiler::BeginBatch() { + GuestMemory::AssertGpuLockHeld("TextureDetiler::BeginBatch"); + // Sets of recorded work that has not completed stay allocated; the pools are recycled once the + // recorder is idle again. + if (const auto* recorder = Recorder::Active(); recorder != nullptr && !recorder->Idle()) return; for (const auto pool : descriptorPools) { Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool texture detiler"); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp index e56ecffee3..9afbe81677 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp @@ -45,8 +45,11 @@ BlockLayout GetBlockLayout(TextureTileMode tileMode, std::uint32_t bytesPerEleme case TextureTileMode::kLinear: throw std::runtime_error("AGC graphics: GetBlockLayout does not apply to linear tiling"); case TextureTileMode::kStandard256B: return {256u, 1u << kLog2BlockThin256B[index].width, 1u << kLog2BlockThin256B[index].height}; case TextureTileMode::kStandard4KB: return {4096u, 1u << kLog2BlockThin4KB[index].width, 1u << kLog2BlockThin4KB[index].height}; - case TextureTileMode::RenderTarget64KB: - case TextureTileMode::kStandard64KB: return {65536u, 1u << kLog2BlockThin64KB[index].width, 1u << kLog2BlockThin64KB[index].height}; + case TextureTileMode::kStandard64KB: + case TextureTileMode::kZ64KBX: + case TextureTileMode::kS64KBX: + case TextureTileMode::kD64KBX: + case TextureTileMode::kR64KBX: return {65536u, 1u << kLog2BlockThin64KB[index].width, 1u << kLog2BlockThin64KB[index].height}; } throw std::runtime_error("AGC graphics: GetBlockLayout encountered an unknown tile mode"); } @@ -92,8 +95,11 @@ bool GetMipTailLayout(TextureTileMode tileMode, const BlockLayout& block, std::u case TextureTileMode::kStandard4KB: out = MakeMipTailLayout(kMipTailThin4KB[index], block.blockWidth >> 1u, block.blockHeight); return true; - case TextureTileMode::RenderTarget64KB: case TextureTileMode::kStandard64KB: + case TextureTileMode::kZ64KBX: + case TextureTileMode::kS64KBX: + case TextureTileMode::kD64KBX: + case TextureTileMode::kR64KBX: out = MakeMipTailLayout(kMipTailThin64KB[index], block.blockWidth >> 1u, block.blockHeight); return true; } @@ -167,8 +173,10 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: const auto paddedWidth = AlignUp(std::max(ShiftCeil(elementsWidth0, level), 1u), block.blockWidth); const auto paddedHeight = AlignUp(std::max(ShiftCeil(elementsHeight0, level), 1u), block.blockHeight); mip.blocksPerRow = paddedWidth / block.blockWidth; + mip.pitchBytes = paddedWidth * bytesPerElement; mip.tiledSize = static_cast(paddedWidth) * paddedHeight * bytesPerElement; - mip.linearSize = static_cast(mip.width) * mip.height * bytesPerElement; + // The detiler writes linear rows at pitchBytes, so the region spans whole padded rows. + mip.linearSize = static_cast(mip.pitchBytes) * mip.height; mip.tail = false; mip.tailX = 0; mip.tailY = 0; @@ -182,8 +190,9 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: mip.width = TexelLevelDimension(width, level, texelWidth); mip.height = TexelLevelDimension(height, level, texelHeight); mip.blocksPerRow = 1u; + mip.pitchBytes = block.blockWidth * bytesPerElement; mip.tiledSize = block.blockSize; - mip.linearSize = static_cast(mip.width) * mip.height * bytesPerElement; + mip.linearSize = static_cast(mip.pitchBytes) * mip.height; mip.tail = true; mip.tailX = tail.locations[level - firstTailLevel].x; mip.tailY = tail.locations[level - firstTailLevel].y; @@ -192,10 +201,17 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: std::uint64_t offset = firstTailLevel < mipCount ? block.blockSize : 0; for (auto level = firstTailLevel; level-- > 0;) { mips[level].tiledOffset = offset; + mips[level].linearOffset = offset; offset += mips[level].tiledSize; } + // Tail mips share one tiled block but each needs its own linear region, placed after the chain. + std::uint64_t linearCursor = 0; + for (auto level = std::uint32_t{0}; level < firstTailLevel; ++level) linearCursor = std::max(linearCursor, mips[level].linearOffset + mips[level].linearSize); + linearCursor = std::max(linearCursor, offset); for (auto level = firstTailLevel; level < mipCount; ++level) { mips[level].tiledOffset = 0; + mips[level].linearOffset = linearCursor; + linearCursor += mips[level].linearSize; } Require(offset == blockSliceSize, "tiled texture mip chain geometry is inconsistent"); @@ -203,15 +219,6 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: Require(mip.width != 0 && mip.height != 0, "computed a zero-sized tiled texture mip level"); Require(mip.tiledSize != 0 && mip.linearSize != 0, "computed a zero-sized tiled texture mip level"); } - std::uint64_t linearOffset = 0; - for (auto& mip : mips) { - const auto alignment = std::max(bytesPerElement, 4u); - linearOffset = (linearOffset + alignment - 1u) / alignment * alignment; - mip.linearOffset = linearOffset; - mip.pitchBytes = mip.width * bytesPerElement; - mip.linearSize = (static_cast(mip.pitchBytes) * mip.height + alignment - 1u) / alignment * alignment; - linearOffset += mip.linearSize; - } return mips; } @@ -225,14 +232,88 @@ std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint3 const auto texelWidth = BlockWidth(format); const auto texelHeight = BlockHeight(format); - if (tileMode == TextureTileMode::RenderTarget64KB) { - Require(!IsBlockCompressed(format), "render target tiling does not support block compressed formats"); - Require(format != 128 && format != 129 && format != 132, "texture format does not support render target tiling"); - } if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, texelWidth, texelHeight, width, height, mipCount); return ComputeTiledMipLayout(tileMode, bytesPerElement, texelWidth, texelHeight, width, height, mipCount); } +std::array ThickBlockExtent(TextureTileMode tileMode, std::uint32_t bytesPerElement) { + // addrlib Block4K_Log2_3d / Block64K_Log2_3d. + constexpr std::uint8_t thick4KB[5][3] = {{4, 4, 4}, {3, 4, 4}, {3, 4, 3}, {3, 3, 3}, {2, 3, 3}}; + constexpr std::uint8_t thick64KB[5][3] = {{6, 5, 5}, {5, 5, 5}, {5, 5, 4}, {5, 4, 4}, {4, 4, 4}}; + Require(std::has_single_bit(bytesPerElement) && bytesPerElement <= 16u, "unsupported bytes per element for thick texture geometry"); + const auto index = static_cast(std::countr_zero(bytesPerElement)); + switch (tileMode) { + case TextureTileMode::kStandard4KB: return {1u << thick4KB[index][0], 1u << thick4KB[index][1], 1u << thick4KB[index][2]}; + case TextureTileMode::kStandard64KB: return {1u << thick64KB[index][0], 1u << thick64KB[index][1], 1u << thick64KB[index][2]}; + default: throw std::runtime_error("AGC graphics: 3D textures are only supported linear or in SW_4KB_S / SW_64KB_S"); + } +} + +ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth) { + Require(width != 0 && height != 0 && depth != 0, "cannot compute layout for a zero-sized 3D texture"); + const auto bytesPerElement = BytesPerElement(format); + Require(BlockWidth(format) == 1u && BlockHeight(format) == 1u, "block-compressed 3D textures are not implemented"); + ThickLayout result{}; + result.depth = depth; + auto& mip = result.mip; + mip.width = width; + mip.height = height; + mip.tail = false; + if (tileMode == TextureTileMode::kLinear) { + const auto paddedWidth = AlignUp(width, CalcLinearBlockWidth(bytesPerElement)); + mip.blocksPerRow = paddedWidth; + mip.pitchBytes = paddedWidth * bytesPerElement; + mip.tiledSize = static_cast(mip.pitchBytes) * height; + result.blockDepth = 1; + result.slabBytes = mip.tiledSize; + } else { + const auto block = ThickBlockExtent(tileMode, bytesPerElement); + const auto paddedWidth = AlignUp(width, block[0]); + const auto paddedHeight = AlignUp(height, block[1]); + mip.blocksPerRow = paddedWidth / block[0]; + mip.pitchBytes = paddedWidth * bytesPerElement; + result.blockDepth = block[2]; + result.slabBytes = static_cast(mip.blocksPerRow) * (paddedHeight / block[1]) * (tileMode == TextureTileMode::kStandard4KB ? 4096u : 65536u); + mip.tiledSize = result.slabBytes; + } + mip.linearSize = static_cast(mip.pitchBytes) * height; + result.sliceLinearBytes = mip.linearSize; + result.guestBytes = result.slabBytes * ((depth + result.blockDepth - 1u) / result.blockDepth); + return result; +} + +SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor) { + SurfaceGeometry geometry; + if (descriptor.dimension == TextureDimension::k3D) { + Require(descriptor.mipCount == 1, "mipmapped 3D textures are not implemented"); + const auto depth = descriptor.depthOrLastArray + 1u; + const auto thick = ComputeThickLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, depth); + geometry.mips = {thick.mip}; + geometry.layers = depth; + geometry.imageDepth = depth; + geometry.guestBytes = thick.guestBytes; + geometry.sliceLinearBytes = thick.sliceLinearBytes; + geometry.thick = descriptor.tileMode != TextureTileMode::kLinear; + geometry.blockDepth = thick.blockDepth; + geometry.layerBytes = thick.slabBytes; + return geometry; + } + geometry.mips = ComputeMipLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, descriptor.mipCount); + const bool layered = descriptor.dimension == TextureDimension::k2DArray || descriptor.dimension == TextureDimension::kCube; + geometry.layers = layered ? descriptor.depthOrLastArray + 1u : 1u; + geometry.imageLayers = geometry.layers; + geometry.guestBytes = ComputeSurfaceSize(geometry.mips, geometry.layers); + geometry.layerBytes = geometry.guestBytes / geometry.layers; + for (const auto& mip : geometry.mips) geometry.sliceLinearBytes = std::max(geometry.sliceLinearBytes, mip.linearOffset + mip.linearSize); + return geometry; +} + +std::vector ComputeElementMipLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount) { + Require(width != 0 && height != 0 && mipCount != 0 && mipCount <= 16u, "invalid surface mip chain"); + if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, 1u, 1u, width, height, mipCount); + return ComputeTiledMipLayout(tileMode, bytesPerElement, 1u, 1u, width, height, mipCount); +} + std::uint64_t ComputeSurfaceSize(const std::vector& mips, std::uint32_t arrayLayers) { Require(!mips.empty(), "cannot compute surface size for an empty mip chain"); Require(arrayLayers != 0, "cannot compute surface size for zero array layers"); diff --git a/core/libs/prx/libSceAgcDriver/PostCompile.cmake b/core/libs/prx/libSceAgcDriver/PostCompile.cmake index 072eeed37f..b48b71293a 100644 --- a/core/libs/prx/libSceAgcDriver/PostCompile.cmake +++ b/core/libs/prx/libSceAgcDriver/PostCompile.cmake @@ -12,25 +12,19 @@ add_custom_command( foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/BufferPool.cpp Graphics/src/CommandCompletion.cpp) - endif() -endforeach() - -foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) - if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Execution/src/GuestGpuRange.cpp) + target_sources(${agcTarget} PRIVATE Graphics/src/BufferPool.cpp) endif() endforeach() foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/TextureDetilerDescriptors.cpp Graphics/src/TextureCache.cpp Graphics/src/RegisteredGpuMemory.cpp) + target_sources(${agcTarget} PRIVATE Graphics/src/TextureDetilerDescriptors.cpp Graphics/src/TextureCache.cpp) endif() endforeach() foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/GpuColorTransfer.cpp Graphics/src/ColorMemoryWriteback.cpp ${agcColorTransferHeader}) + target_sources(${agcTarget} PRIVATE Graphics/src/GpuColorTransfer.cpp ${agcColorTransferHeader}) endif() endforeach() @@ -41,14 +35,6 @@ if(TARGET agc_driver_visual_test) target_sources(agc_driver_visual_test PRIVATE Execution/src/DisplayBuffer.cpp) endif() -foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests) - if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/RenderCache.cpp Graphics/src/RenderMemoryOwnership.cpp Graphics/src/DrawQueue.cpp Graphics/src/DrawCompletion.cpp Graphics/src/RenderTexture.cpp Graphics/src/DescriptorCache.cpp) - endif() -endforeach() - -foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test) - if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Execution/src/PresentationImage.cpp Execution/src/DeviceSynchronization.cpp Graphics/src/GraphicsPipelineCache.cpp) - endif() -endforeach() +# Upstream's GPU-resident draw path (RenderCache, DrawQueue, RenderTexture, PresentationImage, +# GraphicsPipelineCache) targets the single-target Pipeline/GpuColorTransfer API; the multi-target +# draw path in Draw.cpp does not use it yet, so those sources stay out of the build. diff --git a/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp b/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp index 35e29c61c5..4de64c2dd8 100644 --- a/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp +++ b/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp @@ -5,16 +5,26 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +// Resource registration feeds GPU debugging tools, none of which are attached; the game treats this code as benign. +static constexpr int SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE = static_cast(0x8A6C9018); + extern "C" { -int APS5_VABI sceAgcDriverRegisterOwner(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceAgcDriverRegisterOwner(uint32_t* owner_handle, const char* name) { + (void)owner_handle; + (void)name; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } -int APS5_VABI sceAgcDriverRegisterResource(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceAgcDriverRegisterResource(uint32_t* resource_handle, uint32_t owner_handle, const void* memory, size_t size, const char* name, uint32_t type, uint64_t user_data) { + (void)resource_handle; + (void)owner_handle; + (void)memory; + (void)size; + (void)name; + (void)type; + (void)user_data; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } int APS5_VABI sceAgcDriverRegisterWorkloadStream(uint32_t stream_id, const void* stream) { @@ -25,14 +35,13 @@ int APS5_VABI sceAgcDriverRegisterWorkloadStream(uint32_t stream_id, const void* } int APS5_VABI sceAgcDriverUnregisterOwnerAndResources(uint32_t owner_handle) { - (void)owner_handle; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)owner_handle; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } -int APS5_VABI sceAgcDriverUnregisterResource(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceAgcDriverUnregisterResource(uint32_t resource_handle) { + (void)resource_handle; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } } diff --git a/core/libs/prx/libSceAgcDriver/Unimplemented.cpp b/core/libs/prx/libSceAgcDriver/Unimplemented.cpp new file mode 100644 index 0000000000..8166a546fa --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Unimplemented.cpp @@ -0,0 +1,112 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI sceAgcDriverFindResourcesPublic() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetDefaultOwner() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetOwnerName() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceBaseAddressAndSizeInBytes() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceName() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceRegistrationMaxNameLength() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceShaderGuid() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceType() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceUserData() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetShaderDebuggingStatus() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverRegisterDefaultOwner(uint32_t* owner_handle) { + (void)owner_handle; + return static_cast(0x8A6C9018); +} + +int APS5_VABI sceAgcDriverRegisterGdsResource() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverSetResourceUserData() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverSetValidationErrorOutputFrequency() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverUnregisterAllResourcesForOwner() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverUnregisterWorkloadStream() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("+iAOE3jCnkk", sceAgcDriverUnknown__PiAOE3jCnkk); +int APS5_VABI sceAgcDriverUnknown__PiAOE3jCnkk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("ICaGtkEIXTk", sceAgcDriverUnknown_ICaGtkEIXTk); +int APS5_VABI sceAgcDriverUnknown_ICaGtkEIXTk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("WHIOMbb+iIU", sceAgcDriverUnknown_WHIOMbb_PiIU); +int APS5_VABI sceAgcDriverUnknown_WHIOMbb_PiIU() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("chJWZcNSzjk", sceAgcDriverUnknown_chJWZcNSzjk); +int APS5_VABI sceAgcDriverUnknown_chJWZcNSzjk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp index 616cb47b43..fe60064b00 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp @@ -149,21 +149,17 @@ void RunGuestSamplerResourceTests() { badUnorm.forceUnormCoords = true; rejectFields(badUnorm, "unnormalized coordinates"); - Fields badThreshold = base; - badThreshold.anisoThreshold = 1; - rejectFields(badThreshold, "anisotropy threshold override"); - Fields badSrgb = base; badSrgb.forceSrgb = true; rejectFields(badSrgb, "forces sRGB decoding"); - Fields badAnisoBias = base; - badAnisoBias.anisoBias = 1; - rejectFields(badAnisoBias, "anisotropy bias"); - - Fields badTrunc = base; - badTrunc.truncCoord = true; - rejectFields(badTrunc, "coordinate truncation"); + Fields truncated = base; + truncated.truncCoord = true; + truncated.perfMip = 1; + truncated.perfZ = 1; + truncated.anisoThreshold = 1; + truncated.anisoBias = 1; + static_cast(DecodeSamplerResource(pack(truncated))); Fields badCubeWrap = base; badCubeWrap.disableCubeWrap = true; @@ -177,13 +173,6 @@ void RunGuestSamplerResourceTests() { badDegamma.disableDegamma = true; rejectFields(badDegamma, "disables degamma"); - Fields badPerf = base; - badPerf.perfMip = 1; - rejectFields(badPerf, "performance counters"); - badPerf = base; - badPerf.perfZ = 1; - rejectFields(badPerf, "performance counters"); - Fields badLodBiasSec = base; badLodBiasSec.lodBiasSec = 1; rejectFields(badLodBiasSec, "secondary LOD bias"); diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp index e22ad9c316..eedc6e2716 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp @@ -179,9 +179,12 @@ void RunGuestTextureResourceTests() { badMinLod.minLod = 1; rejectFields(badMinLod, "nonzero minimum LOD clamp"); - Fields badMinLodWarn = base; - badMinLodWarn.minLodWarn = 1; - rejectFields(badMinLodWarn, "minimum LOD warning threshold"); + // Streaming feedback fields decode; nothing is reported back. + Fields feedback = base; + feedback.minLodWarn = 1; + feedback.mipStatsCntId = 1; + feedback.mipStatsCntEn = true; + static_cast(DecodeTextureResource(pack(feedback))); const auto unmodulated = DecodeTextureResource(pack(base)); for (std::uint32_t perfMod = 0; perfMod < 8; ++perfMod) { @@ -196,13 +199,6 @@ void RunGuestTextureResourceTests() { rejectFields(modulated, "corner sampling"); } - Fields badMipStats = base; - badMipStats.mipStatsCntId = 1; - rejectFields(badMipStats, "mip statistics counters"); - badMipStats = base; - badMipStats.mipStatsCntEn = true; - rejectFields(badMipStats, "mip statistics counters"); - Fields badCorner = base; badCorner.cornerSample = true; rejectFields(badCorner, "corner sampling"); diff --git a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp new file mode 100644 index 0000000000..4536f474b2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp @@ -0,0 +1,125 @@ +// agc_shader_replay ...: replays compute recompile requests the driver saved with +// APS5_DUMP_SHADERS=1, running the same resource analysis and recompile steps without the game. +#include "ControlFlow/RequestSerializer.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "Recompiler.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" + +#if ANYPS5_ENABLE_SPIRV_TOOLS +#include "spirv-tools/libspirv.hpp" +#endif + +#include +#include +#include +#include +#include + +namespace { + +std::string ReadText(const char* path) { + std::ifstream file(path, std::ios::binary); + std::ostringstream stream; + stream << file.rdbuf(); + auto text = stream.str(); + while (!text.empty() && (text.back() == '\n' || text.back() == '\r' || text.back() == ' ')) text.pop_back(); + return text; +} + +// The error text without the serialized request that Recompile appends. +std::string FirstLine(const std::string& text) { + const auto request = text.find("\nRecompileRequest:"); + auto shown = request == std::string::npos ? text : text.substr(0, request); + if (shown.size() > 2000) shown.resize(2000); + return shown; +} + +bool g_disassemble = false; +bool g_assembly = false; +bool g_memory = false; + +bool Replay(const char* path) { + const auto request = ShaderRecompiler::RequestSerializer{}.Deserialize(ReadText(path)); + std::printf("%s: %zu code words, %zu user data, %zu memory regions, wave%u\n", path, request.request.shader.code.size(), request.request.context.userData.size(), request.request.context.memory.size(), request.request.context.waveSize); + if (request.request.context.compute.has_value()) { + const auto& compute = *request.request.context.compute; + std::printf(" compute: threads %ux%ux%u, lds %u dwords, group ids %d%d%d, tg size %d, thread id components %u\n", compute.numThreads[0], compute.numThreads[1], compute.numThreads[2], compute.ldsSizeDwords, compute.groupIdEnable[0], compute.groupIdEnable[1], compute.groupIdEnable[2], compute.tgSizeEnable, compute.threadIdComponentCount); + } + if (g_memory) { + // --mem: the captured inputs. User data words are printed; each memory region is written to + // mem__.bin next to the request for inspection with other tools. + const auto& context = request.request.context; + for (std::size_t i = 0; i < context.userData.size(); ++i) std::printf(" user[%zu] = 0x%08x\n", i, context.userData[i]); + for (const auto& region : context.memory) { + char name[96]; + std::snprintf(name, sizeof(name), "mem_%llx_%llx.bin", static_cast(request.request.shader.codeAddress), static_cast(region.guestAddress)); + if (std::FILE* file = std::fopen(name, "wb")) { + std::fwrite(region.bytes.data(), 1, region.bytes.size(), file); + std::fclose(file); + } + std::printf(" region 0x%llx + 0x%zx -> %s\n", static_cast(region.guestAddress), region.bytes.size(), name); + } + } + if (g_assembly) { + const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(request.request.shader.code); + // Raw first words carry what the text omits (branch offsets, waitcnt fields). + for (const auto& instruction : program.instructions) std::printf("raw=%08x %s\n", instruction.rawWords[0], ShaderRecompiler::RdnaInstructionToString(instruction).c_str()); + } + try { + auto program = ShaderRecompiler::PrepareResourceProgram(request.request); + constexpr ShaderRecompiler::ResourceMaterializer materializer; + static_cast(materializer.ExtractPlan(program)); + } catch (const std::exception& error) { + std::printf(" resource analysis failed: %s\n", FirstLine(error.what()).c_str()); + return false; + } + try { + const auto result = ShaderRecompiler::Recompile(request.request); + std::printf(" recompiled: %zu SPIR-V words\n", result.spirv.size()); +#if ANYPS5_ENABLE_SPIRV_TOOLS + if (g_disassemble) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string text; + if (tools.Disassemble(result.spirv, &text, SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES)) std::printf("%s\n", text.c_str()); + } +#else + if (g_disassemble) std::printf(" (--dis needs ANYPS5_ENABLE_SPIRV_TOOLS=ON)\n"); +#endif + return true; + } catch (const std::exception& error) { + std::printf(" recompile failed: %s\n", FirstLine(error.what()).c_str()); + return false; + } +} + +} + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: agc_shader_replay [--dis] ...\n the driver writes shader_
.req files when APS5_DUMP_SHADERS is set\n"); + return 2; + } + int failures = 0; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--dis") { + g_disassemble = true; + continue; + } + if (std::string(argv[i]) == "--asm") { + g_assembly = true; + continue; + } + if (std::string(argv[i]) == "--mem") { + g_memory = true; + continue; + } + try { + if (!Replay(argv[i])) ++failures; + } catch (const std::exception& error) { + std::printf("%s: could not load request: %s\n", argv[i], FirstLine(error.what()).c_str()); + ++failures; + } + } + return failures == 0 ? 0 : 1; +} From 91557ebade9cff2729cb04998c9d3c572fc6085b Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:27:49 -0700 Subject: [PATCH 04/93] feat(audio): AudioOut2 SDL output queue, ATRAC9 decoder in libSceAjm.native - AudioOut2: one SDL device per context; the hardware grain queue level is derived from the SDL queued bytes so the title's poll-and-push loop runs at real time; ports are mixed per push with 7.1 folded to stereo. - libSceAjm.native: instance and batch APIs; ATRAC9 decoding with variable-size frames inside superframes, padding, gapless parameters and handle re-creation on re-initialisation. - AudioOut v1: sceAudioOutOutput(handle, NULL) drains instead of throwing. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/libs/prx/libSceAjm.native/CMakeLists.txt | 31 + core/libs/prx/libSceAjm.native/Export.cpp | 194 +----- core/libs/prx/libSceAjm.native/src/Ajm.cpp | 573 ++++++++++++++++++ core/libs/prx/libSceAudioOut/src/AudioOut.cpp | 23 +- .../libSceAudioOut/src/AudioOut2Context.cpp | 265 +++++++- .../libSceAudioOut/src/AudioOut2Internal.hpp | 95 +++ .../prx/libSceAudioOut/src/AudioOut2Port.cpp | 162 ++++- .../src/AudioOut2SpeakerArray.cpp | 10 +- .../libSceAudioOut/src/AudioOut2System.cpp | 5 +- .../prx/libSceAudioOut/src/AudioOut2User.cpp | 10 +- 10 files changed, 1152 insertions(+), 216 deletions(-) create mode 100644 core/libs/prx/libSceAjm.native/CMakeLists.txt create mode 100644 core/libs/prx/libSceAjm.native/src/Ajm.cpp create mode 100644 core/libs/prx/libSceAudioOut/src/AudioOut2Internal.hpp diff --git a/core/libs/prx/libSceAjm.native/CMakeLists.txt b/core/libs/prx/libSceAjm.native/CMakeLists.txt new file mode 100644 index 0000000000..83bd7327e7 --- /dev/null +++ b/core/libs/prx/libSceAjm.native/CMakeLists.txt @@ -0,0 +1,31 @@ +enable_language(C) + +set(LIBATRAC9_DIR ${CMAKE_SOURCE_DIR}/3rdparty/LibAtrac9/C/src) +add_library(atrac9 STATIC EXCLUDE_FROM_ALL + ${LIBATRAC9_DIR}/band_extension.c + ${LIBATRAC9_DIR}/bit_allocation.c + ${LIBATRAC9_DIR}/bit_reader.c + ${LIBATRAC9_DIR}/decinit.c + ${LIBATRAC9_DIR}/decoder.c + ${LIBATRAC9_DIR}/huffCodes.c + ${LIBATRAC9_DIR}/imdct.c + ${LIBATRAC9_DIR}/libatrac9.c + ${LIBATRAC9_DIR}/quantization.c + ${LIBATRAC9_DIR}/scale_factors.c + ${LIBATRAC9_DIR}/tables.c + ${LIBATRAC9_DIR}/unpack.c + ${LIBATRAC9_DIR}/utility.c +) +set_target_properties(atrac9 PROPERTIES POSITION_INDEPENDENT_CODE ON) + +add_library(libSceAjm.native SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/Ajm.cpp +) + +target_include_directories(libSceAjm.native PRIVATE + ${LIBS_INCLUDE_DIR} + ${LIBATRAC9_DIR} +) +target_link_libraries(libSceAjm.native PRIVATE atrac9 libc libkernel) +configure_windows_unwind(libSceAjm.native) diff --git a/core/libs/prx/libSceAjm.native/Export.cpp b/core/libs/prx/libSceAjm.native/Export.cpp index 66117f2b5f..c2150a64a3 100644 --- a/core/libs/prx/libSceAjm.native/Export.cpp +++ b/core/libs/prx/libSceAjm.native/Export.cpp @@ -2,36 +2,22 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include +#include + +// Entry points this library does not implement: reported before the unimplemented-call exception so an +// APS5_TRACE_AJM run shows which one a title reached. +static void AjmStub(const char* name) { + std::fprintf(stderr, "[ajm] unimplemented %s called\n", name); + NotImplemented_nid_no_patch(name); +} extern "C" { int APS5_VABI sceAjmBatchCancel(uint32_t context, uint32_t batch) { (void)context; (void)batch; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmBatchErrorDump(const AjmBatchInfo* info, AjmBatchError* error) { - (void)info; - (void)error; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmBatchInitialize(void* buffer, size_t size, AjmBatchInfo* info) { - (void)buffer; - (void)size; - (void)info; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmBatchJobClearContext(AjmBatchInfo* info, uint32_t instance, void* result) { - (void)info; - (void)instance; - (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -43,7 +29,7 @@ int APS5_VABI sceAjmBatchJobControl(AjmBatchInfo* info, uint32_t instance, uint6 (void)sideband_input_size; (void)sideband_output; (void)sideband_output_size; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -55,7 +41,7 @@ int APS5_VABI sceAjmBatchJobDecode(AjmBatchInfo* info, uint32_t instance, const (void)pcm_output; (void)pcm_output_size; (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -67,7 +53,7 @@ int APS5_VABI sceAjmBatchJobDecodeSingle(AjmBatchInfo* info, uint32_t instance, (void)pcm_output; (void)pcm_output_size; (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -79,7 +65,7 @@ int APS5_VABI sceAjmBatchJobDecodeSplit(AjmBatchInfo* info, uint32_t instance, c (void)output_buffers; (void)output_buffers_num; (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -91,7 +77,7 @@ int APS5_VABI sceAjmBatchJobEncode(AjmBatchInfo* info, uint32_t instance, const (void)bitstream_output; (void)bitstream_output_size; (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -100,7 +86,7 @@ int APS5_VABI sceAjmBatchJobGetCodecInfo(AjmBatchInfo* info, uint32_t instance, (void)instance; (void)result; (void)result_size; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -108,7 +94,7 @@ int APS5_VABI sceAjmBatchJobGetGaplessDecode(AjmBatchInfo* info, uint32_t instan (void)info; (void)instance; (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -116,7 +102,7 @@ int APS5_VABI sceAjmBatchJobGetInfo(AjmBatchInfo* info, uint32_t instance, void* (void)info; (void)instance; (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -124,25 +110,7 @@ int APS5_VABI sceAjmBatchJobGetResampleInfo(AjmBatchInfo* info, uint32_t instanc (void)info; (void)instance; (void)result; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmBatchJobGetStatistics(AjmBatchInfo* info, float interval, void* result) { - (void)info; - (void)interval; - (void)result; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmBatchJobInitialize(AjmBatchInfo* info, uint32_t instance, const void* codec_parameters, size_t codec_parameters_size, void* result) { - (void)info; - (void)instance; - (void)codec_parameters; - (void)codec_parameters_size; - (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -156,31 +124,7 @@ int APS5_VABI sceAjmBatchJobRun(AjmBatchInfo* info, uint32_t instance, uint64_t (void)data_output_size; (void)sideband_output; (void)sideband_output_size; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmBatchJobRunSplit(AjmBatchInfo* info, uint32_t instance, uint64_t flags, const AjmBuffer* input_buffers, size_t input_buffers_num, const AjmBuffer* output_buffers, size_t output_buffers_num, void* sideband_output, size_t sideband_output_size) { - (void)info; - (void)instance; - (void)flags; - (void)input_buffers; - (void)input_buffers_num; - (void)output_buffers; - (void)output_buffers_num; - (void)sideband_output; - (void)sideband_output_size; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmBatchJobSetGaplessDecode(AjmBatchInfo* info, uint32_t instance, const void* gapless_decode, int reset, void* result) { - (void)info; - (void)instance; - (void)gapless_decode; - (void)reset; - (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -190,7 +134,7 @@ int APS5_VABI sceAjmBatchJobSetResampleParameters(AjmBatchInfo* info, uint32_t i (void)ratio; (void)flags; (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } @@ -201,99 +145,19 @@ int APS5_VABI sceAjmBatchJobSetResampleParametersEx(AjmBatchInfo* info, uint32_t (void)ratio_change_per_sample; (void)flags; (void)result; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return 0; } -int APS5_VABI sceAjmBatchStart(uint32_t context, const AjmBatchInfo* info, int priority, AjmBatchError* error, uint32_t* batch) { - (void)context; - (void)info; - (void)priority; +const char* APS5_VABI sceAjmStrError(int error) { (void)error; - (void)batch; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmBatchWait(uint32_t context, uint32_t batch, uint32_t timeout, AjmBatchError* error) { - (void)context; - (void)batch; - (void)timeout; - (void)error; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmDecAt9ParseConfigData(const void* config_data, AjmDecAt9ConfigDataInfo* config_info) { - (void)config_data; - (void)config_info; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmFinalize(uint32_t context) { - (void)context; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmInitialize(int64_t reserved, uint32_t* context) { - (void)reserved; - (void)context; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmInstanceCreate(uint32_t context, uint32_t codec, uint64_t flags, uint32_t* instance) { - (void)context; - (void)codec; - (void)flags; - (void)instance; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmInstanceDestroy(uint32_t context, uint32_t instance) { - (void)context; - (void)instance; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmMemoryRegister(uint32_t context, void* ptr, size_t pages) { - (void)context; - (void)ptr; - (void)pages; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmMemoryUnregister(uint32_t context, void* ptr) { - (void)context; - (void)ptr; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmModuleRegister(uint32_t context, uint32_t codec, int64_t reserved) { - (void)context; - (void)codec; - (void)reserved; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceAjmModuleUnregister(uint32_t context, uint32_t codec) { - (void)context; - (void)codec; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -const char* sceAjmStrError(int error) { - (void)error; - NotImplemented_nid_no_patch(__func__); + AjmStub(__func__); return nullptr; } +int APS5_VABI sceAjmDecMp3ParseFrame() { + AjmStub(__func__); + return 0; +} + } diff --git a/core/libs/prx/libSceAjm.native/src/Ajm.cpp b/core/libs/prx/libSceAjm.native/src/Ajm.cpp new file mode 100644 index 0000000000..1bfc8f760a --- /dev/null +++ b/core/libs/prx/libSceAjm.native/src/Ajm.cpp @@ -0,0 +1,573 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" +#include "libatrac9.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// The batch buffer and its job records are private to this library: the guest only reserves the +// memory and reads the "used bytes" field of AjmBatchInfo. Sideband results written back to guest +// memory follow the SDK layouts. +namespace { + +constexpr int SCE_AJM_ERROR_INVALID_INSTANCE = static_cast(0x80930003); +constexpr int SCE_AJM_ERROR_INVALID_PARAMETER = static_cast(0x80930005); +constexpr int SCE_AJM_ERROR_OUT_OF_RESOURCES = static_cast(0x80930007); + +constexpr std::int32_t AJM_RESULT_NOT_INITIALIZED = 0x00000001; +constexpr std::int32_t AJM_RESULT_INVALID_DATA = 0x00000002; +constexpr std::int32_t AJM_RESULT_INVALID_PARAMETER = 0x00000004; +constexpr std::int32_t AJM_RESULT_PARTIAL_INPUT = 0x00000008; +constexpr std::int32_t AJM_RESULT_NOT_ENOUGH_ROOM = 0x00000010; +constexpr std::int32_t AJM_RESULT_CODEC_ERROR = 0x40000000; + +constexpr std::uint32_t CODEC_AT9 = 1; + +// APS5_TRACE_AJM=1 logs every job the title submits and what the library writes back. +bool TraceEnabled() { + static const bool enabled = std::getenv("APS5_TRACE_AJM") != nullptr; + return enabled; +} + +#define AJM_TRACE(...) \ + do { \ + if (TraceEnabled()) std::fprintf(stderr, __VA_ARGS__); \ + } while (0) + +constexpr std::uint64_t RUN_MULTIPLE_FRAMES = 1ull << 12; +constexpr std::uint64_t SIDEBAND_GAPLESS_DECODE = 1ull << 45; +constexpr std::uint64_t SIDEBAND_FORMAT = 1ull << 46; +constexpr std::uint64_t SIDEBAND_STREAM = 1ull << 47; + +struct SidebandResult { + std::int32_t result; + std::int32_t internalResult; +}; + +struct SidebandStream { + std::int32_t inputConsumed; + std::int32_t outputWritten; + std::uint64_t totalDecodedSamples; +}; + +struct SidebandFormat { + std::uint32_t numChannels; + std::uint32_t channelMask; + std::uint32_t sampleRate; + std::uint32_t sampleFormat; + std::uint32_t bitrate; + std::uint32_t reserved; +}; + +struct SidebandGaplessDecode { + std::uint32_t totalSamples; + std::uint16_t skipSamples; + std::uint16_t skippedSamples; +}; + +struct SidebandMultipleFrames { + std::uint32_t numFrames; + std::uint32_t reserved; +}; + +struct Instance { + std::uint32_t codec = 0; + std::uint64_t flags = 0; + void* decoder = nullptr; + Atrac9CodecInfo info{}; + bool initialized = false; + std::uint32_t superframeRemaining = 0; + std::uint32_t frameInSuperframe = 0; + std::uint64_t totalDecodedSamples = 0; + SidebandGaplessDecode gapless{}; + bool flagsReported = false; + + ~Instance() { + if (decoder) Atrac9ReleaseHandle(decoder); + } +}; + +std::mutex g_lock; +std::map> g_instances; +std::atomic g_nextContext{1}; +std::atomic g_nextInstance{1}; +std::atomic g_nextBatch{1}; + +enum class JobKind : std::uint32_t { + Initialize = 1, + ClearContext = 2, + SetGaplessDecode = 3, + Run = 4, + GetStatistics = 5, +}; + +struct JobHeader { + JobKind kind; + std::uint32_t bytes; + std::uint32_t instance; + std::uint32_t reserved; + std::uint64_t flags; + void* sideband; + std::uint64_t sidebandSize; + std::uint64_t parameterSize; + std::uint32_t inputCount; + std::uint32_t outputCount; + std::uint8_t parameters[16]; +}; + +int Append(AjmBatchInfo* info, const JobHeader& header, const AjmBuffer* inputs, const AjmBuffer* outputs) { + if (!info || !info->p_buffer) return SCE_AJM_ERROR_INVALID_PARAMETER; + const std::size_t bytes = sizeof(JobHeader) + (header.inputCount + header.outputCount) * sizeof(AjmBuffer); + if (bytes > info->size - info->offset) return SCE_AJM_ERROR_OUT_OF_RESOURCES; + auto* cursor = static_cast(info->p_buffer) + info->offset; + JobHeader record = header; + record.bytes = static_cast(bytes); + std::memcpy(cursor, &record, sizeof(record)); + cursor += sizeof(record); + if (header.inputCount) std::memcpy(cursor, inputs, header.inputCount * sizeof(AjmBuffer)); + cursor += header.inputCount * sizeof(AjmBuffer); + if (header.outputCount) std::memcpy(cursor, outputs, header.outputCount * sizeof(AjmBuffer)); + info->offset += bytes; + return 0; +} + +JobHeader MakeHeader(JobKind kind, std::uint32_t instance, void* sideband, std::uint64_t sidebandSize) { + JobHeader header{}; + header.kind = kind; + header.instance = instance; + header.sideband = sideband; + header.sidebandSize = sidebandSize; + return header; +} + +// The SDK's speaker masks (SCE_AJM_CHANNELMASK_*) for a decoded channel count. +std::uint32_t ChannelMask(std::size_t channels) { + switch (channels) { + case 1: return 0x4; + case 2: return 0x3; + case 4: return 0x33; + case 6: return 0x3F; + case 8: return 0x63F; + default: return 0; + } +} + +void WriteResult(void* sideband, std::uint64_t size, std::int32_t result) { + if (!sideband || size < sizeof(SidebandResult)) return; + const SidebandResult value{result, 0}; + std::memcpy(sideband, &value, sizeof(value)); +} + +Instance* Find(std::uint32_t id) { + const auto found = g_instances.find(id); + return found == g_instances.end() ? nullptr : found->second.get(); +} + +// Replaces the instance's decoder with a fresh one for its configuration, at a superframe boundary. The +// decoder handle keeps its position within the superframe, which a re-initialization does not clear. +bool ResetDecoder(Instance& instance) { + if (instance.decoder) Atrac9ReleaseHandle(instance.decoder); + instance.decoder = Atrac9GetHandle(); + instance.superframeRemaining = 0; + instance.frameInSuperframe = 0; + unsigned char config[ATRAC9_CONFIG_DATA_SIZE]; + std::memcpy(config, instance.info.configData, sizeof(config)); + return Atrac9InitDecoder(instance.decoder, config) == 0; +} + +std::int32_t InitializeInstance(Instance& instance, const std::uint8_t* parameters, std::uint64_t size) { + if (instance.codec != CODEC_AT9) { + instance.initialized = true; + return 0; + } + if (size < ATRAC9_CONFIG_DATA_SIZE) return AJM_RESULT_INVALID_PARAMETER; + std::memcpy(instance.info.configData, parameters, ATRAC9_CONFIG_DATA_SIZE); + if (!ResetDecoder(instance)) { + instance.initialized = false; + return AJM_RESULT_INVALID_PARAMETER; + } + Atrac9GetCodecInfo(instance.decoder, &instance.info); + instance.initialized = true; + instance.totalDecodedSamples = 0; + instance.gapless = {}; + return 0; +} + +// Titles may hand the decoder a whole .at9 file. Like the console's decoder, a RIFF/WAVE header at a +// frame boundary is skipped up to the data chunk's payload and counted as consumed input. Returns the +// payload offset, or 0 when the bytes do not start a complete RIFF header. +std::size_t RiffDataOffset(const std::uint8_t* data, std::size_t size) { + if (size < 12 || std::memcmp(data, "RIFF", 4) != 0 || std::memcmp(data + 8, "WAVE", 4) != 0) return 0; + std::size_t cursor = 12; + while (cursor + 8 <= size) { + std::uint32_t chunkBytes = 0; + std::memcpy(&chunkBytes, data + cursor + 4, sizeof(chunkBytes)); + if (std::memcmp(data + cursor, "data", 4) == 0) return cursor + 8; + cursor += 8 + static_cast(chunkBytes) + (chunkBytes & 1u); + } + return 0; +} + +// Whether this is the instance's first run since it was (re)initialized; used to report its flags once. +bool consumedFirstRun(Instance& instance) { + if (instance.flagsReported) return false; + instance.flagsReported = true; + return true; +} + +// Decodes AT9 frames from the concatenated inputs into the concatenated outputs in the instance's PCM +// encoding. +void RunAt9(Instance& instance, const JobHeader& job, const AjmBuffer* inputs, const AjmBuffer* outputs) { + std::vector input; + for (std::uint32_t index = 0; index < job.inputCount; ++index) { + const auto* data = static_cast(inputs[index].ptr); + input.insert(input.end(), data, data + inputs[index].size); + } + std::size_t outputCapacity = 0; + for (std::uint32_t index = 0; index < job.outputCount; ++index) outputCapacity += outputs[index].size; + + const auto channels = static_cast(instance.info.channels); + const auto frameSamples = static_cast(instance.info.frameSamples); + // Instance flags bits 7..9 select the PCM encoding: 0 16-bit, 1 32-bit integer, 2 float. + const auto encoding = static_cast((instance.flags >> 7u) & 7u); + const std::size_t sampleBytes = encoding == 0 ? sizeof(std::int16_t) : sizeof(std::int32_t); + const std::size_t frameBytes = frameSamples * channels * sampleBytes; + std::vector pcm(frameBytes); + static const bool traceFlags = std::getenv("APS5_TRACE_AJM") != nullptr; + if (traceFlags && instance.totalDecodedSamples == 0 && consumedFirstRun(instance)) std::fprintf(stderr, "[ajm] instance %u flags 0x%llx (encoding %u)\n", job.instance, static_cast(instance.flags), encoding); + + std::int32_t result = 0; + int decodeStatus = 0; + std::size_t consumed = 0; + std::size_t produced = 0; + std::uint32_t frames = 0; + std::uint32_t outputIndex = 0; + std::size_t outputOffset = 0; + const auto emit = [&](const std::uint8_t* source, std::size_t bytes) { + while (bytes > 0 && outputIndex < job.outputCount) { + const std::size_t room = outputs[outputIndex].size - outputOffset; + const std::size_t chunk = std::min(room, bytes); + std::memcpy(static_cast(outputs[outputIndex].ptr) + outputOffset, source, chunk); + source += chunk; + bytes -= chunk; + produced += chunk; + outputOffset += chunk; + if (outputOffset == outputs[outputIndex].size) { + ++outputIndex; + outputOffset = 0; + } + } + }; + + // A superframe is a fixed superframeSize bytes holding framesInSuperframe frames packed back to back, + // each byte aligned but of its own size (the encoder moves bits between the frames of a superframe), + // with padding after the last one. The decoder reports each frame's size, so frames are walked with + // it and the padding is skipped after the last frame. Like the console's decoder, a superframe is + // decoded only once it is wholly present: a trailing partial superframe stays unconsumed and is + // reported as partial input, and the title resubmits it from the consumed offset. + const auto superframeSize = static_cast(instance.info.superframeSize); + const auto framesInSuperframe = static_cast(std::max(1, instance.info.framesInSuperframe)); + for (;;) { + if (instance.superframeRemaining == 0) consumed += RiffDataOffset(input.data() + consumed, input.size() - consumed); + const std::size_t needed = instance.superframeRemaining == 0 ? superframeSize : instance.superframeRemaining; + if (input.size() - consumed < needed) { + if (input.size() != consumed) result |= AJM_RESULT_PARTIAL_INPUT; + break; + } + if (outputCapacity - produced < frameBytes) { + if (frames == 0) result |= AJM_RESULT_NOT_ENOUGH_ROOM; + break; + } + if (instance.superframeRemaining == 0) { + instance.superframeRemaining = static_cast(superframeSize); + instance.frameInSuperframe = 0; + } + int used = 0; + switch (encoding) { + case 0: decodeStatus = Atrac9Decode(instance.decoder, input.data() + consumed, reinterpret_cast(pcm.data()), &used, 0); break; + case 1: decodeStatus = Atrac9DecodeS32(instance.decoder, input.data() + consumed, reinterpret_cast(pcm.data()), &used, 0); break; + default: decodeStatus = Atrac9DecodeF32(instance.decoder, input.data() + consumed, reinterpret_cast(pcm.data()), &used, 0); break; + } + if (decodeStatus != 0 || used <= 0 || static_cast(used) > instance.superframeRemaining) { + result |= AJM_RESULT_INVALID_DATA; + // The decoder tracks its position in the superframe; a fresh one starts the next superframe. + ResetDecoder(instance); + break; + } + consumed += static_cast(used); + instance.superframeRemaining -= static_cast(used); + if (++instance.frameInSuperframe == framesInSuperframe) { + consumed += instance.superframeRemaining; + instance.superframeRemaining = 0; + } + ++frames; + + std::size_t first = 0; + std::size_t count = frameSamples; + if (instance.gapless.skippedSamples < instance.gapless.skipSamples) { + const std::size_t skip = std::min(count, instance.gapless.skipSamples - instance.gapless.skippedSamples); + instance.gapless.skippedSamples = static_cast(instance.gapless.skippedSamples + skip); + first += skip; + count -= skip; + } + if (instance.gapless.totalSamples != 0) { + const std::uint64_t remaining = instance.gapless.totalSamples > instance.totalDecodedSamples ? instance.gapless.totalSamples - instance.totalDecodedSamples : 0; + count = static_cast(std::min(count, remaining)); + } + emit(pcm.data() + first * channels * sampleBytes, count * channels * sampleBytes); + instance.totalDecodedSamples += count; + if ((job.flags & RUN_MULTIPLE_FRAMES) == 0) break; + } + + if (TraceEnabled()) { + std::fprintf(stderr, "[ajm] instance %u run flags 0x%llx: %zu input bytes in %u buffers, %zu output bytes in %u buffers, sideband %llu bytes; superframe %d (%d frames), %d channels, %d samples/frame, %d Hz, config %02x %02x %02x %02x -> result 0x%x, decoder status 0x%x, %u frames, consumed %zu, produced %zu, total samples %llu, gapless total %u skip %u skipped %u, input", + job.instance, static_cast(job.flags), input.size(), job.inputCount, outputCapacity, job.outputCount, static_cast(job.sidebandSize), instance.info.superframeSize, instance.info.framesInSuperframe, instance.info.channels, instance.info.frameSamples, instance.info.samplingRate, + instance.info.configData[0], instance.info.configData[1], instance.info.configData[2], instance.info.configData[3], static_cast(result), static_cast(decodeStatus), frames, consumed, produced, static_cast(instance.totalDecodedSamples), instance.gapless.totalSamples, instance.gapless.skipSamples, instance.gapless.skippedSamples); + for (std::size_t i = 0; i < input.size() && i < 16; ++i) std::fprintf(stderr, " %02x", input[i]); + std::fprintf(stderr, "\n"); + } + auto* sideband = static_cast(job.sideband); + std::size_t offset = 0; + const auto write = [&](const void* value, std::size_t size) { + if (!sideband || offset + size > job.sidebandSize) return; + std::memcpy(sideband + offset, value, size); + offset += size; + }; + const SidebandResult status{result, 0}; + write(&status, sizeof(status)); + if (job.flags & SIDEBAND_STREAM) { + const SidebandStream stream{static_cast(consumed), static_cast(produced), instance.totalDecodedSamples}; + write(&stream, sizeof(stream)); + } + if (job.flags & SIDEBAND_FORMAT) { + const SidebandFormat format{static_cast(channels), ChannelMask(channels), static_cast(instance.info.samplingRate), encoding, 0, 0}; + write(&format, sizeof(format)); + } + if (job.flags & SIDEBAND_GAPLESS_DECODE) write(&instance.gapless, sizeof(instance.gapless)); + if (job.flags & RUN_MULTIPLE_FRAMES) { + const SidebandMultipleFrames multiple{frames, 0}; + write(&multiple, sizeof(multiple)); + } +} + +void Execute(const JobHeader& job, const AjmBuffer* inputs, const AjmBuffer* outputs) { + if (job.kind == JobKind::GetStatistics) { + AJM_TRACE("[ajm] statistics job: sideband %llu bytes\n", static_cast(job.sidebandSize)); + if (job.sideband && job.sidebandSize) std::memset(job.sideband, 0, std::min(job.sidebandSize, 24)); + return; + } + std::lock_guard lock(g_lock); + auto* instance = Find(job.instance); + if (!instance) { + AJM_TRACE("[ajm] job kind %u on unknown instance %u\n", static_cast(job.kind), job.instance); + WriteResult(job.sideband, job.sidebandSize, AJM_RESULT_INVALID_PARAMETER); + return; + } + switch (job.kind) { + case JobKind::Initialize: { + const std::int32_t result = InitializeInstance(*instance, job.parameters, job.parameterSize); + AJM_TRACE("[ajm] instance %u initialize: %llu parameter bytes (%02x %02x %02x %02x %02x %02x %02x %02x), sideband %llu bytes -> result 0x%x, %d channels, %d Hz, superframe %d bytes (%d frames of %d samples)\n", job.instance, static_cast(job.parameterSize), job.parameters[0], job.parameters[1], job.parameters[2], job.parameters[3], job.parameters[4], job.parameters[5], job.parameters[6], job.parameters[7], static_cast(job.sidebandSize), static_cast(result), instance->info.channels, instance->info.samplingRate, instance->info.superframeSize, instance->info.framesInSuperframe, instance->info.frameSamples); + WriteResult(job.sideband, job.sidebandSize, result); + break; + } + case JobKind::ClearContext: + AJM_TRACE("[ajm] instance %u clear context (sideband %llu bytes)\n", job.instance, static_cast(job.sidebandSize)); + instance->totalDecodedSamples = 0; + instance->gapless.skippedSamples = 0; + if (instance->decoder && instance->initialized) ResetDecoder(*instance); + WriteResult(job.sideband, job.sidebandSize, 0); + break; + case JobKind::SetGaplessDecode: { + SidebandGaplessDecode gapless{}; + std::memcpy(&gapless, job.parameters, sizeof(gapless)); + AJM_TRACE("[ajm] instance %u set gapless decode: total %u, skip %u, reset %llu (sideband %llu bytes)\n", job.instance, gapless.totalSamples, gapless.skipSamples, static_cast(job.flags), static_cast(job.sidebandSize)); + instance->gapless.totalSamples = gapless.totalSamples; + instance->gapless.skipSamples = gapless.skipSamples; + if (job.flags) instance->gapless.skippedSamples = 0; + WriteResult(job.sideband, job.sidebandSize, 0); + break; + } + case JobKind::Run: + if (!instance->initialized) { + AJM_TRACE("[ajm] instance %u run before initialize\n", job.instance); + WriteResult(job.sideband, job.sidebandSize, AJM_RESULT_NOT_INITIALIZED); + } else if (instance->codec == CODEC_AT9) { + RunAt9(*instance, job, inputs, outputs); + } else { + AJM_TRACE("[ajm] instance %u run on codec %u without a decoder\n", job.instance, instance->codec); + WriteResult(job.sideband, job.sidebandSize, AJM_RESULT_CODEC_ERROR); + } + break; + default: + AJM_TRACE("[ajm] instance %u unknown job kind %u\n", job.instance, static_cast(job.kind)); + WriteResult(job.sideband, job.sidebandSize, AJM_RESULT_INVALID_PARAMETER); + break; + } +} + +} + +extern "C" { + +int APS5_VABI sceAjmInitialize(int64_t reserved, uint32_t* context) { + (void)reserved; + if (!context) return SCE_AJM_ERROR_INVALID_PARAMETER; + *context = g_nextContext.fetch_add(1, std::memory_order_relaxed); + return 0; +} + +int APS5_VABI sceAjmFinalize(uint32_t context) { + (void)context; + return 0; +} + +int APS5_VABI sceAjmModuleRegister(uint32_t context, uint32_t codec, int64_t reserved) { + (void)context; + (void)codec; + (void)reserved; + return 0; +} + +int APS5_VABI sceAjmModuleUnregister(uint32_t context, uint32_t codec) { + (void)context; + (void)codec; + return 0; +} + +int APS5_VABI sceAjmMemoryRegister(uint32_t context, void* ptr, size_t pages) { + (void)context; + (void)ptr; + (void)pages; + return 0; +} + +int APS5_VABI sceAjmMemoryUnregister(uint32_t context, void* ptr) { + (void)context; + (void)ptr; + return 0; +} + +int APS5_VABI sceAjmInstanceCreate(uint32_t context, uint32_t codec, uint64_t flags, uint32_t* instance) { + (void)context; + if (!instance) return SCE_AJM_ERROR_INVALID_PARAMETER; + static std::atomic reportedCodecs{0}; + if (codec != CODEC_AT9 && codec < 32 && (reportedCodecs.fetch_or(1u << codec) & (1u << codec)) == 0) + std::fprintf(stderr, "[ajm] codec %u has no decoder; its jobs report codec errors\n", codec); + auto created = std::make_unique(); + created->codec = codec; + created->flags = flags; + const std::uint32_t id = (codec << 14) | (g_nextInstance.fetch_add(1, std::memory_order_relaxed) & 0x3FFF); + std::lock_guard lock(g_lock); + g_instances[id] = std::move(created); + *instance = id; + AJM_TRACE("[ajm] instance create: context %u, codec %u, flags 0x%llx -> instance %u\n", context, codec, static_cast(flags), id); + return 0; +} + +int APS5_VABI sceAjmInstanceDestroy(uint32_t context, uint32_t instance) { + (void)context; + AJM_TRACE("[ajm] instance %u destroy\n", instance); + std::lock_guard lock(g_lock); + return g_instances.erase(instance) ? 0 : SCE_AJM_ERROR_INVALID_INSTANCE; +} + +int APS5_VABI sceAjmDecAt9ParseConfigData(const void* config_data, AjmDecAt9ConfigDataInfo* config_info) { + if (!config_data || !config_info) return SCE_AJM_ERROR_INVALID_PARAMETER; + void* decoder = Atrac9GetHandle(); + unsigned char config[ATRAC9_CONFIG_DATA_SIZE]; + std::memcpy(config, config_data, sizeof(config)); + const bool valid = Atrac9InitDecoder(decoder, config) == 0; + Atrac9CodecInfo info{}; + if (valid) Atrac9GetCodecInfo(decoder, &info); + Atrac9ReleaseHandle(decoder); + AJM_TRACE("[ajm] parse config %02x %02x %02x %02x -> %s, %d channels, %d Hz, superframe %d bytes (%d frames of %d samples)\n", config[0], config[1], config[2], config[3], valid ? "ok" : "invalid", info.channels, info.samplingRate, info.superframeSize, info.framesInSuperframe, info.frameSamples); + if (!valid) return SCE_AJM_ERROR_INVALID_PARAMETER; + config_info->channels = static_cast(info.channels); + config_info->sample_rate = static_cast(info.samplingRate); + config_info->frame_samples_per_channel = static_cast(info.frameSamples); + config_info->superframe_samples_per_channel = static_cast(info.frameSamples * info.framesInSuperframe); + config_info->superframe_size = static_cast(info.superframeSize); + return 0; +} + +int APS5_VABI sceAjmBatchInitialize(void* buffer, size_t size, AjmBatchInfo* info) { + if (!buffer || !info) return SCE_AJM_ERROR_INVALID_PARAMETER; + info->p_buffer = buffer; + info->offset = 0; + info->size = size; + return 0; +} + +int APS5_VABI sceAjmBatchJobInitialize(AjmBatchInfo* info, uint32_t instance, const void* codec_parameters, size_t codec_parameters_size, void* result) { + auto header = MakeHeader(JobKind::Initialize, instance, result, sizeof(SidebandResult)); + header.parameterSize = std::min(codec_parameters_size, sizeof(header.parameters)); + if (codec_parameters && header.parameterSize) std::memcpy(header.parameters, codec_parameters, header.parameterSize); + return Append(info, header, nullptr, nullptr); +} + +int APS5_VABI sceAjmBatchJobClearContext(AjmBatchInfo* info, uint32_t instance, void* result) { + return Append(info, MakeHeader(JobKind::ClearContext, instance, result, sizeof(SidebandResult)), nullptr, nullptr); +} + +int APS5_VABI sceAjmBatchJobSetGaplessDecode(AjmBatchInfo* info, uint32_t instance, const void* gapless_decode, int reset, void* result) { + auto header = MakeHeader(JobKind::SetGaplessDecode, instance, result, sizeof(SidebandResult)); + if (gapless_decode) std::memcpy(header.parameters, gapless_decode, sizeof(SidebandGaplessDecode)); + header.flags = reset ? 1 : 0; + return Append(info, header, nullptr, nullptr); +} + +int APS5_VABI sceAjmBatchJobRunSplit(AjmBatchInfo* info, uint32_t instance, uint64_t flags, const AjmBuffer* input_buffers, size_t input_buffers_num, const AjmBuffer* output_buffers, size_t output_buffers_num, void* sideband_output, size_t sideband_output_size) { + auto header = MakeHeader(JobKind::Run, instance, sideband_output, sideband_output_size); + header.flags = flags; + header.inputCount = static_cast(input_buffers_num); + header.outputCount = static_cast(output_buffers_num); + return Append(info, header, input_buffers, output_buffers); +} + +int APS5_VABI sceAjmBatchJobGetStatistics(AjmBatchInfo* info, float interval, void* result) { + (void)interval; + return Append(info, MakeHeader(JobKind::GetStatistics, 0, result, 24), nullptr, nullptr); +} + +int APS5_VABI sceAjmBatchStart(uint32_t context, const AjmBatchInfo* info, int priority, AjmBatchError* error, uint32_t* batch) { + (void)context; + (void)priority; + if (!info || !batch) return SCE_AJM_ERROR_INVALID_PARAMETER; + AJM_TRACE("[ajm] batch start: context %u, priority %d, %llu of %llu buffer bytes used\n", context, priority, static_cast(info->offset), static_cast(info->size)); + const auto* cursor = static_cast(info->p_buffer); + const auto* end = cursor + info->offset; + while (cursor < end) { + JobHeader job; + std::memcpy(&job, cursor, sizeof(job)); + const auto* buffers = reinterpret_cast(cursor + sizeof(JobHeader)); + Execute(job, buffers, buffers + job.inputCount); + cursor += job.bytes; + } + if (error) std::memset(error, 0, sizeof(*error)); + *batch = g_nextBatch.fetch_add(1, std::memory_order_relaxed); + return 0; +} + +int APS5_VABI sceAjmBatchWait(uint32_t context, uint32_t batch, uint32_t timeout, AjmBatchError* error) { + (void)context; + (void)batch; + (void)timeout; + if (error) std::memset(error, 0, sizeof(*error)); + return 0; +} + +int APS5_VABI sceAjmBatchErrorDump(const AjmBatchInfo* info, AjmBatchError* error) { + (void)info; + if (error) std::memset(error, 0, sizeof(*error)); + return 0; +} + +} diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut.cpp index 64478c4b47..b9ecce49e0 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut.cpp @@ -185,8 +185,23 @@ static const void* prepareBuffer(const Port& port, const void* data, std::vector } static void queueAudio(Port& port, const void* data) { - if (port.device == 0 || data == nullptr) { - APS5_INVALID_ARG_EX; + // Without an SDL device there is nothing to queue: the callers already sleep for the block's + // duration so the game's timing holds. A null pointer is the documented way to wait until the + // port's queued audio has been output (sceAudioOutOutput(handle, NULL)); it queues nothing. + if (port.device == 0) return; + if (data == nullptr) { + const std::uint64_t waitStart = sceKernelGetProcessTime(); + while (SDL_GetQueuedAudioSize(port.device) > 0) { + if (sceKernelGetProcessTime() - waitStart > DRAIN_TIMEOUT_US) { + SDL_ClearQueuedAudio(port.device); + break; + } + struct timespec req{}; + req.tv_sec = 0; + req.tv_nsec = static_cast(DRAIN_SLEEP_US * 1000ULL); + nanosleep(&req, nullptr); + } + return; } std::vector prepareBuf; @@ -391,12 +406,12 @@ int APS5_VABI sceAudioOutOutputs(AudioOutOutputParam* param, std::uint32_t num) } for (std::uint32_t i = 0; i < num; i++) { - queueAudio(*getPort(param[i].handle), param[i].ptr); + if (auto* port = getPort(param[i].handle)) queueAudio(*port, param[i].ptr); } const std::uint64_t done = sceKernelGetProcessTime(); for (std::uint32_t i = 0; i < num; i++) { - getPort(param[i].handle)->lastOutputTime = done; + if (auto* port = getPort(param[i].handle)) port->lastOutputTime = done; } return static_cast(first.samplesNum); diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp index 8720752dd4..85de18a579 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp @@ -1,64 +1,277 @@ +#include +#include +#include #include #include +#include +#include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "AudioOut2Internal.hpp" + +// An AudioOut2 context is the hardware output queue: every push mixes the ports' current grain +// (num_grains samples) and appends it to a queue of queue_depth grains that plays in real time. The +// title paces its mixer on that queue (Demon's Souls polls GetQueueLevel and pushes non-blocking +// whenever a slot is free), so the level must follow the output clock: with an SDL device open the +// device's own queue is the hardware queue, otherwise a wall-clock model of it stands in. + +using Clock = std::chrono::steady_clock; + +static constexpr std::size_t CONTEXT_MEMORY = 0x10000; +static constexpr std::uint32_t DEFAULT_MAX_PORTS = 16; +// Pushes, polls and advances traced individually before the trace falls back to the per-second summary. +static constexpr std::uint64_t CALL_TRACE_FULL = 16; +// Silence queued ahead of the first grain (and again whenever the SDL queue ran dry) so scheduling +// jitter of the pushing thread does not starve the device; the queue level reported to the title +// counts only what lies beyond it. +static constexpr std::uint32_t CUSHION_MS = 40; +// The SDL queue is not allowed to run further ahead than this; grains beyond it are dropped. +static constexpr std::uint32_t MAX_QUEUED_MS = 250; +// A blocking push on a full queue gives up after this long. +static constexpr std::chrono::milliseconds FULL_WAIT_TIMEOUT{200}; +static constexpr std::chrono::milliseconds FULL_WAIT_STEP{1}; +static constexpr std::uint16_t DEVICE_SAMPLES = 512; +// The summed ports (a 7.1 bed folded to stereo plus the object ports) peak above full scale in the +// intro cutscene (2.1 measured); the device takes float and clips hard, so the mix is attenuated +// and clamped. +static constexpr float MASTER_GAIN = 0.5f; +static constexpr std::size_t OUTPUT_FRAME_BYTES = AUDIO_OUT2_OUTPUT_CHANNELS * sizeof(float); +static constexpr std::uint32_t OUTPUT_BYTES_PER_MS = AUDIO_OUT2_SAMPLE_RATE * OUTPUT_FRAME_BYTES / 1000; + +bool AudioOut2TraceEnabled() { + static const bool enabled = std::getenv("APS5_TRACE_AUDIOOUT2") != nullptr; + return enabled; +} + +double AudioOut2TraceSeconds() { + static const auto start = Clock::now(); + return std::chrono::duration(Clock::now() - start).count(); +} + +static AudioOut2Context* FromHandle(AudioOut2ContextHandle ctx) { + return reinterpret_cast(ctx); +} + +static Clock::duration GrainDuration(const AudioOut2Context& context) { + return std::chrono::duration_cast(std::chrono::duration(static_cast(context.grain) / AUDIO_OUT2_SAMPLE_RATE)); +} + +static std::uint32_t GrainBytes(const AudioOut2Context& context) { + return static_cast(context.grain * OUTPUT_FRAME_BYTES); +} + +static std::uint32_t SdlQueuedMs(const AudioOut2Context& context) { + return context.device ? SDL_GetQueuedAudioSize(context.device) / OUTPUT_BYTES_PER_MS : 0; +} + +// Retires the modelled grains whose playback finished by now. The caller holds the context lock. +static void Drain(AudioOut2Context& context, Clock::time_point now) { + const auto duration = GrainDuration(context); + while (context.queued > 0) { + const auto oldestEnd = context.playHead - duration * (context.queued - 1); + if (oldestEnd > now) break; + context.queued--; + } + if (context.queued == 0 && context.playHead < now) context.playHead = now; +} + +// Grains queued and not yet played, as the title sees them. The caller holds the context lock. +static std::uint32_t QueueLevel(AudioOut2Context& context, Clock::time_point now) { + Drain(context, now); + if (context.device == 0) return context.queued; + const auto queuedBytes = SDL_GetQueuedAudioSize(context.device); + const auto cushionBytes = CUSHION_MS * OUTPUT_BYTES_PER_MS; + const auto pending = queuedBytes > cushionBytes ? queuedBytes - cushionBytes : 0; + const auto grainBytes = GrainBytes(context); + return std::min(context.queueDepth, (pending + grainBytes - 1) / grainBytes); +} + +static void OpenDevice(AudioOut2Context& context) { + if (SDL_InitSubSystem(SDL_INIT_AUDIO) < 0) { + AUDIOOUT2_TRACE("SDL audio init failed: %s\n", SDL_GetError()); + return; + } + SDL_AudioSpec desired{}; + desired.freq = static_cast(AUDIO_OUT2_SAMPLE_RATE); + desired.format = AUDIO_F32SYS; + desired.channels = AUDIO_OUT2_OUTPUT_CHANNELS; + desired.samples = DEVICE_SAMPLES; + desired.callback = nullptr; + SDL_AudioSpec obtained{}; + // No format change is allowed: SDL converts to the device's native format itself, so the queue + // always takes stereo float at 48 kHz. + context.device = SDL_OpenAudioDevice(nullptr, 0, &desired, &obtained, 0); + if (context.device == 0) { + AUDIOOUT2_TRACE("SDL_OpenAudioDevice failed: %s\n", SDL_GetError()); + return; + } + SDL_PauseAudioDevice(context.device, 0); + AUDIOOUT2_TRACE("SDL device %u opened: %d Hz, format 0x%x, %u channels, %u samples\n", context.device, obtained.freq, obtained.format, obtained.channels, obtained.samples); +} + +static void CloseDevice(AudioOut2Context& context) { + if (context.device != 0 && SDL_WasInit(SDL_INIT_AUDIO) != 0) { + SDL_ClearQueuedAudio(context.device); + SDL_CloseAudioDevice(context.device); + } + context.device = 0; +} + +// Mixes the ports' current grain and queues it on the SDL device. The caller holds the lock. +static std::uint32_t Render(AudioOut2Context& context) { + std::fill(context.mix.begin(), context.mix.end(), 0.0f); + const auto mixed = AudioOut2MixPorts(context, context.mix.data(), context.grain); + for (float& sample : context.mix) { + sample = std::clamp(sample * MASTER_GAIN, -1.0f, 1.0f); + if (AudioOut2TraceEnabled()) context.summaryPeak = std::max(context.summaryPeak, std::abs(sample)); + } + if (context.device == 0) return mixed; + const auto queuedBytes = SDL_GetQueuedAudioSize(context.device); + if (queuedBytes > MAX_QUEUED_MS * OUTPUT_BYTES_PER_MS) { + context.dropped++; + return mixed; + } + if (queuedBytes == 0) { + static const std::vector silence(static_cast(CUSHION_MS) * AUDIO_OUT2_SAMPLE_RATE / 1000 * AUDIO_OUT2_OUTPUT_CHANNELS, 0.0f); + context.primes++; + SDL_QueueAudio(context.device, silence.data(), static_cast(silence.size() * sizeof(float))); + } + SDL_QueueAudio(context.device, context.mix.data(), GrainBytes(context)); + return mixed; +} + +static void TraceSummary(AudioOut2Context& context, Clock::time_point now) { + if (!AudioOut2TraceEnabled()) return; + if (context.summaryStart == Clock::time_point{}) { + context.summaryStart = now; + return; + } + const auto elapsed = std::chrono::duration(now - context.summaryStart).count(); + if (elapsed < 1.0) return; + std::fprintf(stderr, "[audioout2] t=%.3f ctx %p: %.1f pushes/s, %.1f advances/s, %.1f queue polls/s, hw queue %u/%u, sdl queue %u ms, mix peak %.3f; totals: pushes %llu (%llu blocking, %llu queue-full rejects), primes %llu, dropped %llu\n", + AudioOut2TraceSeconds(), static_cast(&context), static_cast(context.summaryPushes) / elapsed, static_cast(context.summaryAdvances) / elapsed, + static_cast(context.summaryPolls) / elapsed, QueueLevel(context, now), context.queueDepth, SdlQueuedMs(context), static_cast(context.summaryPeak), + static_cast(context.pushes), static_cast(context.blockingPushes), static_cast(context.fullRejects), + static_cast(context.primes), static_cast(context.dropped)); + context.summaryStart = now; + context.summaryPushes = 0; + context.summaryAdvances = 0; + context.summaryPolls = 0; + context.summaryPeak = 0.0f; +} extern "C" { +// The title's per-tick step between setting the ports' data and pushing. The ports are mixed at push +// time from the buffers they currently point at, and the push paces the clock, so nothing is due here. int APS5_VABI sceAudioOut2ContextAdvance(AudioOut2ContextHandle ctx) { - (void)ctx; - NotImplemented_nid_no_patch(__func__); + auto* context = FromHandle(ctx); + if (!context) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + std::lock_guard lock(context->lock); + context->advances++; + context->summaryAdvances++; + if (context->advances <= CALL_TRACE_FULL) AUDIOOUT2_TRACE("t=%.3f Advance ctx %p (pushes so far %llu)\n", AudioOut2TraceSeconds(), static_cast(context), static_cast(context->pushes)); return 0; } int APS5_VABI sceAudioOut2ContextCreate(const AudioOut2ContextParam* params, void* buffer, size_t buffer_size, AudioOut2ContextHandle* ctx) { - (void)params; - (void)buffer; - (void)buffer_size; - (void)ctx; - NotImplemented_nid_no_patch(__func__); + if (!params || !ctx) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + auto* context = new AudioOut2Context(); + context->grain = params->num_grains ? params->num_grains : AUDIO_OUT2_DEFAULT_GRAIN; + context->queueDepth = params->queue_depth ? params->queue_depth : 1; + context->playHead = Clock::now(); + context->mix.assign(static_cast(context->grain) * AUDIO_OUT2_OUTPUT_CHANNELS, 0.0f); + AUDIOOUT2_TRACE("t=%.3f ContextCreate: max_ports=%u max_object_ports=%u guarantee_object_ports=%u queue_depth=%u num_grains=%u flags=0x%x buffer=%p size=%zu -> ctx %p: %u-sample grains (%.2f ms), %u queued, %u Hz stereo float output\n", + AudioOut2TraceSeconds(), params->max_ports, params->max_object_ports, params->guarantee_object_ports, params->queue_depth, params->num_grains, params->flags, buffer, buffer_size, + static_cast(context), context->grain, 1000.0 * context->grain / AUDIO_OUT2_SAMPLE_RATE, context->queueDepth, AUDIO_OUT2_SAMPLE_RATE); + OpenDevice(*context); + *ctx = reinterpret_cast(context); return 0; } int APS5_VABI sceAudioOut2ContextDestroy(AudioOut2ContextHandle ctx) { - (void)ctx; - NotImplemented_nid_no_patch(__func__); + auto* context = FromHandle(ctx); + if (!context) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + AUDIOOUT2_TRACE("t=%.3f ContextDestroy ctx %p after %llu pushes\n", AudioOut2TraceSeconds(), static_cast(context), static_cast(context->pushes)); + { + std::lock_guard lock(context->lock); + CloseDevice(*context); + } + AudioOut2ReleasePorts(*context); + delete context; return 0; } -int APS5_VABI sceAudioOut2ContextGetQueueLevel(AudioOut2ContextHandle ctx, uint32_t* queue_level, uint32_t* available_queues) { - (void)ctx; - (void)queue_level; - (void)available_queues; - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceAudioOut2ContextGetQueueLevel(AudioOut2ContextHandle ctx, uint32_t* queue_level, uint32_t* available_queue) { + auto* context = FromHandle(ctx); + if (!context) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + std::lock_guard lock(context->lock); + const auto level = QueueLevel(*context, Clock::now()); + if (queue_level) *queue_level = level; + if (available_queue) *available_queue = context->queueDepth - level; + context->queueLevelPolls++; + context->summaryPolls++; + if (context->queueLevelPolls <= CALL_TRACE_FULL) AUDIOOUT2_TRACE("t=%.3f GetQueueLevel ctx %p -> level %u, available %u\n", AudioOut2TraceSeconds(), static_cast(context), level, context->queueDepth - level); return 0; } int APS5_VABI sceAudioOut2ContextPush(AudioOut2ContextHandle ctx, uint32_t blocking) { - (void)ctx; - (void)blocking; - NotImplemented_nid_no_patch(__func__); + auto* context = FromHandle(ctx); + if (!context) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + std::unique_lock lock(context->lock); + auto now = Clock::now(); + const auto waitStart = now; + while (QueueLevel(*context, now) >= context->queueDepth) { + if (!blocking) { + context->fullRejects++; + if (context->fullRejects <= CALL_TRACE_FULL) AUDIOOUT2_TRACE("t=%.3f Push ctx %p non-blocking on a full queue (%u): rejected\n", AudioOut2TraceSeconds(), static_cast(context), context->queueDepth); + return SCE_AUDIO_OUT2_ERROR_QUEUE_FULL; + } + if (now - waitStart > FULL_WAIT_TIMEOUT) break; + lock.unlock(); + std::this_thread::sleep_for(FULL_WAIT_STEP); + lock.lock(); + now = Clock::now(); + } + context->queued++; + context->playHead += GrainDuration(*context); + const auto mixed = Render(*context); + context->pushes++; + context->summaryPushes++; + if (blocking) context->blockingPushes++; + if (context->pushes <= CALL_TRACE_FULL) { + AUDIOOUT2_TRACE("t=%.3f Push ctx %p blocking=%u: grain from %u ports, hw queue %u/%u, sdl queue %u ms\n", AudioOut2TraceSeconds(), static_cast(context), blocking, + mixed, QueueLevel(*context, now), context->queueDepth, SdlQueuedMs(*context)); + } + TraceSummary(*context, now); return 0; } int APS5_VABI sceAudioOut2ContextQueryMemory(const AudioOut2ContextParam* params, size_t* memory_size) { - (void)params; - (void)memory_size; - NotImplemented_nid_no_patch(__func__); + if (!params || !memory_size) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + *memory_size = CONTEXT_MEMORY; return 0; } int APS5_VABI sceAudioOut2ContextResetParam(AudioOut2ContextParam* params) { - (void)params; - NotImplemented_nid_no_patch(__func__); + if (!params) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + std::memset(params, 0, sizeof(*params)); + params->max_ports = DEFAULT_MAX_PORTS; + params->queue_depth = 1; + params->num_grains = AUDIO_OUT2_DEFAULT_GRAIN; return 0; } int APS5_VABI sceAudioOut2ContextSetAttributes(AudioOut2ContextHandle ctx, const AudioOut2Attribute* attributes, uint32_t num) { - (void)ctx; - (void)attributes; - (void)num; - NotImplemented_nid_no_patch(__func__); + auto* context = FromHandle(ctx); + if (!context) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + if (!attributes && num != 0) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + for (uint32_t index = 0; index < num; index++) { + AUDIOOUT2_TRACE("t=%.3f ContextSetAttributes ctx %p: id=0x%x size=%zu value=%p (ignored)\n", AudioOut2TraceSeconds(), static_cast(context), attributes[index].attribute_id, attributes[index].value_size, attributes[index].value); + } return 0; } diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2Internal.hpp b/core/libs/prx/libSceAudioOut/src/AudioOut2Internal.hpp new file mode 100644 index 0000000000..611d3e7268 --- /dev/null +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2Internal.hpp @@ -0,0 +1,95 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAUDIOOUT_SRC_AUDIOOUT2INTERNAL_HPP +#define CORE_LIBS_PRX_LIBSCEAUDIOOUT_SRC_AUDIOOUT2INTERNAL_HPP + +#include +#include +#include +#include +#include +#include + +#include "SDL.h" +#include "SceTypes.hpp" + +// Shared by the AudioOut2 context and port files. +// +// APS5_TRACE_AUDIOOUT2=1 prints the title's use of the API to stderr: context and port parameters, every +// attribute the title sets (first few per port, then sampled), the first pushes and queue polls, and a +// once-per-second summary with the push rate and the SDL queue level. +bool AudioOut2TraceEnabled(); +// Seconds since the first AudioOut2 call, for the trace. +double AudioOut2TraceSeconds(); + +#define AUDIOOUT2_TRACE(...) \ + do { \ + if (AudioOut2TraceEnabled()) std::fprintf(stderr, "[audioout2] " __VA_ARGS__); \ + } while (0) + +static constexpr int SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT = static_cast(0x80260502); +static constexpr int SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE = static_cast(0x80260503); +// Not recovered from a title or SDK header: follows the 0x802605xx pattern of the codes above. Returned +// by a non-blocking push when the modelled hardware queue is full. +static constexpr int SCE_AUDIO_OUT2_ERROR_QUEUE_FULL = static_cast(0x80260507); + +static constexpr std::uint32_t AUDIO_OUT2_SAMPLE_RATE = 48000; +// The grain a push carries, in samples per channel: a context's num_grains (Demon's Souls: 256, 5.33 ms). +static constexpr std::uint32_t AUDIO_OUT2_DEFAULT_GRAIN = 256; +static constexpr std::uint32_t AUDIO_OUT2_OUTPUT_CHANNELS = 2; +static constexpr std::uint32_t AUDIO_OUT2_PORT_CHANNELS_MAX = 8; + +struct AudioOut2Context; + +struct AudioOut2Port { + bool used = false; + AudioOut2Context* context = nullptr; + std::uint16_t type = 0; + std::uint32_t dataFormat = 0; + std::uint32_t samplingFreq = 0; + std::uint32_t flags = 0; + // Channel count decoded from dataFormat; 0 when the format is not understood (the port is then + // not rendered). Samples are float. + std::uint32_t channels = 0; + // The PCM buffer (one grain, float, interleaved) the title last handed over through the data + // attribute. It is guest memory the title rewrites every tick, so it is read when the context + // mixes, not when it is set. + const float* data = nullptr; + float volume[AUDIO_OUT2_PORT_CHANNELS_MAX] = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; + std::uint64_t dataSets = 0; + std::uint64_t attributeTraces = 0; +}; + +struct AudioOut2Context { + std::mutex lock; + std::uint32_t grain = AUDIO_OUT2_DEFAULT_GRAIN; + std::uint32_t queueDepth = 1; + // Fallback hardware queue model for a context without an SDL device: pushes that have not + // finished playing, and the time the last of them finishes; playback is real time, so the queue + // drains as the wall clock advances. With a device the SDL queue is the hardware queue. + std::uint32_t queued = 0; + std::chrono::steady_clock::time_point playHead; + SDL_AudioDeviceID device = 0; + // Stereo float mix of the ports for one push. + std::vector mix; + // Trace counters. + std::uint64_t pushes = 0; + std::uint64_t blockingPushes = 0; + std::uint64_t fullRejects = 0; + std::uint64_t advances = 0; + std::uint64_t queueLevelPolls = 0; + std::uint64_t primes = 0; + std::uint64_t dropped = 0; + std::uint64_t summaryPushes = 0; + std::uint64_t summaryAdvances = 0; + std::uint64_t summaryPolls = 0; + // Largest sample magnitude of the grains mixed since the last summary: the mixed signal's level. + float summaryPeak = 0.0f; + std::chrono::steady_clock::time_point summaryStart; +}; + +// Mixes every port of the context that carries PCM data into out (stereo float, frames frames), summing +// onto the zeroed buffer. Returns the number of ports mixed. +std::uint32_t AudioOut2MixPorts(const AudioOut2Context& context, float* out, std::uint32_t frames); +// Forgets the ports of a context being destroyed. +void AudioOut2ReleasePorts(const AudioOut2Context& context); + +#endif diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp index d90c5b40a8..546ac704e2 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp @@ -1,35 +1,171 @@ +#include #include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "AudioOut2Internal.hpp" + +static constexpr std::uint16_t OUTPUT_MAIN = 1; +static constexpr std::int16_t VOLUME_MAX = 127; +// Attribute sets are traced in full for the first few calls of a port, then once per this many. +static constexpr std::uint64_t ATTRIBUTE_TRACE_FULL = 8; +static constexpr std::uint64_t ATTRIBUTE_TRACE_EVERY = 2000; + +// Port attributes as Demon's Souls sets them every tick (APS5_TRACE_AUDIOOUT2): id 0 carries the +// grain's PCM as an 8-byte value holding the buffer pointer, id 1 one float gain per channel. Ids 5, +// 8 and 9 (a u32, a u32 and a byte on object and voice ports) are not understood and ignored. +static constexpr std::uint32_t ATTRIBUTE_DATA = 0; +static constexpr std::uint32_t ATTRIBUTE_VOLUME = 1; + +// data_format bits 8..11 carry the channel count: the title opens 0x100 (mono object ports), +// 0x200 (stereo) and 0x880 (7.1 bed). The buffers behind them are float, as their spacing shows +// (1024 bytes per 256-sample mono grain). The low byte's meaning is not known. +static constexpr std::uint32_t FORMAT_CHANNELS_SHIFT = 8; +static constexpr std::uint32_t FORMAT_CHANNELS_MASK = 0xFu; + +static std::mutex g_portsLock; +// Grows on demand: the title opens its bed ports plus max_object_ports object ports at once. +static std::vector g_ports; + +static AudioOut2Port* FromHandle(AudioOut2PortHandle handle) { + const auto index = handle - 1; + if (handle == 0 || index >= g_ports.size() || !g_ports[index].used) return nullptr; + return &g_ports[index]; +} + +// 8-channel order FL FR C LFE RL RR SL SR (a swapped rear/side pair order sums the same): the rear +// and side pairs fold into the front at -3 dB, the centre into both sides, and the LFE is dropped. +static constexpr float DOWNMIX_GAIN = 0.7071f; + +static void AccumulatePort(const AudioOut2Port& port, float* out, std::uint32_t frames) { + const auto ch = port.channels; + const float* volume = port.volume; + for (std::uint32_t frame = 0; frame < frames; frame++) { + const float* in = port.data + static_cast(frame) * ch; + float left = 0.0f; + float right = 0.0f; + if (ch == 1) { + left = right = in[0] * volume[0]; + } else if (ch < 8) { + left = in[0] * volume[0]; + right = in[1] * volume[1]; + } else { + const float centre = in[2] * volume[2] * DOWNMIX_GAIN; + left = in[0] * volume[0] + centre + (in[4] * volume[4] + in[6] * volume[6]) * DOWNMIX_GAIN; + right = in[1] * volume[1] + centre + (in[5] * volume[5] + in[7] * volume[7]) * DOWNMIX_GAIN; + } + out[frame * AUDIO_OUT2_OUTPUT_CHANNELS] += left; + out[frame * AUDIO_OUT2_OUTPUT_CHANNELS + 1] += right; + } +} + +std::uint32_t AudioOut2MixPorts(const AudioOut2Context& context, float* out, std::uint32_t frames) { + std::lock_guard lock(g_portsLock); + std::uint32_t mixed = 0; + for (const auto& port : g_ports) { + if (!port.used || port.context != &context || port.data == nullptr || port.channels == 0) continue; + AccumulatePort(port, out, frames); + mixed++; + } + return mixed; +} + +void AudioOut2ReleasePorts(const AudioOut2Context& context) { + std::lock_guard lock(g_portsLock); + for (auto& port : g_ports) { + if (port.used && port.context == &context) port = AudioOut2Port{}; + } +} + +static void TraceAttribute(AudioOut2PortHandle handle, const AudioOut2Port& port, const AudioOut2Attribute& attribute, const char* verdict) { + if (!AudioOut2TraceEnabled()) return; + char values[128] = ""; + if (attribute.value != nullptr) { + std::size_t used = 0; + const auto words = std::min(attribute.value_size / sizeof(std::uint32_t), 4); + for (std::size_t index = 0; index < words && used < sizeof(values); index++) { + std::uint32_t bits = 0; + float value = 0.0f; + std::memcpy(&bits, static_cast(attribute.value) + index * sizeof(bits), sizeof(bits)); + std::memcpy(&value, &bits, sizeof(value)); + used += static_cast(std::snprintf(values + used, sizeof(values) - used, " %08x(%g)", bits, static_cast(value))); + } + } + std::fprintf(stderr, "[audioout2] t=%.3f port %llu set attribute id=0x%x size=%zu value=%p%s: %s (data sets so far %llu)\n", + AudioOut2TraceSeconds(), static_cast(handle), attribute.attribute_id, attribute.value_size, attribute.value, values, verdict, + static_cast(port.dataSets)); +} extern "C" { int APS5_VABI sceAudioOut2PortCreate(AudioOut2ContextHandle ctx, const AudioOut2PortParam* params, AudioOut2PortHandle* port) { - (void)ctx; - (void)params; - (void)port; - NotImplemented_nid_no_patch(__func__); + if (!ctx) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + if (!params || !port) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + std::lock_guard lock(g_portsLock); + std::size_t index = 0; + while (index < g_ports.size() && g_ports[index].used) index++; + if (index == g_ports.size()) g_ports.emplace_back(); + auto& entry = g_ports[index]; + entry = AudioOut2Port{}; + entry.used = true; + entry.context = reinterpret_cast(ctx); + entry.type = params->port_type; + entry.dataFormat = params->data_format; + entry.samplingFreq = params->sampling_freq; + entry.flags = params->flags; + entry.channels = (params->data_format >> FORMAT_CHANNELS_SHIFT) & FORMAT_CHANNELS_MASK; + if (entry.channels > AUDIO_OUT2_PORT_CHANNELS_MAX) entry.channels = 0; + *port = static_cast(index) + 1; + AUDIOOUT2_TRACE("t=%.3f PortCreate ctx=%llx -> port %llu: type=0x%x data_format=0x%x (%u float ch) sampling_freq=%u flags=0x%x user=%llx\n", + AudioOut2TraceSeconds(), static_cast(ctx), static_cast(*port), params->port_type, params->data_format, + entry.channels, params->sampling_freq, params->flags, static_cast(params->user_handle)); return 0; } int APS5_VABI sceAudioOut2PortDestroy(AudioOut2PortHandle port) { - (void)port; - NotImplemented_nid_no_patch(__func__); + std::lock_guard lock(g_portsLock); + auto* entry = FromHandle(port); + if (!entry) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + AUDIOOUT2_TRACE("t=%.3f PortDestroy port %llu (data sets %llu)\n", AudioOut2TraceSeconds(), static_cast(port), static_cast(entry->dataSets)); + *entry = AudioOut2Port{}; return 0; } int APS5_VABI sceAudioOut2PortGetState(AudioOut2PortHandle port, AudioOut2PortState* state) { - (void)port; - (void)state; - NotImplemented_nid_no_patch(__func__); + std::lock_guard lock(g_portsLock); + if (!FromHandle(port)) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + if (!state) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + state->output = OUTPUT_MAIN; + state->num_channels = AUDIO_OUT2_OUTPUT_CHANNELS; + state->volume = VOLUME_MAX; + state->reroute_counter = 0; + state->flags = 0; return 0; } int APS5_VABI sceAudioOut2PortSetAttributes(AudioOut2PortHandle port, const AudioOut2Attribute* attributes, uint32_t num) { - (void)port; - (void)attributes; - (void)num; - NotImplemented_nid_no_patch(__func__); + std::lock_guard lock(g_portsLock); + auto* entry = FromHandle(port); + if (!entry) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; + if (!attributes && num != 0) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + for (uint32_t index = 0; index < num; index++) { + const auto& attribute = attributes[index]; + const char* verdict = "ignored"; + if (attribute.value == nullptr) { + verdict = "null value"; + } else if (attribute.attribute_id == ATTRIBUTE_DATA && attribute.value_size == sizeof(entry->data)) { + std::memcpy(&entry->data, attribute.value, sizeof(entry->data)); + entry->dataSets++; + verdict = "pcm data pointer"; + } else if (attribute.attribute_id == ATTRIBUTE_VOLUME && entry->channels != 0 && attribute.value_size == entry->channels * sizeof(float)) { + std::memcpy(entry->volume, attribute.value, attribute.value_size); + verdict = "volume"; + } + const auto traces = entry->attributeTraces++; + if (traces < ATTRIBUTE_TRACE_FULL || traces % ATTRIBUTE_TRACE_EVERY == 0) TraceAttribute(port, *entry, attribute, verdict); + } return 0; } diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2SpeakerArray.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2SpeakerArray.cpp index 8e3275ad37..5f98507263 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2SpeakerArray.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2SpeakerArray.cpp @@ -49,9 +49,15 @@ size_t APS5_VABI sceAudioOut2GetSpeakerArrayMemorySize(uint32_t num_speakers, ui } int APS5_VABI sceAudioOut2GetSpeakerInfo(AudioOut2SpeakerInfo* info, uint32_t flags) { - (void)info; (void)flags; - NotImplemented_nid_no_patch(__func__); + if (!info) return static_cast(0x80260502); + constexpr std::uint8_t SpeakerTypeStereo = 0; + constexpr std::uint32_t FrontLeftAndRight = 0x3; + *info = AudioOut2SpeakerInfo{}; + info->type = SpeakerTypeStereo; + info->available_bits = FrontLeftAndRight; + info->speaker_angle[0] = {-30, 0}; + info->speaker_angle[1] = {30, 0}; return 0; } diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp index 021ddc4974..b690814dec 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp @@ -4,13 +4,12 @@ extern "C" { int APS5_VABI sceAudioOut2Initialize(void) { - NotImplemented_nid_no_patch(__func__); return 0; } int APS5_VABI sceAudioOut2GetSystemState(AudioOut2SystemState* state) { - (void)state; - NotImplemented_nid_no_patch(__func__); + if (!state) return static_cast(0x80260502); + state->loudness = 0.0f; return 0; } diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2User.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2User.cpp index 8373b6838d..a139d22626 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2User.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2User.cpp @@ -1,19 +1,23 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +static constexpr int SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT = static_cast(0x80260502); + +static std::atomic g_nextUser{1}; extern "C" { int APS5_VABI sceAudioOut2UserCreate(uint32_t user_id, AudioOut2UserHandle* handle) { (void)user_id; - (void)handle; - NotImplemented_nid_no_patch(__func__); + if (!handle) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + *handle = g_nextUser.fetch_add(1, std::memory_order_relaxed); return 0; } int APS5_VABI sceAudioOut2UserDestroy(AudioOut2UserHandle handle) { (void)handle; - NotImplemented_nid_no_patch(__func__); return 0; } From 9c2a75601deec10a2dba1eadd74a89f342d5e793 Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:27:49 -0700 Subject: [PATCH 05/93] feat(libs): agc packet builders, savedata memory, offline service stubs - libSceAgc: SetIndexCount, SetBaseIndirectArgs, DispatchIndirect, wait and write-data patch functions, relaxed checks for what titles send, NOP fillers for LOD stats and wait-until-safe, NotImplemented stubs. - libSceSaveData.native: short mount points, backup events, save-data memory slots. - user/system service, store dialog, video-out flip modes and queue-full error, pad read, fibers, offline behaviour for network/NP modules, and new export-only modules Ampr, JpegEnc, Json2, MsgDialog.native, NpCppWebApi, NpUtility. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/libs/prx/libSceAcm/Export.cpp | 58 +- core/libs/prx/libSceAgc/Acb/src/Control.cpp | 15 +- core/libs/prx/libSceAgc/Acb/src/Dispatch.cpp | 10 +- core/libs/prx/libSceAgc/CMakeLists.txt | 1 + core/libs/prx/libSceAgc/Cb/src/Dispatch.cpp | 2 +- core/libs/prx/libSceAgc/Cb/src/Sync.cpp | 12 +- .../libs/prx/libSceAgc/Command/src/Memory.cpp | 8 +- .../libs/prx/libSceAgc/Command/src/Packet.cpp | 5 +- .../prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp | 8 +- .../prx/libSceAgc/DcbDraw/src/Instancing.cpp | 16 +- core/libs/prx/libSceAgc/DcbDraw/src/Stats.cpp | 5 +- .../prx/libSceAgc/DcbFlow/src/Control.cpp | 8 +- .../prx/libSceAgc/DcbFlow/src/Dispatch.cpp | 8 +- .../libs/prx/libSceAgc/DcbFlow/src/Memory.cpp | 11 +- .../prx/libSceAgc/DcbState/src/Marker.cpp | 52 +- .../prx/libSceAgc/Patch/src/WaitRegMem.cpp | 14 +- core/libs/prx/libSceAgc/Unimplemented.cpp | 172 ++++ core/libs/prx/libSceAjm/Export.cpp | 2 +- core/libs/prx/libSceAjmi/Export.cpp | 2 +- core/libs/prx/libSceAmpr/Export.cpp | 335 ++++++++ core/libs/prx/libSceAppContent/Export.cpp | 28 +- core/libs/prx/libSceCesCs/Export.cpp | 2 +- core/libs/prx/libSceFiber/Export.cpp | 425 ++++++++-- core/libs/prx/libSceFont.native/Export.cpp | 2 +- core/libs/prx/libSceFont/Export.cpp | 2 +- core/libs/prx/libSceGameUpdate/Export.cpp | 31 +- core/libs/prx/libSceHttp/Export.cpp | 269 +++---- core/libs/prx/libSceHttp2/Export.cpp | 249 +++--- core/libs/prx/libSceJpegEnc/Export.cpp | 28 + core/libs/prx/libSceJson2/Export.cpp | 37 + .../prx/libSceMsgDialog.native/Export.cpp | 10 + core/libs/prx/libSceNet/Export.cpp | 356 ++++++--- core/libs/prx/libSceNetCtl/Export.cpp | 73 +- core/libs/prx/libSceNpCommerce/Export.cpp | 57 +- core/libs/prx/libSceNpCppWebApi/Export.cpp | 745 ++++++++++++++++++ .../prx/libSceNpEntitlementAccess/Export.cpp | 39 +- core/libs/prx/libSceNpManager/Export.cpp | 105 ++- .../prx/libSceNpSessionSignaling/Export.cpp | 40 +- core/libs/prx/libSceNpUtility/Export.cpp | 10 + core/libs/prx/libSceNpWebApi2/Export.cpp | 175 ++-- core/libs/prx/libScePad/Export.cpp | 12 +- core/libs/prx/libScePlayGo/Export.cpp | 189 +++-- core/libs/prx/libScePngEnc/Export.cpp | 28 + core/libs/prx/libScePsml_debug/Export.cpp | 86 ++ .../libs/prx/libSceSaveData.native/Export.cpp | 148 +++- core/libs/prx/libSceShare.native/Export.cpp | 59 +- core/libs/prx/libSceShare/Export.cpp | 88 +++ core/libs/prx/libSceSsl/Export.cpp | 22 +- core/libs/prx/libSceSystemGesture/Export.cpp | 67 +- core/libs/prx/libSceSystemService/Export.cpp | 24 +- core/libs/prx/libSceUserService/Export.cpp | 43 +- core/libs/prx/libSceVideoOut/CMakeLists.txt | 1 + .../libs/prx/libSceVideoOut/Unimplemented.cpp | 20 + .../libSceVideoOut/include/VideoOutDriver.hpp | 3 +- core/libs/prx/libSceVideoOut/src/Buffer.cpp | 9 +- core/libs/prx/libSceVideoOut/src/Output.cpp | 9 +- .../prx/libSceVideoOut/src/VideoOutDriver.cpp | 32 +- 57 files changed, 3328 insertions(+), 939 deletions(-) create mode 100644 core/libs/prx/libSceAgc/Unimplemented.cpp create mode 100644 core/libs/prx/libSceAmpr/Export.cpp create mode 100644 core/libs/prx/libSceJpegEnc/Export.cpp create mode 100644 core/libs/prx/libSceJson2/Export.cpp create mode 100644 core/libs/prx/libSceMsgDialog.native/Export.cpp create mode 100644 core/libs/prx/libSceNpCppWebApi/Export.cpp create mode 100644 core/libs/prx/libSceNpUtility/Export.cpp create mode 100644 core/libs/prx/libScePngEnc/Export.cpp create mode 100644 core/libs/prx/libScePsml_debug/Export.cpp create mode 100644 core/libs/prx/libSceShare/Export.cpp create mode 100644 core/libs/prx/libSceVideoOut/Unimplemented.cpp diff --git a/core/libs/prx/libSceAcm/Export.cpp b/core/libs/prx/libSceAcm/Export.cpp index 1a39101632..db47c5fea0 100644 --- a/core/libs/prx/libSceAcm/Export.cpp +++ b/core/libs/prx/libSceAcm/Export.cpp @@ -2,47 +2,55 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +// Audio co-processor batches complete immediately without producing output; audio is silent. +static constexpr int SCE_ACM_ERROR_INVALID_PARAMETER = static_cast(0x80E30002); +static std::atomic g_nextId{1}; extern "C" { int APS5_VABI sceAcmBatchStartBuffer(AcmContextId context, const void* batch_commands, size_t batch_size, AcmBatchError* batch_error, AcmBatchId* batch) { - (void)context; - (void)batch_commands; - (void)batch_size; - (void)batch_error; - (void)batch; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)context; + (void)batch_commands; + (void)batch_size; + (void)batch_error; + if (!batch) return SCE_ACM_ERROR_INVALID_PARAMETER; + *batch = g_nextId.fetch_add(1, std::memory_order_relaxed); + return 0; } int APS5_VABI sceAcmBatchStartBuffers(AcmContextId context, uint32_t batch_info_count, const AcmBatchInfo* const batch_info[], AcmBatchError* batch_error, AcmBatchId* batch) { - (void)context; - (void)batch_info_count; - (void)batch_info; - (void)batch_error; - (void)batch; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)context; + (void)batch_info_count; + (void)batch_info; + (void)batch_error; + if (!batch) return SCE_ACM_ERROR_INVALID_PARAMETER; + *batch = g_nextId.fetch_add(1, std::memory_order_relaxed); + return 0; } int APS5_VABI sceAcmBatchWait(AcmContextId context, AcmBatchId batch, uint32_t timeout) { - (void)context; - (void)batch; - (void)timeout; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)context; + (void)batch; + (void)timeout; + return 0; } int APS5_VABI sceAcmContextCreate(AcmContextId* context) { - (void)context; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!context) return SCE_ACM_ERROR_INVALID_PARAMETER; + *context = g_nextId.fetch_add(1, std::memory_order_relaxed); + return 0; } int APS5_VABI sceAcmContextDestroy(AcmContextId context) { - (void)context; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)context; + return 0; +} + +// Builds a convolution-reverb command into a batch; batches execute as no-ops, so nothing is encoded. +int APS5_VABI sceAcm_ConvReverb_SharedInput(void) { + return 0; } } diff --git a/core/libs/prx/libSceAgc/Acb/src/Control.cpp b/core/libs/prx/libSceAgc/Acb/src/Control.cpp index 4532f538b1..9bcea70414 100644 --- a/core/libs/prx/libSceAgc/Acb/src/Control.cpp +++ b/core/libs/prx/libSceAgc/Acb/src/Control.cpp @@ -30,11 +30,16 @@ uint32_t APS5_VABI sceAgcAcbJumpGetSize(void) { return 0; } -uint32_t* APS5_VABI sceAgcAcbResetQueue(CommandBuffer* buf, uint32_t op) { - (void)buf; - (void)op; - NotImplemented_nid_no_patch(__func__); - return nullptr; +// Encoded as the custom DISPATCH_RESET NOP packet the driver consumes on compute queues. +uint32_t* APS5_VABI sceAgcAcbResetQueue(CommandBuffer* buf, uint32_t op, uint32_t value) { + (void)value; + Agc::Command::Require(buf != nullptr, __func__, "null command buffer"); + constexpr std::uint32_t OpcodeNop = 0x10; + constexpr std::uint32_t CustomDispatchReset = 0x09; + auto* packet = Agc::Command::Allocate(buf, 2, __func__); + packet[0] = Agc::Command::Header(OpcodeNop, 2, CustomDispatchReset << 2); + packet[1] = op; + return packet; } std::uint32_t* APS5_VABI sceAgcAcbRewind(CommandBuffer* buf, std::uint32_t initialState) { diff --git a/core/libs/prx/libSceAgc/Acb/src/Dispatch.cpp b/core/libs/prx/libSceAgc/Acb/src/Dispatch.cpp index 7ceb0ec94e..ae28cda63d 100644 --- a/core/libs/prx/libSceAgc/Acb/src/Dispatch.cpp +++ b/core/libs/prx/libSceAgc/Acb/src/Dispatch.cpp @@ -10,11 +10,11 @@ extern "C" { uint32_t* APS5_VABI sceAgcAcbDispatchIndirect(CommandBuffer* buf, const volatile void* indirect_args, uint32_t modifier) { - (void)buf; - (void)indirect_args; - (void)modifier; - NotImplemented_nid_no_patch(__func__); - return nullptr; + // Compute queues take the argument address directly (DISPATCH_INDIRECT with a 64-bit address). + const auto address = reinterpret_cast(indirect_args); + Agc::Command::CheckAddress(address, 4, __func__); + Agc::Command::CheckBits(modifier, 0xa038u | 0x41u, __func__); + return Agc::Command::Emit(buf, 0x16u, {static_cast(address), static_cast(address >> 32u), modifier | 0x41u}, __func__); } std::uint32_t APS5_VABI sceAgcAcbDispatchIndirectGetSize() { diff --git a/core/libs/prx/libSceAgc/CMakeLists.txt b/core/libs/prx/libSceAgc/CMakeLists.txt index 6961369ba3..d1d99b23c9 100644 --- a/core/libs/prx/libSceAgc/CMakeLists.txt +++ b/core/libs/prx/libSceAgc/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(libSceAgc SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Acb/src/Control.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Acb/src/Dispatch.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Acb/src/Memory.cpp diff --git a/core/libs/prx/libSceAgc/Cb/src/Dispatch.cpp b/core/libs/prx/libSceAgc/Cb/src/Dispatch.cpp index 088b230124..7e3f94a533 100644 --- a/core/libs/prx/libSceAgc/Cb/src/Dispatch.cpp +++ b/core/libs/prx/libSceAgc/Cb/src/Dispatch.cpp @@ -9,7 +9,7 @@ extern "C" { std::uint32_t* APS5_VABI sceAgcCbDispatch(CommandBuffer* buf, std::uint32_t threadGroupX, std::uint32_t threadGroupY, std::uint32_t threadGroupZ, std::uint32_t modifier) { - Agc::Command::CheckBits(modifier, 0xa038u, __func__); + Agc::Command::CheckBits(modifier, 0xa038u | 0x41u, __func__); return Agc::Command::Emit(buf, 0x15u, {threadGroupX, threadGroupY, threadGroupZ, modifier | 0x41u}, __func__); } diff --git a/core/libs/prx/libSceAgc/Cb/src/Sync.cpp b/core/libs/prx/libSceAgc/Cb/src/Sync.cpp index 74115f047d..f418119c55 100644 --- a/core/libs/prx/libSceAgc/Cb/src/Sync.cpp +++ b/core/libs/prx/libSceAgc/Cb/src/Sync.cpp @@ -1,6 +1,8 @@ #include "prx/libSceAgc/Cb/include/Sync.hpp" #include "prx/libSceAgc/Command/include/Packet.hpp" +#include +#include #include #include #include "SceTypes.hpp" @@ -37,10 +39,9 @@ std::uint32_t* APS5_VABI sceAgcCbReleaseMem(CommandBuffer* buf, std::uint8_t act } else if (dataSelect == 5) { Agc::Command::Require(data == 0, __func__, "GDS release cannot use immediate data"); value = gdsOffset | (static_cast(gdsSize) << 16u); - } else { - Agc::Command::Require(gdsOffset == 0 && gdsSize == 0, __func__, "GDS parameters supplied for a non-GDS release"); } - if (dataSelect != 0 && interrupt != 4) { + // A null label is accepted: games emit releases whose destination is never read. + if (dataSelect != 0 && interrupt != 4 && guestAddress != 0) { Agc::Command::CheckAddress(guestAddress, dataSelect == 2 || dataSelect == 3 ? 8 : 4, __func__); } if (dataSelect == 1) { @@ -51,7 +52,10 @@ std::uint32_t* APS5_VABI sceAgcCbReleaseMem(CommandBuffer* buf, std::uint8_t act control |= 0x200u; } const auto eventIndex = action >= 0x2fu ? 6u : 5u; - return Agc::Command::Emit(buf, 0x49u, {action | (eventIndex << 8u) | (control << 12u) | (static_cast(cachePolicy) << 25u), (static_cast(dst) << 16u) | (static_cast(interrupt) << 24u) | (static_cast(dataSelect) << 29u), static_cast(guestAddress), static_cast(guestAddress >> 32u), static_cast(value), static_cast(value >> 32u), interruptContextId}, __func__); + auto* packet = Agc::Command::Emit(buf, 0x49u, {action | (eventIndex << 8u) | (control << 12u) | (static_cast(cachePolicy) << 25u), (static_cast(dst) << 16u) | (static_cast(interrupt) << 24u) | (static_cast(dataSelect) << 29u), static_cast(guestAddress), static_cast(guestAddress >> 32u), static_cast(value), static_cast(value >> 32u), interruptContextId}, __func__); + static const std::uint64_t traced = [] { const char* value = std::getenv("APS5_TRACE_LABEL"); return value ? std::strtoull(value, nullptr, 16) : 0ull; }(); + if (traced != 0 && guestAddress == traced) std::fprintf(stderr, "[agc] RELEASE_MEM for 0x%llx recorded at %p" "\n", static_cast(guestAddress), static_cast(packet)); + return packet; } } diff --git a/core/libs/prx/libSceAgc/Command/src/Memory.cpp b/core/libs/prx/libSceAgc/Command/src/Memory.cpp index 04f08584ec..c1952da411 100644 --- a/core/libs/prx/libSceAgc/Command/src/Memory.cpp +++ b/core/libs/prx/libSceAgc/Command/src/Memory.cpp @@ -53,11 +53,13 @@ std::uint32_t* WriteWait(CommandBuffer* buffer, std::uint8_t size, std::uint8_t CheckBits(cachePolicy, 3, function); Require((pollCycles >> 4u) <= 0xffffu, function, "wait poll interval overflow"); if (size == 0) { - CheckBits(reference, 0xffffffffu, function); - CheckBits(mask, 0xffffffffu, function); + // 32-bit waits only use the low halves; guests commonly pass a full 64-bit mask. + reference &= 0xffffffffu; + mask &= 0xffffffffu; } const auto guestAddress = reinterpret_cast(address); - CheckAddress(guestAddress, size == 0 ? 4 : 8, function); + // The address may be left null and filled in later with sceAgcWaitRegMemPatchAddress. + if (guestAddress != 0) CheckAddress(guestAddress, size == 0 ? 4 : 8, function); CheckBits(guestAddress, 0xffffffffffffull, function); const auto waitSize = size == 0 ? 7u : 9u; auto* packet = Allocate(buffer, waitSize + 7u, function); diff --git a/core/libs/prx/libSceAgc/Command/src/Packet.cpp b/core/libs/prx/libSceAgc/Command/src/Packet.cpp index ac4a85471e..aebb7c0f4f 100644 --- a/core/libs/prx/libSceAgc/Command/src/Packet.cpp +++ b/core/libs/prx/libSceAgc/Command/src/Packet.cpp @@ -15,7 +15,10 @@ void Require(bool condition, const char* function, const char* reason) { } void CheckBits(std::uint64_t value, std::uint64_t mask, const char* function) { - Require((value & ~mask) == 0, function, "reserved bits are set"); + if ((value & ~mask) == 0) return; + char reason[96]; + std::snprintf(reason, sizeof(reason), "reserved bits are set (value 0x%llx, allowed 0x%llx)", static_cast(value), static_cast(mask)); + Require(false, function, reason); } void CheckAddress(std::uint64_t address, std::uint32_t alignment, const char* function) { diff --git a/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp b/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp index e7d38de163..9ff4fb9fdb 100644 --- a/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp +++ b/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp @@ -19,15 +19,11 @@ std::uint32_t APS5_VABI sceAgcDcbSetIndexBufferGetSize() { } uint32_t* APS5_VABI sceAgcDcbSetIndexCount(CommandBuffer* buf, uint32_t index_count) { - (void)buf; - (void)index_count; - NotImplemented_nid_no_patch(__func__); - return nullptr; + return Agc::Command::Emit(buf, 0x13u, {index_count}, __func__); } std::uint32_t APS5_VABI sceAgcDcbSetIndexCountGetSize() { - NotImplemented_nid_no_patch(__func__); - return 0; + return 2; } std::uint32_t* APS5_VABI sceAgcDcbSetIndexSize(CommandBuffer* buf, std::uint8_t indexSize, std::uint8_t cachePolicy) { diff --git a/core/libs/prx/libSceAgc/DcbDraw/src/Instancing.cpp b/core/libs/prx/libSceAgc/DcbDraw/src/Instancing.cpp index c1e7ad9865..35a5fcba87 100644 --- a/core/libs/prx/libSceAgc/DcbDraw/src/Instancing.cpp +++ b/core/libs/prx/libSceAgc/DcbDraw/src/Instancing.cpp @@ -17,11 +17,17 @@ std::uint32_t APS5_VABI sceAgcDcbSetNumInstancesGetSize() { } uint32_t* APS5_VABI sceAgcDcbSetBaseIndirectArgs(CommandBuffer* buf, uint32_t shader_type, const volatile void* indirect_base_addr) { - (void)buf; - (void)shader_type; - (void)indirect_base_addr; - NotImplemented_nid_no_patch(__func__); - return nullptr; + // SET_BASE with base index 1; header bit 1 selects the dispatch (compute) base over the draw base. + Agc::Command::CheckBits(shader_type, 1, __func__); + const auto address = reinterpret_cast(indirect_base_addr); + Agc::Command::CheckAddress(address, 8, __func__); + Agc::Command::CheckBits(address, 0xffffffffffffull, __func__); + auto* packet = Agc::Command::Allocate(buf, 4, __func__); + packet[0] = Agc::Command::Header(0x11u, 4, shader_type != 0 ? 2u : 0u); + packet[1] = 1; + packet[2] = static_cast(address); + packet[3] = static_cast(address >> 32u); + return packet; } } diff --git a/core/libs/prx/libSceAgc/DcbDraw/src/Stats.cpp b/core/libs/prx/libSceAgc/DcbDraw/src/Stats.cpp index 84da7348d6..b3ae56e199 100644 --- a/core/libs/prx/libSceAgc/DcbDraw/src/Stats.cpp +++ b/core/libs/prx/libSceAgc/DcbDraw/src/Stats.cpp @@ -15,9 +15,10 @@ std::uint32_t* APS5_VABI sceAgcDcbGetLodStats(CommandBuffer* buf, std::uint8_t c Agc::Command::CheckBits(reportAndReset, 1, __func__); Agc::Command::CheckBits(reportingIntervalIn100kClocks, 0xffu, __func__); const auto address = reinterpret_cast(buffer); - Agc::Command::CheckAddress(address, 64, __func__); + if (address != 0) Agc::Command::CheckAddress(address, 64, __func__); const auto control = (static_cast(cachePolicy) << 28u) | (static_cast(reportAndReset) << 19u) | (static_cast(forceReset) << 18u) | (resetCount << 10u) | (reportingIntervalIn100kClocks << 2u); - return Agc::Command::Emit(buf, 0x8eu, {bufferSizeInBytes, static_cast(address), static_cast(address >> 32u), control}, __func__); + // LOD statistics are not collected; a same-sized NOP keeps the packet layout without producing data. + return Agc::Command::Emit(buf, 0x10u, {bufferSizeInBytes, static_cast(address), static_cast(address >> 32u), control}, __func__); } std::uint32_t APS5_VABI sceAgcDcbBeginOcclusionQueryGetSize() { diff --git a/core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp b/core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp index 92ef8afc10..d348b2c5b2 100644 --- a/core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp +++ b/core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp @@ -51,11 +51,9 @@ uint32_t APS5_VABI sceAgcDcbRewindGetSize(void) { } uint32_t* APS5_VABI sceAgcDcbWaitUntilSafeForRendering(CommandBuffer* buf, uint32_t video_out_handle, uint32_t display_buffer_index) { - (void)buf; - (void)video_out_handle; - (void)display_buffer_index; - NotImplemented_nid_no_patch(__func__); - return nullptr; + // Presentation copies the display buffer when the flip executes, so the buffer is always safe to + // render into; keep the arguments in a NOP for command-buffer dumps. + return Agc::Command::Emit(buf, 0x10u, {video_out_handle, display_buffer_index}, __func__); } } diff --git a/core/libs/prx/libSceAgc/DcbFlow/src/Dispatch.cpp b/core/libs/prx/libSceAgc/DcbFlow/src/Dispatch.cpp index 2b29438a17..5624c73f03 100644 --- a/core/libs/prx/libSceAgc/DcbFlow/src/Dispatch.cpp +++ b/core/libs/prx/libSceAgc/DcbFlow/src/Dispatch.cpp @@ -10,11 +10,9 @@ extern "C" { uint32_t* APS5_VABI sceAgcDcbDispatchIndirect(CommandBuffer* buf, uint32_t data_offset_in_bytes, uint32_t flags) { - (void)buf; - (void)data_offset_in_bytes; - (void)flags; - NotImplemented_nid_no_patch(__func__); - return nullptr; + // Graphics queues read the arguments at an offset from the base set by SetBaseDispatchIndirectArgs. + Agc::Command::CheckBits(flags, 0xa038u | 0x41u, __func__); + return Agc::Command::Emit(buf, 0x16u, {data_offset_in_bytes, flags | 0x41u}, __func__); } std::uint32_t APS5_VABI sceAgcDcbDispatchIndirectGetSize() { diff --git a/core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp b/core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp index 67c4dcd8ab..4f6b3c805e 100644 --- a/core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp +++ b/core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp @@ -13,10 +13,13 @@ std::uint32_t* APS5_VABI sceAgcDcbAcquireMem(CommandBuffer* buf, std::uint8_t en Agc::Command::CheckBits(engine, 1, __func__); Agc::Command::CheckBits(cbDbOp, 0x7fffffffu, __func__); Agc::Command::CheckBits(gcrControl, 0x7ffffu, __func__); - const auto address = reinterpret_cast(base); - Agc::Command::Require((address & 0xffu) == 0 && (address >> 40u) == 0, __func__, "invalid acquire memory base address"); - const auto wholeAddressSpace = sizeBytes == 0xffffffffffffffffull; - Agc::Command::Require(wholeAddressSpace || ((sizeBytes & 0xffu) == 0 && (sizeBytes >> 40u) == 0), __func__, "invalid acquire memory range size"); + // The range only scopes cache maintenance, so round it outward to the 256-byte granularity the + // packet encodes instead of rejecting unaligned ranges. + const auto requested = reinterpret_cast(base); + auto address = requested & ~static_cast(0xffu); + const auto wholeAddressSpace = sizeBytes == 0xffffffffffffffffull || (address >> 40u) != 0 || sizeBytes > (1ull << 40u); + if (wholeAddressSpace) address = 0; + else sizeBytes = (sizeBytes + (requested - address) + 0xffu) & ~0xffull; Agc::Command::Require(pollCycles / 40u <= 0xffffu, __func__, "acquire poll interval overflow"); return Agc::Command::Emit(buf, 0x58u, {(static_cast(engine) << 31u) | cbDbOp, wholeAddressSpace ? 0u : static_cast(sizeBytes >> 8u), 0, static_cast(address >> 8u), 0, pollCycles / 40u, gcrControl}, __func__); } diff --git a/core/libs/prx/libSceAgc/DcbState/src/Marker.cpp b/core/libs/prx/libSceAgc/DcbState/src/Marker.cpp index d2d66306cb..f457dcc3fe 100644 --- a/core/libs/prx/libSceAgc/DcbState/src/Marker.cpp +++ b/core/libs/prx/libSceAgc/DcbState/src/Marker.cpp @@ -3,31 +3,57 @@ #include "prx/libSceAgc/Command/include/Packet.hpp" #include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +// Debug markers are custom NOP packets: PUSH carries the NUL-terminated text, POP has no payload. +// Marker colors are only meaningful to capture tools and are not encoded. +namespace { + +constexpr std::uint32_t OpcodeNop = 0x10; +constexpr std::uint32_t CustomPushMarker = 0x0b; +constexpr std::uint32_t CustomPopMarker = 0x0c; + +std::uint32_t* PushMarker(CommandBuffer* buf, const char* str, const char* function) { + Agc::Command::Require(buf != nullptr, function, "null command buffer"); + const char* text = str ? str : ""; + const std::size_t bytes = std::strlen(text) + 1; + const auto payload = static_cast((bytes + 3) / 4); + Agc::Command::Require(payload <= 0x4000u, function, "marker text too long"); + auto* packet = Agc::Command::Allocate(buf, payload + 1, function); + packet[0] = Agc::Command::Header(OpcodeNop, payload + 1, CustomPushMarker << 2); + std::memset(packet + 1, 0, payload * sizeof(std::uint32_t)); + std::memcpy(packet + 1, text, bytes); + return packet; +} + +std::uint32_t* PopMarker(CommandBuffer* buf, const char* function) { + Agc::Command::Require(buf != nullptr, function, "null command buffer"); + auto* packet = Agc::Command::Allocate(buf, 2, function); + packet[0] = Agc::Command::Header(OpcodeNop, 2, CustomPopMarker << 2); + packet[1] = 0; + return packet; +} + +} + extern "C" { uint32_t* APS5_VABI sceAgcDcbSetMarker(CommandBuffer* buf, const char* str, uint32_t color) { - (void)buf; - (void)str; - (void)color; - NotImplemented_nid_no_patch(__func__); - return nullptr; + (void)color; + auto* packet = PushMarker(buf, str, __func__); + PopMarker(buf, __func__); + return packet; } uint32_t* APS5_VABI sceAgcDcbPopMarker(CommandBuffer* buf) { - (void)buf; - NotImplemented_nid_no_patch(__func__); - return nullptr; + return PopMarker(buf, __func__); } uint32_t* APS5_VABI sceAgcDcbPushMarker(CommandBuffer* buf, const char* str, uint32_t color) { - (void)buf; - (void)str; - (void)color; - NotImplemented_nid_no_patch(__func__); - return nullptr; + (void)color; + return PushMarker(buf, str, __func__); } } diff --git a/core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp b/core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp index 0baf91ef61..daa5b4216b 100644 --- a/core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp +++ b/core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp @@ -30,9 +30,17 @@ int APS5_VABI sceAgcWaitRegMemPatchReference(std::uint32_t* cmd, std::uint64_t r } int APS5_VABI sceAgcWaitRegMemPatchCompareFunction(std::uint32_t* cmd, std::uint8_t compareFunction) { - (void)cmd; - (void)compareFunction; - NotImplemented_nid_no_patch(__func__); + auto* wait = Agc::Command::ValidateWait(cmd, __func__); + Agc::Command::Require(compareFunction <= 6, __func__, "invalid wait comparison"); + wait[1] = (wait[1] & ~7u) | compareFunction; + return 0; +} + +int APS5_VABI sceAgcWriteDataPatchSetAddressOrOffset(std::uint32_t* cmd, std::uint64_t address) { + Agc::Command::CheckAddress(reinterpret_cast(cmd), 4, __func__); + Agc::Command::Require(((cmd[0] >> 8u) & 0xffu) == 0x37u, __func__, "not a WRITE_DATA packet"); + cmd[2] = static_cast(address); + cmd[3] = static_cast(address >> 32u); return 0; } diff --git a/core/libs/prx/libSceAgc/Unimplemented.cpp b/core/libs/prx/libSceAgc/Unimplemented.cpp new file mode 100644 index 0000000000..b4b94f6a4c --- /dev/null +++ b/core/libs/prx/libSceAgc/Unimplemented.cpp @@ -0,0 +1,172 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI sceAgcAcbPushMarkerSpan() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcAcbSetMarkerSpan() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcAcbSetWorkloadComplete() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcAcbSetWorkloadStreamInactive() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcAcbSetWorkloadsActive() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcAcquireMemSetEngine() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDcbPushMarkerSpan() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDcbSetMarkerSpan() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDcbSetWorkloadStreamInactive() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDebugRaiseException() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcGetDataPacketPayloadRange() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcGetDefaultCxStateFlat() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcGetGsOversubscription() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcGetIsTrinityMode() { + return 0; +} + +int APS5_VABI sceAgcGetSemaphoreLabel() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcSetAmmSemaphoreMemory() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("-KRzWekV120", sceAgcUnknown__MKRzWekV120); +// Unnamed DCB state toggle: games call it as (cb, enable, 0, 0) and ignore the result. It sits +// between SetBaseIndirectArgs and SetIndexBuffer in the import table; treated as a no-op. +std::uint32_t* APS5_VABI sceAgcUnknown__MKRzWekV120(void* buf, std::uint32_t enable, std::uint64_t a2, std::uint64_t a3) { + static bool logged = false; + if (!logged) { + logged = true; + std::fprintf(stderr, "[agc] -KRzWekV120 treated as no-op (enable=%u args=0x%llx,0x%llx)" "\n", enable, static_cast(a2), static_cast(a3)); + } + (void)buf; + return nullptr; +} + +APS5_EXPORT("6nths4DHNrs", sceAgcUnknown_6nths4DHNrs); +int APS5_VABI sceAgcUnknown_6nths4DHNrs() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("7Wa3aeJgeVU", sceAgcUnknown_7Wa3aeJgeVU); +int APS5_VABI sceAgcUnknown_7Wa3aeJgeVU() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("EJBA4dbmvfg", sceAgcUnknown_EJBA4dbmvfg); +int APS5_VABI sceAgcUnknown_EJBA4dbmvfg() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("ICkECTBxrMw", sceAgcUnknown_ICkECTBxrMw); +int APS5_VABI sceAgcUnknown_ICkECTBxrMw() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("Ikfdt-rIqCE", sceAgcUnknown_Ikfdt_MrIqCE); +int APS5_VABI sceAgcUnknown_Ikfdt_MrIqCE() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("RTpj-tIlvZc", sceAgcUnknown_RTpj_MtIlvZc); +int APS5_VABI sceAgcUnknown_RTpj_MtIlvZc() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("SwI6QxqwAC0", sceAgcUnknown_SwI6QxqwAC0); +int APS5_VABI sceAgcUnknown_SwI6QxqwAC0() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("d4NZIlguzv0", sceAgcUnknown_d4NZIlguzv0); +int APS5_VABI sceAgcUnknown_d4NZIlguzv0() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("eAy8eGNsCuU", sceAgcUnknown_eAy8eGNsCuU); +int APS5_VABI sceAgcUnknown_eAy8eGNsCuU() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("rP5xLdOf26k", sceAgcUnknown_rP5xLdOf26k); +int APS5_VABI sceAgcUnknown_rP5xLdOf26k() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("tmy-+rBpspY", sceAgcUnknown_tmy_M_PrBpspY); +int APS5_VABI sceAgcUnknown_tmy_M_PrBpspY() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("y5K5tPktiL8", sceAgcUnknown_y5K5tPktiL8); +int APS5_VABI sceAgcUnknown_y5K5tPktiL8() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceAjm/Export.cpp b/core/libs/prx/libSceAjm/Export.cpp index a7c85810ee..fa38a9b323 100644 --- a/core/libs/prx/libSceAjm/Export.cpp +++ b/core/libs/prx/libSceAjm/Export.cpp @@ -94,7 +94,7 @@ int APS5_VABI sceAjmModuleUnregister(uint32_t context, uint32_t codec) { return 0; } -const char* sceAjmStrError(int error) { +const char* APS5_VABI sceAjmStrError(int error) { (void)error; NotImplemented_nid_no_patch(__func__); return nullptr; diff --git a/core/libs/prx/libSceAjmi/Export.cpp b/core/libs/prx/libSceAjmi/Export.cpp index a7c85810ee..fa38a9b323 100644 --- a/core/libs/prx/libSceAjmi/Export.cpp +++ b/core/libs/prx/libSceAjmi/Export.cpp @@ -94,7 +94,7 @@ int APS5_VABI sceAjmModuleUnregister(uint32_t context, uint32_t codec) { return 0; } -const char* sceAjmStrError(int error) { +const char* APS5_VABI sceAjmStrError(int error) { (void)error; NotImplemented_nid_no_patch(__func__); return nullptr; diff --git a/core/libs/prx/libSceAmpr/Export.cpp b/core/libs/prx/libSceAmpr/Export.cpp new file mode 100644 index 0000000000..0deb6a264c --- /dev/null +++ b/core/libs/prx/libSceAmpr/Export.cpp @@ -0,0 +1,335 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" +#include "prx/libkernel/Apr/include/AprCommandBuffer.hpp" +#include + +static constexpr int SCE_AMPR_ERROR_BUFFER_FULL = 0x8002001C; + +static int Append(Apr::CommandBufferObject* buffer, const void* command, uint32_t bytes) { + if (!buffer->base || bytes > buffer->size - buffer->offset) return SCE_AMPR_ERROR_BUFFER_FULL; + std::memcpy(buffer->base + buffer->offset, command, bytes); + buffer->offset += bytes; + ++buffer->numCommands; + return 0; +} + +extern "C" { + +int APS5_VABI sceAmprAprCommandBufferConstructor(Apr::CommandBufferObject* buffer, uint64_t* gatherState, uint64_t* scatterState) { + buffer->type = Apr::BufferType::Apr; + *gatherState = 0; + *scatterState = 0; + return 0; +} + +int APS5_VABI sceAmprAprCommandBufferDestructor(Apr::CommandBufferObject* buffer, uint64_t* gatherState, uint64_t* scatterState) { + (void)buffer; + (void)gatherState; + (void)scatterState; + return 0; +} + +int APS5_VABI sceAmprAprCommandBufferMapBegin() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprAprCommandBufferMapDirectBegin() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprAprCommandBufferMapEnd() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprAprCommandBufferReadFile(Apr::CommandBufferObject* buffer, uint64_t* gatherState, uint64_t* scatterState, uint32_t fileId, void* destination, uint64_t size, uint64_t offset) { + (void)gatherState; + (void)scatterState; + Apr::ReadFileCommand command{}; + command.header = {Apr::Opcode::ReadFile, sizeof(command)}; + command.fileId = fileId; + command.destination = reinterpret_cast(destination); + command.size = size; + command.offset = offset; + return Append(buffer, &command, sizeof(command)); +} + +int APS5_VABI sceAmprAprCommandBufferReadFileGather() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprAprCommandBufferReadFileGatherScatter() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprAprCommandBufferReadFileScatter() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprAprCommandBufferResetGatherScatterState() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferClearBuffer(Apr::CommandBufferObject* buffer) { + buffer->base = nullptr; + buffer->size = 0; + buffer->offset = 0; + buffer->numCommands = 0; + return 0; +} + +int APS5_VABI sceAmprCommandBufferConstructMarker() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferConstructNop() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferConstructor(Apr::CommandBufferObject* buffer) { + *buffer = {nullptr, 0, 0, 0, Apr::BufferType::Generic}; + return 0; +} + +int APS5_VABI sceAmprCommandBufferDestructor(Apr::CommandBufferObject* buffer) { + (void)buffer; + return 0; +} + +void* APS5_VABI sceAmprCommandBufferGetBufferBaseAddress(const Apr::CommandBufferObject* buffer) { + return buffer->base; +} + +uint32_t APS5_VABI sceAmprCommandBufferGetCurrentOffset(const Apr::CommandBufferObject* buffer) { + return buffer->offset; +} + +uint32_t APS5_VABI sceAmprCommandBufferGetNumCommands(const Apr::CommandBufferObject* buffer) { + return buffer->numCommands; +} + +uint32_t APS5_VABI sceAmprCommandBufferGetSize(const Apr::CommandBufferObject* buffer) { + return buffer->size; +} + +uint32_t APS5_VABI sceAmprCommandBufferGetType(const Apr::CommandBufferObject* buffer) { + return static_cast(buffer->type); +} + +int APS5_VABI sceAmprCommandBufferNop() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferNopWithData() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferPopMarker() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferPushMarker() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferPushMarkerWithColor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferReset(Apr::CommandBufferObject* buffer) { + buffer->offset = 0; + buffer->numCommands = 0; + return 0; +} + +int APS5_VABI sceAmprCommandBufferSetBuffer(Apr::CommandBufferObject* buffer, void* memory, uint32_t size) { + buffer->base = static_cast(memory); + buffer->size = size; + buffer->offset = 0; + buffer->numCommands = 0; + return 0; +} + +int APS5_VABI sceAmprCommandBufferSetMarker() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferSetMarkerWithColor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferWaitOnAddress_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferWaitOnCounter_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferWriteAddressFromCounterPair_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferWriteAddressFromCounter_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferWriteAddressFromTimeCounter_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferWriteAddress_04_00(Apr::CommandBufferObject* buffer, uint64_t* address, uint64_t value, uint32_t flags) { + Apr::WriteAddressCommand command{}; + command.header = {Apr::Opcode::WriteAddress, sizeof(command)}; + command.address = reinterpret_cast(address); + command.value = value; + command.flags = flags; + return Append(buffer, &command, sizeof(command)); +} + +int APS5_VABI sceAmprCommandBufferWriteCounter_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprCommandBufferWriteKernelEventQueue_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeMapBegin() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeMapDirectBegin() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeMapEnd() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeNop() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeNopWithData() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizePopMarker() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizePushMarker() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizePushMarkerWithColor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +uint32_t APS5_VABI sceAmprMeasureCommandSizeReadFile(void) { + return sizeof(Apr::ReadFileCommand); +} + +int APS5_VABI sceAmprMeasureCommandSizeReadFileGather() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeReadFileGatherScatter() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeReadFileScatter() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeResetGatherScatterState() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeSetMarker() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeSetMarkerWithColor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeWaitOnAddress_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeWaitOnCounter_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeWriteAddressFromCounterPair_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeWriteAddressFromCounter_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeWriteAddressFromTimeCounter_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +uint32_t APS5_VABI sceAmprMeasureCommandSizeWriteAddress_04_00(void) { + return sizeof(Apr::WriteAddressCommand); +} + +int APS5_VABI sceAmprMeasureCommandSizeWriteCounter_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAmprMeasureCommandSizeWriteKernelEventQueue_04_00() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceAppContent/Export.cpp b/core/libs/prx/libSceAppContent/Export.cpp index a82776a715..9469c86d6e 100644 --- a/core/libs/prx/libSceAppContent/Export.cpp +++ b/core/libs/prx/libSceAppContent/Export.cpp @@ -2,6 +2,11 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +static constexpr int SCE_APP_CONTENT_ERROR_PARAMETER = static_cast(0x80D90002); +static constexpr uint32_t APPPARAM_ID_SKU_FLAG = 1; +static constexpr int32_t SKU_FLAG_FULL = 3; extern "C" { @@ -20,10 +25,17 @@ int APS5_VABI sceAppContentAddcontUnmount(const AppContentMountPoint* mount_poin } int APS5_VABI sceAppContentAppParamGetInt(uint32_t param_id, int32_t* value) { - (void)param_id; - (void)value; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!value) return SCE_APP_CONTENT_ERROR_PARAMETER; + switch (param_id) { + case APPPARAM_ID_SKU_FLAG: + *value = SKU_FLAG_FULL; + return 0; + case 2: case 3: case 4: case 5: + *value = 0; + return 0; + default: + return SCE_APP_CONTENT_ERROR_PARAMETER; + } } int APS5_VABI sceAppContentDownloadDataGetAvailableSpaceKb(const AppContentMountPoint* mount_point, size_t* available_space_kb) { @@ -34,10 +46,10 @@ int APS5_VABI sceAppContentDownloadDataGetAvailableSpaceKb(const AppContentMount } int APS5_VABI sceAppContentInitialize(const AppContentInitParam* init_param, AppContentBootParam* boot_param) { - (void)init_param; - (void)boot_param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)init_param; + if (!boot_param) return SCE_APP_CONTENT_ERROR_PARAMETER; + std::memset(boot_param, 0, sizeof(*boot_param)); + return 0; } int APS5_VABI sceAppContentTemporaryDataFormat(const AppContentMountPoint* mount_point) { diff --git a/core/libs/prx/libSceCesCs/Export.cpp b/core/libs/prx/libSceCesCs/Export.cpp index 74c0388f7a..aaa150eefb 100644 --- a/core/libs/prx/libSceCesCs/Export.cpp +++ b/core/libs/prx/libSceCesCs/Export.cpp @@ -5,7 +5,7 @@ extern "C" { -const uint8_t* sceCesRefersUcsProfileCp1252(void) { +const uint8_t* APS5_VABI sceCesRefersUcsProfileCp1252(void) { NotImplemented_nid_no_patch(__func__); return nullptr; } diff --git a/core/libs/prx/libSceFiber/Export.cpp b/core/libs/prx/libSceFiber/Export.cpp index 4e805a4937..b24a8ea920 100644 --- a/core/libs/prx/libSceFiber/Export.cpp +++ b/core/libs/prx/libSceFiber/Export.cpp @@ -2,79 +2,398 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +static constexpr int32_t SCE_OK = 0; +static constexpr int32_t SCE_FIBER_ERROR_NULL = static_cast(0x80590001); +static constexpr int32_t SCE_FIBER_ERROR_ALIGNMENT = static_cast(0x80590002); +static constexpr int32_t SCE_FIBER_ERROR_RANGE = static_cast(0x80590003); +static constexpr int32_t SCE_FIBER_ERROR_INVALID = static_cast(0x80590004); +static constexpr int32_t SCE_FIBER_ERROR_PERMISSION = static_cast(0x80590005); +static constexpr int32_t SCE_FIBER_ERROR_STATE = static_cast(0x80590006); + +static constexpr std::size_t FIBER_OBJECT_SIZE = 0x100; +static constexpr std::size_t FIBER_OPT_PARAM_SIZE = 0x80; +static constexpr std::size_t FIBER_MIN_CONTEXT_SIZE = 512; + +#ifdef _WIN32 + +using GuestFiberEntry = void (APS5_VABI*)(std::uint64_t argOnInitialize, std::uint64_t argOnRun); + +// A fiber that is switching away stays Suspending until its context is saved; whoever runs next +// on that host thread publishes Suspended. Fibers migrate between threads, so a resumer must never +// see Suspended before the saved stack pointer is valid. +enum class FiberState : std::uint32_t { + Idle = 1, + Running = 2, + Suspending = 3, + Suspended = 4, +}; + +struct Fiber { + std::uint64_t magic; + std::atomic state; + std::uint32_t reserved; + GuestFiberEntry entry; + std::uint64_t argOnInitialize; + std::uint8_t* context; + std::uint64_t contextSize; + void* savedStack; + char name[FIBER_MAX_NAME_LENGTH + 1]; +}; +static_assert(sizeof(Fiber) <= FIBER_OBJECT_SIZE, "guest reserves 0x100 bytes for SceFiber"); + +static constexpr std::uint64_t FIBER_MAGIC = 0x5245424946355041ull; + +struct StackBounds { + void* base; + void* limit; + void* deallocation; +}; + +struct ThreadFiberState { + Fiber* current = nullptr; + void* threadStack = nullptr; + StackBounds threadBounds{}; + std::uint64_t transfer = 0; + Fiber* pendingSuspend = nullptr; +}; + +static thread_local ThreadFiberState g_thread; + +// A fiber may resume on a different host thread than the one that suspended it, so the +// thread-local state must be looked up again after every stack switch rather than cached. +__attribute__((noinline)) static ThreadFiberState& ThreadState() { + asm volatile("" ::: "memory"); + return g_thread; +} + +static bool TraceFibers() { + static const bool enabled = std::getenv("APS5_TRACE_FIBER") != nullptr; + return enabled; +} + +static void CompletePendingSuspend() { + auto& thread = ThreadState(); + if (thread.pendingSuspend) { + thread.pendingSuspend->state.store(FiberState::Suspended, std::memory_order_release); + thread.pendingSuspend = nullptr; + } +} + +extern "C" void Aps5FiberSwitchStack(void** save, void* load); +extern "C" void Aps5FiberTrampoline(); + +asm(R"( + .text + .globl Aps5FiberSwitchStack + .def Aps5FiberSwitchStack; .scl 2; .type 32; .endef +Aps5FiberSwitchStack: + push %rbp + push %rbx + push %rdi + push %rsi + push %r12 + push %r13 + push %r14 + push %r15 + sub $0xa8, %rsp + movaps %xmm6, 0x00(%rsp) + movaps %xmm7, 0x10(%rsp) + movaps %xmm8, 0x20(%rsp) + movaps %xmm9, 0x30(%rsp) + movaps %xmm10, 0x40(%rsp) + movaps %xmm11, 0x50(%rsp) + movaps %xmm12, 0x60(%rsp) + movaps %xmm13, 0x70(%rsp) + movaps %xmm14, 0x80(%rsp) + movaps %xmm15, 0x90(%rsp) + stmxcsr 0xa0(%rsp) + fnstcw 0xa4(%rsp) + mov %rsp, (%rcx) + mov %rdx, %rsp + movaps 0x00(%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + ldmxcsr 0xa0(%rsp) + fldcw 0xa4(%rsp) + add $0xa8, %rsp + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rsi + pop %rdi + pop %rbx + pop %rbp + ret + + .globl Aps5FiberTrampoline + .def Aps5FiberTrampoline; .scl 2; .type 32; .endef +Aps5FiberTrampoline: + mov %r12, %rcx + and $-16, %rsp + sub $32, %rsp + call Aps5FiberMain + ud2 +)"); + +struct InitialFrame { + std::uint8_t xmm[0xa0]; + std::uint32_t mxcsr; + std::uint16_t fpuControl; + std::uint16_t padding; + std::uint64_t r15, r14, r13, r12, rsi, rdi, rbx, rbp; + std::uint64_t returnAddress; +}; +static_assert(sizeof(InitialFrame) == 0xa8 + 8 * 8 + 8, "initial fiber frame must match Aps5FiberSwitchStack"); + +static StackBounds CurrentBounds() { + auto* tib = reinterpret_cast(NtCurrentTeb()); + auto* teb = reinterpret_cast(tib); + return {tib->StackBase, tib->StackLimit, *reinterpret_cast(teb + 0x1478)}; +} + +static void SetBounds(const StackBounds& bounds) { + auto* tib = reinterpret_cast(NtCurrentTeb()); + auto* teb = reinterpret_cast(tib); + tib->StackBase = bounds.base; + tib->StackLimit = bounds.limit; + *reinterpret_cast(teb + 0x1478) = bounds.deallocation; +} + +static StackBounds FiberBounds(const Fiber* fiber) { + return {fiber->context + fiber->contextSize, fiber->context, fiber->context}; +} + +static Fiber* AsFiber(FiberObject* object) { + auto* fiber = reinterpret_cast(object); + return fiber && fiber->magic == FIBER_MAGIC ? fiber : nullptr; +} + +extern "C" [[noreturn]] void Aps5FiberMain(Fiber* fiber) { + CompletePendingSuspend(); + fiber->entry(fiber->argOnInitialize, ThreadState().transfer); + throw std::runtime_error(std::string("sceFiber: entry function of fiber '") + fiber->name + "' returned"); +} + +static void PrepareInitialStack(Fiber* fiber) { + const auto top = reinterpret_cast(fiber->context + fiber->contextSize) & ~static_cast(15); + auto* frame = reinterpret_cast(top - 256); + std::memset(frame, 0, sizeof(*frame)); + frame->mxcsr = _mm_getcsr(); + std::uint16_t control = 0; + asm volatile("fnstcw %0" : "=m"(control)); + frame->fpuControl = control; + frame->r12 = reinterpret_cast(fiber); + frame->returnAddress = reinterpret_cast(&Aps5FiberTrampoline); + fiber->savedStack = frame; +} + +static bool AcquireForResume(Fiber* target) { + for (;;) { + auto state = target->state.load(std::memory_order_acquire); + if (state == FiberState::Suspending) { + std::this_thread::yield(); + continue; + } + if (state != FiberState::Idle && state != FiberState::Suspended) return false; + if (target->state.compare_exchange_weak(state, FiberState::Running, std::memory_order_acq_rel)) { + if (state == FiberState::Idle) PrepareInitialStack(target); + return true; + } + } +} + +static void Resume(Fiber* target, void** save, std::uint64_t argOnRun) { + const auto* frame = static_cast(target->savedStack); + if (frame->returnAddress == 0) { + std::fprintf(stderr, "[fiber] resuming '%s' with a wiped context: saved=%p context=%p size=0x%llx\n", target->name, target->savedStack, + static_cast(target->context), static_cast(target->contextSize)); + const auto* words = static_cast(target->savedStack); + for (int i = 0; i < 40; i += 4) std::fprintf(stderr, "[fiber] +0x%03x %016llx %016llx %016llx %016llx\n", i * 8, static_cast(words[i]), static_cast(words[i + 1]), static_cast(words[i + 2]), static_cast(words[i + 3])); + std::fflush(stderr); + std::abort(); + } + ThreadState().current = target; + ThreadState().transfer = argOnRun; + SetBounds(FiberBounds(target)); + Aps5FiberSwitchStack(save, target->savedStack); +} extern "C" { -int32_t APS5_VABI sceFiberFinalize(FiberObject* fiber) { - (void)fiber; - NotImplemented_nid_no_patch(__func__); - return 0; +int32_t APS5_VABI _sceFiberInitializeImpl_nid_postfix(FiberObject* object, const char* name, GuestFiberEntry entry, uint64_t arg_on_initialize, void* addr_context, uint64_t size_context, const void* opt_param, uint32_t build_version) { + (void)opt_param; + (void)build_version; + if (!object || !name || !entry) return SCE_FIBER_ERROR_NULL; + if ((reinterpret_cast(object) & 7) != 0) return SCE_FIBER_ERROR_ALIGNMENT; + if (addr_context == nullptr && size_context != 0) return SCE_FIBER_ERROR_INVALID; + if ((reinterpret_cast(addr_context) & 15) != 0 || (size_context & 15) != 0) return SCE_FIBER_ERROR_ALIGNMENT; + if (addr_context == nullptr) return SCE_FIBER_ERROR_INVALID; + if (size_context < FIBER_MIN_CONTEXT_SIZE) return SCE_FIBER_ERROR_RANGE; + auto* fiber = reinterpret_cast(object); + std::memset(object, 0, FIBER_OBJECT_SIZE); + fiber->magic = FIBER_MAGIC; + fiber->state.store(FiberState::Idle, std::memory_order_relaxed); + fiber->entry = entry; + fiber->argOnInitialize = arg_on_initialize; + fiber->context = static_cast(addr_context); + fiber->contextSize = size_context; + std::strncpy(fiber->name, name, FIBER_MAX_NAME_LENGTH); + if (TraceFibers()) std::fprintf(stderr, "[fiber] init %s object=%p context=%p+0x%llx entry=%p\n", fiber->name, static_cast(object), addr_context, static_cast(size_context), reinterpret_cast(entry)); + return SCE_OK; } -int32_t APS5_VABI sceFiberGetInfo(FiberObject* fiber, FiberInfo* fiber_info) { - (void)fiber; - (void)fiber_info; - NotImplemented_nid_no_patch(__func__); - return 0; +int32_t APS5_VABI sceFiberFinalize(FiberObject* object) { + auto* fiber = AsFiber(object); + if (!fiber) return object ? SCE_FIBER_ERROR_INVALID : SCE_FIBER_ERROR_NULL; + const auto state = fiber->state.load(std::memory_order_acquire); + if (state == FiberState::Running || state == FiberState::Suspending) return SCE_FIBER_ERROR_STATE; + fiber->magic = 0; + return SCE_OK; } -int32_t APS5_VABI sceFiberGetSelf(FiberObject** fiber) { - (void)fiber; - NotImplemented_nid_no_patch(__func__); - return 0; +int32_t APS5_VABI sceFiberRun_nid_postfix(FiberObject* object, uint64_t arg_on_run, uint64_t* arg_on_return) { + auto* fiber = AsFiber(object); + if (!fiber) return object ? SCE_FIBER_ERROR_INVALID : SCE_FIBER_ERROR_NULL; + if (ThreadState().current) return SCE_FIBER_ERROR_PERMISSION; + if (!AcquireForResume(fiber)) return SCE_FIBER_ERROR_STATE; + ThreadState().threadBounds = CurrentBounds(); + Resume(fiber, &ThreadState().threadStack, arg_on_run); + CompletePendingSuspend(); + SetBounds(ThreadState().threadBounds); + if (arg_on_return) *arg_on_return = ThreadState().transfer; + return SCE_OK; } -int32_t APS5_VABI sceFiberGetThreadFramePointerAddress(uint64_t* addr_frame_pointer) { - (void)addr_frame_pointer; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int32_t APS5_VABI sceFiberOptParamInitialize(FiberOptParam* opt_param) { - (void)opt_param; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int32_t APS5_VABI sceFiberRename(FiberObject* fiber, const char* name) { - (void)fiber; - (void)name; - NotImplemented_nid_no_patch(__func__); - return 0; +int32_t APS5_VABI sceFiberSwitch(FiberObject* object, uint64_t arg_on_run, uint64_t* arg_on_run_out) { + auto* target = AsFiber(object); + if (!target) return object ? SCE_FIBER_ERROR_INVALID : SCE_FIBER_ERROR_NULL; + auto* self = ThreadState().current; + if (!self) return SCE_FIBER_ERROR_PERMISSION; + if (target == self || !AcquireForResume(target)) return SCE_FIBER_ERROR_STATE; + if (TraceFibers()) { + auto** frame = static_cast(__builtin_frame_address(0)); + void* chain[6] = {}; + auto** guest = static_cast(frame[0]); + for (int depth = 0; depth < 6 && guest; ++depth) { + chain[depth] = guest[1]; + guest = static_cast(guest[0]); + } + std::fprintf(stderr, "[fiber] switch %s -> %s from %p %p %p %p %p %p\n", self->name, target->name, chain[0], chain[1], chain[2], chain[3], chain[4], chain[5]); + } + self->state.store(FiberState::Suspending, std::memory_order_relaxed); + ThreadState().pendingSuspend = self; + Resume(target, &self->savedStack, arg_on_run); + CompletePendingSuspend(); + if (arg_on_run_out) *arg_on_run_out = ThreadState().transfer; + return SCE_OK; } int32_t APS5_VABI sceFiberReturnToThread(uint64_t arg_on_return, uint64_t* arg_on_run) { - (void)arg_on_return; - (void)arg_on_run; - NotImplemented_nid_no_patch(__func__); - return 0; + auto* self = ThreadState().current; + if (!self) return SCE_FIBER_ERROR_PERMISSION; + if (TraceFibers()) std::fprintf(stderr, "[fiber] return %s from %p\n", self->name, __builtin_return_address(0)); + self->state.store(FiberState::Suspending, std::memory_order_relaxed); + ThreadState().pendingSuspend = self; + ThreadState().current = nullptr; + ThreadState().transfer = arg_on_return; + SetBounds(ThreadState().threadBounds); + Aps5FiberSwitchStack(&self->savedStack, ThreadState().threadStack); + CompletePendingSuspend(); + if (arg_on_run) *arg_on_run = ThreadState().transfer; + return SCE_OK; } -int32_t APS5_VABI sceFiberRun_nid_postfix(FiberObject* fiber, uint64_t arg_on_run, uint64_t* arg_on_return) { - (void)fiber; - (void)arg_on_run; - (void)arg_on_return; - NotImplemented_nid_no_patch(__func__); - return 0; +int32_t APS5_VABI sceFiberGetSelf(FiberObject** fiber) { + if (!fiber) return SCE_FIBER_ERROR_NULL; + if (!ThreadState().current) return SCE_FIBER_ERROR_PERMISSION; + *fiber = reinterpret_cast(ThreadState().current); + return SCE_OK; +} + +int32_t APS5_VABI sceFiberGetInfo(FiberObject* object, FiberInfo* fiber_info) { + auto* fiber = AsFiber(object); + if (!fiber || !fiber_info) return object && fiber_info ? SCE_FIBER_ERROR_INVALID : SCE_FIBER_ERROR_NULL; + if (fiber_info->size != sizeof(FiberInfo)) return SCE_FIBER_ERROR_INVALID; + fiber_info->entry = reinterpret_cast(fiber->entry); + fiber_info->arg_on_initialize = fiber->argOnInitialize; + fiber_info->addr_context = fiber->context; + fiber_info->size_context = fiber->contextSize; + std::memcpy(fiber_info->name, fiber->name, sizeof(fiber_info->name)); + fiber_info->size_context_margin = static_cast(-1); + return SCE_OK; +} + +int32_t APS5_VABI sceFiberRename(FiberObject* object, const char* name) { + auto* fiber = AsFiber(object); + if (!fiber || !name) return object && name ? SCE_FIBER_ERROR_INVALID : SCE_FIBER_ERROR_NULL; + std::memset(fiber->name, 0, sizeof(fiber->name)); + std::strncpy(fiber->name, name, FIBER_MAX_NAME_LENGTH); + return SCE_OK; +} + +int32_t APS5_VABI sceFiberOptParamInitialize(FiberOptParam* opt_param) { + if (!opt_param) return SCE_FIBER_ERROR_NULL; + std::memset(opt_param, 0, FIBER_OPT_PARAM_SIZE); + return SCE_OK; +} + +int32_t APS5_VABI sceFiberGetThreadFramePointerAddress(uint64_t* addr_frame_pointer) { + (void)addr_frame_pointer; + NotImplemented_nid_no_patch(__func__); + return 0; } int32_t APS5_VABI sceFiberStartContextSizeCheck(uint32_t flags) { - (void)flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)flags; + NotImplemented_nid_no_patch(__func__); + return 0; } int32_t APS5_VABI sceFiberStopContextSizeCheck(void) { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int32_t APS5_VABI sceFiberSwitch(FiberObject* fiber, uint64_t arg_on_run, uint64_t* arg_on_return) { - (void)fiber; - (void)arg_on_run; - (void)arg_on_return; - NotImplemented_nid_no_patch(__func__); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } } + +#else + +extern "C" { + +int32_t APS5_VABI _sceFiberInitializeImpl_nid_postfix() { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberFinalize(FiberObject*) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberRun_nid_postfix(FiberObject*, uint64_t, uint64_t*) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberSwitch(FiberObject*, uint64_t, uint64_t*) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberReturnToThread(uint64_t, uint64_t*) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberGetSelf(FiberObject**) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberGetInfo(FiberObject*, FiberInfo*) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberRename(FiberObject*, const char*) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberOptParamInitialize(FiberOptParam*) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberGetThreadFramePointerAddress(uint64_t*) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberStartContextSizeCheck(uint32_t) { NotImplemented_nid_no_patch(__func__); return 0; } +int32_t APS5_VABI sceFiberStopContextSizeCheck(void) { NotImplemented_nid_no_patch(__func__); return 0; } + +} + +#endif diff --git a/core/libs/prx/libSceFont.native/Export.cpp b/core/libs/prx/libSceFont.native/Export.cpp index 87b9d24751..33f3d0fa40 100644 --- a/core/libs/prx/libSceFont.native/Export.cpp +++ b/core/libs/prx/libSceFont.native/Export.cpp @@ -497,7 +497,7 @@ int APS5_VABI sceFontWritingLineWritesOrder(FontWritingLine writing_line, uint64 return 0; } -const FontWritingStep* sceFontWritingRefersRenderStep(FontWriting* font_writing) { +const FontWritingStep* APS5_VABI sceFontWritingRefersRenderStep(FontWriting* font_writing) { (void)font_writing; NotImplemented_nid_no_patch(__func__); return nullptr; diff --git a/core/libs/prx/libSceFont/Export.cpp b/core/libs/prx/libSceFont/Export.cpp index 87b9d24751..33f3d0fa40 100644 --- a/core/libs/prx/libSceFont/Export.cpp +++ b/core/libs/prx/libSceFont/Export.cpp @@ -497,7 +497,7 @@ int APS5_VABI sceFontWritingLineWritesOrder(FontWritingLine writing_line, uint64 return 0; } -const FontWritingStep* sceFontWritingRefersRenderStep(FontWriting* font_writing) { +const FontWritingStep* APS5_VABI sceFontWritingRefersRenderStep(FontWriting* font_writing) { (void)font_writing; NotImplemented_nid_no_patch(__func__); return nullptr; diff --git a/core/libs/prx/libSceGameUpdate/Export.cpp b/core/libs/prx/libSceGameUpdate/Export.cpp index 937cda4b62..bc9cbef05c 100644 --- a/core/libs/prx/libSceGameUpdate/Export.cpp +++ b/core/libs/prx/libSceGameUpdate/Export.cpp @@ -3,31 +3,30 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +// Update checks need the network, which is not available; requests fail so the game skips the check. +static constexpr int SCE_GAME_UPDATE_ERROR_OFFLINE = static_cast(0x80D50001); + extern "C" { int APS5_VABI sceGameUpdateAbortRequest(int request_id) { - (void)request_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + return 0; } int APS5_VABI sceGameUpdateCheck(int request_id, const GameUpdateCheckParam* param, GameUpdateCheckResult* result) { - (void)request_id; - (void)param; - (void)result; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)param; + (void)result; + return SCE_GAME_UPDATE_ERROR_OFFLINE; } int APS5_VABI sceGameUpdateCreateRequest(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return SCE_GAME_UPDATE_ERROR_OFFLINE; } int APS5_VABI sceGameUpdateDeleteRequest(int request_id) { - (void)request_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + return 0; } int APS5_VABI sceGameUpdateGetAddcontLatestVersion(uint32_t service_label, const void* entitlement_label, GameUpdateAddcontVersionInfo* info) { @@ -39,13 +38,11 @@ int APS5_VABI sceGameUpdateGetAddcontLatestVersion(uint32_t service_label, const } int APS5_VABI sceGameUpdateInitialize(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } int APS5_VABI sceGameUpdateTerminate(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } } diff --git a/core/libs/prx/libSceHttp/Export.cpp b/core/libs/prx/libSceHttp/Export.cpp index 882ca8b4eb..d3292924eb 100644 --- a/core/libs/prx/libSceHttp/Export.cpp +++ b/core/libs/prx/libSceHttp/Export.cpp @@ -2,40 +2,42 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +// No network is emulated: contexts, templates and requests can be created, but any request +// that would touch the network fails with the library's network error. +static constexpr int ERROR_NETWORK = static_cast(0x80431063); +static std::atomic g_nextHandle{1}; extern "C" { int APS5_VABI sceHttpAbortRequest(int request_id) { - (void)request_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + return 0; } int APS5_VABI sceHttpAddRequestHeader(int id, const char* name, const char* value, uint32_t mode) { - (void)id; - (void)name; - (void)value; - (void)mode; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)name; + (void)value; + (void)mode; + return 0; } int APS5_VABI sceHttpCreateConnection(int tmpl_id, const char* server_name, const char* scheme, uint16_t port, int enable_keep_alive) { - (void)tmpl_id; - (void)server_name; - (void)scheme; - (void)port; - (void)enable_keep_alive; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)tmpl_id; + (void)server_name; + (void)scheme; + (void)port; + (void)enable_keep_alive; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttpCreateConnectionWithURL(int tmpl_id, const char* url, int enable_keep_alive) { - (void)tmpl_id; - (void)url; - (void)enable_keep_alive; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)tmpl_id; + (void)url; + (void)enable_keep_alive; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttpCreateEpoll(int http_ctx_id, HttpEpollHandle* eh) { @@ -46,199 +48,172 @@ int APS5_VABI sceHttpCreateEpoll(int http_ctx_id, HttpEpollHandle* eh) { } int APS5_VABI sceHttpCreateRequest(int conn_id, int method, const char* path, uint64_t content_length) { - (void)conn_id; - (void)method; - (void)path; - (void)content_length; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)conn_id; + (void)method; + (void)path; + (void)content_length; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttpCreateRequestWithURL2(int conn_id, const char* method, const char* url, uint64_t content_length) { - (void)conn_id; - (void)method; - (void)url; - (void)content_length; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)conn_id; + (void)method; + (void)url; + (void)content_length; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttpCreateTemplate(int http_ctx_id, const char* user_agent, int http_ver, int is_auto_proxy_conf) { - (void)http_ctx_id; - (void)user_agent; - (void)http_ver; - (void)is_auto_proxy_conf; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)http_ctx_id; + (void)user_agent; + (void)http_ver; + (void)is_auto_proxy_conf; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttpDeleteConnection(int conn_id) { - (void)conn_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)conn_id; + return 0; } int APS5_VABI sceHttpDeleteRequest(int req_id) { - (void)req_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + return 0; } int APS5_VABI sceHttpDeleteTemplate(int tmpl_id) { - (void)tmpl_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)tmpl_id; + return 0; } int APS5_VABI sceHttpDestroyEpoll(int http_ctx_id, HttpEpollHandle eh) { - (void)http_ctx_id; - (void)eh; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)http_ctx_id; + (void)eh; + return 0; } int APS5_VABI sceHttpGetAllResponseHeaders(int request_id, char** header, size_t* header_size) { - (void)request_id; - (void)header; - (void)header_size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)header; + (void)header_size; + return ERROR_NETWORK; } int APS5_VABI sceHttpGetResponseContentLength(int request_id, int* result, uint64_t* content_length) { - (void)request_id; - (void)result; - (void)content_length; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)result; + (void)content_length; + return ERROR_NETWORK; } int APS5_VABI sceHttpGetStatusCode(int request_id, int* status_code) { - (void)request_id; - (void)status_code; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)status_code; + return ERROR_NETWORK; } int APS5_VABI sceHttpInit_nid_postfix(int memid, int ssl_ctx_id, uint64_t pool_size) { - (void)memid; - (void)ssl_ctx_id; - (void)pool_size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)memid; + (void)ssl_ctx_id; + (void)pool_size; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttpsDisableOption(int id, uint32_t ssl_flags) { - (void)id; - (void)ssl_flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)ssl_flags; + return 0; } int APS5_VABI sceHttpSendRequest(int request_id, const void* post_data, size_t size) { - (void)request_id; - (void)post_data; - (void)size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)post_data; + (void)size; + return ERROR_NETWORK; } int APS5_VABI sceHttpSetAuthEnabled(int id, int enable) { - (void)id; - (void)enable; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)enable; + return 0; } int APS5_VABI sceHttpSetAutoRedirect(int id, int enable) { - (void)id; - (void)enable; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)enable; + return 0; } int APS5_VABI sceHttpSetConnectTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttpSetEpoll(int id, HttpEpollHandle eh, void* user_arg) { - (void)id; - (void)eh; - (void)user_arg; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)eh; + (void)user_arg; + return 0; } int APS5_VABI sceHttpSetNonblock(int id, int enable) { - (void)id; - (void)enable; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)enable; + return 0; } int APS5_VABI sceHttpSetRecvTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttpSetRequestContentLength(int request_id, uint64_t content_length) { - (void)request_id; - (void)content_length; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)content_length; + return 0; } int APS5_VABI sceHttpSetResolveRetry(int id, int32_t retry) { - (void)id; - (void)retry; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)retry; + return 0; } int APS5_VABI sceHttpSetResolveTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttpSetSendTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttpsSetMinSslVersion(int id, uint32_t ssl_version) { - (void)id; - (void)ssl_version; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)ssl_version; + return 0; } int APS5_VABI sceHttpsSetSslCallback(int id, HttpsCallback cbfunc, void* user_arg) { - (void)id; - (void)cbfunc; - (void)user_arg; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)cbfunc; + (void)user_arg; + return 0; } int APS5_VABI sceHttpTerm_nid_postfix(int http_ctx_id) { - (void)http_ctx_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)http_ctx_id; + return 0; } int APS5_VABI sceHttpUnsetEpoll(int id) { - (void)id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + return 0; } int APS5_VABI sceHttpUriBuild(char* out, size_t* require, size_t prepare, const SceHttpUriElement* src_element, uint32_t option) { @@ -271,10 +246,24 @@ int APS5_VABI sceHttpUriParse(SceHttpUriElement* out, const char* src_url, void* } int APS5_VABI sceHttpWaitRequest(HttpEpollHandle eh, HttpNBEvent* nbev, int maxevents, int timeout) { - (void)eh; - (void)nbev; - (void)maxevents; - (void)timeout; + (void)eh; + (void)nbev; + (void)maxevents; + (void)timeout; + return ERROR_NETWORK; +} + +int APS5_VABI sceHttpCreateRequestWithURL() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceHttpReadData() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceHttpSetChunkedTransferEnabled() { NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libSceHttp2/Export.cpp b/core/libs/prx/libSceHttp2/Export.cpp index c1e890b024..95f07e60a1 100644 --- a/core/libs/prx/libSceHttp2/Export.cpp +++ b/core/libs/prx/libSceHttp2/Export.cpp @@ -2,228 +2,209 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +// No network is emulated: contexts, templates and requests can be created, but any request +// that would touch the network fails with the library's network error. +static constexpr int ERROR_NETWORK = static_cast(0x80436063); +static std::atomic g_nextHandle{1}; extern "C" { int APS5_VABI sceHttp2AddRequestHeader(int id, const char* name, const char* value, uint32_t mode) { - (void)id; - (void)name; - (void)value; - (void)mode; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)name; + (void)value; + (void)mode; + return 0; } int APS5_VABI sceHttp2CreateRequestWithURL(int tmpl_id, const char* method, const char* url, uint64_t content_length) { - (void)tmpl_id; - (void)method; - (void)url; - (void)content_length; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)tmpl_id; + (void)method; + (void)url; + (void)content_length; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttp2CreateTemplate(int lib_http2_ctx_id, const char* user_agent, int http_ver, int is_auto_proxy_conf) { - (void)lib_http2_ctx_id; - (void)user_agent; - (void)http_ver; - - (void)is_auto_proxy_conf; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)lib_http2_ctx_id; + (void)user_agent; + (void)http_ver; + (void)is_auto_proxy_conf; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttp2DeleteRequest(int req_id) { - (void)req_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + return 0; } int APS5_VABI sceHttp2DeleteTemplate(int tmpl_id) { - (void)tmpl_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)tmpl_id; + return 0; } int APS5_VABI sceHttp2GetAllResponseHeaders(int req_id, char** header, size_t* header_size) { - (void)req_id; - (void)header; - (void)header_size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + (void)header; + (void)header_size; + return ERROR_NETWORK; } int APS5_VABI sceHttp2GetResponseContentLength(int req_id, int* result, uint64_t* content_length) { - (void)req_id; - (void)result; - (void)content_length; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + (void)result; + (void)content_length; + return ERROR_NETWORK; } int APS5_VABI sceHttp2GetStatusCode(int req_id, int* status_code) { - (void)req_id; - (void)status_code; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + (void)status_code; + return ERROR_NETWORK; } int APS5_VABI sceHttp2Init(int libnet_mem_id, int libssl_ctx_id, size_t pool_size, int max_concurrent_request) { - (void)libnet_mem_id; - (void)libssl_ctx_id; - (void)pool_size; - (void)max_concurrent_request; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)libnet_mem_id; + (void)libssl_ctx_id; + (void)pool_size; + (void)max_concurrent_request; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceHttp2ReadData(int req_id, void* data, size_t size) { - (void)req_id; - (void)data; - (void)size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + (void)data; + (void)size; + return ERROR_NETWORK; } int APS5_VABI sceHttp2ReadDataAsync(int req_id, void* data, size_t size, void* kqueue_option, void* option) { - (void)req_id; - (void)data; - (void)size; - (void)kqueue_option; - (void)option; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + (void)data; + (void)size; + (void)kqueue_option; + (void)option; + return ERROR_NETWORK; } int APS5_VABI sceHttp2SendRequest(int req_id, const void* post_data, size_t size) { - (void)req_id; - (void)post_data; - (void)size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + (void)post_data; + (void)size; + return ERROR_NETWORK; } int APS5_VABI sceHttp2SendRequestAsync(int req_id, const void* post_data, size_t size, void* kqueue_option, void* option) { - (void)req_id; - (void)post_data; - (void)size; - (void)kqueue_option; - (void)option; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + (void)post_data; + (void)size; + (void)kqueue_option; + (void)option; + return ERROR_NETWORK; } int APS5_VABI sceHttp2SetAuthEnabled(int id, int is_enable) { - (void)id; - (void)is_enable; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)is_enable; + return 0; } int APS5_VABI sceHttp2SetAutoRedirect(int id, int enable) { - (void)id; - (void)enable; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)enable; + return 0; } int APS5_VABI sceHttp2SetConnectionWaitTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttp2SetConnectTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttp2SetInflateGZIPEnabled(int id, int enable) { - (void)id; - (void)enable; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)enable; + return 0; } int APS5_VABI sceHttp2SetRecvTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttp2SetRedirectCallback(int id, void* cb_func, void* user_arg) { - (void)id; - (void)cb_func; - (void)user_arg; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)cb_func; + (void)user_arg; + return 0; } int APS5_VABI sceHttp2SetRequestContentLength(int id, uint64_t content_length) { - (void)id; - (void)content_length; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)content_length; + return 0; } int APS5_VABI sceHttp2SetResolveTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttp2SetSendTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttp2SetSslCallback(int id, void* cb_func, void* user_arg) { - (void)id; - (void)cb_func; - (void)user_arg; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)cb_func; + (void)user_arg; + return 0; } int APS5_VABI sceHttp2SetTimeOut(int id, uint32_t usec) { - (void)id; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)usec; + return 0; } int APS5_VABI sceHttp2SslDisableOption(int id, uint32_t ssl_flags) { - (void)id; - (void)ssl_flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)ssl_flags; + return 0; } int APS5_VABI sceHttp2SslEnableOption(int id, uint32_t ssl_flags) { - (void)id; - (void)ssl_flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)id; + (void)ssl_flags; + return 0; } int APS5_VABI sceHttp2Term(int lib_http2_ctx_id) { - (void)lib_http2_ctx_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)lib_http2_ctx_id; + return 0; } int APS5_VABI sceHttp2WaitAsync(int req_id, Http2AsyncResult* result, uint32_t* timeout, void* option) { - (void)req_id; - (void)result; - (void)timeout; - (void)option; + (void)req_id; + (void)result; + (void)timeout; + (void)option; + return ERROR_NETWORK; +} + +int APS5_VABI sceHttp2AbortRequest() { NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libSceJpegEnc/Export.cpp b/core/libs/prx/libSceJpegEnc/Export.cpp new file mode 100644 index 0000000000..8f4b5a78fb --- /dev/null +++ b/core/libs/prx/libSceJpegEnc/Export.cpp @@ -0,0 +1,28 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI sceJpegEncCreate() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceJpegEncDelete() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceJpegEncEncode() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceJpegEncQueryMemorySize() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp new file mode 100644 index 0000000000..848de74a45 --- /dev/null +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -0,0 +1,37 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + + +extern "C" { + +// Initialization succeeds as on an offline console: the title tears its whole web API layer down on +// an init failure and later dereferences it anyway. Requests made through it fail as offline. +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* self, const void* parameter) { + (void)self; + (void)parameter; + return 0; +} + +int APS5_VABI _ZN3sce4Json11InitializerC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce4Json11InitializerD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce4Json12MemAllocatorC2Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void* self) { + (void)self; + return 0; +} + +} diff --git a/core/libs/prx/libSceMsgDialog.native/Export.cpp b/core/libs/prx/libSceMsgDialog.native/Export.cpp new file mode 100644 index 0000000000..dbe06e1d1b --- /dev/null +++ b/core/libs/prx/libSceMsgDialog.native/Export.cpp @@ -0,0 +1,10 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int SceMsgDialogNativeModuleLoaded_nid_no_patch = 1; + +} diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index ef59f2ce72..bb4b9bab3e 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -2,54 +2,112 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include +#include +#include +#include +#include +#include +#include + +// No network is emulated: pools, resolvers and epoll instances exist, but sockets cannot be +// created, so every socket operation fails with the guest's EBADF. +static constexpr int GUEST_EINTR = 4; +static constexpr int GUEST_EBADF = 9; +static constexpr int GUEST_EINVAL = 22; +static constexpr int GUEST_ENOSPC = 28; +static constexpr int GUEST_EAFNOSUPPORT = 47; +static constexpr int GUEST_ENETDOWN = 50; +static constexpr int GUEST_EHOSTUNREACH = 65; +static constexpr int GUEST_AF_INET = 2; + +static thread_local int g_netErrno = 0; +static std::atomic g_nextId{1}; + +struct Epoll { + std::mutex lock; + std::condition_variable changed; + bool aborted = false; +}; + +static std::mutex g_epollLock; +static std::map> g_epolls; + +static int Fail(int error) { + g_netErrno = error; + return static_cast(0x80410100u | static_cast(error)); +} + +static std::shared_ptr FindEpoll(int eid) { + std::lock_guard lock(g_epollLock); + const auto found = g_epolls.find(eid); + return found == g_epolls.end() ? nullptr : found->second; +} + +static void AbortEpoll(Epoll& epoll) { + { + std::lock_guard lock(epoll.lock); + epoll.aborted = true; + } + epoll.changed.notify_all(); +} extern "C" { int APS5_VABI sceNetAccept(int s, void* addr, uint32_t* addrlen) { - (void)s; - (void)addr; - (void)addrlen; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)s; + (void)addr; + (void)addrlen; + return Fail(GUEST_EBADF); } int APS5_VABI sceNetBind_nid_postfix(int s, const void* addr, uint32_t addrlen) { - (void)s; - (void)addr; - (void)addrlen; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)s; + (void)addr; + (void)addrlen; + return Fail(GUEST_EBADF); } int APS5_VABI sceNetEpollControl(int eid, int op, int id, const NetEpollEvent* event) { - (void)eid; - (void)op; - (void)id; - (void)event; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)op; + (void)id; + (void)event; + if (!FindEpoll(eid)) return Fail(GUEST_EBADF); + return Fail(GUEST_EBADF); } int APS5_VABI sceNetEpollCreate(const char* name, int flags) { - (void)name; - (void)flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)name; + (void)flags; + const int id = g_nextId.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(g_epollLock); + g_epolls[id] = std::make_shared(); + return id; } int APS5_VABI sceNetEpollDestroy(int eid) { - (void)eid; - NotImplemented_nid_no_patch(__func__); - return 0; + std::lock_guard lock(g_epollLock); + const auto found = g_epolls.find(eid); + if (found == g_epolls.end()) return Fail(GUEST_EBADF); + AbortEpoll(*found->second); + g_epolls.erase(found); + return 0; } int APS5_VABI sceNetEpollWait(int eid, NetEpollEvent* events, int maxevents, int timeout) { - (void)eid; - (void)events; - (void)maxevents; - (void)timeout; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)events; + (void)maxevents; + auto epoll = FindEpoll(eid); + if (!epoll) return Fail(GUEST_EBADF); + std::unique_lock lock(epoll->lock); + const auto aborted = [&] { return epoll->aborted; }; + if (timeout < 0) epoll->changed.wait(lock, aborted); + else epoll->changed.wait_for(lock, std::chrono::microseconds(timeout), aborted); + if (epoll->aborted) { + epoll->aborted = false; + return Fail(GUEST_EINTR); + } + return 0; } int APS5_VABI sceNetEtherNtostr(const NetEtherAddr* n, char* str, size_t len) { @@ -68,138 +126,214 @@ int APS5_VABI sceNetGetMacAddress(NetEtherAddr* addr, int flags) { } int APS5_VABI sceNetGetSockInfo(int s, void* info, int n, int flags) { - (void)s; - (void)info; - (void)n; - (void)flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)s; + (void)info; + (void)n; + (void)flags; + return Fail(GUEST_EBADF); } int APS5_VABI sceNetGetsockname(int s, void* addr, uint32_t* addrlen) { - (void)s; - (void)addr; - (void)addrlen; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)s; + (void)addr; + (void)addrlen; + return Fail(GUEST_EBADF); } uint32_t APS5_VABI sceNetHtonl_nid_postfix(uint32_t host32) { - (void)host32; - NotImplemented_nid_no_patch(__func__); - return 0; + return __builtin_bswap32(host32); } uint16_t APS5_VABI sceNetHtons_nid_postfix(uint16_t host16) { - (void)host16; - NotImplemented_nid_no_patch(__func__); - return 0; + return __builtin_bswap16(host16); } -const char* sceNetInetNtop(int af, const void* src, char* dst, uint32_t size) { - (void)af; - (void)src; - (void)dst; - (void)size; - NotImplemented_nid_no_patch(__func__); - return nullptr; +const char* APS5_VABI sceNetInetNtop(int af, const void* src, char* dst, uint32_t size) { + if (af != GUEST_AF_INET || !src || !dst) { + Fail(GUEST_EAFNOSUPPORT); + return nullptr; + } + const auto* bytes = static_cast(src); + if (std::snprintf(dst, size, "%u.%u.%u.%u", bytes[0], bytes[1], bytes[2], bytes[3]) >= static_cast(size)) { + Fail(GUEST_ENOSPC); + return nullptr; + } + return dst; } int APS5_VABI sceNetInetPton(int af, const char* src, void* dst) { - (void)af; - (void)src; - (void)dst; - NotImplemented_nid_no_patch(__func__); - return 0; + if (af != GUEST_AF_INET) return Fail(GUEST_EAFNOSUPPORT); + if (!src || !dst) return Fail(GUEST_EINVAL); + unsigned parts[4]; + char trailing; + if (std::sscanf(src, "%u.%u.%u.%u%c", &parts[0], &parts[1], &parts[2], &parts[3], &trailing) != 4) return 0; + auto* bytes = static_cast(dst); + for (int index = 0; index < 4; ++index) { + if (parts[index] > 255) return 0; + bytes[index] = static_cast(parts[index]); + } + return 1; } int APS5_VABI sceNetInit_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } int APS5_VABI sceNetListen(int s, int backlog) { - (void)s; - (void)backlog; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)s; + (void)backlog; + return Fail(GUEST_EBADF); } uint32_t APS5_VABI sceNetNtohl_nid_postfix(uint32_t net32) { - (void)net32; - NotImplemented_nid_no_patch(__func__); - return 0; + return __builtin_bswap32(net32); } uint16_t APS5_VABI sceNetNtohs_nid_postfix(uint16_t net16) { - (void)net16; - NotImplemented_nid_no_patch(__func__); - return 0; + return __builtin_bswap16(net16); } int APS5_VABI sceNetPoolCreate(const char* name, int size, int flags) { - (void)name; - (void)size; - (void)flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)name; + (void)flags; + if (size <= 0) return Fail(GUEST_EINVAL); + return g_nextId.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNetPoolDestroy(int memid) { - (void)memid; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)memid; + return 0; } int APS5_VABI sceNetResolverCreate(const char* name, int memid, int flags) { - (void)name; - (void)memid; - (void)flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)name; + (void)memid; + (void)flags; + return g_nextId.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNetResolverStartNtoa(int rid, const char* hostname, void* addr, int timeout, int retry, int flags) { - (void)rid; - (void)hostname; - (void)addr; - (void)timeout; - (void)retry; - (void)flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)rid; + (void)hostname; + (void)addr; + (void)timeout; + (void)retry; + (void)flags; + return Fail(GUEST_EHOSTUNREACH); } int APS5_VABI sceNetSetsockopt(int s, int level, int optname, const void* optval, uint32_t optlen) { - (void)s; - (void)level; - (void)optname; - (void)optval; - (void)optlen; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)s; + (void)level; + (void)optname; + (void)optval; + (void)optlen; + return Fail(GUEST_EBADF); } int APS5_VABI sceNetShutdown(int s, int how) { - (void)s; - (void)how; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)s; + (void)how; + return Fail(GUEST_EBADF); } int APS5_VABI sceNetSocket(const char* name, int family, int type, int protocol) { - (void)name; - (void)family; - (void)type; - (void)protocol; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)name; + (void)family; + (void)type; + (void)protocol; + return Fail(GUEST_ENETDOWN); } int APS5_VABI sceNetSocketClose(int s) { - (void)s; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)s; + return Fail(GUEST_EBADF); +} + +int APS5_VABI sceNetConnect(int s, const void* addr, uint32_t addrlen) { + (void)s; + (void)addr; + (void)addrlen; + return Fail(GUEST_EBADF); +} + +int APS5_VABI sceNetEpollAbort(int eid, int flags) { + (void)flags; + auto epoll = FindEpoll(eid); + if (!epoll) return Fail(GUEST_EBADF); + AbortEpoll(*epoll); + return 0; +} + +int* APS5_VABI sceNetErrnoLoc(void) { + return &g_netErrno; +} + +int APS5_VABI sceNetGetsockopt(int s, int level, int optname, void* optval, uint32_t* optlen) { + (void)s; + (void)level; + (void)optname; + (void)optval; + (void)optlen; + return Fail(GUEST_EBADF); +} + +int APS5_VABI sceNetRecv(int s, void* buf, size_t len, int flags) { + (void)s; + (void)buf; + (void)len; + (void)flags; + return Fail(GUEST_EBADF); +} + +int APS5_VABI sceNetRecvfrom(int s, void* buf, size_t len, int flags, void* from, uint32_t* fromlen) { + (void)s; + (void)buf; + (void)len; + (void)flags; + (void)from; + (void)fromlen; + return Fail(GUEST_EBADF); +} + +int APS5_VABI sceNetResolverDestroy(int rid) { + (void)rid; + return 0; +} + +int APS5_VABI sceNetResolverStartAton(int rid, const void* addr, char* hostname, int len, int timeout, int retry, int flags) { + (void)rid; + (void)addr; + (void)hostname; + (void)len; + (void)timeout; + (void)retry; + (void)flags; + return Fail(GUEST_EHOSTUNREACH); +} + +int APS5_VABI sceNetSend(int s, const void* msg, size_t len, int flags) { + (void)s; + (void)msg; + (void)len; + (void)flags; + return Fail(GUEST_EBADF); +} + +int APS5_VABI sceNetSendto(int s, const void* msg, size_t len, int flags, const void* to, uint32_t tolen) { + (void)s; + (void)msg; + (void)len; + (void)flags; + (void)to; + (void)tolen; + return Fail(GUEST_EBADF); +} + +int APS5_VABI sceNetSocketAbort(int s, int flags) { + (void)s; + (void)flags; + return Fail(GUEST_EBADF); } } diff --git a/core/libs/prx/libSceNetCtl/Export.cpp b/core/libs/prx/libSceNetCtl/Export.cpp index 3cd6cb69ac..1f7af1cd0a 100644 --- a/core/libs/prx/libSceNetCtl/Export.cpp +++ b/core/libs/prx/libSceNetCtl/Export.cpp @@ -2,67 +2,80 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +// Network control reports an initialized but disconnected interface: no network is emulated. +static constexpr int SCE_NET_CTL_ERROR_CALLBACK_MAX = static_cast(0x80412103); +static constexpr int SCE_NET_CTL_ERROR_INVALID_ID = static_cast(0x80412105); +static constexpr int SCE_NET_CTL_ERROR_INVALID_ADDR = static_cast(0x80412107); +static constexpr int SCE_NET_CTL_ERROR_NOT_CONNECTED = static_cast(0x80412108); +static constexpr int NET_CTL_STATE_DISCONNECTED = 0; +static constexpr int MAX_CALLBACKS = 8; + +static std::mutex g_callbackLock; +static NetCtlCallback g_callbacks[MAX_CALLBACKS] = {}; extern "C" { int APS5_VABI sceNetCtlCheckCallback(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } int APS5_VABI sceNetCtlGetInfo(int code, NetCtlInfo* info) { - (void)code; - (void)info; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)code; + (void)info; + return SCE_NET_CTL_ERROR_NOT_CONNECTED; } int APS5_VABI sceNetCtlGetNatInfo(NetCtlNatInfo* nat_info) { - (void)nat_info; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)nat_info; + return SCE_NET_CTL_ERROR_NOT_CONNECTED; } int APS5_VABI sceNetCtlGetResult(int event_type, int* error_code) { - (void)event_type; - (void)error_code; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)event_type; + if (!error_code) return SCE_NET_CTL_ERROR_INVALID_ADDR; + *error_code = 0; + return 0; } int APS5_VABI sceNetCtlGetState(int* state) { - (void)state; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!state) return SCE_NET_CTL_ERROR_INVALID_ADDR; + *state = NET_CTL_STATE_DISCONNECTED; + return 0; } int APS5_VABI sceNetCtlGetStateV6(int* state) { - (void)state; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!state) return SCE_NET_CTL_ERROR_INVALID_ADDR; + *state = NET_CTL_STATE_DISCONNECTED; + return 0; } int APS5_VABI sceNetCtlInit(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } int APS5_VABI sceNetCtlRegisterCallback(NetCtlCallback func, void* arg, int* cid) { - (void)func; - (void)arg; - (void)cid; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!func || !cid) return SCE_NET_CTL_ERROR_INVALID_ADDR; + std::lock_guard lock(g_callbackLock); + for (int index = 0; index < MAX_CALLBACKS; ++index) { + if (!g_callbacks[index]) { + g_callbacks[index] = func; + *cid = index; + return 0; + } + } + return SCE_NET_CTL_ERROR_CALLBACK_MAX; } void APS5_VABI sceNetCtlTerm(void) { - NotImplemented_nid_no_patch(__func__); } int APS5_VABI sceNetCtlUnregisterCallback(int cid) { - (void)cid; - NotImplemented_nid_no_patch(__func__); - return 0; + if (cid < 0 || cid >= MAX_CALLBACKS) return SCE_NET_CTL_ERROR_INVALID_ID; + std::lock_guard lock(g_callbackLock); + g_callbacks[cid] = nullptr; + return 0; } } diff --git a/core/libs/prx/libSceNpCommerce/Export.cpp b/core/libs/prx/libSceNpCommerce/Export.cpp index e7bdccd9b2..e56b3e8598 100644 --- a/core/libs/prx/libSceNpCommerce/Export.cpp +++ b/core/libs/prx/libSceNpCommerce/Export.cpp @@ -1,12 +1,67 @@ #include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +constexpr int COMMON_DIALOG_STATUS_NONE = 0; +constexpr int COMMON_DIALOG_STATUS_INITIALIZED = 1; +constexpr int COMMON_DIALOG_STATUS_FINISHED = 3; +constexpr int COMMON_DIALOG_RESULT_USER_CANCELED = 1; +constexpr int COMMON_DIALOG_ERROR_NOT_INITIALIZED = static_cast(0x80B80003u); +constexpr int COMMON_DIALOG_ERROR_ALREADY_INITIALIZED = static_cast(0x80B80004u); +constexpr int COMMON_DIALOG_ERROR_NOT_FINISHED = static_cast(0x80B80006u); +constexpr int COMMON_DIALOG_ERROR_ARG_NULL = static_cast(0x80B80009u); + +int g_status = COMMON_DIALOG_STATUS_NONE; + +} + extern "C" { +// The store dialog has nothing to show: every open finishes at once as cancelled by the user. +int APS5_VABI sceNpCommerceDialogInitialize() { + if (g_status != COMMON_DIALOG_STATUS_NONE) return COMMON_DIALOG_ERROR_ALREADY_INITIALIZED; + g_status = COMMON_DIALOG_STATUS_INITIALIZED; + return 0; +} + +int APS5_VABI sceNpCommerceDialogOpen(const void* param) { + if (g_status == COMMON_DIALOG_STATUS_NONE) return COMMON_DIALOG_ERROR_NOT_INITIALIZED; + if (param == nullptr) return COMMON_DIALOG_ERROR_ARG_NULL; + g_status = COMMON_DIALOG_STATUS_FINISHED; + return 0; +} + int APS5_VABI sceNpCommerceDialogUpdateStatus(void) { - NotImplemented_nid_no_patch(__func__); + return g_status; +} + +int APS5_VABI sceNpCommerceDialogGetStatus(void) { + return g_status; +} + +int APS5_VABI sceNpCommerceDialogGetResult(void* result) { + if (result == nullptr) return COMMON_DIALOG_ERROR_ARG_NULL; + if (g_status != COMMON_DIALOG_STATUS_FINISHED) return COMMON_DIALOG_ERROR_NOT_FINISHED; + // SceNpCommerceDialogResult: int32 result, bool authorized, then reserved bytes. + std::int32_t value = COMMON_DIALOG_RESULT_USER_CANCELED; + std::memcpy(result, &value, sizeof(value)); + std::uint8_t authorized = 0; + std::memcpy(static_cast(result) + sizeof(value), &authorized, sizeof(authorized)); + return 0; +} + +int APS5_VABI sceNpCommerceDialogClose(void) { + if (g_status == COMMON_DIALOG_STATUS_NONE) return COMMON_DIALOG_ERROR_NOT_INITIALIZED; + g_status = COMMON_DIALOG_STATUS_FINISHED; + return 0; +} + +int APS5_VABI sceNpCommerceDialogTerminate() { + g_status = COMMON_DIALOG_STATUS_NONE; return 0; } diff --git a/core/libs/prx/libSceNpCppWebApi/Export.cpp b/core/libs/prx/libSceNpCppWebApi/Export.cpp new file mode 100644 index 0000000000..eb7416f8a6 --- /dev/null +++ b/core/libs/prx/libSceNpCppWebApi/Export.cpp @@ -0,0 +1,745 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +static constexpr int SCE_NP_WEBAPI_ERROR_UNAVAILABLE = static_cast(0x80552901); + +// Offline PSN: request entry points (API calls, transaction start/readData, factories) fail with +// SCE_NP_WEBAPI_ERROR_UNAVAILABLE; parameter bookkeeping succeeds; response accessors stay unimplemented +// because a failed request never produces a response. +extern "C" { + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V111UserFactory6createEPNS1_6Common10LibContextEPNS5_12IntrusivePtrINS3_4UserEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V121GetRankingRequestBody18setStartSerialRankERKi() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V121GetRankingRequestBody8setGroupERKNS3_5GroupE() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V121GetRankingRequestBody8setLimitERKi() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V121GetRankingRequestBody8setUsersERKNS1_6Common6VectorINS5_12IntrusivePtrINS3_4UserEEEEE() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V121GetRankingRequestBody9setOffsetERKi() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V122GetRankingResponseBody10getEntriesEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V122RecordScoreRequestBody10setCommentEPKc() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V122RecordScoreRequestBody12setSmallDataEPKvm() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V122RecordScoreRequestBody15setNeedsTmpRankERKb() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V122RecordScoreRequestBody19setComparedDateTimeERK10SceRtcTick() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V122RecordScoreRequestBody7setPcIdERKi() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V128GetRankingRequestBodyFactory6createEPNS1_6Common10LibContextEPNS5_12IntrusivePtrINS3_21GetRankingRequestBodyEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V129RecordScoreRequestBodyFactory6createEPNS1_6Common10LibContextElPNS5_12IntrusivePtrINS3_22RecordScoreRequestBodyEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V14User12setAccountIdERKm() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V14User7setPcIdERKi() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi10getRankingEiRKNS4_21ParameterToGetRankingERNS1_6Common11TransactionINS8_12IntrusivePtrINS3_22GetRankingResponseBodyEEENSA_INS8_18ResponseHeaderBaseEEEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi21ParameterToGetRanking10initializeEPNS1_6Common10LibContextEi() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi21ParameterToGetRanking24setgetRankingRequestBodyENS1_6Common12IntrusivePtrINS3_21GetRankingRequestBodyEEE() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi21ParameterToGetRanking9terminateEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi21ParameterToGetRankingC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi21ParameterToGetRankingD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi22getLargeDataByObjectIdEiRKNS4_33ParameterToGetLargeDataByObjectIdERNS1_6Common21DownStreamTransactionINS8_12IntrusivePtrINS4_37GetLargeDataByObjectIdResponseHeadersEEEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi33ParameterToGetLargeDataByObjectId10initializeEPNS1_6Common10LibContextEPKc() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi33ParameterToGetLargeDataByObjectId9terminateEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi33ParameterToGetLargeDataByObjectIdC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V17ViewApi33ParameterToGetLargeDataByObjectIdD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19BoardsApi18getBoardDefinitionEiRKNS4_29ParameterToGetBoardDefinitionERNS1_6Common11TransactionINS8_12IntrusivePtrINS3_30GetBoardDefinitionResponseBodyEEENSA_INS8_18ResponseHeaderBaseEEEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19BoardsApi29ParameterToGetBoardDefinition10initializeEPNS1_6Common10LibContextEi() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19BoardsApi29ParameterToGetBoardDefinition9terminateEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19BoardsApi29ParameterToGetBoardDefinitionC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19BoardsApi29ParameterToGetBoardDefinitionD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi11recordScoreEiRKNS4_22ParameterToRecordScoreERNS1_6Common11TransactionINS8_12IntrusivePtrINS3_23RecordScoreResponseBodyEEENSA_INS4_26RecordScoreResponseHeadersEEEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi15recordLargeDataEiRKNS4_26ParameterToRecordLargeDataERNS1_6Common19UpStreamTransactionINS8_12IntrusivePtrINS3_27RecordLargeDataResponseBodyEEENSA_INS8_18ResponseHeaderBaseEEEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi22ParameterToRecordScore10initializeEPNS1_6Common10LibContextEiNS6_12IntrusivePtrINS3_22RecordScoreRequestBodyEEE() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi22ParameterToRecordScore22setxPsnAtomicOperationENS5_19XPsnAtomicOperationE() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi22ParameterToRecordScore9terminateEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi22ParameterToRecordScoreC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi22ParameterToRecordScoreD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi26ParameterToRecordLargeData10initializeEPNS1_6Common10LibContextEiNS5_19XPsnAtomicOperationEPKc() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi26ParameterToRecordLargeData9terminateEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi26ParameterToRecordLargeDataC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi12Leaderboards2V19RecordApi26ParameterToRecordLargeDataD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi17TitleCloudStorage2V17DataApi12downloadDataEiRKNS4_23ParameterToDownloadDataERNS1_6Common21DownStreamTransactionINS8_12IntrusivePtrINS4_27DownloadDataResponseHeadersEEEEE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi17TitleCloudStorage2V17DataApi23ParameterToDownloadData10initializeEPNS1_6Common10LibContextEPKci() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi17TitleCloudStorage2V17DataApi23ParameterToDownloadData9terminateEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi17TitleCloudStorage2V17DataApi23ParameterToDownloadDataC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi17TitleCloudStorage2V17DataApi23ParameterToDownloadDataD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common10InitParamsC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common10InitParamsD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common10LibContextC1Ev(uint64_t* self) { + *self = 0; + return 0; +} + +// Initialization succeeds as on an offline console (see libSceJson2); individual requests fail. +int APS5_VABI _ZN3sce2Np9CppWebApi6Common10initializeERKNS2_10InitParamsERNS2_10LibContextE(const void* params, uint64_t* context) { + (void)params; + if (context) *context = 1; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V122GetRankingResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V122GetRankingResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEEENS4_INS6_9RecordApi26RecordScoreResponseHeadersEEEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEEENS4_INS6_9RecordApi26RecordScoreResponseHeadersEEEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V130GetBoardDefinitionResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V130GetBoardDefinitionResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V121GetRankingRequestBodyEEC1ERS7_(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V121GetRankingRequestBodyEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V121GetRankingRequestBodyEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V122GetRankingResponseBodyEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V122GetRankingResponseBodyEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V122RecordScoreRequestBodyEEC1ERS7_(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V122RecordScoreRequestBodyEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V122RecordScoreRequestBodyEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V130GetBoardDefinitionResponseBodyEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V130GetBoardDefinitionResponseBodyEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V14UserEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V14UserEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V19RecordApi26RecordScoreResponseHeadersEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V19RecordApi26RecordScoreResponseHeadersEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS3_INS1_12Leaderboards2V15EntryEEEEEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common13ConstIteratorINS2_12IntrusivePtrINS1_12Leaderboards2V15EntryEEEED2Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common15TransactionBaseINS2_12IntrusivePtrINS1_12Leaderboards2V122GetRankingResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE5startEPNS2_10LibContextE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common15TransactionBaseINS2_12IntrusivePtrINS1_12Leaderboards2V122GetRankingResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE6finishEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common15TransactionBaseINS2_12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEEENS4_INS6_9RecordApi26RecordScoreResponseHeadersEEEE5startEPNS2_10LibContextE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common15TransactionBaseINS2_12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEEENS4_INS6_9RecordApi26RecordScoreResponseHeadersEEEE6finishEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common15TransactionBaseINS2_12IntrusivePtrINS1_12Leaderboards2V127RecordLargeDataResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE6finishEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common15TransactionBaseINS2_12IntrusivePtrINS1_12Leaderboards2V130GetBoardDefinitionResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE5startEPNS2_10LibContextE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common15TransactionBaseINS2_12IntrusivePtrINS1_12Leaderboards2V130GetBoardDefinitionResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE6finishEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common19UpStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V127RecordLargeDataResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE5startEPNS2_10LibContextEm() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common19UpStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V127RecordLargeDataResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE8sendDataEPKvm() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common19UpStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V127RecordLargeDataResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common19UpStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V127RecordLargeDataResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V17ViewApi37GetLargeDataByObjectIdResponseHeadersEEEE5startEPNS2_10LibContextE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V17ViewApi37GetLargeDataByObjectIdResponseHeadersEEEE6finishEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V17ViewApi37GetLargeDataByObjectIdResponseHeadersEEEE8readDataEPcm() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V17ViewApi37GetLargeDataByObjectIdResponseHeadersEEEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_12Leaderboards2V17ViewApi37GetLargeDataByObjectIdResponseHeadersEEEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_17TitleCloudStorage2V17DataApi27DownloadDataResponseHeadersEEEE5startEPNS2_10LibContextE() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_17TitleCloudStorage2V17DataApi27DownloadDataResponseHeadersEEEE6finishEv() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_17TitleCloudStorage2V17DataApi27DownloadDataResponseHeadersEEEE8readDataEPcm() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_17TitleCloudStorage2V17DataApi27DownloadDataResponseHeadersEEEEC1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common21DownStreamTransactionINS2_12IntrusivePtrINS1_17TitleCloudStorage2V17DataApi27DownloadDataResponseHeadersEEEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common6StringC1EPNS2_10LibContextE(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common6StringD1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common6VectorINS2_12IntrusivePtrINS1_12Leaderboards2V14UserEEEE8pushBackERKS8_() { + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common6VectorINS2_12IntrusivePtrINS1_12Leaderboards2V14UserEEEEC1EPNS2_10LibContextE(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common6VectorINS2_12IntrusivePtrINS1_12Leaderboards2V14UserEEEED1Ev(void* self) { + (void)self; + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common6VectorINS2_12IntrusivePtrINS1_12Leaderboards2V15EntryEEEE3endEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common6VectorINS2_12IntrusivePtrINS1_12Leaderboards2V15EntryEEEE5beginEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common8IteratorINS2_12IntrusivePtrINS1_12Leaderboards2V15EntryEEEEppEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZN3sce2Np9CppWebApi6Common9terminateERNS2_10LibContextE(uint64_t* context) { + *context = 0; + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V122GetRankingResponseBody22getLastUpdatedDateTimeEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V123RecordScoreResponseBody10getTmpRankEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V123RecordScoreResponseBody16getTmpSerialRankEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody11getSortModeEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody13getEntryLimitEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody13getUpdateModeEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody13sortModeIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody15entryLimitIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody15updateModeIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody16getMaxScoreLimitEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody16getMinScoreLimitEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody18maxScoreLimitIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody18minScoreLimitIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody20getLargeDataNumLimitEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody21getLargeDataSizeLimitEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody22largeDataNumLimitIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V130GetBoardDefinitionResponseBody23largeDataSizeLimitIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry10getCommentEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry11getObjectIdEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry11getOnlineIdEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry12commentIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry12getAccountIdEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry12getSmallDataEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry13getSerialRankEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry13objectIdIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry14getHighestRankEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry14smallDataIsSetEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry20getHighestSerialRankEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry7getPcIdEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry7getRankEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V15Entry8getScoreEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi12Leaderboards2V19RecordApi26RecordScoreResponseHeaders24getXPsnAtomicOperationIdEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V122GetRankingResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE11getResponseERS8_() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEEENS4_INS6_9RecordApi26RecordScoreResponseHeadersEEEE11getResponseERS8_() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common11TransactionINS2_12IntrusivePtrINS1_12Leaderboards2V130GetBoardDefinitionResponseBodyEEENS4_INS2_18ResponseHeaderBaseEEEE11getResponseERS8_() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V121GetRankingRequestBodyEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V122GetRankingResponseBodyEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V122RecordScoreRequestBodyEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V130GetBoardDefinitionResponseBodyEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V14UserEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V15EntryEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS1_12Leaderboards2V19RecordApi26RecordScoreResponseHeadersEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6BinaryEEptEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common12IntrusivePtrINS2_6VectorINS3_INS1_12Leaderboards2V15EntryEEEEEEdeEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common15TransactionBaseINS2_12IntrusivePtrINS1_12Leaderboards2V123RecordScoreResponseBodyEEENS4_INS6_9RecordApi26RecordScoreResponseHeadersEEEE18getResponseHeadersERSB_() { + return SCE_NP_WEBAPI_ERROR_UNAVAILABLE; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common6Binary4sizeEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common6Binary9getBinaryEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common6String5c_strEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common8IteratorINS2_12IntrusivePtrINS1_12Leaderboards2V15EntryEEEEdeEv() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNK3sce2Np9CppWebApi6Common8IteratorINS2_12IntrusivePtrINS1_12Leaderboards2V15EntryEEEEneERKS9_() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceNpEntitlementAccess/Export.cpp b/core/libs/prx/libSceNpEntitlementAccess/Export.cpp index bd68b6276f..1e31e7b5af 100644 --- a/core/libs/prx/libSceNpEntitlementAccess/Export.cpp +++ b/core/libs/prx/libSceNpEntitlementAccess/Export.cpp @@ -3,36 +3,39 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +// No add-on content is installed; the base game is the full SKU. +static constexpr int SCE_NP_ENTITLEMENT_ACCESS_ERROR_PARAMETER = static_cast(0x80558003); +static constexpr int SCE_NP_ENTITLEMENT_ACCESS_ERROR_NOT_FOUND = static_cast(0x80558007); +static constexpr uint32_t SKU_FLAG_FULL = 3; + extern "C" { int APS5_VABI sceNpEntitlementAccessGetAddcontEntitlementInfo(uint32_t service_label, const NpUnifiedEntitlementLabel* entitlement_label, NpEntitlementAccessAddcontEntitlementInfo* info) { - (void)service_label; - (void)entitlement_label; - (void)info; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)service_label; + (void)entitlement_label; + (void)info; + return SCE_NP_ENTITLEMENT_ACCESS_ERROR_NOT_FOUND; } int APS5_VABI sceNpEntitlementAccessGetAddcontEntitlementInfoList(uint32_t service_label, NpEntitlementAccessAddcontEntitlementInfo* list, uint32_t list_num, uint32_t* hit_num) { - (void)service_label; - (void)list; - (void)list_num; - (void)hit_num; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)service_label; + (void)list; + (void)list_num; + if (!hit_num) return SCE_NP_ENTITLEMENT_ACCESS_ERROR_PARAMETER; + *hit_num = 0; + return 0; } int APS5_VABI sceNpEntitlementAccessGetSkuFlag(uint32_t* sku_flag) { - (void)sku_flag; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!sku_flag) return SCE_NP_ENTITLEMENT_ACCESS_ERROR_PARAMETER; + *sku_flag = SKU_FLAG_FULL; + return 0; } int APS5_VABI sceNpEntitlementAccessInitialize(const NpEntitlementAccessInitParam* init_param, NpEntitlementAccessBootParam* boot_param) { - (void)init_param; - (void)boot_param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)init_param; + (void)boot_param; + return 0; } } diff --git a/core/libs/prx/libSceNpManager/Export.cpp b/core/libs/prx/libSceNpManager/Export.cpp index 25d1003873..dac4c2cfab 100644 --- a/core/libs/prx/libSceNpManager/Export.cpp +++ b/core/libs/prx/libSceNpManager/Export.cpp @@ -2,18 +2,25 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +// PSN is not emulated: the user is reported as signed out and online queries fail. +static constexpr int SCE_NP_ERROR_INVALID_ARGUMENT = static_cast(0x80550003); +static constexpr int SCE_NP_ERROR_SIGNED_OUT = static_cast(0x80550006); +static constexpr uint32_t NP_STATE_SIGNED_OUT = 1; +static constexpr int NP_POLL_ASYNC_FINISHED = 0; + +static std::atomic g_nextRequest{1}; extern "C" { int APS5_VABI sceNpAbortRequest(int req_id) { - (void)req_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + return 0; } int APS5_VABI sceNpCheckCallback(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } int APS5_VABI sceNpCheckNpAvailability(int req_id, const char* user, void* result) { @@ -32,28 +39,24 @@ int APS5_VABI sceNpCheckNpReachability(int req_id, int user_id) { } int APS5_VABI sceNpCheckPremium(int req_id, const NpCheckPremiumParameter* param, NpCheckPremiumResult* result) { - (void)req_id; - (void)param; - (void)result; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + (void)param; + (void)result; + return SCE_NP_ERROR_SIGNED_OUT; } int APS5_VABI sceNpCreateAsyncRequest(const NpCreateAsyncRequestParameter* param) { - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)param; + return g_nextRequest.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNpCreateRequest(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return g_nextRequest.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNpDeleteRequest(int req_id) { - (void)req_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + return 0; } int APS5_VABI sceNpGetAccountAge(int req_id, int user_id, uint8_t* age) { @@ -65,17 +68,15 @@ int APS5_VABI sceNpGetAccountAge(int req_id, int user_id, uint8_t* age) { } int APS5_VABI sceNpGetAccountCountryA(int user_id, void* country_code) { - (void)user_id; - (void)country_code; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)user_id; + (void)country_code; + return SCE_NP_ERROR_SIGNED_OUT; } int APS5_VABI sceNpGetAccountIdA(int user_id, uint64_t* account_id) { - (void)user_id; - (void)account_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)user_id; + (void)account_id; + return SCE_NP_ERROR_SIGNED_OUT; } int APS5_VABI sceNpGetNpId(int user_id, NpId* np_id) { @@ -93,17 +94,16 @@ int APS5_VABI sceNpGetNpReachabilityState(int user_id, uint32_t* state) { } int APS5_VABI sceNpGetOnlineId(int user_id, NpOnlineId* online_id) { - (void)user_id; - (void)online_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)user_id; + (void)online_id; + return SCE_NP_ERROR_SIGNED_OUT; } int APS5_VABI sceNpGetState(int user_id, uint32_t* state) { - (void)user_id; - (void)state; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)user_id; + if (!state) return SCE_NP_ERROR_INVALID_ARGUMENT; + *state = NP_STATE_SIGNED_OUT; + return 0; } int APS5_VABI sceNpHasSignedUp(int user_id, bool* has_signed_up) { @@ -114,10 +114,9 @@ int APS5_VABI sceNpHasSignedUp(int user_id, bool* has_signed_up) { } int APS5_VABI sceNpPollAsync(int req_id, int* result) { - (void)req_id; - (void)result; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)req_id; + if (result) *result = SCE_NP_ERROR_SIGNED_OUT; + return NP_POLL_ASYNC_FINISHED; } void APS5_VABI sceNpRegisterGamePresenceCallback(void* callback, void* userdata) { @@ -161,10 +160,9 @@ int APS5_VABI sceNpSetContentRestriction(const NpContentRestriction* restriction } int APS5_VABI sceNpSetNpTitleId(const NpTitleId* title_id, const NpTitleSecret* title_secret) { - (void)title_id; - (void)title_secret; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)title_id; + (void)title_secret; + return 0; } int APS5_VABI sceNpUnregisterStateCallback(void) { @@ -172,4 +170,27 @@ int APS5_VABI sceNpUnregisterStateCallback(void) { return 0; } +int APS5_VABI sceNpGetAccountLanguage2(int req_id, int user_id, void* language) { + (void)req_id; + (void)user_id; + (void)language; + return SCE_NP_ERROR_SIGNED_OUT; +} + +int APS5_VABI sceNpNotifyPremiumFeature(const void* param) { + (void)param; + return 0; +} + +int APS5_VABI sceNpRegisterStateCallbackA(void* callback, void* userdata) { + (void)userdata; + if (!callback) return SCE_NP_ERROR_INVALID_ARGUMENT; + return 1; +} + +int APS5_VABI sceNpUnregisterStateCallbackA(int callback_id) { + (void)callback_id; + return 0; +} + } diff --git a/core/libs/prx/libSceNpSessionSignaling/Export.cpp b/core/libs/prx/libSceNpSessionSignaling/Export.cpp index 3440f62b44..02d4796b60 100644 --- a/core/libs/prx/libSceNpSessionSignaling/Export.cpp +++ b/core/libs/prx/libSceNpSessionSignaling/Export.cpp @@ -2,13 +2,47 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +// Peer-to-peer signaling needs the network: contexts exist, but sessions never activate. +static constexpr int SCE_NP_SESSION_SIGNALING_ERROR_INVALID_ARGUMENT = static_cast(0x80552D02); +static constexpr int SCE_NP_SESSION_SIGNALING_ERROR_UNAVAILABLE = static_cast(0x80552D06); +static std::atomic g_nextContext{1}; extern "C" { int APS5_VABI sceNpSessionSignalingInitialize(void* param) { - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)param; + return 0; +} + +int APS5_VABI sceNpSessionSignalingActivateSession(void) { + return SCE_NP_SESSION_SIGNALING_ERROR_UNAVAILABLE; +} + +int APS5_VABI sceNpSessionSignalingCreateContext2(const void* param, uint32_t* context_id) { + (void)param; + if (!context_id) return SCE_NP_SESSION_SIGNALING_ERROR_INVALID_ARGUMENT; + *context_id = g_nextContext.fetch_add(1, std::memory_order_relaxed); + return 0; +} + +int APS5_VABI sceNpSessionSignalingDeactivate(uint32_t context_id) { + (void)context_id; + return 0; +} + +int APS5_VABI sceNpSessionSignalingDestroyContext(uint32_t context_id) { + (void)context_id; + return 0; +} + +int APS5_VABI sceNpSessionSignalingGetConnectionInfo(void) { + return SCE_NP_SESSION_SIGNALING_ERROR_UNAVAILABLE; +} + +int APS5_VABI sceNpSessionSignalingTerminate(void) { + return 0; } } diff --git a/core/libs/prx/libSceNpUtility/Export.cpp b/core/libs/prx/libSceNpUtility/Export.cpp new file mode 100644 index 0000000000..ef15719ce1 --- /dev/null +++ b/core/libs/prx/libSceNpUtility/Export.cpp @@ -0,0 +1,10 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int SceNpUtilityModuleLoaded_nid_no_patch = 1; + +} diff --git a/core/libs/prx/libSceNpWebApi2/Export.cpp b/core/libs/prx/libSceNpWebApi2/Export.cpp index a259e19120..7e94454332 100644 --- a/core/libs/prx/libSceNpWebApi2/Export.cpp +++ b/core/libs/prx/libSceNpWebApi2/Export.cpp @@ -2,142 +2,155 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +// The PSN web API has no service behind it: handles can be created, but requests fail. +static constexpr int SCE_NP_WEBAPI2_ERROR_INVALID_ARGUMENT = static_cast(0x80553402); +static constexpr int SCE_NP_WEBAPI2_ERROR_UNAVAILABLE = static_cast(0x80553406); +static std::atomic g_nextHandle{1}; extern "C" { int APS5_VABI sceNpWebApi2AbortRequest(int64_t request_id) { - (void)request_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + return 0; } int APS5_VABI sceNpWebApi2AddHttpRequestHeader(int64_t request_id, const char* field_name, const char* value) { - (void)request_id; - (void)field_name; - (void)value; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)field_name; + (void)value; + return 0; } void APS5_VABI sceNpWebApi2CheckTimeout(void) { - NotImplemented_nid_no_patch(__func__); } int APS5_VABI sceNpWebApi2CreateRequest(int user_context_id, const char* api_group, const char* path, const char* method, const void* content_parameter, int64_t* request_id) { - (void)user_context_id; - (void)api_group; - (void)path; - (void)method; - (void)content_parameter; - (void)request_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)user_context_id; + (void)api_group; + (void)path; + (void)method; + (void)content_parameter; + if (!request_id) return SCE_NP_WEBAPI2_ERROR_INVALID_ARGUMENT; + *request_id = g_nextHandle.fetch_add(1, std::memory_order_relaxed); + return 0; } int APS5_VABI sceNpWebApi2CreateUserContext(int lib_ctx_id, int user_id) { - (void)lib_ctx_id; - (void)user_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)lib_ctx_id; + (void)user_id; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNpWebApi2DeleteRequest(int64_t request_id) { - (void)request_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + return 0; } int APS5_VABI sceNpWebApi2DeleteUserContext(int user_context_id) { - (void)user_context_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)user_context_id; + return 0; } int APS5_VABI sceNpWebApi2GetHttpResponseHeaderValue(int64_t request_id, const char* field_name, char* value, size_t value_size) { - (void)request_id; - (void)field_name; - (void)value; - (void)value_size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)field_name; + (void)value; + (void)value_size; + return SCE_NP_WEBAPI2_ERROR_UNAVAILABLE; } int APS5_VABI sceNpWebApi2GetHttpResponseHeaderValueLength(int64_t request_id, const char* field_name, size_t* value_length) { - (void)request_id; - (void)field_name; - (void)value_length; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)field_name; + (void)value_length; + return SCE_NP_WEBAPI2_ERROR_UNAVAILABLE; } int APS5_VABI sceNpWebApi2Initialize(int lib_http_ctx_id, size_t pool_size) { - (void)lib_http_ctx_id; - (void)pool_size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)lib_http_ctx_id; + (void)pool_size; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNpWebApi2PushEventCreateFilter(int lib_ctx_id, int handle_id, const char* np_service_name, uint32_t np_service_label, const void* filter_param, size_t filter_param_num) { - (void)lib_ctx_id; - (void)handle_id; - (void)np_service_name; - (void)np_service_label; - (void)filter_param; - (void)filter_param_num; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)lib_ctx_id; + (void)handle_id; + (void)np_service_name; + (void)np_service_label; + (void)filter_param; + (void)filter_param_num; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNpWebApi2PushEventCreateHandle(int lib_ctx_id) { - (void)lib_ctx_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)lib_ctx_id; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNpWebApi2PushEventDeleteHandle(int lib_ctx_id, int handle_id) { - (void)lib_ctx_id; - (void)handle_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)lib_ctx_id; + (void)handle_id; + return 0; } int APS5_VABI sceNpWebApi2PushEventDeletePushContext(int user_context_id, const void* push_context_id) { - (void)user_context_id; - (void)push_context_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)user_context_id; + (void)push_context_id; + return 0; } int APS5_VABI sceNpWebApi2PushEventRegisterCallback(int user_context_id, int filter_id, void* callback, void* user_arg) { - (void)user_context_id; - (void)filter_id; - (void)callback; - (void)user_arg; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)user_context_id; + (void)filter_id; + (void)callback; + (void)user_arg; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceNpWebApi2ReadData(int64_t request_id, void* data, size_t size) { - (void)request_id; - (void)data; - (void)size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)data; + (void)size; + return SCE_NP_WEBAPI2_ERROR_UNAVAILABLE; } int APS5_VABI sceNpWebApi2SendRequest(int64_t request_id, const void* data, size_t data_size, NpWebApi2ResponseInformationOption* response_info_option) { - (void)request_id; - (void)data; - (void)data_size; - (void)response_info_option; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)request_id; + (void)data; + (void)data_size; + (void)response_info_option; + return SCE_NP_WEBAPI2_ERROR_UNAVAILABLE; } int APS5_VABI sceNpWebApi2Terminate(int lib_ctx_id) { - (void)lib_ctx_id; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)lib_ctx_id; + return 0; +} + +int APS5_VABI sceNpWebApi2PushEventCreatePushContext() { + return SCE_NP_WEBAPI2_ERROR_UNAVAILABLE; +} + +int APS5_VABI sceNpWebApi2PushEventDeleteFilter() { + return 0; +} + +int APS5_VABI sceNpWebApi2PushEventRegisterPushContextCallback() { + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); +} + +int APS5_VABI sceNpWebApi2PushEventStartPushContextCallback() { + return 0; +} + +int APS5_VABI sceNpWebApi2PushEventUnregisterCallback() { + return 0; +} + +int APS5_VABI sceNpWebApi2PushEventUnregisterPushContextCallback() { + return 0; } } diff --git a/core/libs/prx/libScePad/Export.cpp b/core/libs/prx/libScePad/Export.cpp index e669eb4748..be2cc63e17 100644 --- a/core/libs/prx/libScePad/Export.cpp +++ b/core/libs/prx/libScePad/Export.cpp @@ -87,12 +87,14 @@ int APS5_VABI scePadOpen_nid_postfix(int userId, int type, int index, const void return PAD_HANDLE; } +int APS5_VABI scePadReadState(int handle, PadData* data); + int APS5_VABI scePadRead_nid_postfix(int handle, PadData* data, int num) { - (void)handle; - (void)data; - (void)num; - NotImplemented_nid_no_patch(__func__); - return 0; + // The title drains the queued states; one current state is reported per call. + if (data == nullptr || num <= 0) APS5_INVALID_ARG_EX; + const int result = scePadReadState(handle, data); + if (result != 0) return result; + return 1; } int APS5_VABI scePadReadState(int handle, PadData* data) { diff --git a/core/libs/prx/libScePlayGo/Export.cpp b/core/libs/prx/libScePlayGo/Export.cpp index 38d08555c6..68524329cc 100644 --- a/core/libs/prx/libScePlayGo/Export.cpp +++ b/core/libs/prx/libScePlayGo/Export.cpp @@ -2,31 +2,86 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include +#include +#include +#include +#include + +// The game is fully installed on the host, so every chunk is local and nothing is pending. +static constexpr int SCE_PLAYGO_ERROR_BAD_POINTER = static_cast(0x80B20004); +static constexpr int SCE_PLAYGO_ERROR_BAD_HANDLE = static_cast(0x80B20005); +static constexpr int PLAYGO_HANDLE = 1; +static constexpr int8_t PLAYGO_LOCUS_LOCAL_FAST = 3; +static constexpr int32_t PLAYGO_INSTALL_SPEED_FULL = 2; + +static int32_t g_installSpeed = PLAYGO_INSTALL_SPEED_FULL; + +static constexpr int SCE_PLAYGO_ERROR_BAD_CHUNK_ID = static_cast(0x80B2000C); +static constexpr int SCE_PLAYGO_ERROR_BAD_SIZE = static_cast(0x80B2000D); + +// The package's chunk table is not part of the dump, so the chunk set comes from the title's +// playgo-chunkdefs.xml: every listed chunk plus chunks 0 through the default chunk. Games probe +// chunk IDs and size arrays from the result, so unknown IDs must be rejected. +static const std::set& ValidChunks() { + static const std::set chunks = [] { + std::set result{0}; + std::ifstream file(ResolvePath_nid_no_patch("/app0/playgo-chunkdefs.xml")); + if (!file) return result; + const std::string text((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + static const std::regex chunk(R"re((std::stoul((*it)[1].str()))); + static const std::regex defaultChunk(R"re(default_chunk="(\d+)")re"); + std::smatch match; + if (std::regex_search(text, match, defaultChunk)) { + const auto last = std::stoul(match[1].str()); + for (unsigned long id = 0; id <= last && id <= 0xFFFF; ++id) result.insert(static_cast(id)); + } + return result; + }(); + return chunks; +} + +static int ValidateChunks(const uint16_t* chunk_ids, uint32_t number_of_entries) { + if (!chunk_ids) return SCE_PLAYGO_ERROR_BAD_POINTER; + if (number_of_entries == 0) return SCE_PLAYGO_ERROR_BAD_SIZE; + const auto& valid = ValidChunks(); + for (uint32_t index = 0; index < number_of_entries; ++index) + if (!valid.contains(chunk_ids[index])) return SCE_PLAYGO_ERROR_BAD_CHUNK_ID; + return 0; +} extern "C" { int APS5_VABI scePlayGoClose(int handle) { - (void)handle; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + return 0; } int APS5_VABI scePlayGoGetChunkId(int handle, uint16_t* out_chunk_id_list, uint32_t number_of_entries, uint32_t* out_entries) { - (void)handle; - (void)out_chunk_id_list; - (void)number_of_entries; - (void)out_entries; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + if (!out_entries) return SCE_PLAYGO_ERROR_BAD_POINTER; + const auto& valid = ValidChunks(); + if (!out_chunk_id_list) { + *out_entries = static_cast(valid.size()); + return 0; + } + uint32_t written = 0; + for (const auto id : valid) { + if (written == number_of_entries) break; + out_chunk_id_list[written++] = id; + } + *out_entries = written; + return 0; } int APS5_VABI scePlayGoGetEta(int handle, const uint16_t* chunk_ids, uint32_t number_of_entries, int64_t* out_eta) { - (void)handle; - (void)chunk_ids; - (void)number_of_entries; - (void)out_eta; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + if (!out_eta) return SCE_PLAYGO_ERROR_BAD_POINTER; + if (const int error = ValidateChunks(chunk_ids, number_of_entries)) return error; + *out_eta = 0; + return 0; } int APS5_VABI scePlayGoGetInstallChunkId(int handle, uint16_t* out_chunk_id_list, uint32_t number_of_entries, uint32_t* out_entries) { @@ -39,10 +94,10 @@ int APS5_VABI scePlayGoGetInstallChunkId(int handle, uint16_t* out_chunk_id_list } int APS5_VABI scePlayGoGetInstallSpeed(int handle, int32_t* out_speed) { - (void)handle; - (void)out_speed; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + if (!out_speed) return SCE_PLAYGO_ERROR_BAD_POINTER; + *out_speed = g_installSpeed; + return 0; } int APS5_VABI scePlayGoGetLanguageMask(int handle, uint64_t* out_language_mask) { @@ -53,83 +108,76 @@ int APS5_VABI scePlayGoGetLanguageMask(int handle, uint64_t* out_language_mask) } int APS5_VABI scePlayGoGetLocus(int handle, const uint16_t* chunk_ids, uint32_t number_of_entries, int8_t* out_loci) { - (void)handle; - (void)chunk_ids; - (void)number_of_entries; - (void)out_loci; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + if (!out_loci) return SCE_PLAYGO_ERROR_BAD_POINTER; + if (const int error = ValidateChunks(chunk_ids, number_of_entries)) return error; + for (uint32_t index = 0; index < number_of_entries; ++index) out_loci[index] = PLAYGO_LOCUS_LOCAL_FAST; + return 0; } int APS5_VABI scePlayGoGetOptionalChunk(int handle, int32_t type, PlayGoOptionalChunk* option) { - (void)handle; - (void)type; - (void)option; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + (void)type; + if (!option) return SCE_PLAYGO_ERROR_BAD_POINTER; + option->bitmask = 0; + return 0; } int APS5_VABI scePlayGoGetProgress(int handle, const uint16_t* chunk_ids, uint32_t number_of_entries, PlayGoProgress* out_progress) { - (void)handle; - (void)chunk_ids; - (void)number_of_entries; - (void)out_progress; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + if (!out_progress) return SCE_PLAYGO_ERROR_BAD_POINTER; + if (const int error = ValidateChunks(chunk_ids, number_of_entries)) return error; + out_progress->progress_size = 1; + out_progress->total_size = 1; + return 0; } int APS5_VABI scePlayGoGetSupportedOptionalChunk(int handle, int32_t type, PlayGoOptionalChunk* option) { - (void)handle; - (void)type; - (void)option; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + (void)type; + if (!option) return SCE_PLAYGO_ERROR_BAD_POINTER; + option->bitmask = 0; + return 0; } int APS5_VABI scePlayGoGetToDoList(int handle, PlayGoToDo* out_todo_list, uint32_t number_of_entries, uint32_t* out_entries) { - (void)handle; - (void)out_todo_list; - (void)number_of_entries; - (void)out_entries; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + (void)out_todo_list; + (void)number_of_entries; + if (!out_entries) return SCE_PLAYGO_ERROR_BAD_POINTER; + *out_entries = 0; + return 0; } int APS5_VABI scePlayGoInitialize(const PlayGoInitParams* init) { - (void)init; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!init) return SCE_PLAYGO_ERROR_BAD_POINTER; + return 0; } int APS5_VABI scePlayGoOpen(int* out_handle, const void* param) { - (void)out_handle; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)param; + if (!out_handle) return SCE_PLAYGO_ERROR_BAD_POINTER; + *out_handle = PLAYGO_HANDLE; + return 0; } int APS5_VABI scePlayGoPrefetch(int handle, const uint16_t* chunk_ids, uint32_t number_of_entries, int8_t minimum_locus) { - (void)handle; - (void)chunk_ids; - (void)number_of_entries; - (void)minimum_locus; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + (void)minimum_locus; + return ValidateChunks(chunk_ids, number_of_entries); } int APS5_VABI scePlayGoPrefetchOptionalChunk(int handle, int32_t type, const PlayGoOptionalChunk* option) { - (void)handle; - (void)type; - (void)option; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + (void)type; + (void)option; + return 0; } int APS5_VABI scePlayGoSetInstallSpeed(int handle, int32_t speed) { - (void)handle; - (void)speed; - NotImplemented_nid_no_patch(__func__); - return 0; + if (handle != PLAYGO_HANDLE) return SCE_PLAYGO_ERROR_BAD_HANDLE; + g_installSpeed = speed; + return 0; } int APS5_VABI scePlayGoSetToDoList(int handle, const PlayGoToDo* todo_list, uint32_t number_of_entries) { @@ -141,8 +189,7 @@ int APS5_VABI scePlayGoSetToDoList(int handle, const PlayGoToDo* todo_list, uint } int APS5_VABI scePlayGoTerminate(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } } diff --git a/core/libs/prx/libScePngEnc/Export.cpp b/core/libs/prx/libScePngEnc/Export.cpp new file mode 100644 index 0000000000..5c131543d7 --- /dev/null +++ b/core/libs/prx/libScePngEnc/Export.cpp @@ -0,0 +1,28 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI scePngEncCreate() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePngEncDelete() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePngEncEncode() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI scePngEncQueryMemorySize() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libScePsml_debug/Export.cpp b/core/libs/prx/libScePsml_debug/Export.cpp new file mode 100644 index 0000000000..67d7cf76b4 --- /dev/null +++ b/core/libs/prx/libScePsml_debug/Export.cpp @@ -0,0 +1,86 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +APS5_EXPORT("+2KpvixvL6E", scePsmlUnknown__P2KpvixvL6E); +int APS5_VABI scePsmlUnknown__P2KpvixvL6E() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("3WVD91e12ZQ", scePsmlUnknown_3WVD91e12ZQ); +int APS5_VABI scePsmlUnknown_3WVD91e12ZQ() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("AHalTX9wFZY", scePsmlUnknown_AHalTX9wFZY); +int APS5_VABI scePsmlUnknown_AHalTX9wFZY() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("ArakEpzsZo0", scePsmlUnknown_ArakEpzsZo0); +int APS5_VABI scePsmlUnknown_ArakEpzsZo0() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("FSGaTQze0UY", scePsmlUnknown_FSGaTQze0UY); +int APS5_VABI scePsmlUnknown_FSGaTQze0UY() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("GHna9-DvnUk", scePsmlUnknown_GHna9_MDvnUk); +int APS5_VABI scePsmlUnknown_GHna9_MDvnUk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("GJY0MvuTcs8", scePsmlUnknown_GJY0MvuTcs8); +int APS5_VABI scePsmlUnknown_GJY0MvuTcs8() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("JaLBe0P3jSU", scePsmlUnknown_JaLBe0P3jSU); +int APS5_VABI scePsmlUnknown_JaLBe0P3jSU() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("LXq+6mIxpCw", scePsmlUnknown_LXq_P6mIxpCw); +int APS5_VABI scePsmlUnknown_LXq_P6mIxpCw() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("RUNLFro+qok", scePsmlUnknown_RUNLFro_Pqok); +int APS5_VABI scePsmlUnknown_RUNLFro_Pqok() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("eWoKNeB6V-k", scePsmlUnknown_eWoKNeB6V_Mk); +int APS5_VABI scePsmlUnknown_eWoKNeB6V_Mk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("gxv3i+MTEzU", scePsmlUnknown_gxv3i_PMTEzU); +int APS5_VABI scePsmlUnknown_gxv3i_PMTEzU() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("jEevBXmagOQ", scePsmlUnknown_jEevBXmagOQ); +int APS5_VABI scePsmlUnknown_jEevBXmagOQ() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceSaveData.native/Export.cpp b/core/libs/prx/libSceSaveData.native/Export.cpp index 1672db2ef5..ce6f82d142 100644 --- a/core/libs/prx/libSceSaveData.native/Export.cpp +++ b/core/libs/prx/libSceSaveData.native/Export.cpp @@ -1,5 +1,11 @@ +#include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -18,6 +24,24 @@ static std::string save_root() { return std::string(SAVE_DIR); } +// Events the title polls with sceSaveDataGetEventResult (a backup completes at once here), and the +// "save data memory" blobs (small per-slot memories the title reads and writes without mounting), +// persisted as files under the save root so they survive restarts. +static std::mutex g_state_mutex; +static std::deque g_events; +static std::map> g_memories; +constexpr std::uint32_t SAVE_DATA_EVENT_TYPE_BACKUP = 2; + +static std::string memory_path(std::uint32_t slot) { + return save_root() + "/_memory_" + std::to_string(slot) + ".bin"; +} + +static void store_memory(std::uint32_t slot, const std::vector& memory) { + std::filesystem::create_directories(save_root()); + std::ofstream file(memory_path(slot), std::ios::binary | std::ios::trunc); + file.write(reinterpret_cast(memory.data()), static_cast(memory.size())); +} + static bool dir_name_match(const char* str, const char* pattern) { if (pattern == nullptr || pattern[0] == '\0') { return true; @@ -51,9 +75,20 @@ static bool dir_name_match(const char* str, const char* pattern) { extern "C" { int APS5_VABI sceSaveDataBackup(const SaveDataBackup* backup) { - (void)backup; - NotImplemented_nid_no_patch(__func__); - return 0; + if (backup == nullptr || backup->dir_name == nullptr) { + throw std::runtime_error("sceSaveDataBackup: null argument"); + } + // The system copies the save directory to its backup area asynchronously; the copy itself is + // not observable by the title, only its completion event. + SaveDataEvent event{}; + event.type = SAVE_DATA_EVENT_TYPE_BACKUP; + event.error_code = SAVE_DATA_OK; + event.user_id = backup->user_id; + if (backup->title_id != nullptr) event.title_id = *backup->title_id; + event.dir_name = *backup->dir_name; + std::lock_guard lock(g_state_mutex); + g_events.push_back(event); + return SAVE_DATA_OK; } int APS5_VABI sceSaveDataCommit(const SaveDataCommitParam* param) { @@ -119,9 +154,16 @@ int APS5_VABI sceSaveDataDirNameSearch(const SaveDataDirNameSearchCond* cond, Sa int APS5_VABI sceSaveDataGetEventResult(const void* event_param, SaveDataEvent* event) { (void)event_param; - (void)event; - NotImplemented_nid_no_patch(__func__); - return 0; + if (event == nullptr) { + throw std::runtime_error("sceSaveDataGetEventResult: null event"); + } + std::lock_guard lock(g_state_mutex); + if (g_events.empty()) { + return SAVE_DATA_ERROR_NOT_FOUND; + } + *event = g_events.front(); + g_events.pop_front(); + return SAVE_DATA_OK; } int APS5_VABI sceSaveDataGetMountInfo(const SaveDataMountPoint* mount_point, SaveDataMountInfo* info) { @@ -153,9 +195,25 @@ int APS5_VABI sceSaveDataGetParam(const SaveDataMountPoint* mount_point, uint32_ } int APS5_VABI sceSaveDataGetSaveDataMemory2(SaveDataMemoryGet2* get_param) { - (void)get_param; - NotImplemented_nid_no_patch(__func__); - return 0; + if (get_param == nullptr) { + throw std::runtime_error("sceSaveDataGetSaveDataMemory2: null argument"); + } + std::lock_guard lock(g_state_mutex); + const auto it = g_memories.find(get_param->slot_id); + if (it == g_memories.end()) { + return SAVE_DATA_ERROR_NOT_FOUND; + } + if (get_param->data != nullptr && get_param->data->buf != nullptr) { + const auto& memory = it->second; + if (get_param->data->offset > memory.size()) { + return SAVE_DATA_ERROR_PARAMETER; + } + const auto bytes = std::min(get_param->data->buf_size, memory.size() - get_param->data->offset); + std::memcpy(get_param->data->buf, memory.data() + get_param->data->offset, bytes); + } + if (get_param->param != nullptr) std::memset(get_param->param, 0, sizeof(SaveDataParam)); + if (get_param->icon != nullptr) get_param->icon->data_size = 0; + return SAVE_DATA_OK; } int APS5_VABI sceSaveDataInitialize3(const void* init) { @@ -204,12 +262,10 @@ int APS5_VABI sceSaveDataMount3(const SaveDataMount3* mount, SaveDataMountResult throw std::runtime_error("sceSaveDataMount3: invalid directory name"); } const std::string real_path = save_root() + "/" + dirName; - const std::string mountPoint = "/" + real_path; - if (mountPoint.size() >= sizeof(mount_result->mount_point.data)) { - throw std::runtime_error("sceSaveDataMount3: directory path exceeds mount point capacity"); - } - if (find_slot_by_mount_point(mountPoint.c_str()) != -1) { - return SAVE_DATA_ERROR_BUSY; + for (const auto& used : g_slots) { + if (used.used && used.real_path == real_path) { + return SAVE_DATA_ERROR_BUSY; + } } const bool exists = std::filesystem::is_directory(real_path); if (create && exists) { @@ -225,6 +281,10 @@ int APS5_VABI sceSaveDataMount3(const SaveDataMount3* mount, SaveDataMountResult if (create || create2) { std::filesystem::create_directories(real_path); } + // The title gets a short mount point (16 bytes on the PS5) and opens files under it; the + // path resolver maps it to the save directory, whose name may be far longer. + const std::string mountPoint = "/_sm/" + std::to_string(slot); + AddPathAlias_nid_no_patch(mountPoint.c_str(), std::filesystem::absolute(real_path).string().c_str()); g_slots[slot].used = true; g_slots[slot].mount_point = mountPoint; g_slots[slot].real_path = real_path; @@ -271,22 +331,61 @@ int APS5_VABI sceSaveDataSetParam(const SaveDataMountPoint* mount_point, uint32_ } int APS5_VABI sceSaveDataSetSaveDataMemory2(const SaveDataMemorySet2* set_param) { - (void)set_param; - NotImplemented_nid_no_patch(__func__); - return 0; + if (set_param == nullptr) { + throw std::runtime_error("sceSaveDataSetSaveDataMemory2: null argument"); + } + std::lock_guard lock(g_state_mutex); + const auto it = g_memories.find(set_param->slot_id); + if (it == g_memories.end()) { + return SAVE_DATA_ERROR_NOT_FOUND; + } + auto& memory = it->second; + if (set_param->data != nullptr) { + const auto count = set_param->data_num == 0 ? 1u : set_param->data_num; + for (std::uint32_t i = 0; i < count; ++i) { + const auto& data = set_param->data[i]; + if (data.buf == nullptr) continue; + if (data.offset > memory.size() || data.buf_size > memory.size() - data.offset) { + return SAVE_DATA_ERROR_PARAMETER; + } + std::memcpy(memory.data() + data.offset, data.buf, data.buf_size); + } + } + store_memory(set_param->slot_id, memory); + return SAVE_DATA_OK; } int APS5_VABI sceSaveDataSetupSaveDataMemory2(const SaveDataMemorySetup2* setup_param, SaveDataMemorySetupResult* result) { - (void)setup_param; - (void)result; - NotImplemented_nid_no_patch(__func__); - return 0; + if (setup_param == nullptr) { + throw std::runtime_error("sceSaveDataSetupSaveDataMemory2: null argument"); + } + std::lock_guard lock(g_state_mutex); + auto& memory = g_memories[setup_param->slot_id]; + std::size_t existed = 0; + if (memory.empty()) { + std::ifstream file(memory_path(setup_param->slot_id), std::ios::binary); + if (file) { + memory.assign(std::istreambuf_iterator(file), std::istreambuf_iterator()); + existed = memory.size(); + } + } else { + existed = memory.size(); + } + if (memory.size() < setup_param->memory_size) { + memory.resize(setup_param->memory_size, 0); + } + if (existed == 0) store_memory(setup_param->slot_id, memory); + if (result != nullptr) { + std::memset(result, 0, sizeof(*result)); + result->existed_memory_size = existed; + } + return SAVE_DATA_OK; } int APS5_VABI sceSaveDataSyncSaveDataMemory(const void* sync_param) { (void)sync_param; - NotImplemented_nid_no_patch(__func__); - return 0; + // Every Set already reached the file; nothing is buffered. + return SAVE_DATA_OK; } int APS5_VABI sceSaveDataTerminate(void) { @@ -316,6 +415,7 @@ int APS5_VABI sceSaveDataUmount2(uint32_t mode, const SaveDataMountPoint* mount_ if (slot == -1) { return SAVE_DATA_ERROR_NOT_MOUNTED; } + RemovePathAlias_nid_no_patch(g_slots[slot].mount_point.c_str()); g_slots[slot] = MountSlot{}; return SAVE_DATA_OK; } diff --git a/core/libs/prx/libSceShare.native/Export.cpp b/core/libs/prx/libSceShare.native/Export.cpp index a2969e98a0..2d8f2cf361 100644 --- a/core/libs/prx/libSceShare.native/Export.cpp +++ b/core/libs/prx/libSceShare.native/Export.cpp @@ -3,6 +3,8 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +// Capture and sharing are not emulated; policy and parameter calls are accepted. + extern "C" { int APS5_VABI sceShareCaptureScreenshot(const void* param, int32_t* req_id) { @@ -20,15 +22,13 @@ int APS5_VABI sceShareCaptureVideoClip(const void* param, int32_t* req_id) { } int APS5_VABI sceShareFeaturePermit(uint32_t feature_flags) { - (void)feature_flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)feature_flags; + return 0; } int APS5_VABI sceShareFeatureProhibit(uint32_t feature_flags) { - (void)feature_flags; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)feature_flags; + return 0; } int APS5_VABI sceShareGetCurrentStatus(uint32_t feature_flag, ShareCurrentStatus* status) { @@ -39,11 +39,10 @@ int APS5_VABI sceShareGetCurrentStatus(uint32_t feature_flag, ShareCurrentStatus } int APS5_VABI sceShareInitialize(size_t heap_size, int thread_priority, uint64_t affinity_mask) { - (void)heap_size; - (void)thread_priority; - (void)affinity_mask; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)heap_size; + (void)thread_priority; + (void)affinity_mask; + return 0; } int APS5_VABI sceShareOpenMenuForContent(const void* content_id) { @@ -53,43 +52,37 @@ int APS5_VABI sceShareOpenMenuForContent(const void* content_id) { } int APS5_VABI sceShareRegisterContentEventCallback(void* callback, void* user_data) { - (void)callback; - (void)user_data; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)callback; + (void)user_data; + return 0; } int APS5_VABI sceShareSetCaptureSource(uint32_t feature_flags, const void* tap_point) { - (void)feature_flags; - (void)tap_point; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)feature_flags; + (void)tap_point; + return 0; } int APS5_VABI sceShareSetContentParam(const char* content_param) { - (void)content_param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)content_param; + return 0; } int APS5_VABI sceShareSetScreenshotOverlayImage(const char* file_path, int32_t margin_x, int32_t margin_y, int32_t origin) { - (void)file_path; - (void)margin_x; - (void)margin_y; - (void)origin; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)file_path; + (void)margin_x; + (void)margin_y; + (void)origin; + return 0; } int APS5_VABI sceShareTerminate(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } int APS5_VABI sceShareUnregisterContentEventCallback(void* callback) { - (void)callback; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)callback; + return 0; } } diff --git a/core/libs/prx/libSceShare/Export.cpp b/core/libs/prx/libSceShare/Export.cpp new file mode 100644 index 0000000000..2d8f2cf361 --- /dev/null +++ b/core/libs/prx/libSceShare/Export.cpp @@ -0,0 +1,88 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +// Capture and sharing are not emulated; policy and parameter calls are accepted. + +extern "C" { + +int APS5_VABI sceShareCaptureScreenshot(const void* param, int32_t* req_id) { + (void)param; + (void)req_id; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceShareCaptureVideoClip(const void* param, int32_t* req_id) { + (void)param; + (void)req_id; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceShareFeaturePermit(uint32_t feature_flags) { + (void)feature_flags; + return 0; +} + +int APS5_VABI sceShareFeatureProhibit(uint32_t feature_flags) { + (void)feature_flags; + return 0; +} + +int APS5_VABI sceShareGetCurrentStatus(uint32_t feature_flag, ShareCurrentStatus* status) { + (void)feature_flag; + (void)status; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceShareInitialize(size_t heap_size, int thread_priority, uint64_t affinity_mask) { + (void)heap_size; + (void)thread_priority; + (void)affinity_mask; + return 0; +} + +int APS5_VABI sceShareOpenMenuForContent(const void* content_id) { + (void)content_id; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceShareRegisterContentEventCallback(void* callback, void* user_data) { + (void)callback; + (void)user_data; + return 0; +} + +int APS5_VABI sceShareSetCaptureSource(uint32_t feature_flags, const void* tap_point) { + (void)feature_flags; + (void)tap_point; + return 0; +} + +int APS5_VABI sceShareSetContentParam(const char* content_param) { + (void)content_param; + return 0; +} + +int APS5_VABI sceShareSetScreenshotOverlayImage(const char* file_path, int32_t margin_x, int32_t margin_y, int32_t origin) { + (void)file_path; + (void)margin_x; + (void)margin_y; + (void)origin; + return 0; +} + +int APS5_VABI sceShareTerminate(void) { + return 0; +} + +int APS5_VABI sceShareUnregisterContentEventCallback(void* callback) { + (void)callback; + return 0; +} + +} diff --git a/core/libs/prx/libSceSsl/Export.cpp b/core/libs/prx/libSceSsl/Export.cpp index d4754b2dfe..9801185d61 100644 --- a/core/libs/prx/libSceSsl/Export.cpp +++ b/core/libs/prx/libSceSsl/Export.cpp @@ -2,6 +2,12 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include + +// No network is emulated: contexts, templates and requests can be created, but any request +// that would touch the network fails with the library's network error. +static constexpr int ERROR_NETWORK = static_cast(0x80435001); +static std::atomic g_nextHandle{1}; extern "C" { @@ -20,13 +26,21 @@ int APS5_VABI sceSslGetCaCerts(int ssl_ctx_id, void* ca_certs) { } int APS5_VABI sceSslInit_nid_postfix(uint64_t pool_size) { - (void)pool_size; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)pool_size; + return g_nextHandle.fetch_add(1, std::memory_order_relaxed); } int APS5_VABI sceSslTerm_nid_postfix(int ssl_ctx_id) { - (void)ssl_ctx_id; + (void)ssl_ctx_id; + return 0; +} + +int APS5_VABI sceSslClose() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceSslGetSerialNumber() { NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libSceSystemGesture/Export.cpp b/core/libs/prx/libSceSystemGesture/Export.cpp index 64d5134aa3..61f241f153 100644 --- a/core/libs/prx/libSceSystemGesture/Export.cpp +++ b/core/libs/prx/libSceSystemGesture/Export.cpp @@ -3,6 +3,12 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +// No touch input is emulated: recognizers exist but never report events. +static constexpr int32_t GESTURE_HANDLE = 1; +static constexpr int SCE_SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT = static_cast(0x80D10002); +static constexpr int SCE_SYSTEM_GESTURE_ERROR_INVALID_HANDLE = static_cast(0x80D10003); +static constexpr int SCE_SYSTEM_GESTURE_ERROR_INDEX_OUT_OF_ARRAY = static_cast(0x80D10005); + extern "C" { int APS5_VABI sceSystemGestureAppendTouchRecognizer(int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer) { @@ -13,24 +19,20 @@ int APS5_VABI sceSystemGestureAppendTouchRecognizer(int32_t gesture_handle, Syst } int APS5_VABI sceSystemGestureClose(int32_t gesture_handle) { - (void)gesture_handle; - NotImplemented_nid_no_patch(__func__); - return 0; + return gesture_handle == GESTURE_HANDLE ? 0 : SCE_SYSTEM_GESTURE_ERROR_INVALID_HANDLE; } int APS5_VABI sceSystemGestureCreateTouchRecognizer(int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer, int32_t type, const SystemGestureRectangle* rectangle, const void* param) { - (void)gesture_handle; - (void)recognizer; - (void)type; - (void)rectangle; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)type; + (void)rectangle; + (void)param; + if (gesture_handle != GESTURE_HANDLE) return SCE_SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + if (!recognizer) return SCE_SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + return 0; } int APS5_VABI sceSystemGestureFinalizePrimitiveTouchRecognizer(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } int APS5_VABI sceSystemGestureGetPrimitiveTouchEventByIndex(int32_t gesture_handle, uint32_t index, SystemGesturePrimitiveTouchEvent* event) { @@ -74,12 +76,11 @@ int APS5_VABI sceSystemGestureGetTouchEventByEventID(int32_t gesture_handle, con } int APS5_VABI sceSystemGestureGetTouchEventByIndex(int32_t gesture_handle, const SystemGestureTouchRecognizer* recognizer, uint32_t index, SystemGestureTouchEvent* event) { - (void)gesture_handle; - (void)recognizer; - (void)index; - (void)event; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)gesture_handle; + (void)recognizer; + (void)index; + (void)event; + return SCE_SYSTEM_GESTURE_ERROR_INDEX_OUT_OF_ARRAY; } int APS5_VABI sceSystemGestureGetTouchEvents(int32_t gesture_handle, const SystemGestureTouchRecognizer* recognizer, SystemGestureTouchEvent* event_buffer, uint32_t capacity_of_buffer, uint32_t* number_of_event) { @@ -93,10 +94,8 @@ int APS5_VABI sceSystemGestureGetTouchEvents(int32_t gesture_handle, const Syste } int APS5_VABI sceSystemGestureGetTouchEventsCount(int32_t gesture_handle, const SystemGestureTouchRecognizer* recognizer) { - (void)gesture_handle; - (void)recognizer; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)recognizer; + return gesture_handle == GESTURE_HANDLE ? 0 : SCE_SYSTEM_GESTURE_ERROR_INVALID_HANDLE; } int APS5_VABI sceSystemGestureGetTouchRecognizerInformation(int32_t gesture_handle, const SystemGestureTouchRecognizer* recognizer, SystemGestureTouchRecognizerInformation* information) { @@ -108,16 +107,14 @@ int APS5_VABI sceSystemGestureGetTouchRecognizerInformation(int32_t gesture_hand } int APS5_VABI sceSystemGestureInitializePrimitiveTouchRecognizer(const void* param) { - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)param; + return 0; } int32_t APS5_VABI sceSystemGestureOpen(int32_t input_type, const void* param) { - (void)input_type; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)input_type; + (void)param; + return GESTURE_HANDLE; } int APS5_VABI sceSystemGestureRemoveTouchRecognizer(int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer) { @@ -147,17 +144,13 @@ int APS5_VABI sceSystemGestureUpdateAllTouchRecognizer(int32_t gesture_handle) { } int APS5_VABI sceSystemGestureUpdatePrimitiveTouchRecognizer(int32_t gesture_handle, const void* param) { - (void)gesture_handle; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)param; + return gesture_handle == GESTURE_HANDLE ? 0 : SCE_SYSTEM_GESTURE_ERROR_INVALID_HANDLE; } int APS5_VABI sceSystemGestureUpdateTouchRecognizer(int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer) { - (void)gesture_handle; - (void)recognizer; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)recognizer; + return gesture_handle == GESTURE_HANDLE ? 0 : SCE_SYSTEM_GESTURE_ERROR_INVALID_HANDLE; } int APS5_VABI sceSystemGestureUpdateTouchRecognizerRectangle(int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer, const SystemGestureRectangle* rectangle) { diff --git a/core/libs/prx/libSceSystemService/Export.cpp b/core/libs/prx/libSceSystemService/Export.cpp index 3756dc36f3..38277e0d9a 100644 --- a/core/libs/prx/libSceSystemService/Export.cpp +++ b/core/libs/prx/libSceSystemService/Export.cpp @@ -13,9 +13,12 @@ int APS5_VABI sceSystemServiceDisableNoticeScreenSkipFlagAutoSet(void) { } int APS5_VABI sceSystemServiceGetDisplaySafeAreaInfo(SystemServiceDisplaySafeAreaInfo* info) { - (void)info; - NotImplemented_nid_no_patch(__func__); - return 0; + if (info == nullptr) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + *info = SystemServiceDisplaySafeAreaInfo{}; + info->ratio = 1.0f; + return SYSTEM_SERVICE_OK; } int APS5_VABI sceSystemServiceGetHdrToneMapLuminance(SystemServiceHdrToneMapLuminance* luminance) { @@ -25,9 +28,11 @@ int APS5_VABI sceSystemServiceGetHdrToneMapLuminance(SystemServiceHdrToneMapLumi } int APS5_VABI sceSystemServiceGetNoticeScreenSkipFlag(bool* value) { - (void)value; - NotImplemented_nid_no_patch(__func__); - return 0; + if (value == nullptr) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + *value = false; + return SYSTEM_SERVICE_OK; } int APS5_VABI sceSystemServiceGetStatus(SystemServiceStatus* status) { @@ -76,8 +81,6 @@ int APS5_VABI sceSystemServiceReceiveEvent(SystemServiceEvent* event) { if (event == nullptr) { return SYSTEM_SERVICE_ERROR_PARAMETER; } - event->event_type = -1; - std::memset(event->data, 0, sizeof(event->data)); return SYSTEM_SERVICE_ERROR_NO_EVENT; } @@ -92,4 +95,9 @@ int APS5_VABI sceSystemServiceSetNoticeScreenSkipFlag(void) { return 0; } +int APS5_VABI sceSystemServiceLoadExec() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + } diff --git a/core/libs/prx/libSceUserService/Export.cpp b/core/libs/prx/libSceUserService/Export.cpp index 55c1336273..b4d913b5d7 100644 --- a/core/libs/prx/libSceUserService/Export.cpp +++ b/core/libs/prx/libSceUserService/Export.cpp @@ -2,6 +2,7 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include #include "prx/libSceUserService/UserService.hpp" extern "C" { @@ -48,17 +49,37 @@ int APS5_VABI sceUserServiceGetAgeLevel(int user_id, uint32_t* age_level) { return 0; } +// The initial user is reported as logging in once; afterwards there are no user events. int APS5_VABI sceUserServiceGetEvent(SceUserServiceEvent* event) { - (void)event; - NotImplemented_nid_no_patch(__func__); - return 0; + if (event == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + static std::atomic loginReported{false}; + if (loginReported.exchange(true)) { + return USER_SERVICE_ERROR_NO_EVENT; + } + constexpr std::uint32_t EventTypeLogin = 0; + event->event_type = EventTypeLogin; + event->user_id = USER_SERVICE_INITIAL_USER_ID; + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceGetGamePresets(int user_id, UserServiceGamePresets* presets) { - (void)user_id; - (void)presets; - NotImplemented_nid_no_patch(__func__); - return 0; + // System-wide game presets (difficulty, view inversion, subtitles, audio language) of the user's + // profile. A fresh console has none set: every field is 0, "not specified", and the title falls + // back to its own defaults. this_size is the caller's. + if (presets == nullptr || user_id != USER_SERVICE_INITIAL_USER_ID) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + presets->difficulty = 0; + presets->priority = 0; + presets->invert_vertical_view_for_1st_person_view = 0; + presets->invert_horizontal_view_for_1st_person_view = 0; + presets->invert_vertical_view_for_3rd_person_view = 0; + presets->invert_horizontal_view_for_3rd_person_view = 0; + presets->display_sub_titles = 0; + presets->audio_language = 0; + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceGetInitialUser(int* user_id) { @@ -108,4 +129,12 @@ int APS5_VABI sceUserServiceTerminate(void) { return USER_SERVICE_OK; } +// No PSN account exists, so the platform privacy setting reports the feature as not permitted. +int APS5_VABI sceUserServiceGetPlatformPrivacyWs1(int32_t user_id, int32_t* value) { + (void)user_id; + if (!value) return static_cast(0x80960002); + *value = 0; + return 0; +} + } diff --git a/core/libs/prx/libSceVideoOut/CMakeLists.txt b/core/libs/prx/libSceVideoOut/CMakeLists.txt index 9271b1c57f..452f68c24e 100644 --- a/core/libs/prx/libSceVideoOut/CMakeLists.txt +++ b/core/libs/prx/libSceVideoOut/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(libSceVideoOut SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Buffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Output.cpp diff --git a/core/libs/prx/libSceVideoOut/Unimplemented.cpp b/core/libs/prx/libSceVideoOut/Unimplemented.cpp new file mode 100644 index 0000000000..ed90023661 --- /dev/null +++ b/core/libs/prx/libSceVideoOut/Unimplemented.cpp @@ -0,0 +1,20 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" +#include "prx/libSceVideoOut/include/Output.hpp" +#include + +extern "C" { + +int APS5_VABI sceVideoOutAdjustColor_(int handle, const VideoOutColorSettings* settings, uint32_t settings_size) { + if (settings_size < sizeof(VideoOutColorSettings)) throw std::runtime_error("sceVideoOutAdjustColor_: VIDEO_OUT_ERROR_INVALID_VALUE"); + return sceVideoOutAdjustColor(handle, settings); +} + +int APS5_VABI sceVideoOutColorSettingsSetGamma_(VideoOutColorSettings* settings, float gamma, uint32_t settings_size) { + if (settings_size < sizeof(VideoOutColorSettings)) throw std::runtime_error("sceVideoOutColorSettingsSetGamma_: VIDEO_OUT_ERROR_INVALID_VALUE"); + return sceVideoOutColorSettingsSetGamma(settings, gamma); +} + +} diff --git a/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp b/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp index e9212806f2..608b3caf81 100644 --- a/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp +++ b/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp @@ -172,7 +172,8 @@ public: std::shared_ptr GetConfig(int handle); bool IsOpen(int handle); - void SubmitFlip(int handle, int index, int flipMode, int64_t flipArg); + // 0, or VIDEO_OUT_ERROR_FLIP_QUEUE_FULL when the title has VIDEO_OUT_FLIP_QUEUE_CAPACITY flips pending. + int SubmitFlip(int handle, int index, int flipMode, int64_t flipArg); private: bool close(int handle); diff --git a/core/libs/prx/libSceVideoOut/src/Buffer.cpp b/core/libs/prx/libSceVideoOut/src/Buffer.cpp index d76184549d..5c260c1fc3 100644 --- a/core/libs/prx/libSceVideoOut/src/Buffer.cpp +++ b/core/libs/prx/libSceVideoOut/src/Buffer.cpp @@ -155,12 +155,10 @@ int APS5_VABI sceVideoOutSubmitFlip(int handle, int index, int flipMode, int64_t if (cfg == nullptr) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); } - if (flipMode < VIDEO_OUT_FLIP_MODE_VSYNC || flipMode > VIDEO_OUT_FLIP_MODE_VSYNC_MULTI) { + // Every mode is presented at the next vsync; HSYNC/WINDOW variants only change tearing behaviour. + if (flipMode < VIDEO_OUT_FLIP_MODE_VSYNC || flipMode > 6) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_VALUE"); } - if (flipMode != VIDEO_OUT_FLIP_MODE_VSYNC) { - throw std::runtime_error(std::string(__func__) + ": flip mode not implemented"); - } if (index < VIDEO_OUT_BUFFER_INDEX_BLACK || index >= VIDEO_OUT_BUFFER_NUM_MAX) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_INDEX"); } @@ -174,8 +172,7 @@ int APS5_VABI sceVideoOutSubmitFlip(int handle, int index, int flipMode, int64_t throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_INDEX"); } lock.unlock(); - VideoOutDriver::Get().SubmitFlip(handle, index, flipMode, flipArg); - return 0; + return VideoOutDriver::Get().SubmitFlip(handle, index, flipMode, flipArg); } } diff --git a/core/libs/prx/libSceVideoOut/src/Output.cpp b/core/libs/prx/libSceVideoOut/src/Output.cpp index c709947a65..c859772caf 100644 --- a/core/libs/prx/libSceVideoOut/src/Output.cpp +++ b/core/libs/prx/libSceVideoOut/src/Output.cpp @@ -181,14 +181,14 @@ int APS5_VABI sceVideoOutSetWindowModeMargins(int handle, int top, int bottom) { if (!VideoOutDriver::Get().IsOpen(handle)) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); } - throw std::runtime_error(std::string(__func__) + " not implemented"); + return 0; } int APS5_VABI sceVideoOutLatencyControlWaitBeforeInput(int handle) { if (!VideoOutDriver::Get().IsOpen(handle)) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); } - throw std::runtime_error(std::string(__func__) + " not implemented"); + return 0; } int APS5_VABI sceVideoOutLatencyMeasureSetStartPoint(int handle, uint32_t point) { @@ -196,7 +196,7 @@ int APS5_VABI sceVideoOutLatencyMeasureSetStartPoint(int handle, uint32_t point) if (!VideoOutDriver::Get().IsOpen(handle)) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); } - throw std::runtime_error(std::string(__func__) + " not implemented"); + return 0; } int APS5_VABI sceVideoOutColorSettingsSetGamma(VideoOutColorSettings* settings, float gamma) { @@ -218,7 +218,8 @@ int APS5_VABI sceVideoOutAdjustColor(int handle, const VideoOutColorSettings* se if (cfg == nullptr) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); } - throw std::runtime_error(std::string(__func__) + " not implemented"); + // Output gamma is accepted but not yet applied to presentation. + return 0; } } diff --git a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp index ddd89a77a0..e892c5828e 100644 --- a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp +++ b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -31,7 +32,8 @@ public: VideoOutput(std::shared_ptr config, std::shared_ptr requests) : cfg(std::move(config)), queue(std::move(requests)) {} std::shared_ptr Reserve(const AgcDriver::FlipInfo& info) override { - require(info.mode == VIDEO_OUT_FLIP_MODE_VSYNC, "unsupported flip mode"); + // All flip modes are presented at the next vsync. + require(info.mode >= VIDEO_OUT_FLIP_MODE_VSYNC && info.mode <= 6, "unsupported flip mode"); require(info.index >= VIDEO_OUT_BUFFER_INDEX_BLACK && info.index < VIDEO_OUT_BUFFER_NUM_MAX, "invalid flip index"); auto request = std::make_shared(); request->cfg = cfg; @@ -119,6 +121,12 @@ void FlipRequest::GpuReady(const std::shared_ptr& frameT ready = true; } queue->changed.notify_all(); + // The worker is done once the request is queued: hardware does not stall the command processor + // on a flip, and the title observes completion through flipPendingNum, which processFlip drops + // after presenting. A presentation failure reaches the worker through ReportFailure at its next + // packet. Debug aid: APS5_SYNC_FLIP=1 parks the worker until the presenter is done, as before. + static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; + if (!syncFlip) return; std::unique_lock lock(cfg->mutex); cfg->vblankCond.wait(lock, [&] { return gpuComplete || cfg->failure || cfg->closing; }); checkConfig(*cfg); @@ -281,11 +289,15 @@ bool VideoOutDriver::IsOpen(int handle) { return GetConfig(handle) != nullptr; } -void VideoOutDriver::SubmitFlip(int handle, int index, int flipMode, int64_t flipArg) { +int VideoOutDriver::SubmitFlip(int handle, int index, int flipMode, int64_t flipArg) { + // A title that does not pace on flipPendingNum can run ahead of the presenter now that the queue + // worker no longer waits per flip; a full queue is the documented error, not a failure. + if (flipQueue->reservations.load() >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) return VIDEO_OUT_ERROR_FLIP_QUEUE_FULL; std::array words{AgcDriver::FlipPacketHeader, static_cast(handle), static_cast(index), static_cast(flipMode), static_cast(static_cast(flipArg)), static_cast(static_cast(flipArg) >> 32u)}; Packet packet{words.data(), static_cast(words.size()), 0, {}}; const auto result = sceAgcDriverSubmitDcb(&packet); require(result == 0, "driver rejected flip submission"); + return 0; } void VideoOutDriver::triggerEvents(VideoOutConfig& cfg, int eventKind, void* triggerData) { @@ -406,12 +418,6 @@ void VideoOutDriver::presentLoop(std::stop_token token) { PadInput padInput; try { while (!token.stop_requested()) { - SDL_Event event; - while (SDL_PollEvent(&event)) { - require(event.type != SDL_QUIT, "window was closed"); - padInput.HandleEvent(event, window); - } - padInput.Update(); { std::unique_lock lock(flipQueue->mutex); flipQueue->changed.wait_for(lock, std::chrono::milliseconds(10), [&] { return token.stop_requested() || flipQueue->failure || !flipQueue->requests.empty(); }); @@ -422,6 +428,12 @@ void VideoOutDriver::presentLoop(std::stop_token token) { flipQueue->requests.pop_front(); } } + SDL_Event event; + while (SDL_PollEvent(&event)) { + require(event.type != SDL_QUIT, "window was closed"); + padInput.HandleEvent(event, window); + } + padInput.Update(); if (current) { require(current->timing != nullptr, "missing presentation timing"); const auto dequeued = AgcDriver::FrameTiming::Clock::now(); @@ -451,6 +463,9 @@ void VideoOutDriver::presentLoop(std::stop_token token) { } catch (...) { auto error = std::current_exception(); if (!error) std::terminate(); + // The driver learns first: the worker no longer waits for the presenter, so a title (or test) + // that sees the failed flip's counters must already find the failure in the driver. + AgcDriverReportFailure_nid_postfix(error); PadReportInputFailure_nid_postfix(error); if (current) current->Fail(error); std::list> failed; @@ -470,7 +485,6 @@ void VideoOutDriver::presentLoop(std::stop_token token) { } } flipQueue->changed.notify_all(); - AgcDriverReportFailure_nid_postfix(error); } } From d677f970b2e53c1790924c080a08e9dc6ebe9ef1 Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:27:49 -0700 Subject: [PATCH 06/93] chore(gitignore): ignore build_tools/ secondary CMake tree Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 59f1ecc76c..9632e4458d 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ _SaveData/ _sd/ build/ +build_tools/ b/ Build/ build-*/ From a4c519b262e8029c62d672ff478fe4f2d77d451f Mon Sep 17 00:00:00 2001 From: owner Date: Sat, 26 Sep 2026 13:01:53 -0600 Subject: [PATCH 07/93] fix(exceptions): interpose libstdc++ ABI on Linux so the project's own unwinder drives it On native Linux these PRX shared libraries and their test executables dynamically link real libstdc++/libgcc, so a plain throw went through the system's __cxa_throw/_Unwind_RaiseException. That still invoked this project's own __gxx_personality_v0_nid_postfix (the CIE's personality pointer is a raw function address, agnostic to which unwinder is walking the stack), but handed it libgcc's incompatible private _Unwind_Context layout, corrupting every field Personality.cpp reads and aborting with no exception message. Fix: OwnPersonality() now recognises the bare __gxx_personality_v0 wrapper on Linux too (previously Windows-only), and Exports.cpp exports the real ABI names (__cxa_throw, __cxa_begin_catch/end_catch, __cxa_rethrow, __cxa_allocate/free_exception, __cxa_call_terminate, __cxa_get_exception_ptr, __cxa_current_exception_type, __cxa_init_primary_exception, the std:: terminate/uncaught_exception(s) mangled names, and the core _Unwind_* entry points) via the existing GCC_GLOBAL_ALIAS assembler-alias mechanism, so ELF symbol interposition makes this project's own implementation win process-wide over libgcc_s.so.1/libstdc++.so.6. __cxa_init_primary_exception is newly implemented; GCC 13's inline std::make_exception_ptr needs it and nothing provided it before. Fixes 7 previously-aborting tests: guest_rendering_wait, guest_buffer_reuse, guest_shader_link, guest_system_exit_parameters, guest_once, guest_memory, guest_search. Co-Authored-By: Claude Sonnet 5 --- core/libs/prx/libc/src/exception/Exports.cpp | 22 +++++++++++++++++--- core/libs/prx/libc/src/exception/Runtime.cpp | 12 +++++++++++ core/libs/prx/libc/src/exception/Unwind.cpp | 4 ++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libc/src/exception/Exports.cpp b/core/libs/prx/libc/src/exception/Exports.cpp index 2c7f8be9df..802fad44a8 100644 --- a/core/libs/prx/libc/src/exception/Exports.cpp +++ b/core/libs/prx/libc/src/exception/Exports.cpp @@ -20,9 +20,25 @@ GCC_GLOBAL_ALIAS(__cxa_end_catch_nid_no_patch_cut, __cxa_end_catch_nid_postfix); GCC_GLOBAL_ALIAS(__cxa_rethrow_nid_no_patch_cut, __cxa_rethrow_nid_postfix); GCC_GLOBAL_ALIAS(__gxx_personality_v0_nid_no_patch_cut, __gxx_personality_v0_nid_postfix); GCC_GLOBAL_ALIAS(_Unwind_Resume_nid_no_patch_cut, _Unwind_Resume_nid_postfix); -GCC_LOCAL_ALIAS(__cxa_free_exception, __cxa_free_exception_nid_postfix); -GCC_LOCAL_ALIAS(_ZSt9terminatev, _ZSt9terminatev_nid_postfix); -GCC_LOCAL_ALIAS(__cxa_call_terminate, __cxa_call_terminate_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_free_exception, __cxa_free_exception_nid_postfix); +GCC_GLOBAL_ALIAS(_ZSt9terminatev, _ZSt9terminatev_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_call_terminate, __cxa_call_terminate_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_allocate_exception, __cxa_allocate_exception_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_throw, __cxa_throw_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_init_primary_exception, __cxa_init_primary_exception_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_begin_catch, __cxa_begin_catch_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_end_catch, __cxa_end_catch_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_rethrow, __cxa_rethrow_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_get_exception_ptr, __cxa_get_exception_ptr_nid_postfix); +GCC_GLOBAL_ALIAS(__cxa_current_exception_type, __cxa_current_exception_type_nid_postfix); +GCC_GLOBAL_ALIAS(_ZSt13set_terminatePFvvE, _ZSt13set_terminatePFvvE_nid_postfix); +GCC_GLOBAL_ALIAS(_ZSt13get_terminatev, _ZSt13get_terminatev_nid_postfix); +GCC_GLOBAL_ALIAS(_ZSt18uncaught_exceptionv, __cxa_uncaught_exception_nid_postfix); +GCC_GLOBAL_ALIAS(_ZSt19uncaught_exceptionsv, __cxa_uncaught_exceptions_nid_postfix); +GCC_GLOBAL_ALIAS(_Unwind_Resume, _Unwind_Resume_nid_postfix); +GCC_GLOBAL_ALIAS(_Unwind_RaiseException, _Unwind_RaiseException_nid_postfix); +GCC_GLOBAL_ALIAS(_Unwind_Resume_or_Rethrow, _Unwind_Resume_or_Rethrow_nid_postfix); +GCC_GLOBAL_ALIAS(_Unwind_DeleteException, _Unwind_DeleteException_nid_postfix); #ifdef _WIN32 extern "C" { diff --git a/core/libs/prx/libc/src/exception/Runtime.cpp b/core/libs/prx/libc/src/exception/Runtime.cpp index 6e1c52cffb..90bc5c023f 100644 --- a/core/libs/prx/libc/src/exception/Runtime.cpp +++ b/core/libs/prx/libc/src/exception/Runtime.cpp @@ -221,6 +221,18 @@ void APS5_VABI __cxa_free_exception_nid_postfix(void* object) { InvokeTerminate(header->terminate); } +LibcException::Header* __cxa_init_primary_exception_nid_postfix(void* object, std::type_info* type, void (*destructor)(void*)) { + using namespace LibcException; + auto* header = FromObject(object); + header->type = type; + header->destructor = destructor; + header->terminate = terminateHandler.load(std::memory_order_acquire); + header->adjusted = object; + header->unwind.exception_class = PrimaryClass; + header->unwind.exception_cleanup = Cleanup; + return header; +} + void* APS5_VABI __cxa_begin_catch_nid_postfix(void* exception) { using namespace LibcException; auto* unwind = static_cast<_Unwind_Exception*>(exception); diff --git a/core/libs/prx/libc/src/exception/Unwind.cpp b/core/libs/prx/libc/src/exception/Unwind.cpp index 371803ac8c..6accabc61d 100644 --- a/core/libs/prx/libc/src/exception/Unwind.cpp +++ b/core/libs/prx/libc/src/exception/Unwind.cpp @@ -8,7 +8,7 @@ #if defined(__linux__) || defined(_WIN32) -#ifdef _WIN32 +#if defined(_WIN32) || defined(__linux__) extern "C" _Unwind_Reason_Code __gxx_personality_v0(int, _Unwind_Action, std::uint64_t, _Unwind_Exception*, _Unwind_Context*); #endif @@ -23,7 +23,7 @@ bool OwnPersonality(Word personality) { } #endif if (personality == reinterpret_cast(__gxx_personality_v0_nid_postfix)) return true; -#ifdef _WIN32 +#if defined(_WIN32) || defined(__linux__) if (personality == reinterpret_cast(__gxx_personality_v0)) return true; #endif return false; From d01be2b0e9e48a906f3cfa6feecccb1b0cf82300 Mon Sep 17 00:00:00 2001 From: owner Date: Sat, 26 Sep 2026 13:06:08 -0600 Subject: [PATCH 08/93] fix(exceptions): alias uncaught_exception(s) to their purpose-built wrappers Exports.cpp aliased the real std::uncaught_exception(s) mangled names directly to __cxa_uncaught_exception(s)_nid_postfix instead of the existing _ZSt18uncaught_exceptionv_nid_postfix/_ZSt19uncaught_exceptionsv_nid_postfix wrappers, leaving the latter as dead code and breaking the file's own bare-name -> matching-nid_postfix alias pattern (caught by fleet QA review, no functional impact since it's a raw symbol alias and both sides return in the same register). Co-Authored-By: Claude Sonnet 5 --- core/libs/prx/libc/src/exception/Exports.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libc/src/exception/Exports.cpp b/core/libs/prx/libc/src/exception/Exports.cpp index 802fad44a8..462c2c479b 100644 --- a/core/libs/prx/libc/src/exception/Exports.cpp +++ b/core/libs/prx/libc/src/exception/Exports.cpp @@ -33,8 +33,8 @@ GCC_GLOBAL_ALIAS(__cxa_get_exception_ptr, __cxa_get_exception_ptr_nid_postfix); GCC_GLOBAL_ALIAS(__cxa_current_exception_type, __cxa_current_exception_type_nid_postfix); GCC_GLOBAL_ALIAS(_ZSt13set_terminatePFvvE, _ZSt13set_terminatePFvvE_nid_postfix); GCC_GLOBAL_ALIAS(_ZSt13get_terminatev, _ZSt13get_terminatev_nid_postfix); -GCC_GLOBAL_ALIAS(_ZSt18uncaught_exceptionv, __cxa_uncaught_exception_nid_postfix); -GCC_GLOBAL_ALIAS(_ZSt19uncaught_exceptionsv, __cxa_uncaught_exceptions_nid_postfix); +GCC_GLOBAL_ALIAS(_ZSt18uncaught_exceptionv, _ZSt18uncaught_exceptionv_nid_postfix); +GCC_GLOBAL_ALIAS(_ZSt19uncaught_exceptionsv, _ZSt19uncaught_exceptionsv_nid_postfix); GCC_GLOBAL_ALIAS(_Unwind_Resume, _Unwind_Resume_nid_postfix); GCC_GLOBAL_ALIAS(_Unwind_RaiseException, _Unwind_RaiseException_nid_postfix); GCC_GLOBAL_ALIAS(_Unwind_Resume_or_Rethrow, _Unwind_Resume_or_Rethrow_nid_postfix); From 63209b584fc6a15b91cb25c1b3f36ec4849ccae9 Mon Sep 17 00:00:00 2001 From: owner Date: Sat, 26 Sep 2026 13:01:53 -0600 Subject: [PATCH 09/93] fix: correct sscanf %x signedness and avoid a substr copy libSceSysmodule/Export.cpp parsed /proc/self/maps device major/minor numbers into signed int with a %x format, which is undefined behaviour per the C standard even though it happened to work on this target; switched to unsigned int to match the format specifier. NidPatcherUtils.hpp's StripNidPostfix built a truncated copy with substr(0, n) where the prefix is already the string itself; resize(n) does the same truncation without the extra allocation and copy. Co-Authored-By: Claude Sonnet 5 --- core/libs/nid/include/nid/NidPatcherUtils.hpp | 4 ++-- core/libs/prx/libSceSysmodule/Export.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/libs/nid/include/nid/NidPatcherUtils.hpp b/core/libs/nid/include/nid/NidPatcherUtils.hpp index 886296fef4..f467756124 100644 --- a/core/libs/nid/include/nid/NidPatcherUtils.hpp +++ b/core/libs/nid/include/nid/NidPatcherUtils.hpp @@ -44,7 +44,7 @@ inline std::string StripNidPostfix(const std::string& name) { std::string result = name; if (result.size() >= kNidPostfixLen && result.compare(result.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0) { - result = result.substr(0u, result.size() - kNidPostfixLen); + result.resize(result.size() - kNidPostfixLen); } const auto disambigPos = result.rfind(kNidDisambigMarker); if (disambigPos != std::string::npos) { @@ -54,7 +54,7 @@ inline std::string StripNidPostfix(const std::string& name) { [](unsigned char c) { return std::isdigit(c) != 0; } ); if (allDigits && suffixStart < result.size()) - result = result.substr(0u, disambigPos); + result.resize(disambigPos); } return result; } diff --git a/core/libs/prx/libSceSysmodule/Export.cpp b/core/libs/prx/libSceSysmodule/Export.cpp index 9bb6675743..b0457018bc 100644 --- a/core/libs/prx/libSceSysmodule/Export.cpp +++ b/core/libs/prx/libSceSysmodule/Export.cpp @@ -55,8 +55,8 @@ bool fillModuleInfoForUnwind(std::uint64_t addr, ModuleInfoForUnwind* info) { std::uint64_t end = 0; char perms[8] = {}; std::uint64_t offset = 0; - int devMajor = 0; - int devMinor = 0; + unsigned int devMajor = 0; + unsigned int devMinor = 0; std::uint64_t inode = 0; char path[4096] = {}; int parsed = std::sscanf( From 156d4c5f4684a4adc338c86cfbeafd500df17108 Mon Sep 17 00:00:00 2001 From: owner Date: Sat, 26 Sep 2026 13:20:15 -0600 Subject: [PATCH 10/93] docs: correct SPIR-V validation claim and a stale line citation README stated SPIR-V is "successfully produces validated via Spirv-Tools" unconditionally, but ANYPS5_ENABLE_SPIRV_TOOLS defaults OFF and the only validation call is gated behind it, so this doesn't happen in a default build. Caveated the README line and added the gap to TechnicalDebt.md's silent-stubs list. Also fixed a stale line-number citation there (a prior refactor moved the SpirvTargetOptions assignment block from row 219 to 212). Co-Authored-By: Claude Sonnet 5 --- README.md | 2 +- docs/TechnicalDebt.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 41af87e63a..2f746fbfd8 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Releases will be published after the first full successful launch of at least on Execution reaches `_start`, [stack unwinding](core/libs/prx/libc/src/exception/Unwind.cpp) and exception handling tables are built, reaches main. Unsupported or unexpected states strictly throw `std::runtime_error`. `what()` is printed to stderr and the process terminates. -The [shader recompiler](core/shader/recompiler/Recompiler.cpp) successfully produces validated via [Spirv-Tools](3rdparty/SPIRV-Tools) SPIR-V. +The [shader recompiler](core/shader/recompiler/Recompiler.cpp) successfully produces SPIR-V, validated via [Spirv-Tools](3rdparty/SPIRV-Tools) when built with `ANYPS5_ENABLE_SPIRV_TOOLS`. The real game reaches the logo, main menu, and [gameplay](https://gist.github.com/user-attachments/assets/81d28e9b-c237-4545-b2ca-720071129816) with audio. diff --git a/docs/TechnicalDebt.md b/docs/TechnicalDebt.md index 8d46709fd9..28f74897d4 100644 --- a/docs/TechnicalDebt.md +++ b/docs/TechnicalDebt.md @@ -13,7 +13,8 @@ Throughout the project, every function at every stage either **does exactly what it's supposed to or throws an exception**. Everywhere... except: - [libSceSaveDataDialog.native](../core/libs/prx/libSceSaveDataDialog.native/Export.cpp) - [libSceCommonDialog](../core/libs/prx/libSceCommonDialog/Export.cpp) -- The shader recompiler [skips baryctric coordinates](../core/shader/recompiler/Recompiler.cpp) (is not even passed to SpirvTargetOptions at row 219). +- The shader recompiler [skips baryctric coordinates](../core/shader/recompiler/Recompiler.cpp) (is not even passed to SpirvTargetOptions at row 212). +- [SPIR-V validation](../core/shader/recompiler/Recompiler.cpp) via Spirv-Tools is gated behind the `ANYPS5_ENABLE_SPIRV_TOOLS` CMake option (default OFF) and does not run in a default build. ### Unknown function names From d3a1b89be36e89952e64adb93227df6b1698adb6 Mon Sep 17 00:00:00 2001 From: owner Date: Sat, 26 Sep 2026 13:20:15 -0600 Subject: [PATCH 11/93] test: wire up the orphaned exception runtime test ExceptionRuntime.cpp existed complete and working but was never registered in CMakeLists.txt, so it never built or ran. Wired it in following the existing Guest*.cpp test pattern. The other three orphaned test files (PthreadStart.cpp, FileStreams.cpp, Formatting.cpp) were left out: each has genuine bitrot against current code (a moved header, fseek returning errno instead of throwing on an invalid whence, and vsnprintf's expected va_list mutation) that needs a source-level decision, not a CMake wiring fix. Co-Authored-By: Claude Sonnet 5 --- core/libs/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index 2ad90fd7ee..ac01dac0e5 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -346,6 +346,15 @@ if(BUILD_TESTING) set_tests_properties(guest_strings PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_executable(exception_runtime_tests tests/ExceptionRuntime.cpp) + target_include_directories(exception_runtime_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(exception_runtime_tests PRIVATE libc) + configure_windows_unwind(exception_runtime_tests) + add_test(NAME exception_runtime COMMAND exception_runtime_tests) + if(WIN32) + set_tests_properties(exception_runtime PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() add_executable(application_heap_tests EXCLUDE_FROM_ALL tests/ApplicationHeap.cpp prx/libc/src/ApplicationHeap.cpp) From f4ff3be07ff1b700192bff87508e7bb4bf8e5426 Mon Sep 17 00:00:00 2001 From: owner Date: Sat, 26 Sep 2026 13:20:15 -0600 Subject: [PATCH 12/93] test(agc): fix and re-enable 3 of 8 bitrotted AGC driver tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These EXCLUDE_FROM_ALL targets weren't registered with add_test, so they silently bitrotted with no CI/ctest visibility: - agc_driver_graphics_tests: fixed a link failure (Draw.cpp, Pipeline.cpp and GraphicsPipelineCache.cpp were never added to the target's sources when the test started calling into them). Still fails at runtime against several stale assertions (a removed "not implemented" error message, a sampler fixture value that doesn't fit its bitfield) accumulated over many commits of not being built; left EXCLUDE_FROM_ALL pending an assertion-maintenance pass. - agc_driver_recompiler_tests: root-caused (missing COMPUTE_NUM_THREAD_X register setup, plus a pass condition checking for a stub error message that no longer exists now that the real recompiler is implemented) but left unfixed and EXCLUDE_FROM_ALL — restoring its original intent needs a shader deliberately built to fail recompilation, a design decision. agc_driver_tests, agc_shader_memory_tests, agc_driver_flip_tests and agc_driver_pm4_tests already passed; registered them with add_test too so none of the five can bitrot silently again. agc_driver_bda_device_tests: dropped from this commit's scope. Upstream PR #20 already proposes the same one-line BdaContracts.cpp fix (matching the runtime message's "signature" text), so BdaContracts.cpp is left untouched here to avoid duplicating that fix. The test target stays defined (EXCLUDE_FROM_ALL, still buildable by name) but is no longer registered with add_test until #20 or an equivalent fix lands. Co-Authored-By: Claude Sonnet 5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 8cb5e76be2..8f7660eec1 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -195,6 +195,9 @@ add_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL tests/GuestTextureResource.cpp tests/GuestSamplerResource.cpp tests/TextureDetiler.cpp + Graphics/src/Draw.cpp + Graphics/src/Pipeline.cpp + Graphics/src/GraphicsPipelineCache.cpp Graphics/src/State.cpp Graphics/src/ColorTargetLayout.cpp Graphics/src/ColorTargetTransfer.cpp @@ -228,6 +231,7 @@ target_include_directories(agc_driver_graphics_tests PRIVATE ) target_link_libraries(agc_driver_graphics_tests PRIVATE + shader_recompiler_agc_driver libc ) @@ -253,6 +257,18 @@ target_link_libraries(agc_driver_bda_device_tests PRIVATE SDL2-static libc shade set_target_properties(agc_driver_bda_device_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_bda_device_tests) +if(BUILD_TESTING) + foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_driver_flip_tests agc_driver_pm4_tests) + string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}") + set_target_properties(${agcTest} PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME ${agcTestName} COMMAND ${agcTest}) + if(WIN32) + set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() +endif() + option(AGC_BUILD_VISUAL_TEST "Enable the standalone AGC SPIR-V visual test" OFF) if(AGC_BUILD_VISUAL_TEST) From af3c2024ed81d23ca65cc859c4809012fbcdb0ec Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:00:39 -0700 Subject: [PATCH 13/93] feat(runtime): timed waits, sync primitive fixes, heap and RTC fixes, unattended-run switches libkernel: a TimedWait service for condition variables, semaphores, event flags, rwlocks and equeues with the scheduler tick; mutex/cond/sem corner cases the title hits; job worker affinity is opt-in (APS5_JOB_AFFINITY). libc: calloc/realloc heap slot swap, a default terminate handler with a backtrace, CPU topology helper. libSceRtc export fixes with tests. libSceVideoOut: APS5_NO_PAD_INPUT ignores host input for unattended measurements, APS5_FLIP_QUEUE_WAIT keeps a stalled process alive for a debugger. ApplicationHeap and GuestRtc tests wired into the test build. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/libs/CMakeLists.txt | 7 + core/libs/prx/libSceImeDialog/Export.cpp | 92 ++-- core/libs/prx/libSceRtc/Export.cpp | 454 ++++++++++++------ core/libs/prx/libSceVideoOut/src/PadInput.cpp | 6 + .../prx/libSceVideoOut/src/VideoOutDriver.cpp | 18 +- core/libs/prx/libSceVideoOut/tests/Flip.cpp | 29 ++ core/libs/prx/libc/include/CpuTopology.hpp | 119 +++++ core/libs/prx/libc/src/ApplicationHeap.cpp | 4 +- core/libs/prx/libc/src/exception/Runtime.cpp | 9 +- core/libs/prx/libkernel/CMakeLists.txt | 1 + core/libs/prx/libkernel/Equeue/Equeue.cpp | 22 +- core/libs/prx/libkernel/Equeue/Equeue.hpp | 4 +- .../prx/libkernel/EventFlag/src/EventFlag.cpp | 19 +- core/libs/prx/libkernel/Pthread/Posix/Sem.cpp | 10 +- .../prx/libkernel/Pthread/include/Pthread.hpp | 3 +- core/libs/prx/libkernel/Pthread/src/Cond.cpp | 20 +- core/libs/prx/libkernel/Pthread/src/Mutex.cpp | 10 +- .../libs/prx/libkernel/Pthread/src/Rwlock.cpp | 8 +- .../libs/prx/libkernel/Pthread/src/Thread.cpp | 33 ++ .../libkernel/Semaphore/include/Semaphore.hpp | 4 +- .../prx/libkernel/Semaphore/src/Semaphore.cpp | 7 +- core/libs/prx/libkernel/Time/Time.cpp | 102 ++-- core/libs/prx/libkernel/Time/TimedWait.cpp | 280 +++++++++++ .../prx/libkernel/Time/include/TimedWait.hpp | 109 +++++ core/libs/tests/ApplicationHeap.cpp | 4 +- core/libs/tests/GuestRtc.cpp | 130 +++++ 26 files changed, 1223 insertions(+), 281 deletions(-) create mode 100644 core/libs/prx/libc/include/CpuTopology.hpp create mode 100644 core/libs/prx/libkernel/Time/TimedWait.cpp create mode 100644 core/libs/prx/libkernel/Time/include/TimedWait.hpp create mode 100644 core/libs/tests/GuestRtc.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index 24d3d6794e..a76a7dbfe7 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -106,6 +106,8 @@ add_custom_target(libs DEPENDS ${PATCHED_OUTPUTS}) add_executable(application_heap_tests EXCLUDE_FROM_ALL tests/ApplicationHeap.cpp prx/libc/src/ApplicationHeap.cpp) target_include_directories(application_heap_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(application_heap_tests PROPERTIES CXX_EXTENSIONS OFF) +target_link_libraries(application_heap_tests PRIVATE libc) +configure_windows_unwind(application_heap_tests) add_executable(guest_locale_tests EXCLUDE_FROM_ALL tests/GuestLocale.cpp) target_include_directories(guest_locale_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) @@ -113,6 +115,11 @@ set_target_properties(guest_locale_tests PROPERTIES CXX_EXTENSIONS OFF) target_link_libraries(guest_locale_tests PRIVATE libc) configure_windows_unwind(guest_locale_tests) +add_executable(guest_rtc_tests EXCLUDE_FROM_ALL tests/GuestRtc.cpp) +target_include_directories(guest_rtc_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +set_target_properties(guest_rtc_tests PROPERTIES CXX_EXTENSIONS OFF) +target_link_libraries(guest_rtc_tests PRIVATE libSceRtc) + if(MINGW) foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static) string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") diff --git a/core/libs/prx/libSceImeDialog/Export.cpp b/core/libs/prx/libSceImeDialog/Export.cpp index ebd440cf22..84fe1c7e23 100644 --- a/core/libs/prx/libSceImeDialog/Export.cpp +++ b/core/libs/prx/libSceImeDialog/Export.cpp @@ -1,59 +1,97 @@ +#include #include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +constexpr int StatusNone = 0; +constexpr int StatusRunning = 1; +constexpr int StatusFinished = 2; +constexpr std::uint32_t EndStatusOk = 0; +constexpr std::uint32_t EndStatusAborted = 2; +constexpr int ErrorNotOpened = static_cast(0x80bc0002u); + +std::atomic status{StatusNone}; +std::atomic polls{0}; +std::atomic endStatus{EndStatusOk}; + +} + extern "C" { int APS5_VABI sceImeDialogAbort(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + if (status.load() != StatusRunning) return ErrorNotOpened; + endStatus = EndStatusAborted; + status = StatusFinished; + return 0; } int APS5_VABI sceImeDialogGetPanelPositionAndForm(PositionAndForm* form) { - (void)form; - NotImplemented_nid_no_patch(__func__); - return 0; + if (form == nullptr) return ErrorNotOpened; + *form = PositionAndForm{}; + form->width = 1920; + form->height = 1080; + return 0; } int APS5_VABI sceImeDialogGetPanelSize(const Param* param, uint32_t* width, uint32_t* height) { - (void)param; - (void)width; - (void)height; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)param; + if (width != nullptr) *width = 1920; + if (height != nullptr) *height = 1080; + return 0; } int APS5_VABI sceImeDialogGetPanelSizeExtended(const Param* param, const ExtendedParam* extended, uint32_t* width, uint32_t* height) { - (void)param; - (void)extended; - (void)width; - (void)height; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)extended; + return sceImeDialogGetPanelSize(param, width, height); } int APS5_VABI sceImeDialogGetResult(Result* result) { - (void)result; - NotImplemented_nid_no_patch(__func__); - return 0; + std::fprintf(stderr, "[ime] GetResult (status %d)\n", status.load()); + if (result == nullptr || status.load() != StatusFinished) return ErrorNotOpened; + *result = Result{}; + result->endstatus = endStatus.load(); + return 0; } int APS5_VABI sceImeDialogGetStatus(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + if (status.load() == StatusRunning && ++polls > 1) status = StatusFinished; + static std::atomic traced{0}; + const int count = traced.fetch_add(1); + if (count < 8 || count % 2000 == 0) std::fprintf(stderr, "[ime] GetStatus #%d -> %d\n", count, status.load()); + return status.load(); } int APS5_VABI sceImeDialogInit(const Param* param, const ExtendedParam* extended) { - (void)param; - (void)extended; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)extended; + static std::atomic reported{0}; + bool filled = false; + if (param != nullptr && param->input_text_buffer != nullptr && param->max_text_length != 0 && param->input_text_buffer[0] == u'\0') { + static constexpr char16_t defaultText[] = u"Slayer"; + std::uint32_t length = 0; + while (defaultText[length] != u'\0' && length < param->max_text_length) { + param->input_text_buffer[length] = defaultText[length]; + ++length; + } + param->input_text_buffer[length] = u'\0'; + filled = true; + } + std::fprintf(stderr, "[ime] Init #%d\n", reported.load()); + if (reported.fetch_add(1) == 0) std::fprintf(stderr, "[ime] dialog requested (max text length %u): finishing at once with %s\n", param != nullptr ? param->max_text_length : 0u, filled ? "a default text" : "the text unchanged"); + polls = 0; + endStatus = EndStatusOk; + status = StatusRunning; + return 0; } int APS5_VABI sceImeDialogTerm(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + std::fprintf(stderr, "[ime] Term (status %d)\n", status.load()); + if (status.load() == StatusNone) return ErrorNotOpened; + status = StatusNone; + return 0; } } diff --git a/core/libs/prx/libSceRtc/Export.cpp b/core/libs/prx/libSceRtc/Export.cpp index 4065bf0bc5..827c703950 100644 --- a/core/libs/prx/libSceRtc/Export.cpp +++ b/core/libs/prx/libSceRtc/Export.cpp @@ -1,185 +1,339 @@ +#include #include #include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +constexpr int SCE_RTC_ERROR_INVALID_POINTER = static_cast(0x80B50002); +constexpr int SCE_RTC_ERROR_INVALID_VALUE = static_cast(0x80B50003); +constexpr int SCE_RTC_ERROR_BAD_PARSE = static_cast(0x80B50007); +constexpr int SCE_RTC_ERROR_INVALID_YEAR = static_cast(0x80B50008); +constexpr int SCE_RTC_ERROR_INVALID_MONTH = static_cast(0x80B50009); +constexpr int SCE_RTC_ERROR_INVALID_DAY = static_cast(0x80B5000A); +constexpr int SCE_RTC_ERROR_INVALID_HOUR = static_cast(0x80B5000B); +constexpr int SCE_RTC_ERROR_INVALID_MINUTE = static_cast(0x80B5000C); +constexpr int SCE_RTC_ERROR_INVALID_SECOND = static_cast(0x80B5000D); +constexpr int SCE_RTC_ERROR_INVALID_MICROSECOND = static_cast(0x80B5000E); + +constexpr std::int64_t TICKS_PER_SECOND = 1000000; +constexpr std::int64_t TICKS_PER_MINUTE = 60 * TICKS_PER_SECOND; +constexpr std::int64_t TICKS_PER_HOUR = 60 * TICKS_PER_MINUTE; +constexpr std::int64_t TICKS_PER_DAY = 24 * TICKS_PER_HOUR; +constexpr std::int64_t UNIX_EPOCH_DAYS = 719162; +constexpr std::int64_t WIN32_EPOCH_DAYS = 584388; +constexpr std::uint64_t UNIX_EPOCH_TICK = UNIX_EPOCH_DAYS * TICKS_PER_DAY; +constexpr std::uint64_t WIN32_EPOCH_TICK = WIN32_EPOCH_DAYS * TICKS_PER_DAY; +constexpr std::uint64_t MAX_TICK = 3652059ull * TICKS_PER_DAY - 1; + +bool isLeapYear(std::int64_t year) { + return (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; +} + +int daysInMonth(std::int64_t year, int month) { + static constexpr int days[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + return month == 2 && isLeapYear(year) ? 29 : days[month - 1]; +} + +std::int64_t daysFromCivil(std::int64_t year, unsigned month, unsigned day) { + year -= month <= 2; + const std::int64_t era = (year >= 0 ? year : year - 399) / 400; + const auto yearOfEra = static_cast(year - era * 400); + const unsigned dayOfYear = (153 * (month > 2 ? month - 3 : month + 9) + 2) / 5 + day - 1; + const unsigned dayOfEra = yearOfEra * 365 + yearOfEra / 4 - yearOfEra / 100 + dayOfYear; + return era * 146097 + static_cast(dayOfEra) - 719468 + UNIX_EPOCH_DAYS; +} + +void civilFromDays(std::int64_t days, std::int64_t& year, unsigned& month, unsigned& day) { + days -= UNIX_EPOCH_DAYS - 719468; + const std::int64_t era = (days >= 0 ? days : days - 146096) / 146097; + const auto dayOfEra = static_cast(days - era * 146097); + const unsigned yearOfEra = (dayOfEra - dayOfEra / 1460 + dayOfEra / 36524 - dayOfEra / 146096) / 365; + const unsigned dayOfYear = dayOfEra - (365 * yearOfEra + yearOfEra / 4 - yearOfEra / 100); + const unsigned shiftedMonth = (5 * dayOfYear + 2) / 153; + day = dayOfYear - (153 * shiftedMonth + 2) / 5 + 1; + month = shiftedMonth < 10 ? shiftedMonth + 3 : shiftedMonth - 9; + year = static_cast(yearOfEra) + era * 400 + (month <= 2); +} + +int validate(const RtcDateTime* time) { + if (!time) return SCE_RTC_ERROR_INVALID_POINTER; + if (time->year < 1 || time->year > 9999) return SCE_RTC_ERROR_INVALID_YEAR; + if (time->month < 1 || time->month > 12) return SCE_RTC_ERROR_INVALID_MONTH; + if (time->day < 1 || time->day > daysInMonth(time->year, time->month)) return SCE_RTC_ERROR_INVALID_DAY; + if (time->hour > 23) return SCE_RTC_ERROR_INVALID_HOUR; + if (time->minute > 59) return SCE_RTC_ERROR_INVALID_MINUTE; + if (time->second > 59) return SCE_RTC_ERROR_INVALID_SECOND; + if (time->microsecond >= TICKS_PER_SECOND) return SCE_RTC_ERROR_INVALID_MICROSECOND; + return 0; +} + +std::uint64_t toTick(const RtcDateTime& time) { + return static_cast(daysFromCivil(time.year, time.month, time.day)) * TICKS_PER_DAY + + time.hour * TICKS_PER_HOUR + time.minute * TICKS_PER_MINUTE + time.second * TICKS_PER_SECOND + time.microsecond; +} + +RtcDateTime fromTick(std::uint64_t tick) { + std::int64_t year = 0; + unsigned month = 0; + unsigned day = 0; + civilFromDays(static_cast(tick / TICKS_PER_DAY), year, month, day); + const std::uint64_t timeOfDay = tick % TICKS_PER_DAY; + return RtcDateTime{ + static_cast(year), static_cast(month), static_cast(day), + static_cast(timeOfDay / TICKS_PER_HOUR), static_cast(timeOfDay % TICKS_PER_HOUR / TICKS_PER_MINUTE), + static_cast(timeOfDay % TICKS_PER_MINUTE / TICKS_PER_SECOND), static_cast(timeOfDay % TICKS_PER_SECOND)}; +} + +std::uint64_t currentTick() { + const auto now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + return UNIX_EPOCH_TICK + static_cast(now); +} + +std::int64_t localOffsetSeconds(std::uint64_t utcTick) { + const auto seconds = static_cast((static_cast(utcTick) - static_cast(UNIX_EPOCH_TICK)) / TICKS_PER_SECOND); + std::tm local{}; +#ifdef _WIN32 + if (localtime_s(&local, &seconds) != 0) return 0; +#else + if (!localtime_r(&seconds, &local)) return 0; +#endif + const std::int64_t localSeconds = (daysFromCivil(local.tm_year + 1900, static_cast(local.tm_mon + 1), static_cast(local.tm_mday)) - UNIX_EPOCH_DAYS) * 86400 + + local.tm_hour * 3600 + local.tm_min * 60 + local.tm_sec; + return localSeconds - static_cast(seconds); +} + +int addTicks(RtcTick* dst, const RtcTick* src, std::int64_t count, std::int64_t unit) { + if (!dst || !src) return SCE_RTC_ERROR_INVALID_POINTER; + if (count != 0 && (count > std::numeric_limits::max() / unit || count < std::numeric_limits::min() / unit)) return SCE_RTC_ERROR_INVALID_VALUE; + const std::int64_t delta = count * unit; + if (delta < 0 ? src->tick < static_cast(-(delta + 1)) + 1 : MAX_TICK - src->tick < static_cast(delta)) return SCE_RTC_ERROR_INVALID_VALUE; + dst->tick = src->tick + static_cast(delta); + return 0; +} + +int addMonths(RtcTick* dst, const RtcTick* src, std::int64_t months) { + if (!dst || !src) return SCE_RTC_ERROR_INVALID_POINTER; + RtcDateTime time = fromTick(src->tick); + const std::int64_t monthIndex = time.year * 12 + (time.month - 1) + months; + const std::int64_t year = monthIndex / 12; + if (monthIndex < 12 || year > 9999) return SCE_RTC_ERROR_INVALID_VALUE; + time.year = static_cast(year); + time.month = static_cast(monthIndex % 12 + 1); + const int lastDay = daysInMonth(time.year, time.month); + if (time.day > lastDay) time.day = static_cast(lastDay); + dst->tick = toTick(time); + return 0; +} + +bool parseDigits(const char*& cursor, int count, int& value) { + value = 0; + for (int index = 0; index < count; ++index, ++cursor) { + if (*cursor < '0' || *cursor > '9') return false; + value = value * 10 + (*cursor - '0'); + } + return true; +} + +} + extern "C" { int APS5_VABI sceRtcCheckValid(const RtcDateTime* time) { - (void)time; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcConvertLocalTimeToUtc(const RtcTick* local_time, RtcTick* utc) { - (void)local_time; - (void)utc; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcConvertUtcToLocalTime(const RtcTick* utc, RtcTick* local_time) { - (void)utc; - (void)local_time; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcFormatRFC3339(char* date_time, const RtcTick* utc, int time_zone_minutes) { - (void)date_time; - (void)utc; - (void)time_zone_minutes; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetCurrentClock(RtcDateTime* time, int time_zone_minutes) { - (void)time; - (void)time_zone_minutes; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetCurrentClockLocalTime(RtcDateTime* time) { - (void)time; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetCurrentNetworkTick(RtcTick* tick) { - (void)tick; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetCurrentTick(RtcTick* tick) { - (void)tick; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetDayOfWeek(int year, int month, int day) { - (void)year; - (void)month; - (void)day; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetTick(const RtcDateTime* time, RtcTick* tick) { - (void)time; - (void)tick; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetTickResolution(void) { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetTime_t(const RtcDateTime* time, int64_t* seconds) { - (void)time; - (void)seconds; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcGetWin32FileTime(const RtcDateTime* time, uint64_t* win32_time) { - (void)time; - (void)win32_time; - NotImplemented_nid_no_patch(__func__); - return 0; + return validate(time); } int APS5_VABI sceRtcIsLeapYear(int year) { - (void)year; - NotImplemented_nid_no_patch(__func__); - return 0; + if (year < 1) return SCE_RTC_ERROR_INVALID_YEAR; + return isLeapYear(year) ? 1 : 0; } -int APS5_VABI sceRtcParseRFC3339(RtcTick* utc, const char* date_time) { - (void)utc; - (void)date_time; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceRtcGetDaysInMonth(int year, int month) { + if (year < 1) return SCE_RTC_ERROR_INVALID_YEAR; + if (month < 1 || month > 12) return SCE_RTC_ERROR_INVALID_MONTH; + return daysInMonth(year, month); +} + +int APS5_VABI sceRtcGetDayOfWeek(int year, int month, int day) { + if (year < 1 || year > 9999) return SCE_RTC_ERROR_INVALID_YEAR; + if (month < 1 || month > 12) return SCE_RTC_ERROR_INVALID_MONTH; + if (day < 1 || day > daysInMonth(year, month)) return SCE_RTC_ERROR_INVALID_DAY; + return static_cast((daysFromCivil(year, static_cast(month), static_cast(day)) + 1) % 7); +} + +int APS5_VABI sceRtcGetTickResolution(void) { + return static_cast(TICKS_PER_SECOND); +} + +int APS5_VABI sceRtcGetTick(const RtcDateTime* time, RtcTick* tick) { + if (!tick) return SCE_RTC_ERROR_INVALID_POINTER; + if (const int result = validate(time); result != 0) return result; + tick->tick = toTick(*time); + return 0; } int APS5_VABI sceRtcSetTick(RtcDateTime* time, const RtcTick* tick) { - (void)time; - (void)tick; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!time || !tick) return SCE_RTC_ERROR_INVALID_POINTER; + if (tick->tick > MAX_TICK) return SCE_RTC_ERROR_INVALID_VALUE; + *time = fromTick(tick->tick); + return 0; +} + +int APS5_VABI sceRtcGetCurrentTick(RtcTick* tick) { + if (!tick) return SCE_RTC_ERROR_INVALID_POINTER; + tick->tick = currentTick(); + return 0; +} + +int APS5_VABI sceRtcGetCurrentNetworkTick(RtcTick* tick) { + return sceRtcGetCurrentTick(tick); +} + +int APS5_VABI sceRtcGetCurrentClock(RtcDateTime* time, int time_zone_minutes) { + if (!time) return SCE_RTC_ERROR_INVALID_POINTER; + RtcTick tick{currentTick()}; + if (const int result = addTicks(&tick, &tick, time_zone_minutes, TICKS_PER_MINUTE); result != 0) return result; + *time = fromTick(tick.tick); + return 0; +} + +int APS5_VABI sceRtcGetCurrentClockLocalTime(RtcDateTime* time) { + if (!time) return SCE_RTC_ERROR_INVALID_POINTER; + const std::uint64_t utc = currentTick(); + *time = fromTick(utc + static_cast(localOffsetSeconds(utc) * TICKS_PER_SECOND)); + return 0; +} + +int APS5_VABI sceRtcConvertUtcToLocalTime(const RtcTick* utc, RtcTick* local_time) { + if (!utc || !local_time) return SCE_RTC_ERROR_INVALID_POINTER; + return addTicks(local_time, utc, localOffsetSeconds(utc->tick), TICKS_PER_SECOND); +} + +int APS5_VABI sceRtcConvertLocalTimeToUtc(const RtcTick* local_time, RtcTick* utc) { + if (!local_time || !utc) return SCE_RTC_ERROR_INVALID_POINTER; + const std::int64_t offset = localOffsetSeconds(local_time->tick - static_cast(localOffsetSeconds(local_time->tick) * TICKS_PER_SECOND)); + return addTicks(utc, local_time, -offset, TICKS_PER_SECOND); +} + +int APS5_VABI sceRtcGetTime_t(const RtcDateTime* time, int64_t* seconds) { + if (!seconds) return SCE_RTC_ERROR_INVALID_POINTER; + if (const int result = validate(time); result != 0) return result; + const std::uint64_t tick = toTick(*time); + if (tick < UNIX_EPOCH_TICK) return SCE_RTC_ERROR_INVALID_VALUE; + *seconds = static_cast((tick - UNIX_EPOCH_TICK) / TICKS_PER_SECOND); + return 0; } int APS5_VABI sceRtcSetTime_t(RtcDateTime* time, int64_t seconds) { - (void)time; - (void)seconds; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!time) return SCE_RTC_ERROR_INVALID_POINTER; + if (seconds < 0 || static_cast(seconds) > (MAX_TICK - UNIX_EPOCH_TICK) / TICKS_PER_SECOND) return SCE_RTC_ERROR_INVALID_VALUE; + *time = fromTick(UNIX_EPOCH_TICK + static_cast(seconds) * TICKS_PER_SECOND); + return 0; +} + +int APS5_VABI sceRtcGetWin32FileTime(const RtcDateTime* time, uint64_t* win32_time) { + if (!win32_time) return SCE_RTC_ERROR_INVALID_POINTER; + if (const int result = validate(time); result != 0) return result; + const std::uint64_t tick = toTick(*time); + if (tick < WIN32_EPOCH_TICK) return SCE_RTC_ERROR_INVALID_VALUE; + *win32_time = (tick - WIN32_EPOCH_TICK) * 10; + return 0; } int APS5_VABI sceRtcSetWin32FileTime(RtcDateTime* time, uint64_t win32_time) { - (void)time; - (void)win32_time; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!time) return SCE_RTC_ERROR_INVALID_POINTER; + if (win32_time / 10 > MAX_TICK - WIN32_EPOCH_TICK) return SCE_RTC_ERROR_INVALID_VALUE; + *time = fromTick(WIN32_EPOCH_TICK + win32_time / 10); + return 0; } -int APS5_VABI sceRtcTickAddDays(RtcTick* dst, const RtcTick* src, int32_t days) { - (void)dst; - (void)src; - (void)days; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceRtcFormatRFC3339(char* date_time, const RtcTick* utc, int time_zone_minutes) { + if (!date_time || !utc) return SCE_RTC_ERROR_INVALID_POINTER; + RtcTick local{}; + if (const int result = addTicks(&local, utc, time_zone_minutes, TICKS_PER_MINUTE); result != 0) return result; + const RtcDateTime time = fromTick(local.tick); + int written = std::snprintf(date_time, 23, "%04u-%02u-%02uT%02u:%02u:%02u.%02u", time.year, time.month, time.day, time.hour, time.minute, time.second, time.microsecond / 10000); + if (time_zone_minutes == 0) { + std::snprintf(date_time + written, 2, "Z"); + } else { + const int offset = time_zone_minutes < 0 ? -time_zone_minutes : time_zone_minutes; + std::snprintf(date_time + written, 7, "%c%02d:%02d", time_zone_minutes < 0 ? '-' : '+', offset / 60, offset % 60); + } + return 0; } -int APS5_VABI sceRtcTickAddHours(RtcTick* dst, const RtcTick* src, int32_t hours) { - (void)dst; - (void)src; - (void)hours; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcTickAddMicroseconds(RtcTick* dst, const RtcTick* src, int64_t usec) { - (void)dst; - (void)src; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcTickAddMinutes(RtcTick* dst, const RtcTick* src, int64_t minutes) { - (void)dst; - (void)src; - (void)minutes; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceRtcTickAddSeconds(RtcTick* dst, const RtcTick* src, int64_t seconds) { - (void)dst; - (void)src; - (void)seconds; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceRtcParseRFC3339(RtcTick* utc, const char* date_time) { + if (!utc || !date_time) return SCE_RTC_ERROR_INVALID_POINTER; + const char* cursor = date_time; + int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; + if (!parseDigits(cursor, 4, year) || *cursor++ != '-' || !parseDigits(cursor, 2, month) || *cursor++ != '-' || !parseDigits(cursor, 2, day)) return SCE_RTC_ERROR_BAD_PARSE; + if (*cursor != 'T' && *cursor != 't' && *cursor != ' ') return SCE_RTC_ERROR_BAD_PARSE; + ++cursor; + if (!parseDigits(cursor, 2, hour) || *cursor++ != ':' || !parseDigits(cursor, 2, minute) || *cursor++ != ':' || !parseDigits(cursor, 2, second)) return SCE_RTC_ERROR_BAD_PARSE; + std::uint32_t microsecond = 0; + if (*cursor == '.') { + ++cursor; + std::uint32_t scale = 100000; + if (*cursor < '0' || *cursor > '9') return SCE_RTC_ERROR_BAD_PARSE; + for (; *cursor >= '0' && *cursor <= '9'; ++cursor, scale /= 10) microsecond += static_cast(*cursor - '0') * scale; + } + std::int64_t offsetMinutes = 0; + if (*cursor == 'Z' || *cursor == 'z') { + ++cursor; + } else if (*cursor == '+' || *cursor == '-') { + const int sign = *cursor++ == '-' ? -1 : 1; + int offsetHours = 0, offsetMinute = 0; + if (!parseDigits(cursor, 2, offsetHours) || *cursor++ != ':' || !parseDigits(cursor, 2, offsetMinute)) return SCE_RTC_ERROR_BAD_PARSE; + offsetMinutes = sign * (offsetHours * 60 + offsetMinute); + } else { + return SCE_RTC_ERROR_BAD_PARSE; + } + if (*cursor != 0) return SCE_RTC_ERROR_BAD_PARSE; + const RtcDateTime time{static_cast(year), static_cast(month), static_cast(day), + static_cast(hour), static_cast(minute), static_cast(second), microsecond}; + if (const int result = validate(&time); result != 0) return result; + const RtcTick local{toTick(time)}; + return addTicks(utc, &local, -offsetMinutes, TICKS_PER_MINUTE); } int APS5_VABI sceRtcTickAddTicks(RtcTick* dst, const RtcTick* src, int64_t ticks) { - (void)dst; - (void)src; - (void)ticks; - NotImplemented_nid_no_patch(__func__); - return 0; + return addTicks(dst, src, ticks, 1); +} + +int APS5_VABI sceRtcTickAddMicroseconds(RtcTick* dst, const RtcTick* src, int64_t usec) { + return addTicks(dst, src, usec, 1); +} + +int APS5_VABI sceRtcTickAddSeconds(RtcTick* dst, const RtcTick* src, int64_t seconds) { + return addTicks(dst, src, seconds, TICKS_PER_SECOND); +} + +int APS5_VABI sceRtcTickAddMinutes(RtcTick* dst, const RtcTick* src, int64_t minutes) { + return addTicks(dst, src, minutes, TICKS_PER_MINUTE); +} + +int APS5_VABI sceRtcTickAddHours(RtcTick* dst, const RtcTick* src, int32_t hours) { + return addTicks(dst, src, hours, TICKS_PER_HOUR); +} + +int APS5_VABI sceRtcTickAddDays(RtcTick* dst, const RtcTick* src, int32_t days) { + return addTicks(dst, src, days, TICKS_PER_DAY); } int APS5_VABI sceRtcTickAddWeeks(RtcTick* dst, const RtcTick* src, int32_t weeks) { - (void)dst; - (void)src; - (void)weeks; - NotImplemented_nid_no_patch(__func__); - return 0; + return addTicks(dst, src, weeks, 7 * TICKS_PER_DAY); +} + +int APS5_VABI sceRtcTickAddMonths(RtcTick* dst, const RtcTick* src, int32_t months) { + return addMonths(dst, src, months); +} + +int APS5_VABI sceRtcTickAddYears(RtcTick* dst, const RtcTick* src, int16_t years) { + return addMonths(dst, src, static_cast(years) * 12); } } diff --git a/core/libs/prx/libSceVideoOut/src/PadInput.cpp b/core/libs/prx/libSceVideoOut/src/PadInput.cpp index 66fde9da7d..6631fb5a93 100644 --- a/core/libs/prx/libSceVideoOut/src/PadInput.cpp +++ b/core/libs/prx/libSceVideoOut/src/PadInput.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -27,6 +28,10 @@ void PadInput::HandleEvent(const SDL_Event& event, DisplayWindow& window) { publish(); return; } + // APS5_NO_PAD_INPUT=1 keeps a measurement run from reacting to keys or mouse buttons that reach + // its window (a stray press advances the title into another stage). + static const bool ignoreInput = std::getenv("APS5_NO_PAD_INPUT") != nullptr; + if (ignoreInput && event.type == SDL_MOUSEWHEEL) return; if (event.type == SDL_MOUSEWHEEL) { int direction = (event.wheel.y > 0) - (event.wheel.y < 0); if (event.wheel.direction == SDL_MOUSEWHEEL_FLIPPED) direction = -direction; @@ -44,6 +49,7 @@ void PadInput::HandleEvent(const SDL_Event& event, DisplayWindow& window) { const bool keyboard = event.type == SDL_KEYDOWN || event.type == SDL_KEYUP; const bool mouse = event.type == SDL_MOUSEBUTTONDOWN || event.type == SDL_MOUSEBUTTONUP; if (!keyboard && !mouse) return; + if (ignoreInput) return; if (keyboard && event.key.repeat != 0) return; const bool down = event.type == SDL_KEYDOWN || event.type == SDL_MOUSEBUTTONDOWN; for (std::size_t index = 0; index < Pad::InputMapping.size(); ++index) { diff --git a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp index e892c5828e..c0036f286c 100644 --- a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp +++ b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include @@ -42,6 +44,16 @@ public: request->outputHandle = info.handle; request->flipMode = static_cast(info.mode); request->flipArg = info.argument; + // Debug aid: APS5_FLIP_QUEUE_WAIT=1 waits for room instead of failing the flip, so a + // presenter stall keeps the process alive for a debugger (reported every 5 s). + static const bool waitWhenFull = std::getenv("APS5_FLIP_QUEUE_WAIT") != nullptr; + if (waitWhenFull) { + int waited = 0; + while (queue->reservations.load() >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + if (++waited % 500 == 0) std::fprintf(stderr, "[flip] queue full for %d s; waiting (APS5_FLIP_QUEUE_WAIT)\n", waited / 100); + } + } std::lock_guard queueLock(queue->mutex); if (queue->failure) std::rethrow_exception(queue->failure); require(!queue->stopping, "flip during shutdown"); @@ -123,8 +135,10 @@ void FlipRequest::GpuReady(const std::shared_ptr& frameT queue->changed.notify_all(); // The worker is done once the request is queued: hardware does not stall the command processor // on a flip, and the title observes completion through flipPendingNum, which processFlip drops - // after presenting. A presentation failure reaches the worker through ReportFailure at its next - // packet. Debug aid: APS5_SYNC_FLIP=1 parks the worker until the presenter is done, as before. + // once the presentation is queued behind the frame's GPU work (up to APS5_FLIP_INFLIGHT + // presentations may still be executing then, see Driver::Present). A presentation failure + // reaches the worker through ReportFailure at its next packet. Debug aid: APS5_SYNC_FLIP=1 + // parks the worker until the presenter is done, as before. static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; if (!syncFlip) return; std::unique_lock lock(cfg->mutex); diff --git a/core/libs/prx/libSceVideoOut/tests/Flip.cpp b/core/libs/prx/libSceVideoOut/tests/Flip.cpp index e67aa6a137..99af680448 100644 --- a/core/libs/prx/libSceVideoOut/tests/Flip.cpp +++ b/core/libs/prx/libSceVideoOut/tests/Flip.cpp @@ -238,6 +238,34 @@ void testPresentation(bool expectUnavailable) { LibcRunShutdown_nid_postfix(); } +// Flips of two buffers submitted without waiting for each other: every one completes in order +// (presentations trail on the GPU, APS5_FLIP_INFLIGHT), and the shutdown retires the ones in flight. +void testBackToBack() { + const int handle = sceVideoOutOpen(255, 0, 0, nullptr); + auto cfg = VideoOutDriver::Get().GetConfig(handle); + std::array, 2> allocations{std::vector(6 * 65536 + 65535), std::vector(6 * 65536 + 65535)}; + std::array buffers{}; + for (std::size_t i = 0; i < buffers.size(); ++i) { + const auto storage = alignedBuffer(allocations[i]); + fillBuffer(storage, 259, 137); + buffers[i] = {storage.data(), nullptr, {nullptr, nullptr}}; + } + VideoOutBufferAttribute2 attribute{}; + sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 259, 137, 0, 0, 0); + sceVideoOutRegisterBuffers2(handle, 0, 0, buffers.data(), 2, &attribute, 0, nullptr); + int64_t argument = 1000; + for (int index : {0, 1, 0, 1}) sceVideoOutSubmitFlip(handle, index, 1, ++argument); + std::unique_lock lock(cfg->mutex); + check(cfg->vblankCond.wait_for(lock, std::chrono::seconds(15), [&] { return cfg->failure || cfg->flipStatus.count == 4; }), "back-to-back presentations did not complete"); + if (cfg->failure) std::rethrow_exception(cfg->failure); + check(cfg->flipStatus.flipPendingNum == 0 && cfg->bufferPending[0] == 0 && cfg->bufferPending[1] == 0, "back-to-back flips left pending counts"); + check(cfg->flipStatus.flipArg == argument && cfg->flipStatus.currentBuffer == 1, "back-to-back flip status is wrong"); + lock.unlock(); + sceVideoOutUnregisterBuffers(handle, 0); + sceVideoOutClose(handle); + LibcRunShutdown_nid_postfix(); +} + } int main(int argc, char** argv) { @@ -245,6 +273,7 @@ int main(int argc, char** argv) { if (argc == 2 && std::string(argv[1]) == "decode") testDecode(); else if (argc == 2 && std::string(argv[1]) == "controls") testControls(); else if (argc == 2 && std::string(argv[1]) == "present") testPresentation(false); + else if (argc == 2 && std::string(argv[1]) == "backtoback") testBackToBack(); else if (argc == 2 && std::string(argv[1]) == "unavailable") testPresentation(true); else testLifetime(argc == 2 && std::string(argv[1]) == "reopen"); std::puts("VideoOut flip tests passed"); diff --git a/core/libs/prx/libc/include/CpuTopology.hpp b/core/libs/prx/libc/include/CpuTopology.hpp new file mode 100644 index 0000000000..3150ed4002 --- /dev/null +++ b/core/libs/prx/libc/include/CpuTopology.hpp @@ -0,0 +1,119 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_CPUTOPOLOGY_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_CPUTOPOLOGY_HPP + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +// Hybrid CPU layout (Intel P/E cores) from the OS: which logical processors belong to the lowest and +// the highest efficiency class. Non-hybrid machines report no layout and every pin is a no-op unless +// a mask is given explicitly. Header-only so libkernel and the AGC driver share it without a link. +namespace CpuTopology { + +struct Layout { + std::uint64_t process = 0; + std::uint64_t efficient = 0; + std::uint64_t performant = 0; + bool hybrid = false; +}; + +inline const Layout& Get() { + static const Layout layout = [] { + Layout result{}; +#ifdef _WIN32 + DWORD_PTR processMask = 0; + DWORD_PTR systemMask = 0; + if (GetProcessAffinityMask(GetCurrentProcess(), &processMask, &systemMask)) result.process = static_cast(processMask); + DWORD length = 0; + GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &length); + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER || length == 0) return result; + std::vector buffer(length); + if (!GetLogicalProcessorInformationEx(RelationProcessorCore, reinterpret_cast(buffer.data()), &length)) return result; + std::uint64_t byClass[256] = {}; + bool seen[256] = {}; + for (DWORD offset = 0; offset + 8 <= length;) { + const auto* entry = reinterpret_cast(buffer.data() + offset); + if (entry->Size == 0 || offset + entry->Size > length) break; + if (entry->Relationship == RelationProcessorCore && entry->Processor.GroupCount >= 1 && entry->Processor.GroupMask[0].Group == 0) { + const unsigned cls = entry->Processor.EfficiencyClass; + byClass[cls] |= static_cast(entry->Processor.GroupMask[0].Mask); + seen[cls] = true; + } + offset += entry->Size; + } + int lowest = -1; + int highest = -1; + for (int cls = 0; cls < 256; ++cls) { + if (!seen[cls]) continue; + if (lowest < 0) lowest = cls; + highest = cls; + } + if (lowest < 0 || lowest == highest) return result; + result.efficient = byClass[lowest]; + result.performant = byClass[highest]; + result.hybrid = result.efficient != 0 && result.performant != 0; +#endif + return result; + }(); + return layout; +} + +// A hex mask from the environment, 0 when unset or empty. +inline std::uint64_t MaskFromEnvironment(const char* name) { + const char* text = std::getenv(name); + if (text == nullptr || *text == '\0') return 0; + return std::strtoull(text, nullptr, 16); +} + +inline bool Trace() { + static const bool trace = std::getenv("APS5_TRACE_AFFINITY") != nullptr; + return trace; +} + +inline unsigned long ThreadId(void* handle) { +#ifdef _WIN32 + return handle != nullptr ? GetThreadId(static_cast(handle)) : GetCurrentThreadId(); +#else + (void)handle; + return 0; +#endif +} + +// Restricts the thread (`nullptr` is the calling thread) to `mask` clipped to the process affinity; +// returns the mask applied, 0 when nothing was. +inline std::uint64_t Pin(void* handle, std::uint64_t mask) { +#ifdef _WIN32 + const auto& layout = Get(); + if (layout.process != 0) mask &= layout.process; + if (mask == 0) return 0; + HANDLE thread = handle != nullptr ? static_cast(handle) : GetCurrentThread(); + return SetThreadAffinityMask(thread, static_cast(mask)) != 0 ? mask : 0; +#else + (void)handle; + (void)mask; + return 0; +#endif +} + +// Pin plus the APS5_TRACE_AFFINITY=1 line naming the thread. +inline std::uint64_t PinTraced(const char* role, void* handle, std::uint64_t mask) { + const auto applied = Pin(handle, mask); + if (Trace()) std::fprintf(stderr, "[affinity] %s tid=%lu mask=0x%llx%s\n", role, ThreadId(handle), static_cast(mask), applied != 0 ? "" : " (not applied)"); + return applied; +} + +// Helper threads (deferred release, mirror refresh) are pinned only when APS5_HELPER_AFFINITY_MASK +// names their cores. +inline void PinHelperThread(const char* role) { + static const std::uint64_t mask = MaskFromEnvironment("APS5_HELPER_AFFINITY_MASK"); + if (mask != 0) PinTraced(role, nullptr, mask); +} + +} + +#endif diff --git a/core/libs/prx/libc/src/ApplicationHeap.cpp b/core/libs/prx/libc/src/ApplicationHeap.cpp index ffd62cc5a6..81e17250dd 100644 --- a/core/libs/prx/libc/src/ApplicationHeap.cpp +++ b/core/libs/prx/libc/src/ApplicationHeap.cpp @@ -142,7 +142,7 @@ void* ApplicationHeapReallocate_nid_no_patch(void* pointer, std::size_t bytes) { ApplicationHeapFree_nid_no_patch(pointer); return nullptr; } - const auto reallocate = callback(2); + const auto reallocate = callback(3); CallbackScope scope; return requireAllocation(reallocate(pointer, bytes)); } @@ -158,7 +158,7 @@ void* ApplicationHeapAlign_nid_no_patch(std::size_t alignment, std::size_t bytes void* ApplicationHeapCalloc_nid_no_patch(std::size_t count, std::size_t bytes) { if (bytes != 0 && count > std::numeric_limits::max() / bytes) throw std::length_error("application heap: calloc size overflow"); - const auto calloc = callback(3); + const auto calloc = callback(2); CallbackScope scope; return requireAllocation(calloc(count, bytes)); } diff --git a/core/libs/prx/libc/src/exception/Runtime.cpp b/core/libs/prx/libc/src/exception/Runtime.cpp index c26045b6b0..c04c5a1cf6 100644 --- a/core/libs/prx/libc/src/exception/Runtime.cpp +++ b/core/libs/prx/libc/src/exception/Runtime.cpp @@ -31,17 +31,18 @@ static void RawLog(const char* format, ...) { Header* primary = Primary(globals.caught); const char* typeName = primary->type ? primary->type->name() : nullptr; const char* what = nullptr; - if (primary->adjusted) { + void* exception = primary->adjusted; + if (exception && primary->type && Match(&typeid(std::exception), primary->type, exception)) { struct VtableLayout { std::ptrdiff_t offset; const void* type; void (*destroy)(void*); void (*del)(void*); const char* (*whatFn)(const void*); }; - const void* vtable = *static_cast(primary->adjusted); + const void* vtable = *static_cast(exception); auto* layout = reinterpret_cast(static_cast(vtable) - offsetof(VtableLayout, destroy)); #ifdef _WIN32 if (!primary->_pad) { using GuestWhat = const char* (__attribute__((sysv_abi)) *)(const void*); - what = reinterpret_cast(layout->whatFn)(primary->adjusted); + what = reinterpret_cast(layout->whatFn)(exception); } else #endif - what = layout->whatFn(primary->adjusted); + what = layout->whatFn(exception); } int status = 0; char* demangled = abi::__cxa_demangle(typeName, nullptr, nullptr, &status); diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index cf688c1547..fc0208d6b3 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -35,6 +35,7 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Rtld.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Process.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Time/Time.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Time/TimedWait.cpp ${CMAKE_CURRENT_SOURCE_DIR}/AppMetadata/src/AppMetadata.cpp ${CMAKE_CURRENT_SOURCE_DIR}/AppMetadata/src/ParamJsonParser.cpp ) diff --git a/core/libs/prx/libkernel/Equeue/Equeue.cpp b/core/libs/prx/libkernel/Equeue/Equeue.cpp index 4ccab694a1..ece17ea7af 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.cpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.cpp @@ -41,7 +41,7 @@ void KernelEqueuePrivate::Close() { } } m_events.clear(); - m_cond.notify_all(); + m_cond.NotifyAll(); } void KernelEqueuePrivate::TriggerExpiredTimers(uint64_t nowNs) { @@ -113,8 +113,7 @@ int KernelEqueuePrivate::WaitForEvents(KernelEvent* ev, int num, uint32_t micros if (m_closed) { return EQUEUE_ERROR_EBADF; } - const auto deadline = std::chrono::steady_clock::now() + - std::chrono::microseconds(micros); + const std::uint64_t deadline = TimedWait::DeadlineNanos(micros); for (;;) { TriggerExpiredTimers(MonotonicNs()); int ret = 0; @@ -152,21 +151,16 @@ int KernelEqueuePrivate::WaitForEvents(KernelEvent* ev, int num, uint32_t micros return EQUEUE_ERROR_EBADF; } if (micros == 0) { - m_cond.wait(lock); + m_cond.Wait(lock); } else { uint32_t timerWait = 0; const bool hasTimer = NextTimerWaitMicros(MonotonicNs(), &timerWait); - const auto now = std::chrono::steady_clock::now(); + const std::uint64_t now = TimedWait::NowNanos(); if (now >= deadline) { return 0; } - const auto remaining = std::chrono::duration_cast( - deadline - now - ).count(); - const auto waitUs = hasTimer - ? std::min(static_cast(remaining), timerWait) - : static_cast(remaining); - m_cond.wait_for(lock, std::chrono::microseconds(waitUs)); + const std::uint64_t timerDeadline = now + static_cast(timerWait) * 1000ULL; + m_cond.WaitUntil(lock, hasTimer ? std::min(deadline, timerDeadline) : deadline); } } } @@ -190,7 +184,7 @@ int KernelEqueuePrivate::AddEvent(const KernelEqueueEvent& event) { } else { m_events.push_back(event); } - m_cond.notify_one(); + m_cond.NotifyOne(); return EQUEUE_OK; } @@ -212,7 +206,7 @@ int KernelEqueuePrivate::TriggerEvent(uintptr_t ident, int16_t filter, void* tri } else { it->triggered = true; } - m_cond.notify_one(); + m_cond.NotifyOne(); return EQUEUE_OK; } diff --git a/core/libs/prx/libkernel/Equeue/Equeue.hpp b/core/libs/prx/libkernel/Equeue/Equeue.hpp index 0082a1eddb..cae4073442 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.hpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.hpp @@ -1,7 +1,6 @@ #ifndef CORE_LIBS_PRX_LIBKERNEL_EQUEUE_EQUEUE_HPP #define CORE_LIBS_PRX_LIBKERNEL_EQUEUE_EQUEUE_HPP -#include #include #include #include @@ -12,6 +11,7 @@ #include "SceTypes.hpp" #include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" static constexpr int16_t EVFILT_USER = -11; static constexpr int16_t EVFILT_VIDEO_OUT = -13; @@ -76,7 +76,7 @@ private: std::list m_events; std::mutex m_mutex; - std::condition_variable_any m_cond; + TimedWait::Condition m_cond; std::string m_name; KernelEqueue m_handle; bool m_closed = false; diff --git a/core/libs/prx/libkernel/EventFlag/src/EventFlag.cpp b/core/libs/prx/libkernel/EventFlag/src/EventFlag.cpp index 9706587659..7d4ebe2ea3 100644 --- a/core/libs/prx/libkernel/EventFlag/src/EventFlag.cpp +++ b/core/libs/prx/libkernel/EventFlag/src/EventFlag.cpp @@ -3,8 +3,8 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include "prx/libkernel/Time/include/Time.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" #include -#include #include #include #include @@ -29,7 +29,7 @@ static constexpr uint32_t EVF_WAITMODE_CLEAR_PAT = 0x20; struct KernelEventFlagPrivate { std::mutex _mutex; - std::condition_variable _changed; + TimedWait::Condition _changed; std::string _name; uint64_t _pattern = 0; bool _multipleWaiters = false; @@ -83,7 +83,7 @@ int APS5_VABI sceKernelDeleteEventFlag(KernelEventFlag ef) { if (ef->_deleted) return SCE_KERNEL_ERROR_ESRCH; ef->_deleted = true; if (ef->_waiters != 0) { - ef->_changed.notify_all(); + ef->_changed.NotifyAll(); return SCE_OK; } } @@ -97,7 +97,7 @@ int APS5_VABI sceKernelSetEventFlag(KernelEventFlag ef, uint64_t bit_pattern) { if (ef->_deleted) return SCE_KERNEL_ERROR_ESRCH; ef->_pattern |= bit_pattern; if (TraceSync()) std::fprintf(stderr, "[evf] set %p '%s' |=0x%llx -> 0x%llx\n", static_cast(ef), ef->_name.c_str(), static_cast(bit_pattern), static_cast(ef->_pattern)); - ef->_changed.notify_all(); + ef->_changed.NotifyAll(); return SCE_OK; } @@ -116,7 +116,7 @@ int APS5_VABI sceKernelCancelEventFlag(KernelEventFlag ef, uint64_t set_pattern, if (num_wait_threads) *num_wait_threads = ef->_waiters; ef->_pattern = set_pattern; ++ef->_cancelGeneration; - ef->_changed.notify_all(); + ef->_changed.NotifyAll(); return SCE_OK; } @@ -147,12 +147,11 @@ int APS5_VABI sceKernelWaitEventFlag(KernelEventFlag ef, uint64_t bit_pattern, u bool timedOut = false; const auto waitStart = std::chrono::steady_clock::now(); if (timeout) { - const auto deadline = waitStart + std::chrono::microseconds(*timeout); - timedOut = !ef->_changed.wait_until(lock, deadline, released); - const auto remaining = std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()).count(); - *timeout = remaining > 0 ? static_cast(remaining) : 0; + const std::uint64_t deadline = TimedWait::DeadlineNanos(*timeout); + timedOut = !ef->_changed.WaitUntil(lock, deadline, released); + *timeout = static_cast(TimedWait::RemainingMicros(deadline)); } else { - ef->_changed.wait(lock, released); + ef->_changed.Wait(lock, released); } KernelTraceWait_nid_postfix("evf", __builtin_return_address(0), static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), timedOut); --ef->_waiters; diff --git a/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp b/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp index bb76c83d40..8bbe3735e7 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp @@ -3,16 +3,16 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include "Common.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" #include #include -#include #include #include // The guest sem_t is at least 8 bytes; its first word holds a pointer to the host semaphore. struct PosixSemaphore { std::mutex lock; - std::condition_variable available; + TimedWait::Condition available; unsigned int count; }; @@ -35,9 +35,9 @@ static int WaitFor(PosixSemaphore* semaphore, const KernelUseconds* usec) { std::unique_lock lock(semaphore->lock); const auto ready = [&] { return semaphore->count > 0; }; if (usec) { - if (!semaphore->available.wait_for(lock, std::chrono::microseconds(*usec), ready)) return Fail(PosixThread::GUEST_ETIMEDOUT); + if (!semaphore->available.WaitUntil(lock, TimedWait::DeadlineNanos(*usec), ready)) return Fail(PosixThread::GUEST_ETIMEDOUT); } else { - semaphore->available.wait(lock, ready); + semaphore->available.Wait(lock, ready); } --semaphore->count; return 0; @@ -99,7 +99,7 @@ int APS5_VABI sem_post_nid_postfix(void* sem) { std::lock_guard lock(semaphore->lock); ++semaphore->count; } - semaphore->available.notify_one(); + semaphore->available.NotifyOne(); return 0; } diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index c436f91fbc..c87009f13a 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -3,6 +3,7 @@ #include #include "SceTypes.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" #include #include #include @@ -46,7 +47,7 @@ struct PthreadCondattrPrivate { }; struct PthreadCondPrivate { - std::condition_variable_any _cv; + TimedWait::Condition _cv; int _clockid = 0; }; diff --git a/core/libs/prx/libkernel/Pthread/src/Cond.cpp b/core/libs/prx/libkernel/Pthread/src/Cond.cpp index 2f1a2267aa..23a7263dc4 100644 --- a/core/libs/prx/libkernel/Pthread/src/Cond.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Cond.cpp @@ -1,7 +1,9 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" #include "prx/libkernel/Time/include/Time.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" #include +#include #include #include #include @@ -10,7 +12,7 @@ static constexpr int SCE_OK = 0; static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = 0x8002003C; -static int WaitOn(PthreadCondPrivate* cond, PthreadMutexPrivate* mutex, std::optional timeout, const void* caller) { +static int WaitOn(PthreadCondPrivate* cond, PthreadMutexPrivate* mutex, std::optional deadlineNanos, const void* caller) { const auto self = std::this_thread::get_id(); bool timedOut = false; const auto waitStart = std::chrono::steady_clock::now(); @@ -21,13 +23,13 @@ static int WaitOn(PthreadCondPrivate* cond, PthreadMutexPrivate* mutex, std::opt mutex->_owner.store(std::thread::id{}, std::memory_order_relaxed); if (mutex->_type == MutexType::Recursive) { std::unique_lock lock(mutex->_rmtx, std::adopt_lock); - if (timeout) timedOut = cond->_cv.wait_for(lock, *timeout) == std::cv_status::timeout; - else cond->_cv.wait(lock); + if (deadlineNanos) timedOut = !cond->_cv.WaitUntil(lock, *deadlineNanos); + else cond->_cv.Wait(lock); lock.release(); } else { std::unique_lock lock(mutex->_mtx, std::adopt_lock); - if (timeout) timedOut = cond->_cv.wait_for(lock, *timeout) == std::cv_status::timeout; - else cond->_cv.wait(lock); + if (deadlineNanos) timedOut = !cond->_cv.WaitUntil(lock, *deadlineNanos); + else cond->_cv.Wait(lock); lock.release(); } mutex->_owner.store(self, std::memory_order_relaxed); @@ -69,26 +71,26 @@ int APS5_VABI scePthreadCondDestroy(PthreadCond* cond) { int APS5_VABI scePthreadCondSignal(PthreadCond* cond) { if (!cond || !*cond) throw std::runtime_error("scePthreadCondSignal: null cond"); - (*cond)->_cv.notify_one(); + (*cond)->_cv.NotifyOne(); return SCE_OK; } int APS5_VABI scePthreadCondBroadcast(PthreadCond* cond) { if (!cond || !*cond) throw std::runtime_error("scePthreadCondBroadcast: null cond"); - (*cond)->_cv.notify_all(); + (*cond)->_cv.NotifyAll(); return SCE_OK; } int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, unsigned int usec) { if (!cond || !*cond || !mutex || !*mutex) throw std::runtime_error("scePthreadCondTimedwait: null arg"); - return WaitOn(*cond, *mutex, std::chrono::microseconds(usec), __builtin_return_address(0)); + return WaitOn(*cond, *mutex, TimedWait::DeadlineNanos(usec), __builtin_return_address(0)); } int APS5_VABI scePthreadCondSignalto(PthreadCond* cond, Pthread thread) { (void)thread; if (!cond || !*cond) throw std::runtime_error("scePthreadCondSignalto: null cond"); - (*cond)->_cv.notify_all(); + (*cond)->_cv.NotifyAll(); return SCE_OK; } diff --git a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp index baf24a6a91..b88b85a30c 100644 --- a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp @@ -1,7 +1,9 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" #include #include +#include #include #include @@ -98,9 +100,10 @@ int APS5_VABI scePthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec) if (!mutex || !*mutex) throw std::runtime_error("scePthreadMutexTimedlock: null mutex"); auto* m = *mutex; const auto tid = std::this_thread::get_id(); - const auto timeout = std::chrono::microseconds(usec); + const std::uint64_t deadline = TimedWait::DeadlineNanos(usec); if (m->_type == MutexType::Recursive) { - if (!m->_rmtx.try_lock_for(timeout)) return SCE_KERNEL_ERROR_ETIMEDOUT; + const bool locked = TimedWait::AcquireUntil(deadline, [&] { return m->_rmtx.try_lock(); }, [&](std::uint64_t micros) { return m->_rmtx.try_lock_for(std::chrono::microseconds(micros)); }); + if (!locked) return SCE_KERNEL_ERROR_ETIMEDOUT; m->_owner.store(tid, std::memory_order_relaxed); ++m->_count; return SCE_OK; @@ -108,7 +111,8 @@ int APS5_VABI scePthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec) if (m->_type == MutexType::ErrorCheck) { if (m->_owner.load(std::memory_order_acquire) == tid) return SCE_KERNEL_ERROR_EDEADLK; } - if (!m->_mtx.try_lock_for(timeout)) return SCE_KERNEL_ERROR_ETIMEDOUT; + const bool locked = TimedWait::AcquireUntil(deadline, [&] { return m->_mtx.try_lock(); }, [&](std::uint64_t micros) { return m->_mtx.try_lock_for(std::chrono::microseconds(micros)); }); + if (!locked) return SCE_KERNEL_ERROR_ETIMEDOUT; m->_owner.store(tid, std::memory_order_relaxed); return SCE_OK; } diff --git a/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp b/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp index 424a889b75..9b4bb1713f 100644 --- a/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp @@ -1,6 +1,8 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" #include +#include #include static constexpr int SCE_OK = 0; @@ -51,7 +53,8 @@ int APS5_VABI scePthreadRwlockTryrdlock(PthreadRwlock* rwlock) { int APS5_VABI scePthreadRwlockTimedrdlock(PthreadRwlock* rwlock, KernelUseconds usec) { auto* lock = RequireRwlock(rwlock, __func__); if (OwnsWrite(lock)) return SCE_KERNEL_ERROR_EDEADLK; - return lock->_lock.try_lock_shared_for(std::chrono::microseconds(usec)) ? SCE_OK : SCE_KERNEL_ERROR_ETIMEDOUT; + const bool locked = TimedWait::AcquireUntil(TimedWait::DeadlineNanos(usec), [&] { return lock->_lock.try_lock_shared(); }, [&](std::uint64_t micros) { return lock->_lock.try_lock_shared_for(std::chrono::microseconds(micros)); }); + return locked ? SCE_OK : SCE_KERNEL_ERROR_ETIMEDOUT; } int APS5_VABI scePthreadRwlockWrlock(PthreadRwlock* rwlock) { @@ -72,7 +75,8 @@ int APS5_VABI scePthreadRwlockTrywrlock(PthreadRwlock* rwlock) { int APS5_VABI scePthreadRwlockTimedwrlock(PthreadRwlock* rwlock, KernelUseconds usec) { auto* lock = RequireRwlock(rwlock, __func__); if (OwnsWrite(lock)) return SCE_KERNEL_ERROR_EDEADLK; - if (!lock->_lock.try_lock_for(std::chrono::microseconds(usec))) return SCE_KERNEL_ERROR_ETIMEDOUT; + const bool locked = TimedWait::AcquireUntil(TimedWait::DeadlineNanos(usec), [&] { return lock->_lock.try_lock(); }, [&](std::uint64_t micros) { return lock->_lock.try_lock_for(std::chrono::microseconds(micros)); }); + if (!locked) return SCE_KERNEL_ERROR_ETIMEDOUT; lock->_writer.store(std::this_thread::get_id(), std::memory_order_release); return SCE_OK; } diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index 2cff28a931..a7cfe767e4 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -1,5 +1,6 @@ #include "../include/Pthread.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/CpuTopology.hpp" #include #include #include @@ -63,6 +64,36 @@ static void ReleaseThread(PthreadPrivate* thread) { delete thread; } +// The title's job workers ("BPE JobWorkerThread CPU0".."CPU12") spin at full load on one core each. +// Pinning them to the efficiency cores of a hybrid CPU (APS5_JOB_AFFINITY=1, with the driver's +// worker pinning) measured 0 to -9 % at the video stage, so the default leaves them free; +// APS5_JOB_AFFINITY_MASK= picks the cores. +static constexpr const char* JOB_WORKER_PREFIX = "BPE JobWorkerThread"; + +static std::uint64_t JobWorkerMask() { + static const std::uint64_t mask = [] { + if (std::getenv("APS5_NO_JOB_AFFINITY") != nullptr) return std::uint64_t{0}; + const auto requested = CpuTopology::MaskFromEnvironment("APS5_JOB_AFFINITY_MASK"); + if (requested != 0) return requested; + if (std::getenv("APS5_JOB_AFFINITY") == nullptr) return std::uint64_t{0}; + const auto& layout = CpuTopology::Get(); + return layout.hybrid ? layout.efficient : std::uint64_t{0}; + }(); + return mask; +} + +static void ApplyJobAffinity(PthreadPrivate& thread, void* handle) { + if (thread.name.compare(0, std::strlen(JOB_WORKER_PREFIX), JOB_WORKER_PREFIX) != 0) return; + const auto mask = JobWorkerMask(); + if (mask == 0) return; + static std::once_flag summary; + std::call_once(summary, [mask] { + const auto& layout = CpuTopology::Get(); + std::fprintf(stderr, "[affinity] job worker threads -> 0x%llx (hybrid=%d efficient=0x%llx performant=0x%llx process=0x%llx)\n", static_cast(mask), layout.hybrid ? 1 : 0, static_cast(layout.efficient), static_cast(layout.performant), static_cast(layout.process)); + }); + CpuTopology::PinTraced(thread.name.c_str(), handle, mask); +} + struct NativeThreadArgs { std::unique_ptr guest; std::future start; @@ -90,6 +121,7 @@ static unsigned __stdcall StartNativeThread(void* opaque) { if (!self->name.empty()) { const std::wstring description(self->name.begin(), self->name.end()); SetThreadDescription(GetCurrentThread(), description.c_str()); + ApplyJobAffinity(*self, nullptr); } args->initialized.set_value(); } catch (...) { @@ -280,6 +312,7 @@ int APS5_VABI scePthreadRename(Pthread thread, const char* name) { #ifdef _WIN32 const std::wstring description(thread->name.begin(), thread->name.end()); SetThreadDescription(static_cast(thread->nativeHandle), description.c_str()); + ApplyJobAffinity(*thread, thread->nativeHandle); #endif return SCE_OK; } diff --git a/core/libs/prx/libkernel/Semaphore/include/Semaphore.hpp b/core/libs/prx/libkernel/Semaphore/include/Semaphore.hpp index e7c6e49d26..631fe181dc 100644 --- a/core/libs/prx/libkernel/Semaphore/include/Semaphore.hpp +++ b/core/libs/prx/libkernel/Semaphore/include/Semaphore.hpp @@ -2,13 +2,13 @@ #define CORE_LIBS_PRX_LIBKERNEL_SEMAPHORE_SEMAPHORE_HPP #include -#include #include #include #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" constexpr int KERNEL_SEMA_OK = 0; constexpr int KERNEL_SEMA_ERROR_EINVAL = static_cast(0x80020016); @@ -19,7 +19,7 @@ struct KernelSemaPrivate { KernelSemaPrivate(std::int32_t initCount, std::int32_t maxCount, std::string name, bool isFifo); std::mutex mutex; - std::condition_variable condition; + TimedWait::Condition condition; std::string name; std::int32_t tokenCount; std::int32_t maxCount; diff --git a/core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp b/core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp index 9ac0b16a59..993ccdde24 100644 --- a/core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp +++ b/core/libs/prx/libkernel/Semaphore/src/Semaphore.cpp @@ -45,7 +45,7 @@ int APS5_VABI sceKernelSignalSema(KernelSema sem, int count) { return KERNEL_SEMA_ERROR_EINVAL; } sem->tokenCount += count; - sem->condition.notify_all(); + sem->condition.NotifyAll(); return KERNEL_SEMA_OK; } @@ -60,14 +60,13 @@ int APS5_VABI sceKernelWaitSema(KernelSema sem, int need, KernelUseconds* time) KernelTraceWait_nid_postfix("sema", __builtin_return_address(0), static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), timedOut); }; if (time == nullptr) { - sem->condition.wait(lock, [&] { return sem->tokenCount >= need; }); + sem->condition.Wait(lock, [&] { return sem->tokenCount >= need; }); traceWait(false); sem->tokenCount -= need; return KERNEL_SEMA_OK; } - auto timeout = std::chrono::microseconds(*time); - bool acquired = sem->condition.wait_for(lock, timeout, [&] { return sem->tokenCount >= need; }); + const bool acquired = sem->condition.WaitUntil(lock, TimedWait::DeadlineNanos(*time), [&] { return sem->tokenCount >= need; }); traceWait(!acquired); if (!acquired) { return KERNEL_SEMA_ERROR_ETIMEDOUT; diff --git a/core/libs/prx/libkernel/Time/Time.cpp b/core/libs/prx/libkernel/Time/Time.cpp index 3465b0704a..93fd8c7255 100644 --- a/core/libs/prx/libkernel/Time/Time.cpp +++ b/core/libs/prx/libkernel/Time/Time.cpp @@ -1,4 +1,5 @@ #include "prx/libkernel/Time/include/Time.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" #include "prx/libc/include/General.hpp" #include @@ -6,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -23,14 +25,7 @@ static std::uint64_t RawMonotonicNanos() { #ifdef _WIN32 - static const std::uint64_t freq = [] { - LARGE_INTEGER f{}; - QueryPerformanceFrequency(&f); - return static_cast(f.QuadPart); - }(); - LARGE_INTEGER counter{}; - QueryPerformanceCounter(&counter); - return static_cast(counter.QuadPart) * 1000000000ULL / freq; + return TimedWait::NowNanos(); #else struct timespec ts{}; clock_gettime(CLOCK_MONOTONIC, &ts); @@ -85,41 +80,64 @@ static const bool g_timerResolutionRaised = [] { throttling.ControlMask = PROCESS_POWER_THROTTLING_IGNORE_TIMER_RESOLUTION; throttling.StateMask = 0; SetProcessInformation(GetCurrentProcess(), ProcessPowerThrottling, &throttling, sizeof(throttling)); - std::fprintf(stderr, "[time] scheduler tick %s (was %.2f ms)\n", raised ? "raised to 0.5 ms" : "unchanged", previous / 10000.0); + using NtQueryTimerResolutionFn = LONG(WINAPI*)(PULONG, PULONG, PULONG); + const auto queryResolution = ntdll ? reinterpret_cast(reinterpret_cast(GetProcAddress(ntdll, "NtQueryTimerResolution"))) : nullptr; + ULONG minimum = 0, maximum = 0, current = 0; + if (queryResolution) queryResolution(&minimum, &maximum, ¤t); + std::fprintf(stderr, "[time] scheduler tick %s (was %.2f ms, now %.2f ms, timed waits %s)\n", raised ? "raised to 0.5 ms" : "unchanged", previous / 10000.0, current / 10000.0, TimedWait::Coarse() ? "coarse" : "fine"); return raised; }(); -#endif -static void SleepNanos(std::uint64_t nanos) { - if (nanos == 0) { - return; - } -#ifdef _WIN32 - // Guest spin/backoff loops sleep for a few microseconds; Sleep() would either not sleep at all - // or round up to a whole scheduler tick, so short sleeps yield and longer ones use a - // high-resolution waitable timer. - if (nanos < 50000ULL) { - SwitchToThread(); - return; - } - thread_local HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); - if (timer) { - LARGE_INTEGER due{}; - due.QuadPart = -static_cast(nanos / 100ULL); - if (SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE)) { - WaitForSingleObject(timer, INFINITE); - return; - } - } - Sleep(static_cast((nanos + 999999ULL) / 1000000ULL)); -#else - struct timespec req{}; - req.tv_sec = static_cast(nanos / 1000000000ULL); - req.tv_nsec = static_cast(nanos % 1000000000ULL); - while (nanosleep(&req, &req) == -1 && errno == EINTR) {} -#endif +extern "C" { +int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char* name); +int APS5_VABI scePthreadCondDestroy(PthreadCond* cond); +int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, unsigned int usec); +int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr, const char* name); +int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex); } +static void ReportTimedWait(const char* name, const std::function& body) { + std::vector samples(200); + for (auto& sample : samples) { + const std::uint64_t start = TimedWait::NowNanos(); + body(); + sample = static_cast(TimedWait::NowNanos() - start) / 1000.0; + } + std::sort(samples.begin(), samples.end()); + double sum = 0.0; + for (const double sample : samples) sum += sample; + std::fprintf(stderr, "[timers] %-44s mean %8.1f p50 %8.1f max %8.1f us\n", name, sum / static_cast(samples.size()), samples[samples.size() / 2], samples.back()); +} + +static const bool g_timerSelfTest = [] { + if (std::getenv("APS5_TRACE_TIMERS") == nullptr) return false; + std::mutex mutex; + std::condition_variable cv; + ReportTimedWait("std::condition_variable::wait_for(1 ms)", [&] { std::unique_lock lock(mutex); cv.wait_for(lock, std::chrono::milliseconds(1)); }); + PthreadCond cond = nullptr; + PthreadMutex guestMutex = nullptr; + scePthreadCondInit(&cond, nullptr, nullptr); + scePthreadMutexInit(&guestMutex, nullptr, nullptr); + scePthreadMutexLock(&guestMutex); + ReportTimedWait("scePthreadCondTimedwait(1000 us)", [&] { scePthreadCondTimedwait(&cond, &guestMutex, 1000); }); + ReportTimedWait("scePthreadCondTimedwait(13000 us)", [&] { scePthreadCondTimedwait(&cond, &guestMutex, 13000); }); + scePthreadMutexUnlock(&guestMutex); + scePthreadMutexDestroy(&guestMutex); + scePthreadCondDestroy(&cond); + ReportTimedWait("sceKernelUsleep(1000)", [&] { sceKernelUsleep_nid_postfix(1000); }); + ReportTimedWait("Sleep(1)", [&] { Sleep(1); }); + HANDLE event = CreateEventW(nullptr, TRUE, FALSE, nullptr); + ReportTimedWait("WaitForSingleObject(event, 1)", [&] { WaitForSingleObject(event, 1); }); + CloseHandle(event); + HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); + ReportTimedWait("high-resolution waitable timer 1 ms", [&] { LARGE_INTEGER due{}; due.QuadPart = -10000; SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE); WaitForSingleObject(timer, INFINITE); }); + CloseHandle(timer); + return true; +}(); +#endif + extern "C" { std::uint64_t APS5_VABI sceKernelGetProcessTime() { @@ -199,7 +217,7 @@ void KernelTraceWait_nid_postfix(const char* kind, const void* caller, std::uint int APS5_VABI sceKernelUsleep_nid_postfix(KernelUseconds microseconds) { TraceSleep(__builtin_return_address(0), microseconds); - SleepNanos(static_cast(microseconds) * 1000ULL); + TimedWait::SleepNanos(static_cast(microseconds) * 1000ULL); return 0; } @@ -210,8 +228,8 @@ int APS5_VABI sceKernelNanosleep(const KernelTimespec* rqtp, KernelTimespec* rmt if (rqtp->tv_sec < 0 || rqtp->tv_nsec < 0 || rqtp->tv_nsec >= 1000000000LL) { return -1; } - SleepNanos(static_cast(rqtp->tv_sec) * 1000000000ULL + - static_cast(rqtp->tv_nsec)); + TimedWait::SleepNanos(static_cast(rqtp->tv_sec) * 1000000000ULL + + static_cast(rqtp->tv_nsec)); if (rmtp != nullptr) { rmtp->tv_sec = 0; rmtp->tv_nsec = 0; @@ -421,7 +439,7 @@ uint64_t APS5_VABI sceKernelGetTscFrequency(void) { static const std::uint64_t frequency = [] { const std::uint64_t startNanos = RawMonotonicNanos(); const std::uint64_t startTicks = __rdtsc(); - SleepNanos(20000000ULL); + TimedWait::SleepNanos(20000000ULL); const std::uint64_t elapsedNanos = RawMonotonicNanos() - startNanos; const std::uint64_t elapsedTicks = __rdtsc() - startTicks; return static_cast(static_cast(elapsedTicks) * 1000000000.0L / static_cast(elapsedNanos)); diff --git a/core/libs/prx/libkernel/Time/TimedWait.cpp b/core/libs/prx/libkernel/Time/TimedWait.cpp new file mode 100644 index 0000000000..08c0b0d771 --- /dev/null +++ b/core/libs/prx/libkernel/Time/TimedWait.cpp @@ -0,0 +1,280 @@ +#include "prx/libkernel/Time/include/TimedWait.hpp" + +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#include +#endif + +namespace TimedWait { + +namespace { + +constexpr std::uint64_t YIELD_BELOW_NANOS = 50000ULL; +constexpr std::uint64_t LEAD_NANOS = 500000ULL; + +#ifdef _WIN32 + +void WINAPI DestroyWaiter(void* value); + +DWORD WaiterSlotIndex() { + static const DWORD index = FlsAlloc(DestroyWaiter); + return index; +} + +HANDLE CreateHighResolutionTimer() { + return CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); +} + +bool TimerWait(HANDLE timer, std::uint64_t nanos) { + if (!timer) return false; + LARGE_INTEGER due{}; + due.QuadPart = -static_cast(nanos / 100ULL); + if (due.QuadPart == 0) due.QuadPart = -1; + if (!SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE)) return false; + return WaitForSingleObject(timer, INFINITE) == WAIT_OBJECT_0; +} + +DWORD WholeMilliseconds(std::uint64_t nanos) { + const std::uint64_t millis = nanos / 1000000ULL; + return millis == 0 ? 1u : static_cast(millis > 0xFFFFFFFEULL ? 0xFFFFFFFEULL : millis); +} + +void SleepNanosCoarse(std::uint64_t nanos) { + if (nanos < YIELD_BELOW_NANOS) { + SwitchToThread(); + return; + } + thread_local HANDLE timer = CreateHighResolutionTimer(); + if (TimerWait(timer, nanos)) return; + Sleep(static_cast((nanos + 999999ULL) / 1000000ULL)); +} + +#endif + +} // namespace + +#ifdef _WIN32 + +struct Waiter { + HANDLE event = CreateEventW(nullptr, FALSE, FALSE, nullptr); + HANDLE timer = CreateHighResolutionTimer(); + Waiter* previous = nullptr; + Waiter* next = nullptr; + bool queued = false; + + ~Waiter() { + if (event) CloseHandle(event); + if (timer) CloseHandle(timer); + } +}; + +namespace { + +void WINAPI DestroyWaiter(void* value) { + delete static_cast(value); +} + +Waiter* ThisThreadWaiter() { + thread_local Waiter* waiter = nullptr; + if (!waiter) { + waiter = new Waiter(); + const DWORD slot = WaiterSlotIndex(); + if (slot != FLS_OUT_OF_INDEXES) FlsSetValue(slot, waiter); + } + return waiter; +} + +bool EventSet(HANDLE event, DWORD milliseconds) { + return WaitForSingleObject(event, milliseconds) == WAIT_OBJECT_0; +} + +bool WaitEventUntil(Waiter* waiter, std::uint64_t deadlineNanos) { + for (;;) { + const std::uint64_t now = NowNanos(); + if (now >= deadlineNanos) return EventSet(waiter->event, 0); + const std::uint64_t remaining = deadlineNanos - now; + if (remaining > LEAD_NANOS) { + const std::uint64_t bulk = remaining - LEAD_NANOS; + LARGE_INTEGER due{}; + due.QuadPart = -static_cast(bulk / 100ULL); + if (waiter->timer && SetWaitableTimer(waiter->timer, &due, 0, nullptr, nullptr, FALSE)) { + HANDLE handles[2] = {waiter->event, waiter->timer}; + if (WaitForMultipleObjects(2, handles, FALSE, INFINITE) == WAIT_OBJECT_0) return true; + continue; + } + if (EventSet(waiter->event, WholeMilliseconds(bulk))) return true; + continue; + } + if (EventSet(waiter->event, 0)) return true; + YieldProcessor(); + } +} + +} // namespace + +Waiter* Condition::enqueue() { + Waiter* waiter = ThisThreadWaiter(); + std::lock_guard lock(queueLock); + waiter->queued = true; + waiter->next = nullptr; + waiter->previous = tail; + if (tail) tail->next = waiter; + else head = waiter; + tail = waiter; + return waiter; +} + +void Condition::unlink(Waiter* waiter) { + if (waiter->previous) waiter->previous->next = waiter->next; + else head = waiter->next; + if (waiter->next) waiter->next->previous = waiter->previous; + else tail = waiter->previous; + waiter->previous = nullptr; + waiter->next = nullptr; + waiter->queued = false; +} + +void Condition::waitSignal(Waiter* waiter) { + WaitForSingleObject(waiter->event, INFINITE); +} + +bool Condition::waitSignalUntil(Waiter* waiter, std::uint64_t deadlineNanos) { + if (WaitEventUntil(waiter, deadlineNanos)) return true; + std::lock_guard lock(queueLock); + if (waiter->queued) { + unlink(waiter); + return false; + } + EventSet(waiter->event, 0); + return true; +} + +void Condition::NotifyOne() { + if (Coarse()) { + coarse.notify_one(); + return; + } + std::lock_guard lock(queueLock); + Waiter* waiter = head; + if (!waiter) return; + unlink(waiter); + SetEvent(waiter->event); +} + +void Condition::NotifyAll() { + if (Coarse()) { + coarse.notify_all(); + return; + } + std::lock_guard lock(queueLock); + while (Waiter* waiter = head) { + unlink(waiter); + SetEvent(waiter->event); + } +} + +#else + +void Condition::NotifyOne() { + coarse.notify_one(); +} + +void Condition::NotifyAll() { + coarse.notify_all(); +} + +#endif + +bool Coarse() { +#ifdef _WIN32 + static const bool coarse = std::getenv("APS5_COARSE_TIMED_WAITS") != nullptr; + return coarse; +#else + return true; +#endif +} + +std::uint64_t NowNanos() { +#ifdef _WIN32 + static const std::uint64_t frequency = [] { + LARGE_INTEGER f{}; + QueryPerformanceFrequency(&f); + return static_cast(f.QuadPart); + }(); + LARGE_INTEGER counter{}; + QueryPerformanceCounter(&counter); + const std::uint64_t ticks = static_cast(counter.QuadPart); + return (ticks / frequency) * 1000000000ULL + (ticks % frequency) * 1000000000ULL / frequency; +#else + struct timespec ts{}; + clock_gettime(CLOCK_MONOTONIC, &ts); + return static_cast(ts.tv_sec) * 1000000000ULL + static_cast(ts.tv_nsec); +#endif +} + +std::uint64_t DeadlineNanos(std::uint64_t microseconds) { + const std::uint64_t now = NowNanos(); + if (microseconds > (std::numeric_limits::max() - now) / 1000ULL) return std::numeric_limits::max(); + return now + microseconds * 1000ULL; +} + +std::uint64_t RemainingMicros(std::uint64_t deadlineNanos) { + const std::uint64_t now = NowNanos(); + return deadlineNanos > now ? (deadlineNanos - now) / 1000ULL : 0; +} + +void SleepUntil(std::uint64_t deadlineNanos) { +#ifdef _WIN32 + std::uint64_t now = NowNanos(); + if (deadlineNanos > now + LEAD_NANOS) { + thread_local HANDLE timer = CreateHighResolutionTimer(); + if (!TimerWait(timer, deadlineNanos - now - LEAD_NANOS)) Sleep(WholeMilliseconds(deadlineNanos - now - LEAD_NANOS)); + now = NowNanos(); + } + while (now < deadlineNanos) { + YieldProcessor(); + now = NowNanos(); + } +#else + const std::uint64_t now = NowNanos(); + if (deadlineNanos > now) SleepNanos(deadlineNanos - now); +#endif +} + +void PollSleepUntil(std::uint64_t deadlineNanos) { +#ifdef _WIN32 + const std::uint64_t now = NowNanos(); + if (deadlineNanos <= now) return; + thread_local HANDLE timer = CreateHighResolutionTimer(); + if (!TimerWait(timer, deadlineNanos - now)) Sleep(WholeMilliseconds(deadlineNanos - now)); +#else + SleepUntil(deadlineNanos); +#endif +} + +void SleepNanos(std::uint64_t nanos) { + if (nanos == 0) return; +#ifdef _WIN32 + if (Coarse()) { + SleepNanosCoarse(nanos); + return; + } + if (nanos < YIELD_BELOW_NANOS) { + SwitchToThread(); + return; + } + SleepUntil(NowNanos() + nanos); +#else + struct timespec req{}; + req.tv_sec = static_cast(nanos / 1000000000ULL); + req.tv_nsec = static_cast(nanos % 1000000000ULL); + while (nanosleep(&req, &req) == -1 && errno == EINTR) {} +#endif +} + +} // namespace TimedWait diff --git a/core/libs/prx/libkernel/Time/include/TimedWait.hpp b/core/libs/prx/libkernel/Time/include/TimedWait.hpp new file mode 100644 index 0000000000..774c62dc63 --- /dev/null +++ b/core/libs/prx/libkernel/Time/include/TimedWait.hpp @@ -0,0 +1,109 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_TIME_INCLUDE_TIMEDWAIT_HPP +#define CORE_LIBS_PRX_LIBKERNEL_TIME_INCLUDE_TIMEDWAIT_HPP + +#include +#include +#include +#include + +namespace TimedWait { + +bool Coarse(); +std::uint64_t NowNanos(); +std::uint64_t DeadlineNanos(std::uint64_t microseconds); +std::uint64_t RemainingMicros(std::uint64_t deadlineNanos); +void SleepNanos(std::uint64_t nanos); +void SleepUntil(std::uint64_t deadlineNanos); +void PollSleepUntil(std::uint64_t deadlineNanos); + +constexpr std::uint64_t LOCK_BULK_NANOS = 40000000ULL; +constexpr std::uint64_t LOCK_POLL_NANOS = 500000ULL; +constexpr std::uint64_t LOCK_TAIL_NANOS = 1000000ULL; + +template +bool AcquireUntil(std::uint64_t deadlineNanos, TTryLock tryLock, TTryLockFor tryLockFor) { + if (Coarse()) return tryLockFor(RemainingMicros(deadlineNanos)); + for (;;) { + if (tryLock()) return true; + const std::uint64_t now = NowNanos(); + if (now >= deadlineNanos) return false; + const std::uint64_t remaining = deadlineNanos - now; + if (remaining > LOCK_BULK_NANOS) { + if (tryLockFor((remaining - LOCK_BULK_NANOS) / 1000ULL)) return true; + continue; + } + if (remaining <= LOCK_TAIL_NANOS) SleepUntil(deadlineNanos); + else PollSleepUntil(now + LOCK_POLL_NANOS); + } +} + +struct Waiter; + +class Condition { +public: + Condition() = default; + ~Condition() = default; + Condition(const Condition&) = delete; + Condition& operator=(const Condition&) = delete; + + void NotifyOne(); + void NotifyAll(); + + template + void Wait(TLock& lock) { +#ifdef _WIN32 + if (!Coarse()) { + Waiter* waiter = enqueue(); + lock.unlock(); + waitSignal(waiter); + lock.lock(); + return; + } +#endif + coarse.wait(lock); + } + + template + bool WaitUntil(TLock& lock, std::uint64_t deadlineNanos) { +#ifdef _WIN32 + if (!Coarse()) { + Waiter* waiter = enqueue(); + lock.unlock(); + const bool signaled = waitSignalUntil(waiter, deadlineNanos); + lock.lock(); + return signaled; + } +#endif + return coarse.wait_for(lock, std::chrono::microseconds(RemainingMicros(deadlineNanos))) == std::cv_status::no_timeout; + } + + template + void Wait(TLock& lock, TPredicate predicate) { + while (!predicate()) Wait(lock); + } + + template + bool WaitUntil(TLock& lock, std::uint64_t deadlineNanos, TPredicate predicate) { + while (!predicate()) { + if (!WaitUntil(lock, deadlineNanos)) return predicate(); + } + return true; + } + +private: +#ifdef _WIN32 + Waiter* enqueue(); + void unlink(Waiter* waiter); + void waitSignal(Waiter* waiter); + bool waitSignalUntil(Waiter* waiter, std::uint64_t deadlineNanos); + + std::mutex queueLock; + Waiter* head = nullptr; + Waiter* tail = nullptr; +#endif + std::condition_variable_any coarse; +}; + +} // namespace TimedWait + +#endif diff --git a/core/libs/tests/ApplicationHeap.cpp b/core/libs/tests/ApplicationHeap.cpp index 0d4a8e9008..48009f88ef 100644 --- a/core/libs/tests/ApplicationHeap.cpp +++ b/core/libs/tests/ApplicationHeap.cpp @@ -87,8 +87,8 @@ int main(int argc, char**) { write(replacement, 0x18, &finalize); write(replacement, 0x20, &allocate); write(replacement, 0x28, &release); - write(replacement, 0x30, &reallocate); - write(replacement, 0x38, &allocateZeroed); + write(replacement, 0x30, &allocateZeroed); + write(replacement, 0x38, &reallocate); write(replacement, 0x40, &align); write(replacement, 0x48, &reallocate); write(replacement, 0x50, &posixAlign); diff --git a/core/libs/tests/GuestRtc.cpp b/core/libs/tests/GuestRtc.cpp new file mode 100644 index 0000000000..aa393d4493 --- /dev/null +++ b/core/libs/tests/GuestRtc.cpp @@ -0,0 +1,130 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include + +extern "C" { +int APS5_VABI sceRtcCheckValid(const RtcDateTime*); +int APS5_VABI sceRtcIsLeapYear(int); +int APS5_VABI sceRtcGetDaysInMonth(int, int); +int APS5_VABI sceRtcGetDayOfWeek(int, int, int); +int APS5_VABI sceRtcGetTickResolution(void); +int APS5_VABI sceRtcGetTick(const RtcDateTime*, RtcTick*); +int APS5_VABI sceRtcSetTick(RtcDateTime*, const RtcTick*); +int APS5_VABI sceRtcGetCurrentTick(RtcTick*); +int APS5_VABI sceRtcConvertUtcToLocalTime(const RtcTick*, RtcTick*); +int APS5_VABI sceRtcConvertLocalTimeToUtc(const RtcTick*, RtcTick*); +int APS5_VABI sceRtcGetTime_t(const RtcDateTime*, std::int64_t*); +int APS5_VABI sceRtcSetTime_t(RtcDateTime*, std::int64_t); +int APS5_VABI sceRtcGetWin32FileTime(const RtcDateTime*, std::uint64_t*); +int APS5_VABI sceRtcSetWin32FileTime(RtcDateTime*, std::uint64_t); +int APS5_VABI sceRtcFormatRFC3339(char*, const RtcTick*, int); +int APS5_VABI sceRtcParseRFC3339(RtcTick*, const char*); +int APS5_VABI sceRtcTickAddTicks(RtcTick*, const RtcTick*, std::int64_t); +int APS5_VABI sceRtcTickAddSeconds(RtcTick*, const RtcTick*, std::int64_t); +int APS5_VABI sceRtcTickAddDays(RtcTick*, const RtcTick*, std::int32_t); +int APS5_VABI sceRtcTickAddMonths(RtcTick*, const RtcTick*, std::int32_t); +int APS5_VABI sceRtcTickAddYears(RtcTick*, const RtcTick*, std::int16_t); +} + +static void Require(bool value) { if (!value) std::abort(); } + +static bool Equal(const RtcDateTime& left, const RtcDateTime& right) { + return left.year == right.year && left.month == right.month && left.day == right.day && left.hour == right.hour + && left.minute == right.minute && left.second == right.second && left.microsecond == right.microsecond; +} + +int main() { + constexpr int invalidPointer = static_cast(0x80B50002); + constexpr int invalidValue = static_cast(0x80B50003); + constexpr int badParse = static_cast(0x80B50007); + constexpr int invalidYear = static_cast(0x80B50008); + constexpr int invalidMonth = static_cast(0x80B50009); + constexpr int invalidDay = static_cast(0x80B5000A); + constexpr int invalidHour = static_cast(0x80B5000B); + constexpr int invalidMicrosecond = static_cast(0x80B5000E); + constexpr std::uint64_t unixEpochTick = 62135596800000000ull; + constexpr std::uint64_t leapDayTick = 63844806896789000ull; + constexpr std::uint64_t maxTick = 315537897599999999ull; + + Require(sceRtcGetTickResolution() == 1000000); + Require(sceRtcIsLeapYear(2000) == 1 && sceRtcIsLeapYear(1900) == 0 && sceRtcIsLeapYear(2024) == 1); + Require(sceRtcIsLeapYear(0) == invalidYear); + Require(sceRtcGetDaysInMonth(2023, 2) == 28 && sceRtcGetDaysInMonth(2024, 2) == 29 && sceRtcGetDaysInMonth(2024, 4) == 30); + Require(sceRtcGetDaysInMonth(2024, 13) == invalidMonth); + Require(sceRtcGetDayOfWeek(1, 1, 1) == 1 && sceRtcGetDayOfWeek(2026, 9, 26) == 6); + Require(sceRtcGetDayOfWeek(2023, 2, 29) == invalidDay); + + RtcDateTime leapDay{2024, 2, 29, 12, 34, 56, 789000}; + Require(sceRtcCheckValid(&leapDay) == 0); + Require(sceRtcCheckValid(nullptr) == invalidPointer); + RtcDateTime invalid = leapDay; + invalid.year = 2023; + Require(sceRtcCheckValid(&invalid) == invalidDay); + invalid = leapDay; + invalid.hour = 24; + Require(sceRtcCheckValid(&invalid) == invalidHour); + invalid = leapDay; + invalid.microsecond = 1000000; + Require(sceRtcCheckValid(&invalid) == invalidMicrosecond); + + RtcTick tick{}; + Require(sceRtcGetTick(&leapDay, &tick) == 0 && tick.tick == leapDayTick); + RtcDateTime converted{}; + Require(sceRtcSetTick(&converted, &tick) == 0 && Equal(converted, leapDay)); + tick.tick = maxTick; + Require(sceRtcSetTick(&converted, &tick) == 0 && Equal(converted, RtcDateTime{9999, 12, 31, 23, 59, 59, 999999})); + tick.tick = maxTick + 1; + Require(sceRtcSetTick(&converted, &tick) == invalidValue); + + RtcDateTime epoch{1970, 1, 1, 0, 0, 0, 0}; + std::int64_t seconds = -1; + Require(sceRtcGetTime_t(&epoch, &seconds) == 0 && seconds == 0); + RtcDateTime millennium{2000, 1, 1, 0, 0, 0, 0}; + Require(sceRtcGetTime_t(&millennium, &seconds) == 0 && seconds == 946684800); + Require(sceRtcSetTime_t(&converted, 946684800) == 0 && Equal(converted, millennium)); + Require(sceRtcSetTime_t(&converted, -1) == invalidValue); + std::uint64_t fileTime = 0; + Require(sceRtcGetWin32FileTime(&epoch, &fileTime) == 0 && fileTime == 116444736000000000ull); + Require(sceRtcSetWin32FileTime(&converted, 116444736000000000ull) == 0 && Equal(converted, epoch)); + + char text[32]; + tick.tick = leapDayTick; + Require(sceRtcFormatRFC3339(text, &tick, 0) == 0 && std::strcmp(text, "2024-02-29T12:34:56.78Z") == 0); + Require(sceRtcFormatRFC3339(text, &tick, 90) == 0 && std::strcmp(text, "2024-02-29T14:04:56.78+01:30") == 0); + Require(sceRtcFormatRFC3339(text, &tick, -300) == 0 && std::strcmp(text, "2024-02-29T07:34:56.78-05:00") == 0); + Require(sceRtcParseRFC3339(&tick, "2024-02-29T14:04:56.789+01:30") == 0 && tick.tick == leapDayTick); + Require(sceRtcParseRFC3339(&tick, "2024-02-29t12:34:56.789z") == 0 && tick.tick == leapDayTick); + Require(sceRtcParseRFC3339(&tick, "1970-01-01T00:00:00Z") == 0 && tick.tick == unixEpochTick); + Require(sceRtcParseRFC3339(&tick, "2023-02-29T00:00:00Z") == invalidDay); + Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56") == badParse); + Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56Zjunk") == badParse); + Require(sceRtcParseRFC3339(nullptr, "1970-01-01T00:00:00Z") == invalidPointer); + + RtcTick source{leapDayTick}; + RtcTick result{}; + Require(sceRtcTickAddSeconds(&result, &source, 3600) == 0 && result.tick == leapDayTick + 3600000000ull); + Require(sceRtcTickAddDays(&result, &source, 1) == 0); + Require(sceRtcSetTick(&converted, &result) == 0 && converted.month == 3 && converted.day == 1); + Require(sceRtcTickAddYears(&result, &source, 1) == 0); + Require(sceRtcSetTick(&converted, &result) == 0 && converted.year == 2025 && converted.month == 2 && converted.day == 28); + RtcDateTime endOfJanuary{2024, 1, 31, 8, 0, 0, 0}; + Require(sceRtcGetTick(&endOfJanuary, &source) == 0); + Require(sceRtcTickAddMonths(&result, &source, 1) == 0); + Require(sceRtcSetTick(&converted, &result) == 0 && Equal(converted, RtcDateTime{2024, 2, 29, 8, 0, 0, 0})); + Require(sceRtcTickAddMonths(&result, &source, -12 * 2024) == invalidValue); + source.tick = 5; + Require(sceRtcTickAddTicks(&result, &source, -6) == invalidValue); + Require(sceRtcTickAddTicks(&result, &source, -5) == 0 && result.tick == 0); + source.tick = maxTick; + Require(sceRtcTickAddTicks(&result, &source, 1) == invalidValue); + Require(sceRtcTickAddTicks(nullptr, &source, 1) == invalidPointer); + + RtcTick now{}; + Require(sceRtcGetCurrentTick(&now) == 0 && now.tick > leapDayTick); + RtcTick local{}; + RtcTick back{}; + Require(sceRtcConvertUtcToLocalTime(&now, &local) == 0); + Require(sceRtcConvertLocalTimeToUtc(&local, &back) == 0 && back.tick == now.tick); +} From 01c4e3edf7b7d3eeb717dd4bd3989cbf952dc14f Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:00:39 -0700 Subject: [PATCH 14/93] feat(relinker): --to-intel lowers SSE4a sites to same-length jumps into an appended stub section EXTRQ, INSERTQ, MOVNTSS and MOVNTSD are rewritten at relink time: same-length in-place forms where they exist, otherwise a jmp rel32 into an appended executable section holding a scratch-free SSE body and a jump back, so no code shifts and no RIP-relative or jump-table operand needs adjusting. Decoder fixes for POPCNT and 0F BB (BTC), codegen and elfpatcher tests, README and technical-debt notes. The runtime illegal-instruction trap stays as a fallback. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- README.md | 2 + core/relinker/CMakeLists.txt | 52 +++ .../codegen/include/codegen/CodegenTypes.hpp | 12 + .../x86/Amd64OnlySubstitutionTable.hpp | 35 +- .../x86/Amd64OnlySubstitutionTypes.hpp | 10 + .../codegen/x86/DecodedInstruction.hpp | 1 + .../include/codegen/x86/Sse4aLowering.hpp | 25 ++ .../include/codegen/x86/Sse4aOperands.hpp | 22 ++ .../codegen/x86/X64OpcodeConstants.hpp | 18 + .../codegen/src/Amd64OnlyConverter.cpp | 145 ++++++-- .../codegen/src/InstructionScanner.cpp | 9 +- .../src/x86/Amd64OnlyInstructionMatcher.cpp | 77 +++- .../codegen/src/x86/DecodedInstruction.cpp | 7 + .../codegen/src/x86/Sse4aLowering.cpp | 214 +++++++++++ .../codegen/src/x86/Sse4aOperands.cpp | 84 +++++ .../codegen/src/x86/X64InstructionDecoder.cpp | 28 +- .../codegen/tests/Amd64OnlyConverterTests.cpp | 339 +++++++++++++++++ .../elfpatcher/general/IElfPatcher.hpp | 4 +- .../elfpatcher/linux/LinuxElfPatcher.hpp | 5 +- .../elfpatcher/windows/WindowsElfPatcher.hpp | 2 +- .../windows/WindowsTrampolineBuilder.hpp | 16 + .../elfpatcher/src/linux/LinuxElfPatcher.cpp | 44 ++- .../src/windows/WindowsPePatcher.cpp | 4 +- .../src/windows/WindowsTrampolineBuilder.cpp | 58 +++ .../tests/Amd64OnlyWindowsTests.cpp | 342 ++++++++++++++++++ core/relinker/main.cpp | 27 +- .../relinker/tests/StrictNidFilterTests.cpp | 6 +- docs/TechnicalDebt.md | 4 + 28 files changed, 1515 insertions(+), 77 deletions(-) create mode 100644 core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp create mode 100644 core/relinker/codegen/include/codegen/x86/Sse4aOperands.hpp create mode 100644 core/relinker/codegen/src/x86/Sse4aLowering.cpp create mode 100644 core/relinker/codegen/src/x86/Sse4aOperands.cpp create mode 100644 core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp create mode 100644 core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTrampolineBuilder.hpp create mode 100644 core/relinker/elfpatcher/src/windows/WindowsTrampolineBuilder.cpp create mode 100644 core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp diff --git a/README.md b/README.md index 41af87e63a..e7efe9009b 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Tool for automatic executables porting to Linux and Windows. Includes a [relinker](core/relinker) that converts executable to the target system's native format and implementations of [system prx libraries](core/libs/prx) suitable for dynamic linking. No emulation or separate runtime process. +`relinker --to-intel` rewrites the Zen 2-only instructions a title may contain (EXTRQ, INSERTQ, MOVNTSS, MOVNTSD) at relink time, in place when an equal-length Intel encoding exists and otherwise through a jump to a generated stub section, so Intel hosts need no runtime emulation. + Releases will be published after the first full successful launch of at least one game. ## Status diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index c556e72adb..7325396882 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -7,6 +7,7 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsPeFormat.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsLoadImage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsTlsBuilder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsTrampolineBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsRelocationBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsImportBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp @@ -28,6 +29,8 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/X64InstructionRewriter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/DecodedInstruction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sse4aOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sse4aLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp @@ -69,10 +72,59 @@ if(WIN32) set_target_properties(windows_dependency_diagnostics_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) add_test(NAME windows_dependency_diagnostics COMMAND windows_dependency_diagnostics_tests) endif() + + add_executable(amd64_only_windows_tests EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/tests/Amd64OnlyWindowsTests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsTrampolineBuilder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsLoadImage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsStubEmitter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsPeFormat.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsPeWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/io/src/BufferUtils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/Amd64OnlyConverter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/InstructionScanner.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/X64InstructionDecoder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/X64InstructionRewriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/DecodedInstruction.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sse4aOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sse4aLowering.cpp + ) + target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) + if(BUILD_TESTING) + set_target_properties(amd64_only_windows_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME amd64_only_windows COMMAND amd64_only_windows_tests) + endif() endif() if(BUILD_TESTING) + add_executable(amd64_only_converter_tests + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/Amd64OnlyConverterTests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/Amd64OnlyConverter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/InstructionScanner.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/X64InstructionDecoder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/X64InstructionRewriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/DecodedInstruction.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sse4aOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sse4aLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SectionHeaderTableBuilder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/io/src/ByteWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/io/src/BufferUtils.cpp + ) + target_include_directories(amd64_only_converter_tests PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/domain/include + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include + ${CMAKE_CURRENT_SOURCE_DIR}/io/include + ) + add_test(NAME amd64_only_converter COMMAND amd64_only_converter_tests) + add_executable(strict_nid_filter_tests ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/StrictNidFilterTests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/UnusedNidFilter/StrictNidFilter.cpp diff --git a/core/relinker/codegen/include/codegen/CodegenTypes.hpp b/core/relinker/codegen/include/codegen/CodegenTypes.hpp index 3d8c4cd45c..51d90c0bc5 100644 --- a/core/relinker/codegen/include/codegen/CodegenTypes.hpp +++ b/core/relinker/codegen/include/codegen/CodegenTypes.hpp @@ -1,6 +1,7 @@ #ifndef CODEGEN_CODEGENTYPES_HPP #define CODEGEN_CODEGENTYPES_HPP +#include #include #include #include @@ -28,9 +29,20 @@ struct RewriteResult { std::vector Adjustments; }; +struct TrampolineSite { + Domain::FileByteOffset Offset; + Domain::VirtualAddress Address; + std::size_t Length; + std::vector OriginalBytes; + std::vector Body; + std::size_t ReturnBranchOffset; +}; + struct ConvertResult { std::vector Bytes; std::size_t ReplacedCount; + std::vector Trampolines; + std::vector Reports; }; enum class ControlFlowKind : std::uint8_t { diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp index 2881b934d8..1ec1dbfc2f 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp @@ -17,14 +17,39 @@ struct Entry { inline constexpr std::uint8_t k##name##Bytes[] = {__VA_ARGS__}; \ inline constexpr Entry k##name = {#name, k##name##Bytes, sizeof(k##name##Bytes)}; -AMD64_STUB(Monitorx, 0x0F, 0x01, 0xFA) -AMD64_STUB(Mwaitx, 0x0F, 0x01, 0xFB) -AMD64_STUB(Clzero, 0x0F, 0x01, 0xFC) -AMD64_STUB(Rdpru, 0x0F, 0x01, 0xFD) -AMD64_STUB(Mcommit, 0xF3, 0x0F, 0x01, 0xFA) +AMD64_STUB(JmpRel32, 0xE9, 0x00, 0x00, 0x00, 0x00) +AMD64_STUB(LeaRspBelowRedZone, 0x48, 0x8D, 0xA4, 0x24, 0x70, 0xFF, 0xFF, 0xFF) +AMD64_STUB(LeaRspRestore, 0x48, 0x8D, 0xA4, 0x24, 0x90, 0x00, 0x00, 0x00) +AMD64_STUB(Nop1, 0x90) +AMD64_STUB(Nop2, 0x66, 0x90) +AMD64_STUB(Nop3, 0x0F, 0x1F, 0x00) +AMD64_STUB(Nop4, 0x0F, 0x1F, 0x40, 0x00) +AMD64_STUB(Nop5, 0x0F, 0x1F, 0x44, 0x00, 0x00) +AMD64_STUB(Nop6, 0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00) +AMD64_STUB(Nop7, 0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00) #undef AMD64_STUB +inline constexpr Entry kNops[] = {kNop1, kNop2, kNop3, kNop4, kNop5, kNop6, kNop7}; + +inline constexpr Entry kMonitorx = {"MONITORX", nullptr, 0}; +inline constexpr Entry kMwaitx = {"MWAITX", nullptr, 0}; +inline constexpr Entry kClzero = {"CLZERO", nullptr, 0}; +inline constexpr Entry kRdpru = {"RDPRU", nullptr, 0}; +inline constexpr Entry kMcommit = {"MCOMMIT", nullptr, 0}; +inline constexpr Entry kExtrq = {"EXTRQ", nullptr, 0}; +inline constexpr Entry kInsertq = {"INSERTQ", nullptr, 0}; +inline constexpr Entry kExtrqRegisterForm = {"EXTRQ register form", nullptr, 0}; +inline constexpr Entry kInsertqRegisterForm = {"INSERTQ register form", nullptr, 0}; +inline constexpr Entry kMovntss = {"MOVNTSS", nullptr, 0}; +inline constexpr Entry kMovntsd = {"MOVNTSD", nullptr, 0}; + +inline constexpr std::uint8_t kMovsStoreOpcode = 0x11; +inline constexpr std::uint8_t kPshufbZero = 0x80; +inline constexpr std::uint8_t kTrapFill = 0xCC; +inline constexpr std::size_t kStubAlignment = 16; +inline constexpr std::size_t kRedZoneSpillFrame = 0x90; + } // namespace Amd64OnlySubstitutionTable } // namespace Codegen diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp index aa062cc8e6..98c740afa9 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp @@ -8,10 +8,19 @@ namespace Codegen { +enum class Amd64OnlyLowering : std::uint8_t { + InPlace, + Trampoline, + Unsupported +}; + struct Amd64OnlyMatch { std::string InstructionName; std::size_t Length; + Amd64OnlyLowering Lowering; std::vector ReplacementBytes; + std::vector StubBody; + std::size_t ReturnBranchOffset; }; struct Amd64OnlySubstitutionReport { @@ -19,6 +28,7 @@ struct Amd64OnlySubstitutionReport { Domain::FileByteOffset Offset; std::size_t OriginalLength; std::size_t ReplacementLength; + Amd64OnlyLowering Lowering; }; } diff --git a/core/relinker/codegen/include/codegen/x86/DecodedInstruction.hpp b/core/relinker/codegen/include/codegen/x86/DecodedInstruction.hpp index ac42a23d27..5de292facd 100644 --- a/core/relinker/codegen/include/codegen/x86/DecodedInstruction.hpp +++ b/core/relinker/codegen/include/codegen/x86/DecodedInstruction.hpp @@ -10,6 +10,7 @@ struct DecodedInstruction { const std::uint8_t* Data; std::size_t Length; + [[nodiscard]] std::size_t OpcodeOffset() const; [[nodiscard]] bool IsShaNi() const; [[nodiscard]] bool IsExtrq() const; [[nodiscard]] bool IsInsertq() const; diff --git a/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp b/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp new file mode 100644 index 0000000000..6064b9cca1 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp @@ -0,0 +1,25 @@ +#ifndef CODEGEN_X86_SSE4ALOWERING_HPP +#define CODEGEN_X86_SSE4ALOWERING_HPP + +#include +#include +#include +#include +#include + +namespace Codegen { + +struct LoweredBody { + std::vector Bytes; + std::size_t ReturnBranchOffset; +}; + +class Sse4aLowering { +public: + [[nodiscard]] std::optional> LowerInPlace(const Sse4aOperands& operands, std::size_t originalLength) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const Sse4aOperands& operands) const; +}; + +} + +#endif diff --git a/core/relinker/codegen/include/codegen/x86/Sse4aOperands.hpp b/core/relinker/codegen/include/codegen/x86/Sse4aOperands.hpp new file mode 100644 index 0000000000..6e4322f9f0 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/Sse4aOperands.hpp @@ -0,0 +1,22 @@ +#ifndef CODEGEN_X86_SSE4AOPERANDS_HPP +#define CODEGEN_X86_SSE4AOPERANDS_HPP + +#include +#include + +namespace Codegen { + +struct Sse4aOperands { + bool Insertq; + bool RegisterForm; + std::uint8_t Destination; + std::uint8_t Source; + std::uint8_t Length; + std::uint8_t Index; +}; + +[[nodiscard]] Sse4aOperands DecodeSse4a(const std::uint8_t* data, std::size_t length); + +} + +#endif diff --git a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp index 69bc3598f8..2472fe2585 100644 --- a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp +++ b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp @@ -106,6 +106,10 @@ inline constexpr std::uint8_t OneByteLoop = 0xE0; inline constexpr std::uint8_t OneByteLoopMax = 0xE2; inline constexpr std::uint8_t OneByteInOutImm8Min = 0xE4; inline constexpr std::uint8_t OneByteInOutImm8Max = 0xE7; +inline constexpr std::uint8_t OneByteInt = 0xCD; +inline constexpr std::uint8_t OneByteRetImm16 = 0xC2; +inline constexpr std::uint8_t OneByteRetFarImm16 = 0xCA; +inline constexpr std::uint8_t OneByteEnter = 0xC8; inline constexpr std::uint8_t TwoByteJccRel32Min = 0x80; inline constexpr std::uint8_t TwoByteJccRel32Max = 0x8F; @@ -158,6 +162,19 @@ inline constexpr std::uint8_t TwoByteShufpdImm8 = 0xC6; inline constexpr std::uint8_t TwoByteGrp9 = 0xC7; inline constexpr std::uint8_t TwoByteExtrqInsertqImm8Imm8 = 0x78; inline constexpr std::uint8_t TwoByteExtrqInsertqModRm = 0x79; +inline constexpr std::uint8_t TwoByteMovnts = 0x2B; +inline constexpr std::uint8_t TwoByteGrp6 = 0x00; +inline constexpr std::uint8_t TwoByteLar = 0x02; +inline constexpr std::uint8_t TwoByteLsl = 0x03; +inline constexpr std::uint8_t TwoBytePrefetchw = 0x0D; +inline constexpr std::uint8_t TwoByteMovCrDrMin = 0x20; +inline constexpr std::uint8_t TwoByteMovCrDrMax = 0x23; +inline constexpr std::uint8_t TwoBytePopcnt = 0xB8; +inline constexpr std::uint8_t TwoByteUd1 = 0xB9; +inline constexpr std::uint8_t TwoByteBtc = 0xBB; +inline constexpr std::uint8_t TwoByteXaddRm8 = 0xC0; +inline constexpr std::uint8_t TwoByteMovnti = 0xC3; +inline constexpr std::uint8_t TwoBytePinsrw = 0xC4; inline constexpr std::uint8_t VexNoModRmMin = 0x77; inline constexpr std::uint8_t VexNoModRmMax = 0x77; @@ -171,6 +188,7 @@ inline constexpr std::uint8_t EvexMap0F38 = 0x02; inline constexpr std::size_t ImmSizeNone = 0; inline constexpr std::size_t ImmSize8 = 1; inline constexpr std::size_t ImmSize16 = 2; +inline constexpr std::size_t ImmSizeEnter = 3; inline constexpr std::size_t ImmSize32 = 4; inline constexpr std::size_t ImmSize64 = 8; diff --git a/core/relinker/codegen/src/Amd64OnlyConverter.cpp b/core/relinker/codegen/src/Amd64OnlyConverter.cpp index 74f624611a..41cd3c2116 100644 --- a/core/relinker/codegen/src/Amd64OnlyConverter.cpp +++ b/core/relinker/codegen/src/Amd64OnlyConverter.cpp @@ -1,13 +1,28 @@ #include #include +#include +#include +#include #include #include +#include #include +#include +#include namespace Codegen { namespace { +template +auto _atFileOffset(const Domain::FileByteOffset base, const TOperation& operation) { + try { + return operation(); + } catch (const CodegenException& e) { + throw CodegenException(e.what(), base + e.FailureOffset); + } +} + class Amd64OnlyConverter : public IAmd64OnlyConverter { public: [[nodiscard]] ConvertResult Convert( @@ -16,64 +31,132 @@ public: ) const override; private: + struct Pending { + InstructionMatch Instruction; + Amd64OnlyMatch Substitution; + }; + X64InstructionRewriter _rewriter; std::unique_ptr _matcher = MakeAmd64OnlyInstructionMatcher(); std::unique_ptr _scanner = MakeInstructionScanner(); - std::size_t _convertSegment( + void _convertSegment( std::vector& fileBytes, - const Domain::ProgramHeader& ph + const Domain::ProgramHeader& ph, + ConvertResult& result ) const; + + [[nodiscard]] static std::set _collectBranchTargets( + const std::vector& seg, + const std::vector& matches, + const Domain::ProgramHeader& ph + ); }; -std::size_t Amd64OnlyConverter::_convertSegment( - std::vector& fileBytes, +std::set Amd64OnlyConverter::_collectBranchTargets( + const std::vector& seg, + const std::vector& matches, const Domain::ProgramHeader& ph +) { + const X64InstructionDecoder decoder; + std::set targets; + for (const auto& match : matches) { + const auto info = decoder.DecodeInstruction(seg.data() + match.Offset, match.Length); + if (!info.HasBranchTarget || info.HasRipRelativeDisp) + continue; + const auto target = static_cast(ph.MappedAddress + match.Offset + info.Length) + info.BranchDisp; + if (target >= 0) + targets.insert(static_cast(target)); + } + return targets; +} + +void Amd64OnlyConverter::_convertSegment( + std::vector& fileBytes, + const Domain::ProgramHeader& ph, + ConvertResult& result ) const { const auto segOffset = static_cast(ph.Offset); const auto segSize = static_cast(ph.FileSize); + if (segOffset > fileBytes.size() || segSize > fileBytes.size() - segOffset) + throw CodegenException("Code segment exceeds the file", ph.Offset); std::vector seg( - fileBytes.begin() + segOffset, - fileBytes.begin() + segOffset + segSize + fileBytes.begin() + static_cast(segOffset), + fileBytes.begin() + static_cast(segOffset + segSize) ); - const auto matches = _scanner->ScanCodeSection(seg, 0, seg.size()); - std::int64_t shift = 0; - std::size_t count = 0; + const auto matches = _atFileOffset(ph.Offset, [&] { return _scanner->ScanCodeSection(seg, 0, seg.size()); }); + std::vector pending; for (const auto& match : matches) { - const auto shiftedOffset = static_cast(match.Offset) + shift; - const auto offset = static_cast(shiftedOffset); - const auto result = _matcher->Match(seg.data() + offset, match.Length); - if (!result.has_value()) { - continue; + auto substitution = _atFileOffset(ph.Offset + match.Offset, [&] { return _matcher->Match(seg.data() + match.Offset, match.Length); }); + if (substitution.has_value()) + pending.push_back({match, std::move(*substitution)}); + } + if (pending.empty()) + return; + + const bool needsBranchTargets = std::any_of(pending.begin(), pending.end(), [](const Pending& item) { + return item.Substitution.Lowering == Amd64OnlyLowering::Trampoline; + }); + const auto branchTargets = needsBranchTargets ? _collectBranchTargets(seg, matches, ph) : std::set{}; + + for (const auto& item : pending) { + const auto& match = item.Instruction; + const auto& substitution = item.Substitution; + const auto fileOffset = static_cast(ph.Offset + match.Offset); + const auto address = static_cast(ph.MappedAddress + match.Offset); + std::size_t replacementLength = 0; + + switch (substitution.Lowering) { + case Amd64OnlyLowering::InPlace: { + if (substitution.ReplacementBytes.size() != match.Length) + throw CodegenException("Intel substitution changes the instruction length", fileOffset); + seg = _atFileOffset(ph.Offset, [&] { return _rewriter.Rewrite(seg, {match.Offset, substitution.ReplacementBytes}).Bytes; }); + replacementLength = substitution.ReplacementBytes.size(); + ++result.ReplacedCount; + break; } - const RewriteRequest request{ - static_cast(offset), - result->ReplacementBytes - }; - auto rewriteResult = _rewriter.Rewrite(seg, request); - shift += static_cast(result->ReplacementBytes.size()) - - static_cast(match.Length); - seg = std::move(rewriteResult.Bytes); - ++count; + case Amd64OnlyLowering::Trampoline: { + if (match.Length < Amd64OnlySubstitutionTable::kJmpRel32.Size) + throw CodegenException("AMD-only instruction too short for a jump", fileOffset); + const auto hit = branchTargets.upper_bound(address); + if (hit != branchTargets.end() && *hit < address + match.Length) + throw CodegenException("Branch enters an AMD-only instruction", ph.Offset + (*hit - ph.MappedAddress)); + const auto begin = seg.begin() + static_cast(match.Offset); + result.Trampolines.push_back({ + fileOffset, + address, + match.Length, + std::vector(begin, begin + static_cast(match.Length)), + substitution.StubBody, + substitution.ReturnBranchOffset + }); + replacementLength = substitution.StubBody.size(); + break; + } + case Amd64OnlyLowering::Unsupported: + throw CodegenException("AMD-only instruction without Intel lowering: " + substitution.InstructionName, fileOffset); + } + + result.Reports.push_back({substitution.InstructionName, fileOffset, match.Length, replacementLength, substitution.Lowering}); } - const auto copySize = std::min(seg.size(), segSize); - std::copy(seg.begin(), seg.begin() + copySize, fileBytes.begin() + segOffset); - return count; + if (seg.size() != segSize) + throw CodegenException("Code segment size changed during Intel conversion", ph.Offset); + std::copy(seg.begin(), seg.end(), fileBytes.begin() + static_cast(segOffset)); } ConvertResult Amd64OnlyConverter::Convert( std::vector fileBytes, const std::vector& codeSegments ) const { - std::size_t total = 0; - for (const auto& ph : codeSegments) { - total += _convertSegment(fileBytes, ph); - } - return {std::move(fileBytes), total}; + ConvertResult result{{}, 0, {}, {}}; + for (const auto& ph : codeSegments) + _convertSegment(fileBytes, ph, result); + result.Bytes = std::move(fileBytes); + return result; } } diff --git a/core/relinker/codegen/src/InstructionScanner.cpp b/core/relinker/codegen/src/InstructionScanner.cpp index aaa7ee9942..f76cfeb6f5 100644 --- a/core/relinker/codegen/src/InstructionScanner.cpp +++ b/core/relinker/codegen/src/InstructionScanner.cpp @@ -1,7 +1,9 @@ #include +#include #include #include #include +#include namespace Codegen { @@ -29,7 +31,12 @@ std::vector InstructionScanner::ScanCodeSection( const std::uint8_t* cursor = codeSection.data() + i; const std::size_t available = limit - i; - const std::size_t length = decoder.Decode(cursor, available); + std::size_t length = 0; + try { + length = decoder.Decode(cursor, available); + } catch (const CodegenException& e) { + throw CodegenException(std::string("Cannot decode instruction: ") + e.what(), codeSectionOffset + i); + } matches.push_back({codeSectionOffset + i, length}); diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 70ecb7e3d8..f371299d3a 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -1,14 +1,20 @@ #include #include #include +#include +#include +#include +#include #include namespace Codegen { namespace { -std::vector _bytesOf(const Amd64OnlySubstitutionTable::Entry& e) { - return std::vector(e.Bytes, e.Bytes + e.Size); +using namespace Amd64OnlySubstitutionTable; + +Amd64OnlyMatch _unsupported(const Entry& entry, const std::size_t length) { + return Amd64OnlyMatch{entry.Name, length, Amd64OnlyLowering::Unsupported, {}, {}, 0}; } class Amd64OnlyInstructionMatcher : public IAmd64OnlyInstructionMatcher { @@ -17,34 +23,69 @@ public: const std::uint8_t* data, std::size_t length ) const override; + +private: + Sse4aLowering _lowering; + + [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const; + [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry) const; }; +Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstruction& instr, const Entry& entry) const { + using namespace X64OpcodeConstants; + const auto opcodeOffset = instr.OpcodeOffset(); + if (opcodeOffset + 2 >= instr.Length) + throw CodegenException("MOVNTSS/MOVNTSD truncated before its ModRM byte"); + const auto modrm = instr.Data[opcodeOffset + 2]; + if (((modrm >> ModRmModShift) & ModRmModMask) == ModRmModRegister) + throw CodegenException("MOVNTSS/MOVNTSD with a register operand"); + std::vector replacement(instr.Data, instr.Data + instr.Length); + replacement[opcodeOffset + 1] = kMovsStoreOpcode; + return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0}; +} + +Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry) const { + const auto operands = DecodeSse4a(instr.Data, instr.Length); + if (operands.RegisterForm) + return _unsupported(registerFormEntry, instr.Length); + if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length)) + return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0}; + auto body = _lowering.LowerOutOfLine(operands); + return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; +} + std::optional Amd64OnlyInstructionMatcher::Match( const std::uint8_t* data, std::size_t length ) const { - using namespace Amd64OnlySubstitutionTable; const DecodedInstruction instr{data, length}; - if (instr.IsMonitorx()) { - return Amd64OnlyMatch{kMonitorx.Name, length, _bytesOf(kMonitorx)}; - } + if (instr.IsMovntss()) + return _matchMovnts(instr, kMovntss); - if (instr.IsMwaitx()) { - return Amd64OnlyMatch{kMwaitx.Name, length, _bytesOf(kMwaitx)}; - } + if (instr.IsMovntsd()) + return _matchMovnts(instr, kMovntsd); - if (instr.IsClzero()) { - return Amd64OnlyMatch{kClzero.Name, length, _bytesOf(kClzero)}; - } + if (instr.IsExtrq()) + return _matchSse4a(instr, kExtrq, kExtrqRegisterForm); - if (instr.IsRdpru()) { - return Amd64OnlyMatch{kRdpru.Name, length, _bytesOf(kRdpru)}; - } + if (instr.IsInsertq()) + return _matchSse4a(instr, kInsertq, kInsertqRegisterForm); - if (instr.IsMcommit()) { - return Amd64OnlyMatch{kMcommit.Name, length, _bytesOf(kMcommit)}; - } + if (instr.IsMonitorx()) + return _unsupported(kMonitorx, length); + + if (instr.IsMwaitx()) + return _unsupported(kMwaitx, length); + + if (instr.IsClzero()) + return _unsupported(kClzero, length); + + if (instr.IsRdpru()) + return _unsupported(kRdpru, length); + + if (instr.IsMcommit()) + return _unsupported(kMcommit, length); return std::nullopt; } diff --git a/core/relinker/codegen/src/x86/DecodedInstruction.cpp b/core/relinker/codegen/src/x86/DecodedInstruction.cpp index 3ee2a12854..215d6890b3 100644 --- a/core/relinker/codegen/src/x86/DecodedInstruction.cpp +++ b/core/relinker/codegen/src/x86/DecodedInstruction.cpp @@ -58,6 +58,13 @@ std::size_t DecodedInstruction::_skipPrefixesAndRex( return pos; } +std::size_t DecodedInstruction::OpcodeOffset() const { + bool operandSizeOverride = false; + bool repnePrefix = false; + bool repPrefix = false; + return _skipPrefixesAndRex(&operandSizeOverride, &repnePrefix, &repPrefix); +} + bool DecodedInstruction::IsShaNi() const { bool operandSizeOverride = false; bool repnePrefix = false; diff --git a/core/relinker/codegen/src/x86/Sse4aLowering.cpp b/core/relinker/codegen/src/x86/Sse4aLowering.cpp new file mode 100644 index 0000000000..758028a365 --- /dev/null +++ b/core/relinker/codegen/src/x86/Sse4aLowering.cpp @@ -0,0 +1,214 @@ +#include +#include +#include +#include +#include +#include + +namespace Codegen { + +namespace { + +using namespace Amd64OnlySubstitutionTable; + +using Constant = std::array; + +constexpr std::uint8_t kPrefixPacked = 0x66; +constexpr std::uint8_t kPrefixScalar = 0xF3; +constexpr std::uint8_t kRexBase = 0x40; +constexpr std::uint8_t kRexR = 0x04; +constexpr std::uint8_t kRexB = 0x01; +constexpr std::uint8_t kModRmRegister = 0xC0; +constexpr std::uint8_t kModRmRip = 0x05; +constexpr std::uint8_t kModRmRspBase = 0x04; +constexpr std::uint8_t kSibRsp = 0x24; +constexpr std::uint8_t kShiftRight = 2; +constexpr std::uint8_t kShiftLeft = 6; +constexpr std::uint8_t kFieldBits = 64; + +std::uint8_t _rex(const std::uint8_t reg, const std::uint8_t rm) { + return static_cast(kRexBase | ((reg & 8) != 0 ? kRexR : 0) | ((rm & 8) != 0 ? kRexB : 0)); +} + +void _emit(std::vector& out, const std::uint8_t prefix, const std::uint8_t reg, const std::uint8_t rm, const std::initializer_list opcode, const std::uint8_t modrm) { + out.push_back(prefix); + const auto rex = _rex(reg, rm); + if (rex != kRexBase) + out.push_back(rex); + out.insert(out.end(), opcode.begin(), opcode.end()); + out.push_back(modrm); +} + +void _sse(std::vector& out, const std::uint8_t prefix, const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src) { + _emit(out, prefix, dst, src, opcode, static_cast(kModRmRegister | ((dst & 7) << 3) | (src & 7))); +} + +void _shiftImm(std::vector& out, const std::uint8_t extension, const std::uint8_t reg, const std::uint8_t imm) { + _emit(out, kPrefixPacked, 0, reg, {0x0F, 0x73}, static_cast(kModRmRegister | (extension << 3) | (reg & 7))); + out.push_back(imm); +} + +void _nopFill(std::vector& out, std::size_t count) { + while (count > 0) { + const auto& nop = kNops[std::min(count, std::size(kNops)) - 1]; + out.insert(out.end(), nop.Bytes, nop.Bytes + nop.Size); + count -= nop.Size; + } +} + +std::uint64_t _fieldMask(const std::uint8_t length) { + return length >= kFieldBits ? ~std::uint64_t{0} : ((std::uint64_t{1} << length) - 1); +} + +class BodyBuilder { +public: + void Sse(const std::uint8_t prefix, const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src) { + _sse(_bytes, prefix, opcode, dst, src); + } + + void ShiftImm(const std::uint8_t extension, const std::uint8_t reg, const std::uint8_t imm) { + _shiftImm(_bytes, extension, reg, imm); + } + + void RipOperand(const std::initializer_list opcode, const std::uint8_t reg, const Constant& constant) { + _emit(_bytes, kPrefixPacked, reg, 0, opcode, static_cast(((reg & 7) << 3) | kModRmRip)); + _fixups.push_back({_bytes.size(), _bytes.size() + 4, _constants.size()}); + _bytes.insert(_bytes.end(), 4, 0); + _constants.push_back(constant); + } + + void Spill(const std::uint8_t reg) { + _bytes.insert(_bytes.end(), kLeaRspBelowRedZone.Bytes, kLeaRspBelowRedZone.Bytes + kLeaRspBelowRedZone.Size); + _emit(_bytes, kPrefixScalar, reg, 0, {0x0F, 0x7F}, static_cast(((reg & 7) << 3) | kModRmRspBase)); + _bytes.push_back(kSibRsp); + } + + void Restore(const std::uint8_t reg) { + _emit(_bytes, kPrefixScalar, reg, 0, {0x0F, 0x6F}, static_cast(((reg & 7) << 3) | kModRmRspBase)); + _bytes.push_back(kSibRsp); + _bytes.insert(_bytes.end(), kLeaRspRestore.Bytes, kLeaRspRestore.Bytes + kLeaRspRestore.Size); + } + + LoweredBody Finish() { + const auto returnBranchOffset = _bytes.size(); + _bytes.insert(_bytes.end(), kJmpRel32.Bytes, kJmpRel32.Bytes + kJmpRel32.Size); + std::vector constantOffsets; + for (const auto& constant : _constants) { + while (_bytes.size() % kStubAlignment != 0) + _bytes.push_back(kTrapFill); + constantOffsets.push_back(_bytes.size()); + _bytes.insert(_bytes.end(), constant.begin(), constant.end()); + } + for (const auto& fixup : _fixups) { + const auto displacement = static_cast(constantOffsets[fixup.ConstantIndex]) - static_cast(fixup.InstructionEnd); + const auto value = static_cast(static_cast(displacement)); + for (std::size_t index = 0; index < 4; ++index) + _bytes[fixup.DisplacementOffset + index] = static_cast(value >> (index * 8)); + } + return {std::move(_bytes), returnBranchOffset}; + } + +private: + struct Fixup { + std::size_t DisplacementOffset; + std::size_t InstructionEnd; + std::size_t ConstantIndex; + }; + + std::vector _bytes; + std::vector _fixups; + std::vector _constants; +}; + +} + +std::optional> Sse4aLowering::LowerInPlace(const Sse4aOperands& operands, const std::size_t originalLength) const { + if (operands.RegisterForm) + return std::nullopt; + const auto length = operands.Length; + const auto index = operands.Index; + const auto dst = operands.Destination; + const auto src = operands.Source; + std::vector sequence; + if (operands.Insertq) { + if (dst == src && index == 0) { + } else if (length == kFieldBits && index == 0) { + _sse(sequence, kPrefixScalar, {0x0F, 0x7E}, dst, src); + } else if (index == 0 && length % 16 == 0) { + _sse(sequence, kPrefixPacked, {0x0F, 0x3A, 0x0E}, dst, src); + sequence.push_back(static_cast((1u << (length / 16)) - 1)); + } else { + return std::nullopt; + } + } else { + if (index == 0 && length == kFieldBits) { + } else if (index + length == kFieldBits) { + _shiftImm(sequence, kShiftRight, dst, index); + } else if (index == 0 && (length == 8 || length == 16 || length == 32)) { + const std::uint8_t opcode = length == 8 ? 0x32 : (length == 16 ? 0x34 : 0x35); + _sse(sequence, kPrefixPacked, {0x0F, 0x38, opcode}, dst, dst); + } else { + return std::nullopt; + } + } + if (sequence.size() > originalLength) + return std::nullopt; + _nopFill(sequence, originalLength - sequence.size()); + return sequence; +} + +LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands) const { + if (operands.RegisterForm) + throw CodegenException("EXTRQ/INSERTQ register form has no Intel lowering"); + const auto length = operands.Length; + const auto index = operands.Index; + const auto dst = operands.Destination; + const auto src = operands.Source; + const bool byteAligned = length % 8 == 0 && index % 8 == 0; + BodyBuilder body; + if (!operands.Insertq) { + if (byteAligned) { + Constant mask; + mask.fill(kPshufbZero); + for (std::size_t byte = 0; byte < length / 8; ++byte) + mask[byte] = static_cast(index / 8 + byte); + body.RipOperand({0x0F, 0x38, 0x00}, dst, mask); + } else { + if (index != 0) + body.ShiftImm(kShiftRight, dst, index); + if (length != kFieldBits) { + body.ShiftImm(kShiftLeft, dst, static_cast(kFieldBits - length)); + body.ShiftImm(kShiftRight, dst, static_cast(kFieldBits - length)); + } + } + } else if (byteAligned) { + if (src != dst) + body.Sse(kPrefixPacked, {0x0F, 0x6C}, dst, src); + Constant mask; + mask.fill(kPshufbZero); + for (std::size_t byte = 0; byte < 8; ++byte) + mask[byte] = static_cast(byte); + for (std::size_t byte = index / 8; byte < index / 8 + length / 8; ++byte) + mask[byte] = static_cast((src != dst ? 8 : 0) + (byte - index / 8)); + body.RipOperand({0x0F, 0x38, 0x00}, dst, mask); + } else { + std::uint8_t scratch = 0; + while (scratch == dst || scratch == src) + ++scratch; + body.Spill(scratch); + body.Sse(kPrefixPacked, {0x0F, 0x6F}, scratch, src); + if (index != 0) + body.ShiftImm(kShiftLeft, scratch, index); + body.Sse(kPrefixPacked, {0x0F, 0xEF}, scratch, dst); + Constant hole{}; + const auto holeMask = _fieldMask(length) << index; + for (std::size_t byte = 0; byte < 8; ++byte) + hole[byte] = static_cast(holeMask >> (byte * 8)); + body.RipOperand({0x0F, 0xDB}, scratch, hole); + body.Sse(kPrefixPacked, {0x0F, 0xEF}, dst, scratch); + body.Restore(scratch); + } + return body.Finish(); +} + +} diff --git a/core/relinker/codegen/src/x86/Sse4aOperands.cpp b/core/relinker/codegen/src/x86/Sse4aOperands.cpp new file mode 100644 index 0000000000..1e6feb4226 --- /dev/null +++ b/core/relinker/codegen/src/x86/Sse4aOperands.cpp @@ -0,0 +1,84 @@ +#include +#include +#include + +namespace Codegen { + +using namespace X64OpcodeConstants; + +Sse4aOperands DecodeSse4a(const std::uint8_t* data, const std::size_t length) { + std::size_t pos = 0; + bool operandSizeOverride = false; + bool repnePrefix = false; + + while (pos < length) { + const std::uint8_t b = data[pos]; + if (b == PrefixOperandSize) { + operandSizeOverride = true; + } else if (b == PrefixRepne) { + repnePrefix = true; + } else if (b != PrefixLock && b != PrefixRep && b != PrefixAddressSize && + b != PrefixSegCs && b != PrefixSegSs && b != PrefixSegDs && + b != PrefixSegEs && b != PrefixSegFs && b != PrefixSegGs) { + break; + } + pos += 1; + } + + std::uint8_t rex = 0; + if (pos < length && data[pos] >= RexMin && data[pos] <= RexMax) { + rex = data[pos]; + pos += 1; + } + + if (pos + 3 > length || data[pos] != TwoByteOpcodeEscape || + (data[pos + 1] != TwoByteExtrqInsertqImm8Imm8 && data[pos + 1] != TwoByteExtrqInsertqModRm) || + (!operandSizeOverride && !repnePrefix)) { + throw CodegenException("Not an SSE4a instruction"); + } + + const std::uint8_t opcode = data[pos + 1]; + const std::uint8_t modrm = data[pos + 2]; + pos += 3; + + if (((modrm >> ModRmModShift) & ModRmModMask) != ModRmModRegister) { + throw CodegenException("SSE4a instruction with a memory operand"); + } + + const auto regField = static_cast((modrm >> ModRmRegShift) & ModRmRegMask); + const auto reg = static_cast(regField | (((rex & 0x4) != 0) ? 8 : 0)); + const auto rm = static_cast((modrm & ModRmRmMask) | (((rex & 0x1) != 0) ? 8 : 0)); + + Sse4aOperands operands{}; + operands.Insertq = repnePrefix; + operands.RegisterForm = opcode == TwoByteExtrqInsertqModRm; + operands.Destination = reg; + operands.Source = rm; + + if (operands.RegisterForm) { + return operands; + } + + if (pos + 2 > length) { + throw CodegenException("SSE4a instruction truncated before its immediates"); + } + + if (!operands.Insertq) { + if (regField != 0) { + throw CodegenException("EXTRQ immediate form with a non-zero reg field"); + } + operands.Destination = rm; + } + + const auto rawLength = static_cast(data[pos] & 0x3F); + operands.Length = rawLength == 0 ? 64 : rawLength; + operands.Index = static_cast(data[pos + 1] & 0x3F); + + if (static_cast(operands.Length) + operands.Index > 64) { + throw CodegenException("SSE4a field exceeds 64 bits"); + } + + return operands; +} + +} diff --git a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp index 85c42cb9cb..fe0ee7ef9a 100644 --- a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp +++ b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp @@ -202,9 +202,18 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t if (opcode == OneByteJmpRel8 || opcode == OneByteJrcxz || (opcode >= OneByteLoop && opcode <= OneByteLoopMax) || - (opcode >= OneByteInOutImm8Min && opcode <= OneByteInOutImm8Max)) { + (opcode >= OneByteInOutImm8Min && opcode <= OneByteInOutImm8Max) || + opcode == OneByteInt) { immediateSize = ImmSize8; } + + if (opcode == OneByteRetImm16 || opcode == OneByteRetFarImm16) { + immediateSize = ImmSize16; + } + + if (opcode == OneByteEnter) { + immediateSize = ImmSizeEnter; + } } else { if (opcode == TwoByteExtrqInsertqImm8Imm8) { if (!operandSizeOverride && !repnePrefix) { @@ -227,10 +236,15 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t immediateSize = ImmSize8; } else if (opcode == TwoByteShldImm8 || opcode == TwoByteShrdImm8 || opcode == TwoByteShufpsImm8 || opcode == TwoByteShufpdImm8 || - opcode == TwoBytePextrw) { + opcode == TwoBytePextrw || opcode == TwoBytePinsrw) { hasModRm = true; immediateSize = ImmSize8; - } else if (opcode == TwoByteShldCl || opcode == TwoByteShrdCl) { + } else if (opcode == TwoByteShldCl || opcode == TwoByteShrdCl || + opcode == TwoBytePopcnt || opcode == TwoByteUd1 || opcode == TwoByteBtc || + opcode == TwoBytePrefetchw || opcode == TwoByteXaddRm8 || + opcode == TwoByteMovnti || opcode == TwoByteGrp6 || + opcode == TwoByteLar || opcode == TwoByteLsl || + (opcode >= TwoByteMovCrDrMin && opcode <= TwoByteMovCrDrMax)) { hasModRm = true; } else if ((opcode >= TwoByteCmovRangeMin && opcode <= TwoByteCmovRangeMax) || (opcode >= TwoByteModRmRangeAMin && opcode <= TwoByteModRmRangeAMax) || @@ -503,7 +517,13 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( (op >= TwoByteShiftImm8Min && op <= TwoByteShiftImm8Max) || op == TwoByteShldImm8 || op == TwoByteShrdImm8 || op == TwoByteShufpsImm8 || op == TwoByteShufpdImm8 || - op == TwoBytePextrw || op == TwoByteShldCl || op == TwoByteShrdCl) + op == TwoBytePextrw || op == TwoBytePinsrw || + op == TwoByteShldCl || op == TwoByteShrdCl || + op == TwoBytePopcnt || op == TwoByteUd1 || op == TwoByteBtc || + op == TwoBytePrefetchw || op == TwoByteXaddRm8 || + op == TwoByteMovnti || op == TwoByteGrp6 || + op == TwoByteLar || op == TwoByteLsl || + (op >= TwoByteMovCrDrMin && op <= TwoByteMovCrDrMax)) hasModRm = true; } diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp new file mode 100644 index 0000000000..88a57717d3 --- /dev/null +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -0,0 +1,339 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Bytes = std::vector; + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +void requireFailure(const std::function& operation, const char* message) { + try { + operation(); + } catch (const Codegen::CodegenException&) { + return; + } catch (const Domain::RelinkerException&) { + return; + } + throw std::runtime_error(message); +} + +Domain::FileByteOffset failureOffset(const std::function& operation, const char* message) { + try { + operation(); + } catch (const Codegen::CodegenException& error) { + return error.FailureOffset; + } + throw std::runtime_error(message); +} + +template +void write(Bytes& bytes, std::size_t offset, TValue value) { + if (offset > bytes.size() || sizeof(value) > bytes.size() - offset) throw std::runtime_error("Test fixture write is out of bounds"); + std::memcpy(bytes.data() + offset, &value, sizeof(value)); +} + +template +TValue read(const Bytes& bytes, std::size_t offset) { + TValue value; + if (offset > bytes.size() || sizeof(value) > bytes.size() - offset) throw std::runtime_error("Test fixture read is out of bounds"); + std::memcpy(&value, bytes.data() + offset, sizeof(value)); + return value; +} + +Bytes withReturn(Bytes body, const std::size_t returnBranchOffset) { + body.at(returnBranchOffset) = 0xE9; + for (std::size_t index = 1; index <= 4; ++index) body.at(returnBranchOffset + index) = 0; + return body; +} + +const Bytes kExtrqSite = {0x66, 0x0F, 0x78, 0xC3, 0x08, 0x28}; +const Bytes kInsertqSelfSite = {0xF2, 0x0F, 0x78, 0xDB, 0x08, 0x08}; +const Bytes kInsertqCrossSite = {0xF2, 0x0F, 0x78, 0xC8, 0x08, 0x00}; +const Bytes kInsertqHighSite = {0xF2, 0x44, 0x0F, 0x78, 0xCC, 0x10, 0x10}; +const Bytes kInsertqWordSite = {0xF2, 0x0F, 0x78, 0xDC, 0x10, 0x10}; + +const Bytes kExtrqBody = { + 0x66, 0x0F, 0x38, 0x00, 0x1D, 0x07, 0x00, 0x00, 0x00, 0xE9, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xCC, + 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; +const Bytes kInsertqSelfBody = { + 0x66, 0x0F, 0x38, 0x00, 0x1D, 0x07, 0x00, 0x00, 0x00, 0xE9, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xCC, + 0x00, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; +const Bytes kInsertqCrossBody = { + 0x66, 0x0F, 0x6C, 0xC8, 0x66, 0x0F, 0x38, 0x00, 0x0D, 0x13, 0x00, 0x00, 0x00, 0xE9, 0x00, 0x00, + 0x00, 0x00, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, + 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; +const Bytes kInsertqHighBody = { + 0x66, 0x44, 0x0F, 0x6C, 0xCC, 0x66, 0x44, 0x0F, 0x38, 0x00, 0x0D, 0x11, 0x00, 0x00, 0x00, 0xE9, + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, + 0x00, 0x01, 0x08, 0x09, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; +const Bytes kInsertqWordBody = { + 0x66, 0x0F, 0x6C, 0xDC, 0x66, 0x0F, 0x38, 0x00, 0x1D, 0x13, 0x00, 0x00, 0x00, 0xE9, 0x00, 0x00, + 0x00, 0x00, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, + 0x00, 0x01, 0x08, 0x09, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +void decoderLengths() { + const Codegen::X64InstructionDecoder decoder; + const std::vector instructions = { + kExtrqSite, kInsertqSelfSite, kInsertqCrossSite, kInsertqHighSite, kInsertqWordSite, + {0x66, 0x0F, 0x79, 0xCA}, {0xF2, 0x0F, 0x79, 0xCA}, {0x66, 0x45, 0x0F, 0x79, 0xCA}, + {0xF3, 0x0F, 0xB8, 0xC0}, {0xCD, 0x41}, {0x0F, 0x0D, 0x08}, {0x0F, 0xC0, 0xC1}, {0x0F, 0xC3, 0x07}, + {0x66, 0x0F, 0xC4, 0xC0, 0x01}, {0xC2, 0x08, 0x00}, {0xC8, 0x10, 0x00, 0x00}, {0xF3, 0x0F, 0x2B, 0x07}, + {0xF2, 0x44, 0x0F, 0x2B, 0x4C, 0x24, 0x10}, {0x0F, 0x01, 0xFA}, {0x0F, 0xB9, 0x00}, + {0x41, 0x0F, 0xBB, 0xF7}, {0x0F, 0xBB, 0x47, 0x08}}; + Bytes padded; + for (const auto& instruction : instructions) { + padded = instruction; + padded.insert(padded.end(), 8, 0x90); + require(decoder.Decode(padded.data(), padded.size()) == instruction.size(), "AMD-only or repaired two-byte opcode was decoded with the wrong length"); + } + requireFailure([&] { const Bytes bare = {0x0F, 0x78, 0xC3, 0x08, 0x28}; (void)decoder.Decode(bare.data(), bare.size()); }, "0F 78 without an SSE4a prefix was accepted"); +} + +void sse4aOperands() { + const auto check = [](const Bytes& site, const bool insertq, const int dst, const int src, const int length, const int index) { + const auto operands = Codegen::DecodeSse4a(site.data(), site.size()); + require(operands.Insertq == insertq && !operands.RegisterForm && operands.Destination == dst && operands.Source == src && operands.Length == length && operands.Index == index, "SSE4a operands were decoded incorrectly"); + }; + check(kExtrqSite, false, 3, 3, 8, 40); + check(kInsertqSelfSite, true, 3, 3, 8, 8); + check(kInsertqCrossSite, true, 1, 0, 8, 0); + check(kInsertqHighSite, true, 9, 4, 16, 16); + check(kInsertqWordSite, true, 3, 4, 16, 16); + const Bytes fullField = {0xF2, 0x0F, 0x78, 0xC8, 0x00, 0x00}; + require(Codegen::DecodeSse4a(fullField.data(), fullField.size()).Length == 64, "Zero length does not mean 64"); + const Bytes registerForm = {0x66, 0x45, 0x0F, 0x79, 0xCA}; + const auto decoded = Codegen::DecodeSse4a(registerForm.data(), registerForm.size()); + require(decoded.RegisterForm && !decoded.Insertq && decoded.Destination == 9 && decoded.Source == 10, "Register form operands were decoded incorrectly"); + requireFailure([] { const Bytes bytes = {0x66, 0x0F, 0x78, 0xCB, 0x08, 0x28}; (void)Codegen::DecodeSse4a(bytes.data(), bytes.size()); }, "EXTRQ with a non-zero reg field was accepted"); + requireFailure([] { const Bytes bytes = {0xF2, 0x0F, 0x78, 0x1B, 0x08, 0x08}; (void)Codegen::DecodeSse4a(bytes.data(), bytes.size()); }, "SSE4a memory operand was accepted"); + requireFailure([] { const Bytes bytes = {0xF2, 0x0F, 0x78, 0xC8, 0x20, 0x30}; (void)Codegen::DecodeSse4a(bytes.data(), bytes.size()); }, "Field beyond bit 64 was accepted"); +} + +void matcherSubstitutions() { + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); + const auto match = [&](const Bytes& bytes) { return matcher->Match(bytes.data(), bytes.size()); }; + const auto movntss = match({0xF3, 0x0F, 0x2B, 0x07}); + require(movntss && movntss->Lowering == Codegen::Amd64OnlyLowering::InPlace && movntss->ReplacementBytes == Bytes{0xF3, 0x0F, 0x11, 0x07} && movntss->InstructionName == "MOVNTSS", "MOVNTSS was not rewritten to MOVSS"); + const auto movntsd = match({0xF2, 0x44, 0x0F, 0x2B, 0x4C, 0x24, 0x10}); + require(movntsd && movntsd->Lowering == Codegen::Amd64OnlyLowering::InPlace && movntsd->ReplacementBytes == Bytes{0xF2, 0x44, 0x0F, 0x11, 0x4C, 0x24, 0x10} && movntsd->InstructionName == "MOVNTSD", "MOVNTSD was not rewritten to MOVSD"); + requireFailure([&] { (void)match({0xF3, 0x0F, 0x2B, 0xC1}); }, "MOVNTSS with a register operand was accepted"); + const auto monitorx = match({0x0F, 0x01, 0xFA}); + require(monitorx && monitorx->Lowering == Codegen::Amd64OnlyLowering::Unsupported && monitorx->InstructionName == "MONITORX", "MONITORX was not reported as unsupported"); + const auto registerForm = match({0x66, 0x0F, 0x79, 0xCA}); + require(registerForm && registerForm->Lowering == Codegen::Amd64OnlyLowering::Unsupported, "EXTRQ register form was not reported as unsupported"); + require(!match({0x66, 0x0F, 0x2B, 0x07}) && !match({0x0F, 0x2B, 0x07}) && !match({0x48, 0x8B, 0x05, 0, 0, 0, 0}), "Ordinary instruction was matched"); + const auto stub = match(kInsertqHighSite); + require(stub && stub->Lowering == Codegen::Amd64OnlyLowering::Trampoline && stub->StubBody == kInsertqHighBody && stub->ReturnBranchOffset == 15 && stub->InstructionName == "INSERTQ", "INSERTQ was not lowered through a stub"); + const auto shiftInPlace = match({0x66, 0x0F, 0x78, 0xC3, 0x18, 0x28}); + require(shiftInPlace && shiftInPlace->Lowering == Codegen::Amd64OnlyLowering::InPlace && shiftInPlace->ReplacementBytes == Bytes{0x66, 0x0F, 0x73, 0xD3, 0x28, 0x90}, "Top-aligned EXTRQ was not lowered in place"); +} + +void goldenBodies() { + const Codegen::Sse4aLowering lowering; + const auto outOfLine = [&](const Bytes& site, const Bytes& expected, const std::size_t returnBranchOffset) { + const auto operands = Codegen::DecodeSse4a(site.data(), site.size()); + require(!lowering.LowerInPlace(operands, site.size()).has_value(), "Demon's Souls site unexpectedly qualified for an in-place lowering"); + const auto body = lowering.LowerOutOfLine(operands); + require(body.ReturnBranchOffset == returnBranchOffset, "Stub return branch is at the wrong offset"); + require(body.Bytes == expected, "Stub body differs from the golden encoding"); + }; + outOfLine(kExtrqSite, kExtrqBody, 9); + outOfLine(kInsertqSelfSite, kInsertqSelfBody, 9); + outOfLine(kInsertqCrossSite, kInsertqCrossBody, 13); + outOfLine(kInsertqHighSite, kInsertqHighBody, 15); + outOfLine(kInsertqWordSite, kInsertqWordBody, 13); + const auto inPlace = [&](const Bytes& site, const Bytes& expected) { + const auto operands = Codegen::DecodeSse4a(site.data(), site.size()); + const auto sequence = lowering.LowerInPlace(operands, site.size()); + require(sequence.has_value() && *sequence == expected, "In-place lowering differs from the golden encoding"); + }; + inPlace({0xF2, 0x0F, 0x78, 0xC8, 0x00, 0x00}, {0xF3, 0x0F, 0x7E, 0xC8, 0x66, 0x90}); + inPlace({0x66, 0x0F, 0x78, 0xC3, 0x18, 0x28}, {0x66, 0x0F, 0x73, 0xD3, 0x28, 0x90}); + inPlace({0x66, 0x0F, 0x78, 0xC3, 0x08, 0x00}, {0x66, 0x0F, 0x38, 0x32, 0xDB, 0x90}); + inPlace({0xF2, 0x0F, 0x78, 0xDB, 0x08, 0x00}, {0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00}); + inPlace({0xF2, 0x0F, 0x78, 0xC8, 0x20, 0x00}, {0x66, 0x0F, 0x3A, 0x0E, 0xC8, 0x03}); + inPlace({0xF2, 0x45, 0x0F, 0x78, 0xC8, 0x10, 0x00}, {0x66, 0x45, 0x0F, 0x3A, 0x0E, 0xC8, 0x01}); + const auto highRegisters = Codegen::DecodeSse4a(kInsertqHighSite.data(), kInsertqHighSite.size()); + const auto generic = lowering.LowerOutOfLine(Codegen::Sse4aOperands{true, false, 9, 4, 5, 3}); + require(generic.Bytes[0] == 0x48 && generic.Bytes.size() % 16 == 0 && generic.ReturnBranchOffset < generic.Bytes.size(), "Generic INSERTQ body does not start with the red-zone skip"); + (void)highRegisters; + requireFailure([&] { (void)lowering.LowerOutOfLine(Codegen::Sse4aOperands{true, true, 1, 2, 0, 0}); }, "Register form was lowered out of line"); +} + +Bytes segmentFixture() { + Bytes file(0x300, 0xCC); + const Bytes text = { + 0xF3, 0x0F, 0xB8, 0xC0, + 0xCD, 0x41, + 0xEB, 0x07, + 0xF2, 0x44, 0x0F, 0x78, 0xCC, 0x10, 0x10, + 0xF3, 0x0F, 0x2B, 0x07, + 0xC3}; + std::copy(text.begin(), text.end(), file.begin() + 0x200); + return file; +} + +Domain::ProgramHeader segmentHeader(const std::uint64_t size) { + return {1, 5, 0x200, 0x1000, 0, size, size, 16}; +} + +void converterSegment() { + const auto converter = Codegen::MakeAmd64OnlyConverter(); + const auto file = segmentFixture(); + const auto result = converter->Convert(file, {segmentHeader(20)}); + require(result.ReplacedCount == 1 && result.Reports.size() == 2 && result.Trampolines.size() == 1, "Converter did not classify the segment's AMD-only instructions"); + const auto& site = result.Trampolines[0]; + require(site.Offset == 0x208 && site.Address == 0x1008 && site.Length == 7 && site.OriginalBytes == kInsertqHighSite && site.Body == kInsertqHighBody && site.ReturnBranchOffset == 15, "Trampoline site was recorded incorrectly"); + require(result.Reports[0].InstructionName == "INSERTQ" && result.Reports[0].Offset == 0x208 && result.Reports[0].Lowering == Codegen::Amd64OnlyLowering::Trampoline && result.Reports[0].ReplacementLength == 48, "Trampoline report is wrong"); + require(result.Reports[1].InstructionName == "MOVNTSS" && result.Reports[1].Offset == 0x20F && result.Reports[1].Lowering == Codegen::Amd64OnlyLowering::InPlace && result.Reports[1].ReplacementLength == 4, "In-place report is wrong"); + auto expected = file; + expected[0x211] = 0x11; + require(result.Bytes == expected, "Converter changed bytes other than the MOVNTSS opcode"); + const auto untouched = converter->Convert(Bytes(0x300, 0x90), {segmentHeader(0x100)}); + require(untouched.ReplacedCount == 0 && untouched.Trampolines.empty() && untouched.Reports.empty() && untouched.Bytes == Bytes(0x300, 0x90), "Segment without AMD-only instructions was changed"); + auto branchInside = file; + branchInside[0x207] = 0x02; + requireFailure([&] { (void)converter->Convert(branchInside, {segmentHeader(20)}); }, "Branch into an AMD-only instruction was accepted"); + auto monitorx = file; + monitorx[0x20F] = 0x0F; + monitorx[0x210] = 0x01; + monitorx[0x211] = 0xFA; + monitorx[0x212] = 0x90; + requireFailure([&] { (void)converter->Convert(monitorx, {segmentHeader(20)}); }, "MONITORX was silently kept"); + auto registerForm = file; + const Bytes extrqRegister = {0x66, 0x0F, 0x79, 0xCA}; + std::copy(extrqRegister.begin(), extrqRegister.end(), registerForm.begin() + 0x20F); + requireFailure([&] { (void)converter->Convert(registerForm, {segmentHeader(20)}); }, "EXTRQ register form was silently kept"); + requireFailure([&] { (void)converter->Convert(file, {segmentHeader(0x200)}); }, "Segment exceeding the file was accepted"); +} + +void converterFailureOffsets() { + const auto converter = Codegen::MakeAmd64OnlyConverter(); + const auto file = segmentFixture(); + auto undecodable = file; + const Bytes bareSse4a = {0x0F, 0x78, 0xC0, 0x00}; + std::copy(bareSse4a.begin(), bareSse4a.end(), undecodable.begin() + 0x20F); + require(failureOffset([&] { (void)converter->Convert(undecodable, {segmentHeader(20)}); }, "Undecodable instruction was accepted") == 0x20F, "Decoder failure does not carry the file offset"); + auto memoryForm = file; + memoryForm[0x20C] = 0x08; + require(failureOffset([&] { (void)converter->Convert(memoryForm, {segmentHeader(20)}); }, "INSERTQ memory form was accepted") == 0x208, "SSE4a operand failure does not carry the file offset"); + auto movntsRegister = file; + movntsRegister[0x212] = 0xC1; + require(failureOffset([&] { (void)converter->Convert(movntsRegister, {segmentHeader(20)}); }, "MOVNTSS register form was accepted") == 0x20F, "MOVNTSS failure does not carry the file offset"); + auto monitorx = file; + const Bytes monitorxBytes = {0x0F, 0x01, 0xFA, 0x90}; + std::copy(monitorxBytes.begin(), monitorxBytes.end(), monitorx.begin() + 0x20F); + require(failureOffset([&] { (void)converter->Convert(monitorx, {segmentHeader(20)}); }, "MONITORX was accepted") == 0x20F, "Unsupported instruction failure does not carry the file offset"); +} + +Bytes elfFixture(const Bytes& text) { + Bytes bytes(0x400); + bytes[0] = 0x7F; + bytes[1] = 'E'; + bytes[2] = 'L'; + bytes[3] = 'F'; + bytes[4] = 2; + bytes[5] = 1; + bytes[6] = 1; + write(bytes, 16, 3); + write(bytes, 18, 62); + write(bytes, 24, 0x1000); + write(bytes, 32, 64); + write(bytes, 54, 56); + write(bytes, 56, 6); + write(bytes, 64, 1); + write(bytes, 68, 5); + write(bytes, 72, 0x200); + write(bytes, 80, 0x1000); + write(bytes, 96, 0x100); + write(bytes, 104, 0x100); + write(bytes, 112, 0x1000); + write(bytes, 120, 1); + write(bytes, 124, 6); + write(bytes, 128, 0x300); + write(bytes, 136, 0x2000); + write(bytes, 152, 0x100); + write(bytes, 160, 0x100); + write(bytes, 168, 0x1000); + std::fill(bytes.begin() + 0x200, bytes.begin() + 0x300, 0xCC); + std::copy(text.begin(), text.end(), bytes.begin() + 0x200); + return bytes; +} + +std::vector elfHeaders() { + return {{1, 5, 0x200, 0x1000, 0, 0x100, 0x100, 0x1000}, {1, 6, 0x300, 0x2000, 0, 0x100, 0x100, 0x1000}}; +} + +void linuxPlacement() { + const auto source = elfFixture({0xEB, 0x06, 0xF2, 0x0F, 0x78, 0xDB, 0x08, 0x08, 0xC3}); + const auto headers = elfHeaders(); + const auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(source, {headers[0]}); + require(converted.Trampolines.size() == 1 && converted.Bytes == source, "Linux fixture conversion produced unexpected results"); + const auto byteWriter = std::make_shared(); + Elfpatcher::Linux::LinuxElfPatcher patcher( + std::make_shared(), + std::make_shared(std::make_shared(), byteWriter), + std::make_shared(byteWriter), + byteWriter); + const auto output = patcher.Patch(converted.Bytes, headers, {}, 0, "$ORIGIN/libs", true, false, converted.Trampolines); + require(output[0x202] == 0xE9 && output[0x207] == 0x90, "Linux site was not replaced by a jump"); + const auto target = 0x1002 + 5 + static_cast(read(output, 0x203)); + require(target % 16 == 0 && target > 0x2100, "Linux stub is misaligned or inside the original image"); + const auto phNum = read(output, 56); + std::uint64_t bodyOffset = 0; + bool found = false; + for (std::uint16_t index = 0; index < phNum; ++index) { + const auto header = 64 + index * 56; + if (read(output, header) != 1) continue; + const auto vaddr = read(output, header + 16); + const auto memSize = read(output, header + 40); + if (static_cast(target) < vaddr || static_cast(target) >= vaddr + memSize) continue; + require((read(output, header + 4) & 1) != 0, "Linux stub segment is not executable"); + bodyOffset = read(output, header + 8) + (static_cast(target) - vaddr); + found = true; + } + require(found, "Linux stub is not inside a PT_LOAD segment"); + auto expectedBody = kInsertqSelfBody; + write(expectedBody, 10, static_cast(0x1008 - (target + 9 + 5))); + const Bytes actualBody(output.begin() + static_cast(bodyOffset), output.begin() + static_cast(bodyOffset + expectedBody.size())); + require(actualBody == expectedBody, "Linux stub body or return branch is wrong"); + auto altered = converted.Bytes; + altered[0x205] = 0xDC; + requireFailure([&] { (void)patcher.Patch(altered, headers, {}, 0, "$ORIGIN/libs", true, false, converted.Trampolines); }, "Changed Linux site bytes were accepted"); +} + +} + +int main() { + try { + decoderLengths(); + sse4aOperands(); + matcherSubstitutions(); + goldenBodies(); + converterSegment(); + converterFailureOffsets(); + linuxPlacement(); + std::cout << "AMD64-only converter tests passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/relinker/elfpatcher/include/elfpatcher/general/IElfPatcher.hpp b/core/relinker/elfpatcher/include/elfpatcher/general/IElfPatcher.hpp index 41b667ba67..f01272daa6 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/general/IElfPatcher.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/general/IElfPatcher.hpp @@ -1,6 +1,7 @@ #ifndef ELFPATCHER_IELFPATCHER_HPP #define ELFPATCHER_IELFPATCHER_HPP +#include #include #include #include @@ -19,7 +20,8 @@ public: std::uint64_t originalPltGotVaddr, const std::string& runPath, bool lazyBinding, - bool dependencyDiagnostics + bool dependencyDiagnostics, + const std::vector& trampolines ) = 0; }; diff --git a/core/relinker/elfpatcher/include/elfpatcher/linux/LinuxElfPatcher.hpp b/core/relinker/elfpatcher/include/elfpatcher/linux/LinuxElfPatcher.hpp index 30a9cceef4..ae5782f330 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/linux/LinuxElfPatcher.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/linux/LinuxElfPatcher.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -27,7 +28,8 @@ public: std::uint64_t originalPltGotVaddr, const std::string& runPath, bool lazyBinding, - bool dependencyDiagnostics + bool dependencyDiagnostics, + const std::vector& trampolines ) override; private: @@ -37,6 +39,7 @@ private: std::shared_ptr _byteWriter; void _appendDynEntry(std::vector& buf, std::int64_t tag, std::uint64_t val) const; + void _appendTrampoline(std::vector& buf, const Codegen::TrampolineSite& site, std::uint64_t extraBlockOffset, const std::function& vaddrOfExtraBlockOffset) const; }; } diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsElfPatcher.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsElfPatcher.hpp index 2ee1017a09..2552cba55f 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsElfPatcher.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsElfPatcher.hpp @@ -7,7 +7,7 @@ namespace Elfpatcher::Windows { class WindowsPePatcher : public IElfPatcher { public: - std::vector Patch(const std::vector& sourceElf, const std::vector& originalHeaders, const Domain::SysVDynamicSection& dynamicSection, std::uint64_t originalPltGotVaddr, const std::string& runPath, bool lazyBinding, bool dependencyDiagnostics) override; + std::vector Patch(const std::vector& sourceElf, const std::vector& originalHeaders, const Domain::SysVDynamicSection& dynamicSection, std::uint64_t originalPltGotVaddr, const std::string& runPath, bool lazyBinding, bool dependencyDiagnostics, const std::vector& trampolines) override; }; } diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTrampolineBuilder.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTrampolineBuilder.hpp new file mode 100644 index 0000000000..d374108912 --- /dev/null +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTrampolineBuilder.hpp @@ -0,0 +1,16 @@ +#ifndef ELFPATCHER_WINDOWS_WINDOWSTRAMPOLINEBUILDER_HPP +#define ELFPATCHER_WINDOWS_WINDOWSTRAMPOLINEBUILDER_HPP + +#include +#include + +namespace Elfpatcher::Windows { + +class WindowsTrampolineBuilder { +public: + void Build(const std::vector& sites, const WindowsLoadImage& image, std::vector& sections, std::uint32_t& nextRva) const; +}; + +} + +#endif diff --git a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp index 0172eef324..90ce2bc286 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include #include namespace Elfpatcher::Linux { @@ -24,6 +27,41 @@ void LinuxElfPatcher::_appendDynEntry(std::vector& buf, std::int64 _byteWriter->AppendU64(buf, val); } +void LinuxElfPatcher::_appendTrampoline( + std::vector& buf, + const Codegen::TrampolineSite& site, + const std::uint64_t extraBlockOffset, + const std::function& vaddrOfExtraBlockOffset) const +{ + using namespace Codegen::Amd64OnlySubstitutionTable; + if (site.Length < kJmpRel32.Size || site.OriginalBytes.size() != site.Length || site.Body.size() < kJmpRel32.Size || site.ReturnBranchOffset > site.Body.size() - kJmpRel32.Size || site.Body[site.ReturnBranchOffset] != kJmpRel32.Bytes[0]) + throw Domain::RelinkerException("Invalid AMD-only trampoline site", site.Offset); + if (site.Offset > extraBlockOffset || site.Length > extraBlockOffset - site.Offset) + throw Domain::RelinkerException("AMD-only instruction is outside the original image", site.Offset); + if (!std::equal(site.OriginalBytes.begin(), site.OriginalBytes.end(), buf.begin() + static_cast(site.Offset))) + throw Domain::RelinkerException("AMD-only site bytes changed before patching", site.Offset); + + while (buf.size() % kStubAlignment != 0) + buf.push_back(kTrapFill); + const auto bodyOff = static_cast(buf.size()); + const auto bodyVaddr = vaddrOfExtraBlockOffset(bodyOff); + for (const std::uint8_t b : site.Body) + buf.push_back(b); + + const auto inRange = [](const std::int64_t displacement) { + return displacement >= std::numeric_limits::min() && displacement <= std::numeric_limits::max(); + }; + const auto returnDisplacement = static_cast(site.Address + site.Length) - static_cast(bodyVaddr + site.ReturnBranchOffset + kJmpRel32.Size); + const auto jumpDisplacement = static_cast(bodyVaddr) - static_cast(site.Address + kJmpRel32.Size); + if (!inRange(returnDisplacement) || !inRange(jumpDisplacement)) + throw Domain::RelinkerException("AMD-only stub exceeds rel32 range", site.Offset); + _byteWriter->WriteU32(buf, static_cast(bodyOff + site.ReturnBranchOffset + 1), static_cast(returnDisplacement)); + + std::fill_n(buf.begin() + static_cast(site.Offset), site.Length, kNop1.Bytes[0]); + buf[static_cast(site.Offset)] = kJmpRel32.Bytes[0]; + _byteWriter->WriteU32(buf, static_cast(site.Offset + 1), static_cast(jumpDisplacement)); +} + std::vector LinuxElfPatcher::Patch( const std::vector& sourceElf, const std::vector& originalHeaders, @@ -31,7 +69,8 @@ std::vector LinuxElfPatcher::Patch( const std::uint64_t originalPltGotVaddr, const std::string& runPath, const bool lazyBinding, - const bool dependencyDiagnostics) + const bool dependencyDiagnostics, + const std::vector& trampolines) { if (dependencyDiagnostics) throw Domain::RelinkerException("Linux target does not support --windows-diagnostics"); @@ -123,6 +162,9 @@ std::vector LinuxElfPatcher::Patch( buf.push_back(b); _byteWriter->WriteU64(buf, kEhdrEntryOffset, stubVaddr); + for (const auto& site : trampolines) + _appendTrampoline(buf, site, extraBlockOff, vaddrOfExtraBlockOffset); + static constexpr char kInterp[] = "/lib64/ld-linux-x86-64.so.2"; const auto interpOff = static_cast(buf.size()); constexpr std::uint64_t interpSize = sizeof(kInterp); diff --git a/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp b/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp index 0910e7aad1..68676bc2cf 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,7 @@ void writeGotStub(std::vector& sections, const std::uint32_t targetRv } -std::vector WindowsPePatcher::Patch(const std::vector& sourceElf, const std::vector& originalHeaders, const Domain::SysVDynamicSection& dynamicSection, const std::uint64_t originalPltGotVaddr, const std::string& runPath, const bool lazyBinding, const bool dependencyDiagnostics) { +std::vector WindowsPePatcher::Patch(const std::vector& sourceElf, const std::vector& originalHeaders, const Domain::SysVDynamicSection& dynamicSection, const std::uint64_t originalPltGotVaddr, const std::string& runPath, const bool lazyBinding, const bool dependencyDiagnostics, const std::vector& trampolines) { WindowsLoadImage image(sourceElf, originalHeaders); if (originalPltGotVaddr != 0) image.GetRva(originalPltGotVaddr, 8); @@ -67,6 +68,7 @@ std::vector WindowsPePatcher::Patch(const std::vector +#include +#include +#include +#include +#include + +namespace Elfpatcher::Windows { + +namespace { + +PeSection& findSection(std::vector& sections, const std::uint32_t rva, const std::size_t length) { + for (auto& section : sections) { + if (rva < section.Rva || rva - section.Rva >= section.Data.size()) + continue; + if (length > section.Data.size() - (rva - section.Rva)) + throw Domain::RelinkerException("AMD-only instruction crosses a PE section boundary", rva); + return section; + } + throw Domain::RelinkerException("AMD-only instruction is outside the PE image", rva); +} + +} + +void WindowsTrampolineBuilder::Build(const std::vector& sites, const WindowsLoadImage& image, std::vector& sections, std::uint32_t& nextRva) const { + using namespace Codegen::Amd64OnlySubstitutionTable; + if (sites.empty()) + return; + const auto sectionRva = nextRva; + std::vector bytes; + for (const auto& site : sites) { + const auto siteRva = image.GetRva(site.Address, site.Length); + if (site.Length < kJmpRel32.Size || site.OriginalBytes.size() != site.Length || site.Body.size() < kJmpRel32.Size || site.ReturnBranchOffset > site.Body.size() - kJmpRel32.Size || site.Body[site.ReturnBranchOffset] != kJmpRel32.Bytes[0]) + throw Domain::RelinkerException("Invalid AMD-only trampoline site", siteRva); + auto& section = findSection(sections, siteRva, site.Length); + const auto offset = static_cast(siteRva - section.Rva); + if (!std::equal(site.OriginalBytes.begin(), site.OriginalBytes.end(), section.Data.begin() + offset)) + throw Domain::RelinkerException("AMD-only site bytes changed before patching", siteRva); + bytes.resize(Io::AlignUp(bytes.size(), kStubAlignment), kTrapFill); + const auto stubRva = CheckedRva(sectionRva + bytes.size()); + const auto bodyOffset = bytes.size(); + bytes.insert(bytes.end(), site.Body.begin(), site.Body.end()); + const auto returnRva = CheckedRva(siteRva + site.Length); + const auto displacement = static_cast(returnRva) - (static_cast(stubRva) + static_cast(site.ReturnBranchOffset + kJmpRel32.Size)); + if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) + throw Domain::RelinkerException("AMD-only stub return exceeds rel32 range", returnRva); + Io::WriteU32(bytes, bodyOffset + site.ReturnBranchOffset + 1, static_cast(displacement)); + WindowsStubEmitter jump(siteRva); + jump.Rip({0xe9}, stubRva); + const auto jumpBytes = jump.TakeBytes(); + std::fill_n(section.Data.begin() + offset, site.Length, kNop1.Bytes[0]); + std::copy(jumpBytes.begin(), jumpBytes.end(), section.Data.begin() + offset); + } + sections.push_back({".amdstub", sectionRva, SectionRead | SectionExecute | 0x20u, std::move(bytes)}); + nextRva = AlignRva(sectionRva + sections.back().Data.size()); +} + +} diff --git a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp new file mode 100644 index 0000000000..96c4e11602 --- /dev/null +++ b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp @@ -0,0 +1,342 @@ +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace Elfpatcher::Windows; +using Bytes = std::vector; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +void requireFailure(const std::function& operation, const char* message) { + try { + operation(); + } catch (const Domain::RelinkerException&) { + return; + } + throw std::runtime_error(message); +} + +template +void write(Bytes& bytes, std::size_t offset, TValue value) { + if (offset > bytes.size() || sizeof(value) > bytes.size() - offset) throw std::runtime_error("Test fixture write is out of bounds"); + std::memcpy(bytes.data() + offset, &value, sizeof(value)); +} + +template +TValue read(const Bytes& bytes, std::size_t offset) { + TValue value; + if (offset > bytes.size() || sizeof(value) > bytes.size() - offset) throw std::runtime_error("Test fixture read is out of bounds"); + std::memcpy(&value, bytes.data() + offset, sizeof(value)); + return value; +} + +struct alignas(16) State { + std::uint8_t Xmm[16][16]; + std::uint64_t FlagsIn; + std::uint64_t FlagsOut; + std::uint64_t RspBefore; + std::uint64_t RspAfter; + std::uint8_t RedZone[128]; +}; + +constexpr std::uint64_t kCanary = 0xA5C3E17B9D24F608ull; +constexpr std::uint64_t kStatusFlags = 0x8D5; +constexpr std::uint32_t kXmmOffset = 0; +constexpr std::uint32_t kFlagsInOffset = 256; +constexpr std::uint32_t kFlagsOutOffset = 264; +constexpr std::uint32_t kRspBeforeOffset = 272; +constexpr std::uint32_t kRspAfterOffset = 280; +constexpr std::uint32_t kRedZoneOffset = 288; + +static_assert(offsetof(State, FlagsIn) == kFlagsInOffset && offsetof(State, FlagsOut) == kFlagsOutOffset && offsetof(State, RspBefore) == kRspBeforeOffset && offsetof(State, RspAfter) == kRspAfterOffset && offsetof(State, RedZone) == kRedZoneOffset, "State layout drifted"); + +void emit(Bytes& code, std::initializer_list bytes) { + code.insert(code.end(), bytes.begin(), bytes.end()); +} + +void emitU32(Bytes& code, std::uint32_t value) { + for (std::size_t index = 0; index < 4; ++index) code.push_back(static_cast(value >> (index * 8))); +} + +void movdquXmmRbx(Bytes& code, const std::uint8_t reg, const bool store) { + code.push_back(0xF3); + if (reg >= 8) code.push_back(0x44); + emit(code, {0x0F, static_cast(store ? 0x7F : 0x6F), static_cast(0x83 | ((reg & 7) << 3))}); + emitU32(code, kXmmOffset + reg * 16); +} + +void movdquXmmRsp(Bytes& code, const std::uint8_t reg, const bool store, const std::uint32_t offset) { + code.push_back(0xF3); + if (reg >= 8) code.push_back(0x44); + emit(code, {0x0F, static_cast(store ? 0x7F : 0x6F), static_cast(0x84 | ((reg & 7) << 3)), 0x24}); + emitU32(code, offset); +} + +class Harness { +public: + Harness() : _memory(VirtualAlloc(nullptr, kSize, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE)) { + if (_memory == nullptr) throw std::runtime_error("Cannot allocate executable memory"); + } + + ~Harness() { + VirtualFree(_memory, 0, MEM_RELEASE); + } + + void Run(const Bytes& body, const std::size_t returnBranchOffset, State& state) const { + Bytes code; + emit(code, {0x53, 0x55, 0x56, 0x57, 0x41, 0x54, 0x41, 0x55, 0x41, 0x56, 0x41, 0x57, 0x48, 0x81, 0xEC, 0xA8, 0x00, 0x00, 0x00}); + for (std::uint8_t reg = 6; reg < 16; ++reg) movdquXmmRsp(code, reg, true, (reg - 6) * 16); + emit(code, {0x48, 0x89, 0xCB, 0xFF, 0xB3}); + emitU32(code, kFlagsInOffset); + emit(code, {0x9D, 0x48, 0xB8}); + for (std::size_t index = 0; index < 8; ++index) code.push_back(static_cast(kCanary >> (index * 8))); + for (int slot = 0; slot < 16; ++slot) emit(code, {0x48, 0x89, 0x44, 0x24, static_cast(-128 + slot * 8)}); + emit(code, {0x48, 0x89, 0xA3}); + emitU32(code, kRspBeforeOffset); + for (std::uint8_t reg = 0; reg < 16; ++reg) movdquXmmRbx(code, reg, false); + code.push_back(0xE9); + const auto jumpOffset = code.size(); + emitU32(code, 0); + const auto continuation = code.size(); + for (int slot = 0; slot < 16; ++slot) { + emit(code, {0x48, 0x8B, 0x44, 0x24, static_cast(-128 + slot * 8), 0x48, 0x89, 0x83}); + emitU32(code, kRedZoneOffset + slot * 8); + } + emit(code, {0x48, 0x89, 0xA3}); + emitU32(code, kRspAfterOffset); + emit(code, {0x9C, 0x8F, 0x83}); + emitU32(code, kFlagsOutOffset); + for (std::uint8_t reg = 0; reg < 16; ++reg) movdquXmmRbx(code, reg, true); + for (std::uint8_t reg = 6; reg < 16; ++reg) movdquXmmRsp(code, reg, false, (reg - 6) * 16); + emit(code, {0x48, 0x81, 0xC4, 0xA8, 0x00, 0x00, 0x00, 0x41, 0x5F, 0x41, 0x5E, 0x41, 0x5D, 0x41, 0x5C, 0x5F, 0x5E, 0x5D, 0x5B, 0xC3}); + code.resize(Io::AlignUp(code.size(), std::size_t{16}), 0xCC); + const auto bodyOffset = code.size(); + code.insert(code.end(), body.begin(), body.end()); + write(code, jumpOffset, static_cast(bodyOffset - (jumpOffset + 4))); + write(code, bodyOffset + returnBranchOffset + 1, static_cast(continuation - (bodyOffset + returnBranchOffset + 5))); + if (code.size() > kSize) throw std::runtime_error("Harness code exceeds its page"); + std::memcpy(_memory, code.data(), code.size()); + FlushInstructionCache(GetCurrentProcess(), _memory, code.size()); + reinterpret_cast(_memory)(&state); + } + +private: + static constexpr std::size_t kSize = 4096; + void* _memory; +}; + +std::uint64_t low(const std::uint8_t (&lane)[16]) { + std::uint64_t value; + std::memcpy(&value, lane, 8); + return value; +} + +std::uint64_t referenceLow(const Codegen::Sse4aOperands& operands, const State& input) { + const auto mask = operands.Length >= 64 ? ~std::uint64_t{0} : ((std::uint64_t{1} << operands.Length) - 1); + const auto destination = low(input.Xmm[operands.Destination]); + if (!operands.Insertq) return (destination >> operands.Index) & mask; + const auto hole = mask << operands.Index; + return (destination & ~hole) | ((low(input.Xmm[operands.Source]) & mask) << operands.Index); +} + +std::uint64_t libcReferenceLow(const Codegen::Sse4aOperands& operands, const State& input) { + CONTEXT context{}; + for (unsigned reg = 0; reg < 16; ++reg) { + std::memcpy(&context.FltSave.XmmRegisters[reg].Low, input.Xmm[reg], 8); + std::memcpy(&context.FltSave.XmmRegisters[reg].High, input.Xmm[reg] + 8, 8); + } + sse4a::Instruction instruction; + instruction.op = operands.Insertq ? sse4a::Op::Insertq : sse4a::Op::Extrq; + instruction.destination = operands.Destination; + instruction.source = operands.Source; + instruction.length = static_cast(operands.Length == 64 ? 0 : operands.Length); + instruction.index = operands.Index; + sse4a::Execute(instruction, context); + return static_cast(context.FltSave.XmmRegisters[operands.Destination].Low); +} + +Bytes encode(const bool insertq, const std::uint8_t dst, const std::uint8_t src, const std::uint8_t length, const std::uint8_t index) { + Bytes bytes = {static_cast(insertq ? 0xF2 : 0x66)}; + const std::uint8_t reg = insertq ? dst : 0; + const std::uint8_t rm = insertq ? src : dst; + const auto rex = static_cast(0x40 | (reg >= 8 ? 4 : 0) | (rm >= 8 ? 1 : 0)); + if (rex != 0x40) bytes.push_back(rex); + bytes.insert(bytes.end(), {0x0F, 0x78, static_cast(0xC0 | ((reg & 7) << 3) | (rm & 7)), static_cast(length == 64 ? 0 : length), index}); + return bytes; +} + +struct Case { + bool Insertq; + std::uint8_t Destination; + std::uint8_t Source; + std::uint8_t Length; + std::uint8_t Index; +}; + +std::string describe(const Case& item) { + return std::string(item.Insertq ? "insertq xmm" : "extrq xmm") + std::to_string(item.Destination) + (item.Insertq ? ", xmm" + std::to_string(item.Source) : "") + ", " + std::to_string(item.Length) + ", " + std::to_string(item.Index); +} + +std::size_t g_executions = 0; + +void executeCase(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, const Case& item, std::mt19937_64& random, const int images) { + const auto site = encode(item.Insertq, item.Destination, item.Source, item.Length, item.Index); + const auto match = matcher.Match(site.data(), site.size()); + require(match.has_value() && match->Lowering != Codegen::Amd64OnlyLowering::Unsupported, "Immediate-form SSE4a instruction was not lowered: " + describe(item)); + Bytes body; + std::size_t returnBranchOffset = 0; + if (match->Lowering == Codegen::Amd64OnlyLowering::InPlace) { + require(match->ReplacementBytes.size() == site.size(), "In-place lowering changed the length: " + describe(item)); + body = match->ReplacementBytes; + returnBranchOffset = body.size(); + body.insert(body.end(), {0xE9, 0, 0, 0, 0}); + } else { + body = match->StubBody; + returnBranchOffset = match->ReturnBranchOffset; + require(body.size() % 16 == 0 || body.size() == returnBranchOffset + 5, "Stub body with constants is not padded to 16 bytes: " + describe(item)); + } + const auto operands = Codegen::DecodeSse4a(site.data(), site.size()); + const std::array flags = {0x202, 0x203, 0x246, 0xAC7}; + for (int image = 0; image < images; ++image) { + State input{}; + for (auto& lane : input.Xmm) for (auto& byte : lane) byte = static_cast(random()); + input.FlagsIn = flags[static_cast(image) % flags.size()]; + State state = input; + harness.Run(body, returnBranchOffset, state); + ++g_executions; + const auto expected = referenceLow(operands, input); + require(expected == libcReferenceLow(operands, input), "Transcribed reference disagrees with the libc emulation: " + describe(item)); + require(low(state.Xmm[operands.Destination]) == expected, "Lowered sequence computed the wrong field: " + describe(item)); + for (unsigned reg = 0; reg < 16; ++reg) { + if (reg == operands.Destination) continue; + require(std::memcmp(state.Xmm[reg], input.Xmm[reg], 16) == 0, "Lowered sequence clobbered xmm" + std::to_string(reg) + ": " + describe(item)); + } + require(((state.FlagsOut ^ input.FlagsIn) & kStatusFlags) == 0, "Lowered sequence changed RFLAGS: " + describe(item)); + require(state.RspAfter == state.RspBefore, "Lowered sequence did not restore rsp: " + describe(item)); + for (int slot = 0; slot < 16; ++slot) { + std::uint64_t value; + std::memcpy(&value, state.RedZone + slot * 8, 8); + require(value == kCanary, "Lowered sequence wrote into the red zone: " + describe(item)); + } + } +} + +void cpuExecution() { + const Harness harness; + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); + std::mt19937_64 random(0x5EED); + const std::vector sites = {{false, 3, 3, 8, 40}, {true, 3, 3, 8, 8}, {true, 1, 0, 8, 0}, {true, 9, 4, 16, 16}, {true, 3, 4, 16, 16}}; + for (const auto& site : sites) executeCase(harness, *matcher, site, random, 64); + const std::vector> pairs = {{3, 3}, {1, 0}, {9, 4}, {3, 4}, {15, 15}, {0, 15}, {8, 8}}; + for (std::uint8_t length = 1; length <= 64; ++length) { + for (std::uint8_t index = 0; index + length <= 64; ++index) { + for (const auto& [dst, src] : pairs) executeCase(harness, *matcher, {true, dst, src, length, index}, random, 2); + for (const std::uint8_t dst : std::array{0, 3, 9, 15}) executeCase(harness, *matcher, {false, dst, dst, length, index}, random, 2); + } + } +} + +Bytes elfFixture(const Bytes& text) { + Bytes bytes(0x400); + bytes[0] = 0x7F; + bytes[1] = 'E'; + bytes[2] = 'L'; + bytes[3] = 'F'; + bytes[4] = 2; + bytes[5] = 1; + bytes[6] = 1; + write(bytes, 16, 3); + write(bytes, 18, 62); + write(bytes, 24, 0x1000); + write(bytes, 32, 64); + write(bytes, 54, 56); + write(bytes, 56, 2); + std::fill(bytes.begin() + 0x200, bytes.begin() + 0x300, 0xCC); + std::copy(text.begin(), text.end(), bytes.begin() + 0x200); + return bytes; +} + +std::vector elfHeaders() { + return {{1, 5, 0x200, 0x1000, 0, 0x100, 0x100, 0x1000}, {1, 6, 0x300, 0x2000, 0, 0x100, 0x100, 0x1000}}; +} + +void peBuilder() { + const Bytes site = {0xF2, 0x0F, 0x78, 0xDB, 0x08, 0x08}; + Bytes text = {0xEB, 0x06}; + text.insert(text.end(), site.begin(), site.end()); + text.push_back(0xC3); + const auto source = elfFixture(text); + const auto headers = elfHeaders(); + const auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(source, {headers[0]}); + require(converted.Trampolines.size() == 1, "PE fixture conversion did not produce one stub"); + const WindowsLoadImage image(converted.Bytes, headers); + auto sections = image.BuildSections(); + auto nextRva = image.GetEndRva(); + const auto stubSectionRva = nextRva; + WindowsTrampolineBuilder().Build(converted.Trampolines, image, sections, nextRva); + require(sections.size() == 3 && sections.back().Name == ".amdstub" && sections.back().Rva == stubSectionRva && (sections.back().Characteristics & SectionExecute) != 0 && nextRva == AlignRva(stubSectionRva + sections.back().Data.size()), "Stub section was not appended"); + const auto& text0 = sections[0]; + const auto siteRva = LoadRva + 2; + require(text0.Data[2] == 0xE9 && text0.Data[7] == 0x90 && text0.Data[8] == 0xC3, "PE site was not replaced by a jump"); + const auto stubRva = static_cast(static_cast(siteRva + 5) + read(text0.Data, 3)); + require(stubRva == stubSectionRva && stubRva % 16 == 0, "PE jump does not land on the 16-aligned stub"); + const auto& body = sections.back().Data; + require(body.size() == 32 && body[0] == 0x66 && body[9] == 0xE9 && body[16] == 0x00 && body[17] == 0x00 && body[18] == 0x02, "PE stub body is wrong"); + require(static_cast(stubRva + 9 + 5) + read(body, 10) == siteRva + 6, "PE stub does not return past the site"); + require(read(body, 5) == 7 && (stubRva + 16) % 16 == 0, "PE mask constant is not 16-aligned"); + std::array directories{}; + const auto file = WindowsPeWriter().Write(sections, LoadRva, directories); + require(file.size() > 0x400, "PE writer rejected the stub section"); + auto altered = converted.Bytes; + altered[0x205] = 0xDC; + requireFailure([&] { + const WindowsLoadImage alteredImage(altered, headers); + auto alteredSections = alteredImage.BuildSections(); + auto rva = alteredImage.GetEndRva(); + WindowsTrampolineBuilder().Build(converted.Trampolines, alteredImage, alteredSections, rva); + }, "Changed PE site bytes were accepted"); + requireFailure([&] { + auto unmapped = converted.Trampolines; + unmapped[0].Address = 0x5000; + auto freshSections = image.BuildSections(); + auto rva = image.GetEndRva(); + WindowsTrampolineBuilder().Build(unmapped, image, freshSections, rva); + }, "Unmapped PE site was accepted"); + auto untouched = image.BuildSections(); + auto untouchedRva = image.GetEndRva(); + WindowsTrampolineBuilder().Build({}, image, untouched, untouchedRva); + require(untouched.size() == 2 && untouchedRva == image.GetEndRva(), "Empty site list changed the image"); +} + +} + +int main() { + try { + cpuExecution(); + peBuilder(); + std::cout << "AMD64-only Windows tests passed (" << g_executions << " lowered sequences executed)\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/core/relinker/main.cpp b/core/relinker/main.cpp index 58c2658d38..a7654d9b1d 100644 --- a/core/relinker/main.cpp +++ b/core/relinker/main.cpp @@ -18,10 +18,12 @@ #include #include #include +#include #include #include #include #include +#include int main(const int argc, char* argv[]) { Cli::Args args; @@ -39,17 +41,15 @@ int main(const int argc, char* argv[]) { auto sourceBytes = fileReader.Read(args.inputPath); const std::string absPath = std::filesystem::absolute(args.outputPath).string(); + std::vector trampolines; if (args.toIntel) { - std::cout << "Mode: Intel instruction conversion; system unchanged; unused-filter=" << args.unusedFilterLevel << " (not applied)\n"; - - const Relinker::ElfReader elfReader(sourceBytes); - const auto converter = Codegen::MakeAmd64OnlyConverter(); - auto result = converter->Convert(std::move(sourceBytes), elfReader.ReadCodeSegments()); - - fileWriter.Write(absPath, std::move(result.Bytes)); - std::cout << "OK: " << result.ReplacedCount << " instructions replaced\n"; - - if (args.autorun) return Cli::Autorun(absPath, args.toWindows); + const auto codeSegments = Relinker::ElfReader(sourceBytes).ReadCodeSegments(); + auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(std::move(sourceBytes), codeSegments); + sourceBytes = std::move(converted.Bytes); + trampolines = std::move(converted.Trampolines); + for (const auto& report : converted.Reports) + std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) -> " << (report.Lowering == Codegen::Amd64OnlyLowering::InPlace ? "in place " : "stub ") << report.ReplacementLength << " bytes\n"; + std::cout << "Intel conversion: " << converted.ReplacedCount << " in place, " << trampolines.size() << " stubs\n"; } auto elfReader = std::make_shared(sourceBytes); @@ -95,7 +95,7 @@ int main(const int argc, char* argv[]) { ); } - fileWriter.Write(absPath, patcher->Patch(sourceBytes, result.OriginalHeaders, result.DynamicSection, result.OriginalPltGotVaddr, args.runPath, args.lazyBinding, args.windowsDiagnostics)); + fileWriter.Write(absPath, patcher->Patch(sourceBytes, result.OriginalHeaders, result.DynamicSection, result.OriginalPltGotVaddr, args.runPath, args.lazyBinding, args.windowsDiagnostics, trampolines)); std::cout << "External prx references: " << result.RegistryEntries.size() << "\nOutput file: " << absPath << '\n'; if (args.autorun) return Cli::Autorun(absPath, args.toWindows); @@ -105,6 +105,11 @@ int main(const int argc, char* argv[]) { if (e.FailureOffset != 0) std::cerr << " (offset 0x" << std::hex << e.FailureOffset << ")"; std::cerr << "\n"; return 2; + } catch (const Codegen::CodegenException& e) { + std::cerr << "FAIL: " << e.what(); + if (e.FailureOffset != 0) std::cerr << " (offset 0x" << std::hex << e.FailureOffset << ")"; + std::cerr << "\n"; + return 2; } catch (const std::exception& e) { std::cerr << "FAIL: " << e.what() << "\n"; return 2; diff --git a/core/relinker/relinker/tests/StrictNidFilterTests.cpp b/core/relinker/relinker/tests/StrictNidFilterTests.cpp index fe4bba7a99..63fbdac61d 100644 --- a/core/relinker/relinker/tests/StrictNidFilterTests.cpp +++ b/core/relinker/relinker/tests/StrictNidFilterTests.cpp @@ -153,10 +153,12 @@ void vectorInstructionLengths() { {0xC5, 0xD9, 0x73, 0xD4, 0x20}, {0xC4, 0xE1, 0x79, 0x70, 0xC0, 0x1B}, {0xC4, 0xE1, 0x78, 0x77}, - {0x62, 0xF1, 0x7D, 0x48, 0x72, 0xD0, 0x04} + {0x62, 0xF1, 0x7D, 0x48, 0x72, 0xD0, 0x04}, + {0xF3, 0x0F, 0xB8, 0xC0}, + {0xCD, 0x41} }; for (const auto& instruction : instructions) - require(decoder.Decode(instruction.data(), instruction.size()) == instruction.size(), "Vector immediate or VZEROUPPER was decoded with the wrong length"); + require(decoder.Decode(instruction.data(), instruction.size()) == instruction.size(), "Vector immediate, VZEROUPPER, POPCNT or INT was decoded with the wrong length"); } void exceptionLandingPads() { diff --git a/docs/TechnicalDebt.md b/docs/TechnicalDebt.md index bf4a06b7c4..f72aa68c5e 100644 --- a/docs/TechnicalDebt.md +++ b/docs/TechnicalDebt.md @@ -30,3 +30,7 @@ Throughout the project, every function at every stage either **does exactly what - [Shader recompilation](../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../core/shader/recompiler) was written completely independently from [libs/prx](../core/libs/prx). - The executable file that [relinker](../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) generates opens the console when launched, which is inconvenient for playability. - [Relinker](../core/relinker/elfpatcher/src) doesn't add an icon to the generated executable. This should be done without adding dependencies (only standard). +- `--to-intel` does not lower the register forms of EXTRQ/INSERTQ (`0F 79`, 4 or 5 bytes, too short for a jump) nor MONITORX/MWAITX/CLZERO/RDPRU/MCOMMIT; the [lowering](../core/relinker/codegen/src/x86/Sse4aLowering.cpp) fails the relink instead. SHA-NI is not substituted. +- The length-changing path of the [instruction rewriter](../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust VEX/0F38/0F3A RIP-relative operands, data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length. +- The Linux placement of the `--to-intel` stubs in the [ELF patcher](../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. +- The libc [SSE4a trap emulation](../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it. From 29b4601564ad97edacbf3a2fa917c1ec1aea93af Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:00:40 -0700 Subject: [PATCH 15/93] perf(agc): dispatch and draw recipes, data-only cache hits, unit shadows, bindless image tables Dispatch cache with per-key value variants and data-only hits (flat-SRT leaves the recompiler proves pure are refreshed into the resource template instead of re-captured), dispatch and draw recipes replaying a proven prepare, register-keyed draw entries, template data refresh, unit shadows in device-local slabs, 64 KiB block-unit storage tracking with windowed detile, DCC clear-key proofs, per-class GPU timing, a CPU copy path with batch read tracking, present slot ring, rate-limited reaps and lazy completion counting, plan-failure and poisoned source memos, bindless image tables (runtime-indexed descriptor arrays from a CPU-read table) so the per-frame compute program that failed its plan now compiles. Worker affinity opt-in, per-draw logs behind APS5_TRACE_DRAWS, wait-timeout reports include the recorder's view of the label. Fixes a use-after-free behind BDA-miss dispatch snapshots and the CPU-write stamp of UnchangedSinceCollected. Intro-video stage: 154 -> ~500 presents per 10 s at 4K. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 22 + .../Execution/include/GuestMemory.hpp | 61 +- .../Execution/include/PerformanceTimer.hpp | 50 + .../libSceAgcDriver/Execution/include/Pm4.hpp | 52 + .../Execution/include/Recipe.hpp | 91 + .../Execution/include/ShaderMemory.hpp | 73 +- .../Execution/include/VulkanDevice.hpp | 213 +- .../libSceAgcDriver/Execution/src/Driver.cpp | 4386 +++++++++++++++-- .../Execution/src/GuestMemory.cpp | 393 +- .../prx/libSceAgcDriver/Execution/src/Pm4.cpp | 120 +- .../Execution/src/ShaderMemory.cpp | 219 +- .../Execution/src/VulkanDevice.cpp | 2107 ++++++-- .../Graphics/include/BdaResources.hpp | 13 + .../Graphics/include/BufferPool.hpp | 13 +- .../Graphics/include/Context.hpp | 61 + .../Graphics/include/DccMetadata.hpp | 24 + .../libSceAgcDriver/Graphics/include/Draw.hpp | 47 +- .../Graphics/include/GuestBufferMemory.hpp | 135 +- .../Graphics/include/Pipeline.hpp | 15 +- .../Graphics/include/Recorder.hpp | 464 +- .../Graphics/include/Resources.hpp | 9 + .../Graphics/include/ShaderInputState.hpp | 10 +- .../Graphics/include/ShaderResources.hpp | 133 +- .../Graphics/include/State.hpp | 59 + .../Graphics/include/Texture.hpp | 304 +- .../Graphics/include/TextureDetiler.hpp | 26 +- .../Graphics/include/TextureTiling.hpp | 4 + .../Graphics/include/UnitShadow.hpp | 155 + .../Graphics/include/VertexInput.hpp | 13 + .../Graphics/shaders/TextureDetile.comp | 27 +- .../Graphics/src/BdaResources.cpp | 68 +- .../Graphics/src/BufferPool.cpp | 19 +- .../Graphics/src/DccMetadata.cpp | 147 +- .../prx/libSceAgcDriver/Graphics/src/Draw.cpp | 1177 ++++- .../Graphics/src/GuestBufferMemory.cpp | 863 +++- .../libSceAgcDriver/Graphics/src/Pipeline.cpp | 42 +- .../libSceAgcDriver/Graphics/src/Recorder.cpp | 1459 +++++- .../Graphics/src/Resources.cpp | 39 +- .../Graphics/src/ShaderAddressResources.cpp | 5 + .../Graphics/src/ShaderInputState.cpp | 34 +- .../Graphics/src/ShaderResources.cpp | 923 +++- .../Graphics/src/ShaderValidation.cpp | 23 +- .../libSceAgcDriver/Graphics/src/State.cpp | 214 +- .../libSceAgcDriver/Graphics/src/Texture.cpp | 2231 ++++++++- .../Graphics/src/TextureDetiler.cpp | 30 +- .../Graphics/src/TextureTiling.cpp | 5 + .../Graphics/src/UnitShadow.cpp | 848 ++++ .../libSceAgcDriver/tests/BdaResources.cpp | 35 + .../prx/libSceAgcDriver/tests/Graphics.cpp | 22 +- core/libs/prx/libSceAgcDriver/tests/Pm4.cpp | 56 + .../prx/libSceAgcDriver/tests/Recorder.cpp | 945 ++++ .../libSceAgcDriver/tests/ShaderMemory.cpp | 348 +- .../libSceAgcDriver/tests/TextureDetiler.cpp | 84 +- core/shader/recompiler/CMakeLists.txt | 1 + core/shader/recompiler/CacheKey.hpp | 21 + .../IrMetadata/ControlFlowInfo.hpp | 6 + .../IrMetadata/ResourcePlan.hpp | 3 + .../IrMetadata/ShaderInfo.hpp | 4 + .../IntermediateRepresentation/IrOpcode.hpp | 4 + .../src/IrOpcode.cpp | 9 + .../src/IrProgram.cpp | 3 +- .../Optimization/ResourceMaterializer.hpp | 10 +- .../include/Optimization/ResourceProgram.hpp | 20 + .../include/Optimization/SrtWalker.hpp | 13 + .../SrtWalker/SrtFlatSlotClasses.hpp | 22 + .../Optimization/src/BindingAllocator.cpp | 20 +- .../Optimization/src/ResourceMaterializer.cpp | 389 +- .../Optimization/src/ResourceTracker.cpp | 137 +- .../src/SrtWalker/SrtDescriptorEvaluation.cpp | 13 +- .../src/SrtWalker/SrtEvaluator.cpp | 4 + .../src/SrtWalker/SrtFlatSlotClasses.cpp | 64 + core/shader/recompiler/Recompiler.cpp | 235 +- core/shader/recompiler/Recompiler.hpp | 13 +- .../include/SpirvBackend/SpirvBda.hpp | 8 + .../SpirvBackend/SpirvEmitterHelpers.hpp | 4 +- .../SpirvBackend/SpirvEmitterInstructions.hpp | 3 + .../SpirvBackend/SpirvEmitterState.hpp | 13 + .../SpirvMemory/SpirvDescriptors.hpp | 5 +- .../SpirvBackend/src/SpirvBdaLookup.cpp | 41 +- .../SpirvBackend/src/SpirvBdaRead.cpp | 205 +- .../SpirvBackend/src/SpirvEmitter.cpp | 11 + .../SpirvBackend/src/SpirvFlowEmitter.cpp | 3 + .../SpirvBackend/src/SpirvImageEmitter.cpp | 136 +- .../src/SpirvMemory/SpirvDescriptors.cpp | 56 +- .../SpirvMemory/SpirvMemoryInstructions.cpp | 64 +- .../Translation/EmbeddedVertexFetch.hpp | 6 + .../Translation/src/EmbeddedVertexFetch.cpp | 53 + .../Translation/src/InstructionTranslator.cpp | 4 + .../MemoryInstructions/FlatInstructions.cpp | 31 + 89 files changed, 18605 insertions(+), 1945 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/Recorder.cpp create mode 100644 core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp create mode 100644 core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 8cd0d4edf3..71f2dad5d7 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -79,6 +79,7 @@ add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureTiling.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureDetiler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Texture.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/UnitShadow.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Sampler.cpp ${agcTextureDetileShaderHeader} ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Event.cpp @@ -172,6 +173,25 @@ target_link_libraries(agc_driver_flip_tests PRIVATE set_target_properties(agc_driver_flip_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_flip_tests) +add_executable(agc_driver_recorder_tests EXCLUDE_FROM_ALL + tests/Recorder.cpp +) + +target_include_directories(agc_driver_recorder_tests PRIVATE + ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include +) + +target_link_libraries(agc_driver_recorder_tests PRIVATE + libSceAgcDriver + SDL2-static + libc +) + +set_target_properties(agc_driver_recorder_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_driver_recorder_tests) + add_executable(agc_driver_pm4_tests EXCLUDE_FROM_ALL tests/Pm4.cpp ) @@ -215,6 +235,7 @@ add_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL Graphics/src/TextureTiling.cpp Graphics/src/TextureDetiler.cpp Graphics/src/Texture.cpp + Graphics/src/UnitShadow.cpp Graphics/src/Sampler.cpp ${agcTextureDetileShaderHeader} Execution/src/GuestMemory.cpp @@ -308,6 +329,7 @@ if(AGC_BUILD_VISUAL_TEST) Graphics/src/TextureTiling.cpp Graphics/src/TextureDetiler.cpp Graphics/src/Texture.cpp + Graphics/src/UnitShadow.cpp Graphics/src/Sampler.cpp ${agcTextureDetileShaderHeader} ${visualShaders} diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index 7528b331df..c522b689cd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -30,6 +30,19 @@ void Read(std::uint64_t address, std::span destination, std::size_t a // scratch heaps): uncommitted pages read as zeros, are not compared, and are never stored. void ReadCommitted(std::uint64_t address, std::span destination); bool EqualsCommitted(std::uint64_t address, std::span bytes); +// EqualsCommitted without the flush hook: the caller has decided that the recorded GPU work +// writing the range need not be waited for (the dispatch-cache validation in Driver.cpp). +bool EqualsCommittedUnsynced(std::uint64_t address, std::span bytes); +// EqualsCommittedUnsynced for a caller that treats an unmapped page as a miss (the dispatch-cache +// validation's per-run compare): one walk of the page states, one memcmp per mapped run, no profile +// counter and no reason string. Unmapped: some page of the range is not readable (or the address +// space cannot be queried). +enum class Compare : std::uint8_t { Equal, Differs, Unmapped }; +Compare CompareMapped(std::uint64_t address, std::span bytes); +// CompareMapped's walk with a copy in place of the compare: the bytes at `address` are copied into +// `out` (Equal), or Unmapped as CompareMapped answers it (the dispatch-cache validation's masked +// re-read of a differing run). +Compare CopyMapped(std::uint64_t address, std::span out); void WriteChangedCommitted(std::uint64_t address, std::span current, std::span original); void Write(std::uint64_t address, std::span source, std::size_t alignment = 1); // Stores the parts of `current` that differ from `original` (the guest bytes the GPU started from), so @@ -45,7 +58,16 @@ void WriteChanged(std::uint64_t address, std::span current, std // UnchangedSince is always false, so callers fall back to comparing bytes. std::uint64_t CollectWrites(std::uint64_t address, std::size_t bytes); bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation); -void MarkWritten(std::uint64_t address, std::size_t bytes); +// UnchangedSince for several ranges under one tracker lock: true only when every one holds. +struct UnchangedQuery { + std::uint64_t address; + std::size_t bytes; + std::uint64_t generation; +}; +bool UnchangedSinceAll(std::span queries); +// Returns the generation the blocks were stamped with (0 when the arena is not write-watched): +// UnchangedSince(range, it) holds until the next store over the range. +std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes); // Collect epoch: within one epoch a range already collected is not walked again, CollectWrites // returns the current generation instead. A guest write landing between two collects of the same // epoch is seen by the next epoch, which is the ordering real hardware gives a CPU write made while a @@ -64,6 +86,23 @@ void MarkWritten(std::uint64_t address, std::size_t bytes); void BumpCollectEpoch(); std::uint64_t CollectEpochBumps(); std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes); +// The tracker's current generation (every collect and MarkWritten bumps it): a stamp taken after +// a set of driver stores, for UnchangedSinceCollected to compare against later. Read under the +// tracker mutex, so every collect that bumped before it has finished its walk and every later +// collect stamps newer. +std::uint64_t TrackerGeneration(); +// Whether no CPU write touched the range since `generation`: an uncached resetting collect of the +// range's pages first (a game store not collected yet becomes a block stamp), then the compare over +// the stamps collects make (dirty pages), not the MarkWritten stamps of the driver's own GPU label +// records, which share the title's label blocks. False when the arena is not write-watched. +bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint64_t generation); +// Per 64 KiB tracker block of [address, address + bytes) (block 0 holds `address`): `changed[k]` +// receives whether the block was stamped after generations[k] (UnchangedSince over that block +// alone), `cpu[k]`, when given, whether a collect stamped it after (a CPU store, as +// UnchangedSinceCollected tells them apart from the driver's MarkWritten stamps, but without its +// walk: the caller collected the range first). One tracker lock for the whole range. Outside the +// watched arena, or for a generation of 0, a block reads as changed. +void ChangedBlocks(std::uint64_t address, std::size_t bytes, std::span generations, std::span changed, std::span cpu = {}); // Serializes device work: draws, dispatches, presentation and the deferred write-backs below. The // mutex is recursive; it is wrapped so every acquisition (std::lock_guard at any site) measures how @@ -102,6 +141,9 @@ void SetGpuUnlockHook(void (*hook)()); // only be reached under it (a resource build's stage A runs without it, see ShaderResources). void AssertGpuLockHeld(const char* where); void TagGpuLockThread(std::uint32_t queue); +// The presenter stays untagged (its tag is a queue id to the recorder) but takes its own column +// of the [lock] holder matrix, apart from the game threads. +void MarkPresenterThread(); // The queue tag of the calling thread (0xffffffff when untagged), for reports that name the thread. std::uint32_t GpuLockThreadTag(); // Lock sites (APS5_PROFILE_DRAW): a caller names the site of its next lock() so the [lock] line @@ -111,8 +153,14 @@ std::uint32_t GpuLockThreadTag(); // as acquisitions; the hold they start is timed under "try". Holds are timed per site as well (the // outermost acquisition of a thread until its unlock; APS5_NO_HOLD_PROFILE=1 turns that off), so // the line shows who holds the mutex, not only who waits for it. -enum class GpuLockSite : std::uint8_t { Other = 0, Dispatch, Indirect, Draw, Hook, Wait, Label, Flush, Present, Fill, Try, Count }; +enum class GpuLockSite : std::uint8_t { Other = 0, Dispatch, Indirect, Draw, Hook, Wait, Label, Flush, Present, Fill, Copy, End, Try, Count }; void TagGpuLockSite(GpuLockSite site); +// A GPU wait (a fence or timeline wait that found its work unfinished) made while the calling +// thread holds GpuMutex: counted on the thread's [lock] line as 'locked GPU waits' (APS5_PROFILE_DRAW). +void NoteLockedGpuWait(double ms); +// Bytes the calling thread's write-watch collects walked so far (0 without the memory profile): +// a caller reads the difference across a span of its own work. +std::uint64_t ThreadCollectedBytes(); // A code address (a return address) as an offset into its module, in the form of the [guestmem] // callers (symbolized with nm against the driver's image). Takes the loader lock: for reports only. unsigned long long CodeOffset(const void* address); @@ -123,6 +171,11 @@ unsigned long long CodeOffset(const void* address); // calls FlushGpuWrites itself. void SetFlushHook(void (*hook)(std::uint64_t address, std::size_t bytes)); void FlushGpuWrites(std::uint64_t address, std::size_t bytes); +// Moves with every ForgetPages call (the libc invalidator: memory unmapped or re-registered): odd +// while the call stores its page states, moved past it after, so a reader that loads an even +// value before consulting the page states and reading directly, and the same value after, knows +// the states it consulted were current and no page it read changed its mapping meanwhile. +std::uint64_t ForgetSerial(); // Attribution of guest memory accesses (the [hooksync] line in Recorder.cpp and the read-site // counts of the [guestmem] line, APS5_PROFILE_DRAW): a sync the flush hook makes for an access has @@ -143,8 +196,10 @@ PacketTag CurrentPacket(); // write after a pending GPU write, so whether the bytes change says nothing about its necessity. // MirrorRefresh: an address-based build bringing an image mirror up to date with guest memory // (GuestBufferMemory.cpp: the flush before a writable mirror's compare, a new mirror's fill). +// DrawCache: a draw entry's per-stage value validation (Driver.cpp drawCache), where a draw +// capture's hook waits reappear on a hit. // Every enumerator needs its name in ReadSiteName's table (GuestMemory.cpp). -enum class ReadSite : std::uint8_t { Unknown = 0, Capture, DispatchCache, TextureCompare, TextureRead, BufferUpload, IndexBuffer, VertexBuffer, Registers, IndirectArguments, Wait, Label, Scanout, Store, MirrorRefresh, Count }; +enum class ReadSite : std::uint8_t { Unknown = 0, Capture, DispatchCache, TextureCompare, TextureRead, BufferUpload, IndexBuffer, VertexBuffer, Registers, IndirectArguments, Wait, Label, Scanout, Store, MirrorRefresh, DrawCache, Count }; const char* ReadSiteName(ReadSite site); // Sets the calling thread's read site and returns the previous one (nested scopes restore it). ReadSite SetReadSite(ReadSite site); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp index 1f0b1a6858..b1ebdc2c32 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp @@ -67,6 +67,45 @@ public: flipReached = reached; } + Clock::time_point FlipReached() { + std::lock_guard lock(mutex); + return flipReached; + } + + // Recorder batches at the flip packet (after its submit) and how many had not signaled. + void NoteFlipBatches(std::uint64_t atFlip, std::uint64_t unsignaled) { + std::lock_guard lock(mutex); + batchesAtFlip = atFlip; + unsignaledAtFlip = unsignaled; + } + + // 0 until the flip packet sampled the recorder (the drain paths never do). + std::uint64_t BatchesAtFlip() { + std::lock_guard lock(mutex); + return batchesAtFlip; + } + + // At the presentation's blit: the batches ahead of it, the newest batch's vkQueueSubmit time + // and the blit's own, both measured from the flip packet. + void NoteBlit(std::uint64_t atBlit, Clock::time_point lastSubmitted, Clock::time_point blitSubmitted) { + std::lock_guard lock(mutex); + batchesAtBlit = atBlit; + if (flipReached == Clock::time_point{}) return; + blitSubmitAfterFlip = blitSubmitted - flipReached; + lastSubmitAfterFlip = std::max(lastSubmitted, blitSubmitted) - flipReached; + } + + // A presentation retired during this present (with APS5_FLIP_INFLIGHT=1 the previous frame's): + // the batches submitted behind its blit; and, for a retired presentation whose batches all + // have their completion record by this present (VulkanDevice's SettleRetired: usually one + // present later than the retire), the GPU's busy time and idle gaps over those batches. + void NoteRetired(std::uint64_t afterFlip, double busyMs, double gapMs) { + std::lock_guard lock(mutex); + batchesAfterFlip += afterFlip; + gpuBusyMs += busyMs; + gpuGapMs += gapMs; + } + void Print(std::uint32_t outputHandle, std::int32_t buffer, std::int64_t argument, Clock::time_point finished, Clock::duration interval) { std::lock_guard lock(mutex); if (firstSerial == 0 || flipSerial == 0) throw std::runtime_error("Frame timing: incomplete submission lineage"); @@ -89,6 +128,9 @@ public: } output << " worker_unattributed_ms=" << milliseconds(flipReached - executionStart - accounted); if (interval != Clock::duration::zero()) output << " flip_interval_ms=" << milliseconds(interval); + output << " batches_at_flip=" << batchesAtFlip << " unsignaled_at_flip=" << unsignaledAtFlip << " batches_at_blit=" << batchesAtBlit << " batches_after_flip=" << batchesAfterFlip; + output << " gpu_busy_ms=" << gpuBusyMs << " gpu_gap_ms=" << gpuGapMs; + output << " last_submit_after_flip_ms=" << milliseconds(lastSubmitAfterFlip) << " blit_submit_after_flip_ms=" << milliseconds(blitSubmitAfterFlip); output << " metrics=inclusive(count,sum_ms,max_ms[,bytes])"; for (const auto& [key, metric] : metrics) { if (metric.count == 0) continue; @@ -117,6 +159,14 @@ private: Clock::time_point executionStart{}; Clock::time_point flipReceived{}; Clock::time_point flipReached{}; + std::uint64_t batchesAtFlip = 0; + std::uint64_t unsignaledAtFlip = 0; + std::uint64_t batchesAtBlit = 0; + std::uint64_t batchesAfterFlip = 0; + double gpuBusyMs = 0; + double gpuGapMs = 0; + Clock::duration lastSubmitAfterFlip{}; + Clock::duration blitSubmitAfterFlip{}; }; class PerformanceContext { diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp index 475995cccd..9543e5a628 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp @@ -21,8 +21,60 @@ struct DrawParameters { bool indexed = true; std::uint32_t firstVertex = 0; std::uint32_t firstInstance = 0; + // An indirect draw (DRAW_INDIRECT 0x24, DRAW_INDEX_INDIRECT 0x25 and their multi forms 0x2c and + // 0x38): the CP reads {vertexCount, instanceCount, startVertex, startInstance} (indexed: + // {indexCount, instanceCount, firstIndex, vertexOffset, startInstance}) per record at + // `arguments` + record * stride, writes startVertex / vertexOffset, startInstance and the draw + // index into the SH registers the packet names (0x280: nowhere) and takes the instance count + // from the record. indexCount and instanceCount above are then unknown (indexed draws bind the + // declared INDEX_BUFFER_SIZE range), firstVertex holds GE_INDX_OFFSET. The driver decides per + // dimension how the GPU reproduces the CP's register write for the fixed-function fetch (see + // Driver.cpp): InPlace draws from the record's dword, Constant rewrites it with the constant. + struct IndirectDraw { + std::uint64_t arguments; + std::uint32_t opcode; + std::uint32_t recordBytes; + std::uint32_t stride; + std::uint32_t count; + bool countIndirect; + std::uint64_t countAddress; + std::uint32_t baseVertexLocation; + std::uint32_t startInstanceLocation; + std::uint32_t drawIndexLocation; + bool drawIndexEnabled; + std::uint32_t indxOffset; + enum class Rule : std::uint8_t { InPlace, Constant }; + Rule vertexRule = Rule::Constant; + Rule instanceRule = Rule::Constant; + std::uint32_t vertexConstant = 0; + std::uint32_t instanceConstant = 0; + std::int32_t baseVertexSgpr = -1; + std::int32_t startInstanceSgpr = -1; + std::int32_t drawIndexSgpr = -1; + // Bytes of the record range: (count - 1) * stride + recordBytes, or 0 for no records. + std::uint64_t RangeBytes() const { return count == 0 ? 0 : static_cast(count - 1) * stride + recordBytes; } + std::uint32_t VertexDwordOffset() const { return recordBytes == 20 ? 12u : 8u; } + std::uint32_t InstanceDwordOffset() const { return recordBytes == 20 ? 16u : 12u; } + }; + std::optional indirect; }; +// One indirect draw record as the CP reads it (firstVertexOrIndex is startVertex for the +// non-indexed layout and firstIndex for the indexed one; vertexOffset exists only in the latter). +struct DrawArguments { + std::uint32_t count; + std::uint32_t instances; + std::uint32_t firstVertexOrIndex; + std::uint32_t vertexOffset; + std::uint32_t firstInstance; +}; +// Reads record `record` of an indirect draw, or its count dword, through the checked guest memory +// path (which waits for recorded GPU work that writes them). +DrawArguments ReadDrawArguments(const DrawParameters::IndirectDraw& indirect, std::uint32_t record); +std::uint32_t ReadDrawCount(const DrawParameters::IndirectDraw& indirect); +inline bool IndirectDrawOpcode(std::uint32_t opcode) { return opcode == 0x24 || opcode == 0x25 || opcode == 0x2c || opcode == 0x38; } +inline bool DrawOpcode(std::uint32_t opcode) { return opcode == 0x2d || opcode == 0x35 || IndirectDrawOpcode(opcode); } + std::string Name(std::uint32_t header); // A PM4 type-2 packet is a one-dword filler (command-buffer padding); type 3 and type 0 carry a // dword count in bits 29:16. Type 1 is undefined. diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp new file mode 100644 index 0000000000..51c74e1a0a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp @@ -0,0 +1,91 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_RECIPE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_RECIPE_HPP + +#define VK_NO_PROTOTYPES +#include +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver { + +// A compute dispatch's Vulkan objects (VulkanDevice.cpp), shared by every dispatch of one compiled +// variant. +struct ComputePipelineObjects; + +// What a value-equal dispatch-cache hit needs to record its dispatch without preparing, finding or +// comparing anything again (design_cpu_final M4): the template the resource cache served the +// entry's compiled result with, the pipeline objects of its variant, the push constants and data +// words the compiled result fixes, and the pre-sync answer. Built at the end of a successful device +// call of a reusable object and attached to the dispatch-cache variant; immutable after that. It +// holds weak references only: the resource cache owns the template (its 1024-entry bound stays +// what pins textures and storage images) and the device's pipeline map owns the objects, so a dead +// reference is a recipe miss, never a leak past the caches' budgets. +struct Recipe { + // R10: unusable on any other device. + VkDevice device = VK_NULL_HANDLE; + std::weak_ptr templateRef; + // The template's content key: touches the cache's LRU on a hit. + Graphics::ResourceCache::Key key; + std::weak_ptr objects; + std::array pushBytes{}; + bool pushes = false; + // FNV over the compiled result's ShaderData/FlattenedSrt words (ShaderResources::DataWordsHash): + // the template's data buffers need a refresh iff its own hash differs. + std::uint64_t dataWordsHash = 0; + // The template's PresyncSurfaces() as of the attach and the host-import identity they were + // computed under (a surface is CPU-read only while its memory has no import). + std::vector> presyncSurfaces; + Graphics::HostImportsProof presyncProof; + // Both false for every recipe (only reusable objects get one); kept for the asserts. + bool needsCompletion = false; + bool holdsLease = false; +}; + +// Recorded: the dispatch went into the open batch from the recipe; Rebuild: the proof failed or an +// object went away, the caller runs the ordinary path (the recipe is re-attached after it). +enum class RecipeOutcome { Recorded, Rebuild }; + +// What a recorded draw needs to record again from its draw-cache hit without looking anything up +// (design_cpu_final M8, step 8b): the resource-cache template the draw's stages were served with +// (weak, as a dispatch recipe's), its pipeline and framebuffer (weak, R10: the pipeline store owns +// the pipeline and destroys it with its device (ClearCachedPipelines), the pipeline owns its +// framebuffer list; a recipe outlives a device replacement in the draw cache, so an owning +// reference would destroy the objects against a dead VkDevice), the resident targets (weak: a hit +// proves them on the stored objects while they are still Cached()) with their attachment views, +// and the per-draw derivations of the compiled stages and the state: the vertex input layout, the +// push constant block, the state with the unwritten outputs masked, the fragment outputs and the +// pipeline stages. Built at the end of a recorded, cacheable, reusable, non-indirect draw and +// attached to the draw entry's matched stage variants; immutable. +struct DrawRecipe { + VkDevice device = VK_NULL_HANDLE; + std::weak_ptr templateRef; + Graphics::ResourceCache::Key key; + std::weak_ptr pipeline; + std::weak_ptr framebuffer; + std::vector> targets; + std::vector targetViews; + std::uint64_t passKey = 0; + Graphics::VertexInputLayout vertexInput; + std::array pushBytes{}; + VkShaderStageFlags pushStages = 0; + std::optional masked; + std::set fragmentOutputs; + VkPipelineStageFlags shaderStages = 0; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index ce2ab7bfa5..2c41a06849 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -4,25 +4,70 @@ #include "Recompiler.hpp" #include #include +#include #include #include #include #include +#include #include +namespace ShaderRecompiler { +struct SourceHandle; +} + namespace AgcDriver { // Records the guest words a shader's resource analysis reads, as the memory regions its recompile // request carries. Guest memory is fetched a 4 KiB page at a time; the regions report exactly the -// dwords read, so cache keys built from them do not change with unrelated bytes nearby. +// dwords read, so cache keys built from them do not change with unrelated bytes nearby. A page that +// `pendingWrite` reports as written by recorded GPU work is read word by word instead, so only a +// dword the capture needs waits for that work through the flush hook, not the whole page; a dword +// the query knows to be left unchanged by that work (Driver.cpp's written-element evidence) is read +// raw, without the hook. Debug aid: APS5_NO_WORDWISE_CAPTURE=1 fetches every page whole. class ShaderMemory { public: - explicit ShaderMemory(std::span initial); + // The driver's answer for a range: nothing recorded writes it; recorded work writes it and the + // read must go through the flush hook; recorded work writes it but leaves it unchanged, so the + // bytes are read raw; VerifyRaw reads raw and through the hook, counting a difference; + // KnownValue: recorded work writes it with bytes the driver already knows (a copy HLE's + // destination, Driver.cpp's known-value ring entries), copied into `known` when the caller + // passes a span of exactly `bytes` (empty: the answer alone); VerifyKnownValue serves them and + // reads through the hook too, counting a difference. + enum class PendingWrite : std::uint8_t { None, Sync, Raw, VerifyRaw, KnownValue, VerifyKnownValue }; + using PendingWriteQuery = PendingWrite (*)(std::uint64_t address, std::size_t bytes, std::span known); + // Words served from known values by every capture, and the verified ones and their mismatches + // (APS5_VERIFY_KNOWN_VALUES=1), for Driver.cpp's [copy] line. + struct KnownValueCounts { + std::uint64_t served, verified, mismatches; + }; + static KnownValueCounts KnownValues(); + // Told, for a dword read that went through the hook, whether the bytes read before the hook's + // wait were still there after it (the driver's evidence for later raw reads). + using PendingWriteObserver = void (*)(std::uint64_t address, bool unchanged); + // The calling thread's count of hook syncs that waited for unfinished GPU work: a read is only + // observed when the count moved across it (against finished work both reads see the GPU's + // bytes, and "unchanged" would be no evidence). Null observes every read that went through + // the hook. + using HookWaitCounter = std::uint64_t (*)(); + // The calling thread's GPU waits so far in milliseconds (Recorder::ThreadWaitedMs), set once + // by the driver: the [capture] line charges the waits made inside a capture to it. + using WaitedMsProvider = double (*)(); + static void SetWaitedMsProvider(WaitedMsProvider provider); + explicit ShaderMemory(std::span initial, PendingWriteQuery pendingWrite = nullptr, PendingWriteObserver observe = nullptr, HookWaitCounter hookWaits = nullptr); // Returns what the capture resolved (plan, snapshot, specialization) for // ShaderRecompiler::Recompile(request, capture), which then skips its own materialization. One - // result per call: the draw path captures several stages on one ShaderMemory. - std::shared_ptr Capture(const ShaderRecompiler::RecompileRequest& request); + // result per call: the draw path captures several stages on one ShaderMemory. With `handle` + // (the driver's memoized ShaderRecompiler::ResolveSource result) the capture skips the source + // resolution. + std::shared_ptr Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle = nullptr); [[nodiscard]] std::vector Regions() const; + // The page regions read since the previous call (or construction), a word read again + // included, the initial regions excluded: one stage's own reads on the draw path's shared + // ShaderMemory (Regions() stays the union). The spans point into the pages, as Regions()'s do. + [[nodiscard]] std::vector TakeRecentRegions(); + // The driver's source handle memo outcomes, for the [capture] line (APS5_PROFILE_DRAW). + static void CountHandleMemo(bool hit); private: static constexpr std::size_t PageBytes = 4096; @@ -32,6 +77,8 @@ private: std::array words{}; std::bitset valid; std::bitset read; + std::bitset recent; + bool wordwise = false; }; static bool read(void* context, std::uint64_t address, std::uint32_t* value); @@ -41,8 +88,26 @@ private: // the caller keeps them alive for as long as the capture is used. std::map> initial; std::map pages; + PendingWriteQuery pendingWrite = nullptr; + PendingWriteObserver observe = nullptr; + HookWaitCounter hookWaits = nullptr; }; +// The positions, among a dispatch-cache variant's stored words, of the pure flat-SRT leaves a +// capture read (ShaderRecompiler::SrtReadTrace), for a hit that differs only there. `runs` are the +// variant's [begin, end) byte ranges in address order and `words` their dwords in that order; +// `leaves` are (flat offset, address) and `otherReads` the sorted addresses of every other read. +// A leaf is skipped when its address is among the other reads (a walk read aliases the dword, +// counted `aliased`), is not dword-aligned or lies in no run (counted `unmapped`), or its stored +// word differs from the flattened SRT's at that offset (counted `mismatched`). `positions` +// (sorted, may repeat) and `slots` (the flat offset of each) are parallel. +struct DataWordPositionCounts { + std::uint64_t unmapped = 0; + std::uint64_t mismatched = 0; + std::uint64_t aliased = 0; +}; +DataWordPositionCounts DataWordPositions(std::span> runs, std::span> leaves, std::span otherReads, std::span words, std::span flattenedSrt, std::vector& positions, std::vector& slots); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 2c1fef9ace..c94e666bd0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -6,7 +6,10 @@ #include "Recompiler.hpp" #include "prx/libSceAgcDriver/Execution/include/Presentation.hpp" #include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" +#include "prx/libSceAgcDriver/Execution/include/Recipe.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include #include #include @@ -18,6 +21,10 @@ class StorageTexture; // Stage A of a dispatch's resource build (see VulkanDevice::PrepareDispatch); opaque to callers. struct PreparedDispatch; +// The unlocked pre-checks of a recipe hit (see VulkanDevice::PrepareRecipe); opaque to callers. +struct RecipeHit; +// The record tail shared by an ordinary dispatch and a recipe hit (VulkanDevice::recordDispatch). +struct RecordedDispatch; class VulkanDevice { public: @@ -26,6 +33,11 @@ public: VulkanDevice(const VulkanDevice&) = delete; VulkanDevice& operator=(const VulkanDevice&) = delete; ShaderRecompiler::SpirvTarget Target() const; + // Distinguishes this device from every earlier one in the process (a memo keyed by the device + // cannot rely on the pointer, which a replacement may reuse). + std::uint64_t Serial() const { return serial; } + // The Vulkan device handle (a Recipe names the device it was built on). + VkDevice Device() const; // Drains everything under the caller's GpuMutex: recorder Sync plus vkDeviceWaitIdle. For suspend, // resize, device replacement, CPU fill fallback and APS5_DRAIN_ALL; the packet-loop drains use the // three-step form below so the GPU wait happens without the mutex. @@ -59,7 +71,7 @@ public: // under one lock passes it for the first label only. int WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true); // Pending-label table lookup and open-batch overlap test for WAIT_REG_MEM (see Recorder). - std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; + std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, Graphics::Recorder::LabelRefusal* refusal = nullptr) const; bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; // Fills [address, address + bytes) of host-imported guest memory with a repeating 16-byte pattern, // recorded behind the open batch; false when the range is not imported (the caller stores it). @@ -68,24 +80,126 @@ public: // CPU (a copied buffer's write-back, a deferred label). Debug aid: APS5_FILL_SYNC=1 waits for // every recorded store over the range. bool FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern); + // Copies `bytes` of guest memory from `source` to `destination` (disjoint ranges) in place of the + // engine's memcpy kernel (Driver.cpp copyBuffer). `path` 0: copied on the CPU at once, when + // every test of the rule holds (each a pure query, nothing flushed or recorded before the + // decision): at most `cpuMax` bytes; no storage image with results pending over either range; + // the source settled (no recorded write, or every batch writing it signaled and no completion + // store or label pending in it); no recorded write and no label (recorded or queued) in the + // destination; and no unexecuted batch reads the destination in place (Recorder read tracking; + // with APS5_COPY_READ_TRACKING=0 the recorder must be idle instead). The bytes are then settled: + // later CPU readers never wait for a batch, and no GPU work sees them early. 1: recorded as a + // transfer between the host imports behind the open batch, ordered like FillBuffer (recorded GPU + // stores by its barrier, a completion's CPU store over either range by a wait, counted in + // `synced`/`syncMs`), with the source noted as an in-place read, only for at most `gpuMax` + // bytes (APS5_COPY_HLE_GPU_MAX; a larger copy answers 8 and the caller runs the kernel: its + // dispatch costs the GPU the same as the transfer); 2: a range is not host-imported (the + // caller runs the kernel). `reason` says which test refused the CPU path. + // `queue` is the calling queue (the wait experiment below never waits on queue 0). + // `noteWriter` is called for paths 1 and 3 right BEFORE the destination's pending write is + // noted (the caller's evidence ring entry then precedes the note, as a dispatch's elements do) + // with the destination's post-copy bytes when they are known now (path 1, at most `knownMax` + // bytes, and the source CPU-current at the note: no recorded write, pending image, unpublished + // shadow, pending label or copied writer over it; 0 disables) and the write-watch generation + // MarkWritten(destination) stamped, else an empty span and 0. + // `programAddress` keys the transfer's [gputime] range. Debug aids: APS5_COPY_SYNC=1 waits for + // every recorded store over either range; APS5_COPY_WAIT_SOURCE=1 waits (unlocked) for a + // source whose only unfinished writer is the oldest in-flight batch and decides again + // (`waitedForSource`/`waitMs`); APS5_COPY_VERIFY=1 follows every CPU copy with a full sync and + // byte compares (CopyVerifyCounts). + struct CopyOutcome { + // 0 a CPU copy, 1 a transfer between the host imports, 2 refused (not imported), 3 a + // device copy between the surfaces' storage images (StorageTexture::CopyFrom; the + // destination's write-back is deferred; APS5_NO_COPY_ALIAS=1 disables), 8 refused (over + // `gpuMax`; Driver.cpp's CopyOverGpuMax). + int path; + bool synced; + double syncMs; + // DestinationPending: an unsignaled batch writes the destination; DestinationUnsettled: + // every writer signaled but a completion store of one is still to land (as for the source). + // Shadow: a unit shadow holds unpublished results over either range (the GPU path + // publishes them; see UnitShadow.hpp). + enum Refusal { None = 0, Size, Image, SourcePending, SourceUnsettled, DestinationPending, DestinationUnsettled, Label, Reader, Shadow, Refusals }; + int reason; + // Path 0 taken although a batch had written the source: every such batch had signaled. + bool sourceSettledBySignal; + bool waitedForSource; + double waitMs; + // The reader that refused (reason Reader): batch serial, its queue, the reader kind + // (Graphics::Recorder::ReadKind) and whether it is the open batch. + std::uint64_t readerSerial; + std::uint32_t readerQueue; + int readerKind; + bool readerOpen; + }; + using CopyWriterNote = std::function value, std::uint64_t generation)>; + CopyOutcome CopyBuffer(std::uint64_t destination, std::uint64_t source, std::size_t bytes, std::size_t cpuMax, std::size_t gpuMax, std::size_t knownMax, std::uint64_t programAddress, std::uint32_t queue, const CopyWriterNote& noteWriter); + // APS5_COPY_VERIFY=1 totals: CPU copies verified, whose source bytes changed over the sync (a + // producer the rule missed), whose destination bytes were overwritten (a store the rule + // missed), and whose destination an unsignaled batch read in place (the rule's own answer + // re-checked without the fence shortcut). + struct CopyVerification { + std::uint64_t verified, sourceChanged, destinationChanged, readerMissed; + }; + static CopyVerification CopyVerifyCounts(); + // Aliased copies (path 3): made, destination images created for them, and the transfers + // taken instead because the source had no live image, the destination's image differed in + // shape, or the image copy refused (keys, not imported, released). + struct CopyAliasing { + std::uint64_t aliased, created, noSource, shape, refused; + }; + static CopyAliasing CopyAliasCounts(); void ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable); void* Window() const; void Resize(std::uint32_t width, std::uint32_t height); bool Presentable() const; - // A presentation is four steps so the presenter holds GuestMemory::GpuMutex only while it touches - // the queue: AcquireImage takes the next swapchain image (no mutex needed: the swapchain and its - // fences are the presenter's own, and in FIFO mode this is where a frame waits for the vblank); - // PresentClear/PresentDisplayBuffer record and submit the frame (under the mutex) and return - // whether one is in flight; FinishPresent waits for its render fence (no mutex); QueuePresent - // hands the image to the swapchain (under the mutex again). AcquireImage returns false when the - // swapchain is out of date (the frame is dropped; the next Resize recreates it); a present without - // a prior AcquireImage acquires itself. PresentPixels does all steps itself. + // A presentation is a few steps so the presenter holds GuestMemory::GpuMutex only while it + // touches the queue. Presentations are slots (FlipInFlight() + 1, each with its own command + // buffer, fence, kept resident image and dump buffer): RetirePresents(keep) (no mutex) retires + // the slots whose fence signaled, oldest first, and waits for the oldest ones until at most + // `keep` are in flight (the only CPU wait of the flip path; it returns the time spent waiting); + // AcquireImage takes the next swapchain image (no mutex needed: the swapchain and its fences + // are the presenter's own, and in FIFO mode this is where a frame waits for the vblank); + // PresentClear/PresentDisplayBuffer record and submit the frame into the next slot (under the + // mutex; the blit follows the frame's batches in queue order, so the displayed image is always + // complete) and return whether one was submitted; QueuePresent hands the image to the swapchain + // (under the mutex). FinishPresent = RetirePresents(0): the render fence wait of the + // synchronous paths (APS5_FLIP_INFLIGHT=0, APS5_SYNC_FLIP, PresentPixels). AcquireImage returns + // false when the swapchain is out of date (the frame is dropped; the next Resize recreates it); + // a present without a prior AcquireImage acquires itself, one whose slot is in flight waits for + // it. PresentPixels does all steps itself. Retiring a slot releases its kept image and writes + // its frame dump; WaitIdle, Resize and the destructor retire every slot before destroying what + // an in-flight blit reads. + // APS5_FLIP_INFLIGHT=: presentations whose fence may be unsignaled when the next blit is + // submitted (0: today's render fence wait before the flip completes, 1 default, 2; larger + // values are refused; 0 when APS5_SYNC_FLIP is set). + static std::size_t FlipInFlight(); + double RetirePresents(std::size_t keepInFlight); + // Whether presenting `buffer` will use the single scaler source, staging or upload objects (a + // GPU frame dump, the guest-memory path, a source-size change) while an in-flight blit still + // reads them, i.e. wait for every slot: the presenter then retires them before taking the + // mutex (the same wait under it is only the fallback when the answer changes in between). + bool PresentWaitsForSlots(const DisplayBuffer* buffer) const; bool AcquireImage(); bool PresentClear(std::uint32_t width, std::uint32_t height, bool opaque); void PresentPixels(std::uint32_t width, std::uint32_t height, std::span pixels); bool PresentDisplayBuffer(const DisplayBuffer& buffer); - void FinishPresent(); + double FinishPresent(); void QueuePresent(); + // At the flip packet (under the mutex): the recorder's submissions so far and, under + // APS5_PROFILE_DRAW, how many in flight have not signaled (the frame record's batches_at_flip). + void FlipBatches(std::uint64_t& submissions, std::uint64_t& unsignaled) const; + // Cumulative presentation counters for the [present] line (APS5_PROFILE_DRAW): presentations + // submitted; over the batches the retired blits waited for: GPU busy time, idle gaps and the + // batches whose completion record was missing; sums of the batches submitted between the flip + // packet and the blit (ahead of it) and behind the blit (after_flip), of the last-submit and + // blit-submit times after the flip packet; noted in-place reads checked + // (APS5_FLIP_READ_CHECK=1) and found overwritten before the blit's fence. + struct PresentStatistics { + std::uint64_t presents, gpuUnread, batchesAheadOfBlit, batchesAfterFlip, readsChecked, readsOverwritten; + double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs; + }; + static PresentStatistics PresentCounts(); // Stage A of a dispatch's resource build, run WITHOUT GuestMemory::GpuMutex before Dispatch or // DispatchIndirect: the binding plan, data buffers, imports already serving the guest buffers, // read-only copies and the descriptor set (see ShaderResources). The dispatch completes it @@ -94,7 +208,13 @@ public: // Revalidate stays under the mutex), or APS5_LOCKED_BUILD=1 keeps the whole build under it as // before. `shader` and `snapshots` must outlive the dispatch. std::shared_ptr PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots); - void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr); + // APS5_PROFILE_DRAW: the parts of a PrepareDispatch in milliseconds, in the order key, find, + // precollect, presync, stage A (the driver's 'prepare:' rows). + static std::span PreparePhaseMs(const PreparedDispatch& prepared); + // `recipe`, when given, receives the Recipe a successful call built for its dispatch-cache + // variant (design_cpu_final M4): only when the resource cache served or took the object + // (reusable, cacheable) and recipes are on (APS5_NO_DISPATCH_RECIPE=1 builds none); else null. + void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr, std::shared_ptr* recipe = nullptr); // A dispatch whose group counts are the three dwords at `arguments` in guest memory // (DISPATCH_INDIRECT): the GPU reads them in place from the host import, ordered after everything // recorded before, so the CPU never waits for the shader that wrote them. When the GPU could not @@ -109,18 +229,81 @@ public: int cpuReason; double argumentReadMs; }; - IndirectOutcome DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr); - void Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); + IndirectOutcome DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr, std::shared_ptr* recipe = nullptr); + // A value-equal dispatch-cache hit with a recipe (design_cpu_final M4), in two steps. + // PrepareRecipe, WITHOUT GuestMemory::GpuMutex: the pre-checks (recipes on, the recipe's + // device is this one, its template and pipeline objects still alive), the template's + // precollect and the pre-sync from the recipe's surface list (recomputed from the template when + // the host-import identity it was made under moved). Null when a pre-check fails (counted on + // the [recipe] line by reason): the caller takes the ordinary path. `indirect` files the + // counters under the indirect rows. + std::shared_ptr PrepareRecipe(const std::shared_ptr& recipe, bool indirect); + // DispatchRecipe, under the mutex after the packet's labels: reaps the pre-synced batches, + // proves the template current (ShaderResources::ProveCurrent, the authoritative check), + // touches the resource cache's LRU, decides an indirect dispatch's argument path as + // DispatchIndirect does, and records from the recipe's objects (the template refreshed by + // vkCmdUpdateBuffer only when its data-word hash differs from the recipe's). Rebuild when the + // proof failed (the template is removed from the cache and kept by the batch) or the device + // changed: nothing was recorded, the caller releases the mutex and restarts at + // PrepareDispatch. `arguments` non-zero is a DISPATCH_INDIRECT (`outcome` as DispatchIndirect + // reports it). `verify` (APS5_VERIFY_RECIPE=1): the PrepareDispatch made beside the hit, whose + // found object, the pipeline map's objects and the per-word RefreshData decision must agree + // with the recipe's, else the process aborts. `refreshByWords`: the driver's data-only hit + // (`shader` carries live flat-SRT words the recipe's hash does not name), so the template's + // data refresh is decided by the per-word compare instead of the hash. + RecipeOutcome DispatchRecipe(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, const std::shared_ptr& hit, IndirectOutcome& outcome, const std::shared_ptr& verify = nullptr, bool refreshByWords = false); + // Whether dispatch-cache hits use recipes (APS5_NO_DISPATCH_RECIPE unset) and whether every + // hit is verified (APS5_VERIFY_RECIPE=1). + static bool DispatchRecipes(); + static bool VerifyRecipes(); + // Whether compute templates refresh their data buffers with a hit's words instead of keying + // them (APS5_NO_TEMPLATE_DATA_REFRESH unset); the driver's data-only hits require it. + static bool TemplateDataRefresh(); + // The driver's recipe events for the [recipe] line: a restart after Rebuild, an attach; the + // rows are the dispatch, indirect and draw kinds. NoteDrawRecipeMiss files a draw hit's + // pre-check miss made in the driver (no recipe for the matched variants, or a device mismatch). + enum class RecipeEvent { Restart, Attach }; + enum class RecipeKind : std::size_t { Dispatch, Indirect, Draw }; + static void NoteRecipe(RecipeEvent event, bool indirect); + static void NoteRecipe(RecipeEvent event, RecipeKind kind); + enum class DrawRecipePrecheck : std::size_t { NoRecipe, Device }; + static void NoteDrawRecipeMiss(DrawRecipePrecheck miss); + // `recipe`, when given, receives the DrawRecipe a recorded, cacheable, reusable, direct draw + // built for its draw-cache entry (design_cpu_final M8); null otherwise. + void Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}, std::shared_ptr* recipe = nullptr); + // A draw-cache hit recorded from its recipe (Graphics::DrawWithRecipe), under the mutex after + // the packet's labels; Rebuild when the recipe's device is not this one or DrawWithRecipe + // missed (nothing recorded: the caller runs Draw with the hit's stages and re-attaches). + RecipeOutcome DrawFromRecipe(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, const std::shared_ptr& recipe); + // The indirect draw features the device enabled, for the driver's decision whether an indirect + // draw's records can be read by the GPU (Graphics::Draw) or must be resolved on the CPU. + struct IndirectDrawSupport { + bool firstInstance; + bool multi; + bool count; + }; + IndirectDrawSupport DrawIndirectSupport() const; void EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); private: + // The device's Graphics::Context: a copy of the one built at setup (its instance functions + // resolved then, its function table filled then), or with APS5_NO_CONTEXT_CACHE=1 built anew. Graphics::Context graphicsContext() const; + Graphics::Context buildContext() const; // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups. - IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared); + IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe); + // The stage-A pre-sync over `surfaces` (see PrepareDispatch): the serial waited for, 0 none. + std::uint64_t presync(std::span> surfaces); + // A DISPATCH_INDIRECT's argument path (see DispatchIndirect): GPU-side from the host import, or + // the counts read on the CPU (the record then continues as a direct dispatch). + void decideIndirect(RecordedDispatch& record, IndirectOutcome& outcome, char* groupsText); + // The tail of a dispatch's device call from the open batch's command buffer to the completion + // registration: keeps, the template data refresh, barriers, bind, push, dispatch, marks. + void recordDispatch(RecordedDispatch& record); bool present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display = nullptr, const std::shared_ptr& resident = nullptr, VkFilter residentFilter = VK_FILTER_LINEAR, bool dumpFrame = false); - void writeFrameDump(); struct State; std::unique_ptr state; + std::uint64_t serial; }; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index 2a82386bb1..697d37b5bc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -11,7 +11,11 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libc/include/Shutdown.hpp" +#include "prx/libc/include/CpuTopology.hpp" #include "ControlFlow/RequestSerializer.hpp" +#include "CacheKey.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "Optimization/ShaderStageInputInfo.hpp" #ifdef _WIN32 #include #endif @@ -40,6 +44,7 @@ #include #include #include +#include #include #include @@ -77,24 +82,123 @@ void require(bool condition, const char* reason) { } } +// The recompiler's resolved sources for programs of one registered shader (design13 step 3), keyed +// by the code offset, the device and the request's context (RecompileCacheKey::ContextHash): a +// capture then skips the stage input validation, the code hash, the key build and the source table. +// A snapshot object is replaced on re-registration, so the memo dies with it; a few entries per +// shader (the same program is requested with the same registers), the oldest replaced. Several +// queues dispatch one snapshot: mutexed. A compute resolve that threw (a plan failure, static per +// program and context) is kept as a poisoned entry (no handle, the message) so the dispatch skips +// the program without resolving, capturing or throwing again; `poisoned` counts the entries ever +// poisoned (the common path's test). A draw stage's resolve is never poisoned: its stage input +// validation throws on V# words (base, record count) the key does not cover, so the same key can +// resolve on the next draw. +struct HandleMemos { + struct Entry { + std::uint64_t key = 0; + std::shared_ptr handle; + std::shared_ptr failure; + }; + std::mutex mutex; + std::array entries; + std::size_t next = 0; + std::atomic poisoned{0}; +}; + +// The recompiler's switch (APS5_NO_FAILURE_MEMO=1 rebuilds a failed plan every time) disables the +// poisoned source-handle entries too. +static bool FailureMemo() { + static const bool memo = std::getenv("APS5_NO_FAILURE_MEMO") == nullptr; + return memo; +} + struct ShaderSnapshot { std::uint64_t codeAddress; std::uint64_t headerAddress; std::uint8_t type; std::vector code; std::vector header; + std::unique_ptr handles = std::make_unique(); }; +using ShaderRegistry = std::map>; + +// The source handle for `request` (a program of `snapshot` at `codeOffset`, compiled for the device +// of `deviceSerial`), from the snapshot's memo or resolved and memoized. Null when the memo is +// bypassed: `bypass` (the register probe, whose flag the recompiler's own key carries), +// APS5_NO_SOURCE_HANDLE_CACHE=1 (every capture resolves, as before) or APS5_NO_CAPTURE_REUSE=1 +// (Recompile(request) resolves by itself). Debug aid: APS5_VERIFY_SOURCE_HANDLE=1 resolves on +// every hit as well and fails the dispatch when the memo answered a different source. +// A poisoned entry (see HandleMemos) answers null and, with `poisoned` given, its stored message +// (owned by the memo: valid while the snapshot lives); without `poisoned` it throws the message, +// as the resolve did. +std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, const ShaderRecompiler::RecompileRequest& request, bool bypass, const std::string** poisoned = nullptr) { + static const bool enabled = std::getenv("APS5_NO_SOURCE_HANDLE_CACHE") == nullptr && std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; + static const bool verify = std::getenv("APS5_VERIFY_SOURCE_HANDLE") != nullptr; + if (!enabled || bypass || ShaderRecompiler::DebugProbeActive() || !request.useCache) return nullptr; + std::uint64_t key = 0xcbf29ce484222325ull; + for (const auto value : {static_cast(codeOffset), deviceSerial, ShaderRecompiler::RecompileCacheKey::ContextHash(request)}) { + key ^= value; + key *= 0x100000001b3ull; + } + auto& memos = *snapshot.handles; + { + std::lock_guard lock(memos.mutex); + for (const auto& entry : memos.entries) { + if (entry.key != key || (entry.handle == nullptr && entry.failure == nullptr)) continue; + if (entry.handle == nullptr) { + if (verify) { + bool resolved = true; + try { + static_cast(ShaderRecompiler::ResolveSource(request)); + } catch (const std::exception&) { + resolved = false; + } + if (resolved) throw std::runtime_error("AGC driver: source handle memo poisoned but the source resolved"); + } + ShaderMemory::CountHandleMemo(true); + if (poisoned == nullptr) throw std::runtime_error(*entry.failure); + *poisoned = entry.failure.get(); + return nullptr; + } + if (verify && ShaderRecompiler::ResolveSource(request)->source != entry.handle->source) throw std::runtime_error("AGC driver: source handle memo answered a different source"); + ShaderMemory::CountHandleMemo(true); + return entry.handle; + } + } + std::shared_ptr handle; + try { + handle = ShaderRecompiler::ResolveSource(request); + } catch (const std::exception& error) { + if (FailureMemo() && request.shader.stage == ShaderRecompiler::ShaderStage::Compute) { + std::lock_guard lock(memos.mutex); + memos.entries[memos.next] = {key, nullptr, std::make_shared(error.what())}; + memos.next = (memos.next + 1) % memos.entries.size(); + memos.poisoned.fetch_add(1, std::memory_order_relaxed); + } + throw; + } + ShaderMemory::CountHandleMemo(false); + if (handle == nullptr) return nullptr; + std::lock_guard lock(memos.mutex); + memos.entries[memos.next] = {key, handle, nullptr}; + memos.next = (memos.next + 1) % memos.entries.size(); + return handle; +} + struct Submission { std::uint64_t serial; std::uint32_t queue; std::vector commands; - std::map> shaders; + // The registry as of the submit call (shared, never mutated once shared: Driver::RegisterShader + // copies it first when a submission still holds it). + std::shared_ptr shaders; std::map> flips; bool suspend = false; // Record-order stamp (Driver::eventSerial) taken when the game submitted: a WAIT_REG_MEM of this // submission trusts only labels the recorder noted with a newer stamp (see Recorder::NoteLabel). std::uint64_t received = 0; + std::chrono::steady_clock::time_point enqueuedAt{}; }; std::uint32_t readRegister(const Registers& registers, std::uint32_t offset) { @@ -174,8 +278,31 @@ std::atomic queuedLabels{0}, labelGroups{0}, immediateLabels{0}; // label overlapping the capture, which the packet then redoes), and suspend points (all in the // [labels] line). std::atomic packetLockRecords{0}, packetLockSubmits{0}, packetLockDeferred{0}, packetSubmitDeferred{0}, captureTryRecords{0}, captureRetries{0}, suspendPoints{0}; +// Boundary record-tries not made because the next packet is self-locking and the group's deadline +// has not passed (Driver::flushBetweenPackets, [labels] line). +std::atomic recordTriesSkipped{0}; +// Queue 0's worker sleeps in its untimed idle wait (Driver::run): nothing pending when it went to +// sleep, so a completion registered since has no reaper until work arrives, unless a poller reaps. +std::atomic queue0Dormant{false}; // APS5_PROFILE_DRAW: the time (microseconds) the record and the submit take at the packet's lock. std::atomic packetLockRecordUs{0}, packetLockSubmitUs{0}; +// Failed try_locks of the GPU mutex by site ([labels] line): flushBetweenPackets' label record, +// its flush (deadline submit or size cap) and its boundary reap, a wait's poll service, the idle +// reap, the record after a capture and the record from a poll loop. +enum TrySite { TryLabel = 0, TryFlush, TryReap, TryPoll, TryIdle, TryCapture, TryPollRecord, TrySites }; +std::atomic triesFailed[TrySites] = {}; +// Per-submission fixed costs by queue (APS5_PROFILE_DRAW; nanoseconds, printed as microseconds +// per submission on the queue's [packets] line and reset there): Driver::Submit's validation and +// copy on the game thread, the worker's dequeue latency (enqueue to pop) and its completion +// (markCompleted + notify), the suspend point's label record + submit under the GPU mutex (or +// skipped, nothing open), the submission-end submits made and skipped, the notifies skipped. +struct SubmissionCosts { + std::atomic submissions{0}, validateNs{0}, copyNs{0}, dequeueNs{0}, completeNs{0}, suspendNs{0}, suspends{0}, suspendsSkipped{0}, endSubmits{0}, endSkipped{0}, notifiesSkipped{0}; +}; +SubmissionCosts& submissionCosts(std::uint32_t queue) { + static std::array costs; + return costs[queue == 0 ? 0 : std::min(queue - 0x20 + 1, 63)]; +} // Non-label stores the driver makes for the title (COPY_DATA, DMA_DATA, DUMP_CONST_RAM; see the // store path of Driver::execute): recorded on the GPU, recorded on the GPU and stored again by a // completion action (behind a pending write-back, or memory the GPU has no view of: reasons 5 and @@ -220,6 +347,32 @@ DeferredLabels& deferredLabels() { static thread_local DeferredLabels deferred; return deferred; } +// APS5_WORKER_AFFINITY=1 sends the queue workers and the presenter to the performance cores of a +// hybrid CPU (with libkernel's APS5_JOB_AFFINITY=1 for the title's spinning job workers); pinned +// by default the video stage measured 0 to -9 % (the presenter's GPU wait doubled), so the default +// leaves them free. APS5_WORKER_AFFINITY_MASK= picks the cores; APS5_TRACE_AFFINITY=1 names +// each pinned thread, otherwise the first pin prints one summary line. +std::uint64_t WorkerAffinityMask() { + static const std::uint64_t mask = [] { + if (std::getenv("APS5_NO_WORKER_AFFINITY") != nullptr) return std::uint64_t{0}; + const auto requested = CpuTopology::MaskFromEnvironment("APS5_WORKER_AFFINITY_MASK"); + if (requested != 0) return requested; + if (std::getenv("APS5_WORKER_AFFINITY") == nullptr) return std::uint64_t{0}; + const auto& layout = CpuTopology::Get(); + return layout.hybrid ? layout.performant : std::uint64_t{0}; + }(); + return mask; +} +void PinWorkerThread(const char* role) { + const auto mask = WorkerAffinityMask(); + if (mask == 0) return; + static std::once_flag summary; + std::call_once(summary, [mask] { + const auto& layout = CpuTopology::Get(); + std::fprintf(stderr, "[affinity] queue workers and presenter -> 0x%llx (hybrid=%d efficient=0x%llx performant=0x%llx process=0x%llx)\n", static_cast(mask), layout.hybrid ? 1 : 0, static_cast(layout.efficient), static_cast(layout.performant), static_cast(layout.process)); + }); + CpuTopology::PinTraced(role, nullptr, mask); +} bool DeferLabels() { // APS5_DRAIN_COMPLETION_LABELS=1 implies the locked path too: a label that must drain then // keeps the packet path's unlocked timeline wait instead of a WaitIdle under the mutex, so @@ -227,6 +380,40 @@ bool DeferLabels() { static const bool defer = std::getenv("APS5_LABEL_LOCK_EACH") == nullptr && std::getenv("APS5_DRAIN_COMPLETION_LABELS") == nullptr; return defer; } +// Queued labels stay queued past the packets that follow them: a packet that records device work +// under its own hold records them first (Driver::recordLabelsForPacket), any other packet records +// them only if the mutex happens to be free, and the deadline still bounds the group. Meanwhile a +// WAIT_REG_MEM of the same queue takes a queued label's value from the label table +// (Recorder::NoteQueuedLabel, entered by flushBetweenPackets right after the label packet), a CPU +// access of a queued label's range by this worker has the group recorded by the flush hook +// (Driver::recordQueuedLabelsFromHook), and a worker polling in a wait records its group as soon +// as the mutex is free (Driver::LabelFlushDue). Before, every packet that could touch memory, +// waits included, took the mutex outright to record the group first: queue 0's first acquisition +// after a run of unlocked packets, absorbing the compute queues' whole critical sections ([lock] +// site 'label'). Debug aid: APS5_NO_QUEUED_LABELS=1 restores the outright record; +// APS5_NO_LABEL_BATCHING=1 restores it together with the other label-batching changes (the +// per-packet submit below and the recorder's store runs). +bool QueuedLabelTable() { + static const bool enabled = std::getenv("APS5_NO_QUEUED_LABELS") == nullptr && std::getenv("APS5_NO_LABEL_BATCHING") == nullptr; + return enabled; +} +// A recorded label no longer makes the worker submit the open batch at its next packet: the batch +// goes out at the deadline (APS5_LABEL_FLUSH_US), after APS5_BATCH_CAP dispatches or draws, or when +// a consumer needs it (a wait on its range, a flush-hook sync, the submission's end, a flip). Tens +// of thousands of one-label submissions per 10 s went away with it. Debug aid: +// APS5_NO_LABEL_BATCH_SUBMIT=1 (or APS5_NO_LABEL_BATCHING=1) submits at the next packet as before +// (with the 250 us deadline). +bool LabelBatchSubmit() { + static const bool enabled = std::getenv("APS5_NO_LABEL_BATCH_SUBMIT") == nullptr && std::getenv("APS5_NO_LABEL_BATCHING") == nullptr; + return enabled; +} +// How many of this worker's queued labels are in the label table (the group's first entries). +std::size_t& queuedLabelsNoted() { + static thread_local std::size_t noted = 0; + return noted; +} +// [labels] line: queued labels recorded from a wait's poll loop and from the flush hook. +std::atomic pollLabelRecords{0}, hookLabelRecords{0}; // Whether a packet needs this worker's deferred labels recorded before it runs: everything except // pure register and state writes, NOP, and the events the driver does not execute. A wait, dispatch, // draw, copy, indirect register load, constant RAM dump or flip either reads memory a label may @@ -253,7 +440,7 @@ bool NeedsRecordedLabels(std::uint32_t header) { bool PacketLocksItself(std::uint32_t header) { if (header == FlipPacketHeader) return true; switch ((header >> 8u) & 0xffu) { - case 0x15: case 0x16: case 0x2d: case 0x35: return true; + case 0x15: case 0x16: case 0x2d: case 0x35: case 0x24: case 0x25: case 0x2c: case 0x38: return true; default: return false; } } @@ -301,14 +488,19 @@ public: require(descriptor.flags == 0, "nonzero submission flags are not implemented"); Submission submission{}; submission.queue = queue; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& costs = submissionCosts(queue); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; if (descriptor.dw_num != 0) { require(descriptor.dw_num <= std::numeric_limits::max() / sizeof(std::uint32_t), "command size overflow"); GuestMemory::CheckRange(descriptor.addr, static_cast(descriptor.dw_num) * sizeof(std::uint32_t), alignof(std::uint32_t)); submission.commands.assign(descriptor.addr, descriptor.addr + descriptor.dw_num); } + const auto copied = profile ? std::chrono::steady_clock::now() : start; validate(submission.commands, queue, descriptor.addr); static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr; if (trace) std::fprintf(stderr, "[gpu] %.1f submit queue=0x%x dwords=%zu at %p\n", TraceMs(), queue, submission.commands.size(), static_cast(descriptor.addr)); + const auto validated = profile ? std::chrono::steady_clock::now() : start; { std::lock_guard lock(mutex); rethrowFailure(); @@ -331,6 +523,13 @@ public: // Every CPU store the game made before this call precedes the stamp; labels recorded // later carry a larger one. submission.received = ++eventSerial; + if (profile) { + const auto now = std::chrono::steady_clock::now(); + submission.enqueuedAt = now; + ++costs.submissions; + costs.validateNs += static_cast(std::chrono::duration_cast(validated - copied).count()); + costs.copyNs += static_cast(std::chrono::duration_cast((copied - start) + (now - validated)).count()); + } enqueue(std::move(submission)); ++accepted; } @@ -341,7 +540,9 @@ public: require(!OnWorkerThread(), "worker cannot wait for itself"); std::unique_lock lock(mutex); const auto target = accepted; + ++idleWaiters; changed.wait(lock, [&] { return failure != nullptr || completed >= target; }); + --idleWaiters; rethrowFailure(); } @@ -356,6 +557,7 @@ public: boundary.suspend = true; // The boundary resets graphics state, so it runs in order with the graphics queue. boundary.queue = 0; + boundary.enqueuedAt = std::chrono::steady_clock::now(); enqueue(std::move(boundary)); ++accepted; // The suspend point only marks where the system may suspend the title; it does not wait for @@ -398,6 +600,7 @@ public: for (const auto& [offset, flip] : item.flips) flip->Fail(failure); } worker.pending.clear(); + worker.queued.store(0, std::memory_order_release); } } changed.notify_all(); @@ -408,21 +611,33 @@ public: // thread_local per DLL, so the one the video-out library set is invisible here. PerformanceContext timingContext(window.timing.get()); PerformanceTimer timing("Driver.Present"); + static thread_local bool pinned = false; + if (!pinned) { + pinned = true; + PinWorkerThread("presenter"); + GuestMemory::MarkPresenterThread(); + } CheckFailure(); require(gpuReady != nullptr && context != nullptr, "missing GPU completion callback"); require(window.getDrawableSize != nullptr, "missing window drawable size query"); - // The device is held only while the presentation is recorded and submitted; the swapchain - // acquire (where FIFO mode waits for the vblank) and the render fence wait happen outside - // GpuMutex so the queue workers are not blocked for the present's GPU or display time (the - // presenter submits on the same VkQueue as the recorder, so queue order makes the frame's - // batches complete first). Debug aid: APS5_SYNC_FLIP=1 drains the device and presents - // synchronously under one mutex hold, as before. + // The device is held only while the presentation is recorded, submitted and queued; the + // swapchain acquire (where FIFO mode waits for the vblank) and the wait for older + // presentations happen outside GpuMutex so the queue workers are not blocked for the + // present's GPU or display time. The flip completes once the blit is queued behind the + // frame's batches on the one VkQueue (queue order makes the displayed image complete); + // the CPU waits only when more than APS5_FLIP_INFLIGHT presentations trail on the GPU + // (VulkanDevice::RetirePresents). APS5_FLIP_INFLIGHT=0 waits for the frame's own fence + // before completing the flip, as before; APS5_SYNC_FLIP=1 also drains the device and + // presents synchronously under one mutex hold. static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; + static const std::size_t inFlight = VulkanDevice::FlipInFlight(); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; std::shared_ptr presenting; timing.Mark("validate"); try { bool submitted = false; bool presentable = false; + double waitedMs = 0; { GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); std::lock_guard lock(GuestMemory::GpuMutex()); @@ -443,8 +658,15 @@ public: if (buffer != nullptr) require(buffer->width == window.width && buffer->height == window.height, "display buffer extent differs from output"); } // The swapchain, its fences and semaphores are the presenter's own: only the VkQueue needs - // the mutex, so the acquire (and its vblank wait) is taken first without it. + // the mutex, so the older presentations' retirement and the acquire (with its vblank + // wait) are taken first without it. if (presentable && !syncFlip) { + if (inFlight != 0) { + // A present through the single scaler/staging objects (a frame dump, the + // guest-memory path) waits for every slot: here, not under the mutex. + waitedMs = presenting->RetirePresents(presenting->PresentWaitsForSlots(buffer) ? 0 : inFlight); + timing.Mark("inflight_wait"); + } presentable = presenting->AcquireImage(); timing.Mark("acquire_image"); } @@ -464,15 +686,18 @@ public: timing.Mark("present_clear"); } if (submitted && syncFlip) { - presenting->FinishPresent(); + waitedMs = presenting->FinishPresent(); timing.Mark("render_fence_wait"); + } + if (submitted && (syncFlip || inFlight != 0)) { + // vkQueuePresentKHR only waits GPU-side on the blit's semaphore. presenting->QueuePresent(); timing.Mark("queue_present"); submitted = false; } } if (submitted) { - presenting->FinishPresent(); + waitedMs = presenting->FinishPresent(); timing.Mark("render_fence_wait"); // The single VkQueue is shared with the recorder: presenting on it needs the mutex. GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); @@ -482,6 +707,7 @@ public: } gpuReady(context); timing.Mark("release_and_callback"); + if (profile) reportPresents(waitedMs, inFlight); CheckFailure(); } catch (...) { ReportFailure(std::current_exception()); @@ -489,6 +715,36 @@ public: } } + // The [present] line every 10 s (APS5_PROFILE_DRAW, presenter thread): presentations and the + // p50/p90 of the presenter's one CPU wait (inflight_wait, or render_fence_wait with + // APS5_FLIP_INFLIGHT=0), the GPU's busy time and idle gaps over the batches the retired blits + // waited for, the batches per flip (the recorder serial's advance over the window's flips) and + // per present (ahead of the blit since the flip packet, behind it), and the in-place read check. + void reportPresents(double waitedMs, std::size_t inFlight) { + static std::vector waits; + static VulkanDevice::PresentStatistics previous{}; + static std::uint64_t previousFlips = 0, windowSerial = 0, previousUnsignaled = 0; + static auto lastReport = std::chrono::steady_clock::now(); + waits.push_back(waitedMs); + if (waits.size() == 1) windowSerial = flipSerial.load(); + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::sort(waits.begin(), waits.end()); + const auto percentile = [&](double fraction) { return waits.empty() ? 0.0 : waits[std::min(waits.size() - 1, static_cast(fraction * static_cast(waits.size())))]; }; + const auto counts = VulkanDevice::PresentCounts(); + const auto presents = counts.presents - previous.presents; + const double perPresent = presents != 0 ? 1.0 / static_cast(presents) : 0.0; + const auto flips = flipsCounted.load() - previousFlips; + const double perFlip = flips != 0 ? 1.0 / static_cast(flips) : 0.0; + const auto serial = flipSerial.load(); + std::fprintf(stderr, "[present] %llu presents over 10 s (%zu may trail on the GPU); %s p50 %.2f ms, p90 %.2f ms, max %.2f ms over %zu; per present: GPU busy %.1f ms, idle gaps %.1f ms (%llu batches without a completion record); per flip: %.1f batches recorded (%.1f unsignaled at the flip); per present: %.1f batches ahead of the blit, %.1f after it; after the flip packet: last submit %.1f ms, blit submit %.1f ms; in-place reads overwritten before execution: %llu of %llu checked\n", static_cast(presents), inFlight, inFlight != 0 ? "inflight_wait" : "render_fence_wait", percentile(0.5), percentile(0.9), waits.empty() ? 0.0 : waits.back(), waits.size(), (counts.gpuBusyMs - previous.gpuBusyMs) * perPresent, (counts.gpuGapMs - previous.gpuGapMs) * perPresent, static_cast(counts.gpuUnread - previous.gpuUnread), static_cast(serial - windowSerial) * perFlip, static_cast(flipBatchesUnsignaled.load() - previousUnsignaled) * perFlip, static_cast(counts.batchesAheadOfBlit - previous.batchesAheadOfBlit) * perPresent, static_cast(counts.batchesAfterFlip - previous.batchesAfterFlip) * perPresent, (counts.lastSubmitAfterFlipMs - previous.lastSubmitAfterFlipMs) * perPresent, (counts.blitSubmitAfterFlipMs - previous.blitSubmitAfterFlipMs) * perPresent, static_cast(counts.readsOverwritten - previous.readsOverwritten), static_cast(counts.readsChecked - previous.readsChecked)); + previous = counts; + previousFlips = flipsCounted.load(); + previousUnsignaled = flipBatchesUnsignaled.load(); + waits.clear(); + } + void ReleaseWindow(void* window) { std::lock_guard lock(GuestMemory::GpuMutex()); if (device && device->Window() == window) device.reset(); @@ -520,7 +776,12 @@ public: std::lock_guard lock(mutex); rethrowFailure(); const auto address = snapshot.codeAddress; - shaders.insert_or_assign(address, std::make_shared(std::move(snapshot))); + // Submissions hold the registry by reference (a copy per submission cost a map clone): + // one they still reference is replaced by a copy, one only the driver holds (registration + // bursts at load time) is updated in place. + if (shaders == nullptr) shaders = std::make_shared(); + else if (shaders.use_count() != 1) shaders = std::make_shared(*shaders); + shaders->insert_or_assign(address, std::make_shared(std::move(snapshot))); } private: @@ -531,38 +792,648 @@ private: // Draws and dispatches take GuestMemory::GpuMutex, so device work stays serialized. struct QueueWorker { std::deque pending; + // pending.size(), readable without `mutex` (the worker's submission-end decision). + std::atomic queued{0}; std::thread thread; }; std::map workers; std::uint64_t frameSerial = 0; + // Flip packets seen, the recorder serial at the latest one that sampled it (0 until then; the + // drain paths do not) and, summed over the flips, the batches not signaled then (the [present] + // line; written by the queue-0 worker, read by the presenter). + std::atomic flipsCounted{0}; + std::atomic flipSerial{0}; + std::atomic flipBatchesUnsignaled{0}; // Packets executing now and packets finished, across all queues; waits time out only when neither // moves. On a cache line of their own: the waiting queues poll them while the others update them. alignas(64) std::atomic packetsInFlight{0}; std::atomic packetsDone{0}; // Orders the game's submit calls against the labels the workers record (Submission::received). std::atomic eventSerial{0}; - alignas(64) std::map> shaders; + alignas(64) std::shared_ptr shaders = std::make_shared(); // Dispatch cache: a program with the same user data and shader registers whose captured memory // (SRT chains, descriptors) is unchanged reuses its capture and recompile result. Guarded by - // dispatchCacheMutex; entries are evicted least recently used. - struct DispatchEntry { - std::shared_ptr memory; - std::vector captured; + // dispatchCacheMutex; entries are evicted least recently used. An entry is validated by value: + // the dwords its capture read outside the registered code and header (the key names that + // registration, so those bytes are never compared) are compared in place with guest memory + // under the coherence gate of validateEntry, and nothing is stamped or collected on a hit. + // Debug aid: APS5_DISPATCH_STAMP_VALIDATE=1 validates by write-watch stamps over merged spans + // and EqualsCommitted per captured region as before (the entry then keeps the capture's pages). + // A key holds up to DispatchVariants() value sets (a variant each, most recently hit first): + // a program whose SRT alternates between a few value sets hits the variant that matches + // instead of recapturing. A variant is immutable once inserted (a re-stamp is an atomic + // store); an entry is replaced whole under dispatchCacheMutex, so a validator holding the old + // one validates a consistent object. + struct DispatchVariant { + // The captured dwords outside the code and header: [begin, end) byte ranges in address + // order, and their values in that order. + std::vector> runs; + std::vector words; + std::uint64_t forgetSerial = 0; std::shared_ptr compiled; - std::vector> spans; - std::uint64_t generation = 0; // The registered shader whose code and header the capture references. std::shared_ptr shader; + // The dispatch recipe (design_cpu_final M4), null until the first successful device call + // of a reusable object for this variant; stored atomically, read by every later hit + // outside dispatchCacheMutex. + std::atomic> recipe; + // Data-only hits: the positions in `words` of the pure flat-SRT leaves the capture read + // (sorted; DataWordPositions), the FlattenedSrt binding offset each one feeds, and that + // binding's index in `compiled->bindings`. Empty: the compare is exact, as for draws. + static constexpr std::uint32_t NoFlatBinding = std::numeric_limits::max(); + std::vector dataPositions; + std::vector dataSlots; + std::uint32_t flatBinding = NoFlatBinding; + // Draw entries only: the push constant offset the stage was compiled at (the earlier + // stages' push sizes), which a hit must reproduce, and the vertex stage info the stage + // was compiled with (null for a fragment stage): DecodeVertexStageInfo's guest reads are + // among the runs, so a hit reproduces the info without the decode (design_cpu_final 8a). + std::uint32_t pushOffset = 0; + std::shared_ptr vertexInfo; + // Stamp validation only: the capture's pages, every captured region (code and header + // included), the watched spans and the generation they were last found unchanged at. + std::shared_ptr memory; + std::vector captured; + std::vector> spans; + std::atomic generation{0}; + }; + struct DispatchEntry { + std::vector> variants; + // The hit count when this entry last moved to the front of dispatchOrder (dispatchCacheMutex). + std::uint64_t touched = 0; // This entry's position in dispatchOrder. std::list::iterator order; }; - static constexpr std::size_t DispatchCacheEntries = 4096; - std::unordered_map dispatchCache; + static bool StampValidate() { + static const bool stamp = std::getenv("APS5_DISPATCH_STAMP_VALIDATE") != nullptr; + return stamp; + } + // Data-only hits: a variant whose words differ from guest memory only at pure flat-SRT + // leaves (IrResourcePlan::pureFlatSlots: words the CPU walk never consumes) is a hit whose + // compiled result takes the live words and whose template refreshes its data buffers, so + // per-frame constants no longer cost a capture, a recompile and an insert. Debug aids: + // APS5_NO_DATA_HITS=1 keeps every compare exact; APS5_VERIFY_DATA_HITS=1 captures beside + // every data hit and aborts when the read set, its bytes, the variant or the bindings differ. + // Off with APS5_NO_TEMPLATE_DATA_REFRESH=1 too: a hit's live words reach the GPU only through + // the template refresh (a recipe's template is never re-keyed). + static bool DataHits() { + static const bool hits = std::getenv("APS5_NO_DATA_HITS") == nullptr && VulkanDevice::TemplateDataRefresh(); + return hits; + } + static bool VerifyDataHits() { + static const bool verify = std::getenv("APS5_VERIFY_DATA_HITS") != nullptr; + return verify; + } + static constexpr std::size_t MaxDispatchVariants = 8; + // APS5_DISPATCH_VARIANTS=: value sets kept per key (default 4, at most + // MaxDispatchVariants; 1 = one per key as before). Stamp validation keeps one. + static std::size_t DispatchVariants() { + static const std::size_t variants = [] { + if (StampValidate()) return std::size_t{1}; + const char* text = std::getenv("APS5_DISPATCH_VARIANTS"); + const auto parsed = text != nullptr ? std::strtoull(text, nullptr, 10) : 4ull; + return static_cast(std::clamp(parsed, 1, MaxDispatchVariants)); + }(); + return variants; + } + // APS5_INSERT_COMPARE=1: a fresh entry's captured words are compared with guest memory before + // the insert (captureStable) as before. By default a capture is kept as read: a CPU write + // racing it is used once, as hardware would, and the value compare catches it at the next use. + static bool InsertCompare() { + static const bool compare = std::getenv("APS5_INSERT_COMPARE") != nullptr; + return compare; + } + // 16384 entries by value (about 1 KB per variant; 8192 with more than two variants per key), + // 4096 by stamps (two 4 KiB pages each); APS5_DISPATCH_CACHE_ENTRIES= overrides either. + static std::size_t DispatchCacheEntries() { + static const std::size_t entries = [] { + const char* text = std::getenv("APS5_DISPATCH_CACHE_ENTRIES"); + const auto parsed = text != nullptr ? std::strtoull(text, nullptr, 10) : 0ull; + if (parsed != 0) return static_cast(parsed); + if (StampValidate()) return std::size_t{4096}; + return DispatchVariants() > 2 ? std::size_t{8192} : std::size_t{16384}; + }(); + return entries; + } + std::unordered_map> dispatchCache; // Keys most recently used first. std::list dispatchOrder; std::mutex dispatchCacheMutex; std::uint64_t dispatchCacheHits = 0; std::uint64_t dispatchCacheEvictions = 0; + // Variants held over every entry and their estimated bytes (under dispatchCacheMutex). + std::uint64_t dispatchCacheVariants = 0; + std::uint64_t dispatchCacheVariantBytes = 0; + static std::uint64_t variantBytes(const DispatchVariant& variant) { + return sizeof(DispatchVariant) + variant.words.size() * sizeof(std::uint32_t) + variant.runs.size() * sizeof(std::pair) + variant.captured.size() * sizeof(ShaderRecompiler::MemoryRegion) + (variant.vertexInfo != nullptr ? sizeof(ShaderRecompiler::ShaderVertexStageInfo) : 0); + } + void accountVariant(const DispatchVariant& variant, bool added) { + if (added) { + ++dispatchCacheVariants; + dispatchCacheVariantBytes += variantBytes(variant); + } else { + --dispatchCacheVariants; + dispatchCacheVariantBytes -= variantBytes(variant); + } + } + void eraseDispatchEntry(std::unordered_map>::iterator it) { + for (const auto& variant : it->second->variants) accountVariant(*variant, false); + dispatchOrder.erase(it->second->order); + dispatchCache.erase(it); + } + // Lookup outcomes for the [dispatch-cache] line (APS5_PROFILE_DRAW, every 10 s); under + // dispatchCacheMutex. + struct EntryCounters { + std::uint64_t lookups = 0, absent = 0, equal = 0, differing = 0, inaccessible = 0, queuedLabel = 0, flushingImage = 0, publishMoved = 0, pendingMoved = 0, forgetMoved = 0, imagesFlushed = 0, runsSynced = 0, forgetSinceInsert = 0, replaced = 0, inserts = 0, unstable = 0, touches = 0; + // Runs (compared spans) over the validated entries and over the inserted ones, and the + // gate retries after a moved serial by their outcome. + std::uint64_t runsValidated = 0, runsInserted = 0, retriesEqual = 0, retriesMoved = 0; + double validateUs = 0; + // The 'differing' misses whose fresh capture was compared with the entry it replaced + // (classifyDiffering): by class of the differing dwords (every one a V#/T# base word of the + // fresh capture: address; every one a flattened-SRT word: data; none: walk; else mixed; + // the walk read other addresses: runs changed), those whose fresh words equal one of the + // key's last three replaced value sets, the differing dwords and their positions. + std::uint64_t differingClassified = 0, differingAddress = 0, differingData = 0, differingWalk = 0, differingMixed = 0, differingRunsChanged = 0, differingMatchedPrior = 0, differingWords = 0; + std::array differingWordBuckets{}; + // Variants: hits by the rank of the matching variant (front first), variants compared over + // the validations, variants inserted into an existing entry and evicted beyond the limit. + std::array variantHitsByRank{}; + std::uint64_t variantsCompared = 0, variantsInserted = 0, variantsEvicted = 0; + // Data-only hits: hits, words refreshed, by rank, verified (APS5_VERIFY_DATA_HITS); + // inserts with data positions, positions inserted, leaves skipped by reason. + std::uint64_t dataHits = 0, dataWordsRefreshed = 0, dataVerified = 0, dataInserts = 0, dataPositionsInserted = 0, dataLeavesUnmapped = 0, dataLeavesMismatched = 0, dataLeavesAliased = 0; + std::array dataHitsByRank{}; + std::set differingPositions; + std::size_t differingFirstPosition = std::numeric_limits::max(), differingLastPosition = 0; + std::map differingByProgram; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + }; + EntryCounters entryCounters; + // Misses of variants with data positions on a pending, known-value or stamp-compared run (the + // exact compare there): the data hits the pending-write policy left on the table. + std::atomic dataPendingMisses{0}; + // Per key, the runs and words of the last three entries a 'differing' miss replaced + // (APS5_PROFILE_DRAW only; under dispatchCacheMutex): whether a variant ring would have held + // the fresh values. + using ValueSet = std::pair>, std::vector>; + std::unordered_map> priorValueSets; + // Draw cache (design_cpu_final M7): per draw key one variant list per compiled program, each + // variant a DispatchVariant validated by value over its own runs exactly as a dispatch's is, + // so a hit needs every stage equal and skips both captures and both recompiles (rule RD: R0-R3 + // per stage, the queued-label test and the writer notes over the union). The key (8a, the + // register key) is built before any decode from the register banks and the registry alone: + // every present register of Graphics::DrawKeyRegisters with its offset, the registration + // identities and code offsets the program address registers name, and the device serial. The + // decoded state, pixel stage info, programs and roles are pure functions of it and are kept + // with the entry (DrawDecode), so a hit skips DecodeState, DecodeShaderStages, + // DecodePixelStageInfo and the program prepare; the vertex stage infos read attribute and V# + // tables from guest memory, so each stage variant keeps the info it was compiled with and its + // reads among its value-validated runs. Under APS5_NO_DRAW_KEY=1 the key is M7's (the decoded + // bytes hashed after the decode, which then runs on hits too) and no decode is kept. Guarded by + // drawCacheMutex; entries are immutable and replaced whole, least recently used first out. The + // CPU path of an indirect draw (patched user words) bypasses it. Debug aids: + // APS5_NO_DRAW_SRT_ENTRIES=1 captures every draw as before; APS5_VERIFY_DRAW_ENTRIES=1 + // captures on every hit as well and counts a stage whose fresh capture differs from the + // matched variant (the draw then uses the fresh results); APS5_VERIFY_DRAW_RECIPE=1 decodes + // on every register-key hit and compares the state, pixel and vertex infos, programs and + // roles with the entry's (the draw then uses the fresh decode), and on every miss checks the + // decoders' register facade log against the table; APS5_DRAW_CACHE_ENTRIES= is the entry + // budget (default 4096). Stamp validation (APS5_DISPATCH_STAMP_VALIDATE) has no draw form and + // disables the entries. + struct DrawProgram { + ShaderRecompiler::ShaderBinary binary; + std::uint32_t userDataBase; + std::uint32_t firstUserSgpr = 8; + std::vector userData; + std::array memory; + // The registered shader the binary is a program of, and the program's offset in it. + std::shared_ptr snapshot; + std::size_t codeOffset = 0; + }; + // The products of a draw's decode that the register key fixes. + struct DrawDecode { + Graphics::State state; + ShaderRecompiler::ShaderPixelStageInfo pixel; + std::vector programs; + std::vector roles; + }; + // A draw recipe (8b) with the stage variants (by identity, per program index) whose compiled + // results it was recorded with: valid for a hit that matched exactly those. The identity is + // the variant's control block (weak: a variant evicted beyond k dies while its record may + // still be listed, and a fresh variant may reuse its address; a weak reference keeps the + // block, so an evicted variant never matches again). + struct DrawRecipeRecord { + std::vector> stages; + std::shared_ptr recipe; + bool Matches(const std::vector>& variants) const { + if (stages.size() != variants.size()) return false; + for (std::size_t i = 0; i < stages.size(); ++i) { + if (stages[i].owner_before(variants[i]) || variants[i].owner_before(stages[i])) return false; + } + return true; + } + bool Expired() const { + return std::any_of(stages.begin(), stages.end(), [](const std::weak_ptr& stage) { return stage.expired(); }); + } + }; + struct DrawEntry { + // The decode the register key fixes (null under the decoded-bytes key). + std::shared_ptr decode; + // Per program index of the draw (a GeometryBack slot stays empty), most recently hit first. + std::vector>> stages; + // The recipes attached to this key's matched variant sets, newest first (an immutable + // list replaced whole; carried over when the entry is replaced). + std::atomic>> recipes; + std::uint64_t touched = 0; + std::list::iterator order; + }; + static bool DrawEntries() { + static const bool entries = std::getenv("APS5_NO_DRAW_SRT_ENTRIES") == nullptr && !StampValidate(); + return entries; + } + static bool VerifyDrawEntries() { + static const bool verify = std::getenv("APS5_VERIFY_DRAW_ENTRIES") != nullptr; + return verify; + } + static bool RegisterKey() { + static const bool registerKey = std::getenv("APS5_NO_DRAW_KEY") == nullptr; + return registerKey; + } + static bool VerifyDrawRecipe() { + static const bool verify = std::getenv("APS5_VERIFY_DRAW_RECIPE") != nullptr; + return verify; + } + static std::size_t DrawCacheEntries() { + static const std::size_t entries = [] { + const char* text = std::getenv("APS5_DRAW_CACHE_ENTRIES"); + const auto parsed = text != nullptr ? std::strtoull(text, nullptr, 10) : 0ull; + return parsed != 0 ? static_cast(parsed) : std::size_t{4096}; + }(); + return entries; + } + std::unordered_map> drawCache; + std::list drawOrder; + std::mutex drawCacheMutex; + std::uint64_t drawCacheHits = 0, drawCacheEvictions = 0, drawCacheVariants = 0, drawCacheVariantBytes = 0; + // Why a validated draw entry missed, by the first stage in program order without a match: + // its front variant differing (the front program, the fragment program, another stage), no + // variant compiled at the current push offset (layout), a gate outcome (page not mapped, + // queued label, image being stored, a serial moved), a stage count unlike the key's. + enum class DrawMiss : std::size_t { FrontDiffering, FragmentDiffering, OtherDiffering, Layout, Gate, Stages, Count }; + struct DrawEntryCounters { + std::uint64_t lookups = 0, absent = 0, hits = 0, stageValidations = 0, stageEqual = 0, variantsCompared = 0, inserts = 0, variantsInserted = 0, variantsEvicted = 0, present = 0, unstable = 0, touches = 0, verifyHits = 0, verifyMismatches = 0; + std::array(DrawMiss::Count)> misses{}; + std::array variantHitsByRank{}; + double validateUs = 0; + // The register key (8a): lookups made with it, hits (every stage equal), lookups whose + // entry carried the decode (skipped: the whole decode; partial: the state, pixel and + // programs of a miss), the key builds' time, and the verify counts: decoders' reads the + // table lacks ('facade log != table', must be 0) and decodes that differed from the + // entry's (must be 0). + std::uint64_t registerKeyLookups = 0, registerKeyHits = 0, decodeSkipped = 0, decodePartial = 0, facadeMismatches = 0, verifyDecodes = 0, verifyDecodeMismatches = 0; + double keyUs = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + }; + DrawEntryCounters drawEntryCounters; + void reportDrawCache(DrawEntryCounters& counters) { + const auto count = [](std::uint64_t value) { return static_cast(value); }; + const auto validated = counters.lookups - counters.absent; + const auto miss = [&](DrawMiss reason) { return count(counters.misses[static_cast(reason)]); }; + std::string ranks; + for (std::size_t rank = 0; rank < DispatchVariants(); ++rank) { + char text[32]; + std::snprintf(text, sizeof(text), "%s%llu", rank == 0 ? "" : " / ", count(counters.variantHitsByRank[rank])); + ranks += text; + } + std::fprintf(stderr, "[draw-cache] %llu lookups (10 s): %llu no entry, %llu validated in %.1f us each (key, lookup and every stage): %llu hits (every stage equal; stage hits by variant rank 1..k %s), misses by reason: front stage differing %llu, fragment differing %llu, other stage differing %llu, layout (push offset) %llu, gate %llu, stage count %llu; %llu stage validations (%llu equal, %.2f variants compared each); %llu inserts (%llu variants inserted, %llu evicted beyond k, %llu already present, %llu unstable), %zu entries (%llu variants, ~%.1f MiB), %llu evictions in total, %llu LRU moves; verify: %llu hits captured again, %llu stages differed\n", count(counters.lookups), count(counters.absent), count(validated), validated != 0 ? counters.validateUs / static_cast(validated) : 0.0, count(counters.hits), ranks.c_str(), miss(DrawMiss::FrontDiffering), miss(DrawMiss::FragmentDiffering), miss(DrawMiss::OtherDiffering), miss(DrawMiss::Layout), miss(DrawMiss::Gate), miss(DrawMiss::Stages), count(counters.stageValidations), count(counters.stageEqual), counters.stageValidations != 0 ? static_cast(counters.variantsCompared) / static_cast(counters.stageValidations) : 0.0, count(counters.inserts), count(counters.variantsInserted), count(counters.variantsEvicted), count(counters.present), count(counters.unstable), drawCache.size(), count(drawCacheVariants), static_cast(drawCacheVariantBytes) / (1024.0 * 1024.0), count(drawCacheEvictions), count(counters.touches), count(counters.verifyHits), count(counters.verifyMismatches)); + std::fprintf(stderr, "[draw-cache] register key (10 s): %llu lookups, %llu hits, key build %.1f us each; decode skipped %llu, partial (state/pixel/programs from the entry) %llu; facade log != table %llu; verify: %llu decodes compared, %llu differed\n", count(counters.registerKeyLookups), count(counters.registerKeyHits), counters.registerKeyLookups != 0 ? counters.keyUs / static_cast(counters.registerKeyLookups) : 0.0, count(counters.decodeSkipped), count(counters.decodePartial), count(counters.facadeMismatches), count(counters.verifyDecodes), count(counters.verifyDecodeMismatches)); + counters = DrawEntryCounters{}; + } + void accountDrawVariant(const DispatchVariant& variant, bool added) { + if (added) { + ++drawCacheVariants; + drawCacheVariantBytes += variantBytes(variant); + } else { + --drawCacheVariants; + drawCacheVariantBytes -= variantBytes(variant); + } + } + // A miss's fresh variants (per program index, null where nothing was compiled) go in front of + // the key's stage lists, each list trimmed to DispatchVariants(); a variant equal to one the + // list holds (another worker's insert, a stage that had matched) is not inserted again, and + // `fresh[i]` then becomes the kept one (the identity a recipe is attached to). `decode` is + // kept with the entry under the register key (an entry that has one keeps its own: equal by + // construction). + void insertDrawEntry(std::uint64_t key, std::vector>& fresh, std::shared_ptr decode) { + std::lock_guard cacheLock(drawCacheMutex); + auto& counters = drawEntryCounters; + ++counters.inserts; + const auto found = drawCache.find(key); + auto replacement = std::make_shared(); + replacement->stages.resize(fresh.size()); + replacement->decode = std::move(decode); + if (found != drawCache.end()) { + if (found->second->stages.size() == fresh.size()) replacement->stages = found->second->stages; + if (found->second->decode != nullptr) replacement->decode = found->second->decode; + replacement->recipes.store(found->second->recipes.load()); + } + for (std::size_t i = 0; i < fresh.size(); ++i) { + if (fresh[i] == nullptr) continue; + auto& variants = replacement->stages[i]; + const auto present = std::find_if(variants.begin(), variants.end(), [&](const std::shared_ptr& kept) { return kept->pushOffset == fresh[i]->pushOffset && kept->runs == fresh[i]->runs && kept->words == fresh[i]->words; }); + if (present != variants.end()) { + ++counters.present; + fresh[i] = *present; + continue; + } + accountDrawVariant(*fresh[i], true); + variants.insert(variants.begin(), fresh[i]); + ++counters.variantsInserted; + while (variants.size() > DispatchVariants()) { + accountDrawVariant(*variants.back(), false); + variants.pop_back(); + ++counters.variantsEvicted; + } + } + replacement->touched = drawCacheHits; + if (found == drawCache.end()) { + drawOrder.push_front(key); + replacement->order = drawOrder.begin(); + drawCache.emplace(key, std::move(replacement)); + } else { + replacement->order = found->second->order; + drawOrder.splice(drawOrder.begin(), drawOrder, replacement->order); + found->second = std::move(replacement); + } + while (drawCache.size() > DrawCacheEntries()) { + const auto last = drawCache.find(drawOrder.back()); + for (const auto& variants : last->second->stages) { + for (const auto& variant : variants) accountDrawVariant(*variant, false); + } + drawOrder.erase(last->second->order); + drawCache.erase(last); + ++drawCacheEvictions; + } + } + // The recipe a draw hit that matched exactly `stages` (by identity) can record from, if one + // was attached to the key's current entry; null otherwise. + std::shared_ptr findDrawRecipe(std::uint64_t key, const std::vector>& stages) { + std::shared_ptr> records; + { + std::lock_guard cacheLock(drawCacheMutex); + const auto found = drawCache.find(key); + if (found == drawCache.end()) return nullptr; + records = found->second->recipes.load(); + } + if (records == nullptr) return nullptr; + for (const auto& record : *records) { + if (record.Matches(stages)) return record.recipe; + } + return nullptr; + } + // Attaches (or replaces) the recipe of the matched variant set on the key's current entry: a + // fresh immutable list, newest first, at most DispatchVariants() records (an entry's stage + // lists hold that many variants each), records of an evicted variant dropped. A racing attach + // may lose one record: a recipe is a cache, rebuilt at the next miss. + void attachDrawRecipe(std::uint64_t key, const std::vector>& stages, std::shared_ptr recipe) { + std::shared_ptr entry; + { + std::lock_guard cacheLock(drawCacheMutex); + const auto found = drawCache.find(key); + if (found == drawCache.end()) return; + entry = found->second; + } + const auto old = entry->recipes.load(); + auto records = std::make_shared>(); + records->push_back({std::vector>(stages.begin(), stages.end()), std::move(recipe)}); + if (old != nullptr) { + for (const auto& record : *old) { + if (record.Matches(stages) || record.Expired() || records->size() >= DispatchVariants()) continue; + records->push_back(record); + } + } + entry->recipes.store(std::move(records)); + VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Attach, VulkanDevice::RecipeKind::Draw); + } + // The register key of a draw (design_cpu_final 8a, design_cpu_0 3.5 (a), (b), (e)): FNV over + // every present register of the DrawKeyRegisters ranges with its offset (an absent register + // is simply not mixed: the sequence differs), the registered shader each program address + // register present names (its identity and the program's code offset: a re-registration at + // the same address is another object) and the device serial. No guest memory is read. + static std::uint64_t drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial) { + std::uint64_t key = 0xcbf29ce484222325ull; + const auto mix = [&](std::uint64_t value) { + key ^= value; + key *= 0x100000001b3ull; + }; + mix(deviceSerial); + for (const auto& range : Graphics::DrawKeyRegisters) { + const auto& bank = range.bank == Graphics::RegisterBank::Context ? queue.context : range.bank == Graphics::RegisterBank::Shader ? queue.shader : queue.userConfig; + mix((static_cast(range.bank) << 32u) | range.first); + const auto end = range.first + range.count; + for (auto it = bank.lower_bound(range.first); it != bank.end() && it->first < end; ++it) { + mix(it->first); + mix(it->second); + } + } + for (const auto base : {0x008u, 0x088u, 0x0c8u, 0x108u, 0x148u}) { + const auto low = queue.shader.find(base); + const auto high = queue.shader.find(base + 1); + if (low == queue.shader.end() || high == queue.shader.end()) { + mix(0); + continue; + } + const auto address = (static_cast(low->second) << 8u) | (static_cast(high->second & 0xffu) << 40u); + auto it = registry.upper_bound(address); + if (it == registry.begin()) { + mix(1); + continue; + } + --it; + mix(reinterpret_cast(it->second.get())); + mix(address - it->second->codeAddress); + } + return key; + } + // APS5_VERIFY_DRAW_RECIPE: whether two decodes (the entry's and a fresh one) agree, field by + // field (the structs carry padding, so no memcmp). + static bool sameVertexInfo(const ShaderRecompiler::ShaderVertexStageInfo& a, const ShaderRecompiler::ShaderVertexStageInfo& b) { + if (a.resourcesNum != b.resourcesNum || a.fetchAttribReg != b.fetchAttribReg || a.fetchBufferReg != b.fetchBufferReg || a.fetchEmbedded != b.fetchEmbedded) return false; + for (std::uint32_t i = 0; i < a.resourcesNum && i < a.resources.size(); ++i) { + if (a.resources[i].fields != b.resources[i].fields) return false; + const auto& x = a.resourcesDst[i]; + const auto& y = b.resourcesDst[i]; + if (x.registerStart != y.registerStart || x.registersNum != y.registersNum || x.attrId != y.attrId || x.fetchIndex != y.fetchIndex) return false; + } + return true; + } + static bool sameDecode(const DrawDecode& a, const DrawDecode& b) { + const auto& s = a.state; + const auto& t = b.state; + const auto sameColor = [](const Graphics::ColorTarget& x, const Graphics::ColorTarget& y) { + return x.address == y.address && x.extent.width == y.extent.width && x.extent.height == y.extent.height && x.format == y.format && x.bytes == y.bytes && x.componentMapping == y.componentMapping && x.tileMode == y.tileMode && x.elementBytes == y.elementBytes && x.dccAddress == y.dccAddress && x.dccAlphaOnMsb == y.dccAlphaOnMsb; + }; + const auto sameBlend = [](const VkPipelineColorBlendAttachmentState& x, const VkPipelineColorBlendAttachmentState& y) { + return x.blendEnable == y.blendEnable && x.srcColorBlendFactor == y.srcColorBlendFactor && x.dstColorBlendFactor == y.dstColorBlendFactor && x.colorBlendOp == y.colorBlendOp && x.srcAlphaBlendFactor == y.srcAlphaBlendFactor && x.dstAlphaBlendFactor == y.dstAlphaBlendFactor && x.alphaBlendOp == y.alphaBlendOp && x.colorWriteMask == y.colorWriteMask; + }; + const auto sameMesh = [](const std::optional& x, const std::optional& y) { + if (x.has_value() != y.has_value()) return false; + if (!x) return true; + return x->inputPrimitive == y->inputPrimitive && x->primitivesPerGroup == y->primitivesPerGroup && x->verticesPerGroup == y->verticesPerGroup && x->maxVertices == y->maxVertices && x->maxPrimitives == y->maxPrimitives && x->threadsPerGroup == y->threadsPerGroup && x->ldsSizeDwords == y->ldsSizeDwords && x->provokingVertex == y->provokingVertex; + }; + const auto sameTess = [](const std::optional& x, const std::optional& y) { + if (x.has_value() != y.has_value()) return false; + if (!x) return true; + return x->inputControlPoints == y->inputControlPoints && x->outputControlPoints == y->outputControlPoints && x->domain == y->domain && x->partitioning == y->partitioning && x->outputTopology == y->outputTopology; + }; + if (s.stages.path != t.stages.path || s.stages.registerValue != t.stages.registerValue || s.stages.vertexWaveSize != t.stages.vertexWaveSize || s.stages.fragmentWaveSize != t.stages.fragmentWaveSize || !sameMesh(s.stages.mesh, t.stages.mesh) || !sameTess(s.stages.tessellation, t.stages.tessellation)) return false; + if (!sameColor(s.color, t.color) || s.colors.size() != t.colors.size() || s.blends.size() != t.blends.size()) return false; + for (std::size_t i = 0; i < s.colors.size(); ++i) { + if (!sameColor(s.colors[i], t.colors[i])) return false; + } + for (std::size_t i = 0; i < s.blends.size(); ++i) { + if (!sameBlend(s.blends[i], t.blends[i])) return false; + } + if (s.hasColorTarget != t.hasColorTarget || s.rectList != t.rectList || s.renderExtent.width != t.renderExtent.width || s.renderExtent.height != t.renderExtent.height || s.topology != t.topology || s.negativeOneToOne != t.negativeOneToOne || s.depthClamp != t.depthClamp || s.cullMode != t.cullMode || s.frontFace != t.frontFace || !sameBlend(s.blend, t.blend) || s.blendConstants != t.blendConstants) return false; + if (std::memcmp(&s.viewport, &t.viewport, sizeof(VkViewport)) != 0 || std::memcmp(&s.scissor, &t.scissor, sizeof(VkRect2D)) != 0) return false; + const auto& p = a.pixel; + const auto& q = b.pixel; + if (p.interpolatorCount != q.interpolatorCount || p.interpolatorSettings != q.interpolatorSettings || p.wave32 != q.wave32 || p.perspectiveCenterVgpr != q.perspectiveCenterVgpr || p.hasPerspectiveCenterVgpr != q.hasPerspectiveCenterVgpr || p.posX != q.posX || p.posY != q.posY || p.posZ != q.posZ || p.posW != q.posW || p.frontFace != q.frontFace || p.ancillary != q.ancillary || p.sampleShading != q.sampleShading || p.noPerspective != q.noPerspective || p.pixelKillEnable != q.pixelKillEnable || p.depthExportEnable != q.depthExportEnable || p.sampleMaskExportEnable != q.sampleMaskExportEnable || p.earlyZ != q.earlyZ || p.executeOnNoop != q.executeOnNoop || p.targetOutputMode != q.targetOutputMode || p.targetExportMapping != q.targetExportMapping) return false; + if (a.roles != b.roles || a.programs.size() != b.programs.size()) return false; + for (std::size_t i = 0; i < a.programs.size(); ++i) { + const auto& x = a.programs[i]; + const auto& y = b.programs[i]; + if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset) return false; + } + return true; + } + // The driver's own phases of a dispatch, around VulkanDevice::dispatch's phase rows, for the + // "[dispatch] driver phases" line (APS5_PROFILE_DRAW, every 10 s): what the [gpu] capture and + // device totals are made of. Summed under driverPhasesMutex once per dispatch. + // "validate" is the validation's CPU work; the fence and timeline waits its syncs made (the + // flush hook inside syncPendingRuns and the hook compares) are moved to "validate GPU wait", + // so the two compare paths (APS5_VALIDATE_LEGACY) can be compared by CPU cost. + // The packet bracket: 'prologue' is the packet's start (the worker's timer) to the phase + // timers' start (register reads, DecodeComputeStageInfo, the HLE matching, the device load), + // 'epilogue' the tail to the packet's end (in the packet loop), so the rows sum to the [packets] + // DISPATCH_DIRECT 'real' row; 'prepare:' splits PrepareDispatch (VulkanDevice::PreparePhaseMs; + // 'other' is what the call spent outside its parts); 'recipe pre-check' is reserved. + enum DriverPhase { PhasePrologue, PhaseKey, PhaseLookup, PhaseValidate, PhaseValidateWait, PhaseRelock, PhaseCapture, PhaseRecompile, PhaseInsert, PhaseQueuedLabels, PhaseSnapshots, PhasePrepareKey, PhasePrepareFind, PhasePreparePrecollect, PhasePreparePresync, PhasePrepareStageA, PhasePrepareOther, PhaseRecipePrecheck, PhaseLockWait, PhaseLabels, PhaseNoteWriters, PhaseDevice, PhaseTail, PhaseEpilogue, DriverPhaseCount }; + static constexpr const char* DriverPhaseNames[DriverPhaseCount] = {"prologue", "key", "lookup", "validate", "validate GPU wait", "relock", "capture", "recompile", "insert", "queued-label check", "snapshots", "prepare: key", "prepare: find", "prepare: precollect", "prepare: presync", "prepare: stage A (miss)", "prepare: other", "recipe pre-check", "lock wait", "labels", "note writers", "device call", "tail", "epilogue"}; + // The lines are split by queue and packet: queue 0's direct dispatches (the [packets] + // reconciliation), queue 0's indirect ones, and every other queue's. + enum DispatchClass : std::size_t { Queue0Direct, Queue0Indirect, OtherQueues, DispatchClassCount }; + static constexpr const char* DispatchClassNames[DispatchClassCount] = {"queue 0 direct", "queue 0 indirect", "other queues"}; + struct DriverPhaseTotals { + std::array ms{}; + std::uint64_t dispatches = 0, hits = 0, validations = 0; + }; + std::mutex driverPhasesMutex; + std::array driverPhaseTotals{}; + std::chrono::steady_clock::time_point driverPhasesReport = std::chrono::steady_clock::now(); + // `validated`: the lookup found an entry and validated it (hit or not), what the validate + // phase is spent on. + void addDriverPhases(DispatchClass which, const std::array& ms, bool hit, bool validated) { + std::lock_guard lock(driverPhasesMutex); + auto& totals = driverPhaseTotals[which]; + for (std::size_t i = 0; i < DriverPhaseCount; ++i) totals.ms[i] += ms[i]; + ++totals.dispatches; + if (hit) ++totals.hits; + if (validated) ++totals.validations; + const auto now = std::chrono::steady_clock::now(); + if (now - driverPhasesReport < std::chrono::seconds(10)) return; + driverPhasesReport = now; + for (std::size_t cls = 0; cls < DispatchClassCount; ++cls) { + auto& line = driverPhaseTotals[cls]; + if (line.dispatches == 0) continue; + std::string report; + double total = 0; + for (std::size_t i = 0; i < DriverPhaseCount; ++i) { + char text[64]; + std::snprintf(text, sizeof(text), " %s %.1f", DriverPhaseNames[i], line.ms[i] * 1000 / static_cast(line.dispatches)); + report += text; + total += line.ms[i]; + } + const double perValidation = line.validations != 0 ? 1000 / static_cast(line.validations) : 0.0; + std::fprintf(stderr, "[dispatch] driver phases %s (10 s, %llu dispatches, %llu cache hits), us per dispatch:%s, total %.1f (%.1f ms); validate %.1f us per validation (%llu validations; GPU waits inside it %.1f us per validation apart)\n", DispatchClassNames[cls], static_cast(line.dispatches), static_cast(line.hits), report.c_str(), total * 1000 / static_cast(line.dispatches), total, line.ms[PhaseValidate] * perValidation, static_cast(line.validations), line.ms[PhaseValidateWait] * perValidation); + line = {}; + } + } + // What a dispatch packet became, for the [packets] DISPATCH_DIRECT outcome rows and the phase + // bracket: the dispatch fills the calling worker's slot (the phases at its tail, or the HLE + // outcome when it returned early; a tolerated failure leaves it 'skipped'), and the packet + // loop reads it after the call, adds the epilogue and files the phases by class. + enum class DispatchOutcome : std::size_t { Skipped = 0, Real, CopyHle, FillHle, SkippedMemo, Count }; + static constexpr const char* DispatchOutcomeNames[static_cast(DispatchOutcome::Count)] = {"skipped", "real", "copy HLE", "fill HLE", "skipped (memo)"}; + struct PendingDispatchPhases { + DispatchOutcome outcome = DispatchOutcome::Skipped; + bool phases = false; + bool hit = false; + bool validated = false; + std::array ms{}; + std::chrono::steady_clock::time_point tailAt{}; + }; + static PendingDispatchPhases& pendingDispatchPhases() { + static thread_local PendingDispatchPhases pending; + return pending; + } + // The start of the packet the calling worker executes (its [packets] timer), for the prologue rows. + static std::chrono::steady_clock::time_point& packetStartedAt() { + static thread_local std::chrono::steady_clock::time_point started{}; + return started; + } + // The [draw] driver phases line: Driver::draw's own work around Graphics::Draw (the [draws] + // line), bracketed like a dispatch's: prologue (packet start to the decode), precheck, decode + // (State, pixel and vertex infos), program prepare (the programs' registers and user words), + // capture and the hook waits inside it (kept apart, as the validate row does), recompile, + // rect-list, vectors (snapshots, linked programs, the queued-label check, the writer notes), + // key/lookup/validate (the draw cache's key, lookup, per-stage validations and a miss's + // insert) with the hook waits inside the validations as 'validate GPU wait', lock wait, + // labels, Graphics::Draw, the epilogue; a packet that drew nothing (Nothing, Rejected, a + // tolerated throw) is one 'skipped' row. The rows sum to the [packets] DRAW_INDEX_AUTO time; + // less Graphics::Draw they are the prologue the [draws] line does not see. + enum DrawDriverPhase { DrawRowPrologue, DrawRowPrecheck, DrawRowDecode, DrawRowProgramPrepare, DrawRowCapture, DrawRowCaptureHookWaits, DrawRowRecompile, DrawRowRectList, DrawRowVectors, DrawRowKeyLookupValidate, DrawRowValidateWait, DrawRowLockWait, DrawRowLabels, DrawRowGraphics, DrawRowSkipped, DrawRowEpilogue, DrawDriverPhaseCount }; + static constexpr const char* DrawDriverPhaseNames[DrawDriverPhaseCount] = {"prologue", "precheck", "decode", "program prepare", "capture", "capture hook waits", "recompile", "rect-list", "vectors", "key/lookup/validate", "validate GPU wait", "lock wait", "labels", "Graphics::Draw", "skipped", "epilogue"}; + struct DrawPhaseTotals { + std::array ms{}; + std::uint64_t packets = 0, drawn = 0, captures = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + }; + std::mutex drawPhasesMutex; + DrawPhaseTotals drawPhaseTotals; + struct PendingDrawPhases { + bool phases = false; + std::uint64_t captures = 0; + std::array ms{}; + std::chrono::steady_clock::time_point tailAt{}; + }; + static PendingDrawPhases& pendingDrawPhases() { + static thread_local PendingDrawPhases pending; + return pending; + } + void addDrawPhases(const std::array& ms, bool drawn, std::uint64_t captures) { + std::lock_guard lock(drawPhasesMutex); + auto& totals = drawPhaseTotals; + for (std::size_t i = 0; i < DrawDriverPhaseCount; ++i) totals.ms[i] += ms[i]; + ++totals.packets; + if (drawn) ++totals.drawn; + totals.captures += captures; + const auto now = std::chrono::steady_clock::now(); + if (now - totals.lastReport < std::chrono::seconds(10)) return; + totals.lastReport = now; + std::string report; + double total = 0; + for (std::size_t i = 0; i < DrawDriverPhaseCount; ++i) { + char text[64]; + std::snprintf(text, sizeof(text), " %s %.1f", DrawDriverPhaseNames[i], totals.ms[i] * 1000 / static_cast(totals.packets)); + report += text; + total += totals.ms[i]; + } + std::fprintf(stderr, "[draw] driver phases (10 s, %llu draw packets, %llu drawn, %llu captures), us per packet:%s, total %.1f (%.1f ms; Graphics::Draw %.1f ms, skipped %.1f ms)\n", static_cast(totals.packets), static_cast(totals.drawn), static_cast(totals.captures), report.c_str(), total * 1000 / static_cast(totals.packets), total, totals.ms[DrawRowGraphics], totals.ms[DrawRowSkipped]); + totals.ms = {}; + totals.packets = totals.drawn = totals.captures = 0; + } std::map queues; std::map> outputs; DevicePointer device; @@ -574,6 +1445,8 @@ private: std::atomic failed{false}; bool stopping = false; bool resetGraphics = false; + // Threads in WaitIdle (under `mutex`): a worker's completion notifies only while one waits. + std::uint32_t idleWaiters = 0; static bool& OnWorkerThread() { static thread_local bool worker = false; @@ -585,10 +1458,12 @@ private: const auto queue = submission.queue; auto& worker = workers[queue]; worker.pending.push_back(std::move(submission)); + worker.queued.fetch_add(1, std::memory_order_acq_rel); if (!worker.thread.joinable()) worker.thread = std::thread([this, queue] { run(queue); }); } Driver() { + ShaderMemory::SetWaitedMsProvider(&Graphics::Recorder::ThreadWaitedMs); try { LibcRegisterShutdown_nid_postfix([] { Driver::Get().Shutdown(); }); } catch (...) { @@ -697,18 +1572,21 @@ private: // A draw or dispatch the translator cannot handle is skipped so the rest of the frame still runs; // each distinct failure is reported once. + static void reportSkip(const char* kind, const std::string& what) { + static std::mutex reportedMutex; + static std::set reported; + std::lock_guard lock(reportedMutex); + // APS5_TRACE_SKIPS reports every skip (shortened), not only the first per reason. + static const bool traceSkips = std::getenv("APS5_TRACE_SKIPS") != nullptr; + if (reported.insert(what).second) std::fprintf(stderr, "[gpu] skipped %s: %s\n", kind, what.c_str()); + else if (traceSkips) std::fprintf(stderr, "[gpu] skipped %s again: %.100s\n", kind, what.c_str()); + } template void tolerate(const char* kind, Work&& work) { try { work(); } catch (const std::exception& error) { - static std::mutex reportedMutex; - static std::set reported; - std::lock_guard lock(reportedMutex); - // APS5_TRACE_SKIPS reports every skip (shortened), not only the first per reason. - static const bool traceSkips = std::getenv("APS5_TRACE_SKIPS") != nullptr; - if (reported.insert(error.what()).second) std::fprintf(stderr, "[gpu] skipped %s: %s\n", kind, error.what()); - else if (traceSkips) std::fprintf(stderr, "[gpu] skipped %s again: %.100s\n", kind, error.what()); + reportSkip(kind, error.what()); } } @@ -757,6 +1635,81 @@ private: return type == 0 && stride == 16 && !swizzled && !addTid && dstSel == 0xfacu && format == 0x4bu; } + // A fill covering exactly one resident storage image's surface is done as a GPU clear of that + // image (StorageTexture::FillClear): nothing is stored into host memory and the surface is not + // re-uploaded at its next use; the texels reach guest memory through the deferred write-back + // like a shader's. Debug aid: APS5_NO_FILL_CLEAR=1 stores every fill as before. + static bool fillClearEnabled() { + static const bool enabled = std::getenv("APS5_NO_FILL_CLEAR") == nullptr; + return enabled; + } + + // Debug aid: APS5_FILL_CLEAR_EXACT_ONLY=1 clears only an image no other image overlaps (the + // first form of the conversion): no layer clears, no stale images over the range. + static bool fillClearExactOnly() { + static const bool exactOnly = std::getenv("APS5_FILL_CLEAR_EXACT_ONLY") != nullptr; + return exactOnly; + } + + // APS5_PROFILE_DRAW, every 10 s: [fill-cover] counts every fill HLE call by how its range meets + // the storage images alive (StorageTexture::ClassifyFill) with each class's bytes, the covers + // that had other images over the range, the pending results discarded as dead, and the exact + // and layer covers by guest format; [fill-clear] counts the fills done as clears with the bytes + // they did not store, and the covers refused by reason. + static void fillClearCount(const Graphics::StorageTexture::FillCoverage& coverage, std::size_t bytes, std::span pattern, std::size_t discarded, bool cleared, const char* refusal) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + using FillCover = Graphics::StorageTexture::FillCover; + static constexpr std::array names{"none", "exact", "inside", "around", "straddle", "several", "keys", "layer"}; + static std::mutex countsMutex; + static std::array counts{}, coverBytes{}; + static std::map coveredFormats; + static std::map refusals; + static std::map keyCodes; + static std::uint64_t clears = 0, clearedBytes = 0, layerClears = 0, withOthers = 0, othersInside = 0, discards = 0; + static auto lastReport = std::chrono::steady_clock::now(); + std::lock_guard lock(countsMutex); + const auto index = static_cast(coverage.cover); + ++counts[index]; + coverBytes[index] += bytes; + if (coverage.image != nullptr) { + ++coveredFormats[coverage.image->Descriptor().format]; + if (coverage.others != 0) ++withOthers; + othersInside += coverage.inside; + } + if (coverage.cover == FillCover::Keys) { + // The key byte a metadata fill stores (0x100: not one byte repeated). + const auto word = pattern[0]; + const bool uniform = pattern[1] == word && pattern[2] == word && pattern[3] == word && word == (word & 0xffu) * 0x01010101u; + ++keyCodes[uniform ? word & 0xffu : 0x100u]; + } + discards += discarded; + if (cleared) { + ++clears; + clearedBytes += bytes; + if (coverage.cover == FillCover::Layer) ++layerClears; + } else if (refusal != nullptr) { + ++refusals[refusal]; + } + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::string line = "[fill-cover] fills by cover (count/MiB):"; + for (std::size_t i = 0; i < names.size(); ++i) line += " " + std::string(names[i]) + " " + std::to_string(counts[i]) + "/" + std::to_string(coverBytes[i] >> 20u); + line += "; covers with other images over the range " + std::to_string(withOthers) + " (" + std::to_string(othersInside) + " images wholly inside), pending results discarded as dead " + std::to_string(discards) + "; covers by guest format:"; + for (const auto& [format, count] : coveredFormats) line += " " + std::to_string(format) + ":" + std::to_string(count); + line += "; key fills by code:"; + for (const auto& [code, count] : keyCodes) { + char text[32]; + std::snprintf(text, sizeof(text), " 0x%02x:%llu", code, static_cast(count)); + line += text; + } + std::fprintf(stderr, "%s\n", line.c_str()); + line = "[fill-clear] " + std::to_string(clears) + " fills cleared on resident images (" + std::to_string(clearedBytes >> 20u) + " MiB not stored, " + std::to_string(layerClears) + " one array layer); covers refused:"; + for (const auto& [reason, count] : refusals) line += " " + reason + " " + std::to_string(count); + std::fprintf(stderr, "%s\n", line.c_str()); + } + bool fillBuffer(QueueState& queue, std::uint32_t queueId, std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::shared_ptr& localDevice) { if (!matchesFillKernel(code, userData, compute)) return false; const auto numRecords = userData[2]; @@ -774,17 +1727,82 @@ private: const std::array pattern{userData[4], userData[5], userData[6], userData[7]}; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; static std::atomic fills{0}, filledBytes{0}, cpuFills{0}; + // Fills stored by pattern kind (a uniform dword goes to vkCmdFillBuffer, a 16-byte pattern + // to a seed copy doubled in place: one chain copy per doubling) and their bytes; under the + // GpuMutex like the phases below. + static std::uint64_t uniformFills = 0, uniformBytes = 0, patternFills = 0, patternBytes = 0, chainCopies = 0; + // APS5_PROFILE_DRAW, every 10 s on the [fill] line: the hold's sub-phases in ms (labels, + // check, classify, discard, clear = the FillClear attempt with its flush of straddling + // images, flush = the pre-store FlushPending with its count, device = VulkanDevice:: + // FillBuffer, cpu = the CPU fallback), and the hold time by cover class. Under GpuMutex. + enum FillPhase : std::size_t { FillLabels, FillCheck, FillClassify, FillDiscard, FillClearPhase, FillFlush, FillDevice, FillCpu, FillPhaseCount }; + static constexpr const char* fillPhaseNames[FillPhaseCount] = {"labels", "check", "classify", "discard", "clear", "flush", "device", "cpu"}; + static std::array fillPhaseMs{}; + static std::array holdByCoverMs{}; + static std::array holdByCover{}; + static std::uint64_t flushes = 0, flushed = 0; + static auto fillReport = std::chrono::steady_clock::now(); ++fills; filledBytes += bytes; if (bytes != 0) { GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto holdStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + auto phaseStart = holdStart; + const auto phase = [&](FillPhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + fillPhaseMs[which] += std::chrono::duration(now - phaseStart).count(); + phaseStart = now; + }; // This queue's labels first (queue order), as at every dispatch's lock. recordLabelsForPacket(localDevice.get(), queueId); + phase(FillLabels); GuestMemory::CheckRange(reinterpret_cast(base), bytes, 16, true); + phase(FillCheck); + const auto coverage = Graphics::StorageTexture::ClassifyFill(base, bytes); + phase(FillClassify); + // Debug aid (APS5_TRACE_DCC_KEYS=1): the title's fills of a surface's DCC metadata. + static const bool traceKeys = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; + if (traceKeys && coverage.cover == Graphics::StorageTexture::FillCover::Keys) std::fprintf(stderr, "[dcc-keys] title fills keys 0x%llx+0x%zx with %08x %08x %08x %08x (queue 0x%x)\n", static_cast(base), bytes, pattern[0], pattern[1], pattern[2], pattern[3], queueId); + // Results pending in images wholly inside the range are dead whichever way the fill is + // done (it overwrites every byte of them): dropped rather than stored first. + const auto discarded = fillClearEnabled() ? Graphics::StorageTexture::DiscardPendingInside(base, bytes) : 0u; + phase(FillDiscard); + const char* refusal = nullptr; + bool cleared = false; + if (coverage.image != nullptr) { + if (!fillClearEnabled()) { + refusal = "disabled"; + } else if (fillClearExactOnly() && (coverage.cover != Graphics::StorageTexture::FillCover::Exact || coverage.others != 0)) { + refusal = "exact only"; + } else { + // Results of images straddling the range land first (their bytes outside it + // stay); the covered image's own are superseded by the clear. + Graphics::StorageTexture::FlushPending(base, bytes, coverage.image.get(), "buffer fill", Graphics::PublishScope::PartialUnits); + cleared = coverage.image->FillClear(pattern, coverage.layer, refusal); + } + } + phase(FillClearPhase); + fillClearCount(coverage, bytes, pattern, discarded, cleared, refusal); + noteForeignWriter(base, base + bytes, queueId); // Results still on the GPU for the range would be stored over the fill later. - Graphics::StorageTexture::FlushPending(base, bytes, nullptr, "buffer fill"); - if (!localDevice->FillBuffer(base, bytes, pattern)) { + if (!cleared) { + ++flushes; + // The fill's own units need no publish (it overwrites them); the boundary units + // keep a neighbour's shadowed results, which land first. + if (Graphics::StorageTexture::FlushPending(base, bytes, nullptr, "buffer fill", Graphics::PublishScope::PartialUnits)) ++flushed; + } + phase(FillFlush); + const bool stored = cleared || localDevice->FillBuffer(base, bytes, pattern); + phase(FillDevice); + if (profile && stored && !cleared) { + const bool uniform = pattern[0] == pattern[1] && pattern[1] == pattern[2] && pattern[2] == pattern[3]; + ++(uniform ? uniformFills : patternFills); + (uniform ? uniformBytes : patternBytes) += bytes; + if (!uniform) chainCopies += static_cast(std::bit_width((bytes / 16) - 1)); + } + if (!stored) { ++cpuFills; // Ordered after recorded work like any CPU store from the queue. localDevice->WaitIdle(); @@ -793,12 +1811,286 @@ private: block.resize(chunk); for (std::size_t at = 0; at < chunk; at += 16) std::memcpy(block.data() + at, pattern.data(), 16); for (std::size_t done = 0; done < bytes; done += chunk) GuestMemory::Write(base + done, std::span(block).first(std::min(chunk, bytes - done)), 16); + phase(FillCpu); + } + if (profile) { + const auto now = std::chrono::steady_clock::now(); + const auto cover = static_cast(coverage.cover); + ++holdByCover[cover]; + holdByCoverMs[cover] += std::chrono::duration(now - holdStart).count(); + if (now - fillReport > std::chrono::seconds(10)) { + fillReport = now; + static constexpr std::array coverNames{"none", "exact", "inside", "around", "straddle", "several", "keys", "layer"}; + std::string line; + char text[64]; + for (std::size_t i = 0; i < FillPhaseCount; ++i) { + std::snprintf(text, sizeof(text), " %s %.0f", fillPhaseNames[i], fillPhaseMs[i]); + line += text; + } + line += "; hold by cover (count/ms):"; + for (std::size_t i = 0; i < coverNames.size(); ++i) { + std::snprintf(text, sizeof(text), " %s %llu/%.0f", coverNames[i], static_cast(holdByCover[i]), holdByCoverMs[i]); + line += text; + } + std::fprintf(stderr, "[fill] %llu buffer fills (%.0f MiB), %llu stored by the CPU; hold phases ms (cumulative):%s; pre-store flushes %llu (%llu stored an image); stored on the GPU: uniform %llu (%.0f MiB), pattern %llu (%.0f MiB), chain copies %llu\n", static_cast(fills.load()), filledBytes.load() / 1048576.0, static_cast(cpuFills.load()), line.c_str(), static_cast(flushes), static_cast(flushed), static_cast(uniformFills), uniformBytes / 1048576.0, static_cast(patternFills), patternBytes / 1048576.0, static_cast(chainCopies)); + } } } - if (profile && fills % 500 == 0) std::fprintf(stderr, "[fill] %llu buffer fills (%.0f MiB), %llu stored by the CPU\n", static_cast(fills.load()), filledBytes.load() / 1048576.0, static_cast(cpuFills.load())); return true; } + // The engine's memcpy kernel (38 code words: count and n from a one-record V#, then every thread + // i < count stores src[i mod n] to dst[i]; src in user[0..3], dst in user[4..7], the record in + // user[8..11], stride-4 32_UINT V#s and a stride-16 one). Demon's Souls runs it ~27 times per + // frame at the movie stage, mostly on small records whose destinations later captures read, so + // every such capture waited for the kernel's batch. Matched by the code hash and the exact V# + // control words (Kyty's guard set) and done by VulkanDevice::CopyBuffer: a CPU copy at packet + // time when the copy is small and nothing recorded may still read the destination in place + // (the recorder's read tracking) or write either range (the bytes are settled, with no pending + // write for readers to wait on), else a transfer between the host imports. + // count and n are read on the CPU only when nothing recorded may still change their record; + // otherwise, and whenever a guard fails, the kernel runs as before (never a sync). Debug aids: + // APS5_NO_COPY_HLE=1 runs the kernel always; APS5_COPY_HLE_CPU_MAX= bounds the CPU copies + // (default 262144, 0 keeps every copy on the GPU, as APS5_NO_CPU_COPY=1 does); + // APS5_COPY_HLE_GPU_MAX= bounds the transfers (a larger copy that is neither a CPU copy + // nor an image copy runs the kernel; unset: no bound, since the kernel and the transfer cost + // the GPU the same and the transfer spares the dispatch's CPU work; 0: never a transfer); + // APS5_COPY_READ_TRACKING=0 makes the CPU path require an idle recorder (the rule before read + // tracking); APS5_COPY_SYNC, APS5_COPY_WAIT_SOURCE, APS5_COPY_VERIFY see VulkanDevice::CopyBuffer. + static bool matchesCopyKernel(std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) { + static const bool enabled = std::getenv("APS5_NO_COPY_HLE") == nullptr; + if (!enabled || userData.size() != 12 || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1 || !compute.groupIdEnable[0]) return false; + static constexpr std::size_t copyKernelWords = 38; + static constexpr std::uint64_t copyKernelHash = 0x6ec00fe8aa95f99aull; + if (code.size() < copyKernelWords || code[0] != 0xbfa00002u || code[copyKernelWords - 1] != 0xbf810000u) return false; + std::uint64_t hash = 0xcbf29ce484222325ull; + for (const auto word : code.first(copyKernelWords)) { + hash ^= word; + hash *= 0x100000001b3ull; + } + if (hash != copyKernelHash) return false; + const auto stride = [&](std::size_t word) { return (userData[word] >> 16u) & 0x3fffu; }; + const auto swizzled = [&](std::size_t word) { return ((userData[word] >> 31u) & 1u) != 0; }; + return userData[3] == 0x00014004u && userData[7] == 0x00014004u && userData[11] == 0x0004dfacu && stride(1) == 4 && stride(5) == 4 && stride(9) == 16 && !swizzled(1) && !swizzled(5) && !swizzled(9) && userData[10] >= 1; + } + + // The copies' outcomes for the [copy] line (APS5_PROFILE_DRAW, every 10 s): 0 and 1 are + // VulkanDevice::CopyBuffer's paths and 2 its refusal, the rest are decided here before it. + enum CopyPath { CopyCpu = 0, CopyGpu = 1, CopyNotImported = 2, CopyAliased = 3, CopyRecordPending = 4, CopyShape = 5, CopyOverlapping = 6, CopyInaccessible = 7, CopyOverGpuMax = 8, CopyPaths = 9 }; + // `started` and `locked` are when the packet's handling began and when it got the GPU mutex + // (equal when it never took it), so the line shows what the copies cost the worker. `outcome` + // (paths 0..2) carries the CPU-path refusal and the wait/verify counters. + static void countCopy(int path, std::size_t bytes, std::chrono::steady_clock::time_point started, std::chrono::steady_clock::time_point locked, const VulkanDevice::CopyOutcome* outcome = nullptr) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile || path < 0 || path >= CopyPaths) return; + using Outcome = VulkanDevice::CopyOutcome; + static std::mutex countsMutex; + static std::uint64_t counts[CopyPaths] = {}; + static std::uint64_t refused[Outcome::Refusals] = {}; + static std::uint64_t cpuBytes = 0, gpuBytes = 0, aliasedBytes = 0, syncs = 0, settledBySignal = 0, sourceWaits = 0; + static double syncedMs = 0, totalMs = 0, lockWaitMs = 0, sourceWaitMs = 0; + static auto lastReport = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(countsMutex); + ++counts[path]; + if (path == CopyCpu) cpuBytes += bytes; + if (path == CopyGpu) gpuBytes += bytes; + if (path == CopyAliased) aliasedBytes += bytes; + if (outcome != nullptr) { + if (outcome->synced) { + ++syncs; + syncedMs += outcome->syncMs; + } + if (outcome->reason > Outcome::None && outcome->reason < Outcome::Refusals) ++refused[outcome->reason]; + if (outcome->sourceSettledBySignal) ++settledBySignal; + if (outcome->waitedForSource) { + ++sourceWaits; + sourceWaitMs += outcome->waitMs; + } + } + totalMs += std::chrono::duration(now - started).count(); + lockWaitMs += std::chrono::duration(locked - started).count(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::uint64_t matched = 0; + for (const auto count : counts) matched += count; + const auto verify = VulkanDevice::CopyVerifyCounts(); + const auto aliasing = VulkanDevice::CopyAliasCounts(); + // Known values (cumulative): ring entries carrying a value, reads served from one (capture + // words and validation regions), and the verified ones (APS5_VERIFY_KNOWN_VALUES=1). + const auto known = ShaderMemory::KnownValues(); + const auto load = [](const std::atomic& counter) { return static_cast(counter.load(std::memory_order_relaxed)); }; + std::fprintf(stderr, "[copy] %llu copy kernels matched: cpu %llu (%.1f MiB; source settled by signal %llu), gpu %llu (%.0f MiB; %llu waited %.0f ms), aliased %llu (%.0f MiB; %llu destination images created; transfers instead: no source image %llu, shape %llu, refused %llu); refused cpu: size %llu, image %llu, shadow %llu, source pending %llu, source unsettled %llu, destination pending %llu, destination unsettled %llu, label %llu, reader %llu; kernel run instead: count record pending %llu, shape %llu, not imported %llu, overlapping %llu, inaccessible %llu, over gpu max %llu; waited for the source %llu / %.0f ms; verify: %llu verified, source changed %llu, destination changed %llu, reader missed %llu; known values: %llu entries, %llu reads served, %llu verified, %llu mismatches; took %.0f ms (%.0f ms waiting for the mutex)\n", static_cast(matched), static_cast(counts[CopyCpu]), cpuBytes / 1048576.0, static_cast(settledBySignal), static_cast(counts[CopyGpu]), gpuBytes / 1048576.0, static_cast(syncs), syncedMs, static_cast(counts[CopyAliased]), aliasedBytes / 1048576.0, static_cast(aliasing.created), static_cast(aliasing.noSource), static_cast(aliasing.shape), static_cast(aliasing.refused), static_cast(refused[Outcome::Size]), static_cast(refused[Outcome::Image]), static_cast(refused[Outcome::Shadow]), static_cast(refused[Outcome::SourcePending]), static_cast(refused[Outcome::SourceUnsettled]), static_cast(refused[Outcome::DestinationPending]), static_cast(refused[Outcome::DestinationUnsettled]), static_cast(refused[Outcome::Label]), static_cast(refused[Outcome::Reader]), static_cast(counts[CopyRecordPending]), static_cast(counts[CopyShape]), static_cast(counts[CopyNotImported]), static_cast(counts[CopyOverlapping]), static_cast(counts[CopyInaccessible]), static_cast(counts[CopyOverGpuMax]), static_cast(sourceWaits), sourceWaitMs, static_cast(verify.verified), static_cast(verify.sourceChanged), static_cast(verify.destinationChanged), static_cast(verify.readerMissed), load(knownValueEntries), load(knownValueReads), static_cast(known.verified) + load(knownValueVerified), static_cast(known.mismatches) + load(knownValueMismatches), totalMs, lockWaitMs); + } + + // The evidence ring's entry for a copy recorded on the GPU: the kernel's destination element + // under its program, as noteWrittenBuffers enters it for the kernel itself, so the + // write-evidence policy sees the same writer as before. `value`, when VulkanDevice::CopyBuffer + // knows the destination's post-copy bytes, makes it a known-value entry (classifyPendingWrite's + // KnownValue), current while UnchangedSince(destination, `generation`) holds. + void noteCopyWriter(std::uint64_t program, std::uint64_t begin, std::uint64_t end, std::uint32_t queue, std::span value = {}, std::uint64_t generation = 0) { + if (!(WriteEvidenceEnabled() || TraceCapSync())) return; + std::shared_ptr> known; + if (!value.empty() && value.size() == end - begin && generation != 0) { + known = std::make_shared>(value.begin(), value.end()); + knownValueEntries.fetch_add(1, std::memory_order_relaxed); + } + std::lock_guard lock(writtenBuffersMutex); + writtenBuffers.push_back({program, begin, end, ++writtenBufferSerial, queue, false, std::move(known), generation}); + while (writtenBuffers.size() > WrittenBufferRing) writtenBuffers.pop_front(); + } + // Known-value copy destinations (K): APS5_NO_COPY_KNOWN_VALUES=1 carries no values (the copy + // destinations take the Raw/Sync policies as before); APS5_VERIFY_KNOWN_VALUES=1 serves them + // and reads through the hook as well, counting the mismatches on the [copy] line. + static bool CopyKnownValues() { + static const bool enabled = std::getenv("APS5_NO_COPY_KNOWN_VALUES") == nullptr; + return enabled; + } + static bool KnownValueVerify() { + static const bool verify = std::getenv("APS5_VERIFY_KNOWN_VALUES") != nullptr; + return verify; + } + static constexpr std::size_t KnownValueMaxBytes = 4096; + static inline std::atomic knownValueEntries{0}, knownValueReads{0}, knownValueVerified{0}, knownValueMismatches{0}; + + bool copyBuffer(QueueState& queue, std::uint32_t queueId, std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::shared_ptr& localDevice, std::uint64_t programAddress) { + if (!matchesCopyKernel(code, userData, compute)) return false; + const auto started = std::chrono::steady_clock::now(); + static const std::size_t cpuMax = [] { + if (std::getenv("APS5_NO_CPU_COPY") != nullptr) return std::size_t{0}; + const char* text = std::getenv("APS5_COPY_HLE_CPU_MAX"); + return text ? static_cast(std::strtoull(text, nullptr, 10)) : std::size_t{262144}; + }(); + static const std::size_t gpuMax = [] { + const char* text = std::getenv("APS5_COPY_HLE_GPU_MAX"); + return text ? static_cast(std::strtoull(text, nullptr, 10)) : std::numeric_limits::max(); + }(); + const auto base = [&](std::size_t word) { return userData[word] | (static_cast(userData[word + 1] & 0xffffu) << 32u); }; + const auto source = base(0); + const auto destination = base(4); + const auto record = base(8); + std::array groups{packet[1], packet[2], packet[3]}; + if ((packet[4] & 0x20u) != 0) { + for (std::uint32_t axis = 0; axis < 3; ++axis) { + const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u); + groups[axis] = (groups[axis] + threads - 1) / threads; + } + } + if (groups[1] != 1 || groups[2] != 1) { + countCopy(CopyShape, 0, started, started); + return false; + } + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Copy); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto locked = std::chrono::steady_clock::now(); + // This queue's labels first (queue order), as at every dispatch's lock. + recordLabelsForPacket(localDevice.get(), queueId); + // Batches the GPU already finished are retired first (as FillBuffer does), so a writer of + // the record that is long done does not count as pending below, and CopyBuffer sees the + // recorder idle when it is. Only on the graphics worker: a reap runs completions under the + // mutex, which a compute worker must never do against queue 0 (see WriteLabelOnGpu). + if (queueId == 0) localDevice->ReapRecorded(); + // count and n (the record's first two dwords; the title keeps a GPU-written label in its + // second half) are read raw, so only when no recorded work, pending image result or label + // may still change them (a read through the flush hook would wait for it). + const auto labelPending = [&](std::uint64_t address, std::size_t bytes) { + for (auto dword = address & ~std::uint64_t{3}; dword < address + bytes; dword += 4) { + if (localDevice->PendingLabel(dword, 4, 0).has_value()) return true; + } + return false; + }; + const bool recordAccessible = GuestMemory::Accessible(reinterpret_cast(record), 8); + // A publish of shadowed results over the record is a recorded store like a flush: the raw + // read below would miss it. + bool published = false; + if (!recordAccessible || Graphics::Recorder::SnapshotWriteOverlaps(record, 8) || Graphics::StorageTexture::FlushPending(record, 8, nullptr, "buffer copy count", Graphics::PublishScope::Whole, &published) || published || labelPending(record, 8)) { + if (traceCopy()) traceCopyPending(recordAccessible ? "count record" : "inaccessible count record", record, 8, source, destination, 0, localDevice); + countCopy(CopyRecordPending, 0, started, locked); + return false; + } + std::uint32_t count = 0, period = 0; + std::memcpy(&count, reinterpret_cast(record), 4); + std::memcpy(&period, reinterpret_cast(record + 4), 4); + const auto bytes = static_cast(count) * 4u; + // A plain copy only: n >= count (no repetition), every record inside both V#s, exactly the + // groups for the count (Kyty's guard), at most 16 MiB, dword-aligned ranges, and the count + // record outside the destination (every wave of the kernel re-reads it after other waves + // stored, while the HLE reads it once). + if (count == 0 || period < count || groups[0] != (count + 63u) / 64u || count > userData[2] || count > userData[6] || bytes > (16u << 20u) || source % 4 != 0 || destination % 4 != 0 || (record < destination + bytes && destination < record + 8)) { + countCopy(CopyShape, bytes, started, locked); + return false; + } + if (source < destination + bytes && destination < source + bytes) { + countCopy(CopyOverlapping, bytes, started, locked); + return false; + } + if (!GuestMemory::Accessible(reinterpret_cast(source), bytes) || !GuestMemory::Accessible(reinterpret_cast(destination), bytes, true)) { + countCopy(CopyInaccessible, bytes, started, locked); + return false; + } + // The CPU decision and, failing it, the flushes of pending image results and the transfer; + // a transfer's ring entry (with the known destination bytes when the source is current) + // is noted ahead of its pending write, as noteWrittenBuffers precedes a dispatch's note. + const auto outcome = localDevice->CopyBuffer(destination, source, bytes, cpuMax, gpuMax, CopyKnownValues() ? KnownValueMaxBytes : 0, programAddress, queueId, [&](std::span value, std::uint64_t generation) { noteCopyWriter(programAddress, destination, destination + bytes, queueId, value, generation); }); + countCopy(outcome.path, bytes, started, locked, &outcome); + if (traceCopy() && bytes <= cpuMax && outcome.path != CopyCpu) traceCopyRefused(outcome, source, destination, bytes, localDevice); + if (outcome.path == CopyNotImported || outcome.path == CopyOverGpuMax) return false; + // A CPU copy leaves no pending write, so the evidence ring's dword streaks under the + // kernel's program must not carry over to a later GPU copy of the same record: a foreign + // entry makes the next pending write over it sync until a dispatch element writes it again. + if (outcome.path == CopyCpu) noteForeignWriter(destination, destination + bytes, queueId); + return true; + } + + // The [copyhle] trace of a small copy that missed the CPU path: the refusing rule and, for a + // reader, the batch that reads the destination in place; for a pending range, its writer. + void traceCopyRefused(const VulkanDevice::CopyOutcome& outcome, std::uint64_t source, std::uint64_t destination, std::size_t bytes, const std::shared_ptr& localDevice) { + using Outcome = VulkanDevice::CopyOutcome; + static constexpr const char* rules[Outcome::Refusals] = {"none", "size", "pending image", "source pending", "source unsettled", "destination pending", "destination unsettled", "label", "reader", "shadowed"}; + static constexpr const char* readers[] = {"dispatch element", "gpu copy", "address-based", "indirect", "storage upload", "copy source"}; + if (outcome.reason == Outcome::SourcePending || outcome.reason == Outcome::SourceUnsettled) { + traceCopyPending(rules[outcome.reason], source, bytes, source, destination, bytes, localDevice); + return; + } + if (outcome.reason == Outcome::DestinationPending || outcome.reason == Outcome::DestinationUnsettled || outcome.reason == Outcome::Label) { + traceCopyPending(rules[outcome.reason], destination, bytes, source, destination, bytes, localDevice); + return; + } + static std::atomic shown{0}; + if (shown.fetch_add(1) >= 160 || outcome.reason <= Outcome::None || outcome.reason >= Outcome::Refusals) return; + if (outcome.reason == Outcome::Reader) { + const auto kind = outcome.readerKind >= 0 && outcome.readerKind < 6 ? readers[outcome.readerKind] : "?"; + std::fprintf(stderr, "[copyhle] reader refused the CPU copy 0x%llx -> 0x%llx (0x%zx bytes): batch %llu%s of queue 0x%x reads the destination in place (%s)\n", static_cast(source), static_cast(destination), bytes, static_cast(outcome.readerSerial), outcome.readerOpen ? " (open)" : "", outcome.readerQueue, kind); + return; + } + std::fprintf(stderr, "[copyhle] %s refused the CPU copy 0x%llx -> 0x%llx (0x%zx bytes)\n", rules[outcome.reason], static_cast(source), static_cast(destination), bytes); + } + + // Debug aid: APS5_TRACE_COPY_HLE=1 prints the first copies the HLE could not make on the CPU + // because of a pending write ([copyhle] lines): the range, the copy, the newest recorded batch + // writing it (as the flush hook would see it), pending label dwords in it, and the evidence + // ring's writers of it. + static bool traceCopy() { + static const bool enabled = std::getenv("APS5_TRACE_COPY_HLE") != nullptr; + return enabled; + } + void traceCopyPending(const char* what, std::uint64_t address, std::size_t bytes, std::uint64_t source, std::uint64_t destination, std::size_t copyBytes, const std::shared_ptr& localDevice) { + static std::atomic shown{0}; + if (shown.fetch_add(1) >= 160) return; + char pending[160]; + std::snprintf(pending, sizeof(pending), " no recorded write"); + if (auto* recorder = Graphics::Recorder::Active()) { + if (const auto info = recorder->DescribePendingWrite(address, bytes)) { + std::snprintf(pending, sizeof(pending), " batch %llu%s%s range 0x%llx+0x%llx (%zu batches to finish)", static_cast(info->serial), info->open ? " open" : "", info->signaled ? " signaled" : " unsignaled", static_cast(info->rangeBegin), static_cast(info->rangeEnd - info->rangeBegin), info->batchesToFinish); + } + } + std::uint32_t labels = 0; + for (auto dword = address & ~std::uint64_t{3}; dword < address + bytes; dword += 4) { + if (localDevice->PendingLabel(dword, 4, 0).has_value()) ++labels; + } + std::fprintf(stderr, "[copyhle] %s 0x%llx+0x%zx pending (copy 0x%llx -> 0x%llx, 0x%zx bytes):%s; label dwords %u; writers:%s\n", what, static_cast(address), bytes, static_cast(source), static_cast(destination), copyBytes, pending, labels, describeWriters(address, address + bytes).c_str()); + } + // How DISPATCH_INDIRECT group counts were resolved, for the [indirect] line (APS5_PROFILE_DRAW, // every 10 s). 0..3 are VulkanDevice::DispatchIndirect's outcomes (0 recorded GPU-side), the rest // are decided here before the dispatch is prepared. `readMs` is the CPU read's time (its sync). @@ -821,12 +2113,934 @@ private: std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), readWaitedMs / 1000); } + // Recorded GPU writes over captured words. The recompiler marks a V# element written when the + // program stores through it anywhere, so a small descriptor or SRT table a kernel binds that + // way is a pending write for every capture and dispatch-cache validation that reads it, and + // the flush hook waited for the kernel's batch before each such read although the kernel never + // changed the dwords read (thousands of syncs per run, all unchanged). The elements a dispatch + // marks written are remembered (writtenBuffers, newest first), and every read that does wait on + // one compares its dwords before and after the wait (dwordEvidence). A dword that came out + // unchanged from its last WriteEvidenceAfter waits is then read without the wait, raw, as long + // as the newest recorded writer of it is a dispatch's element of at most WriteEvidenceMaxBytes, + // the pending range is that element (a larger one, a fill, shows at the element's edges) and + // no GPU label is pending in it: ShaderMemory reads the dword raw (classifyPendingWrite) and + // the dispatch-cache validation compares the captured words raw (validateCaptured). Every + // WriteEvidenceSampleEvery-th capture or validation that could read raw waits and observes + // anyway, so a kernel that starts changing a dword is noticed and its reads wait again (the + // raw reads before that saw the old bytes). The validation also declares a miss without + // waiting when the CPU-visible words already differ from the capture (a miss recaptures + // through the hook, so that is always safe). + // A streak belongs to the writer it was observed under (program and element): memory reused + // by another kernel's element starts over. A recorded write the ring knows no program for (a + // fill, a draw's written elements: program 0, "foreign") always syncs. An observation counts + // only when the hook waited for unfinished GPU work across the read (Recorder::ThreadHookWaits): + // against finished work both reads hold the GPU's bytes and "unchanged" proves nothing. + // Debug aids: APS5_NO_WRITE_EVIDENCE=1 neither observes nor skips (APS5_NO_VALIDATE_SKIP=1 + // keeps the validation's compares and misses synced, APS5_NO_WORDWISE_CAPTURE=1 the page reads + // whole); APS5_VALIDATE_SYNC_SKIP=1 makes every skipped read wait anyway and counts the ones + // whose raw bytes differed from the waited ones ([validate] line, 10 s; the [capture] line for + // dword reads); APS5_TRACE_CAPSYNC=1 prints every compare and capture that met a pending write, + // with the recorded writers ([capsync]); APS5_NO_WRITER_KEYED_EVIDENCE=1 keys the streaks by + // address alone (and notes a dispatch's elements after its record, as before); + // APS5_NO_FOREIGN_WRITERS=1 leaves fills and draws out of the ring; APS5_NO_OBSERVATION_GUARD=1 + // observes every hook read and every pending region of a validation; APS5_NO_SITE_SAMPLING=1 + // samples captures and validations from one shared count. + // A copy HLE transfer whose source was CPU-current at its note enters the ring with the + // destination's post-copy bytes (`value`, at most KnownValueMaxBytes) and the write-watch + // generation MarkWritten(destination) stamped: while UnchangedSince(destination, generation) + // holds (a memo'd collect first, so a later CPU store is seen) the newest such writer answers + // a read with the bytes the GPU will leave (KnownValue), before the evidence and without the + // sample, since the copy reads a source nothing recorded changes; a later ring entry over the + // dword supersedes it by serial as any writer does. + struct WrittenBuffer { + std::uint64_t program; + std::uint64_t begin; + std::uint64_t end; + std::uint64_t serial; + std::uint32_t queue; + bool atomic; + std::shared_ptr> value; + std::uint64_t generation = 0; + }; + struct DwordEvidence { + std::uint32_t streak = 0; + std::uint32_t changed = 0; + // The newest writer of the dword when the streak was observed. + std::uint64_t program = 0; + std::uint64_t begin = 0; + std::uint64_t end = 0; + }; + // Every WriteEvidenceSampleEvery-th capture, and apart from those every + // WriteEvidenceSampleEvery-th validation, that could read raw waits and observes instead + // (SampledReadScope brackets the capture; the validation sets it itself). + std::atomic evidenceReads{0}; + std::atomic evidenceValidations{0}; + static bool& SampledRead() { + static thread_local bool sampled = false; + return sampled; + } + class SampledReadScope { + public: + explicit SampledReadScope(std::atomic& reads) : previous(SampledRead()) { + SampledRead() = WriteEvidenceEnabled() && reads.fetch_add(1, std::memory_order_relaxed) % WriteEvidenceSampleEvery() == 0; + } + ~SampledReadScope() { SampledRead() = previous; } + SampledReadScope(const SampledReadScope&) = delete; + SampledReadScope& operator=(const SampledReadScope&) = delete; + + private: + bool previous; + }; + static constexpr std::size_t WrittenBufferRing = 4096; + static constexpr std::size_t DwordEvidenceEntries = 65536; + std::mutex writtenBuffersMutex; + std::deque writtenBuffers; + std::uint64_t writtenBufferSerial = 0; + std::unordered_map dwordEvidence; + std::atomic observedUnchanged{0}; + std::atomic observedChanged{0}; + // Pending regions of validations not observed: compared after no GPU wait, or not compared + // at all (a region before them differed). + std::atomic observationsNoWait{0}; + std::atomic observationsNotReached{0}; + struct ValidateCounters { + std::uint64_t pending = 0, unsyncedMisses = 0, skipped = 0, syncedNoWriter = 0, syncedForeign = 0, syncedLargeRange = 0, syncedLabel = 0, syncedEvidence = 0, syncedWriterChanged = 0, syncedSample = 0, syncedImage = 0, syncedShadow = 0, syncedOff = 0, verified = 0, mismatches = 0, verifiedMissesHit = 0, knownValue = 0; + double syncedWaitMs = 0, verifiedWaitMs = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + }; + std::mutex validateMutex; + ValidateCounters validateCounters; + + static bool WriteEvidenceEnabled() { + static const bool enabled = std::getenv("APS5_NO_WRITE_EVIDENCE") == nullptr; + return enabled; + } + static bool WriterKeyedEvidence() { + static const bool keyed = std::getenv("APS5_NO_WRITER_KEYED_EVIDENCE") == nullptr; + return keyed; + } + static bool ForeignWriters() { + static const bool noted = std::getenv("APS5_NO_FOREIGN_WRITERS") == nullptr; + return noted; + } + static bool ObservationGuard() { + static const bool guard = std::getenv("APS5_NO_OBSERVATION_GUARD") == nullptr; + return guard; + } + static bool SiteSampling() { + static const bool perSite = std::getenv("APS5_NO_SITE_SAMPLING") == nullptr; + return perSite; + } + static std::uint64_t HookWaits() { + return Graphics::Recorder::ThreadHookWaits(); + } + // ShaderMemory's hook-wait counter (null under APS5_NO_OBSERVATION_GUARD: every read observed). + static ShaderMemory::HookWaitCounter HookWaitCounter() { + return ObservationGuard() ? &HookWaits : nullptr; + } + static bool ValidateSkipEnabled() { + static const bool enabled = std::getenv("APS5_NO_VALIDATE_SKIP") == nullptr; + return enabled; + } + // Debug aid: APS5_VALIDATE_LEGACY=1 restores the value validation's old compare path: the + // per-run Accessible walk and EqualsCommittedUnsynced (each with its profile counters) instead + // of one CompareMapped per run, a pending-write snapshot load per run and per edge test instead + // of one per validation, and no retry of the gate after a moved serial. + static bool ValidateLegacy() { + static const bool legacy = std::getenv("APS5_VALIDATE_LEGACY") != nullptr; + return legacy; + } + static bool GateRetry() { + static const bool retry = std::getenv("APS5_NO_GATE_RETRY") == nullptr && !ValidateLegacy(); + return retry; + } + // A validation's view of the recorder's pending-write snapshot: loaded once (Load) and consulted + // for every run and edge test, so all of them see the same snapshot (design13 R1's p0: loaded + // after the gate's generation load, a later publish fails the generation re-check). A reload + // is skipped while the publish generation read before the load has not moved (a publish + // stores its snapshot before bumping it, so the loaded one is the newest stored or the gate's + // re-check catches the bump): the atomic shared_ptr load is a locked RMW on a line every queue + // worker shares. Under the legacy switch every query loads the current snapshot as before. + struct PendingView { + std::shared_ptr snapshot; + std::uint64_t generation = 0; + bool loaded = false; + + void Load() { + if (ValidateLegacy()) return; + const auto current = Graphics::Recorder::PublishGeneration(); + if (loaded && current == generation) return; + snapshot = Graphics::Recorder::PendingWriteSnapshot(); + generation = current; + loaded = true; + } + bool Overlaps(std::uint64_t address, std::size_t bytes) const { + return ValidateLegacy() ? Graphics::Recorder::SnapshotWriteOverlaps(address, bytes) : Graphics::Recorder::SnapshotOverlaps(snapshot.get(), address, bytes); + } + }; + static bool ValidateSkipVerify() { + static const bool verify = std::getenv("APS5_VALIDATE_SYNC_SKIP") != nullptr; + return verify; + } + static std::uint32_t WriteEvidenceAfter() { + static const std::uint32_t value = [] { const char* text = std::getenv("APS5_WRITE_EVIDENCE_AFTER"); return text ? static_cast(std::atoi(text)) : 4u; }(); + return value; + } + static std::uint32_t WriteEvidenceSampleEvery() { + static const std::uint32_t value = [] { const char* text = std::getenv("APS5_WRITE_EVIDENCE_SAMPLE"); const auto parsed = text ? std::atoi(text) : 16; return parsed > 0 ? static_cast(parsed) : 16u; }(); + return value; + } + static std::uint64_t WriteEvidenceMaxBytes() { + static const std::uint64_t value = [] { const char* text = std::getenv("APS5_WRITE_EVIDENCE_MAX_KIB"); return (text ? std::strtoull(text, nullptr, 10) : 16ull) << 10u; }(); + return value; + } + static bool TraceCapSync() { + static const bool trace = std::getenv("APS5_TRACE_CAPSYNC") != nullptr; + return trace; + } + // The written V# elements of a compiled program, as the resource build notes them (bufferWritten). + template + static void forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compiled, Visit&& visit) { + for (const auto& binding : compiled.bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; + if (!written || binding.guestDescriptor.size() < (static_cast(element) + 1) * 4) continue; + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4); + const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; + const auto base = descriptor.Base48(); + const auto size = descriptor.GetSize(); + if (base == 0 || size == 0) continue; + visit(element, base, base + size, element < binding.bufferAtomic.size() && binding.bufferAtomic[element]); + } + } + } + // As a dispatch is recorded (under GuestMemory::GpuMutex, before its record, so a + // classification never finds the previous writer under a note that already includes it): + // its written elements join the ring. + void noteWrittenBuffers(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled) { + std::lock_guard lock(writtenBuffersMutex); + const auto serial = ++writtenBufferSerial; + forEachWrittenBuffer(compiled, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool atomic) { + writtenBuffers.push_back({program, begin, end, serial, queue, atomic}); + }); + while (writtenBuffers.size() > WrittenBufferRing) writtenBuffers.pop_front(); + } + // A recorded write the evidence does not cover (a fill, a draw's written elements): a foreign + // entry (program 0) of the ring, so a dword under it goes through the hook until a dispatch + // writes it again. + void noteForeignWriter(std::uint64_t begin, std::uint64_t end, std::uint32_t queue) { + if (!ForeignWriters() || !(WriteEvidenceEnabled() || TraceCapSync()) || end <= begin) return; + std::lock_guard lock(writtenBuffersMutex); + writtenBuffers.push_back({0, begin, end, ++writtenBufferSerial, queue, false}); + while (writtenBuffers.size() > WrittenBufferRing) writtenBuffers.pop_front(); + } + void noteDrawWriters(std::span stages, std::uint32_t queue) { + for (const auto& stage : stages) { + forEachWrittenBuffer(*stage.program, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { noteForeignWriter(begin, end, queue); }); + } + } + std::optional newestWriterLocked(std::uint64_t begin, std::uint64_t end) const { + for (auto it = writtenBuffers.rbegin(); it != writtenBuffers.rend(); ++it) { + if (begin < it->end && it->begin < end) return *it; + } + return std::nullopt; + } + std::optional newestWriter(std::uint64_t begin, std::uint64_t end) { + std::lock_guard lock(writtenBuffersMutex); + return newestWriterLocked(begin, end); + } + // A read that waited: whether its dword still holds the bytes read before the wait. + void observeDword(std::uint64_t address, bool unchanged) { + if (!WriteEvidenceEnabled()) return; + (unchanged ? observedUnchanged : observedChanged).fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(writtenBuffersMutex); + if (dwordEvidence.size() >= DwordEvidenceEntries && !dwordEvidence.contains(address)) dwordEvidence.clear(); + auto& evidence = dwordEvidence[address]; + if (WriterKeyedEvidence()) { + const auto writer = newestWriterLocked(address, address + 4); + const std::uint64_t program = writer ? writer->program : 0, begin = writer ? writer->begin : 0, end = writer ? writer->end : 0; + if (program != evidence.program || begin != evidence.begin || end != evidence.end) evidence = DwordEvidence{0, 0, program, begin, end}; + } + if (!unchanged) { + evidence.streak = 0; + ++evidence.changed; + } else if (evidence.streak < (1u << 30u)) { + ++evidence.streak; + } + } + void observeRange(std::uint64_t address, std::span before) { + if (before.empty() || !GuestMemory::Accessible(reinterpret_cast(address), before.size())) return; + const auto* now = reinterpret_cast(address); + for (std::size_t offset = 0; offset + 4 <= before.size(); offset += 4) { + observeDword(address + offset, std::memcmp(before.data() + offset, now + offset, 4) == 0); + } + } + static void ObservePendingWrite(std::uint64_t address, bool unchanged) { + Get().observeDword(address, unchanged); + } + // Whether a known-value entry's bytes are still what the GPU leaves: no store over the + // destination since the copy's stamp (a game store not collected yet becomes a stamp first). + static bool knownValueCurrent(const WrittenBuffer& writer) { + const auto bytes = static_cast(writer.end - writer.begin); + GuestMemory::CollectWrites(writer.begin, bytes); + return GuestMemory::UnchangedSince(writer.begin, bytes, writer.generation); + } + // The policy for a read of [address, address + bytes) (ShaderMemory's query, the validation): + // None when nothing recorded writes it; KnownValue when its newest recorded writer is a copy + // HLE entry carrying the destination's bytes that is still current (copied into `known` when + // the caller passes a span of `bytes`; an empty span asks for the answer alone) and no GPU + // label is pending in it; Raw when its newest recorded writer is a dispatch's small element, + // the pending range is that element, no GPU label is pending in it and every dword has an + // unchanged streak observed under that writer, unless this is one of the sampled reads; + // VerifyRaw/VerifyKnownValue under the verify switches; Sync otherwise, with `reason` naming + // why for the [validate] line. + ShaderMemory::PendingWrite classifyPendingWrite(std::uint64_t address, std::size_t bytes, std::uint64_t ValidateCounters::*& reason, const PendingView& pending, std::span known = {}) { + using Policy = ShaderMemory::PendingWrite; + if (!pending.Overlaps(address, bytes)) return Policy::None; + // Shadowed results over the range are not in the import's bytes: a raw read would miss + // them, the hook's read publishes them first. + if (Graphics::AnyShadowedOverlaps(address, bytes)) { + reason = &ValidateCounters::syncedShadow; + return Policy::Sync; + } + reason = &ValidateCounters::syncedOff; + if (!WriteEvidenceEnabled()) return Policy::Sync; + const auto writer = newestWriter(address, address + bytes); + if (!writer) { + reason = &ValidateCounters::syncedNoWriter; + return Policy::Sync; + } + if (writer->program == 0) { + reason = &ValidateCounters::syncedForeign; + return Policy::Sync; + } + if (writer->end - writer->begin > WriteEvidenceMaxBytes() || writer->begin < 4 || pending.Overlaps(writer->begin - 4, 4) || pending.Overlaps(writer->end, 4)) { + reason = &ValidateCounters::syncedLargeRange; + return Policy::Sync; + } + const auto first = address & ~3ull; + const auto limit = address + bytes; + for (std::uint64_t dword = first; dword < limit; dword += 4) { + if (Graphics::Recorder::LookupLabelValue(dword, 4, 0).has_value()) { + reason = &ValidateCounters::syncedLabel; + return Policy::Sync; + } + } + if (writer->value && writer->begin <= address && limit <= writer->end && (known.empty() || known.size() == bytes) && knownValueCurrent(*writer)) { + if (!known.empty()) { + std::memcpy(known.data(), writer->value->data() + (address - writer->begin), bytes); + knownValueReads.fetch_add(1, std::memory_order_relaxed); + } + return KnownValueVerify() ? Policy::VerifyKnownValue : Policy::KnownValue; + } + { + std::lock_guard lock(writtenBuffersMutex); + for (std::uint64_t dword = first; dword < limit; dword += 4) { + const auto found = dwordEvidence.find(dword); + if (found == dwordEvidence.end() || found->second.streak < WriteEvidenceAfter()) { + reason = &ValidateCounters::syncedEvidence; + return Policy::Sync; + } + if (WriterKeyedEvidence() && (found->second.program != writer->program || found->second.begin != writer->begin || found->second.end != writer->end)) { + reason = &ValidateCounters::syncedWriterChanged; + return Policy::Sync; + } + } + } + if (SampledRead()) { + reason = &ValidateCounters::syncedSample; + return Policy::Sync; + } + return ValidateSkipVerify() ? Policy::VerifyRaw : Policy::Raw; + } + static ShaderMemory::PendingWrite ClassifyPendingWrite(std::uint64_t address, std::size_t bytes, std::span known) { + std::uint64_t ValidateCounters::*reason = nullptr; + PendingView pending; + pending.Load(); + return Get().classifyPendingWrite(address, bytes, reason, pending, known); + } + std::string describeWriters(std::uint64_t begin, std::uint64_t end) { + std::string text; + std::lock_guard lock(writtenBuffersMutex); + int shown = 0; + for (auto it = writtenBuffers.rbegin(); it != writtenBuffers.rend() && shown < 3; ++it) { + if (begin >= it->end || it->begin >= end) continue; + char item[128]; + if (it->program == 0) std::snprintf(item, sizeof(item), " [foreign q0x%x 0x%llx+0x%llx age %llu]", it->queue, static_cast(it->begin), static_cast(it->end - it->begin), static_cast(writtenBufferSerial - it->serial)); + else std::snprintf(item, sizeof(item), " [0x%llx q0x%x 0x%llx+0x%llx%s%s age %llu]", static_cast(it->program), it->queue, static_cast(it->begin), static_cast(it->end - it->begin), it->atomic ? " atomic" : "", it->value ? " known" : "", static_cast(writtenBufferSerial - it->serial)); + text += item; + ++shown; + } + if (shown == 0) return " none"; + std::uint32_t streakMin = 0xffffffffu, streakMax = 0, changed = 0, unseen = 0; + for (std::uint64_t dword = begin & ~3ull; dword < end; dword += 4) { + const auto found = dwordEvidence.find(dword); + if (found == dwordEvidence.end()) { + ++unseen; + continue; + } + streakMin = std::min(streakMin, found->second.streak); + streakMax = std::max(streakMax, found->second.streak); + changed += found->second.changed; + } + char item[96]; + std::snprintf(item, sizeof(item), " dwords: streak %u..%u changed %u unseen %u", streakMin == 0xffffffffu ? 0u : streakMin, streakMax, changed, unseen); + return text + item; + } + static std::string describeSelf(const ShaderRecompiler::RecompileResult& compiled, std::uint64_t begin, std::uint64_t end) { + std::string text; + forEachWrittenBuffer(compiled, [&](std::uint32_t element, std::uint64_t base, std::uint64_t limit, bool atomic) { + if (begin >= limit || base >= end) return; + char item[96]; + std::snprintf(item, sizeof(item), " own element %u 0x%llx+0x%llx%s", element, static_cast(base), static_cast(limit - base), atomic ? " atomic" : ""); + text += item; + }); + return text; + } + static bool traceBudget() { + static std::atomic lines{0}; + return lines.fetch_add(1) < 600; + } + // A capture that waited: the captured regions recorded dispatches write, by page (what a + // whole-page fetch meets) and by the exact dwords. + void traceCapture(const char* what, std::uint64_t program, std::uint32_t queue, std::span regions, double waitedMs) { + if (waitedMs < 0.05 || !traceBudget()) return; + std::fprintf(stderr, "[capsync] %s q0x%x program 0x%llx waited %.1f ms over %zu regions\n", what, queue, static_cast(program), waitedMs, regions.size()); + for (const auto& region : regions) { + const auto begin = region.guestAddress; + const auto end = begin + region.bytes.size(); + const auto pageBegin = begin & ~static_cast(4095); + const auto pageEnd = (end + 4095) & ~static_cast(4095); + if (!newestWriter(pageBegin, pageEnd)) continue; + std::fprintf(stderr, "[capsync] region 0x%llx+0x%zx: writers of its page(s):%s; of its dwords:%s\n", static_cast(begin), region.bytes.size(), describeWriters(pageBegin, pageEnd).c_str(), describeWriters(begin, end).c_str()); + } + } + void reportValidation(ValidateCounters& counters) { + const auto count = [](std::uint64_t value) { return static_cast(value); }; + std::size_t tracked = 0, eligible = 0; + { + std::lock_guard lock(writtenBuffersMutex); + tracked = dwordEvidence.size(); + for (const auto& [address, evidence] : dwordEvidence) { + if (evidence.streak >= WriteEvidenceAfter()) ++eligible; + } + } + std::fprintf(stderr, "[validate] dispatch-cache compares over pending GPU writes (10 s): %llu; missed without the sync %llu, skipped the sync %llu, synced %llu waiting %.0f ms (no dispatch writer %llu, foreign writer %llu, larger range %llu, label %llu, evidence short %llu, writer changed %llu, sampled %llu, image %llu, shadow %llu, off %llu); known-value reads %llu; verify: %llu would-skip compares synced anyway (%.0f ms), %llu mismatches, %llu misses a sync made hits; dwords observed over waits: %llu unchanged, %llu changed (regions not observed: %llu after no GPU wait, %llu not compared); %zu tracked, %zu with a streak\n", count(counters.pending), count(counters.unsyncedMisses), count(counters.skipped), count(counters.syncedNoWriter + counters.syncedForeign + counters.syncedLargeRange + counters.syncedLabel + counters.syncedEvidence + counters.syncedWriterChanged + counters.syncedSample + counters.syncedImage + counters.syncedShadow + counters.syncedOff), counters.syncedWaitMs, count(counters.syncedNoWriter), count(counters.syncedForeign), count(counters.syncedLargeRange), count(counters.syncedLabel), count(counters.syncedEvidence), count(counters.syncedWriterChanged), count(counters.syncedSample), count(counters.syncedImage), count(counters.syncedShadow), count(counters.syncedOff), count(counters.knownValue), count(counters.verified), counters.verifiedWaitMs, count(counters.mismatches), count(counters.verifiedMissesHit), count(observedUnchanged.exchange(0)), count(observedChanged.exchange(0)), count(observationsNoWait.exchange(0)), count(observationsNotReached.exchange(0)), tracked, eligible); + counters = ValidateCounters{}; + } + // The stability compare of a freshly captured entry (bytes that already differ show a write + // racing the capture, and nothing is kept): the pending regions the capture just read raw are + // compared raw as well, the others through the hook as before. + bool captureStable(std::span captured) { + using Policy = ShaderMemory::PendingWrite; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + std::uint64_t ValidateCounters::*reason = &ValidateCounters::syncedOff; + bool stable = true; + std::uint64_t rawRegions = 0, mismatches = 0; + double waited = 0; + PendingView pending; + pending.Load(); + thread_local std::vector known; + for (const auto& region : captured) { + if (!stable) break; + known.resize(region.bytes.size()); + const auto policy = ValidateSkipEnabled() ? classifyPendingWrite(region.guestAddress, region.bytes.size(), reason, pending, known) : Policy::Sync; + if (policy == Policy::KnownValue || policy == Policy::VerifyKnownValue) { + stable = std::memcmp(known.data(), region.bytes.data(), known.size()) == 0; + if (policy != Policy::VerifyKnownValue) continue; + const bool synced = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + knownValueVerified.fetch_add(1, std::memory_order_relaxed); + if (synced != stable) knownValueMismatches.fetch_add(1, std::memory_order_relaxed); + stable = synced; + continue; + } + if (policy != Policy::Raw && policy != Policy::VerifyRaw) { + stable = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + continue; + } + ++rawRegions; + stable = GuestMemory::EqualsCommittedUnsynced(region.guestAddress, region.bytes); + if (policy != Policy::VerifyRaw) continue; + const auto start = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + const bool synced = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + if (profile) waited += Graphics::Recorder::ThreadWaitedMs() - start; + if (synced != stable) ++mismatches; + stable = synced; + } + if (!profile || rawRegions == 0) return stable; + std::lock_guard lock(validateMutex); + auto& counters = validateCounters; + ++counters.pending; + if (ValidateSkipVerify()) { + ++counters.verified; + counters.verifiedWaitMs += waited; + counters.mismatches += mismatches; + } else { + ++counters.skipped; + } + return stable; + } + // Compares a hit entry's captured regions with guest memory (true: unchanged). `inPlace`: the + // regions no recorded work writes are compared without the flush hook (validateEntry has + // checked the queued labels and the pending images for all of them already); a region of + // theirs, or of the raw-compared pending ones, over an unmapped page is a miss that also sets + // `unmapped` (validateVariant's Inaccessible). `view` is the caller's pending-write snapshot. + // `sampling`: the caller's evidence sample, drawn here at the first pending region if it has + // not been (one draw per validation, however many variants it compares). `data` (a variant + // with data positions): a differing dword of an in-place region at one of its positions + // (word positions over `captured` in order) is accepted and reported in `live`; the pending + // and known-value regions keep the exact compare. + struct DataMask { + std::span positions; + std::vector>* live; + }; + bool validateCaptured(std::uint64_t program, std::uint32_t queue, std::span captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped = nullptr, std::optional* sampling = nullptr, const DataMask* data = nullptr) { + using Policy = ShaderMemory::PendingWrite; + // Reused per thread: empty on nearly every hit, so it never allocates then. + thread_local std::vector pending; + pending.clear(); + for (std::size_t i = 0; i < captured.size(); ++i) { + const auto& region = captured[i]; + if (view.Overlaps(region.guestAddress, region.bytes.size())) pending.push_back(i); + } + // The masked re-read of a differing in-place region: its live bytes are copied (the same + // page-state walk as the compare, inside the caller's gate window) and compared dword by + // dword; a difference at a data position is recorded, any other is the miss. + const auto compareMasked = [&](const ShaderRecompiler::MemoryRegion& region, std::size_t first) { + thread_local std::vector live; + live.resize(region.bytes.size()); + const auto copied = GuestMemory::CopyMapped(region.guestAddress, live); + if (copied != GuestMemory::Compare::Equal) return copied; + const auto count = region.bytes.size() / sizeof(std::uint32_t); + for (std::size_t i = 0; i < count; ++i) { + std::uint32_t stored = 0, fresh = 0; + std::memcpy(&stored, region.bytes.data() + i * sizeof(std::uint32_t), sizeof(stored)); + std::memcpy(&fresh, live.data() + i * sizeof(std::uint32_t), sizeof(fresh)); + if (stored == fresh) continue; + const auto position = static_cast(first + i); + if (!std::binary_search(data->positions.begin(), data->positions.end(), position)) return GuestMemory::Compare::Differs; + data->live->emplace_back(position, fresh); + } + return GuestMemory::Compare::Equal; + }; + // Only the pending regions bypass the hook in a raw compare; the others keep the hook's + // image flush (they overlap no noted write, so it never waits for them) unless the caller + // compares in place. A pending region with a known value (`knownOffsets`, its bytes in + // `knownBytes`) is compared against that value in both passes, never through the hook (the + // value is what the GPU leaves; under the verify switch the hook's answer is taken too and + // a disagreement counted). Of the pending ones a hook compare reached (a differing region + // ends the compare), `observable` are those compared after a GPU wait on this thread since + // the copies below were taken: the only ones whose bytes before and after say anything. + static constexpr std::size_t NoKnownValue = std::numeric_limits::max(); + thread_local std::vector policies; + thread_local std::vector knownOffsets; + thread_local std::vector knownBytes; + policies.assign(pending.size(), Policy::None); + knownOffsets.assign(pending.size(), NoKnownValue); + knownBytes.clear(); + std::uint64_t knownServed = 0, knownMismatches = 0; + std::vector observable; + std::size_t reachedPending = 0; + std::uint64_t hookWaitsAtCopy = 0; + const auto compare = [&](bool rawPending) { + std::size_t next = 0; + std::size_t first = 0; + observable.clear(); + reachedPending = 0; + if (data != nullptr) data->live->clear(); + for (std::size_t i = 0; i < captured.size(); ++i) { + const auto& region = captured[i]; + const bool isPending = next < pending.size() && pending[next] == i; + if (isPending) ++next; + const bool unsynced = isPending ? rawPending : inPlace; + const bool known = isPending && knownOffsets[next - 1] != NoKnownValue; + bool same = false; + bool masked = false; + if (known) { + same = std::memcmp(knownBytes.data() + knownOffsets[next - 1], region.bytes.data(), region.bytes.size()) == 0; + if (!rawPending && policies[next - 1] == Policy::VerifyKnownValue) { + const bool synced = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + if (synced != same) ++knownMismatches; + same = synced; + } + } else if (!unsynced) same = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + else if (ValidateLegacy()) same = GuestMemory::EqualsCommittedUnsynced(region.guestAddress, region.bytes); + else { + auto outcome = GuestMemory::CompareMapped(region.guestAddress, region.bytes); + masked = !isPending && data != nullptr; + if (outcome == GuestMemory::Compare::Differs && masked) outcome = compareMasked(region, first); + if (outcome == GuestMemory::Compare::Unmapped && unmapped != nullptr) *unmapped = true; + same = outcome == GuestMemory::Compare::Equal; + } + first += region.bytes.size() / sizeof(std::uint32_t); + if (isPending && !rawPending && !known) { + ++reachedPending; + if (Graphics::Recorder::ThreadHookWaits() != hookWaitsAtCopy) observable.push_back(next - 1); + } + if (!same) { + if (data != nullptr && !masked) dataPendingMisses.fetch_add(1, std::memory_order_relaxed); + return false; + } + } + return true; + }; + if (pending.empty()) return compare(false); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const bool verify = ValidateSkipVerify(); + std::optional ownSampling; + auto& sample = sampling != nullptr ? *sampling : ownSampling; + if (!sample) sample.emplace(SiteSampling() ? evidenceValidations : evidenceReads); + std::uint64_t ValidateCounters::*reason = &ValidateCounters::syncedOff; + bool skip = ValidateSkipEnabled(); + // The policies first (the known values among them decide what a region is compared + // against); `reason` is the first sync's. + for (std::size_t i = 0; ValidateSkipEnabled() && i < pending.size(); ++i) { + const auto& region = captured[pending[i]]; + std::uint64_t ValidateCounters::*regionReason = &ValidateCounters::syncedOff; + const auto offset = knownBytes.size(); + knownBytes.resize(offset + region.bytes.size()); + auto policy = classifyPendingWrite(region.guestAddress, region.bytes.size(), regionReason, view, std::span(knownBytes).subspan(offset)); + const bool known = policy == Policy::KnownValue || policy == Policy::VerifyKnownValue; + if (!known) knownBytes.resize(offset); + if (policy != Policy::Sync && policy != Policy::None) { + // A publish of shadowed results is a recorded store over the region like a flush + // (over a known value too: it lands after the copy). + bool published = false; + const bool stored = Graphics::StorageTexture::FlushPending(region.guestAddress, region.bytes.size(), nullptr, "memory access", Graphics::PublishScope::Whole, &published); + if (stored || published) { + regionReason = &ValidateCounters::syncedImage; + policy = Policy::Sync; + if (known) knownBytes.resize(offset); + } else if (known) { + knownOffsets[i] = offset; + ++knownServed; + } + } + policies[i] = policy; + if (policy == Policy::Sync && skip) { + skip = false; + reason = regionReason; + } + } + // The raw compare first: words the CPU already changed are a miss whatever the GPU does. + const bool rawSame = skip ? compare(true) : false; + const bool unsyncedMiss = skip && !rawSame; + if (unsyncedMiss) skip = false; + const bool verifyKnown = KnownValueVerify() && knownServed != 0; + double waited = 0; + bool same = rawSame; + if (!(skip || unsyncedMiss) || verify || verifyKnown) { + // The pending regions' bytes before the wait, for the dword evidence. + std::vector> before; + if (WriteEvidenceEnabled()) { + before.reserve(pending.size()); + for (const auto index : pending) { + const auto& region = captured[index]; + std::vector copy; + if (GuestMemory::Accessible(reinterpret_cast(region.guestAddress), region.bytes.size())) { + copy.resize(region.bytes.size()); + std::memcpy(copy.data(), reinterpret_cast(region.guestAddress), copy.size()); + } + before.push_back(std::move(copy)); + } + } + const auto start = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + hookWaitsAtCopy = Graphics::Recorder::ThreadHookWaits(); + same = compare(false); + if (profile) waited = Graphics::Recorder::ThreadWaitedMs() - start; + if (!ObservationGuard()) { + for (std::size_t i = 0; i < before.size(); ++i) observeRange(captured[pending[i]].guestAddress, before[i]); + } else if (!before.empty()) { + for (const auto index : observable) observeRange(captured[pending[index]].guestAddress, before[index]); + observationsNoWait.fetch_add(reachedPending - observable.size(), std::memory_order_relaxed); + observationsNotReached.fetch_add(pending.size() - knownServed - reachedPending, std::memory_order_relaxed); + } + } + if (verifyKnown) { + knownValueVerified.fetch_add(knownServed, std::memory_order_relaxed); + knownValueMismatches.fetch_add(knownMismatches, std::memory_order_relaxed); + } + if (profile) { + std::lock_guard lock(validateMutex); + auto& counters = validateCounters; + ++counters.pending; + counters.knownValue += knownServed; + if (verify && (skip || unsyncedMiss)) { + ++counters.verified; + counters.verifiedWaitMs += waited; + if (skip && !same) ++counters.mismatches; + if (unsyncedMiss && same) ++counters.verifiedMissesHit; + } else if (unsyncedMiss) { + ++counters.unsyncedMisses; + } else if (skip) { + ++counters.skipped; + } else { + ++(counters.*reason); + counters.syncedWaitMs += waited; + } + if (std::chrono::steady_clock::now() - counters.lastReport > std::chrono::seconds(10)) reportValidation(counters); + } + if (TraceCapSync() && traceBudget()) { + const char* outcome = verify && (skip || unsyncedMiss) ? "verify" : unsyncedMiss ? "missed raw" : skip ? "skipped" : reason == &ValidateCounters::syncedEvidence ? "synced (evidence short)" : reason == &ValidateCounters::syncedSample ? "synced (sampled)" : reason == &ValidateCounters::syncedNoWriter ? "synced (no dispatch writer)" : reason == &ValidateCounters::syncedForeign ? "synced (foreign writer)" : reason == &ValidateCounters::syncedWriterChanged ? "synced (writer changed)" : reason == &ValidateCounters::syncedLargeRange ? "synced (larger range)" : reason == &ValidateCounters::syncedLabel ? "synced (label)" : reason == &ValidateCounters::syncedImage ? "synced (image)" : reason == &ValidateCounters::syncedShadow ? "synced (shadow)" : "synced (off)"; + for (const auto index : pending) { + const auto& region = captured[index]; + const auto begin = region.guestAddress; + const auto end = begin + region.bytes.size(); + std::fprintf(stderr, "[capsync] dispatch-cache q0x%x program 0x%llx region 0x%llx+0x%zx (%zu of %zu pending) %s: waited %.1f ms, same %d raw %d;%s; writers:%s\n", queue, static_cast(program), static_cast(begin), region.bytes.size(), pending.size(), captured.size(), outcome, waited, same ? 1 : 0, rawSame ? 1 : 0, describeSelf(compiled, begin, end).c_str(), describeWriters(begin, end).c_str()); + } + } + if (verify || verifyKnown) return same; + return skip || unsyncedMiss ? rawSame : same; + } + // EqualData: equal apart from data words (a hit with the live words in the caller's list). + enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved }; + // The captured dwords of a variant as regions over its stored values (or over `words`, a + // data hit's live words in the same layout), in address order. + static void appendEntryRegions(const DispatchVariant& variant, std::vector& regions, const std::vector* words = nullptr) { + const auto& source = words != nullptr ? *words : variant.words; + regions.reserve(regions.size() + variant.runs.size()); + std::size_t offset = 0; + for (const auto& [begin, end] : variant.runs) { + const auto count = static_cast((end - begin) / sizeof(std::uint32_t)); + regions.push_back({begin, std::as_bytes(std::span(source).subspan(offset, count))}); + offset += count; + } + } + // The runs of a variant recorded GPU work still writes whose pending-write policy is a sync: + // waited for through the flush hook BEFORE the gate below loads its serials, so the sync's + // own publish precedes them and each such run is then compared in place with the others + // (compared through validateCaptured after the loads, every such hit ended as PublishMoved). + // The bytes before the wait feed the dword evidence as validateCaptured's compare does, and + // under the skip policy a run the CPU already changed is a miss without the wait (false). + // `pending` is reloaded after every sync (the sync's finish publishes a snapshot without the + // range), so the later runs are not synced against a stale one. `sampling` is the + // validation's one evidence sample (validateCaptured), drawn here if a pending run comes first. + bool syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span regions, std::uint64_t& synced, PendingView& pending, std::optional& sampling) { + using Policy = ShaderMemory::PendingWrite; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + for (const auto& region : regions) { + if (!pending.Overlaps(region.guestAddress, region.bytes.size())) continue; + if (!sampling) sampling.emplace(SiteSampling() ? evidenceValidations : evidenceReads); + std::uint64_t ValidateCounters::*reason = &ValidateCounters::syncedOff; + const auto policy = ValidateSkipEnabled() ? classifyPendingWrite(region.guestAddress, region.bytes.size(), reason, pending) : Policy::Sync; + if (policy != Policy::Sync) continue; + if (ValidateSkipEnabled() && (ValidateLegacy() ? !GuestMemory::EqualsCommittedUnsynced(region.guestAddress, region.bytes) : GuestMemory::CompareMapped(region.guestAddress, region.bytes) != GuestMemory::Compare::Equal)) { + if (profile) { + std::lock_guard lock(validateMutex); + ++validateCounters.pending; + ++validateCounters.unsyncedMisses; + } + return false; + } + std::vector before; + if (WriteEvidenceEnabled() && GuestMemory::Accessible(reinterpret_cast(region.guestAddress), region.bytes.size())) { + before.resize(region.bytes.size()); + std::memcpy(before.data(), reinterpret_cast(region.guestAddress), before.size()); + } + const auto waitsBefore = Graphics::Recorder::ThreadHookWaits(); + const bool timed = profile || TraceCapSync(); + const auto waitedBefore = timed ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + GuestMemory::FlushGpuWrites(region.guestAddress, region.bytes.size()); + const auto waited = timed ? Graphics::Recorder::ThreadWaitedMs() - waitedBefore : 0.0; + pending.Load(); + if (!before.empty()) { + if (!ObservationGuard() || Graphics::Recorder::ThreadHookWaits() != waitsBefore) observeRange(region.guestAddress, before); + else observationsNoWait.fetch_add(1, std::memory_order_relaxed); + } + ++synced; + if (profile) { + std::lock_guard lock(validateMutex); + auto& counters = validateCounters; + ++counters.pending; + ++(counters.*reason); + counters.syncedWaitMs += waited; + if (std::chrono::steady_clock::now() - counters.lastReport > std::chrono::seconds(10)) reportValidation(counters); + } + if (TraceCapSync() && traceBudget()) { + const auto begin = region.guestAddress; + const auto end = begin + region.bytes.size(); + std::fprintf(stderr, "[capsync] dispatch-cache q0x%x program 0x%llx region 0x%llx+0x%zx synced before the gate: waited %.1f ms;%s; writers:%s\n", queue, static_cast(program), static_cast(begin), region.bytes.size(), waited, describeSelf(compiled, begin, end).c_str(), describeWriters(begin, end).c_str()); + } + } + return true; + } + // Whether a variant still describes the memory its capture read: its words are compared with + // guest memory in place, and the compare counts only if nothing that could have changed the + // bytes under it moved meanwhile: the pending-write snapshot (Recorder::PublishGeneration, + // bumped after every publish), the pending storage registry (StorageTexture::PendingSerial, + // bumped after every store) and the page mappings (GuestMemory::ForgetSerial, odd while a + // forget stores). The serials are loaded before the registry is scanned, so a mutation between + // the two is caught by one test or the other. A run recorded GPU work still writes is synced + // first (syncPendingRuns); one over an image whose results are pending is stored first + // (FlushPending, the serials taken again behind the store); one over an image another thread + // is storing right now, or under a label this worker still has queued (the miss's capture + // reads it through the hook, which records the label first), is a miss, as is a page that is + // not mapped. A miss never yields a wrong hit: a writer ordered before this dispatch had + // published its note, marked its image or forgotten its pages before the gate's first load, + // so the compare sees its bytes; one landing during the compare is unordered on hardware too + // and still moves one of the serials. Each variant of an entry is gated over ITS runs (never + // the union: a synced run only a non-matching variant reads must not wait); `sampling` is the + // validation's one evidence sample, shared by every variant it compares. `live` (a dispatch + // variant): the compare accepts differing data words (DataMask) and reports them there; a + // hit with any is EqualData. The live words are read inside the same gate window, by the + // same page-checked path, as the words the compare accepts. + EntryOutcome validateVariant(std::uint64_t program, std::uint32_t queue, const DispatchVariant& variant, std::span regions, std::uint64_t& imagesFlushed, std::uint64_t& runsSynced, std::optional& sampling, std::vector>* live = nullptr) { + if (live != nullptr) live->clear(); + const bool masked = live != nullptr && !variant.dataPositions.empty(); + const DataMask mask{variant.dataPositions, live}; + PendingView pending; + pending.Load(); + if (!syncPendingRuns(program, queue, *variant.compiled, regions, runsSynced, pending, sampling)) return EntryOutcome::Differing; + auto publish = Graphics::Recorder::PublishGeneration(); + auto pendingSerial = Graphics::StorageTexture::PendingSerial(); + auto forget = GuestMemory::ForgetSerial(); + // Results a unit shadow holds over a run are not in the import's bytes either: the flush + // publishes them (a recorded store the reload below then syncs on). + if (Graphics::StorageTexture::AnyPendingOverlaps(variant.runs) || Graphics::AnyShadowedOverlaps(variant.runs)) { + for (const auto& region : regions) { + if (Graphics::StorageTexture::FlushPending(region.guestAddress, region.bytes.size(), nullptr, "dispatch-cache entry")) ++imagesFlushed; + } + pending.Load(); + if (!syncPendingRuns(program, queue, *variant.compiled, regions, runsSynced, pending, sampling)) return EntryOutcome::Differing; + publish = Graphics::Recorder::PublishGeneration(); + pendingSerial = Graphics::StorageTexture::PendingSerial(); + forget = GuestMemory::ForgetSerial(); + if (Graphics::StorageTexture::AnyPendingOverlaps(variant.runs) || Graphics::AnyShadowedOverlaps(variant.runs)) return EntryOutcome::FlushingImage; + } + if ((forget & 1) != 0) return EntryOutcome::ForgetMoved; + for (const auto& label : deferredLabels().labels) { + for (const auto& [begin, end] : variant.runs) { + if (label.address < end && begin < label.address + label.size) return EntryOutcome::QueuedLabel; + } + } + if (ValidateLegacy()) { + for (const auto& region : regions) { + if (!GuestMemory::Accessible(reinterpret_cast(region.guestAddress), region.bytes.size())) return EntryOutcome::Inaccessible; + } + } + // The snapshot the compare consults: loaded after the final generation load above. + pending.Load(); + bool unmapped = false; + if (!validateCaptured(program, queue, regions, *variant.compiled, true, pending, &unmapped, &sampling, masked ? &mask : nullptr)) return unmapped ? EntryOutcome::Inaccessible : EntryOutcome::Differing; + if (Graphics::Recorder::PublishGeneration() != publish) return EntryOutcome::PublishMoved; + if (Graphics::StorageTexture::PendingSerial() != pendingSerial) return EntryOutcome::PendingMoved; + if (GuestMemory::ForgetSerial() != forget) return EntryOutcome::ForgetMoved; + return masked && !live->empty() ? EntryOutcome::EqualData : EntryOutcome::Equal; + } + // A 'differing' miss's fresh variant against the front variant it goes ahead of (under + // dispatchCacheMutex; see EntryCounters): the differing dwords are classified by matching + // their fresh values against the capture's V#/T# base words (address) and its flattened SRT + // (data). + void classifyDiffering(std::uint64_t program, std::uint64_t key, const DispatchVariant& old, const DispatchVariant& fresh, const ShaderRecompiler::ResourceCapture* capture, EntryCounters& counters) { + ++counters.differingClassified; + ++counters.differingByProgram[program]; + auto& ring = priorValueSets[key]; + if (std::any_of(ring.begin(), ring.end(), [&](const ValueSet& set) { return set.first == fresh.runs && set.second == fresh.words; })) ++counters.differingMatchedPrior; + ring.emplace_front(old.runs, old.words); + while (ring.size() > 3) ring.pop_back(); + if (priorValueSets.size() > DispatchCacheEntries()) priorValueSets.clear(); + if (old.runs != fresh.runs || old.words.size() != fresh.words.size()) { + ++counters.differingRunsChanged; + ++counters.differingWalk; + return; + } + const auto baseWord = [&](const std::vector& values, std::uint32_t value) { + return std::any_of(values.begin(), values.end(), [&](const ShaderRecompiler::DescriptorValue& descriptor) { return descriptor.dwordCount >= 2 && (descriptor.dwords[0] == value || descriptor.dwords[1] == value); }); + }; + std::size_t words = 0, addressWords = 0, dataWords = 0; + for (std::size_t i = 0; i < fresh.words.size(); ++i) { + if (old.words[i] == fresh.words[i]) continue; + ++words; + counters.differingPositions.insert(i); + counters.differingFirstPosition = std::min(counters.differingFirstPosition, i); + counters.differingLastPosition = std::max(counters.differingLastPosition, i); + const auto value = fresh.words[i]; + if (capture != nullptr && (baseWord(capture->snapshot.buffers, value) || baseWord(capture->snapshot.images, value))) ++addressWords; + else if (capture != nullptr && std::find(capture->snapshot.flattenedSrt.begin(), capture->snapshot.flattenedSrt.end(), value) != capture->snapshot.flattenedSrt.end()) ++dataWords; + } + counters.differingWords += words; + ++counters.differingWordBuckets[words <= 1 ? 0 : words <= 4 ? 1 : words <= 16 ? 2 : 3]; + if (words == 0) ++counters.differingWalk; + else if (addressWords == words) ++counters.differingAddress; + else if (dataWords == words) ++counters.differingData; + else if (addressWords + dataWords == 0) ++counters.differingWalk; + else ++counters.differingMixed; + } + void reportDispatchCache(EntryCounters& counters) { + const auto count = [](std::uint64_t value) { return static_cast(value); }; + const auto validated = counters.lookups - counters.absent; + std::fprintf(stderr, "[dispatch-cache] %llu lookups (10 s): %llu no entry, %llu validated by value in %.1f us each: %llu equal, %llu differing, %llu page not mapped, %llu queued label, %llu image being stored, %llu publish generation moved, %llu pending serial moved, %llu forget serial moved (%llu pending images stored first, %llu pending runs synced first, %llu equal entries inserted before a forget; gate retried %llu: %llu equal, %llu moved again); runs per entry %.1f validated / %.1f inserted; %llu misses found the entry replaced meanwhile; %llu inserts (%llu captures not kept: unstable), %llu evictions in total, %zu entries, %llu LRU moves\n", count(counters.lookups), count(counters.absent), count(validated), validated != 0 ? counters.validateUs / static_cast(validated) : 0.0, count(counters.equal), count(counters.differing), count(counters.inaccessible), count(counters.queuedLabel), count(counters.flushingImage), count(counters.publishMoved), count(counters.pendingMoved), count(counters.forgetMoved), count(counters.imagesFlushed), count(counters.runsSynced), count(counters.forgetSinceInsert), count(counters.retriesEqual + counters.retriesMoved), count(counters.retriesEqual), count(counters.retriesMoved), validated != 0 ? static_cast(counters.runsValidated) / static_cast(validated) : 0.0, counters.inserts != 0 ? static_cast(counters.runsInserted) / static_cast(counters.inserts) : 0.0, count(counters.replaced), count(counters.inserts), count(counters.unstable), count(dispatchCacheEvictions), dispatchCache.size(), count(counters.touches)); + std::vector> programs(counters.differingByProgram.begin(), counters.differingByProgram.end()); + std::sort(programs.begin(), programs.end(), [](const auto& a, const auto& b) { return a.second > b.second; }); + std::string top; + for (std::size_t i = 0; i < programs.size() && i < 8; ++i) { + char text[48]; + std::snprintf(text, sizeof(text), " 0x%llx x%llu", static_cast(programs[i].first), count(programs[i].second)); + top += text; + } + std::fprintf(stderr, "[dispatch-cache] differing by class (10 s, %llu classified): address-only %llu, data-only %llu, walk %llu, mixed %llu (runs changed %llu); matched one of the last 3 value sets %llu; differing words %llu in total (misses with 1 / 2-4 / 5-16 / >16 words: %llu / %llu / %llu / %llu), %zu distinct positions (first %zu, last %zu); top programs by differing:%s\n", count(counters.differingClassified), count(counters.differingAddress), count(counters.differingData), count(counters.differingWalk), count(counters.differingMixed), count(counters.differingRunsChanged), count(counters.differingMatchedPrior), count(counters.differingWords), count(counters.differingWordBuckets[0]), count(counters.differingWordBuckets[1]), count(counters.differingWordBuckets[2]), count(counters.differingWordBuckets[3]), counters.differingPositions.size(), counters.differingPositions.empty() ? std::size_t{0} : counters.differingFirstPosition, counters.differingLastPosition, top.c_str()); + std::string ranks; + for (std::size_t rank = 0; rank < DispatchVariants(); ++rank) { + char text[32]; + std::snprintf(text, sizeof(text), "%s%llu", rank == 0 ? "" : " / ", count(counters.variantHitsByRank[rank])); + ranks += text; + } + std::fprintf(stderr, "[dispatch-cache] variants (10 s, k = %zu): hits by rank 1..k %s; %.2f variants compared per validation; %llu inserted into an entry, %llu evicted beyond k; %.2f variants per entry (%llu over %zu entries, ~%.1f MiB)\n", DispatchVariants(), ranks.c_str(), validated != 0 ? static_cast(counters.variantsCompared) / static_cast(validated) : 0.0, count(counters.variantsInserted), count(counters.variantsEvicted), dispatchCache.empty() ? 0.0 : static_cast(dispatchCacheVariants) / static_cast(dispatchCache.size()), count(dispatchCacheVariants), dispatchCache.size(), static_cast(dispatchCacheVariantBytes) / (1024.0 * 1024.0)); + std::string dataRanks; + for (std::size_t rank = 0; rank < DispatchVariants(); ++rank) { + char text[32]; + std::snprintf(text, sizeof(text), "%s%llu", rank == 0 ? "" : " / ", count(counters.dataHitsByRank[rank])); + dataRanks += text; + } + std::fprintf(stderr, "[dispatch-cache] data hits (10 s): %llu (%llu words refreshed; by rank 1..k %s; %llu data variants missed on pending runs), verified %llu; inserts with data positions %llu of %llu (%.1f positions each), leaves skipped: unmapped %llu, mismatched %llu, aliased %llu\n", count(counters.dataHits), count(counters.dataWordsRefreshed), dataRanks.c_str(), count(dataPendingMisses.exchange(0, std::memory_order_relaxed)), count(counters.dataVerified), count(counters.dataInserts), count(counters.inserts), counters.dataInserts != 0 ? static_cast(counters.dataPositionsInserted) / static_cast(counters.dataInserts) : 0.0, count(counters.dataLeavesUnmapped), count(counters.dataLeavesMismatched), count(counters.dataLeavesAliased)); + counters = EntryCounters{}; + } + + // APS5_VERIFY_DATA_HITS=1: the miss path's capture and recompile beside a data-only hit. The + // read set (the captured regions outside the code and header), its bytes, the variant and the + // bindings must agree with what the hit reused, else the process aborts: a disagreement would + // be a stale read set served silently. + void verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, ShaderRecompiler::RecompileRequest request, std::span memory, std::uint64_t address, const DispatchVariant& variant, std::span liveWords, const ShaderRecompiler::RecompileResult& patched) { + const auto fail = [&](const char* what, std::size_t position, std::size_t slot) { + std::fprintf(stderr, "[dispatch-cache] APS5_VERIFY_DATA_HITS: %s disagrees (program 0x%llx, position %zu, slot %zu)\n", what, static_cast(address), position, slot); + std::fflush(stderr); + std::abort(); + }; + auto verifyMemory = std::make_shared(memory, &ClassifyPendingWrite, &ObservePendingWrite, HookWaitCounter()); + const auto handle = SourceHandleFor(snapshot, codeOffset, deviceSerial, request, false); + const auto capture = verifyMemory->Capture(request, handle.get()); + const auto regions = verifyMemory->Regions(); + request.context.memory = regions; + const auto result = ShaderRecompiler::Recompile(request, *capture); + std::size_t position = 0, run = 0; + for (const auto& region : regions) { + const bool registered = std::any_of(memory.begin(), memory.end(), [&](const auto& known) { return region.guestAddress >= known.guestAddress && region.guestAddress < known.guestAddress + known.bytes.size(); }); + if (registered) continue; + if (run >= variant.runs.size() || variant.runs[run].first != region.guestAddress || variant.runs[run].second != region.guestAddress + region.bytes.size()) fail("the read set", position, run); + const auto count = region.bytes.size() / sizeof(std::uint32_t); + if (position + count > liveWords.size() || std::memcmp(region.bytes.data(), liveWords.data() + position, region.bytes.size()) != 0) fail("the captured bytes", position, run); + position += count; + ++run; + } + if (run != variant.runs.size()) fail("the read set", position, run); + if (result->variantId != patched.variantId) fail("the variant", 0, 0); + if (result->bindings.size() != patched.bindings.size() || result->pushConstants != patched.pushConstants) fail("the bindings", 0, 0); + for (std::size_t b = 0; b < result->bindings.size(); ++b) { + const auto& fresh = result->bindings[b]; + const auto& reused = patched.bindings[b]; + if (fresh.kind != reused.kind || fresh.role != reused.role || fresh.binding != reused.binding || fresh.count != reused.count || fresh.guestDescriptor != reused.guestDescriptor) fail("the bindings", b, 0); + } + std::lock_guard cacheLock(dispatchCacheMutex); + ++entryCounters.dataVerified; + } + // `indirectArguments` (non-zero): a DISPATCH_INDIRECT whose group counts, at that guest address, // are not read here; `packet` then carries only the initiator (see dispatchIndirect). void dispatch(QueueState& queue, std::span packet, const Submission& submission, std::uint64_t indirectArguments = 0) { const auto address = (static_cast(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast(readRegister(queue.shader, 0x20d) & 0xffu) << 40u); - auto it = submission.shaders.upper_bound(address); - require(it != submission.shaders.begin(), "compute program does not belong to a registered shader"); + auto it = submission.shaders->upper_bound(address); + require(it != submission.shaders->begin(), "compute program does not belong to a registered shader"); --it; const auto& snapshot = *it->second; require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "compute program is outside registered shader code"); @@ -861,7 +3075,14 @@ private: packet = resolved; indirectArguments = 0; } - if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) return; + if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) { + pendingDispatchPhases().outcome = DispatchOutcome::FillHle; + return; + } + if (indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { + pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; + return; + } ShaderRecompiler::RecompileRequest request{ {ShaderRecompiler::ShaderStage::Compute, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header}, {(packet[4] & 0x8000u) != 0 ? 32u : 64u, 0, userData, compute, std::nullopt, std::nullopt, memory}, @@ -879,6 +3100,15 @@ private: lap = now; return ms; }; + std::array phaseMs{}; + auto phaseLap = lap; + if (profile && packetStartedAt() != std::chrono::steady_clock::time_point{}) phaseMs[PhasePrologue] = std::chrono::duration(lap - packetStartedAt()).count(); + const auto phase = [&](DriverPhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + phaseMs[which] += std::chrono::duration(now - phaseLap).count(); + phaseLap = now; + }; // Debug aid: APS5_NO_DISPATCH_CACHE=1 captures and recompiles every dispatch. static const bool noDispatchCacheEnv = std::getenv("APS5_NO_DISPATCH_CACHE") != nullptr; // Debug aid: APS5_PROBE_DISPATCH=: applies the recompiler's APS5_PROBE @@ -899,6 +3129,16 @@ private: probeThis = dispatchesSeen.fetch_add(1) == probeDispatch.second; if (probeThis) std::fprintf(stderr, "[gpu] probing dispatch %llu of 0x%llx\n", static_cast(probeDispatch.second), static_cast(address)); } + // A program whose plan build failed before (the poisoned source-handle memo) is skipped + // here, before the key, the capture, the ShaderMemory and the lock, without a throw; the + // first failure took the full path and reported it. A probed dispatch bypasses the memo. + if (FailureMemo() && snapshot.handles->poisoned.load(std::memory_order_relaxed) != 0) { + const std::string* poisoned = nullptr; + if (SourceHandleFor(snapshot, codeOffset, localDevice->Serial(), request, probeThis, &poisoned) == nullptr && poisoned != nullptr) { + pendingDispatchPhases().outcome = DispatchOutcome::SkippedMemo; + return; + } + } const bool noDispatchCache = noDispatchCacheEnv || probeThis; std::uint64_t key = 0xcbf29ce484222325ull; const auto mix = [&](std::uint64_t value) { @@ -925,12 +3165,30 @@ private: } } std::shared_ptr compiledResult; - std::shared_ptr keepMemory; - // The captured code and header spans point into the entry's snapshot, which may differ - // from this submission's (re-registered, identical bytes); a hit keeps it past eviction. - std::shared_ptr keepShader; + // A hit's captured regions point into the variant (its stored words; under stamp + // validation its capture's pages and its snapshot's code): the variant is kept past + // eviction. + std::shared_ptr keepVariant; + // A miss's inserted variant (or the equal one another worker inserted meanwhile): the + // recipe the device call builds is attached to it. + std::shared_ptr attachVariant; + // A miss's captured regions point into its capture's pages, and the snapshots made from + // them are compared by an address-based build's Complete() under the device lock: the + // capture lives for the whole dispatch, not only until the entry is made from it. + std::shared_ptr shaderMemory; std::vector captured; + // A data-only hit: the matched variant's words with the differing data words at their + // live values; `captured` points into them, so they live for the whole dispatch (an + // address-based build's Complete() compares the snapshots under the device lock). + std::vector liveWords; + bool dataHit = false; bool cached = false; + bool validated = false; + // A miss keeps the entry it validated: the fresh variant goes in front of its variants. + // A 'differing' one also keeps its own capture for classifyDiffering (profile only). + std::shared_ptr missedEntry; + bool missedDiffering = false; + std::shared_ptr capture; // Debug aid: APS5_TRACE_DISPATCH_CACHE reports, per program, what changed between dispatches. static const bool traceCache = std::getenv("APS5_TRACE_DISPATCH_CACHE") != nullptr; if (traceCache) { @@ -963,75 +3221,214 @@ private: previous.shader = std::map(queue.shader.begin(), queue.shader.end()); previous.key = key; } + // The key names the registered shader object (RegisterShader makes a new one per + // registration, identical bytes included), so an entry's code and header are never + // compared: the recompile result was made from that object's copy of them. Under stamp + // validation the entry compares them itself, as before. + if (!StampValidate()) mix(reinterpret_cast(it->second.get())); + phase(PhaseKey); if (!noDispatchCache) { - // The entry is copied out and validated without the cache lock: CollectWrites and - // EqualsCommitted (which can wait for the GpuMutex through the flush hook) would - // otherwise stall every other queue's lookup behind this one. The write watch keeps - // concurrent validations of one entry correct, so the re-lock only has to skip an entry - // another worker changed meanwhile. Debug aid: APS5_NO_UNLOCKED_VALIDATE=1 holds the lock. + // The entry is validated without the cache lock: the compares (which can wait for the + // GpuMutex through the flush hook over a pending write) would otherwise stall every + // other queue's lookup behind this one. An entry is immutable once inserted (a + // re-stamp is an atomic store), so the re-lock only has to skip an entry another worker + // replaced meanwhile. Debug aid: APS5_NO_UNLOCKED_VALIDATE=1 holds the lock. static const bool validateUnlocked = std::getenv("APS5_NO_UNLOCKED_VALIDATE") == nullptr; std::unique_lock cacheLock(dispatchCacheMutex); - if (const auto found = dispatchCache.find(key); found != dispatchCache.end()) { - auto entry = found->second; - if (validateUnlocked) cacheLock.unlock(); - bool current = true; - for (const auto& [begin, bytes] : entry.spans) { - GuestMemory::CollectWrites(begin, bytes); - if (!GuestMemory::UnchangedSince(begin, bytes, entry.generation)) { - current = false; - break; - } - } + ++entryCounters.lookups; + const auto found = dispatchCache.find(key); + std::shared_ptr entry = found != dispatchCache.end() ? found->second : nullptr; + if (entry == nullptr) ++entryCounters.absent; + if (validateUnlocked) cacheLock.unlock(); + phase(PhaseLookup); + if (entry != nullptr) { + validated = true; + const auto& variants = entry->variants; + // The matched variant and its rank; the front variant under stamp validation. + std::shared_ptr variant; + std::size_t rank = 0; + const auto generation = variants.front()->generation.load(std::memory_order_acquire); + bool current = false; std::uint64_t restamped = 0; - if (!current) { - // A write landed in a watched span (spans merge regions up to 64 KiB apart, so it - // is often unrelated): the captured bytes themselves decide, and an unchanged - // capture is re-stamped at a fresh generation. The generation is collected - // BEFORE the compare, as on insert: a write landing after the collect dirties - // the pages again and fails the next validation, whereas one landing between a - // compare and a later collect would be stamped into the entry and never seen. - std::uint64_t generation = 0; - for (const auto& [begin, bytes] : entry.spans) generation = std::max(generation, GuestMemory::CollectWrites(begin, bytes)); - bool same = generation != 0; + // The front variant's outcome, as the single entry's was; Equal for any hit. + auto outcome = EntryOutcome::Differing; + std::uint64_t imagesFlushed = 0, runsSynced = 0, retriesEqual = 0, retriesMoved = 0, compared = 0; + std::vector regions; + // The differing data words of the matched variant (position, live word). + std::vector> liveData; + const auto waitedBeforeValidate = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + if (!StampValidate()) { const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); - for (const auto& region : entry.captured) { - if (!same) break; - same = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + // One evidence sample for the whole validation, drawn at its first pending run. + std::optional sampling; + // Most recently hit first; the first Equal is the hit. A Differing variant says + // nothing about the others (they are independent), so every one is compared + // before a miss. + for (std::size_t i = 0; i < variants.size(); ++i) { + regions.clear(); + appendEntryRegions(*variants[i], regions); + ++compared; + auto result = validateVariant(address, submission.queue, *variants[i], regions, imagesFlushed, runsSynced, sampling, &liveData); + // A serial that moved during an otherwise equal compare means "compare + // again", not "changed": one fresh pass of the same gate, a second move is + // the miss. + if (GateRetry() && (result == EntryOutcome::PublishMoved || result == EntryOutcome::PendingMoved)) { + result = validateVariant(address, submission.queue, *variants[i], regions, imagesFlushed, runsSynced, sampling, &liveData); + ++(result == EntryOutcome::Equal || result == EntryOutcome::EqualData ? retriesEqual : retriesMoved); + } + if (i == 0) outcome = result; + if (result == EntryOutcome::Equal || result == EntryOutcome::EqualData) { + outcome = result; + variant = variants[i]; + rank = i; + dataHit = result == EntryOutcome::EqualData; + break; + } } - if (same) { - restamped = generation; - current = true; + current = variant != nullptr; + } else { + variant = variants.front(); + compared = 1; + current = true; + for (const auto& [begin, bytes] : variant->spans) { + GuestMemory::CollectWrites(begin, bytes); + if (!GuestMemory::UnchangedSince(begin, bytes, generation)) { + current = false; + break; + } } + if (!current) { + // A write landed in a watched span (spans merge regions up to 64 KiB apart, + // so it is often unrelated): the captured bytes themselves decide, and an + // unchanged capture is re-stamped at a fresh generation. The generation is + // collected BEFORE the compare, as on insert: a write landing after the + // collect dirties the pages again and fails the next validation, whereas + // one landing between a compare and a later collect would be stamped into + // the entry and never seen. + std::uint64_t collected = 0; + for (const auto& [begin, bytes] : variant->spans) collected = std::max(collected, GuestMemory::CollectWrites(begin, bytes)); + bool same = collected != 0; + if (same) { + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); + PendingView pending; + pending.Load(); + same = validateCaptured(address, submission.queue, variant->captured, *variant->compiled, false, pending); + } + if (same) { + restamped = collected; + current = true; + } + } + if (current) outcome = EntryOutcome::Equal; + } + phase(PhaseValidate); + if (profile) { + const auto waited = std::min(Graphics::Recorder::ThreadWaitedMs() - waitedBeforeValidate, phaseMs[PhaseValidate]); + phaseMs[PhaseValidate] -= waited; + phaseMs[PhaseValidateWait] += waited; } if (!cacheLock.owns_lock()) cacheLock.lock(); - // Only an entry still at the copied generation is re-stamped or erased; the validated - // copy is used either way. + // Only the entry still in the map, at the copied generation, is re-stamped or + // moved; the validated variant is used either way. const auto again = dispatchCache.find(key); - const bool untouched = again != dispatchCache.end() && again->second.generation == entry.generation; + const bool untouched = again != dispatchCache.end() && again->second == entry && variants.front()->generation.load(std::memory_order_acquire) == generation; + auto& counters = entryCounters; + counters.validateUs += phaseMs[PhaseValidate] * 1000; + counters.imagesFlushed += imagesFlushed; + counters.runsSynced += runsSynced; + for (std::size_t i = 0; i < compared && i < variants.size(); ++i) counters.runsValidated += variants[i]->runs.size(); + counters.retriesEqual += retriesEqual; + counters.retriesMoved += retriesMoved; + counters.variantsCompared += compared; + switch (outcome) { + case EntryOutcome::Equal: ++counters.equal; break; + case EntryOutcome::EqualData: ++counters.equal; break; + case EntryOutcome::Differing: ++counters.differing; break; + case EntryOutcome::Inaccessible: ++counters.inaccessible; break; + case EntryOutcome::QueuedLabel: ++counters.queuedLabel; break; + case EntryOutcome::FlushingImage: ++counters.flushingImage; break; + case EntryOutcome::PublishMoved: ++counters.publishMoved; break; + case EntryOutcome::PendingMoved: ++counters.pendingMoved; break; + case EntryOutcome::ForgetMoved: ++counters.forgetMoved; break; + } if (current) { - compiledResult = std::move(entry.compiled); - keepMemory = std::move(entry.memory); - keepShader = std::move(entry.shader); - captured = std::move(entry.captured); + ++counters.variantHitsByRank[rank]; + compiledResult = variant->compiled; + if (dataHit) { + // The live words: the regions, the snapshots and the label check see them + // as a fresh capture would. The compiled result is a copy with the + // FlattenedSrt binding patched to them; the variant id, hence the + // pipeline, is unchanged (no pure word reaches the specialization). The + // recipe or the template hit then refreshes the data buffers by words. + liveWords = variant->words; + for (const auto& [position, value] : liveData) liveWords[position] = value; + regions.clear(); + appendEntryRegions(*variant, regions, &liveWords); + auto patched = std::make_shared(*variant->compiled); + auto& descriptor = patched->bindings[variant->flatBinding].guestDescriptor; + for (std::size_t k = 0; k < variant->dataPositions.size(); ++k) { + if (variant->dataSlots[k] < descriptor.size()) descriptor[variant->dataSlots[k]] = liveWords[variant->dataPositions[k]]; + } + compiledResult = std::move(patched); + ++counters.dataHits; + counters.dataWordsRefreshed += liveData.size(); + ++counters.dataHitsByRank[rank]; + } + if (StampValidate()) { + captured = variant->captured; + } else { + // `regions` holds the matched variant's (the loop stopped at it). + captured.reserve(memory.size() + regions.size()); + captured.assign(memory.begin(), memory.end()); + captured.insert(captured.end(), regions.begin(), regions.end()); + if (variant->forgetSerial != GuestMemory::ForgetSerial()) ++counters.forgetSinceInsert; + } + keepVariant = variant; cached = true; ++dispatchCacheHits; if (untouched) { - if (restamped != 0) again->second.generation = restamped; - dispatchOrder.splice(dispatchOrder.begin(), dispatchOrder, again->second.order); + if (restamped != 0) variant->generation.store(restamped, std::memory_order_release); + // A hit behind the front reorders by replacing the entry with a rotated + // copy (the entry is immutable). + if (rank != 0) { + auto rotated = std::make_shared(); + rotated->variants.reserve(variants.size()); + rotated->variants.push_back(variant); + for (std::size_t i = 0; i < variants.size(); ++i) { + if (i != rank) rotated->variants.push_back(variants[i]); + } + rotated->touched = entry->touched; + rotated->order = entry->order; + again->second = std::move(rotated); + } + // The move to the front is amortized: an entry hit within the newest eighth + // of the cache's worth of hits stays where it is. + if (dispatchCacheHits - again->second->touched > DispatchCacheEntries() / 8) { + dispatchOrder.splice(dispatchOrder.begin(), dispatchOrder, again->second->order); + again->second->touched = dispatchCacheHits; + ++counters.touches; + } } } else { if (traceCache) std::fprintf(stderr, "[dispatch-cache] 0x%llx captured memory changed\n", static_cast(address)); - if (untouched) { - dispatchOrder.erase(again->second.order); - dispatchCache.erase(again); - } + // The entry stays until the insert puts the fresh variant in front of its + // variants (a slot another worker replaced meanwhile is as good). + missedEntry = entry; + missedDiffering = outcome == EntryOutcome::Differing; + if (!untouched) ++counters.replaced; } + if (profile && std::chrono::steady_clock::now() - counters.lastReport > std::chrono::seconds(10)) { + counters.lastReport = std::chrono::steady_clock::now(); + reportDispatchCache(counters); + } + phase(PhaseRelock); } } if (cached) { captureMs += elapsed(); } else { - auto shaderMemory = std::make_shared(memory); + shaderMemory = std::make_shared(memory, &ClassifyPendingWrite, &ObservePendingWrite, HookWaitCounter()); + std::uint64_t forgetAtCapture = 0; // Debug aid: APS5_DUMP_SHADERS=1 saves every request, failing ones included, for agc_shader_replay. static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; try { @@ -1042,17 +3439,26 @@ private: explicit ProbeScope(bool active) : active(active) { if (active) ShaderRecompiler::SetDebugProbeActive(true); } ~ProbeScope() { if (active) ShaderRecompiler::SetDebugProbeActive(false); } } probeScope{probeThis}; - const auto capture = shaderMemory->Capture(request); + const auto waitedBefore = TraceCapSync() ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + forgetAtCapture = GuestMemory::ForgetSerial(); + const auto handle = SourceHandleFor(snapshot, codeOffset, localDevice->Serial(), request, probeThis); + capture = [&] { + const SampledReadScope sampling(evidenceReads); + return shaderMemory->Capture(request, handle.get()); + }(); captured = shaderMemory->Regions(); request.context.memory = captured; + if (TraceCapSync()) traceCapture("dispatch-capture", address, submission.queue, captured, Graphics::Recorder::ThreadWaitedMs() - waitedBefore); captureMs += elapsed(); + phase(PhaseCapture); if (dumpShaders) static_cast(dumpRequest(address, request)); const auto started = std::chrono::steady_clock::now(); // The recompile reuses the capture's plan and materialization rather than walking // the captured regions again. Debug aid: APS5_NO_CAPTURE_REUSE=1 walks them again. static const bool reuseCapture = std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; - compiledResult = std::make_shared(reuseCapture ? ShaderRecompiler::Recompile(request, *capture) : ShaderRecompiler::Recompile(request)); - if (compiledResult->cacheHit) ++cacheHits; + bool memoHit = false; + compiledResult = reuseCapture ? ShaderRecompiler::Recompile(request, *capture, &memoHit) : std::make_shared(ShaderRecompiler::Recompile(request)); + if (compiledResult->cacheHit || memoHit) ++cacheHits; const auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); static double totalMs = 0; totalMs += elapsed; @@ -1068,53 +3474,136 @@ private: throw std::runtime_error(where + reason); } recompileMs += elapsed(); + phase(PhaseRecompile); if (!noDispatchCache) { - // The captured regions, merged into spans when close, are watched for writes from now - // on; bytes that already differ show a write racing the capture, and nothing is kept. - DispatchEntry fresh{shaderMemory, captured, compiledResult, {}, 0, it->second}; + auto fresh = std::make_shared(); + fresh->compiled = compiledResult; + fresh->shader = it->second; + fresh->forgetSerial = forgetAtCapture; + // The captured dwords outside the registered code and header (the capture served + // those from the registration, so no page region lies inside them), as the entry + // stores them. + std::vector srtRegions; for (const auto& region : captured) { - const auto begin = region.guestAddress; - const auto end = begin + region.bytes.size(); - if (!fresh.spans.empty() && begin <= fresh.spans.back().first + fresh.spans.back().second + 65536) { - auto& last = fresh.spans.back(); - last.second = static_cast(std::max(last.first + last.second, end) - last.first); - } else { - fresh.spans.emplace_back(begin, static_cast(end - begin)); + const bool registered = std::any_of(memory.begin(), memory.end(), [&](const auto& known) { return region.guestAddress >= known.guestAddress && region.guestAddress < known.guestAddress + known.bytes.size(); }); + if (registered) continue; + srtRegions.push_back(region); + fresh->runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size()); + const auto count = region.bytes.size() / sizeof(std::uint32_t); + const auto offset = fresh->words.size(); + fresh->words.resize(offset + count); + std::memcpy(fresh->words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t)); + } + // The positions of the pure flat-SRT leaves among the stored words (data-only + // hits): none under stamp validation, without a FlattenedSrt binding, or when the + // capture traced no leaf (no pure slot in the plan). + DataWordPositionCounts dataCounts; + if (DataHits() && !StampValidate() && capture != nullptr && !capture->readTrace.leaves.empty()) { + for (std::size_t b = 0; b < compiledResult->bindings.size(); ++b) { + if (compiledResult->bindings[b].role != ShaderRecompiler::DescriptorRole::FlattenedSrt) continue; + fresh->flatBinding = static_cast(b); + dataCounts = DataWordPositions(fresh->runs, capture->readTrace.leaves, capture->readTrace.otherReads, fresh->words, compiledResult->bindings[b].guestDescriptor, fresh->dataPositions, fresh->dataSlots); + break; } } - std::uint64_t generation = 0; - for (const auto& [begin, bytes] : fresh.spans) generation = std::max(generation, GuestMemory::CollectWrites(begin, bytes)); - bool stable = generation != 0; - const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); - for (const auto& region : captured) { - if (!stable) break; - stable = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + bool stable = true; + if (StampValidate()) { + // The captured regions, merged into spans when close, are watched for writes + // from now on; bytes that already differ show a write racing the capture, and + // nothing is kept. + fresh->memory = shaderMemory; + fresh->captured = captured; + for (const auto& region : captured) { + const auto begin = region.guestAddress; + const auto end = begin + region.bytes.size(); + if (!fresh->spans.empty() && begin <= fresh->spans.back().first + fresh->spans.back().second + 65536) { + auto& last = fresh->spans.back(); + last.second = static_cast(std::max(last.first + last.second, end) - last.first); + } else { + fresh->spans.emplace_back(begin, static_cast(end - begin)); + } + } + std::uint64_t generation = 0; + for (const auto& [begin, bytes] : fresh->spans) generation = std::max(generation, GuestMemory::CollectWrites(begin, bytes)); + stable = generation != 0; + if (stable) { + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); + stable = captureStable(captured); + } + fresh->generation.store(generation, std::memory_order_release); + } else if (InsertCompare()) { + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); + stable = captureStable(srtRegions); } + std::lock_guard cacheLock(dispatchCacheMutex); + if (profile && missedEntry != nullptr && missedDiffering) classifyDiffering(address, key, *missedEntry->variants.front(), *fresh, capture.get(), entryCounters); if (stable) { - fresh.generation = generation; - std::lock_guard cacheLock(dispatchCacheMutex); - // Another worker may have inserted the key meanwhile; its entry is as good. - const auto [inserted, isNew] = dispatchCache.try_emplace(key, std::move(fresh)); - if (isNew) { + ++entryCounters.inserts; + entryCounters.runsInserted += fresh->runs.size(); + if (!fresh->dataPositions.empty()) { + ++entryCounters.dataInserts; + entryCounters.dataPositionsInserted += fresh->dataPositions.size(); + } + entryCounters.dataLeavesUnmapped += dataCounts.unmapped; + entryCounters.dataLeavesMismatched += dataCounts.mismatched; + entryCounters.dataLeavesAliased += dataCounts.aliased; + attachVariant = fresh; + const auto found = dispatchCache.find(key); + if (found == dispatchCache.end()) { + auto created = std::make_shared(); + created->touched = dispatchCacheHits; + accountVariant(*fresh, true); + created->variants.push_back(std::move(fresh)); dispatchOrder.push_front(key); - inserted->second.order = dispatchOrder.begin(); + created->order = dispatchOrder.begin(); + dispatchCache.emplace(key, std::move(created)); + } else if (found->second != missedEntry && !found->second->variants.empty() && found->second->variants.front()->runs == fresh->runs && found->second->variants.front()->words == fresh->words) { + // Another worker inserted this value set meanwhile; its variant is as good. + attachVariant = found->second->variants.front(); + } else { + // The fresh variant goes in front of the slot's variants (the ones this + // miss validated, or another worker's: as good); the least recently hit + // beyond DispatchVariants() go. The entry is replaced whole. + const auto& old = found->second; + auto replacement = std::make_shared(); + replacement->variants.reserve(DispatchVariants()); + accountVariant(*fresh, true); + replacement->variants.push_back(std::move(fresh)); + for (const auto& kept : old->variants) { + if (replacement->variants.size() < DispatchVariants()) { + replacement->variants.push_back(kept); + } else { + accountVariant(*kept, false); + ++entryCounters.variantsEvicted; + } + } + ++entryCounters.variantsInserted; + replacement->touched = dispatchCacheHits; + replacement->order = old->order; + dispatchOrder.splice(dispatchOrder.begin(), dispatchOrder, replacement->order); + found->second = std::move(replacement); } // Debug aid: APS5_NO_DISPATCH_LRU=1 clears the cache when it fills, as before. static const bool lru = std::getenv("APS5_NO_DISPATCH_LRU") == nullptr; - if (dispatchCache.size() > DispatchCacheEntries) { + if (dispatchCache.size() > DispatchCacheEntries()) { if (lru) { - dispatchCache.erase(dispatchOrder.back()); - dispatchOrder.pop_back(); + eraseDispatchEntry(dispatchCache.find(dispatchOrder.back())); ++dispatchCacheEvictions; } else { dispatchCacheEvictions += dispatchCache.size(); dispatchCache.clear(); dispatchOrder.clear(); + dispatchCacheVariants = 0; + dispatchCacheVariantBytes = 0; } } + } else { + ++entryCounters.unstable; } + phase(PhaseInsert); } } + if (VerifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, localDevice->Serial(), request, memory, address, *keepVariant, liveWords, *compiledResult); // A label this queue still has queued over a captured region: recorded now, and the dispatch // starts over (the cache entry just validated or inserted fails its next validation, the // label store marks the tracker). The queue is empty after, so this happens once. @@ -1122,6 +3611,7 @@ private: dispatch(queue, packet, submission, indirectArguments); return; } + phase(PhaseQueuedLabels); const auto& compiled = *compiledResult; std::vector snapshots; for (const auto& region : captured) snapshots.push_back({region.guestAddress, region.bytes}); @@ -1152,33 +3642,116 @@ private: std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast(address)); throw std::runtime_error(where + std::string(error.what())); }; - // Stage A of the resource build (copies, buffers, the descriptor set) runs before the - // lock, which then covers only stage B (texture lookups, imports, descriptor writes) and - // the record: a compute queue's build no longer stalls the graphics queue for its whole - // resources phase. Null when the resource cache may serve this dispatch, or with - // APS5_LOCKED_BUILD=1 (the whole build under the lock as before). - std::shared_ptr prepared; - try { - prepared = localDevice->PrepareDispatch(compiled, snapshots); - } catch (const std::exception& error) { - rethrow(error); + phase(PhaseSnapshots); + // The matched variant's recipe (design_cpu_final M4): a value-equal hit whose variant + // carries one skips PrepareDispatch (the content key, the cache find, stage A), the + // pipeline lookup and the data-word compare; its pre-checks run here without the mutex, + // its proof under it. Stamp validation keeps no recipes (one variant, the stamps' path). + // APS5_NO_DISPATCH_RECIPE=1 takes the ordinary path for every hit. + std::shared_ptr recipeHit; + if (cached && keepVariant != nullptr && !StampValidate()) { + recipeHit = localDevice->PrepareRecipe(keepVariant->recipe.load(std::memory_order_acquire), indirectArguments != 0); + phase(PhaseRecipePrecheck); } - GuestMemory::TagGpuLockSite(indirectArguments != 0 ? GuestMemory::GpuLockSite::Indirect : GuestMemory::GpuLockSite::Dispatch); - std::lock_guard gpuLock(GuestMemory::GpuMutex()); - // This queue's labels first (queue order), and their batch out before the dispatch. Not - // under the shader's name: a label that fails to record is not a shader failure. - recordLabelsForPacket(localDevice.get(), submission.queue); - try { - if (indirectArguments != 0) { - const auto outcome = localDevice->DispatchIndirect(compiled, indirectArguments, snapshots, address, std::move(prepared)); - countIndirect(outcome.cpuReason, outcome.argumentReadMs); - } else { - localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared)); + // The recipe the device call builds for the variant (a hit without a usable recipe, or + // the miss's fresh variant); attached after the call. + std::shared_ptr builtRecipe; + auto* const attachTo = StampValidate() ? nullptr : keepVariant != nullptr ? keepVariant.get() : attachVariant.get(); + bool writersNoted = false; + const bool noteWrites = WriteEvidenceEnabled() || TraceCapSync(); + // A Rebuild from the recipe path (a failed proof, a replaced device) restarts here at + // PrepareDispatch with the mutex released, so stage A stays unlocked; once at most, since + // the second pass has no recipe. + for (;;) { + // Stage A of the resource build (copies, buffers, the descriptor set) runs before the + // lock, which then covers only stage B (texture lookups, imports, descriptor writes) and + // the record: a compute queue's build no longer stalls the graphics queue for its whole + // resources phase. Null when the resource cache may serve this dispatch, or with + // APS5_LOCKED_BUILD=1 (the whole build under the lock as before). A recipe hit skips + // it, except under APS5_VERIFY_RECIPE=1 (the prepared object is then compared). + std::shared_ptr prepared; + if (recipeHit == nullptr || VulkanDevice::VerifyRecipes()) { + try { + prepared = localDevice->PrepareDispatch(compiled, snapshots); + } catch (const std::exception& error) { + rethrow(error); + } + if (profile) { + const auto now = std::chrono::steady_clock::now(); + const auto prepareMs = std::chrono::duration(now - phaseLap).count(); + phaseLap = now; + double parts = 0; + if (prepared != nullptr) { + const auto phases = VulkanDevice::PreparePhaseMs(*prepared); + phaseMs[PhasePrepareKey] += phases[0]; + phaseMs[PhasePrepareFind] += phases[1]; + phaseMs[PhasePreparePrecollect] += phases[2]; + phaseMs[PhasePreparePresync] += phases[3]; + phaseMs[PhasePrepareStageA] += phases[4]; + for (const auto part : phases) parts += part; + } + phaseMs[PhasePrepareOther] += std::max(0.0, prepareMs - parts); + } } - } catch (const std::exception& error) { - rethrow(error); + GuestMemory::TagGpuLockSite(indirectArguments != 0 ? GuestMemory::GpuLockSite::Indirect : GuestMemory::GpuLockSite::Dispatch); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + phase(PhaseLockWait); + // This queue's labels first (queue order), and their batch out before the dispatch. Not + // under the shader's name: a label that fails to record is not a shader failure. + recordLabelsForPacket(localDevice.get(), submission.queue); + phase(PhaseLabels); + if (noteWrites && WriterKeyedEvidence() && !writersNoted) { + noteWrittenBuffers(address, submission.queue, compiled); + writersNoted = true; + } + phase(PhaseNoteWriters); + try { + if (recipeHit != nullptr) { + VulkanDevice::IndirectOutcome outcome{0, 0}; + const auto result = localDevice->DispatchRecipe(compiled, groups[0], groups[1], groups[2], indirectArguments, address, recipeHit, outcome, VulkanDevice::VerifyRecipes() ? prepared : nullptr, dataHit); + if (result == RecipeOutcome::Rebuild) { + // Nothing was recorded: the ordinary path, re-attaching its recipe. + recipeHit = nullptr; + VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Restart, indirectArguments != 0); + continue; + } + if (indirectArguments != 0) countIndirect(outcome.cpuReason, outcome.argumentReadMs); + } else if (indirectArguments != 0) { + const auto outcome = localDevice->DispatchIndirect(compiled, indirectArguments, snapshots, address, std::move(prepared), attachTo != nullptr ? &builtRecipe : nullptr); + countIndirect(outcome.cpuReason, outcome.argumentReadMs); + } else { + localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), attachTo != nullptr ? &builtRecipe : nullptr); + } + } catch (const std::exception& error) { + rethrow(error); + } + if (builtRecipe != nullptr) { + if (dataHit) { + // Built from the patched result: the hash must name the variant's own words, + // else a later exact hit of the variant would find the template's (this hit's) + // words "the same" and skip the refresh back. + auto own = std::make_shared(*builtRecipe); + own->dataWordsHash = Graphics::ShaderResources::DataWordsHash({ShaderRecompiler::ShaderStage::Compute, keepVariant->compiled.get(), 0}); + builtRecipe = std::move(own); + } + attachTo->recipe.store(std::move(builtRecipe), std::memory_order_release); + VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Attach, indirectArguments != 0); + } + break; } + phase(PhaseDevice); + if (noteWrites && !WriterKeyedEvidence()) noteWrittenBuffers(address, submission.queue, compiled); deviceMs += elapsed(); + phase(PhaseTail); + if (profile) { + auto& pending = pendingDispatchPhases(); + pending.outcome = DispatchOutcome::Real; + pending.phases = true; + pending.hit = cached; + pending.validated = validated; + pending.ms = phaseMs; + pending.tailAt = phaseLap; + } if (profile && ++dispatches % 100 == 0) std::fprintf(stderr, "[gpu] %llu dispatches (%llu dispatch cache hits, %llu evictions, %llu recompile cache hits): capture %.1f s, cache key %.1f s, recompile %.1f s, device %.1f s\n", static_cast(dispatches), static_cast(dispatchCacheHits), static_cast(dispatchCacheEvictions), static_cast(cacheHits), captureMs / 1000, keyMs / 1000, recompileMs / 1000, deviceMs / 1000); } @@ -1210,109 +3783,70 @@ private: dispatch(queue, unresolved, submission, arguments); } - void draw(QueueState& queue, std::span packet, const Submission& submission) { + // Drawn; Nothing: the packet draws nothing (an empty count, or no color writes and no pixel + // shader); Rejected: the register precheck refused it before the decode, `rejected` holding the + // message the decode would have thrown (APS5_NO_DRAW_PRECHECK=1 leaves every rejection to the + // decode's throw, as before). + enum class DrawVerdict { Drawn, Nothing, Rejected }; + static bool DrawPrecheck() { + static const bool precheck = std::getenv("APS5_NO_DRAW_PRECHECK") == nullptr; + return precheck; + } + DrawVerdict draw(QueueState& queue, std::span packet, const Submission& submission, std::string& rejected) { PerformanceTimer timing("Driver.Draw"); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + std::array phaseMs{}; + std::uint64_t captures = 0; + auto phaseLap = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + if (profile && packetStartedAt() != std::chrono::steady_clock::time_point{}) phaseMs[DrawRowPrologue] = std::chrono::duration(phaseLap - packetStartedAt()).count(); + const auto phase = [&](DrawDriverPhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + phaseMs[which] += std::chrono::duration(now - phaseLap).count(); + phaseLap = now; + }; + // The packet loop reads the phases after the call (see pendingDrawPhases). + const auto drawn = [&] { + phase(DrawRowVectors); + if (!profile) return DrawVerdict::Drawn; + auto& pending = pendingDrawPhases(); + pending.phases = true; + pending.captures = captures; + pending.ms = phaseMs; + pending.tailAt = phaseLap; + return DrawVerdict::Drawn; + }; auto drawParameters = Pm4::ResolveDraw(packet, queue); - if (!drawParameters.indexed && (drawParameters.indexCount == 0 || drawParameters.instanceCount == 0)) return; + if (!drawParameters.indirect && !drawParameters.indexed && (drawParameters.indexCount == 0 || drawParameters.instanceCount == 0)) return DrawVerdict::Nothing; + static const bool traceIndirect = std::getenv("APS5_TRACE_INDIRECT_DRAWS") != nullptr; + if (traceIndirect && drawParameters.indirect) std::fprintf(stderr, "[draw] indirect packet %s args 0x%llx count %u reached\n", Pm4::Name(packet[0]).c_str(), static_cast(drawParameters.indirect->arguments), drawParameters.indirect->count); // Depth/stencil-only passes (no color writes, no pixel shader) have no effect without depth // targets, which are not emulated. { const auto targetMask = queue.context.find(0x8e); const auto shaderMask = queue.context.find(0x8f); const bool colorWrites = targetMask != queue.context.end() && shaderMask != queue.context.end() && (targetMask->second & shaderMask->second) != 0; - if (!colorWrites && !queue.shader.contains(0x8)) return; + if (!colorWrites && !queue.shader.contains(0x8)) return DrawVerdict::Nothing; } - const auto graphics = Graphics::DecodeState(queue); - struct Program { - ShaderRecompiler::ShaderBinary binary; - std::uint32_t userDataBase; - std::uint32_t firstUserSgpr = 8; - std::vector userData; - std::array memory; - }; - const auto programAddress = [&](std::uint32_t base) { - const auto high = readRegister(queue.shader, base + 1); - require((high & ~0xffu) == 0, "reserved graphics program address bits are set"); - return (static_cast(readRegister(queue.shader, base)) << 8u) | (static_cast(high) << 40u); - }; - const auto prepare = [&](std::uint64_t address, std::uint8_t type, ShaderRecompiler::ShaderStage stage, std::uint32_t rsrc2, std::uint32_t userDataBase) { - auto it = submission.shaders.upper_bound(address); - require(it != submission.shaders.begin(), "graphics program does not belong to a registered shader"); - --it; - const auto& snapshot = *it->second; - require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "graphics program is outside registered shader code"); - require(snapshot.type == type, "graphics program refers to an incompatible shader binary type"); - const auto resources = readRegister(queue.shader, rsrc2); - const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u); - require(userCount <= 32, "graphics user SGPR count exceeds the register bank"); - const auto codeOffset = static_cast((address - snapshot.codeAddress) / sizeof(std::uint32_t)); - Program result{ - {stage, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header}, - userDataBase, - 8, - {}, - {{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}} - }; - for (std::uint32_t i = 0; i < userCount; ++i) result.userData.push_back(readRegister(queue.shader, userDataBase + i)); - return result; - }; + if (DrawPrecheck()) { + rejected = Graphics::DrawRejection(queue); + if (!rejected.empty()) return DrawVerdict::Rejected; + } + phase(DrawRowPrecheck); using Stage = ShaderRecompiler::ShaderStage; using Role = ShaderRecompiler::ProgramRole; - std::vector programs; - std::vector roles; - programs.reserve(5); - roles.reserve(5); - const auto append = [&](std::uint32_t base, std::uint8_t type, Stage stage, std::uint32_t resources, std::uint32_t users, Role role) { - programs.push_back(prepare(programAddress(base), type, stage, resources, users)); - roles.push_back(role); - }; - const auto initializeMerged = [&](Program& program, std::uint32_t pointerBase, bool pointerRequired) { - program.firstUserSgpr = 0; - program.userData.insert(program.userData.begin(), 8, 0); - if (pointerRequired) { - const auto low = readRegister(queue.shader, pointerBase); - const auto high = readRegister(queue.shader, pointerBase + 1); - const auto address = static_cast(low) | (static_cast(high) << 32u); - require(address != 0, "merged shader user-data address is null"); - GuestMemory::CheckRange(reinterpret_cast(address), 8, 4); - program.userData[0] = low; - program.userData[1] = high; - } - }; - if (graphics.stages.path == Graphics::ShaderPath::Tessellation) { - append(0x148, 5, Stage::Local, 0x10b, 0x10c, Role::Local); - append(0x108, 7, Stage::TessellationControl, 0x10b, 0x10c, Role::Hull); - initializeMerged(programs.back(), 0x102, true); - append(0x0c8, 2, Stage::TessellationEvaluation, 0x08b, 0x08c, Role::Domain); - } else if (graphics.stages.path == Graphics::ShaderPath::Geometry) { - const auto frontAddress = programAddress(0xc8); - auto snapshot = submission.shaders.upper_bound(frontAddress); - require(snapshot != submission.shaders.begin(), "geometry front program is not registered"); - --snapshot; - const auto type = snapshot->second->type; - require(type == 2 || type == 4, "invalid geometry front binary type"); - append(0xc8, type, Stage::Mesh, 0x8b, 0x8c, Role::Main); - initializeMerged(programs.back(), 0x82, type == 4); - if (type == 4) append(0x88, 6, Stage::Mesh, 0x8b, 0x8c, Role::GeometryBack); - } else { - append(0xc8, 2, Stage::Vertex, 0x8b, 0x8c, Role::Main); - } - append(0x008, 1, Stage::Fragment, 0x00b, 0x00c, Role::Fragment); - programs.back().firstUserSgpr = 0; - const auto pixel = Graphics::DecodePixelStageInfo(queue.context, graphics.hasColorTarget, graphics.color.componentMapping); - std::vector memory; - std::vector linked; - for (std::size_t i = 0; i < programs.size(); ++i) { - const auto& program = programs[i]; - memory.insert(memory.end(), program.memory.begin(), program.memory.end()); - linked.push_back({roles[i], program.binary, program.userDataBase, program.firstUserSgpr, program.userData}); - } - timing.Mark("prepare"); + // Debug aid: APS5_DUMP_DRAW_SHADERS= saves the requests of draws into that + // target as shader_
.req for agc_shader_replay. + static const std::uint64_t dumpTarget = [] { const char* text = std::getenv("APS5_DUMP_DRAW_SHADERS"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); + // Debug aid: APS5_DUMP_DRAW_SLOT1= saves the requests and registers of draws whose + // second color target (slot 1) is at that address, as shader_
.req and draw_slot1.regs. + static const std::uint64_t dumpSlot1 = [] { const char* text = std::getenv("APS5_DUMP_DRAW_SLOT1"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); // As in dispatch, the capture, the recompiles and the rect-list shaders run without the // GpuMutex (guest memory is read through the self-locking flush hook, the recompiler has its // own locks); only the device creation and the device Draw take it, so the compute workers // do not pile up behind a draw's whole preparation. The device is held by this shared_ptr - // meanwhile (DevicePointer's arrow form is for holders of the mutex only). Debug aid: + // meanwhile (DevicePointer's arrow form is for holders of the mutex only); it is taken + // before the decode because the draw key names its serial. Debug aid: // APS5_LOCKED_DRAW_PREPARE=1 holds the mutex across the preparation as before. static const bool lockedPrepare = std::getenv("APS5_LOCKED_DRAW_PREPARE") != nullptr; std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); @@ -1323,10 +3857,12 @@ private: GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); gpuLock.lock(); timing.Mark("gpu_mutex_wait"); + phase(DrawRowLockWait); if (device == nullptr) device = std::make_shared(); localDevice = device; // This queue's labels first (queue order), as at the unlocked path's acquisition below. recordLabelsForPacket(localDevice.get(), submission.queue); + phase(DrawRowLabels); } else if ((localDevice = device.load()) == nullptr) { GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); std::lock_guard createLock(GuestMemory::GpuMutex()); @@ -1334,37 +3870,474 @@ private: localDevice = device; } timing.Mark("device_setup"); - ShaderMemory shaderMemory(memory); + phase(DrawRowVectors); + // The draw cache lookup (see drawCache): under the register key the entry is found before + // any decode and its decode products serve the draw; every stage is validated over its own + // runs under the DrawCache read site, the matched variants kept for the whole draw (a hit's + // regions, results and vertex infos point into them). + const bool useDrawEntries = DrawEntries() && !ShaderRecompiler::DebugProbeActive() && dumpTarget == 0 && dumpSlot1 == 0; + const bool registerKey = useDrawEntries && RegisterKey(); + std::uint64_t drawKey = 0; + std::shared_ptr entry; + std::shared_ptr decode; + if (registerKey) { + const auto keyStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + drawKey = drawRegisterKey(queue, *submission.shaders, localDevice->Serial()); + std::lock_guard cacheLock(drawCacheMutex); + ++drawEntryCounters.lookups; + ++drawEntryCounters.registerKeyLookups; + if (profile) drawEntryCounters.keyUs += std::chrono::duration(std::chrono::steady_clock::now() - keyStart).count(); + const auto found = drawCache.find(drawKey); + if (found != drawCache.end()) { + entry = found->second; + decode = entry->decode; + } else { + ++drawEntryCounters.absent; + } + } + phase(DrawRowKeyLookupValidate); + // The decode (DecodeState with DecodeShaderStages, the program prepare, DecodePixelStageInfo), + // skipped when the entry carries it. In verify mode (APS5_VERIFY_DRAW_RECIPE=1) it runs on + // every draw: with the register facade's log on, checked against DrawKeyRegisters, and + // compared with the entry's decode when there is one (the fresh decode is used then). + const auto programAddress = [&](std::uint32_t base) { + Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, base); + Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, base + 1); + const auto high = readRegister(queue.shader, base + 1); + require((high & ~0xffu) == 0, "reserved graphics program address bits are set"); + return (static_cast(readRegister(queue.shader, base)) << 8u) | (static_cast(high) << 40u); + }; + const auto prepare = [&](std::uint64_t address, std::uint8_t type, Stage stage, std::uint32_t rsrc2, std::uint32_t userDataBase) { + auto it = submission.shaders->upper_bound(address); + require(it != submission.shaders->begin(), "graphics program does not belong to a registered shader"); + --it; + const auto& snapshot = *it->second; + require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "graphics program is outside registered shader code"); + require(snapshot.type == type, "graphics program refers to an incompatible shader binary type"); + Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, rsrc2); + const auto resources = readRegister(queue.shader, rsrc2); + const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u); + require(userCount <= 32, "graphics user SGPR count exceeds the register bank"); + const auto codeOffset = static_cast((address - snapshot.codeAddress) / sizeof(std::uint32_t)); + DrawProgram result{ + {stage, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header}, + userDataBase, + 8, + {}, + {{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}, + it->second, + codeOffset + }; + for (std::uint32_t i = 0; i < userCount; ++i) { + Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, userDataBase + i); + result.userData.push_back(readRegister(queue.shader, userDataBase + i)); + } + return result; + }; + const auto decodeAll = [&] { + auto product = std::make_shared(); + product->state = Graphics::DecodeState(queue); + auto& programs = product->programs; + auto& roles = product->roles; + programs.reserve(5); + roles.reserve(5); + const auto append = [&](std::uint32_t base, std::uint8_t type, Stage stage, std::uint32_t resources, std::uint32_t users, Role role) { + programs.push_back(prepare(programAddress(base), type, stage, resources, users)); + roles.push_back(role); + }; + const auto initializeMerged = [&](DrawProgram& program, std::uint32_t pointerBase, bool pointerRequired) { + program.firstUserSgpr = 0; + program.userData.insert(program.userData.begin(), 8, 0); + if (pointerRequired) { + Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, pointerBase); + Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, pointerBase + 1); + const auto low = readRegister(queue.shader, pointerBase); + const auto high = readRegister(queue.shader, pointerBase + 1); + const auto address = static_cast(low) | (static_cast(high) << 32u); + require(address != 0, "merged shader user-data address is null"); + GuestMemory::CheckRange(reinterpret_cast(address), 8, 4); + program.userData[0] = low; + program.userData[1] = high; + } + }; + const auto& graphics = product->state; + if (graphics.stages.path == Graphics::ShaderPath::Tessellation) { + append(0x148, 5, Stage::Local, 0x10b, 0x10c, Role::Local); + append(0x108, 7, Stage::TessellationControl, 0x10b, 0x10c, Role::Hull); + initializeMerged(programs.back(), 0x102, true); + append(0x0c8, 2, Stage::TessellationEvaluation, 0x08b, 0x08c, Role::Domain); + } else if (graphics.stages.path == Graphics::ShaderPath::Geometry) { + const auto frontAddress = programAddress(0xc8); + auto snapshot = submission.shaders->upper_bound(frontAddress); + require(snapshot != submission.shaders->begin(), "geometry front program is not registered"); + --snapshot; + const auto type = snapshot->second->type; + require(type == 2 || type == 4, "invalid geometry front binary type"); + append(0xc8, type, Stage::Mesh, 0x8b, 0x8c, Role::Main); + initializeMerged(programs.back(), 0x82, type == 4); + if (type == 4) append(0x88, 6, Stage::Mesh, 0x8b, 0x8c, Role::GeometryBack); + } else { + append(0xc8, 2, Stage::Vertex, 0x8b, 0x8c, Role::Main); + } + append(0x008, 1, Stage::Fragment, 0x00b, 0x00c, Role::Fragment); + programs.back().firstUserSgpr = 0; + product->pixel = Graphics::DecodePixelStageInfo(queue.context, graphics.hasColorTarget, graphics.color.componentMapping); + return product; + }; + if (decode == nullptr || VerifyDrawRecipe()) { + std::vector readLog; + struct LogScope { + explicit LogScope(std::vector* log) { Graphics::RegisterReadLog() = log; } + ~LogScope() { Graphics::RegisterReadLog() = nullptr; } + } logScope(VerifyDrawRecipe() && registerKey ? &readLog : nullptr); + auto fresh = decodeAll(); + if (VerifyDrawRecipe() && registerKey) { + std::uint64_t facadeMismatches = 0; + for (const auto read : readLog) { + if (Graphics::DrawKeyCovers(read)) continue; + ++facadeMismatches; + static std::atomic reports{0}; + if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: the decoders read %s register 0x%x, which DrawKeyRegisters lacks\n", Graphics::RegisterBankName(read.bank), read.offset); + } + std::uint64_t decodeMismatches = 0; + if (decode != nullptr && !sameDecode(*decode, *fresh)) { + decodeMismatches = 1; + static std::atomic reports{0}; + if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: the entry's decode differs from a fresh decode (key 0x%llx, target 0x%llx)\n", static_cast(drawKey), static_cast(fresh->state.color.address)); + } + std::lock_guard cacheLock(drawCacheMutex); + drawEntryCounters.facadeMismatches += facadeMismatches; + if (decode != nullptr) ++drawEntryCounters.verifyDecodes; + drawEntryCounters.verifyDecodeMismatches += decodeMismatches; + } + decode = std::move(fresh); + } else if (profile) { + std::lock_guard cacheLock(drawCacheMutex); + ++drawEntryCounters.decodeSkipped; + } + // The draw's copies of the products: the CPU path of an indirect draw patches user words. + const auto& graphics = decode->state; + const auto& pixel = decode->pixel; + std::vector programs = decode->programs; + const std::vector& roles = decode->roles; + phase(DrawRowDecode); + // The user word an SH register named by an indirect draw packet lands in: the program whose + // bank holds it and the index into its user data (a merged program carries eight hidden + // words in front of its bank), or nothing for the sentinel 0x280 and a register no active + // stage loads (the CP writes it; no shader sees it). + const auto locate = [&](std::uint32_t location) -> std::optional> { + if (location == 0x280u) return std::nullopt; + for (std::size_t i = 0; i < programs.size(); ++i) { + if (roles[i] == Role::Fragment || roles[i] == Role::GeometryBack || location < programs[i].userDataBase) continue; + const auto word = location - programs[i].userDataBase + (8u - programs[i].firstUserSgpr); + if (word < programs[i].userData.size()) return std::make_pair(i, static_cast(word)); + } + return std::nullopt; + }; + if (drawParameters.indirect) { + auto& indirect = *drawParameters.indirect; + const auto sgprOf = [&](std::uint32_t location) -> std::int32_t { + const auto word = locate(location); + if (!word || word->first != 0) return -1; + return static_cast(programs.front().firstUserSgpr + word->second); + }; + indirect.baseVertexSgpr = sgprOf(indirect.baseVertexLocation); + indirect.startInstanceSgpr = sgprOf(indirect.startInstanceLocation); + indirect.drawIndexSgpr = sgprOf(indirect.drawIndexLocation); + } + std::vector memory; + std::vector linked; + for (std::size_t i = 0; i < programs.size(); ++i) { + const auto& program = programs[i]; + memory.insert(memory.end(), program.memory.begin(), program.memory.end()); + linked.push_back({roles[i], program.binary, program.userDataBase, program.firstUserSgpr, program.userData}); + } + timing.Mark("prepare"); + phase(DrawRowProgramPrepare); + // The vertex stage infos: the decode reads the attribute and V# tables from guest memory + // outside the capture (its reads are kept per stage, DecodeRead, and become runs of the + // stage's variant), so under the register key a stage is decoded only when no variant + // matched it (a matched variant carries the info it was compiled with); under the + // decoded-bytes key every stage is decoded before the key. The CPU path of an indirect + // draw decodes a program again once its user words are patched. + std::vector> vertexInfos(programs.size()); + std::vector> decodeReads(programs.size()); + const auto decodeVertexInfo = [&](std::size_t i) { + const auto& program = programs[i]; + if (program.binary.stage == Stage::Fragment || roles[i] == Role::GeometryBack) return; + decodeReads[i].clear(); + vertexInfos[i] = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, &decodeReads[i]); + }; + if (!registerKey) { + for (std::size_t i = 0; i < programs.size(); ++i) decodeVertexInfo(i); + phase(DrawRowDecode); + } + ShaderMemory shaderMemory(memory, &ClassifyPendingWrite, &ObservePendingWrite, HookWaitCounter()); std::vector results; std::vector stages; results.reserve(programs.size() + (graphics.rectList ? 2u : 0u)); stages.reserve(programs.size()); std::uint32_t pushCursorBytes = 0; - for (std::size_t i = 0; i < programs.size(); ++i) { - if (roles[i] == Role::GeometryBack) continue; + // The compiled result of each program (null for GeometryBack): a miss's copy in `results`, + // a hit's shared object held by the matched variant. + std::vector programResults(programs.size(), nullptr); + // A miss's per-stage capture, what the draw entry's variant is made of. + struct StageCapture { + std::shared_ptr compiled; + std::vector regions; + std::uint64_t forgetSerial = 0; + std::uint32_t pushOffset = 0; + }; + std::vector stageCaptures(programs.size()); + std::vector> matched(programs.size()); + std::vector> matchedRegions(programs.size()); + // A miss's fresh variants (per program index), the kept objects after the insert. + std::vector> fresh(programs.size()); + // The stages `compile` below captured on the shared ShaderMemory: a hit's other stages + // keep their matched regions in the draw's union. + std::vector recompiled(programs.size(), false); + bool drawHit = false; + bool verifyHit = false; + if (useDrawEntries) { + if (!registerKey) { + drawKey = 0xcbf29ce484222325ull; + const auto mix = [&](std::uint64_t value) { + drawKey ^= value; + drawKey *= 0x100000001b3ull; + }; + mix(localDevice->Serial()); + mix(static_cast(graphics.stages.path)); + mix(graphics.stages.registerValue); + mix(graphics.stages.vertexWaveSize); + mix(graphics.stages.fragmentWaveSize); + mix(graphics.stages.mesh.has_value()); + if (graphics.stages.mesh) { + const auto& mesh = *graphics.stages.mesh; + for (const auto value : {mesh.inputPrimitive, mesh.primitivesPerGroup, mesh.verticesPerGroup, mesh.maxVertices, mesh.maxPrimitives, mesh.threadsPerGroup, mesh.ldsSizeDwords, mesh.provokingVertex}) mix(value); + } + mix(graphics.stages.tessellation.has_value()); + if (graphics.stages.tessellation) { + const auto& tess = *graphics.stages.tessellation; + for (const auto value : {tess.inputControlPoints, tess.outputControlPoints, tess.domain, tess.partitioning, tess.outputTopology}) mix(value); + } + mix(graphics.rectList); + mix(programs.size()); + for (std::size_t i = 0; i < programs.size(); ++i) { + const auto& program = programs[i]; + mix(reinterpret_cast(program.snapshot.get())); + mix(program.codeOffset); + mix(static_cast(roles[i])); + mix(static_cast(program.binary.stage)); + mix(program.userDataBase); + mix(program.firstUserSgpr); + mix(program.userData.size()); + for (const auto word : program.userData) mix(word); + mix(vertexInfos[i].has_value()); + if (!vertexInfos[i]) continue; + const auto& vertex = *vertexInfos[i]; + require(vertex.resourcesNum <= vertex.resources.size(), "vertex stage info resource count exceeds its table"); + mix(vertex.resourcesNum); + mix(vertex.fetchAttribReg); + mix(vertex.fetchBufferReg); + mix(vertex.fetchEmbedded); + for (std::uint32_t r = 0; r < vertex.resourcesNum; ++r) { + for (const auto field : vertex.resources[r].fields) mix(field); + const auto& destination = vertex.resourcesDst[r]; + mix(static_cast(destination.registerStart)); + mix(static_cast(destination.registersNum)); + mix(static_cast(destination.attrId)); + mix(destination.fetchIndex); + } + } + require(pixel.interpolatorCount <= pixel.interpolatorSettings.size(), "pixel stage info interpolator count exceeds its table"); + mix(pixel.interpolatorCount); + for (std::uint32_t i = 0; i < pixel.interpolatorCount; ++i) mix(pixel.interpolatorSettings[i]); + mix(pixel.perspectiveCenterVgpr); + for (const bool flag : {pixel.wave32, pixel.hasPerspectiveCenterVgpr, pixel.posX, pixel.posY, pixel.posZ, pixel.posW, pixel.frontFace, pixel.ancillary, pixel.sampleShading, pixel.noPerspective, pixel.pixelKillEnable, pixel.depthExportEnable, pixel.sampleMaskExportEnable, pixel.earlyZ, pixel.executeOnNoop}) mix(flag); + for (const auto value : pixel.targetOutputMode) mix(value); + for (const auto value : pixel.targetExportMapping) mix(value); + std::lock_guard cacheLock(drawCacheMutex); + ++drawEntryCounters.lookups; + const auto found = drawCache.find(drawKey); + if (found != drawCache.end()) entry = found->second; + else ++drawEntryCounters.absent; + } + if (entry != nullptr) { + const auto waitedBeforeValidate = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + std::optional miss; + std::vector ranks(programs.size(), 0); + std::uint64_t stageValidations = 0, stageEqual = 0, compared = 0, imagesFlushed = 0, runsSynced = 0; + if (entry->stages.size() != programs.size()) miss = DrawMiss::Stages; + // The push offset the stage must have been compiled at: the matched earlier + // stages' push sizes. + std::uint32_t cursor = 0; + { + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DrawCache); + // One evidence sample for the whole validation, as a dispatch's. + std::optional sampling; + for (std::size_t i = 0; !miss && i < programs.size(); ++i) { + if (roles[i] == Role::GeometryBack) continue; + ++stageValidations; + const auto& variants = entry->stages[i]; + auto outcome = EntryOutcome::Differing; + bool anyLayout = false; + for (std::size_t rank = 0; rank < variants.size(); ++rank) { + const auto& variant = variants[rank]; + if (variant->pushOffset != cursor) continue; + auto& regions = matchedRegions[i]; + regions.clear(); + appendEntryRegions(*variant, regions); + ++compared; + auto result = validateVariant(programs[i].binary.codeAddress, submission.queue, *variant, regions, imagesFlushed, runsSynced, sampling); + if (GateRetry() && (result == EntryOutcome::PublishMoved || result == EntryOutcome::PendingMoved)) result = validateVariant(programs[i].binary.codeAddress, submission.queue, *variant, regions, imagesFlushed, runsSynced, sampling); + if (!anyLayout) outcome = result; + anyLayout = true; + if (result != EntryOutcome::Equal) continue; + matched[i] = variant; + ranks[i] = rank; + ++stageEqual; + cursor += static_cast(variant->compiled->pushConstants.size()); + break; + } + if (matched[i] != nullptr) continue; + if (!anyLayout) miss = DrawMiss::Layout; + else if (outcome != EntryOutcome::Differing) miss = DrawMiss::Gate; + else miss = roles[i] == Role::Fragment ? DrawMiss::FragmentDiffering : i == 0 ? DrawMiss::FrontDiffering : DrawMiss::OtherDiffering; + } + } + drawHit = !miss; + std::lock_guard cacheLock(drawCacheMutex); + auto& counters = drawEntryCounters; + counters.stageValidations += stageValidations; + counters.stageEqual += stageEqual; + counters.variantsCompared += compared; + if (drawHit) { + ++counters.hits; + if (registerKey) ++counters.registerKeyHits; + ++drawCacheHits; + bool rotate = false; + for (std::size_t i = 0; i < programs.size(); ++i) { + if (matched[i] == nullptr) continue; + ++counters.variantHitsByRank[ranks[i]]; + if (ranks[i] != 0) rotate = true; + } + // Only the entry still in the map is reordered: a stage hit behind its front + // moves to the front by replacing the entry with a rotated copy. + const auto again = drawCache.find(drawKey); + if (again != drawCache.end() && again->second == entry) { + if (rotate) { + auto rotated = std::make_shared(); + rotated->decode = entry->decode; + rotated->stages = entry->stages; + rotated->recipes.store(entry->recipes.load()); + for (std::size_t i = 0; i < programs.size(); ++i) { + if (ranks[i] == 0) continue; + auto& variants = rotated->stages[i]; + variants.erase(variants.begin() + static_cast(ranks[i])); + variants.insert(variants.begin(), matched[i]); + } + rotated->touched = entry->touched; + rotated->order = entry->order; + again->second = std::move(rotated); + } + if (drawCacheHits - again->second->touched > DrawCacheEntries() / 8) { + drawOrder.splice(drawOrder.begin(), drawOrder, again->second->order); + again->second->touched = drawCacheHits; + ++counters.touches; + } + } + if (VerifyDrawEntries()) { + ++counters.verifyHits; + verifyHit = true; + drawHit = false; + } + } else { + ++counters.misses[static_cast(*miss)]; + if (registerKey && entry->decode != nullptr) ++counters.decodePartial; + } + if (profile) { + // The hook waits inside the validations are their own row. + phase(DrawRowKeyLookupValidate); + const auto waited = std::min(Graphics::Recorder::ThreadWaitedMs() - waitedBeforeValidate, phaseMs[DrawRowKeyLookupValidate]); + phaseMs[DrawRowKeyLookupValidate] -= waited; + phaseMs[DrawRowValidateWait] += waited; + counters.validateUs += phaseMs[DrawRowKeyLookupValidate] * 1000; + if (std::chrono::steady_clock::now() - counters.lastReport > std::chrono::seconds(10)) { + counters.lastReport = std::chrono::steady_clock::now(); + reportDrawCache(counters); + } + } + } + phase(DrawRowKeyLookupValidate); + } + if (registerKey) { + // On a hit every stage's info is its variant's; a miss (a partial one included: its + // matched stages are compiled again, and the fresh variant's runs must carry the + // decode's reads, rule RD) and a verified hit (for the compare) decode every stage now. + for (std::size_t i = 0; i < programs.size(); ++i) { + if (matched[i] != nullptr && drawHit && !VerifyDrawRecipe()) { + if (matched[i]->vertexInfo != nullptr) vertexInfos[i] = *matched[i]->vertexInfo; + continue; + } + decodeVertexInfo(i); + if (matched[i] != nullptr && VerifyDrawRecipe() && (vertexInfos[i].has_value() != (matched[i]->vertexInfo != nullptr) || (vertexInfos[i] && !sameVertexInfo(*vertexInfos[i], *matched[i]->vertexInfo)))) { + static std::atomic reports{0}; + if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: stage %zu (program 0x%llx) of a hit has a vertex stage info unlike its variant's\n", i, static_cast(programs[i].binary.codeAddress)); + std::lock_guard cacheLock(drawCacheMutex); + ++drawEntryCounters.verifyDecodeMismatches; + } + } + phase(DrawRowDecode); + } + // One program's capture and recompile at a push constant offset; the CPU path of an + // indirect draw repeats it for a program whose user words the CP would have patched. + const auto compile = [&](std::size_t i, std::uint32_t pushOffset) { + phase(DrawRowVectors); const auto& program = programs[i]; const auto waveSize = program.binary.stage == Stage::Fragment ? graphics.stages.fragmentWaveSize : graphics.stages.vertexWaveSize; ShaderRecompiler::RecompileRequest request{ program.binary, - {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, program.binary.stage == Stage::Fragment ? std::nullopt : std::optional(Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData)), memory}, + {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, vertexInfos[i], memory}, localDevice->Target(), - {0, 0, pushCursorBytes, Graphics::PipelinePushConstantBytes - pushCursorBytes}, + {0, 0, pushOffset, Graphics::PipelinePushConstantBytes - pushOffset}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}} }; - const auto capture = shaderMemory.Capture(request); + const auto waitedBefore = TraceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + const auto handle = SourceHandleFor(*program.snapshot, program.codeOffset, localDevice->Serial(), request, false); + auto& stageCapture = stageCaptures[i]; + stageCapture.forgetSerial = GuestMemory::ForgetSerial(); + stageCapture.pushOffset = pushOffset; + const auto capture = [&] { + const SampledReadScope sampling(evidenceReads); + return shaderMemory.Capture(request, handle.get()); + }(); + // This stage's own reads (a page the earlier stage fetched included), the entry's runs. + stageCapture.regions = shaderMemory.TakeRecentRegions(); + recompiled[i] = true; memory = shaderMemory.Regions(); + // Regions() holds only the stages captured here: on a hit (the CPU path of an + // indirect draw recompiling a patched program) the other stages' validated regions + // stay in the union (rule RD). + if (drawHit) { + for (std::size_t j = 0; j < programs.size(); ++j) { + if (matched[j] != nullptr && !recompiled[j]) memory.insert(memory.end(), matchedRegions[j].begin(), matchedRegions[j].end()); + } + } request.context.memory = memory; - // Debug aid: APS5_DUMP_DRAW_SHADERS= saves the requests of draws into that - // target as shader_
.req for agc_shader_replay. - static const std::uint64_t dumpTarget = [] { const char* text = std::getenv("APS5_DUMP_DRAW_SHADERS"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); + if (TraceCapSync()) traceCapture("draw-capture", program.binary.codeAddress, submission.queue, memory, Graphics::Recorder::ThreadWaitedMs() - waitedBefore); + if (profile) { + ++captures; + phase(DrawRowCapture); + // The hook waits inside the capture are their own row, as the validate row's are. + const auto waited = std::min(Graphics::Recorder::ThreadWaitedMs() - waitedBefore, phaseMs[DrawRowCapture]); + phaseMs[DrawRowCapture] -= waited; + phaseMs[DrawRowCaptureHookWaits] += waited; + } if (dumpTarget != 0) { // Match the first enabled target or the raw slot-0 base, which skip reports name. const auto slot0 = (static_cast(readRegister(queue.context, 0x390)) << 40u) | (static_cast(readRegister(queue.context, 0x318)) << 8u); if ((graphics.hasColorTarget && graphics.color.address == dumpTarget) || slot0 == dumpTarget) static_cast(dumpRequest(program.binary.codeAddress, request)); } - // Debug aid: APS5_DUMP_DRAW_SLOT1= saves the requests and registers of draws whose - // second color target (slot 1) is at that address, as shader_
.req and draw_slot1.regs. - static const std::uint64_t dumpSlot1 = [] { const char* text = std::getenv("APS5_DUMP_DRAW_SLOT1"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); if (dumpSlot1 != 0) { const auto value = [&](std::uint32_t offset) -> std::uint64_t { const auto it = queue.context.find(offset); return it == queue.context.end() ? 0u : it->second; }; const auto slot1 = (value(0x391) << 40u) | (value(0x327) << 8u); @@ -1380,43 +4353,211 @@ private: } // As in dispatch: the recompile reuses the capture (APS5_NO_CAPTURE_REUSE=1 restores). static const bool reuseCapture = std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; - results.push_back(reuseCapture ? ShaderRecompiler::Recompile(request, *capture) : ShaderRecompiler::Recompile(request)); - const auto& result = results.back(); - if (!drawParameters.indexed && i == 0) { - const auto offsetValue = [&](std::int32_t sgpr) { - require(sgpr >= 0 && static_cast(sgpr) >= program.firstUserSgpr, "invalid draw offset SGPR"); - const auto index = static_cast(sgpr) - program.firstUserSgpr; - require(index < program.userData.size(), "draw offset SGPR exceeds user data"); - return program.userData[index]; + phase(DrawRowCapture); + // The memo'd result is copied into this draw's results (stages point into them and + // the CPU indirect path rewrites slots); the descriptor population is what the memo + // saves. The shared object is what the draw entry's variant keeps. + stageCapture.compiled = reuseCapture ? ShaderRecompiler::Recompile(request, *capture) : std::make_shared(ShaderRecompiler::Recompile(request)); + ShaderRecompiler::RecompileResult result = *stageCapture.compiled; + phase(DrawRowRecompile); + return result; + }; + // The front program's user word an SGPR holds (the recompiler numbers user SGPRs from + // firstUserSgpr). + const auto userWord = [&](std::int32_t sgpr) { + const auto& program = programs.front(); + require(sgpr >= 0 && static_cast(sgpr) >= program.firstUserSgpr, "invalid draw offset SGPR"); + const auto index = static_cast(sgpr) - program.firstUserSgpr; + require(index < program.userData.size(), "draw offset SGPR exceeds user data"); + return program.userData[index]; + }; + // The fixed-function fetch starts where the hardware's does: the shader adds its base + // vertex / start instance SGPR to the id VGPR itself (the analyzer names it), so that user + // word is folded into the draw's first vertex / instance, on top of GE_INDX_OFFSET, which + // the hardware adds as well (APS5_INDX_OFFSET_SKIP_FOLD=1 restores the skip of the fold + // when it is set), and for indexed draws as the vertex offset (APS5_NO_INDEXED_OFFSET_FOLD=1 + // restores the old no-fold). + static const bool indxOffsetSkipFold = std::getenv("APS5_INDX_OFFSET_SKIP_FOLD") != nullptr; + static const bool indexedOffsetFold = std::getenv("APS5_NO_INDEXED_OFFSET_FOLD") == nullptr; + const auto fold = [&](const ShaderRecompiler::RecompileResult& main, Pm4::DrawParameters& parameters) { + if (parameters.indexed && !indexedOffsetFold) return; + if (main.vertexOffsetSgpr >= 0 && (parameters.firstVertex == 0 || !indxOffsetSkipFold)) { + const auto offset = userWord(main.vertexOffsetSgpr); + require(offset <= std::numeric_limits::max() - parameters.firstVertex, "draw vertex offset overflow"); + parameters.firstVertex += offset; + } + if (main.instanceOffsetSgpr >= 0) parameters.firstInstance = userWord(main.instanceOffsetSgpr); + }; + // Why an indirect draw's records must be read on the CPU (nothing: the GPU reads them). + std::optional indirectCpu; + std::vector pushOffsets(programs.size(), 0); + std::vector resultIndex(programs.size(), 0); + for (std::size_t i = 0; i < programs.size(); ++i) { + if (roles[i] == Role::GeometryBack) continue; + const auto& program = programs[i]; + pushOffsets[i] = pushCursorBytes; + if (drawHit) { + // The matched variant's shared result and its validated regions (the draw's + // union, rule RD). + programResults[i] = matched[i]->compiled.get(); + memory.insert(memory.end(), matchedRegions[i].begin(), matchedRegions[i].end()); + } else { + resultIndex[i] = results.size(); + results.push_back(compile(i, pushCursorBytes)); + programResults[i] = &results.back(); + } + const auto& result = *programResults[i]; + if (i == 0 && drawParameters.indirect) { + // Per dimension (vertex, instance) how the GPU reproduces the CP's register write + // for the fixed-function fetch, with k the SGPR the shader adds to the dimension's + // id VGPR, p the SGPR the packet names for the CP's write and X GE_INDX_OFFSET (the + // vertex dimension of non-indexed draws): a sentinel p leaves the shader's add + // reading the game's word, so the record's dword is rewritten with X + user[k] + // (Constant); k == p read nowhere else means the record's value is exactly the fold + // (InPlace; X must be 0, the GPU cannot add it to a record); anything else needs the + // CP's write in the user word, which only the CPU path below makes. + auto& indirect = *drawParameters.indirect; + using Rule = Pm4::DrawParameters::IndirectDraw::Rule; + using Path = Graphics::IndirectDrawPath; + const auto support = localDevice->DrawIndirectSupport(); + const auto fail = [&](Path reason) { if (!indirectCpu) indirectCpu = reason; }; + if (graphics.stages.path != Graphics::ShaderPath::Vertex || graphics.rectList) fail(Path::NonVertexPath); + if (indirect.drawIndexEnabled && indirect.drawIndexSgpr >= 0) fail(Path::DrawIndex); + if (indirect.countIndirect && !support.count) fail(Path::FeatureGap); + if (indirect.countIndirect && indirect.count > 1 && !support.multi) fail(Path::FeatureGap); + const auto dimension = [&](std::int32_t k, std::int32_t p, bool shared, bool conflict, std::uint32_t x, Rule& rule, std::uint32_t& constant) { + if (p < 0) { + rule = Rule::Constant; + const auto offset = k >= 0 ? userWord(k) : 0u; + require(offset <= std::numeric_limits::max() - x, "draw vertex offset overflow"); + constant = x + offset; + } else if (conflict) fail(Path::FetchUnknown); + else if (k != p || shared) fail(Path::NotFolded); + else if (x != 0) fail(Path::IndxOffset); + else rule = Rule::InPlace; }; - if (drawParameters.firstVertex == 0 && result.vertexOffsetSgpr >= 0) drawParameters.firstVertex = offsetValue(result.vertexOffsetSgpr); - if (result.instanceOffsetSgpr >= 0) drawParameters.firstInstance = offsetValue(result.instanceOffsetSgpr); + dimension(result.vertexOffsetSgpr, indirect.baseVertexSgpr, result.vertexOffsetShared, result.vertexOffsetConflict, drawParameters.indexed ? 0u : indirect.indxOffset, indirect.vertexRule, indirect.vertexConstant); + dimension(result.instanceOffsetSgpr, indirect.startInstanceSgpr, result.instanceOffsetShared, result.instanceOffsetConflict, 0u, indirect.instanceRule, indirect.instanceConstant); + if (!support.firstInstance && (indirect.instanceRule == Rule::InPlace || indirect.instanceConstant != 0)) fail(Path::FeatureGap); + if ((indirect.vertexRule == Rule::Constant || indirect.instanceRule == Rule::Constant) && indirect.count > 256) fail(Path::FeatureGap); + // The GPU path copies every vertex descriptor's whole range (the counts are in the + // records): APS5_INDIRECT_VERTEX_MIB (default 64) caps it. + static const std::uint64_t vertexCap = [] { const char* text = std::getenv("APS5_INDIRECT_VERTEX_MIB"); return (text ? std::strtoull(text, nullptr, 10) : 64ull) << 20u; }(); + for (const auto& attribute : result.vertexAttributes) { + const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu; + const auto extent = stride == 0 ? static_cast(attribute.resource.fields[2]) : static_cast(attribute.resource.fields[2]) * stride; + if (extent > vertexCap) fail(Path::VertexRange); + } + if (traceIndirect) { + const auto location = [](std::uint32_t value, std::int32_t sgpr) { char text[24]; if (value == 0x280u) std::snprintf(text, sizeof(text), "none"); else std::snprintf(text, sizeof(text), "0x%x(s%d)", value, sgpr); return std::string(text); }; + const auto rule = [](Rule value, std::uint32_t constant) { char text[24]; if (value == Rule::InPlace) std::snprintf(text, sizeof(text), "in-place"); else std::snprintf(text, sizeof(text), "const %u", constant); return std::string(text); }; + std::string decision = indirectCpu ? "cpu (" + std::string(Graphics::IndirectDrawPathName(*indirectCpu)) + ")" : "gpu vertex=" + rule(indirect.vertexRule, indirect.vertexConstant) + " instance=" + rule(indirect.instanceRule, indirect.instanceConstant); + std::fprintf(stderr, "[draw] indirect 0x%x args 0x%llx stride %u count %u%s locs base=%s inst=%s idx=%s%s analyzer v=%d i=%d shared=%d/%d conflict=%d/%d indx=%u -> %s\n", indirect.opcode, static_cast(indirect.arguments), indirect.stride, indirect.count, indirect.countIndirect ? " (indirect)" : "", location(indirect.baseVertexLocation, indirect.baseVertexSgpr).c_str(), location(indirect.startInstanceLocation, indirect.startInstanceSgpr).c_str(), location(indirect.drawIndexLocation, indirect.drawIndexSgpr).c_str(), indirect.drawIndexEnabled ? " (enabled)" : "", result.vertexOffsetSgpr, result.instanceOffsetSgpr, result.vertexOffsetShared ? 1 : 0, result.instanceOffsetShared ? 1 : 0, result.vertexOffsetConflict ? 1 : 0, result.instanceOffsetConflict ? 1 : 0, indirect.indxOffset, decision.c_str()); + } + } else if (i == 0) { + fold(result, drawParameters); } require(result.pushConstants.size() <= Graphics::PipelinePushConstantBytes - pushCursorBytes, "stage push constants exceed the pipeline push constant block"); stages.push_back({program.binary.stage, &result, result.pushConstants.empty() ? 0u : pushCursorBytes}); pushCursorBytes += static_cast(result.pushConstants.size()); } + // A miss inserts its stages' fresh variants (the CPU path of an indirect draw recompiles + // on patched words below: nothing of it is kept); a verified hit compares them with the + // matched variants first. + if (useDrawEntries && !drawHit && !(drawParameters.indirect && indirectCpu)) { + phase(DrawRowVectors); + std::uint64_t unstable = 0, mismatches = 0; + for (std::size_t i = 0; i < programs.size(); ++i) { + const auto& stageCapture = stageCaptures[i]; + if (stageCapture.compiled == nullptr) continue; + auto variant = std::make_shared(); + variant->compiled = stageCapture.compiled; + variant->shader = programs[i].snapshot; + variant->forgetSerial = stageCapture.forgetSerial; + variant->pushOffset = stageCapture.pushOffset; + if (vertexInfos[i]) variant->vertexInfo = std::make_shared(*vertexInfos[i]); + // The stage's runs: its capture's regions and the vertex decode's guest reads + // (rule RD), in address order. + std::vector regions(stageCapture.regions.begin(), stageCapture.regions.end()); + for (const auto& read : decodeReads[i]) regions.push_back({read.address, std::as_bytes(std::span(read.bytes))}); + std::stable_sort(regions.begin(), regions.end(), [](const ShaderRecompiler::MemoryRegion& a, const ShaderRecompiler::MemoryRegion& b) { return a.guestAddress < b.guestAddress; }); + for (const auto& region : regions) { + variant->runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size()); + const auto count = region.bytes.size() / sizeof(std::uint32_t); + const auto offset = variant->words.size(); + variant->words.resize(offset + count); + std::memcpy(variant->words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t)); + } + if (verifyHit && matched[i] != nullptr && (matched[i]->runs != variant->runs || matched[i]->words != variant->words)) { + ++mismatches; + static std::atomic reports{0}; + if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: stage %zu (program 0x%llx) of a hit captured differently: %zu runs / %zu words matched, %zu / %zu fresh\n", i, static_cast(programs[i].binary.codeAddress), matched[i]->runs.size(), matched[i]->words.size(), variant->runs.size(), variant->words.size()); + } + if (InsertCompare()) { + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DrawCache); + if (!captureStable(stageCapture.regions)) { + ++unstable; + continue; + } + } + fresh[i] = std::move(variant); + } + insertDrawEntry(drawKey, fresh, registerKey ? decode : nullptr); + if (unstable != 0 || mismatches != 0) { + std::lock_guard cacheLock(drawCacheMutex); + drawEntryCounters.unstable += unstable; + drawEntryCounters.verifyMismatches += mismatches; + } + phase(DrawRowKeyLookupValidate); + } timing.Mark("shader_compile_and_link"); + // The vertex decode's guest reads join the union the label test and the snapshots cover + // (a matched stage's are among its regions already). + for (const auto& reads : decodeReads) { + for (const auto& read : reads) memory.push_back({read.address, std::as_bytes(std::span(read.bytes))}); + } // As in dispatch: a label this queue still has queued over a captured region is recorded // now and the draw starts over (nothing queued on the locked-prepare path, recorded above). - if (recordQueuedLabelsAfterCapture(submission.queue, memory)) { - draw(queue, packet, submission); - return; - } - if (graphics.rectList) { + if (recordQueuedLabelsAfterCapture(submission.queue, memory)) return draw(queue, packet, submission, rejected); + // The rect-list stages are generated from the front and fragment results (rebuilt when the + // CPU path below recompiles the front program). + bool rectListBuilt = false; + // Where the rect-list results sit in `results` (behind a miss's program results; first + // on a hit). + std::size_t rectIndex = 0; + const auto buildRectList = [&] { + phase(DrawRowVectors); + require(programs.size() == 2 && programResults[0] != nullptr && programResults[1] != nullptr, "rect-list requires vertex and fragment programs"); + auto rectangle = ShaderRecompiler::BuildRectListShaders(*programResults[0], *programResults[1], localDevice->Target()); + if (rectListBuilt) { + results[rectIndex] = std::move(rectangle.control); + results[rectIndex + 1] = std::move(rectangle.evaluation); + phase(DrawRowRectList); + return; + } require(stages.size() == 2, "rect-list requires vertex and fragment programs"); - auto rectangle = ShaderRecompiler::BuildRectListShaders(results[0], results[1], localDevice->Target()); + rectIndex = results.size(); results.push_back(std::move(rectangle.control)); results.push_back(std::move(rectangle.evaluation)); - stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[2], 0}, {Stage::TessellationEvaluation, &results[3], 0}}); - } + stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[rectIndex], 0}, {Stage::TessellationEvaluation, &results[rectIndex + 1], 0}}); + rectListBuilt = true; + phase(DrawRowRectList); + }; + if (graphics.rectList) buildRectList(); std::vector snapshots; - for (const auto& region : memory) snapshots.push_back({region.guestAddress, region.bytes}); + const auto snapshot = [&] { + snapshots.clear(); + for (const auto& region : memory) snapshots.push_back({region.guestAddress, region.bytes}); + }; + snapshot(); timing.Mark("post_compile_prepare"); - if (!gpuLock.owns_lock()) { + const auto lockForDraw = [&] { + if (gpuLock.owns_lock()) return; + phase(DrawRowVectors); GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); gpuLock.lock(); timing.Mark("gpu_mutex_wait"); + phase(DrawRowLockWait); // Present replaces a windowless device by a windowed one (after WaitIdle) while draws // may be preparing unlocked: the draw then records on the current device, whose // recorder Recorder::Active() already is, rather than mixing the old device's images @@ -1429,9 +4570,115 @@ private: } // This queue's labels first (queue order), and their batch out before the draw. recordLabelsForPacket(localDevice.get(), submission.queue); + phase(DrawRowLabels); + }; + if (drawParameters.indirect && indirectCpu) { + // The CPU path of an indirect draw: the records are read through the flush hook (after + // this queue's queued labels went in, as for a DISPATCH_INDIRECT resolved here), the + // CP's register writes are made in the user words of the programs whose banks the + // packet names, those programs are recompiled on the patched words (a cache hit: only + // their push constants change) and every record becomes a direct draw with the fold + // rule, so the fetch and every read of a patched SGPR match the hardware; the id VGPR + // after a folded add keeps the direct path's double add. + const auto indirect = *drawParameters.indirect; + if (drawHit) { + // A hit's results are the variants' shared objects; this path rewrites slots, so + // it works on copies of them (the reserve above holds them). + for (std::size_t i = 0; i < programs.size(); ++i) { + if (programResults[i] == nullptr) continue; + resultIndex[i] = results.size(); + results.push_back(ShaderRecompiler::RecompileResult(*programResults[i])); + for (auto& stage : stages) { + if (stage.program == programResults[i]) stage.program = &results[resultIndex[i]]; + } + programResults[i] = &results[resultIndex[i]]; + } + } + recordQueuedLabelsBeforeRead(submission.queue); + const auto readStart = std::chrono::steady_clock::now(); + const auto count = std::min(indirect.countIndirect ? Pm4::ReadDrawCount(indirect) : indirect.count, indirect.count); + std::vector records; + for (std::uint32_t record = 0; record < count; ++record) records.push_back(Pm4::ReadDrawArguments(indirect, record)); + Graphics::CountIndirectDraw(*indirectCpu, std::chrono::duration(std::chrono::steady_clock::now() - readStart).count()); + const auto baseVertexWord = locate(indirect.baseVertexLocation); + const auto startInstanceWord = locate(indirect.startInstanceLocation); + const auto drawIndexWord = indirect.drawIndexEnabled ? locate(indirect.drawIndexLocation) : std::nullopt; + for (std::uint32_t record = 0; record < records.size(); ++record) { + const auto& arguments = records[record]; + if (traceIndirect) std::fprintf(stderr, "[draw] record %u: count %u instances %u first %u vertexOffset %u startInstance %u\n", record, arguments.count, arguments.instances, arguments.firstVertexOrIndex, arguments.vertexOffset, arguments.firstInstance); + if (arguments.count == 0 || arguments.instances == 0) continue; + std::set patched; + const auto patch = [&](const std::optional>& word, std::uint32_t value) { + if (!word) return; + programs[word->first].userData[word->second] = value; + patched.insert(word->first); + }; + patch(baseVertexWord, indirect.recordBytes == 20 ? arguments.vertexOffset : arguments.firstVertexOrIndex); + patch(startInstanceWord, arguments.firstInstance); + patch(drawIndexWord, record); + Pm4::DrawParameters direct{0, arguments.count, 0, arguments.instances, drawParameters.flags, drawParameters.indexed, 0, 0}; + if (drawParameters.indexed) { + // The CP clamps the index range to INDEX_BUFFER_SIZE (the declared range). + if (arguments.firstVertexOrIndex >= drawParameters.indexCount) continue; + direct.indexAddress = drawParameters.indexAddress + static_cast(arguments.firstVertexOrIndex) * drawParameters.indexSize; + direct.indexCount = std::min(arguments.count, drawParameters.indexCount - arguments.firstVertexOrIndex); + direct.indexSize = drawParameters.indexSize; + } else { + direct.firstVertex = indirect.indxOffset; + } + for (const auto programIndex : patched) { + auto& result = results[resultIndex[programIndex]]; + const auto pushBytes = result.pushConstants.size(); + decodeVertexInfo(programIndex); + result = compile(programIndex, pushOffsets[programIndex]); + require(result.pushConstants.size() == pushBytes, "patched program changed its push constant layout"); + } + fold(*programResults[0], direct); + if (graphics.rectList && patched.contains(0)) buildRectList(); + snapshot(); + lockForDraw(); + noteDrawWriters(stages, submission.queue); + phase(DrawRowVectors); + localDevice->Draw(graphics, direct, stages, snapshots); + phase(DrawRowGraphics); + } + timing.Mark("draw_and_resource_release"); + return drawn(); } - localDevice->Draw(graphics, drawParameters, stages, snapshots); + // The draw recipe (design_cpu_final M8, step 8b): the stage variants this draw records + // with, by identity (a hit's matched ones, a miss's kept fresh ones), name the recipe a + // later hit of the same set records from. Recipes need the register key (the entry then + // fixes the state the recipe's pipeline and targets were built for); the CPU path of an + // indirect draw never reaches here, and a GPU-side indirect draw takes the ordinary path. + std::vector> recipeStages; + if (registerKey && !drawParameters.indirect && Graphics::DrawRecipes()) { + recipeStages.reserve(programs.size()); + for (std::size_t i = 0; i < programs.size(); ++i) recipeStages.push_back(drawHit ? matched[i] : fresh[i]); + if (std::all_of(recipeStages.begin(), recipeStages.end(), [](const std::shared_ptr& variant) { return variant == nullptr; })) recipeStages.clear(); + } + std::shared_ptr recipe; + if (drawHit && !recipeStages.empty()) { + recipe = findDrawRecipe(drawKey, recipeStages); + if (recipe == nullptr) VulkanDevice::NoteDrawRecipeMiss(VulkanDevice::DrawRecipePrecheck::NoRecipe); + } + lockForDraw(); + noteDrawWriters(stages, submission.queue); + phase(DrawRowVectors); + if (recipe != nullptr) { + if (localDevice->DrawFromRecipe(graphics, drawParameters, stages, snapshots, recipe) == RecipeOutcome::Recorded) { + phase(DrawRowGraphics); + timing.Mark("draw_and_resource_release"); + return drawn(); + } + // Nothing recorded: the ordinary path with the variant's stages, then the re-attach. + VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Restart, VulkanDevice::RecipeKind::Draw); + } + std::shared_ptr built; + localDevice->Draw(graphics, drawParameters, stages, snapshots, recipeStages.empty() ? nullptr : &built); + phase(DrawRowGraphics); + if (built != nullptr) attachDrawRecipe(drawKey, recipeStages, std::move(built)); timing.Mark("draw_and_resource_release"); + return drawn(); } // Serials complete out of order when a wait runs other queues' work; `completed` stays the @@ -1497,6 +4744,48 @@ private: return value; } + // The WAIT_REG_MEM poll loop's service calls, per queue worker (the [poll] line printed with its + // [packets] line, APS5_PROFILE_DRAW): service calls, waits ended from the label table, mutex + // tries and failed tries, submits and reaps made for the producer (with the reaps that retired + // a batch), the table's refusals by reason (Recorder::LabelRefusal), and the try holds. + struct PollStats { + std::uint64_t ticks = 0, tableHits = 0, tries = 0, triesFailed = 0, submits = 0, reaps = 0, reapsWithWork = 0; + // Reaps made for a label refused as BehindCompletion (the one case a poll cannot converge + // without one) and the rest (queue 0's poller standing in for its boundary reap, a compute + // poller while queue 0 is dormant, every poller with APS5_POLL_REAP=1). + std::uint64_t reapsBehindCompletion = 0, reapsOther = 0; + std::array refusals{}; + double longestTryHoldUs = 0; + double tryHoldUs = 0; + }; + static PollStats& pollStats() { + static thread_local PollStats stats; + return stats; + } + static constexpr const char* LabelRefusalNames[6] = {"none", "trust-off", "queued", "overwritten", "unclosed", "behind-completion"}; + // A compute poller reaps only for a label refused as BehindCompletion (design_cpu_final M5 + // P1) or while queue 0 is dormant; queue 0's poller reaps whenever completions are pending + // (its boundary reap cannot run inside a wait). APS5_POLL_REAP=1 makes every poller reap + // whenever completions are pending, as before. + static bool PollReapAll() { + static const bool all = std::getenv("APS5_POLL_REAP") != nullptr; + return all; + } + // A poller tries the GPU mutex only when the recorder's write generation moved, or at most once + // per PollTryInterval when only a label deadline or a reap is due (M5 P2); APS5_POLL_TRY_EACH=1 + // tries at every service call, as before. + static bool PollTryEach() { + static const bool each = std::getenv("APS5_POLL_TRY_EACH") != nullptr; + return each; + } + static constexpr std::chrono::microseconds PollTryInterval{1000}; + // flushBetweenPackets skips the label record-try when the next packet records the group at its + // own lock (M5 L1); APS5_LABEL_TRY_EACH_PACKET=1 tries before every packet, as before. + static bool LabelTryEachPacket() { + static const bool each = std::getenv("APS5_LABEL_TRY_EACH_PACKET") != nullptr; + return each; + } + // How WAIT_REG_MEM packets were satisfied, per queue worker (reported in its [packets] line): // already true when reached; from the recorder's pending-label table (and how many of those by // a label of this same queue); by polling memory (the GPU, another queue or the CPU wrote it); @@ -1504,15 +4793,27 @@ private: // fromRecorderUnlocked: table hits (at entry or polling) that took no GPU mutex; entriesUnlocked: // waits that went to polling without the GPU mutex (nothing recorded writes the range); // entryTriesFailed: overlapping entries whose try of the mutex failed and went to polling - // instead of waiting for it (see waitMemory). + // instead of waiting for it (see waitMemory). fromRecorderLate: table hits taken under the late + // rule (Recorder::NoteLabel); lateRefusedCpuStore: late hits refused because a CPU store + // touched the dwords since the label's group closed. struct WaitOutcomes { std::uint64_t atEntry = 0, fromRecorder = 0, fromRecorderSameQueue = 0, fromRecorderPolling = 0, polled = 0, timedOut = 0, pollSubmits = 0, pollReaps = 0; - std::uint64_t fromRecorderUnlocked = 0, entriesUnlocked = 0, entryTriesFailed = 0; + std::uint64_t fromRecorderUnlocked = 0, entriesUnlocked = 0, entryTriesFailed = 0, fromRecorderLate = 0, lateRefusedCpuStore = 0; }; static WaitOutcomes& waitOutcomes() { static thread_local WaitOutcomes outcomes; return outcomes; } + // The late counters of Recorder::LateCounts (cumulative, per thread) at this worker's last + // [packets] line, for the deltas it prints. + static Graphics::Recorder::LateStatistics& lateCountsSeen() { + static thread_local Graphics::Recorder::LateStatistics seen{}; + return seen; + } + static bool TraceLateLabels() { + static const bool trace = std::getenv("APS5_TRACE_LATE_LABELS") != nullptr; + return trace; + } // Collect-epoch bumps by ordering point, per queue worker (reported in its [packets] line, see // GuestMemory::BumpCollectEpoch): the start of a submission, a wait the CPU may have satisfied, @@ -1534,27 +4835,49 @@ private: } // Labels recorded on the GPU are not submitted one by one (see VulkanDevice::WriteLabelOnGpu): - // the open batch goes out at the next non-label packet of the recording worker, and at the - // latest APS5_LABEL_FLUSH_US (default 250) after its first label, checked by every worker - // between packets and by waiting workers inside their poll loop, so a game thread polling the - // label is never left waiting on an unsubmitted batch for long. The deadline is read lock-free. + // the open batch goes out at the latest APS5_LABEL_FLUSH_US (default 1000; 250 with + // APS5_NO_LABEL_BATCH_SUBMIT=1, where it also goes out at the next non-label packet of the + // recording worker) after its first label, checked by every worker between packets and by + // waiting workers inside their poll loop, so a game thread polling the label is never left + // waiting on an unsubmitted batch for long. The deadline is read lock-free. static std::chrono::microseconds LabelFlushDeadline() { static const std::chrono::microseconds value = [] { const char* text = std::getenv("APS5_LABEL_FLUSH_US"); - return std::chrono::microseconds(text ? std::atoi(text) : 250); + return std::chrono::microseconds(text ? std::atoi(text) : (LabelBatchSubmit() ? 1000 : 250)); }(); return value; } + // Asked by the poll loop of a WAIT_REG_MEM (Driver::waitMemory): whether the open batch's first + // label passed its deadline, so the poller submits it. With the queued-label table it records + // this worker's queued labels first whenever the mutex is free: a wait of this queue on another + // queue's label can last milliseconds, and a label this worker queued before it (which only + // this queue's waits can take from the table) must reach its other consumers meanwhile, after + // which the deadline submits it like any recorded label. static bool LabelFlushDue() { + if (QueuedLabelTable() && !deferredLabels().labels.empty()) Get().recordQueuedLabelsByTry(GuestMemory::GpuLockThreadTag(), pollLabelRecords); const auto since = Graphics::Recorder::PendingLabelSince(); return since.has_value() && std::chrono::steady_clock::now() - *since >= LabelFlushDeadline(); } + // Whether some unfinished batch holds a completion the CPU must reap for: a label store or a + // write-back (the gate of every reap made outside a packet's own hold). + static bool CompletionsPending() { + return Graphics::Recorder::PendingCompletionLabels() != 0 || Graphics::Recorder::PendingWriteBackCompletions() != 0; + } + // Boundary reaps (flushBetweenPackets) are tried at most this often per worker; a reap only + // bounds how late a completion lands for a CPU poller (reads sync through the hook anyway). + static constexpr std::chrono::microseconds ReapInterval{250}; + static bool ReapEachPacket() { + // Debug aid: APS5_REAP_EACH_PACKET=1 tries a boundary reap at every packet of every queue, as before. + static const bool each = std::getenv("APS5_REAP_EACH_PACKET") != nullptr; + return each; + } // Submits the open batch after this many dispatches/draws even without a label, so the GPU - // starts on them while the CPU records the rest (APS5_BATCH_CAP, default 16; 0 disables). + // starts on them while the CPU records the rest (APS5_BATCH_CAP, default 64, the previous 16 + // measured no better on the GPU and worse on the presenter's tail; 0 disables). static std::uint64_t BatchCap() { static const std::uint64_t value = [] { const char* text = std::getenv("APS5_BATCH_CAP"); - return text ? std::strtoull(text, nullptr, 10) : 16ull; + return text ? std::strtoull(text, nullptr, 10) : 64ull; }(); return value; } @@ -1572,16 +4895,30 @@ private: void recordDeferredLabels(VulkanDevice* localDevice, std::uint32_t queue) { auto& deferred = deferredLabels(); if (deferred.labels.empty()) return; + // The group leaves the queue before it is recorded: a completion store inside the record + // (queue 0's reap before the first label) can reach the flush hook, whose queued-label + // callback (recordQueuedLabelsFromHook) must then find nothing to record a second time. + static thread_local std::vector labels; + labels.clear(); + labels.swap(deferred.labels); struct Clear { std::vector& labels; std::uint32_t queue; ~Clear() { - if (std::uncaught_exceptions() != 0) std::fprintf(stderr, "[gpu] queue 0x%x dropped %zu queued labels: their record failed\n", queue, labels.size()); + const bool failed = std::uncaught_exceptions() != 0; + if (failed) std::fprintf(stderr, "[gpu] queue 0x%x dropped %zu queued labels: their record failed\n", queue, labels.size()); labels.clear(); + // The group's table entries close with the tracker generation after the group's + // last MarkWritten (a failed record leaves them unclosed, never late-trusted). + Graphics::Recorder::CloseLabelGroup(failed ? 0 : GuestMemory::TrackerGeneration()); + // The recorded labels replaced their queued table entries; the rest (stored by + // the CPU, or dropped) leave the table with the per-thread ranges. + Graphics::Recorder::ForgetQueuedLabels(); + queuedLabelsNoted() = 0; } - } clear{deferred.labels, queue}; + } clear{labels, queue}; bool first = true; - for (const auto& label : deferred.labels) { + for (const auto& label : labels) { const auto bytes = std::span(label.bytes).first(label.size); const int reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label.address, bytes, ++eventSerial, queue, first) : 4; first = false; @@ -1608,7 +4945,7 @@ private: // critical sections ([lock] site 'label'), after which the packet's own came uncontended. // Returns whether the batch was submitted (the flip then needs no submit of its own). bool recordLabelsForPacket(VulkanDevice* localDevice, std::uint32_t queue) { - static const bool packetFlush = std::getenv("APS5_NO_LABEL_PACKET_FLUSH") == nullptr; + static const bool packetFlush = !LabelBatchSubmit() && std::getenv("APS5_NO_LABEL_PACKET_FLUSH") == nullptr; // APS5_PROFILE_DRAW: what this costs inside the packet's own hold ([labels] line): the // record (with queue 0's reap of finished batches before the first label) and the submit // (with another reap on queue 0), both of which run completions, so a dispatch's or @@ -1664,68 +5001,147 @@ private: } GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); - if (!gpuLock.try_lock()) return false; + if (!gpuLock.try_lock()) { + ++triesFailed[TryCapture]; + return false; + } const auto localDevice = device.load(); recordDeferredLabels(localDevice.get(), queue); ++captureTryRecords; return false; } + // Enters this worker's queued labels that are not in the label table yet (the group's tail + // since the last call) as queued entries, see QueuedLabelTable. The stamp taken now is older + // than the one the record takes, so a wait the queued entry does not satisfy is satisfied by + // the recorded one at the latest, never the reverse. + void noteQueuedLabels(std::uint32_t queue) { + static const bool installed = (Graphics::Recorder::SetQueuedLabelRecorder(&Driver::recordQueuedLabelsFromHook), true); + (void)installed; + const auto& labels = deferredLabels().labels; + for (auto& noted = queuedLabelsNoted(); noted < labels.size(); ++noted) { + const auto& label = labels[noted]; + Graphics::Recorder::NoteQueuedLabel(label.address, std::span(label.bytes).first(label.size), ++eventSerial, queue); + } + } + + // Records this worker's queued labels if the mutex is free (a try): from the poll loop of a + // wait (LabelFlushDue), so the group reaches its other consumers while this queue waits. + void recordQueuedLabelsByTry(std::uint32_t queue, std::atomic& counter) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + if (!gpuLock.try_lock()) { + ++triesFailed[TryPollRecord]; + return; + } + const auto localDevice = device.load(); + recordDeferredLabels(localDevice.get(), queue); + ++counter; + } + + // The flush hook's callback (Recorder::SetQueuedLabelRecorder): a CPU access by this worker + // overlaps a label it queued, which precedes the access in queue order, so the group is + // recorded outright now; the hook then syncs on the recorded store like on any other. + static void recordQueuedLabelsFromHook() { + if (deferredLabels().labels.empty()) return; + auto& driver = Get(); + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = driver.device.load(); + driver.recordDeferredLabels(localDevice.get(), GuestMemory::GpuLockThreadTag()); + ++hookLabelRecords; + } + // Between packets: records this worker's deferred labels before a packet that needs them (or - // once their deadline passed), submits the open batch when a label is pending (at once on a - // non-label packet, else once the deadline passed) or the batch holds enough work, and reaps - // finished batches whose completion actions still hold labels. The mutex is taken outright when - // the packet needs the labels or a deadline passed, and by the graphics worker for a pending - // submit; otherwise it is only tried (a register-only packet must not queue behind a 2-4 ms - // dispatch for this) and the next packet retries. A packet that takes the mutex itself - // (PacketLocksItself) gets both the record and the submit at its own acquisition - // (recordLabelsForPacket), so for it the mutex is only tried here as well: a try that succeeds - // records at once (the packet's unlocked prologue then reads through a hook that knows the - // labels, as before), one that fails leaves the group to the packet. Only a deadline still - // locks outright before such a packet: its prologue may take milliseconds, and pollers of the - // label are promised the deadline. APS5_NO_LABEL_PACKET_FLUSH=1 leaves recorded labels pending - // until the deadline (the batch then absorbs the following work); APS5_LABEL_OWN_LOCK=1 takes - // the mutex outright before every packet that needs the labels, as before. + // once their deadline passed), submits the open batch once a label's deadline passed (at once + // on a non-label packet with APS5_NO_LABEL_BATCH_SUBMIT=1, see LabelBatchSubmit) or the batch + // holds enough work, and reaps finished batches whose completion actions still hold labels. + // The mutex is taken outright when the packet needs the labels (without the queued-label + // table, see QueuedLabelTable) or a deadline is well past (four times the deadline, twice the + // cap: a try that fails is retried at the next packet within microseconds, while a wait behind + // a compute queue's hold costs queue 0 milliseconds); otherwise it is only tried (a + // register-only packet must not queue behind a 2-4 ms dispatch for this) and the next packet + // retries. A packet that takes the mutex itself (PacketLocksItself) gets the record (and the + // submit) at its own acquisition (recordLabelsForPacket), so for it the mutex is only tried + // here as well: a try that succeeds records at once (the packet's unlocked prologue then reads + // through a hook that knows the labels, as before), one that fails leaves the group to the + // packet. APS5_NO_LABEL_PACKET_FLUSH=1 leaves recorded labels pending until the deadline with + // the old submit policy; APS5_LABEL_OWN_LOCK=1 takes the mutex outright before every packet + // that needs the labels, as before. void flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool labelPacket) { - static const bool packetFlush = std::getenv("APS5_NO_LABEL_PACKET_FLUSH") == nullptr; + static const bool packetFlush = !LabelBatchSubmit() && std::getenv("APS5_NO_LABEL_PACKET_FLUSH") == nullptr; static const bool ownLock = std::getenv("APS5_LABEL_OWN_LOCK") != nullptr; - static std::atomic packetFlushes{0}, deadlineFlushes{0}, capFlushes{0}, boundaryReaps{0}; + static std::atomic packetFlushes{0}, deadlineFlushes{0}, capFlushes{0}, boundaryReaps{0}, overdueLocks{0}; auto& deferred = deferredLabels(); const bool queued = !deferred.labels.empty(); + const bool queuedTable = queued && QueuedLabelTable(); + if (queuedTable) noteQueuedLabels(queue); const bool selfLocking = !ownLock && PacketLocksItself(header); - bool submit = false, record = false, mustLock = false, deferredDue = false, pendingDue = false; + bool submit = false, record = false, needsRecord = false, deferredDue = false, pendingDue = false; const auto pending = Graphics::Recorder::PendingLabelSince(); - // The clock is read only while some label waits (every packet passes here). - const auto now = queued || pending.has_value() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + // Boundary reaps: queue 0 only (a compute worker must not run completions against queue + // 0; it reaps in its poll loops and the idle wake), at most once per ReapInterval whether + // the try succeeds or not (design13 step 6: the try at every packet was the mutex's main + // contender). APS5_REAP_EACH_PACKET=1 restores the try at every packet on every queue. + thread_local std::chrono::steady_clock::time_point lastReapTry{}; + const bool completions = CompletionsPending(); + const bool reapDue = completions && (ReapEachPacket() || queue == 0); + // The clock is read only while some label or reap waits (every packet passes here). + const auto now = queued || pending.has_value() || reapDue ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + // A self-locking packet records the group at its own acquisition within one packet, before + // its deadline can matter, so a queued group alone is no reason to try the mutex before it; + // a try made for another reason (submit, cap, reap) still records it. + bool recordAtLock = false; if (queued) { if (!labelPacket) { record = true; - mustLock = NeedsRecordedLabels(header) && !selfLocking; + // With the queued-label table the packet's reads and waits are covered without the + // record (the hook and the table), so the mutex is only tried, as for a self-locking packet. + needsRecord = !queuedTable && NeedsRecordedLabels(header) && !selfLocking; + } + if (now - deferred.since >= LabelFlushDeadline()) record = deferredDue = true; + if (record && !deferredDue && selfLocking && !LabelTryEachPacket()) { + record = false; + recordAtLock = true; } - if (now - deferred.since >= LabelFlushDeadline()) record = mustLock = deferredDue = true; } if (pending.has_value()) { if (!labelPacket && packetFlush) submit = true; if (now - *pending >= LabelFlushDeadline()) submit = pendingDue = true; } - const bool capped = !submit && !record && BatchCap() != 0 && Graphics::Recorder::RecordedWorkSinceSubmit() >= BatchCap(); - const bool reap = !submit && !record && !capped && Graphics::Recorder::PendingCompletionLabels() != 0; - if (!submit && !record && !capped && !reap) return; + const auto work = Graphics::Recorder::RecordedWorkSinceSubmit(); + const bool capped = !submit && !record && BatchCap() != 0 && work >= BatchCap(); + const bool reap = !submit && !record && !capped && reapDue && (ReapEachPacket() || now - lastReapTry >= ReapInterval); + if (!submit && !record && !capped && !reap) { + if (recordAtLock) ++recordTriesSkipped; + return; + } + if (reap) lastReapTry = now; std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); // The site names the hold too (a try's hold is charged to it; a reap alone shows as 'try'). if (record) GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); else if (submit || capped) GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); // A reap alone is never worth waiting for the mutex (the next packet retries); neither is a // submit the self-locking packet makes itself, unless the label's deadline passed. - if (mustLock || (queue == 0 && (capped || (submit && (!selfLocking || pendingDue))))) { + bool outright = needsRecord; + if (LabelBatchSubmit()) { + const auto grace = 4 * LabelFlushDeadline(); + const bool overdue = (deferredDue && now - deferred.since >= grace) || (pendingDue && now - *pending >= grace) || (capped && work >= 2 * BatchCap()); + if (overdue) ++overdueLocks; + outright = outright || overdue; + } else if (deferredDue || (queue == 0 && (capped || (submit && (!selfLocking || pendingDue))))) { + outright = true; + } + if (outright) { gpuLock.lock(); } else if (!gpuLock.try_lock()) { + ++triesFailed[record ? TryLabel : submit || capped ? TryFlush : TryReap]; if (selfLocking && record) ++packetLockDeferred; else if (selfLocking && submit) ++packetSubmitDeferred; return; } const auto localDevice = device.load(); - if (record) { + if (record || recordAtLock) { recordDeferredLabels(localDevice.get(), queue); // The group goes out with the packet flush (or at once past the deadline), as a label // recorded at its own packet would have. @@ -1745,8 +5161,15 @@ private: ++boundaryReaps; } static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - static std::atomic total{0}; - if (profile && (++total & 4095u) == 0) std::fprintf(stderr, "[labels] batch flushes between packets: %llu at the next packet, %llu by deadline, %llu by size cap; %llu boundary reaps; at the packet's own lock: %llu label groups (%.0f ms), %llu submits (%.0f ms incl. queue 0's reaps), left to it by a failed try: %llu groups, %llu submits; after a capture: %llu groups recorded by a try, %llu packets redone for a queued label over the capture; %llu suspend points\n", static_cast(packetFlushes.load()), static_cast(deadlineFlushes.load()), static_cast(capFlushes.load()), static_cast(boundaryReaps.load()), static_cast(packetLockRecords.load()), packetLockRecordUs.load() / 1000.0, static_cast(packetLockSubmits.load()), packetLockSubmitUs.load() / 1000.0, static_cast(packetLockDeferred.load()), static_cast(packetSubmitDeferred.load()), static_cast(captureTryRecords.load()), static_cast(captureRetries.load()), static_cast(suspendPoints.load())); + // Every 10 s (under the mutex still, so the clock is a plain static), cumulative counts. + static std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + if (!profile) return; + const auto reportNow = std::chrono::steady_clock::now(); + if (reportNow - lastReport < std::chrono::seconds(10)) return; + lastReport = reportNow; + const auto stores = Graphics::Recorder::StoreCounts(); + const auto failed = [](TrySite site) { return static_cast(triesFailed[site].load()); }; + std::fprintf(stderr, "[labels] batch flushes between packets: %llu at the next packet, %llu by deadline, %llu by size cap (%llu well overdue, waited for the mutex); %llu boundary reaps; record tries skipped (self-locking next) %llu; failed tries: label %llu, flush %llu, reap %llu, poll %llu, idle %llu, capture %llu, poll-record %llu; at the packet's own lock: %llu label groups (%.0f ms), %llu submits (%.0f ms incl. queue 0's reaps), left to it by a failed try: %llu groups, %llu submits; after a capture: %llu groups recorded by a try, %llu packets redone for a queued label over the capture; %llu suspend points; stores: %llu in %llu runs (%llu joined, %llu replaced, %llu WAW barriers, %llu joins refused; %llu runs recorded at submit, %llu in place before a writer or reader); DCC key stores: %llu queued in %llu runs (%llu before a writer, %llu joined a queued range); queued labels: %llu noted (%llu over a recorded entry), %llu same-queue wait hits, %llu groups recorded from a poll loop, %llu from the flush hook (%llu hook accesses, %llu inside a completion skipped)\n", static_cast(packetFlushes.load()), static_cast(deadlineFlushes.load()), static_cast(capFlushes.load()), static_cast(overdueLocks.load()), static_cast(boundaryReaps.load()), static_cast(recordTriesSkipped.load()), failed(TryLabel), failed(TryFlush), failed(TryReap), failed(TryPoll), failed(TryIdle), failed(TryCapture), failed(TryPollRecord), static_cast(packetLockRecords.load()), packetLockRecordUs.load() / 1000.0, static_cast(packetLockSubmits.load()), packetLockSubmitUs.load() / 1000.0, static_cast(packetLockDeferred.load()), static_cast(packetSubmitDeferred.load()), static_cast(captureTryRecords.load()), static_cast(captureRetries.load()), static_cast(suspendPoints.load()), static_cast(stores.stores), static_cast(stores.runs), static_cast(stores.joined), static_cast(stores.replaced), static_cast(stores.wawBarriers), static_cast(stores.joinsRefused), static_cast(stores.runsAtSubmit), static_cast(stores.runsForced), static_cast(stores.keyStores), static_cast(stores.keyStoreRuns), static_cast(stores.keyStoreRunsForWriter), static_cast(stores.keyStoresJoined), static_cast(stores.queuedNoted), static_cast(stores.queuedOverRecorded), static_cast(stores.queuedHits), static_cast(pollLabelRecords.load()), static_cast(hookLabelRecords.load()), static_cast(stores.queuedHookRecords), static_cast(stores.queuedHookInCompletion)); } // WAIT_REG_MEM: another queue or the CPU produces the value; the other queues run on their own threads. @@ -1796,6 +5219,51 @@ private: static const bool overlapSubmit = std::getenv("APS5_NO_WAIT_OVERLAP_SUBMIT") == nullptr; // APS5_WAIT_LOCK=1 takes the GPU mutex at every wait's entry and poll check, as before. static const bool waitLock = std::getenv("APS5_WAIT_LOCK") != nullptr; + // The wait side of the late rule (Recorder::NoteLabel), shared by the entry lookup and the + // poll loop's: a hit must satisfy the packet's compare; a late hit is taken only when no + // CPU store touched the awaited dwords since its group closed (an uncached resetting + // collect of the page, then the block stamps: the tracker mutex alone, after the table + // mutex was released), and a refusal is remembered per generation so the poll loop does + // not repeat the syscall every 200 us. A late hit's submission orders the game's CPU + // stores before it, so the epoch bump stays even for a label of this same queue. + // APS5_TRACE_LATE_LABELS=1 prints the first 200 late decisions. + std::uint64_t refusedGeneration = 0; + // The last table lookup's verdict: the awaited label is stored by a completion action of + // its batch, so this poller must reap for it (the one reap a compute poll loop makes + // while queue 0 can reap: M5 P1). + bool behindCompletion = false; + const auto traceLate = [&](const char* outcome, const std::optional& hit, Graphics::Recorder::LabelRefusal refusal) { + static std::atomic shown{0}; + if (shown.fetch_add(1) >= 200) return; + std::fprintf(stderr, "[late] queue 0x%x wait on 0x%llx (function %u ref 0x%x mask 0x%x) received %llu: %s; entry value 0x%llx stamp %llu generation %llu, refusal %d\n", queue, static_cast(awaited), packet[1] & 7u, packet[4], packet[5], static_cast(received), outcome, + static_cast(hit ? hit->value : 0), static_cast(hit ? hit->stamp : 0), static_cast(hit ? hit->generation : 0), static_cast(refusal)); + }; + const auto takeLabel = [&](const std::optional& hit, Graphics::Recorder::LabelRefusal refusal, bool polling, bool unlockedHit) { + behindCompletion = refusal == Graphics::Recorder::LabelRefusal::BehindCompletion; + const bool trace = TraceLateLabels() && (refusal != Graphics::Recorder::LabelRefusal::None || (hit.has_value() && hit->late)); + if (!hit.has_value() || !Pm4::WaitComparesValue(packet, hit->value)) { + if (trace) traceLate(hit.has_value() ? "value mismatch" : "refused by the table", hit, refusal); + return false; + } + if (hit->late) { + if (hit->generation == refusedGeneration) return false; + if (!GuestMemory::UnchangedSinceCollected(awaited, awaitedBytes, hit->generation)) { + ++outcomes.lateRefusedCpuStore; + refusedGeneration = hit->generation; + if (trace) traceLate("refused, CPU store since the group closed", hit, refusal); + return false; + } + ++outcomes.fromRecorderLate; + if (trace) traceLate("trusted", hit, refusal); + } + ++(polling ? outcomes.fromRecorderPolling : outcomes.fromRecorder); + if (unlockedHit) ++outcomes.fromRecorderUnlocked; + if (hit->queue == queue) { + ++outcomes.fromRecorderSameQueue; + if (!hit->late) epochPoint.bump = false; + } + return true; + }; // Read before the checks below: a write noted before the (locked or snapshot) check is seen // by that check, one noted after it changes the generation the poll loop compares against. std::uint64_t seenGeneration = Graphics::Recorder::WriteGeneration(); @@ -1810,16 +5278,8 @@ private: const bool unlocked = !waitLock && overlapSubmit; if (unlocked) { if (labelShortcut) { - std::uint32_t producer = 0; - if (const auto value = Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { - ++outcomes.fromRecorder; - ++outcomes.fromRecorderUnlocked; - if (producer == queue) { - ++outcomes.fromRecorderSameQueue; - epochPoint.bump = false; - } - return; - } + Graphics::Recorder::LabelRefusal refusal{}; + if (takeLabel(Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, &refusal), refusal, false, true)) return; } } const bool lockedEntry = !unlocked || Graphics::Recorder::SnapshotWriteOverlaps(awaited, awaitedBytes); @@ -1849,16 +5309,9 @@ private: // A label the recorder holds for this range, recorded after the game submitted this // packet, will store the value in order (see Recorder::NoteLabel): the wait is over // without the GPU round trip. The label's batch is submitted by the flush rules. - std::uint32_t producer = 0; if (labelShortcut) { - if (const auto value = localDevice->PendingLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { - ++outcomes.fromRecorder; - if (producer == queue) { - ++outcomes.fromRecorderSameQueue; - epochPoint.bump = false; - } - return; - } + Graphics::Recorder::LabelRefusal refusal{}; + if (takeLabel(localDevice->PendingLabel(awaited, awaitedBytes, received, &refusal), refusal, false, false)) return; } // Recorded work may write the awaited value: send it to the GPU; the poll below sees // the store once it lands in the imported memory. Work not writing the range stays @@ -1883,31 +5336,67 @@ private: // An unlocked entry decided from the snapshot alone (or one whose try failed): the first // check re-does the entry's overlap test under the mutex (a note whose snapshot was not yet // published when the entry looked must still get its batch submitted; see the entry comment). + static const bool pollProfile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& poll = pollStats(); + const auto pollRefused = [&](Graphics::Recorder::LabelRefusal refusal) { + if (pollProfile && refusal != Graphics::Recorder::LabelRefusal::None && static_cast(refusal) < poll.refusals.size()) ++poll.refusals[static_cast(refusal)]; + }; + // A compute poller reaps only for a label the table refused as BehindCompletion (its + // batch's completion action is the store the poll waits for); every other completion + // belongs to queue 0's rate-limited reap, which queue 0's own poller continues inside a + // wait (its boundary reap cannot run there, and the awaited value may itself land only + // through a completion the table cannot describe: a copied buffer's write-back). While + // queue 0 sleeps untimed nobody else reaps, so a compute poller does. Without the table + // (APS5_NO_LABEL_SHORTCUT) no lookup can say BehindCompletion, so the old rule (reap + // whenever completions are pending) stays, as with APS5_POLL_REAP=1. + const bool reapAll = PollReapAll() || !labelShortcut || queue == 0; + std::chrono::steady_clock::time_point lastTry{}; const auto pollService = [&](bool sleeping) { + if (pollProfile) ++poll.ticks; const auto generation = Graphics::Recorder::WriteGeneration(); const bool changed = generation != seenGeneration || recheck; const bool labelDue = LabelFlushDue(); - const bool reap = sleeping && Graphics::Recorder::PendingCompletionLabels() != 0; + const bool reap = sleeping && (behindCompletion || ((reapAll || queue0Dormant.load(std::memory_order_relaxed)) && CompletionsPending())); if (!changed && !labelDue && !reap) return false; + // Only a deadline or a reap due: the try is rate-limited (the producer's record shows as + // a generation change and is tried at once). + if (!changed && !PollTryEach()) { + const auto now = std::chrono::steady_clock::now(); + if (now - lastTry < PollTryInterval) return false; + lastTry = now; + } if (changed && unlocked && labelShortcut) { // The table first, without the mutex: a hit ends the wait. A miss is not final (the // producer may be between its generation bump and its table entry or snapshot), so // the generation is consumed only below, under a successful try_lock, where the // table, the open batch and the snapshot are consistent (a try_lock never waits, // so this costs no mutex wait; a failed one leaves the generation for the next check). - std::uint32_t producer = 0; - if (const auto value = Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { - ++outcomes.fromRecorderPolling; - ++outcomes.fromRecorderUnlocked; - if (producer == queue) { - ++outcomes.fromRecorderSameQueue; - epochPoint.bump = false; - } + Graphics::Recorder::LabelRefusal refusal{}; + if (takeLabel(Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, &refusal), refusal, true, true)) { + if (pollProfile) ++poll.tableHits; return true; } + pollRefused(refusal); } + if (pollProfile) ++poll.tries; std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::try_to_lock); - if (!gpuLock.owns_lock()) return false; + if (!gpuLock.owns_lock()) { + ++triesFailed[TryPoll]; + if (pollProfile) ++poll.triesFailed; + return false; + } + // Destroyed before the unique_lock releases: the hold as the other threads saw it. + struct TryHold { + bool enabled; + PollStats& poll; + std::chrono::steady_clock::time_point start; + ~TryHold() { + if (!enabled) return; + const auto us = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + poll.tryHoldUs += us; + poll.longestTryHoldUs = std::max(poll.longestTryHoldUs, us); + } + } tryHold{pollProfile, poll, pollProfile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}}; const auto localDevice = device.load(); if (localDevice == nullptr) { seenGeneration = generation; @@ -1919,15 +5408,12 @@ private: seenGeneration = generation; recheck = false; if (labelShortcut) { - std::uint32_t producer = 0; - if (const auto value = localDevice->PendingLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { - ++outcomes.fromRecorderPolling; - if (producer == queue) { - ++outcomes.fromRecorderSameQueue; - epochPoint.bump = false; - } + Graphics::Recorder::LabelRefusal refusal{}; + if (takeLabel(localDevice->PendingLabel(awaited, awaitedBytes, received, &refusal), refusal, true, false)) { + if (pollProfile) ++poll.tableHits; return true; } + pollRefused(refusal); } if (localDevice->OpenWriteOverlaps(awaited, awaitedBytes)) { localDevice->SubmitRecorded(false); @@ -1938,10 +5424,19 @@ private: localDevice->SubmitRecorded(false); submitted = true; } - if (submitted) ++outcomes.pollSubmits; + if (submitted) { + ++outcomes.pollSubmits; + if (pollProfile) ++poll.submits; + } if (reap) { + const auto withWorkBefore = pollProfile ? Graphics::Recorder::ReapsWithWork() : 0; localDevice->ReapRecorded(); ++outcomes.pollReaps; + if (pollProfile) { + ++poll.reaps; + ++(behindCompletion ? poll.reapsBehindCompletion : poll.reapsOther); + poll.reapsWithWork += Graphics::Recorder::ReapsWithWork() - withWorkBefore; + } } return false; }; @@ -1995,6 +5490,17 @@ private: if (record.length != 0 && record.target <= address && address < record.target + std::max(record.length, 4)) std::fprintf(stderr, "[gpu] earlier write by queue 0x%x opcode 0x%x at 0x%llx+0x%llx\n", record.queue, record.opcode, static_cast(record.target), static_cast(record.length)); } + // The recorder's view of the awaited range: what a poller could still do for it. + if (const auto reportDevice = device.load(); reportDevice != nullptr) { + Graphics::Recorder::LabelRefusal refusal{}; + const auto hit = reportDevice->PendingLabel(awaited, awaitedBytes, received, &refusal); + const auto since = Graphics::Recorder::PendingLabelSince(); + const auto age = since.has_value() ? std::chrono::duration(std::chrono::steady_clock::now() - *since).count() : -1.0; + std::fprintf(stderr, "[gpu] recorder: table %s (refusal %u), open batch writes it %d, snapshot write overlaps %d, pending completion labels %llu, write-back completions %llu, pending label age %.1f ms, write generation %llu seen %llu, queue 0 dormant %d, completions pending %d, label due %d\n", + hit.has_value() ? "hit" : "miss", static_cast(refusal), reportDevice->OpenWriteOverlaps(awaited, awaitedBytes) ? 1 : 0, Graphics::Recorder::SnapshotWriteOverlaps(awaited, awaitedBytes) ? 1 : 0, + static_cast(Graphics::Recorder::PendingCompletionLabels()), static_cast(Graphics::Recorder::PendingWriteBackCompletions()), age, + static_cast(Graphics::Recorder::WriteGeneration()), static_cast(seenGeneration), queue0Dormant.load(std::memory_order_relaxed) ? 1 : 0, CompletionsPending() ? 1 : 0, LabelFlushDue() ? 1 : 0); + } context.Each([](const std::string& line) { std::fprintf(stderr, "[gpu] preceding packet %s\n", line.c_str()); }); } if (warned) return; @@ -2012,13 +5518,23 @@ private: // reset that follows needs no idle GPU (recorded work keeps what it uses). // Debug aid: APS5_SUSPEND_DRAIN=1 drains as before. static const bool suspendDrain = std::getenv("APS5_SUSPEND_DRAIN") != nullptr; - GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); - std::lock_guard gpuLock(GuestMemory::GpuMutex()); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; ++suspendPoints; - if (const auto localDevice = device.load()) { - recordDeferredLabels(localDevice.get(), submission.queue); - if (suspendDrain) localDevice->WaitIdle(); - else localDevice->SubmitRecorded(submission.queue == 0); + auto& costs = submissionCosts(submission.queue); + // Nothing queued or recorded (the lock-free test of the submission end): no mutex. + if (suspendDrain || !deferredLabels().labels.empty() || Graphics::Recorder::PendingLabelSince().has_value() || Graphics::Recorder::RecordedWorkSinceSubmit() != 0) { + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (const auto localDevice = device.load()) { + recordDeferredLabels(localDevice.get(), submission.queue); + if (suspendDrain) localDevice->WaitIdle(); + else localDevice->SubmitRecorded(submission.queue == 0); + } + ++costs.suspends; + if (profile) costs.suspendNs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); + } else { + ++costs.suspendsSkipped; } resetGraphics = true; return; @@ -2066,6 +5582,9 @@ private: // Time in flushBetweenPackets (deferred label submits, reaps and, on queue 0, the mutex // wait for them), kept apart from the packets so their buckets show their own cost. double flushMs = 0; + // DISPATCH_DIRECT packets by outcome (count, ms): real dispatches, the copy and fill HLEs, + // and packets a tolerated failure skipped. + std::array, static_cast(DispatchOutcome::Count)> dispatchOutcomes{}; }; thread_local PacketProfile packetProfile; ++packetProfile.submissions; @@ -2108,14 +5627,68 @@ private: } auto& waits = waitOutcomes(); auto& epochs = epochBumps(); - std::fprintf(stderr, "[packets] queue 0x%x %llu submissions, time by packet (10 s):%s, flush %.0fms; waits satisfied: at entry %llu, from recorder %llu (%llu while polling, same queue %llu, %llu without the GPU mutex), polled %llu (%llu entered without the GPU mutex, %llu entry tries failed), timed out %llu; poll submits %llu, poll reaps %llu; epoch bumps: submissions %llu, waits %llu, drains %llu, reaps %llu, packets %llu\n", queue, static_cast(profile.submissions), report.c_str(), profile.flushMs, static_cast(waits.atEntry), static_cast(waits.fromRecorder + waits.fromRecorderPolling), static_cast(waits.fromRecorderPolling), static_cast(waits.fromRecorderSameQueue), static_cast(waits.fromRecorderUnlocked), static_cast(waits.polled), static_cast(waits.entriesUnlocked), static_cast(waits.entryTriesFailed), static_cast(waits.timedOut), static_cast(waits.pollSubmits), static_cast(waits.pollReaps), static_cast(epochs.submissions), static_cast(epochs.waits), static_cast(epochs.drains), static_cast(epochs.reaps), static_cast(epochs.packets)); + // The late counters are cumulative per thread: this window's deltas. + const auto late = Graphics::Recorder::LateCounts(); + auto& lateSeen = lateCountsSeen(); + // The per-submission costs of this queue over the window (Driver::Submit's on the + // game thread, the rest on this worker), reset here. + auto& costs = submissionCosts(queue); + const auto submitted = costs.submissions.exchange(0); + const auto perSubmission = [&](std::atomic& ns) { const auto total = ns.exchange(0); return submitted != 0 ? static_cast(total) / 1000.0 / static_cast(submitted) : 0.0; }; + const auto validateUs = perSubmission(costs.validateNs), copyUs = perSubmission(costs.copyNs), dequeueUs = perSubmission(costs.dequeueNs), completeUs = perSubmission(costs.completeNs); + const auto suspends = costs.suspends.exchange(0); + const auto suspendUs = suspends != 0 ? static_cast(costs.suspendNs.exchange(0)) / 1000.0 / static_cast(suspends) : 0.0; + std::string outcomes; + for (std::size_t i = 0; i < profile.dispatchOutcomes.size(); ++i) { + char text[64]; + std::snprintf(text, sizeof(text), " %s %llu %.0fms", DispatchOutcomeNames[i], static_cast(profile.dispatchOutcomes[i].first), profile.dispatchOutcomes[i].second); + outcomes += text; + } + auto& poll = pollStats(); + std::fprintf(stderr, "[poll] queue 0x%x (10 s): ticks %llu, table hits %llu, tries %llu / failed %llu, submits %llu, reaps %llu (%llu with work; by reason: behind completion %llu / other %llu), refusals: %s %llu, %s %llu, %s %llu, %s %llu, %s %llu; longest try hold %.1f us, try holds %.1f ms\n", queue, static_cast(poll.ticks), static_cast(poll.tableHits), static_cast(poll.tries), static_cast(poll.triesFailed), static_cast(poll.submits), static_cast(poll.reaps), static_cast(poll.reapsWithWork), static_cast(poll.reapsBehindCompletion), static_cast(poll.reapsOther), LabelRefusalNames[1], static_cast(poll.refusals[1]), LabelRefusalNames[2], static_cast(poll.refusals[2]), LabelRefusalNames[3], static_cast(poll.refusals[3]), LabelRefusalNames[4], static_cast(poll.refusals[4]), LabelRefusalNames[5], static_cast(poll.refusals[5]), poll.longestTryHoldUs, poll.tryHoldUs / 1000.0); + poll = PollStats{}; + std::fprintf(stderr, "[packets] queue 0x%x %llu submissions, time by packet (10 s):%s, flush %.0fms; DISPATCH_DIRECT by outcome:%s; per submission (%llu submitted): validate %.1f us, copy %.1f us, dequeue %.1f us, complete %.1f us; suspend points %llu x %.1f us, %llu skipped (nothing open); end submits %llu made, %llu skipped, %llu notifies skipped; waits satisfied: at entry %llu, from recorder %llu (%llu while polling, same queue %llu, %llu without the GPU mutex, %llu late-trusted), polled %llu (%llu entered without the GPU mutex, %llu entry tries failed; late candidates %llu: refused cpu-store %llu, overwritten %llu, unclosed %llu, queued %llu), timed out %llu; poll submits %llu, poll reaps %llu; epoch bumps: submissions %llu, waits %llu, drains %llu, reaps %llu, packets %llu\n", queue, static_cast(profile.submissions), report.c_str(), profile.flushMs, outcomes.c_str(), static_cast(submitted), validateUs, copyUs, dequeueUs, completeUs, static_cast(suspends), suspendUs, static_cast(costs.suspendsSkipped.exchange(0)), static_cast(costs.endSubmits.exchange(0)), static_cast(costs.endSkipped.exchange(0)), static_cast(costs.notifiesSkipped.exchange(0)), static_cast(waits.atEntry), static_cast(waits.fromRecorder + waits.fromRecorderPolling), static_cast(waits.fromRecorderPolling), static_cast(waits.fromRecorderSameQueue), static_cast(waits.fromRecorderUnlocked), static_cast(waits.fromRecorderLate), static_cast(waits.polled), static_cast(waits.entriesUnlocked), static_cast(waits.entryTriesFailed), static_cast(late.candidates - lateSeen.candidates), static_cast(waits.lateRefusedCpuStore), static_cast(late.overwritten - lateSeen.overwritten), static_cast(late.unclosed - lateSeen.unclosed), static_cast(late.queued - lateSeen.queued), static_cast(waits.timedOut), static_cast(waits.pollSubmits), static_cast(waits.pollReaps), static_cast(epochs.submissions), static_cast(epochs.waits), static_cast(epochs.drains), static_cast(epochs.reaps), static_cast(epochs.packets)); + lateSeen = late; waits = WaitOutcomes{}; epochs = EpochBumps{}; profile.byOpcode.clear(); profile.submissions = 0; profile.flushMs = 0; + profile.dispatchOutcomes = {}; } } packetTimer{profilePackets, header == FlipPacketHeader ? 0xffffu : opcode, submission.queue, packetProfile, std::chrono::steady_clock::now()}; + // The packet bracket of the dispatch and draw phase lines (see pendingDispatchPhases). + if (profilePackets) { + packetStartedAt() = packetTimer.start; + pendingDispatchPhases() = {}; + pendingDrawPhases() = {}; + } + const auto finishDispatchPacket = [&](bool indirect) { + if (!profilePackets) return; + const auto now = std::chrono::steady_clock::now(); + auto& pending = pendingDispatchPhases(); + if (pending.phases) { + pending.ms[PhaseEpilogue] = std::chrono::duration(now - pending.tailAt).count(); + addDriverPhases(submission.queue != 0 ? OtherQueues : indirect ? Queue0Indirect : Queue0Direct, pending.ms, pending.hit, pending.validated); + } + if (indirect) return; + auto& row = packetProfile.dispatchOutcomes[static_cast(pending.outcome)]; + ++row.first; + row.second += std::chrono::duration(now - packetTimer.start).count(); + }; + const auto finishDrawPacket = [&](bool drawn) { + if (!profilePackets) return; + const auto now = std::chrono::steady_clock::now(); + auto& pending = pendingDrawPhases(); + std::array ms{}; + if (drawn && pending.phases) { + ms = pending.ms; + ms[DrawRowEpilogue] = std::chrono::duration(now - pending.tailAt).count(); + } else { + ms[DrawRowSkipped] = std::chrono::duration(now - packetTimer.start).count(); + } + addDrawPhases(ms, drawn && pending.phases, pending.captures); + }; if (profilePackets) packetProfile.flushMs += std::chrono::duration(packetTimer.start - flushStart).count(); // Label writes are recorded on the GPU behind the work they signal whenever possible, so // the CPU never waits for them. Otherwise packets that write guest memory in order with @@ -2166,6 +5739,7 @@ private: GuestMemory::Write(label->address, bytes, 4); wroteOnGpu = true; } + Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); countLabelOutcome(reason); ++immediateLabels; } @@ -2207,7 +5781,7 @@ private: // first, like a buffer fill (a GPU-direct write-back is recorded before the // store and ordered by its ALL_COMMANDS -> TRANSFER barrier). The CPU and // drained paths get this through the flush hook of GuestMemory::Write. - Graphics::StorageTexture::FlushPending(store->address, bytes.size(), nullptr, "packet store"); + Graphics::StorageTexture::FlushPending(store->address, bytes.size(), nullptr, "packet store", Graphics::PublishScope::PartialUnits); // Earlier labels of this queue go first (queue order); the stamp is taken // under the mutex like a label's. recordDeferredLabels(localDevice.get(), submission.queue); @@ -2223,6 +5797,7 @@ private: wroteOnGpu = true; drained = false; } + Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); } } if (drained) ++storesDrained; @@ -2231,7 +5806,7 @@ private: // a synchronous draw's batch submits the recorder first (CommandBatch); the guest memory a // draw's preparation reads goes through the flush hook. Debug aid: APS5_DRAW_DRAIN=1 restores. static const bool drawDrain = std::getenv("APS5_DRAW_DRAIN") != nullptr; - const bool drawPacket = opcode == 0x2d || opcode == 0x35; + const bool drawPacket = Pm4::DrawOpcode(opcode); // Flips no longer drain either: the frame's batches are submitted and the presenter's blit // follows them on the same queue (see Driver::Present). Debug aid: APS5_SYNC_FLIP=1 restores. static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; @@ -2309,11 +5884,12 @@ private: recent.Record(cursor); if (header == FlipPacketHeader) { CheckFailure(); + std::uint64_t batchesAtFlip = 0, unsignaledAtFlip = 0; if (!drains) { // Frame N's recorded batches go to the GPU now; the presenter's own submission // follows them in queue order, so no wait is needed here and GpuReady returns at - // once (the title paces on flipPendingNum, which the presenter drops after showing - // the frame). + // once (the title paces on flipPendingNum, which the presenter drops after queuing + // the frame's presentation). GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); std::lock_guard gpuLock(GuestMemory::GpuMutex()); const auto localDevice = device.load(); @@ -2321,20 +5897,27 @@ private: // then everything goes out (an empty open batch submits nothing) unless the // record submitted already (a second call would only scan the fences again). if (!recordLabelsForPacket(localDevice.get(), submission.queue) && localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0); + if (localDevice != nullptr) localDevice->FlipBatches(batchesAtFlip, unsignaledAtFlip); } + ++flipsCounted; + if (batchesAtFlip != 0) flipSerial = batchesAtFlip; + flipBatchesUnsignaled += unsignaledAtFlip; // Each flip carries a fresh frame timing record; the per-stage frame statistics upstream // collects are not gathered on this path. auto frame = std::make_shared(++frameSerial); const auto now = FrameTiming::Clock::now(); frame->IncludeSubmission(submission.serial, now, now, now, true); frame->SetFlip(submission.serial, cursor, now, now); + frame->NoteFlipBatches(batchesAtFlip, unsignaledAtFlip); submission.flips.at(cursor)->GpuReady(frame); } else if (opcode == 0x15) { timed(&WorkerProfile::dispatchMs, [&] { tolerate("dispatch", [&] { dispatch(queue, packet, submission); }); }); Graphics::Recorder::CountRecordedWork(); + finishDispatchPacket(false); } else if (opcode == 0x16) { timed(&WorkerProfile::dispatchMs, [&] { tolerate("indirect dispatch", [&] { dispatchIndirect(queue, packet, submission); }); }); Graphics::Recorder::CountRecordedWork(); + finishDispatchPacket(true); } else if (opcode == 0x3c || opcode == 0x93) { static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; const auto waitStart = std::chrono::steady_clock::now(); @@ -2351,27 +5934,32 @@ private: next += nextCount; } } - } else if (opcode == 0x35 || opcode == 0x2d) { + } else if (drawPacket) { + bool drawn = false; timed(&WorkerProfile::drawMs, [&] { tolerate("draw", [&] { static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; + static const bool profileDraws = std::getenv("APS5_PROFILE_DRAW") != nullptr; const auto color = (static_cast(readRegister(queue.context, 0x390)) << 40u) | (static_cast(readRegister(queue.context, 0x318)) << 8u); - try { - draw(queue, packet, submission); - if (traceDraws) std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x ok\n",static_cast(color), readRegister(queue.context, 0x8e)); - } catch (const std::exception& error) { - if (traceDraws) std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x failed: %.160s\n",static_cast(color), readRegister(queue.context, 0x8e), error.what()); - // Name the color target so skipped draws can be matched against the scanout buffer. - char suffix[48]; - std::snprintf(suffix, sizeof(suffix), " [color target 0x%llx]", static_cast(color)); + const auto started = profileDraws ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto countSkip = [&](Graphics::DrawSkip kind) { + if (profileDraws) Graphics::CountDrawSkip(kind, std::chrono::duration(std::chrono::steady_clock::now() - started).count()); + }; + // A draw the decode refused (prechecked without a throw, or thrown): reported + // once per message, and the first per color target and reason dumps its banks. + const auto skipped = [&](const std::string& what) { + if (traceDraws) std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x failed: %.160s\n",static_cast(color), readRegister(queue.context, 0x8e), what.c_str()); + // Name the packet and the color target so skipped draws can be matched against the scanout buffer. + char suffix[80]; + std::snprintf(suffix, sizeof(suffix), " [%s, color target 0x%llx]", Pm4::Name(header).c_str(), static_cast(color)); // Debug aid: the first failing draw per color target and reason saves its register banks to // draw__.regs. static std::set> dumpedTargets; - const std::string reason = std::string(error.what()).substr(0, 48); + const std::string reason = what.substr(0, 48); if (dumpedTargets.insert({color, reason}).second) { char name[64]; std::snprintf(name, sizeof(name), "draw_%llx_%08x.regs", static_cast(color), static_cast(std::hash{}(reason))); if (std::FILE* file = std::fopen(name, "w")) { - std::fprintf(file, "# %s\n", error.what()); + std::fprintf(file, "# %s\n", what.c_str()); recent.Each([&](const std::string& line) { std::fprintf(file, "# packet %s\n", line.c_str()); }); for (const auto& [offset, value] : queue.context) std::fprintf(file, "context %x %08x\n", offset, value); for (const auto& [offset, value] : queue.userConfig) std::fprintf(file, "uconfig %x %08x\n", offset, value); @@ -2379,9 +5967,26 @@ private: std::fclose(file); } } - throw std::runtime_error(error.what() + std::string(suffix)); + reportSkip("draw", what + suffix); + }; + try { + std::string rejected; + const auto verdict = draw(queue, packet, submission, rejected); + drawn = verdict == DrawVerdict::Drawn; + if (verdict == DrawVerdict::Rejected) { + skipped(rejected); + countSkip(Graphics::DrawSkip::Prechecked); + } else if (verdict == DrawVerdict::Nothing) { + countSkip(Graphics::DrawSkip::Nothing); + } else if (traceDraws) { + std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x ok\n",static_cast(color), readRegister(queue.context, 0x8e)); + } + } catch (const std::exception& error) { + skipped(error.what()); + countSkip(Graphics::DrawSkip::Thrown); } }); }); + finishDrawPacket(drawn); } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { if (!wroteOnGpu) Pm4::Execute(packet, queue); } @@ -2392,8 +5997,21 @@ private: // (Driver::WaitIdle and the game threads polling the labels must not depend on a later // packet of some queue). Checked lock-free first: an empty open batch needs no submit. A // compute worker submits without reaping (it must not wait under the mutex for queue 0). + // A queue-0 submission whose successor is already queued leaves its labels queued and the + // batch open to that successor's packets (recorded and submitted at their flush or own + // lock, the mid-submission rule; the label deadline still bounds pollers): the last one + // of a burst submits, so WaitIdle and polling after an idle queue keep their timing. A + // compute queue always submits (its tail would otherwise wait for a poller). + // Debug aid: APS5_SUBMIT_AT_END=1 submits at every submission's end, as before. + static const bool submitAtEnd = std::getenv("APS5_SUBMIT_AT_END") != nullptr; if (!deferredLabels().labels.empty() || Graphics::Recorder::PendingLabelSince().has_value() || Graphics::Recorder::RecordedWorkSinceSubmit() != 0) { - GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + auto& costs = submissionCosts(submission.queue); + if (!submitAtEnd && submission.queue == 0 && workerQueued() != nullptr && workerQueued()->load(std::memory_order_acquire) != 0) { + ++costs.endSkipped; + return; + } + ++costs.endSubmits; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::End); std::lock_guard gpuLock(GuestMemory::GpuMutex()); const auto localDevice = device.load(); // This worker's deferred labels are recorded first (they belong to this submission). @@ -2402,16 +6020,25 @@ private: } } + // The calling worker's pending count (QueueWorker::queued), set by run(); null off a worker. + static const std::atomic*& workerQueued() { + static thread_local const std::atomic* queued = nullptr; + return queued; + } + // An idle worker retires labels waiting in completion actions (see Recorder::AfterCompletions): // without it a label whose only consumer is a game thread, submitted last before the game waits // for it, would never land once every worker sleeps. Non-blocking (a fence status check under // the mutex, tried only); the caller polls it every millisecond while such labels exist. void reapCompletionLabels() { std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::try_to_lock); - if (!gpuLock.owns_lock()) return; + if (!gpuLock.owns_lock()) { + ++triesFailed[TryIdle]; + return; + } // Every idle worker polls this; once another worker reaped the labels there is nothing to // do, and the epoch bump below would only inflate the "reaps" and [guestmem] epoch counts. - if (Graphics::Recorder::PendingCompletionLabels() == 0) return; + if (!CompletionsPending()) return; // The write-backs the reap runs collect outside any submission: a fresh epoch, so no walk // of this worker's last packet is reused for them. bumpEpoch(&EpochBumps::reaps); @@ -2420,27 +6047,41 @@ private: void run(std::uint32_t id) noexcept { OnWorkerThread() = true; + { + char role[32]; + std::snprintf(role, sizeof role, "queue worker 0x%x", id); + PinWorkerThread(role); + } // Named from the start, so a worker's fence waits before its first WAIT_REG_MEM land in its // own row of the [recorder] "fence waits by thread" report and "untagged" is exactly the // presenter and the game threads. GuestMemory::TagGpuLockThread(id); if (id == 0) StartWorkerSampler(); Submission submission; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& costs = submissionCosts(id); try { for (;;) { submission = Submission{}; static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; { std::unique_lock lock(mutex); - auto& pending = workers.at(id).pending; + auto& worker = workers.at(id); + auto& pending = worker.pending; + workerQueued() = &worker.queued; if (traceGpu && pending.empty()) std::fprintf(stderr, "[gpu] %.1f idle queue=0x%x\n", TraceMs(), id); const auto ready = [&] { return failure || stopping || !pending.empty(); }; - // While completion labels are pending, the idle wait wakes every millisecond to - // reap them (the driver mutex is dropped for that: it is never held with the GPU - // mutex); otherwise it sleeps until work arrives. + // While completions are pending, the idle wait wakes every millisecond to reap + // them (the driver mutex is dropped for that: it is never held with the GPU + // mutex); otherwise it sleeps until work arrives. A compute worker wakes for + // completion labels only (a game thread may poll one); write-backs are queue 0's. + // Queue 0's untimed sleep is published: a completion registered during it has + // no reaper until work arrives, so a sleeping poller reaps meanwhile. while (!ready()) { - if (Graphics::Recorder::PendingCompletionLabels() == 0) { + if (!CompletionsPending() || (id != 0 && Graphics::Recorder::PendingCompletionLabels() == 0)) { + if (id == 0) queue0Dormant.store(true, std::memory_order_relaxed); changed.wait(lock, ready); + if (id == 0) queue0Dormant.store(false, std::memory_order_relaxed); break; } if (changed.wait_for(lock, std::chrono::milliseconds(1), ready)) break; @@ -2454,15 +6095,24 @@ private: } submission = std::move(pending.front()); pending.pop_front(); + worker.queued.fetch_sub(1, std::memory_order_acq_rel); + if (profile && submission.enqueuedAt != std::chrono::steady_clock::time_point{}) costs.dequeueNs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - submission.enqueuedAt).count()); } execute(submission); if (traceGpu) std::fprintf(stderr, "[gpu] %.1f done serial=%llu queue=0x%x\n", TraceMs(), static_cast(submission.serial), id); + const auto completeStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + bool notify = true; { std::lock_guard lock(mutex); rethrowFailure(); markCompleted(submission.serial); + // Only WaitIdle waits for a completion (idle workers wait for work, notified at + // its enqueue): the notify is skipped while nobody is in it. + notify = idleWaiters != 0; } - changed.notify_all(); + if (notify) changed.notify_all(); + else ++costs.notifiesSkipped; + if (profile) costs.completeNs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - completeStart).count()); } } catch (...) { const auto error = std::current_exception(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 0789c853b5..561ef3b504 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #ifdef _WIN32 #ifndef NOMINMAX @@ -54,6 +55,8 @@ bool writableProtection(DWORD protection) { namespace { std::atomic forgetCalls{0}; +// See ForgetSerial: odd while a ForgetPages call stores its page states. +std::atomic forgetSerial{0}; std::atomic forgetBytes{0}; std::atomic collectMemoHits{0}; std::atomic collectEpochBumps{0}; @@ -69,21 +72,57 @@ std::size_t ImagePagesSize(); } // Profile (APS5_PROFILE_DRAW): calls, bytes and time of the guest memory copies and comparisons the -// driver makes, printed every 10 s as [guestmem]. +// driver makes, printed every 10 s as [guestmem]. Accumulated per thread (the calling thread is the +// only writer: plain loads and stores, no locked RMW on a cache line every queue worker shares) and +// folded at report time; a finished thread's totals move to `retired`. Nothing is counted without +// the profile switch (the counters are only ever printed under it). +// Debug aid: APS5_SHARED_MEMORY_COUNTERS=1 accumulates into one shared set with atomic RMWs as before. +enum MemoryCounterKind { CounterRead, CounterCompare, CounterWrite, CounterChangedWrite, CounterCollect, CounterVerify, CounterQuery, MemoryCounterCount }; +constexpr const char* MemoryCounterNames[MemoryCounterCount] = {"read", "compare", "write", "write-changed", "collect", "verify", "query"}; + struct MemoryCounter { std::atomic calls{0}; std::atomic bytes{0}; std::atomic nanoseconds{0}; + + void Add(std::uint64_t moreBytes, std::uint64_t moreNanoseconds, bool shared) { + if (shared) { + calls.fetch_add(1, std::memory_order_relaxed); + bytes.fetch_add(moreBytes, std::memory_order_relaxed); + nanoseconds.fetch_add(moreNanoseconds, std::memory_order_relaxed); + return; + } + calls.store(calls.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + bytes.store(bytes.load(std::memory_order_relaxed) + moreBytes, std::memory_order_relaxed); + nanoseconds.store(nanoseconds.load(std::memory_order_relaxed) + moreNanoseconds, std::memory_order_relaxed); + } +}; + +struct MemoryCounters { + std::array counters; + + void FoldInto(std::array, MemoryCounterCount>& totals) const { + for (std::size_t i = 0; i < MemoryCounterCount; ++i) { + totals[i][0] += counters[i].calls.load(std::memory_order_relaxed); + totals[i][1] += counters[i].bytes.load(std::memory_order_relaxed); + totals[i][2] += counters[i].nanoseconds.load(std::memory_order_relaxed); + } + } + // Under the profile's threadsMutex (the only writer of `retired`). + void AddFrom(const MemoryCounters& other) { + for (std::size_t i = 0; i < MemoryCounterCount; ++i) { + counters[i].calls.store(counters[i].calls.load(std::memory_order_relaxed) + other.counters[i].calls.load(std::memory_order_relaxed), std::memory_order_relaxed); + counters[i].bytes.store(counters[i].bytes.load(std::memory_order_relaxed) + other.counters[i].bytes.load(std::memory_order_relaxed), std::memory_order_relaxed); + counters[i].nanoseconds.store(counters[i].nanoseconds.load(std::memory_order_relaxed) + other.counters[i].nanoseconds.load(std::memory_order_relaxed), std::memory_order_relaxed); + } + } }; struct MemoryProfile { - MemoryCounter read; - MemoryCounter compare; - MemoryCounter write; - MemoryCounter changedWrite; - MemoryCounter collect; - MemoryCounter verify; - MemoryCounter query; + std::mutex threadsMutex; + std::vector threads; + MemoryCounters retired; + MemoryCounters shared; std::atomic lastReport{0}; // Sampled callers of Read and Write (module offsets), to name what touches guest memory most. std::mutex callersMutex; @@ -91,12 +130,51 @@ struct MemoryProfile { std::array, 16> writeCallers{}; }; - +// Never destroyed: worker threads' counters unregister at thread exit, which can follow static +// destruction at process exit. MemoryProfile& Profile() { - static MemoryProfile profile; + static MemoryProfile& profile = *new MemoryProfile(); return profile; } +bool SharedMemoryCounters() { + static const bool shared = std::getenv("APS5_SHARED_MEMORY_COUNTERS") != nullptr; + return shared; +} + +struct ThreadMemoryCounters { + MemoryCounters counters; + + ThreadMemoryCounters() { + auto& profile = Profile(); + std::lock_guard lock(profile.threadsMutex); + profile.threads.push_back(&counters); + } + ~ThreadMemoryCounters() { + auto& profile = Profile(); + std::lock_guard lock(profile.threadsMutex); + std::erase(profile.threads, &counters); + profile.retired.AddFrom(counters); + } +}; + +MemoryCounter& CounterFor(MemoryCounterKind kind) { + if (SharedMemoryCounters()) return Profile().shared.counters[kind]; + thread_local ThreadMemoryCounters thread; + return thread.counters.counters[kind]; +} + +// Every thread's counters, the retired and the shared ones summed: calls, bytes, nanoseconds per kind. +std::array, MemoryCounterCount> FoldMemoryCounters() { + auto& profile = Profile(); + std::array, MemoryCounterCount> totals{}; + std::lock_guard lock(profile.threadsMutex); + for (const auto* thread : profile.threads) thread->FoldInto(totals); + profile.retired.FoldInto(totals); + profile.shared.FoldInto(totals); + return totals; +} + void CountCaller(std::array, 16>& callers, std::uint32_t& sampled, std::uint32_t every, const void* returnAddress) { if (++sampled % every != 0) return; auto& state = Profile(); @@ -127,7 +205,7 @@ PacketTag CurrentPacket() { } const char* ReadSiteName(ReadSite site) { - static const char* const names[static_cast(ReadSite::Count)] = {"unknown", "capture", "dispatch-cache", "texture-compare", "texture-read", "buffer-upload", "index-buffer", "vertex-buffer", "registers", "indirect-args", "wait", "label", "scanout", "store"}; + static const char* const names[static_cast(ReadSite::Count)] = {"unknown", "capture", "dispatch-cache", "texture-compare", "texture-read", "buffer-upload", "index-buffer", "vertex-buffer", "registers", "indirect-args", "wait", "label", "scanout", "store", "mirror-refresh", "draw-cache"}; const auto index = static_cast(site); return index < static_cast(ReadSite::Count) ? names[index] : "?"; } @@ -187,32 +265,27 @@ void CountWriteCaller(const void* returnAddress) { CountCaller(Profile().writeCallers, sampled, 16, returnAddress); } +bool MemoryProfiled() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + class TimedAccess { public: - TimedAccess(MemoryCounter& counter, std::uint64_t bytes) : counter(counter), bytes(bytes), start(std::chrono::steady_clock::now()) {} + TimedAccess(MemoryCounterKind kind, std::uint64_t bytes) : kind(kind), bytes(bytes), start(MemoryProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}) {} ~TimedAccess() { + if (!MemoryProfiled()) return; const auto now = std::chrono::steady_clock::now(); - counter.calls += 1; - counter.bytes += bytes; - counter.nanoseconds += static_cast(std::chrono::duration_cast(now - start).count()); - static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - if (!profile) return; + CounterFor(kind).Add(bytes, static_cast(std::chrono::duration_cast(now - start).count()), SharedMemoryCounters()); auto& state = Profile(); const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); auto last = state.lastReport.load(); if (nowMs - last < 10000 || !state.lastReport.compare_exchange_strong(last, nowMs)) return; - const auto line = [](const char* name, const MemoryCounter& c) { - const double seconds = c.nanoseconds / 1e9; - std::fprintf(stderr, " %s %llu calls %.0f MiB %.1f s", name, static_cast(c.calls.load()), c.bytes / 1048576.0, seconds); - }; + const auto totals = FoldMemoryCounters(); std::fprintf(stderr, "[guestmem]"); - line("read", state.read); - line("compare", state.compare); - line("write", state.write); - line("write-changed", state.changedWrite); - line("collect", state.collect); - line("verify", state.verify); - line("query", state.query); + for (std::size_t i = 0; i < MemoryCounterCount; ++i) { + std::fprintf(stderr, " %s %llu calls %.0f MiB %.1f s", MemoryCounterNames[i], static_cast(totals[i][0]), totals[i][1] / 1048576.0, totals[i][2] / 1e9); + } std::fprintf(stderr, " collect-memo hits %llu, collect epochs %llu", static_cast(collectMemoHits.load()), static_cast(collectEpochBumps.load())); std::fprintf(stderr, " collect-dirty %llu tracker waits %llu / %llu", static_cast(collectDirty.load()), static_cast(trackerWaits.load()), static_cast(trackerAcquisitions.load())); std::fprintf(stderr, " | arena 0x%llx+0x%llx image 0x%llx+0x%llx forgets %llu (%.0f MiB)", static_cast(PagesBase()), static_cast(PagesSize()), static_cast(ImagePagesBase()), static_cast(ImagePagesSize()), static_cast(forgetCalls.load()), forgetBytes.load() / 1048576.0); @@ -235,7 +308,7 @@ public: std::fprintf(stderr, "\n"); } private: - MemoryCounter& counter; + MemoryCounterKind kind; std::uint64_t bytes; std::chrono::steady_clock::time_point start; }; @@ -356,7 +429,9 @@ std::size_t ImagePagesSize() { void ForgetPages(std::uintptr_t address, std::size_t bytes) { forgetCalls.fetch_add(1, std::memory_order_relaxed); forgetBytes.fetch_add(bytes, std::memory_order_relaxed); + forgetSerial.fetch_add(1, std::memory_order_release); Pages().forget(address, bytes); + forgetSerial.fetch_add(1, std::memory_order_release); } void PageStates::initialize() { @@ -411,7 +486,7 @@ bool describePages(std::uintptr_t address, std::size_t bytes, Emit&& emit) { } } #ifdef _WIN32 - const TimedAccess timed(Profile().query, 0); + const TimedAccess timed(CounterQuery, 0); MEMORY_BASIC_INFORMATION memory{}; const auto queryStart = std::chrono::steady_clock::now(); // A mutation that changes these pages while they are being queried bumps the generation @@ -477,7 +552,7 @@ bool describePages(std::uintptr_t address, std::size_t bytes, Emit&& emit) { // Checks that [address, address + bytes) is mapped with read (and, if asked, write) access. Returns // an empty string when it is, otherwise why it is not. std::string verify(std::uintptr_t address, std::size_t bytes, bool writable) { - const TimedAccess timed(Profile().verify, bytes); + const TimedAccess timed(CounterVerify, bytes); std::string reason; const bool queried = describePages(address, bytes, [&](const PageRun& run) { if (!run.readable) { @@ -514,9 +589,13 @@ bool Accessible(const void* pointer, std::size_t bytes, bool writable) { return address != 0 && bytes <= std::numeric_limits::max() - address && verify(address, bytes, writable).empty(); } +std::uint64_t ForgetSerial() { + return forgetSerial.load(std::memory_order_acquire); +} + Commitment DescribeCommitted(std::uint64_t address, std::size_t bytes, bool writable) { require(bytes <= std::numeric_limits::max() - address, "address range overflow"); - const TimedAccess timed(Profile().verify, bytes); + const TimedAccess timed(CounterVerify, bytes); Commitment result; const bool queried = describePages(static_cast(address), bytes, [&](const PageRun& run) { if (run.readable && (!writable || run.writable)) { @@ -546,6 +625,10 @@ struct WriteTracker { std::size_t size = 0; // Generation of the last collected write per 64 KiB block of the arena. std::vector blocks; + // The same, stamped by collects only (dirty pages = CPU stores), never by MarkWritten: the + // late label rule asks whether the CPU touched a label's block, and the driver's own GPU label + // records in the same block (the title's per-job slots are 0x20 apart) must not count. + std::vector cpuBlocks; // Atomic only so a per-thread memo hit can read it without the mutex (every mutation and every // stamp still happen under it): a hit must return the current value, including MarkWritten // bumps, or an image refreshed after a stamp would keep failing UnchangedSince until the next @@ -575,6 +658,7 @@ struct WriteTracker { watched = size != 0 && GuestArena::GuestArenaWriteWatched_nid_postfix(); if (!watched) return; blocks.assign(size / WriteBlockBytes + 1, 0); + cpuBlocks.assign(size / WriteBlockBytes + 1, 0); pages.resize(1u << 16); } }; @@ -669,7 +753,7 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi } } } - const TimedAccess timed(Profile().collect, bytes); + const TimedAccess timed(CounterCollect, bytes); ++tracker.generation; // One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing // after the walk passed a page is reported by the next walk, and a clean page is not touched by the @@ -685,7 +769,11 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi return 0; } if (count != 0) dirty = true; - for (ULONG_PTR i = 0; i < count; ++i) tracker.blocks[(reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes] = tracker.generation; + for (ULONG_PTR i = 0; i < count; ++i) { + const auto block = (reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes; + tracker.blocks[block] = tracker.generation; + tracker.cpuBlocks[block] = tracker.generation; + } if (count < tracker.pages.size()) break; cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); } @@ -697,7 +785,11 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi ULONG_PTR count = tracker.pages.size(); DWORD granularity = 0; if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, reinterpret_cast(cursor), static_cast(stop - cursor), tracker.pages.data(), &count, &granularity) != 0) return 0; - for (ULONG_PTR i = 0; i < count; ++i) tracker.blocks[(reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes] = tracker.generation; + for (ULONG_PTR i = 0; i < count; ++i) { + const auto block = (reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes; + tracker.blocks[block] = tracker.generation; + tracker.cpuBlocks[block] = tracker.generation; + } if (count < tracker.pages.size()) break; cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); } @@ -747,15 +839,71 @@ bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t gene return true; } -void MarkWritten(std::uint64_t address, std::size_t bytes) { +bool UnchangedSinceAll(std::span queries) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched) return false; + for (const auto& [address, bytes, generation] : queries) { + if (generation == 0 || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return false; + const auto first = (address - tracker.base) / WriteBlockBytes; + const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + for (auto block = first; block <= last; ++block) { + if (tracker.blocks[block] > generation) return false; + } + } + return true; +} + +std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return 0; + const auto first = (address - tracker.base) / WriteBlockBytes; + const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + ++tracker.generation; + for (auto block = first; block <= last; ++block) tracker.blocks[block] = tracker.generation; + return tracker.generation; +} + +std::uint64_t TrackerGeneration() { + auto& tracker = Tracker(); + // Under the mutex: a collect bumps the generation before its walk, and a walk still running + // when the value is read absorbs stores made after it with a stamp not newer than the value. + const auto lock = lockTracker(tracker); + return tracker.generation; +} + +bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { + if (CollectWritesUncached(address, bytes) == 0) return false; + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + if (!tracker.watched || generation == 0 || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return false; + const auto first = (address - tracker.base) / WriteBlockBytes; + const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + for (auto block = first; block <= last; ++block) { + if (tracker.cpuBlocks[block] > generation) return false; + } + return true; +} + +void ChangedBlocks(std::uint64_t address, std::size_t bytes, std::span generations, std::span changed, std::span cpu) { + std::fill(changed.begin(), changed.end(), std::uint8_t{1}); + std::fill(cpu.begin(), cpu.end(), std::uint8_t{1}); auto& tracker = Tracker(); const auto lock = lockTracker(tracker); tracker.initialize(); if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return; const auto first = (address - tracker.base) / WriteBlockBytes; const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; - ++tracker.generation; - for (auto block = first; block <= last; ++block) tracker.blocks[block] = tracker.generation; + for (auto block = first; block <= last; ++block) { + const auto k = block - first; + if (k >= generations.size() || k >= changed.size()) break; + const auto generation = generations[k]; + changed[k] = generation == 0 || tracker.blocks[block] > generation ? 1 : 0; + if (k < cpu.size()) cpu[k] = generation == 0 || tracker.cpuBlocks[block] > generation ? 1 : 0; + } } namespace { @@ -763,7 +911,48 @@ namespace { // Per-thread GpuMutex wait accounting (APS5_PROFILE_DRAW): a queue worker tags its thread, other // threads (the presenter) report untagged. constexpr std::size_t GpuLockSiteCount = static_cast(GpuLockSite::Count); -constexpr const char* GpuLockSiteNames[GpuLockSiteCount] = {"other", "dispatch", "indirect", "draw", "hook", "wait", "label", "flush", "present", "fill", "try"}; +constexpr const char* GpuLockSiteNames[GpuLockSiteCount] = {"other", "dispatch", "indirect", "draw", "hook", "wait", "label", "flush", "present", "fill", "copy", "end", "try"}; + +// Hold-side holder matrix: nanoseconds held per (holder thread, site), added by EndHold. Column 0 +// is every untagged thread (the presenter, game threads); the queue workers take a column each on +// their first hold. A contended lock() snapshots the matrix before and after its wait and credits +// the deltas (the holds that ended during the wait) to the waiter's site. +constexpr std::size_t HolderColumns = 12; +constexpr std::uint32_t UnassignedHolder = 0xfffffffeu; +constexpr std::uint32_t PresenterHolderTag = 0xfffffffdu; +constexpr int PresenterColumn = 1; +using HolderMatrixValues = std::array, HolderColumns>; +struct HolderMatrix { + std::array, HolderColumns> tags{}; + std::array, GpuLockSiteCount>, HolderColumns> heldNs{}; + HolderMatrix() { + for (auto& tag : tags) tag.store(UnassignedHolder, std::memory_order_relaxed); + tags[0].store(0xffffffffu, std::memory_order_relaxed); + tags[PresenterColumn].store(PresenterHolderTag, std::memory_order_relaxed); + } + void Snapshot(HolderMatrixValues& into) const { + for (std::size_t h = 0; h < HolderColumns; ++h) { + for (std::size_t s = 0; s < GpuLockSiteCount; ++s) into[h][s] = heldNs[h][s].load(std::memory_order_relaxed); + } + } +}; + +HolderMatrix& Holders() { + static HolderMatrix matrix; + return matrix; +} + +std::size_t HolderColumnFor(std::uint32_t tag) { + if (tag == 0xffffffffu) return 0; + auto& holders = Holders(); + for (std::size_t i = 1; i < HolderColumns; ++i) { + auto expected = UnassignedHolder; + if (holders.tags[i].load(std::memory_order_acquire) == tag) return i; + if (holders.tags[i].compare_exchange_strong(expected, tag, std::memory_order_acq_rel)) return i; + if (expected == tag) return i; + } + return 0; +} struct GpuLockStats { std::uint32_t tag = 0xffffffffu; @@ -784,6 +973,14 @@ struct GpuLockStats { std::array siteMaxHeldMs{}; std::size_t holdSite = 0; std::chrono::steady_clock::time_point holdStart; + // This thread's column of the holder matrix (-1 until its first hold), and what its contended + // acquisitions waited behind: holds by (holder, site) that ended during the waits. + int holderColumn = -1; + HolderMatrixValues waitedBehindNs{}; + std::uint64_t tries = 0; + std::uint64_t triesFailed = 0; + std::uint64_t lockedGpuWaits = 0; + double lockedGpuWaitMs = 0; std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; @@ -812,10 +1009,41 @@ void BeginHold(std::size_t site) { void EndHold() { auto& stats = LockStats(); - const auto heldMs = std::chrono::duration(std::chrono::steady_clock::now() - stats.holdStart).count(); + const auto heldNs = std::chrono::duration_cast(std::chrono::steady_clock::now() - stats.holdStart).count(); + const auto heldMs = static_cast(heldNs) / 1e6; ++stats.siteHolds[stats.holdSite]; stats.siteHeldMs[stats.holdSite] += heldMs; stats.siteMaxHeldMs[stats.holdSite] = std::max(stats.siteMaxHeldMs[stats.holdSite], heldMs); + if (stats.holderColumn < 0) stats.holderColumn = static_cast(HolderColumnFor(stats.tag)); + Holders().heldNs[static_cast(stats.holderColumn)][stats.holdSite].fetch_add(static_cast(heldNs), std::memory_order_relaxed); +} + +const char* HolderName(std::size_t column, char* text, std::size_t size) { + const auto tag = Holders().tags[column].load(std::memory_order_relaxed); + if (tag == 0xffffffffu) std::snprintf(text, size, "untagged"); + else if (tag == PresenterHolderTag) std::snprintf(text, size, "presenter"); + else std::snprintf(text, size, "q0x%x", tag); + return text; +} + +// The waiter's matrix as text, largest first, entries of at least a millisecond. +std::string WaitedBehindReport(const HolderMatrixValues& waited) { + std::vector> entries; + for (std::size_t h = 0; h < HolderColumns; ++h) { + for (std::size_t s = 0; s < GpuLockSiteCount; ++s) { + if (waited[h][s] >= 1000000) entries.emplace_back(waited[h][s], h, s); + } + } + std::sort(entries.begin(), entries.end(), [](const auto& a, const auto& b) { return std::get<0>(a) > std::get<0>(b); }); + std::string report; + char text[96]; + char holder[24]; + for (std::size_t i = 0; i < entries.size() && i < 16; ++i) { + const auto& [ns, h, s] = entries[i]; + std::snprintf(text, sizeof(text), " %s/%s %.0f", HolderName(h, holder, sizeof(holder)), GpuLockSiteNames[s], static_cast(ns) / 1e6); + report += text; + } + return report; } } @@ -843,7 +1071,11 @@ bool GpuMutexType::try_lock() { // acquisition); an untagged try's hold is charged to 'try' (a boundary reap, an idle worker). auto& stats = LockStats(); const auto tagged = std::exchange(stats.nextSite, GpuLockSite::Other); - if (!mutex.try_lock()) return false; + ++stats.tries; + if (!mutex.try_lock()) { + ++stats.triesFailed; + return false; + } acquired(); if (depth == 1 && GpuHoldProfiled()) BeginHold(static_cast(tagged == GpuLockSite::Other ? GpuLockSite::Try : tagged)); return true; @@ -901,11 +1133,21 @@ void GpuMutexType::lock() { // An uncontended (or recursive) acquisition costs no clock reads for the wait; only a wait is // timed. The hold timer (outermost acquisitions only) starts once the mutex is held. if (!mutex.try_lock()) { + const bool holders = GpuHoldProfiled(); + HolderMatrixValues before{}; + if (holders) Holders().Snapshot(before); const auto start = std::chrono::steady_clock::now(); mutex.lock(); acquired(); const auto now = std::chrono::steady_clock::now(); - if (depth == 1 && GpuHoldProfiled()) { + if (holders) { + HolderMatrixValues after{}; + Holders().Snapshot(after); + for (std::size_t h = 0; h < HolderColumns; ++h) { + for (std::size_t s = 0; s < GpuLockSiteCount; ++s) stats.waitedBehindNs[h][s] += after[h][s] - before[h][s]; + } + } + if (depth == 1 && holders) { stats.holdSite = site; stats.holdStart = now; } @@ -942,8 +1184,12 @@ void GpuMutexType::lock() { std::snprintf(text, sizeof(text), " %s %llu %.0f ms (max %.1f)", GpuLockSiteNames[i], static_cast(stats.siteHolds[i]), stats.siteHeldMs[i], stats.siteMaxHeldMs[i]); holds += text; } + holds += "; waited behind (holder/site ms):" + WaitedBehindReport(stats.waitedBehindNs); } - if (stats.tag == 0xffffffffu) std::fprintf(stderr, "[lock] untagged thread waited %.0f ms for the GPU mutex in %llu of %llu acquisitions (%.0f s); by site (acquisitions/waits, waited):%s%s\n", stats.waitedMs, static_cast(stats.waits), static_cast(stats.acquisitions), interval, sites.c_str(), holds.c_str()); + char counts[160]; + std::snprintf(counts, sizeof(counts), "; lock() %llu / try %llu / try failed %llu; locked GPU waits %llu / %.0f ms", static_cast(stats.acquisitions), static_cast(stats.tries), static_cast(stats.triesFailed), static_cast(stats.lockedGpuWaits), stats.lockedGpuWaitMs); + holds += counts; + if (stats.tag == 0xffffffffu) std::fprintf(stderr, "[lock] %s waited %.0f ms for the GPU mutex in %llu of %llu acquisitions (%.0f s); by site (acquisitions/waits, waited):%s%s\n", stats.holderColumn == PresenterColumn ? "presenter" : "untagged thread", stats.waitedMs, static_cast(stats.waits), static_cast(stats.acquisitions), interval, sites.c_str(), holds.c_str()); else std::fprintf(stderr, "[lock] queue 0x%x waited %.0f ms for the GPU mutex in %llu of %llu acquisitions (%.0f s); by site (acquisitions/waits, waited):%s%s\n", stats.tag, stats.waitedMs, static_cast(stats.waits), static_cast(stats.acquisitions), interval, sites.c_str(), holds.c_str()); stats.waitedMs = 0; stats.waits = 0; @@ -954,6 +1200,11 @@ void GpuMutexType::lock() { stats.siteHolds.fill(0); stats.siteHeldMs.fill(0); stats.siteMaxHeldMs.fill(0); + for (auto& row : stats.waitedBehindNs) row.fill(0); + stats.tries = 0; + stats.triesFailed = 0; + stats.lockedGpuWaits = 0; + stats.lockedGpuWaitMs = 0; return; } acquired(); @@ -966,6 +1217,22 @@ void TagGpuLockSite(GpuLockSite site) { LockStats().nextSite = site; } +void MarkPresenterThread() { + LockStats().holderColumn = PresenterColumn; +} + +void NoteLockedGpuWait(double ms) { + if (!GpuLockProfiled() || GpuMutex().DepthOnThisThread() == 0) return; + auto& stats = LockStats(); + ++stats.lockedGpuWaits; + stats.lockedGpuWaitMs += ms; +} + +std::uint64_t ThreadCollectedBytes() { + if (!MemoryProfiled()) return 0; + return CounterFor(CounterCollect).bytes.load(std::memory_order_relaxed); +} + std::uint32_t GpuLockThreadTag() { return LockStats().tag; } @@ -998,7 +1265,7 @@ void FlushGpuWrites(std::uint64_t address, std::size_t bytes) { void Read(std::uint64_t address, std::span destination, std::size_t alignment) { if (destination.empty()) return; FlushGpuWrites(address, destination.size()); - const TimedAccess timed(Profile().read, destination.size()); + const TimedAccess timed(CounterRead, destination.size()); CountReadCaller(__builtin_return_address(0)); const auto* source = reinterpret_cast(address); try { @@ -1014,7 +1281,7 @@ void Read(std::uint64_t address, std::span destination, std::size_t a void ReadCommitted(std::uint64_t address, std::span destination) { if (destination.empty()) return; FlushGpuWrites(address, destination.size()); - const TimedAccess timed(Profile().read, destination.size()); + const TimedAccess timed(CounterRead, destination.size()); CountReadCaller(__builtin_return_address(0)); if (Accessible(reinterpret_cast(address), destination.size())) { std::memcpy(destination.data(), reinterpret_cast(address), destination.size()); @@ -1027,7 +1294,12 @@ void ReadCommitted(std::uint64_t address, std::span destination) { bool EqualsCommitted(std::uint64_t address, std::span bytes) { if (bytes.empty()) return true; FlushGpuWrites(address, bytes.size()); - const TimedAccess timed(Profile().compare, bytes.size()); + return EqualsCommittedUnsynced(address, bytes); +} + +bool EqualsCommittedUnsynced(std::uint64_t address, std::span bytes) { + if (bytes.empty()) return true; + const TimedAccess timed(CounterCompare, bytes.size()); if (Accessible(reinterpret_cast(address), bytes.size())) return std::memcmp(reinterpret_cast(address), bytes.data(), bytes.size()) == 0; for (const auto& [begin, end] : CommittedRanges(address, bytes.size())) { if (std::memcmp(reinterpret_cast(begin), bytes.data() + (begin - address), static_cast(end - begin)) != 0) return false; @@ -1035,6 +1307,31 @@ bool EqualsCommitted(std::uint64_t address, std::span bytes) { return true; } +Compare CompareMapped(std::uint64_t address, std::span bytes) { + if (bytes.empty()) return Compare::Equal; + if (address == 0 || bytes.size() > std::numeric_limits::max() - address) return Compare::Unmapped; + auto outcome = Compare::Equal; + // Each run's page state is consulted right before its bytes are compared. + const bool queried = describePages(static_cast(address), bytes.size(), [&](const PageRun& run) { + if (!run.readable) outcome = Compare::Unmapped; + else if (std::memcmp(reinterpret_cast(run.begin), bytes.data() + (run.begin - address), static_cast(run.end - run.begin)) != 0) outcome = Compare::Differs; + return outcome == Compare::Equal; + }); + return queried ? outcome : Compare::Unmapped; +} + +Compare CopyMapped(std::uint64_t address, std::span out) { + if (out.empty()) return Compare::Equal; + if (address == 0 || out.size() > std::numeric_limits::max() - address) return Compare::Unmapped; + auto outcome = Compare::Equal; + const bool queried = describePages(static_cast(address), out.size(), [&](const PageRun& run) { + if (!run.readable) outcome = Compare::Unmapped; + else std::memcpy(out.data() + (run.begin - address), reinterpret_cast(run.begin), static_cast(run.end - run.begin)); + return outcome == Compare::Equal; + }); + return queried ? outcome : Compare::Unmapped; +} + void WriteChangedCommitted(std::uint64_t address, std::span current, std::span original) { require(current.size() == original.size(), "write-back snapshot sizes differ"); if (Accessible(reinterpret_cast(address), current.size(), true)) { @@ -1055,7 +1352,7 @@ void WriteChanged(std::uint64_t address, std::span current, std // site above it was doing (the [hooksync] line counts stores apart). const ReadSiteScope site(ReadSite::Store); FlushGpuWrites(address, current.size()); - const TimedAccess timed(Profile().changedWrite, current.size()); + const TimedAccess timed(CounterChangedWrite, current.size()); auto* destination = reinterpret_cast(address); CheckRange(destination, current.size(), 1, true); constexpr std::size_t block = 256; @@ -1081,7 +1378,7 @@ void Write(std::uint64_t address, std::span source, std::size_t // A store, as in WriteChanged. const ReadSiteScope site(ReadSite::Store); FlushGpuWrites(address, source.size()); - const TimedAccess timed(Profile().write, source.size()); + const TimedAccess timed(CounterWrite, source.size()); CountWriteCaller(__builtin_return_address(0)); auto* destination = reinterpret_cast(address); CheckRange(destination, source.size(), alignment, true); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index 6e23d0166c..cc8578d619 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -41,6 +41,20 @@ bool TraceContextState() { return value; } +// Kill switch: APS5_NO_INDIRECT_DRAW=1 rejects the indirect draw packets at submission validation as +// before they were implemented (the driver then dumps the submission as it did). +bool IndirectDrawsDisabled() { + static const bool value = std::getenv("APS5_NO_INDIRECT_DRAW") != nullptr; + return value; +} + +// An SH register an indirect draw packet may name for the CP's patch: the sentinel 0x280 (write +// nowhere) or a user-data register of the vertex-side stage banks (0x8c.. for the vertex / geometry +// programs, 0x10c.. for the local / hull programs). +bool drawLocation(std::uint32_t value) { + return value == 0x280u || value - 0x8cu < 32u || value - 0x10cu < 32u; +} + Registers& registersFor(QueueState& queue, std::uint32_t opcode) { if (opcode == 0x69 || opcode == 0x9f) return queue.context; if (opcode == 0x76 || opcode == 0x63) return queue.shader; @@ -128,7 +142,10 @@ std::string_view UnsupportedReason(std::uint32_t header) { case 0x2a: case 0x2d: case 0x2f: case 0x35: case 0x37: case 0x40: case 0x42: case 0x46: case 0x50: case 0x58: case 0x63: case 0x64: case 0x69: case 0x76: case 0x79: case 0x7a: case 0x81: case 0x83: case 0x9f: return {}; - case 0x24: case 0x25: case 0x27: case 0x2c: case 0x38: case 0x3a: case 0x8d: + case 0x24: case 0x25: case 0x2c: case 0x38: + if (IndirectDrawsDisabled()) return "graphics draw, shader stages and guest render-target materialization are not implemented"; + return {}; + case 0x27: case 0x3a: case 0x8d: return "graphics draw, shader stages and guest render-target materialization are not implemented"; case 0x20: return "GPU query predication is not implemented"; case 0x22: return "conditional command execution and conditional flip reservation are not implemented"; @@ -214,6 +231,26 @@ void Validate(std::span packet, std::uint32_t queue) { require(packet[3] <= packet[1], "index count exceeds maximum index size"); require((packet[4] & ~0x20u) == 0, "unsupported indexed draw flags"); break; + case 0x24: case 0x25: + graphics(); + size(5); + require((packet[1] & 3u) == 0, "misaligned indirect draw offset"); + require((packet[2] >> 16u) == 0 && (packet[3] >> 16u) == 0, "start-index location semantics are not implemented"); + require(drawLocation(packet[2]) && drawLocation(packet[3]), "invalid indirect draw register location"); + require((packet[4] & ~0x20u) == (opcode == 0x24 ? 2u : 0u), "unsupported indirect draw initiator"); + break; + case 0x2c: case 0x38: + graphics(); + size(10); + require((packet[1] & 3u) == 0, "misaligned indirect draw offset"); + require((packet[2] >> 16u) == 0 && (packet[3] >> 16u) == 0, "start-index location semantics are not implemented"); + require(drawLocation(packet[2]) && drawLocation(packet[3]) && drawLocation(packet[4] & 0xffffu), "invalid indirect draw register location"); + require((packet[4] & ~(0xffffu | (1u << 30u) | (1u << 31u))) == 0, "indirect multi-draw control bits are not implemented"); + if ((packet[4] & (1u << 30u)) != 0) require((packet[6] & 3u) == 0 && address(packet[6], packet[7]) != 0, "invalid indirect draw count address"); + else require(address(packet[6], packet[7]) == 0, "count address without an indirect draw count"); + require((packet[8] & 3u) == 0 && packet[8] >= (opcode == 0x2c ? 16u : 20u), "invalid indirect draw stride"); + require((packet[9] & ~0x20u) == (opcode == 0x2c ? 2u : 0u), "unsupported indirect draw initiator"); + break; case 0x15: size(5); if ((packet[4] & ~0x8020u) != 0x41u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break; case 0x16: require(packet.size() == 3 || packet.size() == 4, "invalid indirect dispatch size"); @@ -433,7 +470,7 @@ bool UsesGpuCacheBarrier(std::span packet) { bool AccessesMemory(std::uint32_t header) { switch ((header >> 8u) & 0xffu) { - case 0x3c: case 0x93: case 0x49: case 0x16: case 0x2d: case 0x35: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true; + case 0x3c: case 0x93: case 0x49: case 0x16: case 0x2d: case 0x35: case 0x24: case 0x25: case 0x2c: case 0x38: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true; default: return false; } } @@ -457,8 +494,83 @@ std::array ResolveDispatch(std::span pack return ReadDispatchArguments(DispatchArgumentAddress(packet, queue), packet.back()); } +namespace { + +// The index buffer state an indexed draw binds: the index size for the journaled INDEX_TYPE and the +// checks every indexed draw makes on INDEX_BASE. +std::uint32_t indexSizeOf(const QueueState& queue) { + require(queue.indexType <= 2, "unsupported index type"); + const std::uint32_t indexSize = queue.indexType == 0 ? 2 : queue.indexType == 1 ? 4 : 1; + require(queue.indexBase != 0 && queue.indexBase % indexSize == 0, "null or misaligned index base"); + return indexSize; +} + +DrawParameters resolveIndirectDraw(std::span packet, const QueueState& queue) { + const auto opcode = (packet[0] >> 8u) & 0xffu; + const bool indexed = opcode == 0x25 || opcode == 0x38; + const bool multi = opcode == 0x2c || opcode == 0x38; + require(queue.drawIndirectBase != 0, "indirect draw base has not been set"); + require(packet[1] <= std::numeric_limits::max() - queue.drawIndirectBase, "indirect draw address overflow"); + DrawParameters::IndirectDraw indirect{}; + indirect.arguments = queue.drawIndirectBase + packet[1]; + indirect.opcode = opcode; + indirect.recordBytes = indexed ? 20 : 16; + indirect.stride = multi ? packet[8] : indirect.recordBytes; + indirect.count = multi ? packet[5] : 1; + indirect.countIndirect = multi && (packet[4] & (1u << 30u)) != 0; + indirect.countAddress = multi ? address(packet[6], packet[7]) : 0; + indirect.baseVertexLocation = packet[2]; + indirect.startInstanceLocation = packet[3]; + indirect.drawIndexLocation = multi ? packet[4] & 0xffffu : 0x280u; + indirect.drawIndexEnabled = multi && (packet[4] >> 31u) != 0; + indirect.indxOffset = 0; + require(indirect.count <= (std::numeric_limits::max() - indirect.arguments - indirect.recordBytes) / indirect.stride, "indirect draw record range overflow"); + // The counts are unknown here (the CP takes them from the record; NUM_INSTANCES is ignored). + DrawParameters draw{0, 0, 0, 0, packet.back() & 0x20u, indexed, 0, 0}; + if (!indexed) { + const auto offset = queue.userConfig.find(0x24a); + require(offset != queue.userConfig.end(), "missing GE_INDX_OFFSET register"); + draw.firstVertex = offset->second; + indirect.indxOffset = offset->second; + } else { + // The CP clamps firstIndex + indexCount to INDEX_BUFFER_SIZE: the declared range is the + // hardware's view of the index buffer, so that is what gets bound. + const auto indexSize = indexSizeOf(queue); + require(queue.indexBufferSize != 0, "INDEX_BUFFER_SIZE has not been set"); + const auto bytes = static_cast(queue.indexBufferSize) * indexSize; + require(bytes <= std::numeric_limits::max() && bytes <= std::numeric_limits::max() - queue.indexBase, "index range size overflow"); + GuestMemory::CheckRange(reinterpret_cast(queue.indexBase), static_cast(bytes), indexSize); + draw.indexAddress = queue.indexBase; + draw.indexCount = queue.indexBufferSize; + draw.indexSize = indexSize; + } + draw.indirect = indirect; + return draw; +} + +} + +DrawArguments ReadDrawArguments(const DrawParameters::IndirectDraw& indirect, std::uint32_t record) { + require(record < indirect.count, "indirect draw record index exceeds the packet's count"); + std::array words{}; + // Named for the [hooksync] attribution (the read goes through the flush hook). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::IndirectArguments); + GuestMemory::Read(indirect.arguments + static_cast(record) * indirect.stride, std::as_writable_bytes(std::span(words)).first(indirect.recordBytes), 4); + if (indirect.recordBytes == 20) return {words[0], words[1], words[2], words[3], words[4]}; + return {words[0], words[1], words[2], 0, words[3]}; +} + +std::uint32_t ReadDrawCount(const DrawParameters::IndirectDraw& indirect) { + require(indirect.countIndirect && indirect.countAddress != 0, "indirect draw has no count address"); + std::uint32_t count = 0; + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::IndirectArguments); + GuestMemory::Read(indirect.countAddress, std::as_writable_bytes(std::span(&count, 1)), 4); + return count; +} + DrawParameters ResolveDraw(std::span packet, const QueueState& queue) { Validate(packet, 0); + if (IndirectDrawOpcode((packet[0] >> 8u) & 0xffu)) return resolveIndirectDraw(packet, queue); if (((packet[0] >> 8u) & 0xffu) == 0x2d) { const auto offset = queue.userConfig.find(0x24a); require(offset != queue.userConfig.end(), "missing GE_INDX_OFFSET register"); @@ -467,9 +579,7 @@ DrawParameters ResolveDraw(std::span packet, const QueueSta return {0, packet[1], 0, queue.instanceCount, packet[2] & 0x20u, false, firstVertex, 0}; } require(((packet[0] >> 8u) & 0xffu) == 0x35, "expected DRAW_INDEX_OFFSET_2 packet"); - require(queue.indexType <= 2, "unsupported index type"); - const std::uint32_t indexSize = queue.indexType == 0 ? 2 : queue.indexType == 1 ? 4 : 1; - require(queue.indexBase != 0 && queue.indexBase % indexSize == 0, "null or misaligned index base"); + const auto indexSize = indexSizeOf(queue); const auto offset = static_cast(packet[2]) * indexSize; require(offset <= std::numeric_limits::max() - queue.indexBase, "index address overflow"); const auto address = queue.indexBase + offset; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index 03853d0796..78bc557d74 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -3,6 +3,7 @@ #include "Optimization/RequestMemoryView.hpp" #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" +#include #include #include #include @@ -10,6 +11,7 @@ #include #include #include +#include namespace AgcDriver { namespace { @@ -20,16 +22,71 @@ struct CaptureProfile { std::atomic pages{0}; // The whole CaptureResources call: the plan lookup and the materialization are one call. std::atomic captureNanoseconds{0}; + // Pages read word by word because recorded GPU work writes them, and the word reads of those + // pages that waited (a read taking longer than WordWaitNanoseconds went through a hook sync), + // were read raw on the driver's evidence, or were read both ways (verify) and differed. + std::atomic pagesWordwise{0}; + std::atomic wordWaits{0}; + std::atomic wordWaitNanoseconds{0}; + std::atomic wordsRaw{0}; + std::atomic wordsVerified{0}; + std::atomic wordMismatches{0}; + // Words served from the driver's known values (never profile-gated: the [copy] line reads them). + std::atomic wordsKnown{0}; + std::atomic wordsKnownVerified{0}; + std::atomic wordsKnownMismatches{0}; + // Hook reads not reported to the observer because no GPU wait happened across them. + std::atomic observationsSkipped{0}; + // Sub-phases of a capture: the source lookup (the stage inputs, the recompiler's key over the + // code and the plan, timed by CaptureResources itself: ResourceCapture::sourceNanoseconds), + // whole-page fetches, per-word fetches (their hook waits included) and the GPU waits the flush + // hook made anywhere inside the capture (Recorder::ThreadWaitedMs). The walk itself is what + // remains of the capture time after the lookup, the fetches and the specialization + // (ResourceMaterializer). + std::atomic resolveNanoseconds{0}; + std::atomic pageReadNanoseconds{0}; + std::atomic wordReadNanoseconds{0}; + std::atomic hookWaitNanoseconds{0}; + // The driver's per-shader source handle memo: captures served from it, and resolves it made. + std::atomic handleHits{0}; + std::atomic handleMisses{0}; }; +constexpr std::uint64_t WordWaitNanoseconds = 50000; + +bool WordwisePages() { + static const bool wordwise = std::getenv("APS5_NO_WORDWISE_CAPTURE") == nullptr; + return wordwise; +} + +bool CaptureProfiled() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + +std::uint64_t NanosecondsSince(std::chrono::steady_clock::time_point started) { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); +} + CaptureProfile& CaptureTotals() { static CaptureProfile profile; return profile; } +std::atomic waitedMsProvider{nullptr}; + +double WaitedMs() { + const auto provider = waitedMsProvider.load(std::memory_order_acquire); + return provider != nullptr ? provider() : 0.0; } -ShaderMemory::ShaderMemory(std::span regions) { +} + +void ShaderMemory::SetWaitedMsProvider(WaitedMsProvider provider) { + waitedMsProvider.store(provider, std::memory_order_release); +} + +ShaderMemory::ShaderMemory(std::span regions, PendingWriteQuery pendingWrite, PendingWriteObserver observe, HookWaitCounter hookWaits) : pendingWrite(pendingWrite), observe(observe), hookWaits(hookWaits) { const ShaderRecompiler::RequestMemoryView validated(regions); for (const auto& region : regions) { initial.emplace(region.guestAddress, region.bytes); @@ -41,9 +98,17 @@ ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) { if (found != pages.end()) return found->second; auto& page = pages[base]; ++CaptureTotals().pages; - // A page is either mapped whole or read word by word where the guest mapped less than a page. + // A page is either mapped whole or read word by word where the guest mapped less than a page, + // or where recorded GPU work writes into it (a whole-page read would wait for that work). if (GuestMemory::Accessible(reinterpret_cast(base), PageBytes)) { + if (WordwisePages() && pendingWrite != nullptr && pendingWrite(base, PageBytes, {}) != PendingWrite::None) { + page.wordwise = true; + ++CaptureTotals().pagesWordwise; + return page; + } + const auto started = CaptureProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; GuestMemory::Read(base, std::as_writable_bytes(std::span(page.words)), sizeof(std::uint32_t)); + if (CaptureProfiled()) CaptureTotals().pageReadNanoseconds += NanosecondsSince(started); page.valid.set(); } return page; @@ -72,18 +137,86 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val const auto index = static_cast((address % PageBytes) / sizeof(*value)); if (!page.valid.test(index)) { std::uint32_t word = 0; - GuestMemory::Read(address, std::as_writable_bytes(std::span(&word, 1)), alignof(std::uint32_t)); + auto policy = PendingWrite::Sync; + if (page.wordwise) { + policy = self.pendingWrite(address, sizeof(word), std::as_writable_bytes(std::span(&word, 1))); + if (policy != PendingWrite::None && policy != PendingWrite::Sync && !GuestMemory::Accessible(reinterpret_cast(address), sizeof(word))) policy = PendingWrite::Sync; + } + // A read is evidence only when the hook waited for unfinished GPU work across it. + const auto hookWaits = [&] { return self.hookWaits != nullptr ? self.hookWaits() : 0; }; + const auto report = [&](std::uint64_t waitsBefore, bool unchanged) { + if (self.hookWaits == nullptr || self.hookWaits() != waitsBefore) self.observe(address, unchanged); + else ++CaptureTotals().observationsSkipped; + }; + const auto fetchStarted = CaptureProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + if (policy == PendingWrite::KnownValue || policy == PendingWrite::VerifyKnownValue) { + // The query stored the known dword in `word`. + ++CaptureTotals().wordsKnown; + if (policy == PendingWrite::VerifyKnownValue) { + std::uint32_t waited = 0; + const auto waitsBefore = hookWaits(); + GuestMemory::Read(address, std::as_writable_bytes(std::span(&waited, 1)), alignof(std::uint32_t)); + ++CaptureTotals().wordsKnownVerified; + if (waited != word) ++CaptureTotals().wordsKnownMismatches; + if (self.observe != nullptr) report(waitsBefore, waited == word); + word = waited; + } + } else if (policy == PendingWrite::Raw || policy == PendingWrite::VerifyRaw) { + std::memcpy(&word, reinterpret_cast(address), sizeof(word)); + ++CaptureTotals().wordsRaw; + if (policy == PendingWrite::VerifyRaw) { + std::uint32_t waited = 0; + const auto waitsBefore = hookWaits(); + GuestMemory::Read(address, std::as_writable_bytes(std::span(&waited, 1)), alignof(std::uint32_t)); + ++CaptureTotals().wordsVerified; + if (waited != word) ++CaptureTotals().wordMismatches; + if (self.observe != nullptr) report(waitsBefore, waited == word); + word = waited; + } + } else { + // On a word-wise page the bytes before the hook's wait are kept for the observer. + const bool observed = page.wordwise && policy == PendingWrite::Sync && self.observe != nullptr && GuestMemory::Accessible(reinterpret_cast(address), sizeof(word)); + std::uint32_t before = 0; + if (observed) std::memcpy(&before, reinterpret_cast(address), sizeof(before)); + const auto waitsBefore = observed ? hookWaits() : 0; + const auto started = page.wordwise ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + GuestMemory::Read(address, std::as_writable_bytes(std::span(&word, 1)), alignof(std::uint32_t)); + if (page.wordwise) { + const auto nanoseconds = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); + if (nanoseconds >= WordWaitNanoseconds) { + ++CaptureTotals().wordWaits; + CaptureTotals().wordWaitNanoseconds += nanoseconds; + } + } + if (observed) report(waitsBefore, before == word); + } + if (CaptureProfiled()) CaptureTotals().wordReadNanoseconds += NanosecondsSince(fetchStarted); page.words[index] = word; page.valid.set(index); } page.read.set(index); + page.recent.set(index); *value = page.words[index]; return true; } -std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request) { +ShaderMemory::KnownValueCounts ShaderMemory::KnownValues() { + auto& totals = CaptureTotals(); + return {totals.wordsKnown.load(std::memory_order_relaxed), totals.wordsKnownVerified.load(std::memory_order_relaxed), totals.wordsKnownMismatches.load(std::memory_order_relaxed)}; +} + +void ShaderMemory::CountHandleMemo(bool hit) { + if (!CaptureProfiled()) return; + (hit ? CaptureTotals().handleHits : CaptureTotals().handleMisses).fetch_add(1, std::memory_order_relaxed); +} + +std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle) { // The capture's word and page reads (through `read`) are attributed to it ([hooksync], [guestmem]). const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Capture); + const bool profile = CaptureProfiled(); + auto& totals = CaptureTotals(); + // The hook's GPU waits made anywhere inside the capture. + const auto waitedBefore = profile ? WaitedMs() : 0.0; const auto started = std::chrono::steady_clock::now(); ShaderRecompiler::SrtRuntime runtime; runtime.userData = request.context.userData; @@ -91,15 +224,24 @@ std::shared_ptr ShaderMemory::Capture(c runtime.userContext = this; runtime.readMemory = &read; runtime.readSpecializationMemory = &read; - auto capture = ShaderRecompiler::CaptureResources(request, runtime); - static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - auto& totals = CaptureTotals(); + auto capture = handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); if (profile) { - const auto done = std::chrono::steady_clock::now(); - totals.captureNanoseconds += static_cast(std::chrono::duration_cast(done - started).count()); + totals.captureNanoseconds += NanosecondsSince(started); + totals.resolveNanoseconds += capture->sourceNanoseconds; + totals.hookWaitNanoseconds += static_cast((WaitedMs() - waitedBefore) * 1e6); std::uint64_t initialBytes = 0; for (const auto& [address, bytes] : initial) initialBytes += bytes.size(); - if (++totals.captures % 500 == 0) std::fprintf(stderr, "[capture] %llu captures: %llu word reads, %llu pages fetched, capture (plan lookup + materialize) %.1f s, %llu KiB registered code this capture\n", static_cast(totals.captures.load()), static_cast(totals.reads.load()), static_cast(totals.pages.load()), totals.captureNanoseconds.load() / 1e9, static_cast(initialBytes / 1024)); + if (++totals.captures % 500 == 0) { + const auto captures = static_cast(totals.captures.load()); + const auto captureNs = totals.captureNanoseconds.load(); + const auto resolveNs = totals.resolveNanoseconds.load(); + const auto pageNs = totals.pageReadNanoseconds.load(); + const auto wordNs = totals.wordReadNanoseconds.load(); + const auto specializationNs = ShaderRecompiler::ResourceMaterializer::SpecializationNanoseconds(); + const auto hookNs = totals.hookWaitNanoseconds.load(); + const auto walkNs = captureNs > resolveNs + pageNs + wordNs + specializationNs ? captureNs - resolveNs - pageNs - wordNs - specializationNs : 0; + std::fprintf(stderr, "[capture] %llu captures: %llu word reads, %llu pages fetched (%llu read word by word over pending GPU writes: %llu words waited %.2f s, %llu read raw on evidence, %llu verified with %llu mismatches, %llu served from known values, %llu hook reads not observed: no GPU wait), capture (plan lookup + materialize) %.1f s, %llu KiB registered code this capture; sub-phases (s, us per capture): source resolve %.2f/%.1f, walk %.2f/%.1f, specialization (every materialize) %.2f/%.1f, page reads %.2f/%.1f, word reads %.2f/%.1f, hook GPU waits inside %.2f/%.1f; source handle memo %llu hits, %llu resolves\n", static_cast(totals.captures.load()), static_cast(totals.reads.load()), static_cast(totals.pages.load()), static_cast(totals.pagesWordwise.load()), static_cast(totals.wordWaits.load()), totals.wordWaitNanoseconds.load() / 1e9, static_cast(totals.wordsRaw.load()), static_cast(totals.wordsVerified.load()), static_cast(totals.wordMismatches.load()), static_cast(totals.wordsKnown.load()), static_cast(totals.observationsSkipped.load()), captureNs / 1e9, static_cast(initialBytes / 1024), resolveNs / 1e9, resolveNs / captures / 1e3, walkNs / 1e9, walkNs / captures / 1e3, specializationNs / 1e9, specializationNs / captures / 1e3, pageNs / 1e9, pageNs / captures / 1e3, wordNs / 1e9, wordNs / captures / 1e3, hookNs / 1e9, hookNs / captures / 1e3, static_cast(totals.handleHits.load()), static_cast(totals.handleMisses.load())); + } } return capture; } @@ -128,4 +270,61 @@ std::vector ShaderMemory::Regions() const { return result; } +std::vector ShaderMemory::TakeRecentRegions() { + std::vector result; + for (auto& [base, page] : pages) { + if (page.recent.none()) continue; + for (std::size_t index = 0; index < PageWords;) { + if (!page.recent.test(index)) { + ++index; + continue; + } + const auto first = index; + while (index < PageWords && page.recent.test(index)) ++index; + result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, index - first))}); + } + page.recent.reset(); + } + return result; +} + +DataWordPositionCounts DataWordPositions(std::span> runs, std::span> leaves, std::span otherReads, std::span words, std::span flattenedSrt, std::vector& positions, std::vector& slots) { + DataWordPositionCounts counts; + positions.clear(); + slots.clear(); + std::vector prefix(runs.size() + 1, 0); + for (std::size_t i = 0; i < runs.size(); ++i) prefix[i + 1] = prefix[i] + static_cast((runs[i].second - runs[i].first) / sizeof(std::uint32_t)); + std::vector> found; + for (const auto& [slot, address] : leaves) { + if ((address & 3u) != 0) { + ++counts.unmapped; + continue; + } + if (std::binary_search(otherReads.begin(), otherReads.end(), address)) { + ++counts.aliased; + continue; + } + auto run = std::upper_bound(runs.begin(), runs.end(), address, [](std::uint64_t value, const std::pair& candidate) { return value < candidate.first; }); + if (run == runs.begin() || address + sizeof(std::uint32_t) > (run - 1)->second) { + ++counts.unmapped; + continue; + } + --run; + const auto position = prefix[static_cast(run - runs.begin())] + static_cast((address - run->first) / sizeof(std::uint32_t)); + if (position >= words.size() || slot >= flattenedSrt.size() || words[position] != flattenedSrt[slot]) { + ++counts.mismatched; + continue; + } + found.emplace_back(static_cast(position), slot); + } + std::sort(found.begin(), found.end()); + positions.reserve(found.size()); + slots.reserve(found.size()); + for (const auto& [position, slot] : found) { + positions.push_back(position); + slots.push_back(slot); + } + return counts; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 651bd0bac3..500b035946 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -18,8 +18,10 @@ #ifdef _WIN32 #include #endif +#include #include #include +#include #include #include #include @@ -27,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -49,8 +52,21 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("Vulkan presentation: ") + reason); } +// The ShaderResources content cache dispatches share with recorded draws: Graphics::ResourceCache, +// one process-wide instance (see SharedResourceCache) that the State references so this file keeps +// its Find/Insert/Remove/Clear calls; the device clears it at teardown before its descriptor caches +// go. APS5_NO_RESOURCE_CACHE=1 builds every dispatch's resources as before. +using ResourceCache = Graphics::ResourceCache; + +void WriteFrameBmp(int index, std::uint32_t fullWidth, std::uint32_t fullHeight, std::span full); + +// The [present] line's counters (see the header); mutated under State::presentMutex. +VulkanDevice::PresentStatistics presentCounters{}; + +} + // A compute dispatch's Vulkan objects, shared by every dispatch of the same compiled variant and -// kept by the recorder until the batches using them completed. +// kept by the recorder until the batches using them completed (a Recipe references them weakly). struct ComputePipelineObjects { VkDevice device = VK_NULL_HANDLE; PFN_vkDestroyShaderModule destroyModule = nullptr; @@ -66,14 +82,6 @@ struct ComputePipelineObjects { } }; -// The ShaderResources content cache dispatches share with recorded draws: Graphics::ResourceCache, -// one process-wide instance (see SharedResourceCache) that the State references so this file keeps -// its Find/Insert/Remove/Clear calls; the device clears it at teardown before its descriptor caches -// go. APS5_NO_RESOURCE_CACHE=1 builds every dispatch's resources as before. -using ResourceCache = Graphics::ResourceCache; - -} - struct VulkanDevice::State { void* library = nullptr; PFN_vkGetInstanceProcAddr instanceProc = nullptr; @@ -89,14 +97,12 @@ struct VulkanDevice::State { VkExtent2D extent{}; std::vector images; VkFence acquireFence = VK_NULL_HANDLE; - VkFence renderFence = VK_NULL_HANDLE; struct RetiredSwapchain { VkSwapchainKHR swapchain = VK_NULL_HANDLE; std::vector rendered; }; std::vector rendered; std::vector retiredSwapchains; - VkCommandBuffer clearCommands = VK_NULL_HANDLE; VkPhysicalDeviceMemoryProperties memoryProperties{}; VkBuffer uploadBuffer = VK_NULL_HANDLE; VkDeviceMemory uploadMemory = VK_NULL_HANDLE; @@ -109,6 +115,9 @@ struct VulkanDevice::State { bool tessellationShader = false; bool meshShader = false; bool fragmentShaderBarycentric = false; + // VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in + // graphics stages, and compute workgroups wider than a wave). + bool descriptorIndexing = false; bool depthClipControl = false; bool depthClamp = false; VkDeviceSize hostImportAlignment = 0; @@ -117,8 +126,26 @@ struct VulkanDevice::State { bool textureCompressionBC = false; // VK_KHR_timeline_semaphore enabled: the recorder's unlocked waits are available. bool timelineSemaphores = false; + // Indirect draw features enabled (see Graphics::Context). + bool drawIndirectFirstInstance = false; + bool multiDrawIndirect = false; + bool drawIndirectCount = false; + // The device's resolved entry points (Graphics::DeviceFunctions) and the Graphics::Context + // built once after setup (see graphicsContext); the context's pool reference is dropped before + // the pool at teardown. + Graphics::DeviceFunctions deviceFunctions; + bool functionsReady = false; + Graphics::Context context; + bool contextReady = false; std::unique_ptr detiler; std::unique_ptr recorder; + // Device-local buffers of a repeated 16-byte fill pattern (FillBuffer), most recently used + // last; each is filled once by a doubling chain in device memory. + std::vector, std::shared_ptr>> patternBuffers; + // Compute pipeline objects by variant (and push-constant use), under their own mutex: the + // dispatch's find-or-insert and the verify switch's lookups touch the map, recipes hold weak + // references to its objects. + std::mutex computePipelinesMutex; std::map> computePipelines; std::unique_ptr colorTransfer; std::shared_ptr bufferPool; @@ -138,16 +165,48 @@ struct VulkanDevice::State { // The swapchain image AcquireImage took for the next present(), consumed by that present. bool imageAcquired = false; std::uint32_t acquiredIndex = 0; - // The presentation in flight between present() and QueuePresent(): its swapchain image, and the - // resident image it blits from, alive until the render fence has been waited for. - bool presentPending = false; - std::uint32_t presentIndex = 0; - std::shared_ptr presentKept; - // APS5_DUMP_FRAMES: the presented frame is read back into this buffer by the same submission and - // written as a BMP by the writer thread after the render fence. - std::unique_ptr dumpBuffer; - bool dumpRecorded = false; - int dumpIndex = 0; + // Presentations in flight (see the header): FlipInFlight() + 1 slots used round robin from + // `presentCursor` (the slot there is the oldest). A slot's kept resident image and its dump + // buffer (APS5_DUMP_FRAMES: the frame read back by the blit's submission) live until its fence + // signaled. `presentMutex` orders the presenter's unlocked RetirePresents against a WaitIdle + // from a queue worker; it is never taken before GuestMemory::GpuMutex on a thread. + struct PresentSlot { + VkFence fence = VK_NULL_HANDLE; + VkCommandBuffer commands = VK_NULL_HANDLE; + std::shared_ptr kept; + std::unique_ptr dumpBuffer; + bool dumpRecorded = false; + int dumpIndex = 0; + std::uint32_t imageIndex = 0; + bool inFlight = false; + // The blit reads the single scaler source, staging or upload objects (not a direct blit + // from the resident image): the next use of those waits for this slot. + bool shared = false; + // Recorder::Submissions() when the blit was submitted and the previous slot's: the batches + // in between ran ahead of this blit. + std::uint64_t recorderSerial = 0; + std::uint64_t previousSerial = 0; + std::chrono::steady_clock::time_point submittedAt{}; + // APS5_PROFILE_GPU: the blit's own two timestamps (the [gputime] present-blit class), read + // when the slot retires, and the swapchain bytes it wrote. + VkQueryPool queries = VK_NULL_HANDLE; + std::uint64_t blitBytes = 0; + }; + std::vector presentSlots; + std::size_t presentCursor = 0; + std::uint64_t lastPresentSerial = 0; + // The batch ranges of retired slots whose GPU accounting waits for the batches' completion + // records (SettleRetired); under presentMutex. + struct RetiredRange { + std::uint64_t afterSerial; + std::uint64_t throughSerial; + }; + std::deque retiredRanges; + std::mutex presentMutex; + // The presentation present() submitted and QueuePresent has not handed to the swapchain. + bool queuePending = false; + std::uint32_t queueIndex = 0; + int nextDumpIndex = 0; std::thread dumpWriter; template @@ -161,6 +220,7 @@ struct VulkanDevice::State { template TFunction DeviceFunction(const char* name) const { + ++Graphics::DeviceProcLookups(); auto function = reinterpret_cast(deviceProc(device, name)); if (function == nullptr) { throw std::runtime_error(std::string("Vulkan device function missing: ") + name); @@ -209,6 +269,18 @@ struct VulkanDevice::State { APS5_LOG_OUT("Upload memcpy complete bytes=%zu", pixels.size()); } + bool SharedSlotInFlight() { + std::lock_guard lock(presentMutex); + return std::any_of(presentSlots.begin(), presentSlots.end(), [](const auto& slot) { return slot.inFlight && slot.shared; }); + } + + // Whether a recorded dispatch's or draw's copied write-back (listed until it ran) stores into + // the range on the CPU; under GpuMutex. + bool CopiedWriterOverlaps(std::uint64_t address, std::size_t bytes) const { + const auto overlaps = [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }; + return std::any_of(copiedWriters->begin(), copiedWriters->end(), overlaps) || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), overlaps); + } + void DestroyRetiredSwapchains() { if (retiredSwapchains.empty()) return; const auto destroySemaphore = DeviceFunction("vkDestroySemaphore"); @@ -222,19 +294,135 @@ struct VulkanDevice::State { retiredSwapchains.clear(); } - ~State() { + // A presentation slot whose fence signaled: its frame dump is written, its kept image released + // and its statistics taken (under presentMutex). + void RetireSlot(PresentSlot& slot) { + slot.inFlight = false; + slot.kept.reset(); + if (slot.dumpRecorded) { + slot.dumpRecorded = false; + WriteFrameDump(slot); + } + if (slot.queries != VK_NULL_HANDLE) { + std::uint64_t stamps[2] = {}; + if (DeviceFunction("vkGetQueryPoolResults")(device, slot.queries, 0, 2, sizeof(stamps), stamps, sizeof(std::uint64_t), VK_QUERY_RESULT_64_BIT) == VK_SUCCESS && stamps[1] > stamps[0]) { + Graphics::Recorder::AddGpuTiming(Graphics::Recorder::CommandClass::PresentBlit, static_cast(stamps[1] - stamps[0]) * properties.limits.timestampPeriod, slot.blitBytes); + } + } + if (!recorder) return; + const auto afterFlip = recorder->NewestSubmitted() - slot.recorderSerial; + presentCounters.batchesAfterFlip += afterFlip; + if (auto* frame = PerformanceContext::Current()) frame->NoteRetired(afterFlip, 0, 0); + retiredRanges.push_back({slot.previousSerial, slot.recorderSerial}); + SettleRetired(false); + } + + // The GPU accounting of retired slots ([present] line, frame record): a batch's completion + // record is written when a queue worker finishes it, which can be well after the blit's fence + // signaled (compute queues never reap), so a range read at the retire was mostly unread. Each + // range waits until every serial in it has a record, or until the ring could have dropped its + // oldest one (`force`: at teardown). Under presentMutex. + void SettleRetired(bool force) { + const auto newest = recorder->NewestSubmitted(); + while (!retiredRanges.empty()) { + const auto range = retiredRanges.front(); + std::size_t missing = 0; + const auto batches = recorder->CompletedBatches(range.afterSerial, range.throughSerial, missing); + if (missing != 0 && !force && newest < range.afterSerial + Graphics::Recorder::CompletedRingSize / 2) break; + retiredRanges.pop_front(); + double busyMs = 0, firstNs = 0, lastNs = 0; + bool stamped = false; + std::uint64_t checked = 0, overwritten = 0; + for (const auto& batch : batches) { + if (batch.gpuEndNs > batch.gpuStartNs) { + busyMs += (batch.gpuEndNs - batch.gpuStartNs) / 1e6; + firstNs = stamped ? std::min(firstNs, batch.gpuStartNs) : batch.gpuStartNs; + lastNs = stamped ? std::max(lastNs, batch.gpuEndNs) : batch.gpuEndNs; + stamped = true; + } + if (batch.readGeneration == 0) continue; + for (const auto& [begin, end] : batch.reads) { + ++checked; + GuestMemory::CollectWritesUncached(begin, end - begin); + if (!GuestMemory::UnchangedSince(begin, end - begin, batch.readGeneration)) ++overwritten; + } + } + const double gapMs = stamped ? (lastNs - firstNs) / 1e6 - busyMs : 0; + if (auto* frame = PerformanceContext::Current()) frame->NoteRetired(0, busyMs, gapMs); + presentCounters.gpuBusyMs += busyMs; + presentCounters.gpuGapMs += gapMs; + presentCounters.gpuUnread += missing; + presentCounters.readsChecked += checked; + presentCounters.readsOverwritten += overwritten; + } + } + + void WriteFrameDump(const PresentSlot& slot) { + require(slot.dumpBuffer != nullptr && scaler != nullptr, "frame dump was not recorded"); + // The readback buffer is reused by the slot's next dump, so its bytes are copied out before + // the writer thread (one at a time) downscales and writes them. + std::vector full(slot.dumpBuffer->Bytes().begin(), slot.dumpBuffer->Bytes().end()); if (dumpWriter.joinable()) dumpWriter.join(); + const auto index = slot.dumpIndex; + const auto width = scaler->SourceWidth(); + const auto height = scaler->SourceHeight(); + static const auto start = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, width, height, std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + dumpWriter = std::thread([index, width, height, pixels = std::move(full)] { WriteFrameBmp(index, width, height, pixels); }); + } + + // Blocks on the fence, or polls it every 50 us with APS5_PRESENT_POLL_FENCE=1 (the difference + // in the wait is the presenter's wake-up latency). + void WaitPresentFence(VkFence fence) { + static const bool poll = std::getenv("APS5_PRESENT_POLL_FENCE") != nullptr; + if (!poll) { + check(DeviceFunction("vkWaitForFences")(device, 1, &fence, VK_TRUE, std::numeric_limits::max()), "vkWaitForFences present"); + return; + } + const auto status = DeviceFunction("vkGetFenceStatus"); + for (;;) { + const auto result = status(device, fence); + if (result == VK_SUCCESS) return; + if (result != VK_NOT_READY) check(result, "vkGetFenceStatus present"); + const auto until = std::chrono::steady_clock::now() + std::chrono::microseconds(50); + while (std::chrono::steady_clock::now() < until) std::this_thread::yield(); + } + } + + ~State() { + contextReady = false; if (device != VK_NULL_HANDLE) { const auto idle = reinterpret_cast(deviceProc(device, "vkDeviceWaitIdle"))(device); if (idle != VK_SUCCESS && idle != VK_ERROR_DEVICE_LOST) std::terminate(); - // A resident image kept for an unfinished presentation goes before the caches it came from. - presentKept.reset(); - dumpBuffer.reset(); + // Resident images kept for unfinished presentations go before the caches they came from. + { + std::lock_guard lock(presentMutex); + for (auto& slot : presentSlots) { + if (slot.inFlight) RetireSlot(slot); + } + if (recorder) SettleRetired(true); + } + if (dumpWriter.joinable()) dumpWriter.join(); + for (auto& slot : presentSlots) slot.dumpBuffer.reset(); + // Results still in the unit shadows reach guest memory before the device goes; the + // slabs (kept by the publishing batch) are freed by the recorder's teardown. + if (recorder) { + try { + Graphics::PublishAllShadows(context, Graphics::PublishReason::Teardown); + recorder->Sync(); + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] unit shadow teardown: %s\n", error.what()); + } + } + Graphics::DestroyShadows(device); recorder.reset(); // Cached graphics pipelines (with their framebuffers, modules, render passes and layouts) // belong to this device and must be destroyed while it lives. Graphics::ClearCachedPipelines(device); - computePipelines.clear(); + { + std::lock_guard pipelines(computePipelinesMutex); + computePipelines.clear(); + } // Every ShaderResources (kept by the recorder or the resource cache) is gone now, so the // sets and samplers they borrowed can go. resourceCache.Clear(); @@ -245,10 +433,15 @@ struct VulkanDevice::State { colorTransfer.reset(); scaler.reset(); pipelineCache.reset(); + context.bufferPool.reset(); bufferPool.reset(); const auto destroyFence = reinterpret_cast(deviceProc(device, "vkDestroyFence")); if (acquireFence) destroyFence(device, acquireFence, nullptr); - if (renderFence) destroyFence(device, renderFence, nullptr); + const auto destroyQueryPool = reinterpret_cast(deviceProc(device, "vkDestroyQueryPool")); + for (const auto& slot : presentSlots) { + if (slot.fence) destroyFence(device, slot.fence, nullptr); + if (slot.queries) destroyQueryPool(device, slot.queries, nullptr); + } const auto destroySemaphore = reinterpret_cast(deviceProc(device, "vkDestroySemaphore")); for (auto semaphore : rendered) { if (semaphore) destroySemaphore(device, semaphore, nullptr); @@ -275,7 +468,7 @@ struct VulkanDevice::State { } }; -VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_unique()) { +VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_unique()), serial([] { static std::atomic serials{0}; return serials.fetch_add(1, std::memory_order_relaxed) + 1; }()) { APS5_LOG_OUT("VulkanDevice constructor window=%p", static_cast(window)); #ifdef _WIN32 state->library = SDL_LoadObject("vulkan-1.dll"); @@ -444,6 +637,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u // drops those writes instead of faulting the device. const bool imageRobustness = hasExtension(VK_EXT_IMAGE_ROBUSTNESS_EXTENSION_NAME); if (imageRobustness) deviceExtensions.push_back(VK_EXT_IMAGE_ROBUSTNESS_EXTENSION_NAME); + // Indirect draws with a GPU-side count (DRAW_INDIRECT_MULTI with count_indirect); a device + // without it resolves such draws on the CPU. + state->drawIndirectCount = hasExtension(VK_KHR_DRAW_INDIRECT_COUNT_EXTENSION_NAME); + if (state->drawIndirectCount) deviceExtensions.push_back(VK_KHR_DRAW_INDIRECT_COUNT_EXTENSION_NAME); // Guest memory is host memory: importing it lets address-based shaders use it in place instead of // copying every registered allocation per draw. if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) && std::getenv("APS5_NO_HOST_IMPORT") == nullptr) { @@ -497,6 +694,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u // Guest shaders routinely read past descriptor ranges; robust access turns that into zeros // instead of a GPU fault that loses the device. enabled.robustBufferAccess = available.robustBufferAccess; + // Indirect draw records with a non-zero first instance, and several records per call. + enabled.drawIndirectFirstInstance = available.drawIndirectFirstInstance; + enabled.multiDrawIndirect = available.multiDrawIndirect; + state->drawIndirectFirstInstance = enabled.drawIndirectFirstInstance == VK_TRUE; + state->multiDrawIndirect = enabled.multiDrawIndirect == VK_TRUE; // PA_CL_CLIP_CNTL near/far clip disable maps to depth clamping. enabled.depthClamp = available.depthClamp; state->depthClamp = enabled.depthClamp == VK_TRUE; @@ -508,6 +710,27 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u if (enabled.shaderImageGatherExtended) state->capabilities.push_back(spv::CapabilityImageGatherExtended); if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat); if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat); + // Bindless image tables index an image array with a wave-uniform runtime slot. + enabled.shaderSampledImageArrayDynamicIndexing = available.shaderSampledImageArrayDynamicIndexing; + enabled.shaderStorageImageArrayDynamicIndexing = available.shaderStorageImageArrayDynamicIndexing; + if (enabled.shaderSampledImageArrayDynamicIndexing) state->capabilities.push_back(spv::CapabilitySampledImageArrayDynamicIndexing); + if (enabled.shaderStorageImageArrayDynamicIndexing) state->capabilities.push_back(spv::CapabilityStorageImageArrayDynamicIndexing); + VkPhysicalDeviceDescriptorIndexingFeaturesEXT descriptorIndexingFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT}; + if (hasExtension(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &descriptorIndexingFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + state->descriptorIndexing = descriptorIndexingFeatures.shaderSampledImageArrayNonUniformIndexing == VK_TRUE && descriptorIndexingFeatures.shaderStorageImageArrayNonUniformIndexing == VK_TRUE; + } + descriptorIndexingFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT}; + if (state->descriptorIndexing) { + descriptorIndexingFeatures.shaderSampledImageArrayNonUniformIndexing = VK_TRUE; + descriptorIndexingFeatures.shaderStorageImageArrayNonUniformIndexing = VK_TRUE; + deviceExtensions.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME); + state->capabilities.push_back(spv::CapabilityShaderNonUniform); + state->capabilities.push_back(spv::CapabilitySampledImageArrayNonUniformIndexing); + state->capabilities.push_back(spv::CapabilityStorageImageArrayNonUniformIndexing); + state->spirvExtensions.push_back("SPV_EXT_descriptor_indexing"); + } state->samplerAnisotropy = true; state->textureCompressionBC = true; deviceInfo.pEnabledFeatures = &enabled; @@ -531,6 +754,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u imageRobustnessFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &imageRobustnessFeatures; } + if (state->descriptorIndexing) { + descriptorIndexingFeatures.pNext = byteFeatures.pNext; + byteFeatures.pNext = &descriptorIndexingFeatures; + } // Timeline semaphores let a queue worker wait for recorded batches without holding the GPU mutex // (see Recorder::WaitSerial). The instance is 1.1, so the KHR extension is used even on 1.2+ // devices. Debug aid: APS5_NO_TIMELINE=1 leaves it off (drains wait under the mutex as before). @@ -569,6 +796,16 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->samplerCache = std::make_unique(); state->recorder = std::make_unique(graphicsContext(), state->timelineSemaphores); state->recorder->Activate(); + // The entry points every record site uses, resolved once (APS5_NO_PROC_TABLE=1: per call, as + // before), and the context copy graphicsContext() hands out from here on: built after the + // recorder and the descriptor cache exist, so it carries them. + static const bool procTable = std::getenv("APS5_NO_PROC_TABLE") == nullptr; + if (procTable) { + Graphics::FillDeviceFunctions(graphicsContext(), state->deviceFunctions); + state->functionsReady = true; + } + state->context = buildContext(); + state->contextReady = true; if (window != nullptr) { require(window->getDrawableSize != nullptr, "missing window drawable size query"); std::uint32_t drawableWidth = 0; @@ -617,16 +854,19 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u check(getImages(state->device, state->swapchain, &imageCount, state->images.data()), "vkGetSwapchainImagesKHR"); APS5_LOG_OUT("Swapchain created swapchain=%p extent=%ux%u images=%u", reinterpret_cast(state->swapchain), state->extent.width, state->extent.height, imageCount); VkFenceCreateInfo fence{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; - for (auto* destination : {&state->acquireFence, &state->renderFence}) { - check(state->DeviceFunction("vkCreateFence")(state->device, &fence, nullptr, destination), "vkCreateFence"); - } + const auto createFence = state->DeviceFunction("vkCreateFence"); + check(createFence(state->device, &fence, nullptr, &state->acquireFence), "vkCreateFence"); state->images.resize(imageCount); state->rendered.resize(imageCount, VK_NULL_HANDLE); VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; allocation.commandPool = state->pool; allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; allocation.commandBufferCount = 1; - check(state->DeviceFunction("vkAllocateCommandBuffers")(state->device, &allocation, &state->clearCommands), "vkAllocateCommandBuffers"); + state->presentSlots.resize(FlipInFlight() + 1); + for (auto& slot : state->presentSlots) { + check(createFence(state->device, &fence, nullptr, &slot.fence), "vkCreateFence present"); + check(state->DeviceFunction("vkAllocateCommandBuffers")(state->device, &allocation, &slot.commands), "vkAllocateCommandBuffers"); + } state->scaler = std::make_unique(graphicsContext(), VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM); } } @@ -635,6 +875,7 @@ VulkanDevice::~VulkanDevice() = default; void VulkanDevice::WaitIdle() { APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle begin"); + RetirePresents(0); if (state->recorder) { // Announced with this call's return address: the [recorder] site table then names the // driver site that drains (suspend point, label fallback, fill fallback, device replacement). @@ -691,9 +932,9 @@ void VulkanDevice::ReapRecorded() { if (state->recorder) state->recorder->Reap(); } -std::optional VulkanDevice::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { +std::optional VulkanDevice::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, Graphics::Recorder::LabelRefusal* refusal) const { if (!state->recorder) return std::nullopt; - return state->recorder->PendingLabel(address, bytes, afterStamp, queue); + return state->recorder->PendingLabel(address, bytes, afterStamp, refusal); } bool VulkanDevice::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const { @@ -722,17 +963,22 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::spanDeviceFunction("vkCmdUpdateBuffer")(commands, import->buffer, address - import->base, bytes.size(), bytes.data()); - Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT); - }; - if (recorder.HasCompletions()) { + // and it is visible to the host after. The barriers belong to the recorder's store run: one + // pair for a whole group of labels recorded back to back (Recorder::RecordStore orders the + // stores of a run against each other and against the work before and after, the queued DCC + // key stores included). + const auto recordStore = [&] { recorder.RecordStore(import->buffer, address - import->base, bytes, address); }; + // Only a write-back that can land over these bytes (a listed copied writer of the range, as + // FillBuffer tests) puts the label behind the completions; any other pending completion + // stores elsewhere. Debug aid: APS5_LABEL_GATE_ALL=1 gates on any completion, as before (a + // completion label there registered its own completion, so every later label followed). + static const bool gateAll = std::getenv("APS5_LABEL_GATE_ALL") != nullptr; + if (gateAll ? recorder.HasCompletions() : state->CopiedWriterOverlaps(address, bytes.size())) { // A copied buffer's write-back is still to run: the store must land after it. It is stored // by a completion action of the batch (a CPU memcpy when the batch is reaped) instead of // draining the device here. The same bytes are also recorded on the GPU into the open batch @@ -752,10 +998,8 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span std::chrono::seconds(10)) { lastReport = now; - std::fprintf(stderr, "[fill-sync] fills: %llu synced (%llu with a completion label pending) waited %.0f ms, %llu ordered by barrier, %llu with no pending write\n", static_cast(synced), static_cast(labelSynced), syncedMs, static_cast(ordered), static_cast(fills - synced - ordered)); + std::fprintf(stderr, "[fill-sync] fills: %llu synced (%llu with a completion label pending) waited %.0f ms, %llu ordered by barrier, %llu with no pending write; phases ms: decide %.0f record %.0f notes %.0f; %llu uniform, %llu pattern (%llu doubling steps in place; pattern buffers reused %llu, made %llu)\n", static_cast(synced), static_cast(labelSynced), syncedMs, static_cast(ordered), static_cast(fills - synced - ordered), decideMs, recordMs, notesMs, static_cast(uniformFills), static_cast(patternFills), static_cast(doublingSteps), static_cast(patternBuffersReused), static_cast(patternBuffersMade)); } } - const auto commands = recorder.Commands(); + auto phaseStart = fillStart; + const auto phase = [&](double& total) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + total += std::chrono::duration(now - phaseStart).count(); + phaseStart = now; + }; + phase(decideMs); + using CommandClass = Graphics::Recorder::CommandClass; + // A queued DCC key store over the range must land before the fill. + recorder.FlushKeyStoresOverlapping(address, bytes); + const bool uniform = pattern[0] == pattern[1] && pattern[1] == pattern[2] && pattern[2] == pattern[3]; + const std::array patternWords{pattern[0], pattern[1], pattern[2], pattern[3]}; + std::shared_ptr patternBuffer; + if (!uniform && !chainInPlace) { + auto& buffers = state->patternBuffers; + const auto found = std::find_if(buffers.begin(), buffers.end(), [&](const auto& entry) { return entry.first == patternWords; }); + if (found != buffers.end()) { + patternBuffer = found->second; + std::rotate(found, std::next(found), buffers.end()); + ++patternBuffersReused; + } else { + patternBuffer = std::make_shared(context, PatternBufferBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + auto seed = std::make_shared(context, 16, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(seed->Bytes().data(), pattern.data(), 16); + recorder.Keep(seed); + const auto chain = recorder.Commands(); + const auto copyBuffer = context.Resolved(&Graphics::DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); + const VkBufferCopy first{0, 0, 16}; + copyBuffer(chain, seed->Handle(), patternBuffer->Handle(), 1, &first); + for (std::size_t done = 16; done < PatternBufferBytes; done *= 2) { + Graphics::RecordMemoryBarrier(context, chain, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + const VkBufferCopy copy{0, done, done}; + copyBuffer(chain, patternBuffer->Handle(), patternBuffer->Handle(), 1, ©); + } + Graphics::RecordMemoryBarrier(context, chain, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + if (buffers.size() >= PatternBuffersKept) buffers.erase(buffers.begin()); + buffers.emplace_back(patternWords, patternBuffer); + ++patternBuffersMade; + } + // The buffer outlives its use in this batch even when evicted from the cache meanwhile. + recorder.Keep(patternBuffer); + } + // A queued label store over the range lands before the fill (program order). + recorder.FlushStoresOverlapping(address, bytes); + VkAccessFlags covered = 0; + const auto commands = recorder.Commands(&covered); + const auto timing = recorder.BeginGpuTiming(CommandClass::Fill); + if (Graphics::Recorder::BarrierValidate()) { + const std::pair range{address, address + bytes}; + recorder.NoteAccess(CommandClass::Fill, Graphics::Recorder::Access{{}, std::span(&range, 1), {}, VK_PIPELINE_STAGE_TRANSFER_BIT}); + } // Every earlier recorded read or write of the range precedes the fill (WAR by the execution - // dependency, WAW by the writes made available), whichever stage made it. - Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); - const auto offset = address - import->base; - if (pattern[0] == pattern[1] && pattern[1] == pattern[2] && pattern[2] == pattern[3]) { - state->DeviceFunction("vkCmdFillBuffer")(commands, import->buffer, offset, bytes, pattern[0]); + // dependency, WAW by the writes made available), whichever stage made it; the previous + // command's trailing barrier already did that when it covered transfer writes. + if ((covered & VK_ACCESS_TRANSFER_WRITE_BIT) != 0 && Graphics::Recorder::MergeBarriers()) { + Graphics::Recorder::CountMerged(CommandClass::Fill); } else { + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + Graphics::Recorder::CountBarriers(CommandClass::Fill); + } + const auto offset = address - import->base; + if (uniform) { + ++uniformFills; + context.Resolved(&Graphics::DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer")(commands, import->buffer, offset, bytes, pattern[0]); + } else if (patternBuffer != nullptr) { + ++patternFills; + std::vector copies; + for (std::size_t done = 0; done < bytes; done += PatternBufferBytes) copies.push_back({0, offset + done, std::min(PatternBufferBytes, bytes - done)}); + context.Resolved(&Graphics::DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, patternBuffer->Handle(), import->buffer, static_cast(copies.size()), copies.data()); + } else { + ++patternFills; // A 16-byte pattern is seeded once and doubled in place until the range is covered. auto seed = std::make_shared(context, 16, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); std::memcpy(seed->Bytes().data(), pattern.data(), 16); recorder.Keep(seed); const VkBufferCopy first{0, offset, 16}; - state->DeviceFunction("vkCmdCopyBuffer")(commands, seed->Handle(), import->buffer, 1, &first); + const auto copyBuffer = context.Resolved(&Graphics::DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); + copyBuffer(commands, seed->Handle(), import->buffer, 1, &first); for (std::size_t done = 16; done < bytes;) { const auto chunk = std::min(done, bytes - done); Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + Graphics::Recorder::CountBarriers(CommandClass::Fill); const VkBufferCopy copy{offset, offset + done, chunk}; - state->DeviceFunction("vkCmdCopyBuffer")(commands, import->buffer, import->buffer, 1, ©); + copyBuffer(commands, import->buffer, import->buffer, 1, ©); done += chunk; + ++doublingSteps; } } // The filled bytes are visible to everything recorded after: shaders, transfers, the host, and // an indirect dispatch reading its group counts in place (DispatchIndirect adds its own // ALL_COMMANDS -> DRAW_INDIRECT barrier as well). A later GPU label store over the range is // ordered behind the fill by its ALL_COMMANDS -> TRANSFER barrier (WriteLabelOnGpu). - Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT); + constexpr VkAccessFlags filledAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT; + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, filledAccess); + Graphics::Recorder::CountBarriers(CommandClass::Fill); + recorder.EndGpuTiming(timing, bytes); + recorder.MarkCovered(filledAccess); + phase(recordMs); recorder.NotePendingWrite(address, bytes); GuestMemory::MarkWritten(address, bytes); + phase(notesMs); return true; } +namespace { + +std::atomic copiesVerified{0}, copySourceChanged{0}, copyDestinationChanged{0}, copyReaderMissed{0}; +std::atomic copiesAliased{0}, copyAliasCreated{0}, copyAliasNoSource{0}, copyAliasShape{0}, copyAliasRefused{0}; + +// The copy as a device copy between the surfaces' storage images (StorageTexture::CopyFrom): the +// source range must be exactly a live image's surface, and the destination range either exactly +// a live image of the same shape or none (one is made from the source's descriptor rebased). Under +// GuestMemory::GpuMutex. Debug aid: APS5_NO_COPY_ALIAS=1 keeps every such copy a transfer. +bool AliasCopy(const Graphics::Context& context, std::uint64_t destination, std::uint64_t source, std::size_t bytes) { + static const bool disabled = std::getenv("APS5_NO_COPY_ALIAS") != nullptr; + if (disabled) return false; + auto from = Graphics::StorageTexture::FindLive(source, bytes); + if (from == nullptr) { + copyAliasNoSource.fetch_add(1, std::memory_order_relaxed); + return false; + } + auto to = Graphics::StorageTexture::FindLive(destination, bytes); + if (to != nullptr && !to->SameSurfaceShape(*from)) { + copyAliasShape.fetch_add(1, std::memory_order_relaxed); + return false; + } + if (to == nullptr) { + auto resource = from->Descriptor(); + resource.baseAddress = destination; + resource.dccAddress = 0; + try { + to = Graphics::CachedStorageSurface(context, resource); + } catch (const std::exception& error) { + static std::atomic reported{0}; + if (reported.fetch_add(1) < 8) std::fprintf(stderr, "[copy] no storage image for the copy destination 0x%llx: %s\n", static_cast(destination), error.what()); + copyAliasRefused.fetch_add(1, std::memory_order_relaxed); + return false; + } + copyAliasCreated.fetch_add(1, std::memory_order_relaxed); + } + const char* refusal = nullptr; + if (!to->CopyFrom(*from, refusal)) { + static std::atomic reported{0}; + if (reported.fetch_add(1) < 8) std::fprintf(stderr, "[copy] image copy 0x%llx -> 0x%llx (0x%zx bytes) refused: %s\n", static_cast(source), static_cast(destination), bytes, refusal); + copyAliasRefused.fetch_add(1, std::memory_order_relaxed); + return false; + } + copiesAliased.fetch_add(1, std::memory_order_relaxed); + return true; +} + +} + +VulkanDevice::CopyVerification VulkanDevice::CopyVerifyCounts() { + return {copiesVerified.load(std::memory_order_relaxed), copySourceChanged.load(std::memory_order_relaxed), copyDestinationChanged.load(std::memory_order_relaxed), copyReaderMissed.load(std::memory_order_relaxed)}; +} + +VulkanDevice::CopyAliasing VulkanDevice::CopyAliasCounts() { + return {copiesAliased.load(std::memory_order_relaxed), copyAliasCreated.load(std::memory_order_relaxed), copyAliasNoSource.load(std::memory_order_relaxed), copyAliasShape.load(std::memory_order_relaxed), copyAliasRefused.load(std::memory_order_relaxed)}; +} + +VulkanDevice::CopyOutcome VulkanDevice::CopyBuffer(std::uint64_t destination, std::uint64_t source, std::size_t bytes, std::size_t cpuMax, std::size_t gpuMax, std::size_t knownMax, std::uint64_t programAddress, std::uint32_t queue, const CopyWriterNote& noteWriter) { + CopyOutcome outcome{2, false, 0, CopyOutcome::None, false, false, 0, 0, 0, 0, false}; + if (!state->recorder || bytes == 0) return outcome; + auto& recorder = *state->recorder; + // Finished batches were retired by the caller (Driver.cpp copyBuffer, on the graphics worker + // only, as WriteLabelOnGpu reaps), so their writes neither refuse the CPU copy nor force a wait. + static const bool alwaysSync = std::getenv("APS5_COPY_SYNC") != nullptr; + static const bool waitForSource = std::getenv("APS5_COPY_WAIT_SOURCE") != nullptr; + static const bool verify = std::getenv("APS5_COPY_VERIFY") != nullptr; + // A store a batch's completion makes on the CPU over the range (a copied buffer's write-back, + // a completion-deferred label): it lands only when that batch is reaped, whatever its fence. + const auto completionStoreOverlaps = [&](std::uint64_t address) { + if (alwaysSync || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }) + || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(address, bytes); })) return true; + return Graphics::Recorder::PendingCompletionLabels() != 0 && recorder.PendingLabelIn(address, bytes); + }; + // The CPU decision (see the header): pure queries, in the order that makes the first failing + // test the reason; a signaled writer of the source is fine (its in-place stores are final in + // host memory) as long as no completion store or label of it is still to land. + bool settledBySignal = false; + const auto refusal = [&]() -> int { + settledBySignal = false; + if (bytes > cpuMax) return CopyOutcome::Size; + // Both lists: an image whose write-back is in progress on another thread (moved to flushing, + // waiting for the GPU mutex) still stores over the range once this hold ends. + const std::array, 2> ranges{{{source, source + bytes}, {destination, destination + bytes}}}; + if (Graphics::StorageTexture::AnyPendingOverlaps(ranges)) return CopyOutcome::Image; + // Results retiled into a unit shadow are not in the import's bytes: the GPU path publishes them. + if (Graphics::AnyShadowedOverlaps(ranges)) return CopyOutcome::Shadow; + if (recorder.PendingWriteOverlaps(source, bytes)) { + if (!recorder.PendingWriteSettled(source, bytes)) return CopyOutcome::SourcePending; + if (completionStoreOverlaps(source) || recorder.PendingLabelIn(source, bytes)) return CopyOutcome::SourceUnsettled; + settledBySignal = true; + } + if (recorder.PendingWriteOverlaps(destination, bytes)) { + // As for the source: a signaled writer's in-place stores are final and the memcpy lands + // after them; only a completion store of it still to run refuses (compute queues never + // reap, so a finished GPU copy into the same record stays in flight for a while). + // Debug aid: APS5_COPY_DESTINATION_STRICT=1 refuses any pending writer, as before. + static const bool strict = std::getenv("APS5_COPY_DESTINATION_STRICT") != nullptr; + if (strict || !recorder.PendingWriteSettled(destination, bytes)) return CopyOutcome::DestinationPending; + if (completionStoreOverlaps(destination)) return CopyOutcome::DestinationUnsettled; + } + if (recorder.PendingLabelIn(destination, bytes)) return CopyOutcome::Label; + if (Graphics::Recorder::ReadTracking() ? recorder.PendingReadOverlaps(destination, bytes) : !recorder.Idle()) return CopyOutcome::Reader; + return CopyOutcome::None; + }; + outcome.reason = refusal(); + if (outcome.reason == CopyOutcome::SourcePending && waitForSource && queue != 0) { + // The experiment: only when the producer is the oldest batch in flight (the wait finishes + // nothing else), never on the frame-critical queue 0, unlocked (this hold is the outermost). + if (const auto info = recorder.DescribePendingWrite(source, bytes); info.has_value() && !info->open && info->batchesToFinish == 1) { + const auto waitStart = std::chrono::steady_clock::now(); + Graphics::Recorder::CountSync(4); + recorder.SyncThrough(source, bytes, true); + outcome.waitedForSource = true; + outcome.waitMs = std::chrono::duration(std::chrono::steady_clock::now() - waitStart).count(); + outcome.reason = refusal(); + } + } + if (outcome.reason == CopyOutcome::None) { + // A plain CPU store, stamped for the write watch, with no pending write to note (nothing is + // outstanding on the range) and nothing recorded. + std::vector expected; + if (verify) expected.assign(reinterpret_cast(source), reinterpret_cast(source) + bytes); + std::memcpy(reinterpret_cast(destination), reinterpret_cast(source), bytes); + GuestMemory::MarkWritten(destination, bytes); + outcome.path = 0; + outcome.sourceSettledBySignal = settledBySignal; + if (verify) { + // The rule's reader answer without the fence shortcut: a reader still unsignaled now + // would have been unsignaled at the decision a moment ago. + if (const auto reader = recorder.DescribePendingRead(destination, bytes); reader.has_value() && !reader->signaled) copyReaderMissed.fetch_add(1, std::memory_order_relaxed); + Graphics::Recorder::CountSync(4); + recorder.Sync(); + copiesVerified.fetch_add(1, std::memory_order_relaxed); + if (GuestMemory::Accessible(reinterpret_cast(source), bytes) && std::memcmp(expected.data(), reinterpret_cast(source), bytes) != 0) copySourceChanged.fetch_add(1, std::memory_order_relaxed); + if (GuestMemory::Accessible(reinterpret_cast(destination), bytes) && std::memcmp(expected.data(), reinterpret_cast(destination), bytes) != 0) copyDestinationChanged.fetch_add(1, std::memory_order_relaxed); + } + return outcome; + } + if (outcome.reason == CopyOutcome::Reader) { + if (const auto reader = recorder.DescribePendingRead(destination, bytes)) { + outcome.readerSerial = reader->serial; + outcome.readerQueue = reader->queue; + outcome.readerKind = static_cast(reader->kind); + outcome.readerOpen = reader->open; + } + } + const auto context = graphicsContext(); + // The ordering decision of FillBuffer, for both ranges: GPU stores recorded earlier are ordered + // before the transfer (or the device copy below) by its barrier; a store a batch's completion + // makes on the CPU must land before the copy reads the source or writes the destination, so + // the copy waits for that batch (SyncThrough) first. + const bool waitSource = recorder.PendingWriteOverlaps(source, bytes) && completionStoreOverlaps(source); + const bool waitDestination = recorder.PendingWriteOverlaps(destination, bytes) && completionStoreOverlaps(destination); + if (waitSource || waitDestination) { + outcome.synced = true; + const auto syncStart = std::chrono::steady_clock::now(); + if (waitSource) { + Graphics::Recorder::CountSync(4); + recorder.SyncThrough(source, bytes); + } + if (waitDestination) { + Graphics::Recorder::CountSync(4); + recorder.SyncThrough(destination, bytes); + } + outcome.syncMs = std::chrono::duration(std::chrono::steady_clock::now() - syncStart).count(); + } + // A whole surface copied into another: a device copy between their images, whose results reach + // the destination's memory through the image's deferred write-back (nothing crosses to host + // memory for the copy itself). + if (AliasCopy(context, destination, source, bytes)) { + noteWriter({}, 0); + recorder.NotePendingWrite(destination, bytes); + outcome.path = 3; + return outcome; + } + if (bytes > gpuMax) { + outcome.path = 8; + return outcome; + } + // The GPU path. Results still on the GPU: the source's must be stored before the transfer reads + // it, the destination's would be stored over the copy later (a flush into an import records + // the retile and notes it as a pending write, which is why it comes after the CPU decision). + Graphics::StorageTexture::FlushPending(source, bytes, nullptr, "buffer copy source", Graphics::PublishScope::Whole); + Graphics::StorageTexture::FlushPending(destination, bytes, nullptr, "buffer copy destination", Graphics::PublishScope::PartialUnits); + // Looked up after the sync (its completions' write-backs can refresh the import table and + // retire an import) and the flushes (which may import memory themselves). + const Graphics::HostImport* destinationImport = Graphics::HostImportFor(context, destination, bytes); + const Graphics::HostImport* sourceImport = destinationImport != nullptr ? Graphics::HostImportFor(context, source, bytes) : nullptr; + if (destinationImport == nullptr || sourceImport == nullptr) return outcome; + using CommandClass = Graphics::Recorder::CommandClass; + // A queued DCC key store over either range must land before the transfer reads or writes it. + recorder.FlushKeyStoresOverlapping(source, bytes); + recorder.FlushKeyStoresOverlapping(destination, bytes); + // A queued label store in either range lands before the transfer reads or writes it. + recorder.FlushStoresOverlapping(source, bytes); + recorder.FlushStoresOverlapping(destination, bytes); + VkAccessFlags covered = 0; + const auto commands = recorder.Commands(&covered); + // The class range covers the barriers too; the program-keyed range inside it is the transfer. + const auto classTiming = recorder.BeginGpuTiming(CommandClass::Copy); + if (Graphics::Recorder::BarrierValidate()) { + const std::pair read{source, source + bytes}, written{destination, destination + bytes}; + recorder.NoteAccess(CommandClass::Copy, Graphics::Recorder::Access{std::span(&read, 1), std::span(&written, 1), {}, VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + // Every earlier recorded read or write of either range precedes the transfer, whichever stage + // made it, and host stores (the CPU copies above, the game's) are visible to it; the previous + // command's trailing barrier already did that when it covered transfer reads and writes. + constexpr VkAccessFlags transferAccess = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + if ((covered & transferAccess) == transferAccess && Graphics::Recorder::MergeBarriers()) { + Graphics::Recorder::CountMerged(CommandClass::Copy); + } else { + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, transferAccess); + Graphics::Recorder::CountBarriers(CommandClass::Copy); + } + const auto gpuTiming = recorder.BeginGpuTiming(programAddress); + const VkBufferCopy region{source - sourceImport->base, destination - destinationImport->base, bytes}; + context.Resolved(&Graphics::DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, sourceImport->buffer, destinationImport->buffer, 1, ®ion); + recorder.EndGpuTiming(gpuTiming, bytes); + // The copied bytes are visible to everything recorded after and to the host, as after a fill. + constexpr VkAccessFlags copiedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT; + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, copiedAccess); + Graphics::Recorder::CountBarriers(CommandClass::Copy); + recorder.EndGpuTiming(classTiming, bytes); + recorder.MarkCovered(copiedAccess); + recorder.NotePendingRead(source, bytes, Graphics::Recorder::ReadKind::CopySource); + // The destination's bytes after the transfer are the source's now when nothing recorded, + // pending or still to land can change the source before the transfer reads it (a CPU store + // to it in between is a hardware race too): the caller's ring entry carries them, stamped + // with the generation this store gets, and goes in ahead of the note. + const auto generation = GuestMemory::MarkWritten(destination, bytes); + const std::array, 1> sourceRange{{{source, source + bytes}}}; + const bool sourceCurrent = bytes <= knownMax && generation != 0 && !recorder.PendingWriteOverlaps(source, bytes) && !recorder.PendingLabelIn(source, bytes) && !completionStoreOverlaps(source) && !Graphics::StorageTexture::AnyPendingOverlaps(sourceRange) && !Graphics::AnyShadowedOverlaps(sourceRange); + noteWriter(sourceCurrent ? std::span(reinterpret_cast(source), bytes) : std::span{}, sourceCurrent ? generation : 0); + recorder.NotePendingWrite(destination, bytes); + outcome.path = 1; + return outcome; +} + void* VulkanDevice::Window() const { return state->window; } @@ -1017,6 +1585,38 @@ bool ResidentPresentable(const Graphics::Context& context, const Graphics::Stora return true; } +// The resident image a display buffer is blitted from, or null; `pending` tells a buffer without +// a pending image from one whose image is unsuitable. +std::shared_ptr PresentableResident(const Graphics::Context& context, const DisplayBuffer& buffer, VkFilter& filter, bool& pending) { + auto resident = Graphics::StorageTexture::FindPending(buffer.address, DisplayBufferSize(buffer)); + pending = resident != nullptr; + if (resident != nullptr && !ResidentPresentable(context, *resident, buffer, filter)) resident.reset(); + return resident; +} + +// Debug aid: APS5_NO_RESIDENT_PRESENT=1 always presents through guest memory. +bool NoResidentPresent() { + static const bool no = std::getenv("APS5_NO_RESIDENT_PRESENT") != nullptr; + return no; +} + +// Debug aid: APS5_DUMP_FRAMES= saves the first n presented display buffers as frame_.bmp, +// read back by the blit's own submission and written after its fence (RetireSlot), or with +// APS5_NO_GPU_DUMP=1 decoded from the tiled guest buffer on the CPU, as before. +struct FrameDumps { + int limit; + bool cpu; + int dumped = 0; +}; + +FrameDumps& Dumps() { + static FrameDumps dumps{[] { + const char* value = std::getenv("APS5_DUMP_FRAMES"); + return value ? std::atoi(value) : 0; + }(), std::getenv("APS5_NO_GPU_DUMP") != nullptr}; + return dumps; +} + } bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { @@ -1024,19 +1624,18 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { // The display buffer is usually a resident render target whose results are still on the GPU: it // is blitted from that image, which skips the write-back retile, the 33 MB read of guest memory, // the upload and the detile dispatch. The deferred write-back stays pending as for any storage - // image. Debug aid: APS5_NO_RESIDENT_PRESENT=1 always goes through guest memory. - static const bool noResidentPresent = std::getenv("APS5_NO_RESIDENT_PRESENT") != nullptr; + // image. static std::uint64_t residentPresents = 0, refreshedPresents = 0, notPending = 0, unsuitable = 0, gpuDumps = 0; static auto lastReport = std::chrono::steady_clock::now(); const auto bytes = DisplayBufferSize(buffer); std::shared_ptr resident; VkFilter filter = VK_FILTER_LINEAR; - if (!noResidentPresent) { - resident = Graphics::StorageTexture::FindPending(buffer.address, bytes); - if (resident == nullptr) { + if (!NoResidentPresent()) { + bool pending = false; + resident = PresentableResident(graphicsContext(), buffer, filter, pending); + if (!pending) { ++notPending; - } else if (!ResidentPresentable(graphicsContext(), *resident, buffer, filter)) { - resident.reset(); + } else if (resident == nullptr) { ++unsuitable; } else { // 64 KiB blocks the CPU wrote since the image last matched guest memory would be shown @@ -1052,22 +1651,14 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { ++residentPresents; } } - // Debug aid: APS5_DUMP_FRAMES= also covers the GPU display path: the presented frame is read - // back by the same submission and written after its fence (see writeFrameDump). - // APS5_NO_GPU_DUMP=1 decodes the tiled guest buffer on the CPU instead, as before. - static const int dumpLimit = [] { - const char* value = std::getenv("APS5_DUMP_FRAMES"); - return value ? std::atoi(value) : 0; - }(); - static const bool cpuDump = std::getenv("APS5_NO_GPU_DUMP") != nullptr; - static int dumped = 0; + auto& dumps = Dumps(); bool dumpFrame = false; - if (dumped < dumpLimit) { - if (cpuDump) { + if (dumps.dumped < dumps.limit) { + if (dumps.cpu) { const auto full = ReadDisplayBuffer(buffer); - WriteFrameBmp(dumped++, buffer.width, buffer.height, full); + WriteFrameBmp(dumps.dumped++, buffer.width, buffer.height, full); } else { - state->dumpIndex = dumped++; + state->nextDumpIndex = dumps.dumped++; dumpFrame = true; ++gpuDumps; } @@ -1078,7 +1669,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { } if (!present(buffer.width, buffer.height, true, {}, &buffer, resident, filter, dumpFrame)) { // A dropped frame (swapchain out of date) keeps the dump numbering contiguous. - if (dumpFrame) --dumped; + if (dumpFrame) --dumps.dumped; return false; } return true; @@ -1088,13 +1679,11 @@ bool VulkanDevice::AcquireImage() { PerformanceTimer timing("Vulkan.Acquire"); require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); - require(!state->presentPending, "the previous presentation was not finished"); require(!state->imageAcquired, "the previously acquired image was not presented"); auto wait = state->DeviceFunction("vkWaitForFences"); auto reset = state->DeviceFunction("vkResetFences"); - const std::array fences{state->acquireFence, state->renderFence}; - check(reset(state->device, static_cast(fences.size()), fences.data()), "vkResetFences"); - APS5_LOG_CHARS_OUT_DEBUG("present fences reset"); + check(reset(state->device, 1, &state->acquireFence), "vkResetFences"); + APS5_LOG_CHARS_OUT_DEBUG("acquire fence reset"); std::uint32_t index = 0; timing.Mark("fence_reset"); // An out-of-date swapchain (the window changed) drops this frame; the next Resize recreates it. @@ -1128,7 +1717,8 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu APS5_LOG_OUT_DEBUG("present begin width=%u height=%u opaque=%u pixels=%zu", width, height, static_cast(opaque), pixels.size()); require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); - require(!state->presentPending, "the previous presentation was not finished"); + require(!state->queuePending, "the previous presentation was not queued"); + require(!state->presentSlots.empty(), "device has no presentation slots"); // The game path acquired the image before taking GpuMutex (Driver::Present); tests, tools and // APS5_SYNC_FLIP=1 acquire here. if (!state->imageAcquired && !AcquireImage()) return false; @@ -1136,22 +1726,54 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu const auto index = state->acquiredIndex; require(index < state->images.size() && index < state->rendered.size(), "acquired image index is out of range"); auto& rendered = state->rendered[index]; + const bool clearOnly = pixels.empty() && display == nullptr; + // A resident image is blitted straight to the swapchain; a frame dump goes through the + // scaler's BGRA8 image so the readback has one format. + const bool direct = resident != nullptr && !dumpFrame; + // The scaler's source image, the color transfer's staging and the upload buffer are single + // objects an in-flight blit through them may still read: the paths using or re-creating them + // wait for every slot first. The game path did so before taking the mutex + // (PresentWaitsForSlots); this is the fallback. + const bool shared = !clearOnly && !direct; + if ((shared || (!clearOnly && (state->scaler == nullptr || state->scaler->SourceWidth() != width || state->scaler->SourceHeight() != height))) && state->SharedSlotInFlight()) RetirePresents(0); + // The next slot is the oldest; a caller that did not retire it first (PresentPixels) waits here. + if (state->presentSlots[state->presentCursor].inFlight) RetirePresents(state->presentSlots.size() - 1); + auto& slot = state->presentSlots[state->presentCursor]; + require(!slot.inFlight, "presentation slot is still in flight"); if (display != nullptr && resident == nullptr) { static_cast(DisplayBufferSize(*display)); state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget); } if (!pixels.empty()) state->Upload(pixels); // The frame's recorded work (and a refresh of the resident image) must reach the queue before the - // presentation's own submission, which reads the image in queue order. - SubmitRecorded(); + // presentation's own submission, which reads the image in queue order. No reap first: queue 0 + // reaps at its next packet, so the presenter's hold carries no completions. + SubmitRecorded(false); + std::chrono::steady_clock::time_point lastSubmittedAt{}; + const std::uint64_t batchesAtBlit = state->recorder ? state->recorder->NewestSubmitted(&lastSubmittedAt) : 0; timing.Mark("pixel_upload"); APS5_LOG_OUT_DEBUG("present source=%s bytes=%zu", pixels.empty() ? "clear" : "pixels", pixels.size()); - auto commands = state->clearCommands; + auto commands = slot.commands; check(state->DeviceFunction("vkResetCommandBuffer")(commands, 0), "vkResetCommandBuffer"); VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; check(state->DeviceFunction("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); APS5_LOG_OUT_DEBUG("Presentation command buffer begin commands=%p image=%p", reinterpret_cast(commands), reinterpret_cast(state->images[index])); + using CommandClass = Graphics::Recorder::CommandClass; + Graphics::Recorder::CountPresent(); + if (Graphics::Recorder::GpuTimingEnabled()) { + if (slot.queries == VK_NULL_HANDLE) { + VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; + info.queryType = VK_QUERY_TYPE_TIMESTAMP; + info.queryCount = 2; + if (state->DeviceFunction("vkCreateQueryPool")(state->device, &info, nullptr, &slot.queries) != VK_SUCCESS) slot.queries = VK_NULL_HANDLE; + } + if (slot.queries != VK_NULL_HANDLE) { + state->DeviceFunction("vkCmdResetQueryPool")(commands, slot.queries, 0, 2); + state->DeviceFunction("vkCmdWriteTimestamp")(commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, slot.queries, 0); + } + slot.blitBytes = static_cast(state->extent.width) * state->extent.height * 4; + } VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; @@ -1162,6 +1784,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; auto pipelineBarrier = state->DeviceFunction("vkCmdPipelineBarrier"); pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + Graphics::Recorder::CountBarriers(CommandClass::PresentBlit); if (pixels.empty() && display == nullptr) { APS5_LOG_OUT_DEBUG("Recording swapchain clear opaque=%u image=%p", static_cast(opaque), reinterpret_cast(barrier.image)); VkClearColorValue clear{}; @@ -1171,9 +1794,6 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu APS5_LOG_OUT_DEBUG("Recording swapchain scaled blit width=%u height=%u bytes=%zu buffer=%p image=%p", width, height, pixels.size(), reinterpret_cast(state->uploadBuffer), reinterpret_cast(barrier.image)); require(state->scaler != nullptr, "presentation scaler is unavailable"); state->scaler->EnsureSourceImage(width, height); - // A resident image is blitted straight to the swapchain; a frame dump goes through the - // scaler's BGRA8 image so the readback has one format. - const bool direct = resident != nullptr && !dumpFrame; VkImageMemoryBarrier residentBarrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; if (resident != nullptr) { // The draws and dispatches that produced the image were submitted before this command @@ -1188,6 +1808,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu residentBarrier.image = resident->Image(); residentBarrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; pipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &residentBarrier); + Graphics::Recorder::CountBarriers(CommandClass::PresentBlit); if (!direct) state->scaler->RecordBlitInto(commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, residentFilter); } else { if (display != nullptr) { @@ -1212,6 +1833,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu letterboxBarrier.image = barrier.image; letterboxBarrier.subresourceRange = barrier.subresourceRange; pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &letterboxBarrier); + Graphics::Recorder::CountBarriers(CommandClass::PresentBlit); if (direct) PresentationScaler::RecordBlitFrom(graphicsContext(), commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, width, height, residentFilter, barrier.image, state->extent.width, state->extent.height); else state->scaler->RecordBlit(commands, barrier.image, state->extent.width, state->extent.height); if (resident != nullptr) { @@ -1220,11 +1842,12 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu residentBarrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; residentBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &residentBarrier); + Graphics::Recorder::CountBarriers(CommandClass::PresentBlit); } if (dumpFrame) { const auto dumpBytes = static_cast(state->scaler->SourceWidth()) * state->scaler->SourceHeight() * 4; - if (!state->dumpBuffer || state->dumpBuffer->Bytes().size() != dumpBytes) state->dumpBuffer = std::make_unique(graphicsContext(), dumpBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); - state->scaler->RecordReadback(commands, state->dumpBuffer->Handle()); + if (!slot.dumpBuffer || slot.dumpBuffer->Bytes().size() != dumpBytes) slot.dumpBuffer = std::make_unique(graphicsContext(), dumpBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + state->scaler->RecordReadback(commands, slot.dumpBuffer->Handle()); } } barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; @@ -1232,6 +1855,8 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; barrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + Graphics::Recorder::CountBarriers(CommandClass::PresentBlit); + if (slot.queries != VK_NULL_HANDLE) state->DeviceFunction("vkCmdWriteTimestamp")(commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, slot.queries, 1); check(state->DeviceFunction("vkEndCommandBuffer")(commands), "vkEndCommandBuffer"); APS5_LOG_CHARS_OUT_DEBUG("Presentation command buffer recorded"); VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; @@ -1240,54 +1865,121 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu submit.signalSemaphoreCount = 1; submit.pSignalSemaphores = &rendered; timing.Mark("command_record_scale"); - check(state->DeviceFunction("vkQueueSubmit")(state->queue, 1, &submit, state->renderFence), "vkQueueSubmit clear"); + // The slot's fence signaled (retired above) or was never used: the reset cannot block. + check(state->DeviceFunction("vkResetFences")(state->device, 1, &slot.fence), "vkResetFences present"); + check(state->DeviceFunction("vkQueueSubmit")(state->queue, 1, &submit, slot.fence), "vkQueueSubmit clear"); + const auto submittedAt = std::chrono::steady_clock::now(); timing.Mark("queue_submit"); APS5_LOG_CHARS_OUT_DEBUG("Presentation vkQueueSubmit OK"); - // The image stays alive until FinishPresent has waited for the blit (storage-cache eviction only - // drops the cache's reference). - state->presentKept = resident; - state->presentIndex = index; - state->presentPending = true; - // Only a submitted readback is written after the fence (a failed submit tears the device down). - state->dumpRecorded = dumpFrame; + { + std::lock_guard lock(state->presentMutex); + // The image stays alive until the slot is retired (storage-cache eviction only drops the + // cache's reference). + slot.kept = resident; + slot.imageIndex = index; + slot.inFlight = true; + slot.shared = shared; + // Only a submitted readback is written after the fence (a failed submit tears the device down). + slot.dumpRecorded = dumpFrame; + slot.dumpIndex = state->nextDumpIndex; + slot.recorderSerial = batchesAtBlit; + slot.previousSerial = state->lastPresentSerial; + slot.submittedAt = submittedAt; + state->lastPresentSerial = batchesAtBlit; + state->presentCursor = (state->presentCursor + 1) % state->presentSlots.size(); + state->queuePending = true; + state->queueIndex = index; + ++presentCounters.presents; + if (auto* frame = PerformanceContext::Current()) { + frame->NoteBlit(batchesAtBlit, lastSubmittedAt, submittedAt); + // The serials are cumulative: only their difference (the batches other queues put + // ahead of the blit since the flip packet) is summed; a record without a flip sample + // (the drain paths) contributes nothing. + const auto atFlip = frame->BatchesAtFlip(); + if (atFlip != 0 && batchesAtBlit >= atFlip) presentCounters.batchesAheadOfBlit += batchesAtBlit - atFlip; + if (frame->FlipReached() != std::chrono::steady_clock::time_point{}) { + presentCounters.blitSubmitAfterFlipMs += std::chrono::duration(submittedAt - frame->FlipReached()).count(); + presentCounters.lastSubmitAfterFlipMs += std::chrono::duration(std::max(lastSubmittedAt, submittedAt) - frame->FlipReached()).count(); + } + } + } return true; } -void VulkanDevice::FinishPresent() { - if (!state->presentPending) return; - PerformanceTimer timing("Vulkan.PresentWait"); - const auto wait = state->DeviceFunction("vkWaitForFences"); - check(wait(state->device, 1, &state->renderFence, VK_TRUE, std::numeric_limits::max()), "vkWaitForFences clear"); - timing.Mark("render_fence_wait"); - APS5_LOG_CHARS_OUT_DEBUG("Presentation render fence complete"); - if (state->dumpRecorded) { - state->dumpRecorded = false; - writeFrameDump(); - timing.Mark("frame_dump"); - } +std::size_t VulkanDevice::FlipInFlight() { + static const std::size_t count = [] { + if (std::getenv("APS5_SYNC_FLIP") != nullptr) return std::size_t{0}; + const char* value = std::getenv("APS5_FLIP_INFLIGHT"); + if (value == nullptr) return std::size_t{1}; + const long parsed = std::strtol(value, nullptr, 10); + if (parsed >= 0 && parsed <= 2) return static_cast(parsed); + std::fprintf(stderr, "[present] APS5_FLIP_INFLIGHT=%s refused (0, 1 or 2 presentations may trail on the GPU); using 1\n", value); + return std::size_t{1}; + }(); + return count; } -void VulkanDevice::writeFrameDump() { - require(state->dumpBuffer != nullptr && state->scaler != nullptr, "frame dump was not recorded"); - // The readback buffer is reused by the next dump, so its bytes are copied out before the writer - // thread (one at a time) downscales and writes them. - std::vector full(state->dumpBuffer->Bytes().begin(), state->dumpBuffer->Bytes().end()); - if (state->dumpWriter.joinable()) state->dumpWriter.join(); - const auto index = state->dumpIndex; - const auto width = state->scaler->SourceWidth(); - const auto height = state->scaler->SourceHeight(); - static const auto start = std::chrono::steady_clock::now(); - std::fprintf(stderr, "[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, width, height, std::chrono::duration(std::chrono::steady_clock::now() - start).count()); - state->dumpWriter = std::thread([index, width, height, pixels = std::move(full)] { WriteFrameBmp(index, width, height, pixels); }); +VulkanDevice::PresentStatistics VulkanDevice::PresentCounts() { + return presentCounters; +} + +bool VulkanDevice::PresentWaitsForSlots(const DisplayBuffer* buffer) const { + if (buffer == nullptr || !state->SharedSlotInFlight()) return false; + const auto& dumps = Dumps(); + if ((dumps.dumped < dumps.limit && !dumps.cpu) || NoResidentPresent()) return true; + VkFilter filter = VK_FILTER_LINEAR; + bool pending = false; + if (PresentableResident(graphicsContext(), *buffer, filter, pending) == nullptr) return true; + return state->scaler == nullptr || state->scaler->SourceWidth() != buffer->width || state->scaler->SourceHeight() != buffer->height; +} + +void VulkanDevice::FlipBatches(std::uint64_t& submissions, std::uint64_t& unsignaled) const { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + submissions = state->recorder ? state->recorder->Submissions() : 0; + unsignaled = profile && state->recorder ? state->recorder->UnsignaledBatches() : 0; +} + +double VulkanDevice::RetirePresents(std::size_t keepInFlight) { + auto& slots = state->presentSlots; + if (slots.empty()) return 0; + PerformanceTimer timing("Vulkan.Retire"); + std::lock_guard lock(state->presentMutex); + const auto status = state->DeviceFunction("vkGetFenceStatus"); + auto inFlight = static_cast(std::count_if(slots.begin(), slots.end(), [](const auto& slot) { return slot.inFlight; })); + double waitedMs = 0; + // Oldest first from the cursor; the fences of one queue signal in submission order, so the + // first unsignaled one ends the scan, and the ones beyond the bound are the oldest. + for (std::size_t i = 0; i < slots.size() && inFlight != 0; ++i) { + auto& slot = slots[(state->presentCursor + i) % slots.size()]; + if (!slot.inFlight) continue; + if (inFlight <= keepInFlight) { + const auto result = status(state->device, slot.fence); + if (result == VK_NOT_READY) break; + check(result, "vkGetFenceStatus present"); + timing.Mark("poll"); + } else { + const auto start = std::chrono::steady_clock::now(); + state->WaitPresentFence(slot.fence); + waitedMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + timing.Mark("wait"); + } + const bool dump = slot.dumpRecorded; + state->RetireSlot(slot); + if (dump) timing.Mark("dump"); + --inFlight; + } + return waitedMs; +} + +double VulkanDevice::FinishPresent() { + return RetirePresents(0); } void VulkanDevice::QueuePresent() { - if (!state->presentPending) return; + if (!state->queuePending) return; PerformanceTimer timing("Vulkan.QueuePresent"); - state->presentPending = false; - // The finished submission was the last reader of the resident image. - state->presentKept.reset(); - const auto index = state->presentIndex; + state->queuePending = false; + const auto index = state->queueIndex; require(index < state->rendered.size(), "presented image index is out of range"); VkPresentInfoKHR present{VK_STRUCTURE_TYPE_PRESENT_INFO_KHR}; present.waitSemaphoreCount = 1; @@ -1314,6 +2006,12 @@ ShaderRecompiler::SpirvTarget VulkanDevice::Target() const { } Graphics::Context VulkanDevice::graphicsContext() const { + static const bool noCache = std::getenv("APS5_NO_CONTEXT_CACHE") != nullptr; + if (state->contextReady && !noCache) return state->context; + return buildContext(); +} + +Graphics::Context VulkanDevice::buildContext() const { auto context = Graphics::Context{ state->device, state->physical, @@ -1345,14 +2043,27 @@ Graphics::Context VulkanDevice::graphicsContext() const { context.recorder = state->recorder.get(); context.descriptorCache = state->descriptorCache.get(); context.samplerCache = state->samplerCache.get(); + context.drawIndirectFirstInstance = state->drawIndirectFirstInstance; + context.multiDrawIndirect = state->multiDrawIndirect; + context.drawIndirectCount = state->drawIndirectCount; + context.copiedWriters = state->copiedWriters.get(); + context.functions = state->functionsReady ? &state->deviceFunctions : nullptr; + context.descriptorIndexing = state->descriptorIndexing; return context; } -void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { - APS5_LOG_OUT("VulkanDevice::Draw indices=%u instances=%u indexSize=%u address=0x%llx shaders=%zu colorTarget=%u", draw.indexCount, draw.instanceCount, draw.indexSize, static_cast(draw.indexAddress), shaders.size(), static_cast(graphics.hasColorTarget)); +VulkanDevice::IndirectDrawSupport VulkanDevice::DrawIndirectSupport() const { + return {state->drawIndirectFirstInstance, state->multiDrawIndirect, state->drawIndirectCount}; +} + +void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, std::shared_ptr* recipe) { + // Two stdout lines per draw cost ~1.3 ms per frame of the queue-0 worker (part of it under the + // GPU mutex); APS5_TRACE_DRAWS=1 restores them. + static const bool trace = std::getenv("APS5_TRACE_DRAWS") != nullptr; + if (trace) APS5_LOG_OUT("VulkanDevice::Draw indices=%u instances=%u indexSize=%u address=0x%llx shaders=%zu colorTarget=%u", draw.indexCount, draw.instanceCount, draw.indexSize, static_cast(draw.indexAddress), shaders.size(), static_cast(graphics.hasColorTarget)); const auto context = graphicsContext(); - Graphics::Draw(context, graphics, draw, shaders, snapshots); - APS5_LOG_CHARS_OUT("VulkanDevice::Draw complete"); + Graphics::Draw(context, graphics, draw, shaders, snapshots, recipe); + if (trace) APS5_LOG_CHARS_OUT("VulkanDevice::Draw complete"); } namespace { @@ -1366,31 +2077,71 @@ bool ResourceCacheEnabled() { return !noResourceCache && !noTextureCache; } +// Compute templates serve dispatches whose ShaderData/FlattenedSrt words differ (the words leave +// the key; a hit refreshes the template's data buffers with the dispatch's words at the record, +// ShaderResources::RefreshData). APS5_NO_TEMPLATE_DATA_REFRESH=1 keys the words as before. +bool TemplateDataRefresh() { + static const bool enabled = std::getenv("APS5_NO_TEMPLATE_DATA_REFRESH") == nullptr; + return enabled; +} + // The content key of a compute stage, naming the device: the cache is process-wide and the driver // replaces the headless device with the windowed one while workers may still use the old one, so // the key names the device that built the entry (its descriptor set and pooled buffers belong to it). ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device) { - auto key = Graphics::ShaderResources::ContentKey(shader); + auto key = Graphics::ShaderResources::ContentKey(shader, !TemplateDataRefresh()); const auto deviceHandle = reinterpret_cast(device); key.push_back(static_cast(deviceHandle)); key.push_back(static_cast(deviceHandle >> 32u)); return key; } +// The phases of VulkanDevice::dispatch (APS5_PROFILE_DRAW), followed by one row per image lookup +// outcome (Graphics::LookupOutcomes) on the [indirect] line. +enum DispatchPhase : std::size_t { PhaseReap, PhaseResources, PhaseResourcesInsert, PhaseResourcesComplete, PhaseResourcesImages, PhaseResourcesUpload, PhaseResourcesDescriptors, PhaseResourcesOther, PhaseResourcesALocked, PhaseResourcesAddress, PhaseResourcesAUnlocked, PhaseResourcesRevalidate, PhaseResourcesFullBuild, PhaseResourcesHookWaits, PhaseProof, PhasePipeline, PhaseDecide, PhaseArgumentRead, PhaseRecordCommands, PhaseRecordKeeps, PhaseRecordDataRefresh, PhaseRecordBind, PhaseRecordMarks, PhaseRecordCompletion, PhaseRecord, PhaseSync, DispatchPhaseCount }; +constexpr std::array DispatchPhaseNames{"reap", "resources", "resources: cache insert", "resources: complete (wall)", "resources: B images", "resources: B upload", "resources: B descriptors", "resources: B bda+other", "resources: A locked (bda)", "resources: address bindings (bda)", "resources A (unlocked)", "resources: revalidate", "resources: full build (locked)", "resources: hook waits", "proof", "pipeline", "decide", "argument read", "record: commands", "record: keeps", "record: data refresh", "record: bind+dispatch", "record: marks", "record: completion", "record", "sync"}; +constexpr std::size_t DispatchRows = static_cast(DispatchPhaseCount) + static_cast(Graphics::LookupOutcomes::Count); + +const char* DispatchRowName(std::size_t row) { + if (row < DispatchPhaseCount) return DispatchPhaseNames[row]; + static const std::array imageRows = [] { + std::array rows; + for (std::size_t kind = 0; kind < Graphics::LookupOutcomes::Count; ++kind) rows[kind] = std::string("images: ") + Graphics::LookupOutcomes::Name(static_cast(kind)); + return rows; + }(); + return imageRows[row - DispatchPhaseCount].c_str(); +} + } struct PreparedDispatch { // Stage A done; stage B (Complete) runs in the dispatch under the mutex. Null when the resource - // cache held the key's object at prepare time: the dispatch looks it up and revalidates it. + // cache held the key's object at prepare time: the dispatch revalidates `cached` instead. std::shared_ptr resources; + std::shared_ptr cached; ResourceCache::Key key; - // APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch. + // APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch, and + // the prepare's parts for the driver's 'prepare:' rows (PreparePhase order). double prepareMs = 0; + enum PreparePhase : std::size_t { PrepareKey, PrepareFind, PreparePrecollect, PreparePresync, PrepareStageA, PreparePhaseCount }; + std::array phaseMs{}; // The stage-A pre-sync (see PrepareDispatch): the newest recorder serial waited for without the // mutex, whose batches the dispatch reaps under it before stage B (0: nothing waited for). std::uint64_t presyncSerial = 0; }; +struct RecipeHit { + std::shared_ptr recipe; + // The recipe's template and pipeline objects, locked by the pre-check. + std::shared_ptr resources; + std::shared_ptr objects; + // The pre-sync's serial (0: nothing waited for), reaped by DispatchRecipe. + std::uint64_t presyncSerial = 0; + bool indirect = false; + // APS5_PROFILE_DRAW: the pre-check's time (the driver's 'recipe pre-check' row). + double precheckMs = 0; +}; + namespace { // APS5_PROFILE_DRAW: the pre-syncs made and the time they waited (unlocked), as [presync] every 10 s. @@ -1406,6 +2157,350 @@ PresyncCounters& Presyncs() { return counters; } +// A cached object's Revalidate collects the write watch over its surfaces under the mutex; the +// walk is made before the lock instead, so those collects are memo hits (as stage A's precollect +// does for a build). APS5_NO_CACHED_PRECOLLECT=1 leaves the walks under the lock. +bool CachedPrecollect() { + static const bool cachedPrecollect = std::getenv("APS5_NO_CACHED_PRECOLLECT") == nullptr; + return cachedPrecollect; +} + +// APS5_NO_PRESYNC=1 disables the stage-A pre-sync. +bool NoPresync() { + static const bool noPresync = std::getenv("APS5_NO_PRESYNC") != nullptr; + return noPresync; +} + +// Debug aid: APS5_SYNC_DISPATCH=1 waits for every dispatch, as before batching. +bool SyncEachDispatch() { + static const bool syncEachDispatch = std::getenv("APS5_SYNC_DISPATCH") != nullptr; + return syncEachDispatch; +} + +// Debug aid: APS5_TRACE_DISPATCH_IO prints every dispatch's resources (after write-back when synced). +bool TraceDispatchIo() { + static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr; + return traceIo; +} + +// The [dispatch] phases for indirect dispatches alone, every 10 s on an [indirect] line: what the +// 'indirect' GpuMutex hold ([lock] line) spends its time on inside the device call (the label +// record before it is timed in the driver, [labels] line), with the longest hold seen and the +// copied-writer lists the indirect decision scans. All under the mutex, like the rest. +struct IndirectHold { + std::array phaseMs{}; + // Rows with a count (the cache insert's evictions, the image lookups by outcome). + std::array phaseCounts{}; + std::uint64_t count = 0; + double totalMs = 0; + double maxMs = 0; + // The phases and lookup outcomes of the longest call, so the maximum is named. + std::string maxPhases; + std::uint64_t writersScanned = 0; + std::uint64_t drawWritersScanned = 0; + std::size_t maxWriters = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +// The running totals of the device call (APS5_PROFILE_DRAW), shared by dispatch, DispatchRecipe +// and recordDispatch; mutated under GuestMemory::GpuMutex like the calls themselves. The +// [rescache] and [vk] counters: template hits by outcome, vkGetDeviceProcAddr lookups made inside +// the device call (this thread's), the pre-dispatch barriers recorded or skipped (Recorder::Commands). +struct DispatchCounters { + std::array phaseTotals{}; + std::uint64_t profiledDispatches = 0; + IndirectHold indirectHold; + std::uint64_t cacheHits = 0, cacheMisses = 0, cacheInvalidated = 0, templateRefreshed = 0, templateSameWords = 0; + double templateRevalidateMs = 0; + std::uint64_t procLookups = 0, preBarriersRecorded = 0, preBarriersSkipped = 0; + std::chrono::steady_clock::time_point cacheReport = std::chrono::steady_clock::now(); +}; + +DispatchCounters& Dispatches() { + static DispatchCounters counters; + return counters; +} + +// The [recipe] line (APS5_PROFILE_DRAW, every 10 s), dispatch and indirect rows apart. The +// pre-check runs without the mutex, so the counters are atomic. +enum RecipeMiss : std::size_t { MissNoRecipe, MissDevice, MissTemplateGone, MissObjectsGone, MissNotRecordable, RecipeMissCount }; +constexpr std::array RecipeMissNames{"no recipe", "device", "template gone", "objects gone", "not recordable"}; +// The draw rows name the miss reasons of DrawWithRecipe (a target or a pipeline gone is the +// objects-gone slot). +constexpr std::array DrawRecipeMissNames{"no recipe", "device", "template gone", "targets gone", "not recordable"}; +constexpr std::array(Graphics::ShaderResources::ProofPath::Count)> ProofPathNames{"fast", "T1 refreshed", "full"}; +constexpr std::array(Graphics::ShaderResources::ProofFailure::Count)> ProofFailureNames{"none", "imports", "evicted", "pending", "changed", "keys", "other"}; + +struct RecipeCounters { + struct Kind { + std::atomic hits{0}; + std::array, RecipeMissCount> precheckMisses{}; + std::array, static_cast(Graphics::ShaderResources::ProofPath::Count)> proofPaths{}; + std::array, static_cast(Graphics::ShaderResources::ProofFailure::Count)> rebuilds{}; + std::atomic rebuildDevice{0}; + std::atomic presyncsFromRecipe{0}, presyncsRecomputed{0}, presyncWaits{0}; + std::atomic dataRefreshed{0}, dataSkipped{0}, refreshByWords{0}; + std::atomic restarts{0}, attaches{0}, verified{0}, verifyReplaced{0}; + std::atomic precheckNs{0}, proofNs{0}, recordNs{0}; + }; + std::array kinds; + std::atomic lastReport{0}; +}; + +RecipeCounters& Recipes() { + static RecipeCounters counters; + return counters; +} + +void reportRecipes() { + auto& recipes = Recipes(); + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = recipes.lastReport.load(); + if (nowMs - last < 10000 || !recipes.lastReport.compare_exchange_strong(last, nowMs)) return; + const auto take = [](std::atomic& counter) { return static_cast(counter.exchange(0, std::memory_order_relaxed)); }; + for (std::size_t kind = 0; kind < 3; ++kind) { + auto& counters = recipes.kinds[kind]; + const auto hits = take(counters.hits); + std::string misses, proofs, rebuilds; + char text[64]; + for (std::size_t i = 0; i < RecipeMissCount; ++i) { + std::snprintf(text, sizeof(text), "%s%s %llu", i == 0 ? "" : ", ", (kind == 2 ? DrawRecipeMissNames : RecipeMissNames)[i], take(counters.precheckMisses[i])); + misses += text; + } + for (std::size_t i = 0; i < ProofPathNames.size(); ++i) { + std::snprintf(text, sizeof(text), "%s%s %llu", i == 0 ? "" : ", ", ProofPathNames[i], take(counters.proofPaths[i])); + proofs += text; + } + for (std::size_t i = 1; i < ProofFailureNames.size(); ++i) { + std::snprintf(text, sizeof(text), "%s%s %llu", i == 1 ? "" : ", ", ProofFailureNames[i], take(counters.rebuilds[i])); + rebuilds += text; + } + std::snprintf(text, sizeof(text), ", device %llu", take(counters.rebuildDevice)); + rebuilds += text; + const auto perHit = [&](std::atomic& ns) { return hits != 0 ? static_cast(take(ns)) / 1000.0 / static_cast(hits) : 0.0; }; + const auto precheckUs = perHit(counters.precheckNs); + const auto proofUs = perHit(counters.proofNs); + const auto recordUs = perHit(counters.recordNs); + std::fprintf(stderr, "[recipe] %s (10 s): hits %llu; pre-check misses: %s; proofs: %s; rebuilds: %s; presyncs from the recipe %llu (recomputed %llu, waited %llu); data refreshes recorded %llu / skipped by hash %llu (%llu decided by words: data hits); restarts %llu, attaches %llu; us per hit: pre-check %.1f, proof %.1f, record %.1f; verified %llu (object replaced %llu)\n", kind == 0 ? "dispatch" : kind == 1 ? "indirect" : "draw", hits, misses.c_str(), proofs.c_str(), rebuilds.c_str(), take(counters.presyncsFromRecipe), take(counters.presyncsRecomputed), take(counters.presyncWaits), take(counters.dataRefreshed), take(counters.dataSkipped), take(counters.refreshByWords), take(counters.restarts), take(counters.attaches), precheckUs, proofUs, recordUs, take(counters.verified), take(counters.verifyReplaced)); + } +} + +// The phases of one device call (APS5_PROFILE_DRAW): this call's rows (formatted only for a slow +// call or a new [indirect] maximum) and the running totals of Dispatches(); `indirect` charges the +// [indirect] rows too (cleared when a CPU-resolved indirect dispatch continues as a direct one). +struct DispatchTimer { + bool profile; + bool indirect; + std::size_t spirvWords; + std::uint64_t programAddress; + std::array callMs{}; + std::chrono::steady_clock::time_point start; + std::chrono::steady_clock::time_point phaseStart; + DispatchTimer(bool profile, bool indirect, std::size_t spirvWords, std::uint64_t programAddress) : profile(profile), indirect(indirect), spirvWords(spirvWords), programAddress(programAddress), start(std::chrono::steady_clock::now()), phaseStart(start) {} + void add(DispatchPhase which, double ms) { + if (!profile) return; + auto& d = Dispatches(); + callMs[which] += ms; + d.phaseTotals[which] += ms; + if (indirect) d.indirectHold.phaseMs[which] += ms; + } + void phase(DispatchPhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - phaseStart).count(); + add(which, ms); + phaseStart = now; + // Slow phases are reported as they finish, so a dispatch that never completes shows where it is. + if (ms > 1000) std::fprintf(stderr, "[dispatch] %s took %.0f ms (%zu words, program 0x%llx)\n", DispatchRowName(which), ms, spirvWords, static_cast(programAddress)); + } + void restart() { phaseStart = std::chrono::steady_clock::now(); } + // The call's phases as text, only when wanted: a slow call, or a new [indirect] maximum. + std::string formatCall() const { + std::string text; + char row[96]; + for (std::size_t i = 0; i < DispatchRows; ++i) { + if (callMs[i] == 0) continue; + std::snprintf(row, sizeof(row), " %s=%.2fms", DispatchRowName(i), callMs[i]); + text += row; + } + return text; + } + // Every 1000 profiled dispatches: the totals per phase. + static void countDispatch(bool profile) { + if (!profile) return; + auto& d = Dispatches(); + if (++d.profiledDispatches % 1000 != 0) return; + std::string report; + for (std::size_t row = 0; row < DispatchPhaseCount; ++row) { + if (d.phaseTotals[row] != 0) report += " " + std::string(DispatchRowName(row)) + "=" + std::to_string(static_cast(d.phaseTotals[row] / 1000)) + "s"; + } + std::fprintf(stderr, "[dispatch] %llu dispatches, phase totals:%s\n", static_cast(d.profiledDispatches), report.c_str()); + } + // The call's end: the proc lookups it made, the slow-call report, the [indirect] hold + // accounting (`indirectHold`: the call was an indirect dispatch, CPU-resolved or not) and line. + void finish(std::uint64_t lookupsBefore, const char* groupsText, bool indirectHold) const { + if (!profile) return; + auto& d = Dispatches(); + d.procLookups += Graphics::DeviceProcLookups() - lookupsBefore; + const auto now = std::chrono::steady_clock::now(); + const auto totalMs = std::chrono::duration(now - start).count(); + if (totalMs > 100) std::fprintf(stderr, "[dispatch] %s %zu words:%s\n", groupsText, spirvWords, formatCall().c_str()); + if (!indirectHold) return; + auto& hold = d.indirectHold; + ++hold.count; + hold.totalMs += totalMs; + if (totalMs > hold.maxMs) { + hold.maxMs = totalMs; + hold.maxPhases = formatCall(); + } + if (now - hold.lastReport <= std::chrono::seconds(10)) return; + hold.lastReport = now; + std::string report; + for (std::size_t i = 0; i < DispatchRows; ++i) { + if (hold.phaseMs[i] == 0 && hold.phaseCounts[i] == 0) continue; + char text[112]; + if (hold.phaseCounts[i] != 0) std::snprintf(text, sizeof(text), " %s %llux %.0f ms", DispatchRowName(i), static_cast(hold.phaseCounts[i]), hold.phaseMs[i]); + else std::snprintf(text, sizeof(text), " %s %.0f ms", DispatchRowName(i), hold.phaseMs[i]); + report += text; + } + std::fprintf(stderr, "[indirect] %llu indirect dispatches spent %.0f ms inside the device call (10 s; max %.1f ms:%s), by phase (the 'resources: ...' rows split 'resources'; 'hook waits' overlaps them; 'images: ...' rows are the lookups by outcome, count x ms, inside 'B images'):%s; copied-writer lists scanned per decision: dispatch avg %.1f (max %zu), draw avg %.1f\n", static_cast(hold.count), hold.totalMs, hold.maxMs, hold.maxPhases.c_str(), report.c_str(), hold.count != 0 ? static_cast(hold.writersScanned) / hold.count : 0.0, hold.maxWriters, hold.count != 0 ? static_cast(hold.drawWritersScanned) / hold.count : 0.0); + hold.phaseMs = {}; + hold.phaseCounts = {}; + hold.count = 0; + hold.totalMs = 0; + hold.maxMs = 0; + hold.maxPhases.clear(); + hold.writersScanned = 0; + hold.drawWritersScanned = 0; + hold.maxWriters = 0; + } +}; + +} + +struct RecordedDispatch { + const Graphics::Context* context; + const Graphics::CompiledShader* shader; + std::shared_ptr resources; + std::shared_ptr objects; + VkShaderStageFlags pushStages; + const std::array* pushBytes; + std::uint32_t x, y, z; + // The DISPATCH_INDIRECT arguments (0: direct) and their host import when the GPU reads them. + std::uint64_t arguments; + const Graphics::HostImport* argumentImport; + std::uint64_t programAddress; + // What decides the template's data refresh: None (the object was built by this call), Words + // (a template hit: RefreshData compares every buffer's words), Hash (a recipe: the template's + // DataWordsHash against the recipe's, RefreshData only when they differ). + enum class DataRefresh { None, Words, Hash }; + DataRefresh dataRefresh; + std::uint64_t dataWordsHash; + // A template hit's revalidate time, charged to the [rescache] refresh accounting. + double revalidateMs; + DispatchTimer* timer; + // Whether the data refresh recorded anything. + bool refreshed = false; +}; + +VkDevice VulkanDevice::Device() const { + return state->device; +} + +bool VulkanDevice::DispatchRecipes() { + static const bool noDispatchRecipe = std::getenv("APS5_NO_DISPATCH_RECIPE") != nullptr; + return !noDispatchRecipe; +} + +bool VulkanDevice::VerifyRecipes() { + static const bool verifyRecipe = std::getenv("APS5_VERIFY_RECIPE") != nullptr; + return verifyRecipe; +} + +bool VulkanDevice::TemplateDataRefresh() { + return AgcDriver::TemplateDataRefresh(); +} + +void VulkanDevice::NoteRecipe(RecipeEvent event, bool indirect) { + NoteRecipe(event, indirect ? RecipeKind::Indirect : RecipeKind::Dispatch); +} + +void VulkanDevice::NoteRecipe(RecipeEvent event, RecipeKind kind) { + auto& counters = Recipes().kinds[static_cast(kind)]; + (event == RecipeEvent::Restart ? counters.restarts : counters.attaches).fetch_add(1, std::memory_order_relaxed); +} + +void VulkanDevice::NoteDrawRecipeMiss(DrawRecipePrecheck miss) { + auto& counters = Recipes().kinds[static_cast(RecipeKind::Draw)]; + counters.precheckMisses[miss == DrawRecipePrecheck::NoRecipe ? MissNoRecipe : MissDevice].fetch_add(1, std::memory_order_relaxed); +} + +RecipeOutcome VulkanDevice::DrawFromRecipe(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, const std::shared_ptr& recipe) { + PerformanceTimer timing("Vulkan.DrawFromRecipe"); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& counters = Recipes().kinds[static_cast(RecipeKind::Draw)]; + // R10: the recipe names the device it was built on (the driver may have replaced the device + // between the hit and the lock). + if (recipe->device != state->device) { + counters.rebuildDevice.fetch_add(1, std::memory_order_relaxed); + return RecipeOutcome::Rebuild; + } + const auto context = graphicsContext(); + const auto outcome = Graphics::DrawWithRecipe(context, graphics, draw, shaders, snapshots, *recipe); + if (!outcome.recorded) { + switch (outcome.miss) { + case Graphics::DrawRecipeMiss::NotRecordable: counters.precheckMisses[MissNotRecordable].fetch_add(1, std::memory_order_relaxed); break; + case Graphics::DrawRecipeMiss::TargetGone: + case Graphics::DrawRecipeMiss::ObjectsGone: counters.precheckMisses[MissObjectsGone].fetch_add(1, std::memory_order_relaxed); break; + case Graphics::DrawRecipeMiss::TemplateGone: counters.precheckMisses[MissTemplateGone].fetch_add(1, std::memory_order_relaxed); break; + case Graphics::DrawRecipeMiss::Proof: counters.rebuilds[static_cast(outcome.proof.failure)].fetch_add(1, std::memory_order_relaxed); break; + default: break; + } + return RecipeOutcome::Rebuild; + } + counters.proofPaths[static_cast(outcome.proof.path)].fetch_add(1, std::memory_order_relaxed); + counters.hits.fetch_add(1, std::memory_order_relaxed); + if (profile) { + counters.proofNs.fetch_add(static_cast(outcome.proofUs * 1000.0), std::memory_order_relaxed); + counters.recordNs.fetch_add(static_cast(outcome.recordUs * 1000.0), std::memory_order_relaxed); + reportRecipes(); + } + return RecipeOutcome::Recorded; +} + +std::uint64_t VulkanDevice::presync(std::span> surfaces) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& counters = Presyncs(); + bool overlaps = false; + for (const auto& [address, bytes] : surfaces) overlaps = overlaps || Graphics::Recorder::SnapshotWriteOverlaps(address, static_cast(bytes)); + std::uint64_t serial = 0; + if (overlaps) { + // No reap here (SubmitAndEpoch's would run completions, which a compute worker must not do + // on queue 0's behalf, see SubmitRecorded): the dispatch reaps after the wait. + std::lock_guard lock(GuestMemory::GpuMutex()); + if (state->recorder) { + bool open = false; + for (const auto& [address, bytes] : surfaces) { + const auto info = state->recorder->DescribePendingWrite(address, static_cast(bytes)); + if (!info.has_value()) continue; + open = open || info->open; + serial = std::max(serial, info->serial); + } + if (open) serial = state->recorder->SubmitAndEpoch(); + } + } + if (serial != 0) { + const auto waitStart = std::chrono::steady_clock::now(); + WaitRecorded(serial); + counters.presyncs.fetch_add(1, std::memory_order_relaxed); + counters.waitedUs.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), std::memory_order_relaxed); + } + if (profile) { + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(); + if (nowMs - last >= 10000 && counters.lastReport.compare_exchange_strong(last, nowMs)) std::fprintf(stderr, "[presync] %llu dispatches checked, %llu pre-syncs waited %.0f ms (cumulative)\n", static_cast(counters.checked.load()), static_cast(counters.presyncs.load()), counters.waitedUs.load() / 1000.0); + } + return serial; } std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots) { @@ -1421,11 +2516,28 @@ std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderReco // looks it up again by the key made here; the object stays local, only its surfaces matter // below. Anything inserted between here and the dispatch is simply replaced by this build. std::shared_ptr cached; + auto phaseStart = start; + const auto phase = [&](PreparedDispatch::PreparePhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + prepared->phaseMs[which] += std::chrono::duration(now - phaseStart).count(); + phaseStart = now; + }; if (ResourceCacheEnabled() && shader.variantId != 0) { prepared->key = DispatchContentKey(compute, context.device); + phase(PreparedDispatch::PrepareKey); cached = state->resourceCache.Find(prepared->key); + prepared->cached = cached; + phase(PreparedDispatch::PrepareFind); + } + if (cached == nullptr) { + prepared->resources = std::make_shared(context, compute, snapshots, true); + phase(PreparedDispatch::PrepareStageA); + } + if (cached != nullptr && CachedPrecollect()) { + cached->PrecollectSurfaces(); + phase(PreparedDispatch::PreparePrecollect); } - if (cached == nullptr) prepared->resources = std::make_shared(context, compute, snapshots, true); // The pre-sync. Stage B's image lookups that read guest memory on the CPU (PresyncSurfaces) // wait, through the flush hook and under the mutex, for the recorded work writing those // surfaces: every other worker queues behind that wait. So the wait is made here instead, @@ -1435,59 +2547,257 @@ std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderReco // dispatch reaps those batches at its own lock so the hook then finds nothing pending. Work // noted in between falls back to the locked wait as before. A cached object's Revalidate // repeats the same lookups when its stamps fail, so it gets the same pre-sync. Needs timeline - // semaphores. APS5_NO_PRESYNC=1 disables it. - static const bool noPresync = std::getenv("APS5_NO_PRESYNC") != nullptr; - if (!noPresync && CanWaitUnlocked()) { - auto& counters = Presyncs(); - counters.checked.fetch_add(1, std::memory_order_relaxed); + // semaphores. + if (!NoPresync() && CanWaitUnlocked()) { + Presyncs().checked.fetch_add(1, std::memory_order_relaxed); const auto surfaces = cached != nullptr ? cached->PresyncSurfaces() : prepared->resources->PresyncSurfaces(); - bool overlaps = false; - for (const auto& [address, bytes] : surfaces) overlaps = overlaps || Graphics::Recorder::SnapshotWriteOverlaps(address, static_cast(bytes)); - if (overlaps) { - std::uint64_t serial = 0; - { - // No reap here (SubmitAndEpoch's would run completions, which a compute worker must - // not do on queue 0's behalf, see SubmitRecorded): the dispatch reaps after the wait. - std::lock_guard lock(GuestMemory::GpuMutex()); - if (state->recorder) { - bool open = false; - for (const auto& [address, bytes] : surfaces) { - const auto info = state->recorder->DescribePendingWrite(address, static_cast(bytes)); - if (!info.has_value()) continue; - open = open || info->open; - serial = std::max(serial, info->serial); - } - if (open) serial = state->recorder->SubmitAndEpoch(); - } - } - if (serial != 0) { - const auto waitStart = std::chrono::steady_clock::now(); - WaitRecorded(serial); - prepared->presyncSerial = serial; - counters.presyncs.fetch_add(1, std::memory_order_relaxed); - counters.waitedUs.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), std::memory_order_relaxed); - } - } - if (profile) { - const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); - auto last = counters.lastReport.load(); - if (nowMs - last >= 10000 && counters.lastReport.compare_exchange_strong(last, nowMs)) std::fprintf(stderr, "[presync] %llu dispatches checked, %llu pre-syncs waited %.0f ms (cumulative)\n", static_cast(counters.checked.load()), static_cast(counters.presyncs.load()), counters.waitedUs.load() / 1000.0); - } + prepared->presyncSerial = presync(surfaces); + phase(PreparedDispatch::PreparePresync); } if (profile) prepared->prepareMs = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); return prepared; } -void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { - static_cast(dispatch(shader, x, y, z, 0, snapshots, programAddress, std::move(prepared))); +std::span VulkanDevice::PreparePhaseMs(const PreparedDispatch& prepared) { + static_assert(PreparedDispatch::PreparePhaseCount == 5); + return std::span(prepared.phaseMs); } -VulkanDevice::IndirectOutcome VulkanDevice::DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { - return dispatch(shader, 0, 0, 0, arguments, snapshots, programAddress, std::move(prepared)); +std::shared_ptr VulkanDevice::PrepareRecipe(const std::shared_ptr& recipe, bool indirect) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& counters = Recipes().kinds[indirect ? 1 : 0]; + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto miss = [&](RecipeMiss reason) { + counters.precheckMisses[reason].fetch_add(1, std::memory_order_relaxed); + return std::shared_ptr{}; + }; + if (!DispatchRecipes() || recipe == nullptr) return miss(MissNoRecipe); + if (recipe->device != state->device) return miss(MissDevice); + auto hit = std::make_shared(); + hit->recipe = recipe; + hit->indirect = indirect; + hit->resources = recipe->templateRef.lock(); + if (hit->resources == nullptr) return miss(MissTemplateGone); + hit->objects = recipe->objects.lock(); + if (hit->objects == nullptr) return miss(MissObjectsGone); + if (CachedPrecollect()) hit->resources->PrecollectSurfaces(); + // The pre-sync as PrepareDispatch makes it, over the recipe's surface list while the import + // table still has the identity the list was computed under (a surface is CPU-read only while + // its memory has no import), else over the template's current list. + if (!NoPresync() && CanWaitUnlocked()) { + Presyncs().checked.fetch_add(1, std::memory_order_relaxed); + if (Graphics::HostImportsUnchanged(graphicsContext(), recipe->presyncProof)) { + counters.presyncsFromRecipe.fetch_add(1, std::memory_order_relaxed); + hit->presyncSerial = presync(recipe->presyncSurfaces); + } else { + counters.presyncsRecomputed.fetch_add(1, std::memory_order_relaxed); + hit->presyncSerial = presync(hit->resources->PresyncSurfaces()); + } + if (hit->presyncSerial != 0) counters.presyncWaits.fetch_add(1, std::memory_order_relaxed); + } + if (profile) { + const auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); + hit->precheckMs = static_cast(ns) / 1e6; + counters.precheckNs.fetch_add(static_cast(ns), std::memory_order_relaxed); + } + return hit; } -VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { +void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe) { + static_cast(dispatch(shader, x, y, z, 0, snapshots, programAddress, std::move(prepared), recipe)); +} + +VulkanDevice::IndirectOutcome VulkanDevice::DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe) { + return dispatch(shader, 0, 0, 0, arguments, snapshots, programAddress, std::move(prepared), recipe); +} + +void VulkanDevice::decideIndirect(RecordedDispatch& record, IndirectOutcome& outcome, char* groupsText) { + auto& recorder = *state->recorder; + auto& timer = *record.timer; + const auto& context = *record.context; + const auto arguments = record.arguments; + const auto* limit = state->properties.limits.maxComputeWorkGroupCount; + // Decided here, after the resource build (which may retire imports and wait for recorded + // work) and before the dispatch itself is recorded, so the import cannot be dropped before + // the record. The build already recorded the copy-ins of its misaligned and staged regions + // (GuestBufferMemory::recordGpuCopies), so the CPU fallback's sync below drains them in one + // batch and records the dispatch with its copy-backs in the next; the completions the reap + // between them runs store to the CPU outside host imports (copied regions' write-backs) or + // onto label dwords, so none lands inside a copied range for the copy-back to roll back. + // A label over the argument dwords (a completion-deferred one lands by a CPU store the GPU + // read would miss; a GPU one is ordered but rare enough to share the fallback). Only those + // dwords: a global "any completion label pending" test would send most indirect dispatches + // to the CPU, as ~5 such labels are pending per frame. Stamp 0 matches every live entry. + const auto labelPending = [&] { + for (std::uint64_t dword = arguments; dword < arguments + 12; dword += 4) { + if (recorder.PendingLabel(dword, 4, 0).has_value()) return true; + } + return false; + }; + if (Graphics::StorageTexture::FlushPending(arguments, 12, nullptr, "indirect dispatch arguments")) { + // As the flush hook does: the stores were only recorded and the CPU is about to read them. + outcome.cpuReason = 1; + Graphics::Recorder::CountSync(2); + recorder.Sync(); + } else if (labelPending() || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); }) + || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); })) { + outcome.cpuReason = 2; + } else if ((record.argumentImport = Graphics::HostImportFor(context, arguments, 12)) == nullptr) { + // Valid until the next refreshImports, which only runs under GuestMemory::GpuMutex (held + // here, by every Dispatch caller) and Keeps a retired VkBuffer while the recorder is busy. + outcome.cpuReason = 3; + } + if (timer.profile) { + auto& hold = Dispatches().indirectHold; + hold.writersScanned += state->copiedWriters->size(); + hold.drawWritersScanned += Graphics::DrawCopiedWriters()->size(); + hold.maxWriters = std::max(hold.maxWriters, state->copiedWriters->size()); + } + timer.phase(PhaseDecide); + if (outcome.cpuReason == 0) return; + // The read waits for the producing batch through the flush hook, as the driver's resolve + // did for every indirect dispatch before; the counts then go through the direct checks. + const auto readStart = std::chrono::steady_clock::now(); + std::array groups{}; + GuestMemory::Read(arguments, std::as_writable_bytes(std::span(groups)), 4); + outcome.argumentReadMs = std::chrono::duration(std::chrono::steady_clock::now() - readStart).count(); + record.x = groups[0]; + record.y = groups[1]; + record.z = groups[2]; + // Timed before `arguments` is cleared, so the read counts as an indirect phase. + timer.phase(PhaseArgumentRead); + // From here on the phases of a CPU-resolved indirect dispatch are charged as direct ones. + record.arguments = 0; + timer.indirect = false; + std::snprintf(groupsText, 40, "%ux%ux%u", record.x, record.y, record.z); + if (record.x > limit[0] || record.y > limit[1] || record.z > limit[2]) throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits"); +} + +void VulkanDevice::recordDispatch(RecordedDispatch& record) { + auto& d = Dispatches(); + auto& recorder = *state->recorder; + auto& timer = *record.timer; + const auto& context = *record.context; + auto& resources = *record.resources; + const auto arguments = record.arguments; + const auto* argumentImport = record.argumentImport; + // The record phase split (APS5_PROFILE_DRAW, rows "record: ..." of the [dispatch] totals): + // opening the batch, the keeps, the data refresh, the barriers with the bind and the dispatch + // itself, the pending-write notes and marks (MarkGpuWrites), and the completion registration. + auto recordFrom = timer.profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto recordStep = [&](DispatchPhase which) { + if (!timer.profile) return; + const auto now = std::chrono::steady_clock::now(); + timer.add(which, std::chrono::duration(now - recordFrom).count()); + recordFrom = now; + }; + // Whether the last command recorded into the batch left every earlier write visible to this + // dispatch's reads and writes (a closed label-store run, the previous dispatch's own trailing + // barrier, a fill's or copy's): the pre-dispatch barrier below then adds nothing and is + // skipped. APS5_NO_BARRIER_ELISION=1 or APS5_FULL_BARRIERS=1 records it every time. + const bool barrierElision = Graphics::Recorder::MergeBarriers(); + // A queued DCC key store over memory this dispatch writes or reads in place (unknown for an + // address-based build), or over its indirect arguments, must land before it: the title's clear + // kernels store keys through V#s, its decompress kernels read them. A queued label store over + // such memory likewise (the dispatch would read a stale dword, or its write would lose to + // the label). + const auto touches = [&](std::uint64_t begin, std::uint64_t end) { + const auto bytes = static_cast(end - begin); + return resources.WritesOverlap(begin, bytes) || resources.ReadsOverlap(begin, bytes) || (argumentImport != nullptr && begin < arguments + 12 && arguments < end); + }; + if (recorder.HasQueuedKeyStores() && (resources.HoldsLease() || recorder.AnyQueuedKeyStore(touches))) recorder.FlushKeyStores(); + if (recorder.HasQueuedStores() && (resources.HoldsLease() || recorder.AnyQueuedStore(touches))) recorder.FlushStores(); + VkAccessFlags covered = 0; + const auto commands = recorder.Commands(&covered); + recordStep(PhaseRecordCommands); + recorder.Keep(record.objects); + recorder.Keep(record.resources); + recordStep(PhaseRecordKeeps); + if (record.dataRefresh != RecordedDispatch::DataRefresh::None) { + // The template's data buffers take this dispatch's words: a transfer write the pre-dispatch + // barrier makes visible (so it is recorded whatever the previous command covered), ordered + // after an earlier dispatch's reads of the buffers by that dispatch's trailing barrier. A + // recipe compares the two 64-bit hashes first: equal hashes mean the buffers hold the words. + const bool differs = record.dataRefresh == RecordedDispatch::DataRefresh::Words || resources.DataWordsHash() != record.dataWordsHash; + if (differs && resources.RefreshData(commands, *record.shader, &recorder)) { + covered = 0; + ++d.templateRefreshed; + d.templateRevalidateMs += record.revalidateMs; + record.refreshed = true; + } else { + ++d.templateSameWords; + } + recordStep(PhaseRecordDataRefresh); + } + using CommandClass = Graphics::Recorder::CommandClass; + if (Graphics::Recorder::BarrierValidate()) { + if (argumentImport != nullptr) { + const std::pair argumentRange{arguments, arguments + 12}; + recorder.NoteAccess(CommandClass::IndirectArguments, Graphics::Recorder::Access{std::span(&argumentRange, 1), {}, {}, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT}); + } + const auto reads = resources.InPlaceReads(); + const auto images = resources.StorageImages(); + recorder.NoteAccess(CommandClass::DispatchLeading, Graphics::Recorder::Access{reads, resources.GpuWrites(), images, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, resources.HoldsLease()}); + } + if (argumentImport != nullptr) { + // The group counts were stored by earlier recorded work (a dispatch in place, a fill) or the + // host; the indirect read follows all of it. + const auto timing = recorder.BeginGpuTiming(CommandClass::IndirectArguments); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT); + Graphics::Recorder::CountBarriers(CommandClass::IndirectArguments); + recorder.EndGpuTiming(timing, 12); + recorder.NotePendingRead(arguments, 12, Graphics::Recorder::ReadKind::Indirect); + } + // Results of earlier recorded work are visible to this dispatch, its own to everything after. + constexpr VkAccessFlags shaderAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + if ((covered & shaderAccess) == shaderAccess && barrierElision) { + ++d.preBarriersSkipped; + Graphics::Recorder::CountMerged(CommandClass::DispatchLeading); + } else { + const auto timing = recorder.BeginGpuTiming(CommandClass::DispatchLeading); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + Graphics::Recorder::CountBarriers(CommandClass::DispatchLeading); + recorder.EndGpuTiming(timing); + ++d.preBarriersRecorded; + } + context.Resolved(&Graphics::DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->pipeline); + resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout); + if (record.pushStages != 0) { + context.Resolved(&Graphics::DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, record.objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes, record.pushBytes->data()); + } + const auto gpuTiming = recorder.BeginGpuTiming(record.programAddress != 0 ? record.programAddress : record.shader->program->variantId); + if (argumentImport != nullptr) context.Resolved(&Graphics::DeviceFunctions::cmdDispatchIndirect, "vkCmdDispatchIndirect")(commands, argumentImport->buffer, arguments - argumentImport->base); + else context.Resolved(&Graphics::DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, record.x, record.y, record.z); + recorder.EndGpuTiming(gpuTiming); + const auto trailingTiming = recorder.BeginGpuTiming(CommandClass::DispatchTrailing); + constexpr VkAccessFlags dispatchedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT; + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT, dispatchedAccess); + Graphics::Recorder::CountBarriers(CommandClass::DispatchTrailing); + recorder.EndGpuTiming(trailingTiming); + recorder.MarkCovered(dispatchedAccess); + recordStep(PhaseRecordBind); + resources.MarkGpuWrites(recorder); + recordStep(PhaseRecordMarks); + // Only copied written buffers (and BDA fault checks) need work once the GPU is done; without them + // the batch can signal its labels from the GPU. + if (resources.NeedsCompletion()) { + // Listed for DispatchIndirect until the write-back ran (a non-reusable object, so once). + auto writers = state->copiedWriters; + auto kept = record.resources; + recorder.OnComplete([kept, writers] { + // Delisted before the write-back: one that fails must not keep indirect dispatches on the CPU. + writers->erase(std::remove(writers->begin(), writers->end(), kept), writers->end()); + kept->WriteBackBuffers(); + }); + // Listed after the registration: a throw there leaves nothing that would pin the CPU path forever. + writers->push_back(kept); + } + recordStep(PhaseRecordCompletion); +} + +VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipeOut) { PerformanceTimer timing("Vulkan.Dispatch"); + if (recipeOut != nullptr) *recipeOut = nullptr; // Group counts for the trace lines; an indirect dispatch does not know them. char groupsText[40]; if (arguments != 0) std::snprintf(groupsText, sizeof(groupsText), "indirect"); @@ -1512,78 +2822,59 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec const std::uint64_t pipelineKey = shader.variantId != 0 ? (shader.variantId << 1u) | (pushStages != 0 ? 1u : 0u) : 0u; std::shared_ptr objects; if (pipelineKey != 0) { + std::lock_guard pipelines(state->computePipelinesMutex); if (const auto found = state->computePipelines.find(pipelineKey); found != state->computePipelines.end()) objects = found->second; } static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - // Debug aid: APS5_SYNC_DISPATCH=1 waits for every dispatch, as before batching. - static const bool syncEachDispatch = std::getenv("APS5_SYNC_DISPATCH") != nullptr; - auto phaseStart = std::chrono::steady_clock::now(); - const auto dispatchStart = phaseStart; - std::string phases; - // Totals per phase across dispatches, reported every 1000 dispatches under APS5_PROFILE_DRAW. - static std::map phaseTotals; - static std::uint64_t profiledDispatches = 0; - // The same phases for indirect dispatches alone, every 10 s on an [indirect] line: what the - // 'indirect' GpuMutex hold ([lock] line) spends its time on inside this function (the label - // record before it is timed in the driver, [labels] line), with the longest hold seen and the - // copied-writer lists the indirect decision scans. All under the mutex, like the rest. - struct IndirectHold { - std::map phaseMs; - std::uint64_t count = 0; - double totalMs = 0; - double maxMs = 0; - std::uint64_t writersScanned = 0; - std::uint64_t drawWritersScanned = 0; - std::size_t maxWriters = 0; - std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); - }; - static IndirectHold indirectHold; - const auto phase = [&](const char* name) { - if (!profile) return; - const auto now = std::chrono::steady_clock::now(); - const auto ms = std::chrono::duration(now - phaseStart).count(); - char text[64]; - std::snprintf(text, sizeof(text), " %s=%.0fms", name, ms); - phases += text; - phaseTotals[name] += ms; - if (arguments != 0) indirectHold.phaseMs[name] += ms; - phaseStart = now; - // Slow phases are reported as they finish, so a dispatch that never completes shows where it is. - if (ms > 1000) std::fprintf(stderr, "[dispatch] %s took %.0f ms (%zu words)\n", name, ms, shader.spirv.size()); - }; - if (profile && ++profiledDispatches % 1000 == 0) { - std::string report; - for (const auto& [name, ms] : phaseTotals) report += " " + name + "=" + std::to_string(static_cast(ms / 1000)) + "s"; - std::fprintf(stderr, "[dispatch] %llu dispatches, phase totals:%s\n", static_cast(profiledDispatches), report.c_str()); - } - // A dispatch whose compiled content repeats an earlier one reuses that build's descriptor set - // when it is still valid (see ResourceCache); push constants still come from this dispatch. - static std::uint64_t cacheHits = 0, cacheMisses = 0, cacheInvalidated = 0; - static auto cacheReport = std::chrono::steady_clock::now(); + auto& d = Dispatches(); + // This call's phases (indexed by DispatchPhase), formatted only for a slow call or a new + // [indirect] maximum; the totals per phase across dispatches are reported every 1000 dispatches + // under APS5_PROFILE_DRAW. + DispatchTimer timer(profile, arguments != 0, shader.spirv.size(), programAddress); + DispatchTimer::countDispatch(profile); + const auto lookupsBefore = Graphics::DeviceProcLookups(); std::shared_ptr resources; ResourceCache::Key contentKey; const bool cacheable = ResourceCacheEnabled() && shader.variantId != 0; + // The object came from the resource cache: a compute template whose data buffers take this + // dispatch's words at the record (ShaderResources::RefreshData). + bool fromCache = false; + double revalidateMs = 0; // The batches the pre-sync (PrepareDispatch) waited for are reaped first: their completions // (CPU write-backs) land before stage B or a cached object's Revalidate reads, and the flush // hook then finds nothing pending over their surfaces. if (prepared != nullptr && prepared->presyncSerial != 0) { // Timed alone: the phase entry exists only when a reap ran, and measures just the reap. - phaseStart = std::chrono::steady_clock::now(); + timer.restart(); ReapRecorded(prepared->presyncSerial); - phase("reap"); + timer.phase(PhaseReap); } // The 'resources' phase below, split (APS5_PROFILE_DRAW) into what runs under the mutex: the // sub-phases are extra rows named "resources: ..." in the [dispatch] totals and the [indirect] // line, so the hold's biggest part (stage B's image lookups, the whole build of an address-based // shader) is named, and "resources: hook waits" says how much of the build was the nested flush // hook waiting for recorded work (a locked wait: it overlaps the other rows). - const auto subPhase = [&](const char* name, double ms) { - if (!profile) return; - phaseTotals[name] += ms; - if (arguments != 0) indirectHold.phaseMs[name] += ms; - }; const auto subPhaseStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; const auto waitedBefore = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + // The image lookups of this call by outcome (stage B, a Revalidate or a locked build), for the + // [indirect] line: reset here, read after the resources phase. + if (profile && arguments != 0) Graphics::ThreadLookupOutcomes() = {}; + // The resource cache insert evicts the oldest entry, destroying its ShaderResources here when the + // cache was the last owner: timed apart from the build. + const auto insert = [&] { + const auto insertStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + // The displaced entries go to the open batch: their destruction (descriptor sets, pooled + // buffers, texture references) then runs on the release thread once the batch completed, + // not here under the mutex. APS5_NO_DEFERRED_EVICTION=1 destroys them here as before. + static const bool deferEviction = std::getenv("APS5_NO_DEFERRED_EVICTION") == nullptr; + std::vector> evicted; + state->resourceCache.Insert(contentKey, resources, deferEviction ? &evicted : nullptr); + for (auto& object : evicted) state->recorder->Keep(std::move(object)); + if (profile) { + timer.add(PhaseResourcesInsert, std::chrono::duration(std::chrono::steady_clock::now() - insertStart).count()); + if (arguments != 0) d.indirectHold.phaseCounts[PhaseResourcesInsert] += evicted.size(); + } + }; if (prepared != nullptr && prepared->resources != nullptr) { // Stage A ran without the mutex (PrepareDispatch); stage B completes the build here. resources = std::move(prepared->resources); @@ -1591,57 +2882,86 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec // The build's own sub-phase totals before and after: the differences are stage B's parts // (an address-based build also runs its stage A here, under the mutex: 'A locked'). const auto before = profile ? resources->Timing() : Graphics::ShaderResources::BuildTiming{}; + const auto completeStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; resources->Complete(); if (profile) { + const auto completeWall = std::chrono::duration(std::chrono::steady_clock::now() - completeStart).count(); + timer.add(PhaseResourcesComplete, completeWall); const auto& after = resources->Timing(); const auto images = after.bindingsMs - before.bindingsMs; const auto upload = after.uploadMs - before.uploadMs; const auto descriptors = after.descriptorsMs - before.descriptorsMs; const auto stageA = after.prepareMs - before.prepareMs; - subPhase("resources: B images", images); - subPhase("resources: B upload", upload); - subPhase("resources: B descriptors", descriptors); - subPhase("resources: B bda+other", std::max(0.0, after.completeMs - before.completeMs - images - upload - descriptors)); - subPhase("resources: A locked (bda)", stageA); - char text[64]; - std::snprintf(text, sizeof(text), " resourcesA=%.0fms", prepared->prepareMs); - phases += text; - phaseTotals["resources A (unlocked)"] += prepared->prepareMs; + const auto stageB = after.completeMs - before.completeMs; + timer.add(PhaseResourcesImages, images); + timer.add(PhaseResourcesUpload, upload); + timer.add(PhaseResourcesDescriptors, descriptors); + timer.add(PhaseResourcesOther, std::max(0.0, stageB - images - upload - descriptors)); + timer.add(PhaseResourcesALocked, stageA); + // What Complete() spends outside both stages' totals: an address-based build's lease + // and mirror acquisition (prepareAddressBindings) before its locked stage A. + timer.add(PhaseResourcesAddress, std::max(0.0, completeWall - stageA - stageB)); + timer.callMs[PhaseResourcesAUnlocked] += prepared->prepareMs; + d.phaseTotals[PhaseResourcesAUnlocked] += prepared->prepareMs; } if (cacheable) { - ++cacheMisses; - if (resources->Reusable()) state->resourceCache.Insert(contentKey, resources); + ++d.cacheMisses; + if (resources->Reusable()) insert(); } } else if (cacheable) { - // PrepareDispatch made the key already when it found the cached object. + // PrepareDispatch made the key already when it found the cached object, and carries the + // object it found: revalidated as a map hit would be (an entry replaced meanwhile is left + // alone by the pointer-conditional Remove). APS5_NO_PREPARED_FIND=1 looks the key up again. + static const bool preparedFind = std::getenv("APS5_NO_PREPARED_FIND") == nullptr; contentKey = prepared != nullptr && !prepared->key.empty() ? std::move(prepared->key) : DispatchContentKey(shaders[0], context.device); - if (auto cached = state->resourceCache.Find(contentKey)) { + auto cached = preparedFind && prepared != nullptr && prepared->cached != nullptr ? std::move(prepared->cached) : state->resourceCache.Find(contentKey); + if (cached != nullptr) { if (cached->Revalidate(shaders[0])) { resources = std::move(cached); - ++cacheHits; + fromCache = true; + ++d.cacheHits; } else { - state->resourceCache.Remove(contentKey); - ++cacheInvalidated; + state->resourceCache.Remove(contentKey, cached.get()); + ++d.cacheInvalidated; + // Off the mutex with its batch, as an evicted entry (insert()). + state->recorder->Keep(std::move(cached)); + } + if (profile) { + revalidateMs = std::chrono::duration(std::chrono::steady_clock::now() - subPhaseStart).count(); + timer.add(PhaseResourcesRevalidate, revalidateMs); } - if (profile) subPhase("resources: revalidate", std::chrono::duration(std::chrono::steady_clock::now() - subPhaseStart).count()); } } if (resources == nullptr) { const auto buildStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; resources = std::make_shared(context, shaders[0], snapshots); - if (profile) subPhase("resources: full build (locked)", std::chrono::duration(std::chrono::steady_clock::now() - buildStart).count()); + if (profile) timer.add(PhaseResourcesFullBuild, std::chrono::duration(std::chrono::steady_clock::now() - buildStart).count()); if (cacheable) { - ++cacheMisses; - if (resources->Reusable()) state->resourceCache.Insert(contentKey, resources); + ++d.cacheMisses; + if (resources->Reusable()) insert(); } } - if (profile) subPhase("resources: hook waits", Graphics::Recorder::ThreadWaitedMs() - waitedBefore); - if (profile && std::chrono::steady_clock::now() - cacheReport > std::chrono::seconds(10)) { - cacheReport = std::chrono::steady_clock::now(); - std::fprintf(stderr, "[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw)\n", static_cast(cacheHits), static_cast(cacheMisses), static_cast(cacheInvalidated), state->resourceCache.Size()); + if (profile) timer.add(PhaseResourcesHookWaits, Graphics::Recorder::ThreadWaitedMs() - waitedBefore); + // The lookup outcomes of this call: rows with counts on the [indirect] line, and a summary for + // the longest call. + if (profile && arguments != 0) { + const auto& outcomes = Graphics::ThreadLookupOutcomes(); + for (std::size_t kind = 0; kind < Graphics::LookupOutcomes::Count; ++kind) { + if (outcomes.counts[kind] == 0) continue; + const auto row = DispatchPhaseCount + kind; + timer.callMs[row] += outcomes.ms[kind]; + d.indirectHold.phaseMs[row] += outcomes.ms[kind]; + d.indirectHold.phaseCounts[row] += outcomes.counts[kind]; + } + } + if (profile && std::chrono::steady_clock::now() - d.cacheReport > std::chrono::seconds(10)) { + d.cacheReport = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast(d.cacheHits), static_cast(d.cacheMisses), static_cast(d.cacheInvalidated), state->resourceCache.Size(), static_cast(d.templateRefreshed), d.templateRevalidateMs, static_cast(d.templateSameWords), static_cast(ResourceCache::Finds()), static_cast(ResourceCache::Touches())); + std::fprintf(stderr, "[vk] deviceProc lookups inside the device call: %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu\n", static_cast(d.procLookups), d.profiledDispatches != 0 ? static_cast(d.procLookups) / static_cast(d.profiledDispatches) : 0.0, static_cast(d.preBarriersRecorded), static_cast(d.preBarriersSkipped)); + reportRecipes(); } timing.Mark("shader_resources"); - phase("resources"); + timer.phase(PhaseResources); if (objects == nullptr) { objects = std::make_shared(); objects->device = state->device; @@ -1667,117 +2987,38 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec pipelineInfo.stage.module = objects->module; pipelineInfo.stage.pName = "main"; pipelineInfo.layout = objects->layout; - if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words\n", shader.spirv.size()); + if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words (program 0x%llx)\n", shader.spirv.size(), static_cast(programAddress)); check(state->DeviceFunction("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &objects->pipeline), "vkCreateComputePipelines"); timing.Mark("pipeline_create"); - if (pipelineKey != 0) state->computePipelines[pipelineKey] = objects; - phase("pipeline"); + if (pipelineKey != 0) { + // Find-or-insert: objects another dispatch of the variant mapped meanwhile serve this + // one too (this call's go with the batch that keeps them). + std::lock_guard pipelines(state->computePipelinesMutex); + objects = state->computePipelines.try_emplace(pipelineKey, objects).first->second; + } + timer.phase(PhasePipeline); } // Recorded into the device's open batch: the CPU moves on to the next command while the GPU // works; the batch is waited for where the guest expects results (labels, flips, CPU reads). auto& recorder = *state->recorder; IndirectOutcome outcome{0, 0}; - const Graphics::HostImport* argumentImport = nullptr; - if (arguments != 0) { - // Decided here, after the resource build (which may retire imports and wait for recorded - // work) and before anything of this dispatch is recorded, so the CPU fallback's sync cannot - // split the dispatch across two batches and the import cannot be dropped before the record. - // A label over the argument dwords (a completion-deferred one lands by a CPU store the GPU - // read would miss; a GPU one is ordered but rare enough to share the fallback). Only those - // dwords: a global "any completion label pending" test would send most indirect dispatches - // to the CPU, as ~5 such labels are pending per frame. Stamp 0 matches every live entry. - const auto labelPending = [&] { - std::uint32_t labelQueue = 0; - for (std::uint64_t dword = arguments; dword < arguments + 12; dword += 4) { - if (recorder.PendingLabel(dword, 4, 0, labelQueue).has_value()) return true; - } - return false; - }; - if (Graphics::StorageTexture::FlushPending(arguments, 12, nullptr, "indirect dispatch arguments")) { - // As the flush hook does: the stores were only recorded and the CPU is about to read them. - outcome.cpuReason = 1; - Graphics::Recorder::CountSync(2); - recorder.Sync(); - } else if (labelPending() || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); }) - || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); })) { - outcome.cpuReason = 2; - } else if ((argumentImport = Graphics::HostImportFor(context, arguments, 12)) == nullptr) { - // Valid until the next refreshImports, which only runs under GuestMemory::GpuMutex (held - // here, by every Dispatch caller) and Keeps a retired VkBuffer while the recorder is busy. - outcome.cpuReason = 3; - } - if (profile) { - indirectHold.writersScanned += state->copiedWriters->size(); - indirectHold.drawWritersScanned += Graphics::DrawCopiedWriters()->size(); - indirectHold.maxWriters = std::max(indirectHold.maxWriters, state->copiedWriters->size()); - } - phase("decide"); - if (outcome.cpuReason != 0) { - // The read waits for the producing batch through the flush hook, as the driver's resolve - // did for every indirect dispatch before; the counts then go through the direct checks. - const auto readStart = std::chrono::steady_clock::now(); - std::array groups{}; - GuestMemory::Read(arguments, std::as_writable_bytes(std::span(groups)), 4); - outcome.argumentReadMs = std::chrono::duration(std::chrono::steady_clock::now() - readStart).count(); - x = groups[0]; - y = groups[1]; - z = groups[2]; - // Timed before `arguments` is cleared, so the read counts as an indirect phase. - phase("argument read"); - arguments = 0; - std::snprintf(groupsText, sizeof(groupsText), "%ux%ux%u", x, y, z); - if (x > limit[0] || y > limit[1] || z > limit[2]) throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits"); - } - } - // From here on the phases of a CPU-resolved indirect dispatch are charged as direct ones - // (`arguments` is 0); the indirect hold total below still covers the whole call. - const bool indirect = arguments != 0 || outcome.cpuReason != 0; - const auto commands = recorder.Commands(); - recorder.Keep(objects); - recorder.Keep(resources); - if (argumentImport != nullptr) { - // The group counts were stored by earlier recorded work (a dispatch in place, a fill) or the - // host; the indirect read follows all of it. - Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT); - } - // Results of earlier recorded work are visible to this dispatch, its own to everything after. - Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - state->DeviceFunction("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, objects->pipeline); - resources->Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, objects->layout); - if (pushStages != 0) { - state->DeviceFunction("vkCmdPushConstants")(commands, objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes, pushBytes.data()); - } - const auto gpuTiming = recorder.BeginGpuTiming(programAddress != 0 ? programAddress : shader.variantId); - if (argumentImport != nullptr) state->DeviceFunction("vkCmdDispatchIndirect")(commands, argumentImport->buffer, arguments - argumentImport->base); - else state->DeviceFunction("vkCmdDispatch")(commands, x, y, z); - Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT); - recorder.EndGpuTiming(gpuTiming); - resources->MarkGpuWrites(recorder); - // Only copied written buffers (and BDA fault checks) need work once the GPU is done; without them - // the batch can signal its labels from the GPU. - if (resources->NeedsCompletion()) { - // Listed for DispatchIndirect until the write-back ran (a non-reusable object, so once). - auto writers = state->copiedWriters; - recorder.OnComplete([resources, writers] { - // Delisted before the write-back: one that fails must not keep indirect dispatches on the CPU. - writers->erase(std::remove(writers->begin(), writers->end(), resources), writers->end()); - resources->WriteBackBuffers(); - }); - // Listed after the registration: a throw there leaves nothing that would pin the CPU path forever. - writers->push_back(resources); - } + RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer}; + if (arguments != 0) decideIndirect(record, outcome, groupsText); + // The indirect hold total below still covers the whole call of a CPU-resolved one. + const bool indirect = record.arguments != 0 || outcome.cpuReason != 0; + recordDispatch(record); timing.Mark("command_record"); - phase("record"); + timer.phase(PhaseRecord); APS5_LOG_OUT_DEBUG("Dispatch recorded groups=%s", groupsText); // Address-based shaders pin guest allocations until their write-back, which the completion runs // when the batch finished (deferred lease release, see Graphics::SyncLeaseWork): a guest thread // that needs a leased allocation syncs the recorder itself through the registry's pin waiter. // APS5_SYNC_LEASE_DISPATCH=1 completes such dispatches at once, as before. - if (syncEachDispatch || (resources->HoldsLease() && Graphics::SyncLeaseWork())) { + if (SyncEachDispatch() || (resources->HoldsLease() && Graphics::SyncLeaseWork())) { Graphics::Recorder::CountSync(3); const auto syncStart = std::chrono::steady_clock::now(); recorder.Sync(); - phase("sync"); + timer.phase(PhaseSync); if (resources->HoldsLease()) Graphics::CountLeaseOutcome(true, 0); if (profile) { // APS5_PROFILE_DRAW: the [recorder] line's "address-based" syncs by program, every 10 s, @@ -1809,38 +3050,146 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec // finishes the recorder up to that batch's serial. Prints the [address-sync] leases line. Graphics::CountLeaseOutcome(false, recorder.Submissions() + 1); } - // Debug aid: APS5_TRACE_DISPATCH_IO prints every dispatch's resources (after write-back when synced). - static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr; - if (traceIo) std::fprintf(stderr, "[dispatch-io] %s:%s\n", groupsText, resources->Describe().c_str()); - if (profile) { - const auto now = std::chrono::steady_clock::now(); - const auto totalMs = std::chrono::duration(now - dispatchStart).count(); - if (totalMs > 100) std::fprintf(stderr, "[dispatch] %s %zu words:%s\n", groupsText, shader.spirv.size(), phases.c_str()); - if (indirect) { - auto& hold = indirectHold; - ++hold.count; - hold.totalMs += totalMs; - hold.maxMs = std::max(hold.maxMs, totalMs); - if (now - hold.lastReport > std::chrono::seconds(10)) { - hold.lastReport = now; - std::string report; - for (const auto& [name, ms] : hold.phaseMs) { - char text[80]; - std::snprintf(text, sizeof(text), " %s %.0f ms", name.c_str(), ms); - report += text; - } - std::fprintf(stderr, "[indirect] %llu indirect dispatches spent %.0f ms inside the device call (10 s; max %.1f ms), by phase (the 'resources: ...' rows split 'resources'; 'hook waits' overlaps them):%s; copied-writer lists scanned per decision: dispatch avg %.1f (max %zu), draw avg %.1f\n", static_cast(hold.count), hold.totalMs, hold.maxMs, report.c_str(), hold.count != 0 ? static_cast(hold.writersScanned) / hold.count : 0.0, hold.maxWriters, hold.count != 0 ? static_cast(hold.drawWritersScanned) / hold.count : 0.0); - hold.phaseMs.clear(); - hold.count = 0; - hold.totalMs = 0; - hold.maxMs = 0; - hold.writersScanned = 0; - hold.drawWritersScanned = 0; - hold.maxWriters = 0; - } + if (TraceDispatchIo()) std::fprintf(stderr, "[dispatch-io] %s:%s\n", groupsText, resources->Describe().c_str()); + // The recipe for the caller's dispatch-cache variant (design_cpu_final M4, rule R3): only an + // object the resource cache serves under this content key (reusable: no lease, no copied + // writes, every direct region import- or mirror-served), so a hit's proof is the template's + // Revalidate and nothing needs completion work. + if (recipeOut != nullptr && DispatchRecipes() && cacheable && resources->Reusable() && objects != nullptr && !contentKey.empty()) { + auto recipe = std::make_shared(); + recipe->device = state->device; + recipe->templateRef = resources; + recipe->key = contentKey; + recipe->objects = objects; + recipe->pushBytes = pushBytes; + recipe->pushes = pushStages != 0; + recipe->dataWordsHash = Graphics::ShaderResources::DataWordsHash(shaders[0]); + if (!NoPresync() && CanWaitUnlocked()) { + recipe->presyncSurfaces = resources->PresyncSurfaces(); + recipe->presyncProof = Graphics::HostImportsIdentity(context); } + recipe->needsCompletion = resources->NeedsCompletion(); + recipe->holdsLease = resources->HoldsLease(); + *recipeOut = std::move(recipe); } + timer.finish(lookupsBefore, groupsText, indirect); return outcome; } +RecipeOutcome VulkanDevice::DispatchRecipe(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, const std::shared_ptr& hit, IndirectOutcome& outcome, const std::shared_ptr& verify, bool refreshByWords) { + PerformanceTimer timing("Vulkan.DispatchRecipe"); + outcome = {0, 0}; + char groupsText[40]; + if (arguments != 0) std::snprintf(groupsText, sizeof(groupsText), "indirect"); + else std::snprintf(groupsText, sizeof(groupsText), "%ux%ux%u", x, y, z); + const auto& recipe = *hit->recipe; + auto& counters = Recipes().kinds[hit->indirect ? 1 : 0]; + auto& d = Dispatches(); + const std::array shaders{{{ShaderRecompiler::ShaderStage::Compute, &shader, 0}}}; + const auto context = graphicsContext(); + const auto* limit = state->properties.limits.maxComputeWorkGroupCount; + if (arguments == 0 && (x > limit[0] || y > limit[1] || z > limit[2])) { + throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits"); + } + if (recipe.needsCompletion || recipe.holdsLease || !hit->resources->Reusable()) throw std::runtime_error("Vulkan dispatch: recipe over a non-reusable template"); + // Without the refresh the template holds the words it was keyed with: a data-only hit takes + // the ordinary path, whose word-keyed lookup serves or builds the object for the live words. + if (refreshByWords && !TemplateDataRefresh()) return RecipeOutcome::Rebuild; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + DispatchTimer timer(profile, arguments != 0, shader.spirv.size(), programAddress); + DispatchTimer::countDispatch(profile); + const auto lookupsBefore = Graphics::DeviceProcLookups(); + // R10: the recipe names the device it was built on (the pre-check read it too, but the driver + // may replace the device between the two). + if (recipe.device != state->device) { + counters.rebuildDevice.fetch_add(1, std::memory_order_relaxed); + return RecipeOutcome::Rebuild; + } + if (hit->presyncSerial != 0) { + timer.restart(); + ReapRecorded(hit->presyncSerial); + timer.phase(PhaseReap); + } + if (verify != nullptr) { + // APS5_VERIFY_RECIPE=1: the ordinary path's answers beside the recipe's. The found object + // is the cache's under the content key (PrepareDispatch's Find), the pipeline objects the + // map's, and the data refresh decision the per-word compare RefreshData makes; the proof + // itself is the same Revalidate on the same object, so it needs no second run. + // A found object that is not the recipe's template is no disagreement: the cache no longer + // serves the template under its key (removed by a sibling variant's failed proof, evicted, + // replaced by another worker's Insert) while the batch keeps it alive, so the ordinary path + // would use a content-equal object and the comparison has no counterpart. Counted, and the + // ordinary path runs with the prepared find (C10: the template goes to the batch). + if (verify->cached.get() != hit->resources.get()) { + counters.verifyReplaced.fetch_add(1, std::memory_order_relaxed); + state->recorder->Keep(std::move(hit->resources)); + return RecipeOutcome::Rebuild; + } + std::shared_ptr mapped; + const auto pipelineKey = (shader.variantId << 1u) | (recipe.pushes ? 1u : 0u); + { + std::lock_guard pipelines(state->computePipelinesMutex); + if (const auto found = state->computePipelines.find(pipelineKey); found != state->computePipelines.end()) mapped = found->second; + } + // A data-only hit's words are the shader's, not the recipe's: its hash decision is + // computed from the shader, as the record below decides by words. + const bool byHash = TemplateDataRefresh() && hit->resources->DataWordsHash() != (refreshByWords ? Graphics::ShaderResources::DataWordsHash(shaders[0]) : recipe.dataWordsHash); + const bool byWords = TemplateDataRefresh() && hit->resources->DataWordsDiffer(shaders[0]); + const char* disagreement = mapped != hit->objects ? "the pipeline objects" : byHash != byWords ? "the RefreshData decision" : nullptr; + if (disagreement != nullptr) { + std::fprintf(stderr, "[recipe] APS5_VERIFY_RECIPE: %s of the recipe disagrees with the ordinary path (program 0x%llx)\n", disagreement, static_cast(programAddress)); + std::fflush(stderr); + std::abort(); + } + counters.verified.fetch_add(1, std::memory_order_relaxed); + } + // The proof (rules R6/R7): the template's Revalidate, T1 included; a failure removes the + // template from the cache (the batch keeps it) and the caller rebuilds. + const auto waitedBefore = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + const auto proofStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + Graphics::ShaderResources::ProofReport report; + const bool proved = hit->resources->ProveCurrent(shaders[0], &report); + if (profile) { + counters.proofNs.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - proofStart).count()), std::memory_order_relaxed); + timer.add(PhaseResourcesHookWaits, Graphics::Recorder::ThreadWaitedMs() - waitedBefore); + } + timer.phase(PhaseProof); + if (!proved) { + state->resourceCache.Remove(recipe.key, hit->resources.get()); + ++d.cacheInvalidated; + // Off the mutex with its batch, as an evicted entry. + state->recorder->Keep(std::move(hit->resources)); + counters.rebuilds[static_cast(report.failure)].fetch_add(1, std::memory_order_relaxed); + return RecipeOutcome::Rebuild; + } + counters.proofPaths[static_cast(report.path)].fetch_add(1, std::memory_order_relaxed); + ++d.cacheHits; + state->resourceCache.Touch(recipe.key); + // The recipe's hash names the words it was attached with, not a data-only hit's: that hit + // refreshes by the per-word compare (RefreshData keeps the template's own hash exact, so a + // later exact hit of a sibling variant compares correctly against it). + const auto dataRefresh = !TemplateDataRefresh() ? RecordedDispatch::DataRefresh::None : refreshByWords ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::Hash; + if (dataRefresh == RecordedDispatch::DataRefresh::Words) counters.refreshByWords.fetch_add(1, std::memory_order_relaxed); + RecordedDispatch record{&context, &shaders[0], hit->resources, hit->objects, recipe.pushes ? VkShaderStageFlags{VK_SHADER_STAGE_COMPUTE_BIT} : VkShaderStageFlags{0}, &recipe.pushBytes, x, y, z, arguments, nullptr, programAddress, dataRefresh, recipe.dataWordsHash, 0, &timer}; + if (arguments != 0) decideIndirect(record, outcome, groupsText); + const bool indirect = record.arguments != 0 || outcome.cpuReason != 0; + const auto recordStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + recordDispatch(record); + if (profile) counters.recordNs.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - recordStart).count()), std::memory_order_relaxed); + timing.Mark("command_record"); + timer.phase(PhaseRecord); + counters.hits.fetch_add(1, std::memory_order_relaxed); + if (record.dataRefresh != RecordedDispatch::DataRefresh::None) (record.refreshed ? counters.dataRefreshed : counters.dataSkipped).fetch_add(1, std::memory_order_relaxed); + APS5_LOG_OUT_DEBUG("Dispatch recorded from recipe groups=%s", groupsText); + if (SyncEachDispatch()) { + Graphics::Recorder::CountSync(3); + state->recorder->Sync(); + timer.phase(PhaseSync); + } + if (TraceDispatchIo()) std::fprintf(stderr, "[dispatch-io] %s:%s\n", groupsText, hit->resources->Describe().c_str()); + timer.finish(lookupsBefore, groupsText, indirect); + if (profile) reportRecipes(); + return RecipeOutcome::Recorded; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp index 6e525bf902..32028ea798 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp @@ -18,6 +18,19 @@ public: std::uint64_t hits = 0; std::uint64_t misses = 0; std::size_t held = 0; + // Hits served by the cached address space's serial alone (no hash, no compare; see + // GuestBufferMemory::CachedAddressTable), counted in `hits` too. + std::uint64_t spaceTables = 0; + // Why the most recently used table did not serve a build ([bda-table] first-differing entry + // class): its owners were gone (expired weak reference), or its word hash differed, split by + // whether the first differing range begins below 0x10000000 (a low mapping: the exe image, + // the system heaps) or in the guest heap above; `sameHash` is a hash match whose ranges + // still differed, `empty` a lookup with no entry held. + std::uint64_t firstExpired = 0; + std::uint64_t firstDiffersLow = 0; + std::uint64_t firstDiffersHeap = 0; + std::uint64_t firstSameHash = 0; + std::uint64_t firstEmpty = 0; }; static TableCacheStats TableCacheCounters(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index efc1bb176f..73639a91a5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -39,6 +39,11 @@ struct BufferAllocation { // the 256-byte copied-region buffers to fit, and those then missed on every build (81869 misses // and 80850 evictions per run, each a Vulkan create or destroy under the pool mutex, 3.5 s per // run). APS5_BUFFER_POOL_SHARED=1 keeps one tier as before. +// +// Device-local allocations (the detiler's scratch buffers and the staging shadows of written guest +// buffers, see GuestBufferMemory) are retained in a third tier with a budget of their own, video +// memory instead of pinned host memory: APS5_STAGING_POOL_MIB (default 512); 0 keeps them in the +// two host tiers as before. class BufferPool { public: explicit BufferPool(const Context& context); @@ -65,8 +70,11 @@ private: std::uint64_t misses = 0; std::uint64_t evictions = 0; }; - // The tier a buffer of `capacity` is retained in (the large one for everything when shared). - Tier& tierFor(std::size_t capacity); + // The tier a buffer of `capacity` and `properties` is retained in (the large one for everything + // when shared; the device tier for device-local memory while it has a budget). + Tier& tierFor(std::size_t capacity, VkMemoryPropertyFlags properties); + // The device tier's budget (APS5_STAGING_POOL_MIB), read once. + static VkDeviceSize DeviceBudget(); void destroy(const BufferAllocation& allocation) noexcept; // Moves the tier's least recently used slot to `evicted`; the caller destroys those after // releasing the mutex, so builds taking buffers on other threads do not wait behind the @@ -82,6 +90,7 @@ private: // Not `small`/`large`: (via ) defines `small` as a macro. Tier smallTier; Tier largeTier; + Tier deviceTier; std::uint64_t clock = 0; static constexpr VkDeviceSize budget = 512ull * 1024 * 1024; // The small tier's own budget (512 slots of at most half a MiB each): pinned host memory the diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index a1c7ab64d0..db6ae2af55 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace AgcDriver::Graphics { @@ -22,6 +23,7 @@ class GraphicsPipelineCache; class Recorder; class DescriptorCache; class SamplerCache; +class ShaderResources; inline void Require(bool condition, const std::string& reason) { if (!condition) throw std::runtime_error("AGC graphics: " + reason); @@ -35,6 +37,43 @@ inline void Require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("AGC graphics: ") + reason); } +// Device entry points resolved once per device (VulkanDevice's State fills it after setup): the +// loader's vkGetDeviceProcAddr is a name lookup under its global mutex per call, paid at every +// record site otherwise. Null members (tests, APS5_NO_PROC_TABLE=1) resolve per call (Resolved). +struct DeviceFunctions { + PFN_vkCmdPipelineBarrier cmdPipelineBarrier = nullptr; + PFN_vkCmdCopyBuffer cmdCopyBuffer = nullptr; + PFN_vkCmdUpdateBuffer cmdUpdateBuffer = nullptr; + PFN_vkCmdFillBuffer cmdFillBuffer = nullptr; + PFN_vkCmdBindPipeline cmdBindPipeline = nullptr; + PFN_vkCmdBindDescriptorSets cmdBindDescriptorSets = nullptr; + PFN_vkCmdPushConstants cmdPushConstants = nullptr; + PFN_vkCmdDispatch cmdDispatch = nullptr; + PFN_vkCmdDispatchIndirect cmdDispatchIndirect = nullptr; + PFN_vkCmdBeginRenderPass cmdBeginRenderPass = nullptr; + PFN_vkCmdEndRenderPass cmdEndRenderPass = nullptr; + PFN_vkCmdSetViewport cmdSetViewport = nullptr; + PFN_vkCmdSetScissor cmdSetScissor = nullptr; + PFN_vkCmdBindVertexBuffers cmdBindVertexBuffers = nullptr; + PFN_vkCmdBindIndexBuffer cmdBindIndexBuffer = nullptr; + PFN_vkCmdDraw cmdDraw = nullptr; + PFN_vkCmdDrawIndexed cmdDrawIndexed = nullptr; + PFN_vkCmdDrawIndirect cmdDrawIndirect = nullptr; + PFN_vkCmdDrawIndexedIndirect cmdDrawIndexedIndirect = nullptr; + PFN_vkCmdCopyBufferToImage cmdCopyBufferToImage = nullptr; + PFN_vkCmdCopyImageToBuffer cmdCopyImageToBuffer = nullptr; + PFN_vkCmdClearColorImage cmdClearColorImage = nullptr; + PFN_vkUpdateDescriptorSets updateDescriptorSets = nullptr; + PFN_vkAllocateDescriptorSets allocateDescriptorSets = nullptr; + PFN_vkGetFenceStatus getFenceStatus = nullptr; +}; + +// Per-thread count of vkGetDeviceProcAddr lookups made through Context::Function (the [vk] line). +inline std::uint64_t& DeviceProcLookups() { + thread_local std::uint64_t count = 0; + return count; +} + struct Context { VkDevice device; VkPhysicalDevice physical; @@ -72,15 +111,37 @@ struct Context { // layouts, descriptor pools, samplers); null (tests) means every build makes and destroys its own. DescriptorCache* descriptorCache = nullptr; SamplerCache* samplerCache = nullptr; + // Indirect draw features of the device: records with a non-zero first instance, several records + // per call, and a GPU-side draw count (VK_KHR_draw_indirect_count). + bool drawIndirectFirstInstance = false; + bool multiDrawIndirect = false; + bool drawIndirectCount = false; + // The device's list of recorded dispatches whose copied written buffers await a CPU write-back + // (VulkanDevice's State::copiedWriters; the draw counterpart is DrawCopiedWriters): an indirect + // draw whose records one of them produces reads them on the CPU. Null in tests. + const std::vector>* copiedWriters = nullptr; + // The device's resolved entry points (see DeviceFunctions); null until the device set up. + const DeviceFunctions* functions = nullptr; + // VK_EXT_descriptor_indexing with non-uniform sampled/storage image array indexing enabled + // (bindless image tables in graphics stages). + bool descriptorIndexing = false; template TFunction Function(const char* name) const { Require(deviceProc != nullptr, "missing Vulkan device function resolver"); + ++DeviceProcLookups(); const auto function = reinterpret_cast(deviceProc(device, name)); if (function == nullptr) throw std::runtime_error(std::string("AGC graphics: missing Vulkan function: ") + name); return function; } + // The table's entry point, or a per-call lookup while the table is absent or lacks it. + template + TFunction Resolved(TFunction DeviceFunctions::*member, const char* name) const { + if (functions != nullptr && functions->*member != nullptr) return functions->*member; + return Function(name); + } + std::uint32_t MemoryType(std::uint32_t mask, VkMemoryPropertyFlags flags) const { for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) { if ((mask & (1u << i)) != 0 && (memory.memoryTypes[i].propertyFlags & flags) == flags) return i; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp index 80961435d5..ccd64b1431 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp @@ -45,6 +45,30 @@ bool DccAlphaOnMsb(VkFormat format, std::uint32_t componentSwap); // The keys of a texture's DCC surface when they fast-clear it to a value encodable here, else // Uncompressed (the texels are read as stored; other keys are reported once). DccKeys TextureClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes); +// The last stable key scan of one surface's metadata (ProvedClearKeys): the keys read and the write +// generation the key range was collected at before the scan. Held by the image or texture of the +// surface, read and written under GuestMemory::GpuMutex only. +struct DccKeyProof { + DccKeys keys = DccKeys::Uncompressed; + std::uint64_t generation = 0; +}; +// TextureClearKeys with the scan skipped while the key range is unchanged since `proof` was taken +// (CollectWrites over the keys, then UnchangedSince its generation), so a surface's keys are scanned +// once per change instead of on every use. A scan becomes the proof only when it read the bytes +// (not the pending-store memo) and no recorded work still writes them: a recorded key store stamps +// the range at its note, before it lands, so a proof taken meanwhile would outlive the change. Every +// other key writer stamps the range too (CPU writes through the page write watch, GPU writes at +// their record), which makes the proof exactly as sound as UnchangedSince over the texels. +// APS5_NO_KEY_FAST_PATH=1 scans on every call and stores nothing. +DccKeys ProvedClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes, DccKeyProof& proof); +bool KeyFastPath(); +// Cumulative outcomes of ProvedClearKeys: calls answered by a proof, scans made, and scans not kept. +struct DccKeyProofCounts { + std::uint64_t proved; + std::uint64_t scanned; + std::uint64_t unstable; +}; +DccKeyProofCounts KeyProofCounts(); // A surface's texels as a read sees them: the guest bytes, or the clear value of fast-cleared keys. void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp index e3016a2aec..63724bec11 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp @@ -3,12 +3,57 @@ #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include "prx/libSceAgcDriver/Execution/include/Recipe.hpp" #include #include namespace AgcDriver::Graphics { -void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); +// `recipe`, when given, receives the DrawRecipe a recorded, cacheable, reusable, non-indirect draw +// built for its draw-cache entry (design_cpu_final M8; null otherwise, and always under +// APS5_NO_DRAW_RECIPE=1). +void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}, std::shared_ptr* recipe = nullptr); + +// Why DrawWithRecipe did not record from the recipe (the caller then runs Draw): the draw is not +// recordable (no recorder, APS5_SYNC_DRAWS, APS5_DUMP_TARGETS), a resident target is gone from the +// storage cache, the template is gone, the pipeline is gone from the pipeline store, or the +// template's proof failed (ShaderResources::ProveCurrent; the template is removed from the +// resource cache and kept by the batch). +enum class DrawRecipeMiss : std::uint8_t { None, NotRecordable, TargetGone, TemplateGone, ObjectsGone, Proof, Count }; +const char* DrawRecipeMissName(DrawRecipeMiss miss); +struct DrawRecipeOutcome { + bool recorded = false; + DrawRecipeMiss miss = DrawRecipeMiss::None; + ShaderResources::ProofReport proof; + // APS5_PROFILE_DRAW: the proof's and the record's time. + double proofUs = 0; + double recordUs = 0; +}; +// A draw-cache hit recorded from its recipe, under GuestMemory::GpuMutex after the packet's labels +// (design_cpu_final M8, step 8b): the validation with the recipe's stored outputs, the index and +// vertex work as Draw's, the resident-target proof on the stored images (Refresh while Cached()), +// the template's proof (ProveCurrent) with the alias checks repeated, the pipeline and framebuffer +// from the recipe, then the record as Draw's (pass continued or begun, Kept, MarkGpuWrites). A +// miss records nothing. +DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, const DrawRecipe& recipe); +// Whether recorded draws build recipes and draw-cache hits use them (APS5_NO_DRAW_RECIPE unset). +bool DrawRecipes(); + +// How an indirect draw's records were read, for the [draws] line (APS5_PROFILE_DRAW): Gpu is +// vkCmdDrawIndirect from the host import; every other value names why the CPU read them instead, +// either decided by the driver before the draw (its patched-SGPR fold rules, the shader path, the +// draw index, the vertex ranges, the device features, APS5_NO_GPU_INDIRECT_DRAW) or under the device +// lock in Draw (the memory state of the records). `readMs` is the CPU read's time (its sync); +// `rewritten` counts a GPU-side draw whose records were copied and patched with a constant. +enum class IndirectDrawPath : std::uint8_t { Gpu, NotFolded, FetchUnknown, NonVertexPath, IndxOffset, DrawIndex, VertexRange, FeatureGap, PendingImage, PendingLabelOrCopy, NotImported, Disabled, Count }; +void CountIndirectDraw(IndirectDrawPath path, double readMs, bool rewritten = false); +const char* IndirectDrawPathName(IndirectDrawPath path); + +// Draw packets that drew nothing, for the [draws] line: Nothing (an empty count, or no color writes +// and no pixel shader), Prechecked (State.hpp's DrawRejection refused it before the decode) and +// Thrown (the decode or the build threw); `us` is the packet's time in the driver. +enum class DrawSkip : std::uint8_t { Nothing, Prechecked, Thrown, Count }; +void CountDrawSkip(DrawSkip kind, double us); // Recorded draws whose written guest buffers were copied (their results reach guest memory by a CPU // write-back when their batch completes), listed until that write-back ran. It is the draw diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 06484200d7..bfe9cc0e04 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -36,6 +36,9 @@ const HostImport* HostImportFor(const Context& context, std::uint64_t address, s // made outside the device lock (a sampled texture's path, a dispatch's pre-sync); the path taken // re-checks with HostImportFor when it binds the import. bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes); +// Whether a readable registered allocation contains [address, address + bytes) right now (one +// registry lease): a storage image whose memory was freed or re-registered has nothing to store to. +bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes); // A persistent device copy of one registered range of the main guest image (which cannot be host // imported); see GuestBufferMemory.cpp. @@ -50,6 +53,19 @@ class Recorder; // work using the import is recorded) the imports are left as they are. std::uint64_t HostImportSerial(const Context& context, std::uint64_t address, std::size_t bytes, bool reconcile); +// The import table's identity (the fast Revalidate): serials proved under one identity stand +// while the table still has it, i.e. the same device, the same epoch (bumped by every retire) +// and the registry generation it was reconciled with, which must still be the live one. +struct HostImportsProof { + VkDevice device = VK_NULL_HANDLE; + std::uint64_t epoch = 0; + std::uint64_t refreshedGeneration = 0; +}; +// Whether the table still matches `proof` (one lock); false for an empty or stale proof. +bool HostImportsUnchanged(const Context& context, const HostImportsProof& proof); +// The table's identity when it is reconciled with the live registry (one lock), else empty. +HostImportsProof HostImportsIdentity(const Context& context); + // Identity of the read-only image mirror serving [address, address + bytes) (the counterpart of // HostImportSerial for main-image ranges, which no import serves): a serial unique for the life of // one mirror with its top bit set, 0 when no read-only mirror whose registered range still exists @@ -74,25 +90,66 @@ struct LeaseStats { std::uint64_t synced = 0; // Pin-contention waits made by guest threads (the waiter calls), how many of them finished the // newest lease batch, how many had to drain the whole recorder instead (no lease batch noted), - // and their total time. + // how many only dropped the cached address space (no GPU wait, see AddressSpaceStats), and + // their total time. std::uint64_t contentionWaits = 0; std::uint64_t contentionSyncs = 0; std::uint64_t contentionDrains = 0; + std::uint64_t cacheDrops = 0; double contentionMs = 0; }; LeaseStats LeaseCounters(); +// The cached address space of address-based builds (see GuestBufferMemory.cpp): builds served by +// the cached space, rebuilds by reason, builds whose space could not be published (an import +// retired while it was built) and the pin waiter's drops (cumulative). `enabled` is false under +// APS5_NO_ADDRESS_SPACE_CACHE=1, when every build takes the per-build path. +struct AddressSpaceStats { + bool enabled = true; + std::uint64_t hits = 0; + std::uint64_t rebuiltFirst = 0; + std::uint64_t rebuiltGeneration = 0; + std::uint64_t rebuiltEpoch = 0; + std::uint64_t rebuiltDevice = 0; + std::uint64_t rebuiltWaiterDrop = 0; + std::uint64_t unpublished = 0; + // Builds that left the space for the per-build path: a region partially overlapping one of + // its regions, or an import retired under it before the upload. + std::uint64_t dissolvedOverlap = 0; + std::uint64_t dissolvedImports = 0; + std::uint64_t waiterDrops = 0; +}; +AddressSpaceStats AddressSpaceCounters(); + class GuestBufferMemory { public: explicit GuestBufferMemory(const Context& context); + ~GuestBufferMemory(); void AcquireRegistered(); + // APS5_PROFILE_DRAW: the parts of one address-based build ([address] line every 10 s): the + // registry lease, the imports pass, the mirror preparation and the mirror compare (blocks and + // copied blocks) are timed by AcquireRegistered on the calling thread; the caller adds the + // snapshot compares it made after it (ShaderResources::prepareAddressBindings). + static void CountAddressBuild(double snapshotsUs); // A guest range bound through a descriptor. Both read live guest memory at upload and bind the // same way (a storage buffer); AddWritable also notes the range in Writes(), so it gets the // write-back's reference copy, a write-back, the recorder's pending-write note and the // direct-write marks. AddReadable is for an element the shader is proved never to store to - // (DescriptorBinding::bufferWritten): the CPU never has to wait for it. - void AddWritable(std::uint64_t address, std::size_t bytes); + // (DescriptorBinding::bufferWritten): the CPU never has to wait for it. `atomic` marks an + // element the shader updates atomically (DescriptorBinding::bufferAtomic), staged in device + // memory whatever its size (see AllowDeviceStaging). + void AddWritable(std::uint64_t address, std::size_t bytes, bool atomic = false); void AddReadable(std::uint64_t address, std::size_t bytes); + // Device-local staging of written and atomic elements inside host imports (see + // GuestBufferMemory.cpp): allowed only for a build whose every use records its work and then + // calls RecordCopyBacks (a dispatch), since a staged region's results reach guest memory by + // that copy alone. Call before Upload. + void AllowDeviceStaging() { stagingAllowed = true; } + // Records the copy-in of every staged region anew for another use of this upload (a resource + // cache hit, from ShaderResources::Revalidate, under GuestMemory::GpuMutex, once the imports + // were confirmed unchanged): the previous use's copy-back left the shadow behind, and the next + // RecordCopyBacks copies it back again. Nothing to do without staged regions. + void RecordStagingCopies(Recorder& recorder); void AddSnapshot(const GuestMemorySnapshot& snapshot); // Upload is the two stages below back to back. UploadPrepare needs no device lock: it merges the // regions, binds the image mirrors and host imports that already serve them (an import pointer is @@ -106,6 +163,19 @@ public: void UploadFinish(bool addressable); VkDescriptorBufferInfo Descriptor(std::uint64_t address, std::size_t bytes) const; std::vector AddressRanges() const; + // The BDA table of the cached address space when it serves this upload alone (an address-based + // build with no region outside it): its ranges, immutable while the space lives, and the + // space's serial, by which BdaResources reuses the table buffer built for it without a + // compare. Nothing when a region of this build is not in the space (AddressRanges then merges). + struct CachedTable { + std::uint64_t serial; + const std::vector* ranges; + }; + std::optional CachedAddressTable() const; + // The registered memory an address-based build maps, shared by consecutive builds (see + // GuestBufferMemory.cpp): its lease and the sorted, immutable regions of the ranges served in + // place by imports and mirrors. Opaque outside GuestBufferMemory.cpp. + struct AddressSpace; void WriteBack(); bool WritesOverlap(std::uint64_t address, std::size_t bytes) const; // Guest ranges the shader may write through descriptors, as [begin, end). @@ -123,11 +193,18 @@ public: // regions was recorded (ShaderResources::MarkGpuWrites), under GuestMemory::GpuMutex. A use that // never calls it (a synchronous draw) stores the staging bytes from the CPU in WriteBack. void RecordCopyBacks(Recorder& recorder); - // Whether registered allocations are pinned until write-back (address-based shaders). - bool HoldsLease() const { return !lease.empty(); } - // Every uploaded region as [begin, end) when all of them are served by host imports (nothing was - // copied, so the upload can serve a later identical build), else nothing. + // Whether registered allocations are pinned until write-back (address-based shaders): by this + // build's own lease, or by the cached address space it holds. + bool HoldsLease() const { return !lease.empty() || space != nullptr; } + // Every uploaded region as [begin, end) when all of them are served by host imports, in place or + // through a device-local staging copy of the import (nothing was copied through the CPU, so the + // upload can serve a later identical build), else nothing. std::optional>> DirectRegions() const; + // Every region the recorded work reads in place through a host import, as [begin, end): the + // regions bound in place (`direct`), an address-based build's leased heaps included. A region + // the GPU copies out of an import (gpuCopy) notes its read itself when the copy is recorded. + // For the recorder's read tracking (ShaderResources::MarkGpuWrites); nothing once committed. + std::vector> InPlaceReads() const; private: struct Region { @@ -168,11 +245,42 @@ private: VkBuffer copySource = VK_NULL_HANDLE; std::uint64_t copySourceBase = 0; bool copiedBack = false; + // An element the shader updates atomically lies inside (AddWritable's `atomic`). + bool atomic = false; + // The gpuCopy buffer is a device-local staging shadow (see stagingEligible): no host + // mapping, so nothing is ever stored from it by the CPU, and the region is taken even when + // the import could bind it in place. + bool deviceLocal = false; + // The device refused the shadow (memory or allocations exhausted): the region takes the + // path it would take without staging, in both upload stages. + bool unstaged = false; }; + // How [begin, end) lies against the space's base regions. + enum class BaseOverlap { None, Inside, Partial }; + BaseOverlap baseOverlap(std::uint64_t begin, std::uint64_t end, const Region** owner) const; + // Copies the space's base regions into this build's own regions (today's per-build form) and + // drops the space: for a region that partially overlaps a base region, and for a build whose + // imports changed under the space (`resolve`: the direct regions are re-resolved by UploadFinish). + void dissolveSpace(bool resolve); + // The region containing `address`: this build's own regions (sorted after UploadPrepare) or + // the space's base regions, or null. + const Region* owner(std::uint64_t address) const; + // Builds this build's regions from a lease as the per-build path always did: `base` gets the + // ranges served in place (imports, mirrors), `copied` the ones copied per build. + struct CopiedRange { + std::uint64_t begin; + std::uint64_t end; + bool writable; + }; + void addCopiedRange(const CopiedRange& range); + // The BDA table entry of an uploaded region. + static ShaderRecompiler::BdaAbi::Range addressRange(const Region& region); void validate(std::uint64_t address, std::size_t bytes) const; - // The region of a descriptor-bound range (AddWritable/AddReadable), committed pages only. - void addDescriptorRegion(std::uint64_t address, std::size_t bytes); + // The region of a descriptor-bound range (AddWritable/AddReadable), committed pages only. A + // range inside a base region of the space adds nothing: the region serves it, as today's merge + // of the two did. + void addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic); // Gives a region a buffer of its own with its bytes (guest memory for host-backed and writable // ranges, plus the write-back's reference copy for a range a descriptor writes; else its snapshot). void copyRegion(Region& region, bool addressable); @@ -180,11 +288,20 @@ private: // instead of the CPU (see Region::gpuCopy): live guest bytes, not sparse, at most the size // APS5_GPU_COPY_MAX_KIB allows, and APS5_CPU_COPIES unset. bool gpuCopyEligible(const Region& region) const; + // Whether a region inside a host import is staged in device memory instead of bound in place + // (see GuestBufferMemory.cpp): staging allowed, not address-based, a written element inside, + // and an atomic element or a size within the written-shadow window. Independent of the import, + // so UploadPrepare and UploadFinish decide alike. + bool stagingEligible(const Region& region, bool addressable) const; // Records the import-to-buffer copies of the given gpuCopy regions into the open batch, with // the barriers that order them after earlier recorded writes and before the shaders reading them. void recordGpuCopies(std::span copies, bool addressable); Context context; + bool stagingAllowed = false; GuestAllocations::Lease lease; + // The cached address space this build maps through (its lease pins the ranges); `regions` then + // holds only the regions outside it (V#s, snapshots, ranges copied per build). + std::shared_ptr space; // Import registry epoch when `direct` pointers were taken at acquire time; they are reused while // no import was destroyed since. std::uint64_t importsEpoch = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp index a0722e1488..efa0a95140 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp @@ -32,8 +32,10 @@ class Pipeline { public: // `state` carries the blend states with the outputs the pixel shader lacks already masked and // `vertexInput` the layout of the draw's vertex descriptors; the shaders were validated by the - // caller (ValidateShaders). - Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders); + // caller (ValidateShaders). `attachmentLayout` is the layout the color attachments are in + // before, during and after the pass (GENERAL for resident targets, which then need no + // transitions). + Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); ~Pipeline(); Pipeline(const Pipeline&) = delete; Pipeline& operator=(const Pipeline&) = delete; @@ -44,7 +46,12 @@ public: std::shared_ptr AcquireFramebuffer(std::span targets, std::span> owners, VkExtent2D extent); // Begins the render pass on the framebuffer, binds the pipeline and sets viewport and scissor. void Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, VkExtent2D extent, const VkViewport& viewport, const VkRect2D& scissor) const; + // The same inside a render pass another pipeline of the same attachments began (compatible by + // construction: the attachment formats alone decide). + void Continue(VkCommandBuffer commands, const VkViewport& viewport, const VkRect2D& scissor) const; void PushConstants(VkCommandBuffer commands, std::span shaders) const; + // The same from an assembled block (a draw recipe's, made by AssemblePushConstants once). + void PushConstants(VkCommandBuffer commands, VkShaderStageFlags stages, std::span bytes) const; // Forgets the Vulkan objects without destroying them: for entries of a device that is already gone. void Abandon() noexcept; @@ -71,7 +78,7 @@ private: // A configuration whose stage has no variant id gets a private pipeline. Entries are LRU-bounded and // only evicted once no recorded draw holds them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per // draw as before. -std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders); +std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); // Destroys the cached pipelines of a device; to be called before the device goes away. Without it, // entries of a gone device are recognised by their buffer pool (made and reset with the device, so // it tells device instances apart when the loader reuses a VkDevice handle) and forgotten unused. @@ -81,7 +88,7 @@ void ValidateViewport(const Context& context, const VkViewport& viewport); void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment); // Returns the color attachment locations the pixel shader writes. -std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric); +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index 0f37b4497a..80ea18a886 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -2,6 +2,7 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RECORDER_HPP #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include #include #include #include @@ -9,10 +10,11 @@ #include #include #include +#include #include #include #include -#include +#include #include #include @@ -41,8 +43,42 @@ public: Recorder(const Recorder&) = delete; Recorder& operator=(const Recorder&) = delete; - // The open batch's command buffer, starting a batch when none is open. - VkCommandBuffer Commands(); + // The open batch's command buffer, starting a batch when none is open. `coveredAccess`, when + // given, receives the destination access mask of the last recorded command's trailing barrier + // when that barrier had ALL_COMMANDS as its destination stage (MarkCovered), else 0: every + // write recorded so far is then available and visible to those accesses of any later stage, + // so a dispatch, fill or copy whose leading barrier asks for a subset of them may skip it + // (the [barriers] line counts the merges). The call clears the mask (whatever the caller + // records is assumed uncovered until it marks); a new batch starts uncovered. + VkCommandBuffer Commands(VkAccessFlags* coveredAccess = nullptr); + // After a trailing barrier with ALL_COMMANDS as its destination stage: `access` is its + // destination access mask. + void MarkCovered(VkAccessFlags access); + void MarkShaderReadsCovered() { MarkCovered(VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); } + // A recorded draw's render pass (Draw.cpp) is left open after the draw: the next draw of the + // same attachments (`key`: the views and the extent) continues it when nothing was recorded in + // between and the earlier draw allowed it (`continuable`: it wrote nothing but its + // attachments, so no barrier is owed inside the pass), and anything else recorded first ends + // it (vkCmdEndRenderPass, one trailing barrier for the pass, the end of the draw class range + // `timing`): Commands(), RecordStore and Submit end it; CommandsInRenderPass hands a + // continuing draw the command buffer without ending it. + bool ContinuesRenderPass(std::uint64_t key) const; + VkCommandBuffer CommandsInRenderPass(); + void LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable); + // DCC "uncompressed" key stores (DccMetadata.cpp StoreUncompressedOnGpu): queued on the open + // batch and recorded as one run (one barrier pair for every queued fill) at Submit, before a + // label store (RecordStore), or before a command that writes or reads a queued range (the + // fill must precede a later key writer or reader in the batch: FlushKeyStores by the callers + // that know their ranges). `seed` (kept) supplies an + // unaligned head or tail; [begin, end) is the guest range. Debug aid: APS5_DCC_KEYS_EACH=1 + // records every store at once, as before (QueueKeyStore then records and returns). + void QueueKeyStore(VkBuffer buffer, VkDeviceSize first, VkDeviceSize last, std::shared_ptr seed, std::uint64_t begin, std::uint64_t end); + bool HasQueuedKeyStores() const { return open != nullptr && !open->keyStores.empty(); } + bool QueuedKeyStoreOverlaps(std::uint64_t address, std::size_t bytes) const; + // Whether `overlaps(begin, end)` holds for a queued store's range. + bool AnyQueuedKeyStore(const std::function& overlaps) const; + void FlushKeyStores(); + void FlushKeyStoresOverlapping(std::uint64_t address, std::size_t bytes) { if (HasQueuedKeyStores() && QueuedKeyStoreOverlaps(address, bytes)) FlushKeyStores(); } bool Recording() const { return open != nullptr; } bool Idle() const { return open == nullptr && inFlight.empty(); } // Whether recorded work still has completion actions (write-backs the CPU must see) to run. @@ -60,6 +96,44 @@ public: // Notes several [begin, end) ranges and publishes the snapshot once (a dispatch writes many buffers). void NotePendingWrites(std::span> ranges); bool PendingWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + // Whether no unfinished batch writes the range any more: no open-batch overlap, and every + // overlapping in-flight batch's fence has signaled (its in-place GPU stores are final in host + // memory; its completion stores are the caller's business, see VulkanDevice::CopyBuffer). + bool PendingWriteSettled(std::uint64_t address, std::size_t bytes) const; + // Batch read tracking: the guest ranges the recorded work reads IN PLACE through a host import + // and the GPU has not executed yet (a V# element bound in place, a region the GPU copies out of + // an import, an address-based build's leased heaps, indirect arguments, a GPU-direct storage + // upload, the copy HLE's source). A CPU store into such a range before the batch ran would be + // seen by the reader early (WAR); VulkanDevice::CopyBuffer refuses its CPU path on a hit. Noted + // on the open batch, no snapshot: the only reader holds the mutex. `kind` names the reader for + // the [recorder] hit counters. APS5_COPY_READ_TRACKING=0 notes nothing (ReadTracking() is then + // false and the CPU copy falls back to Idle()). + enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, Count }; + static bool ReadTracking(); + void NotePendingRead(std::uint64_t address, std::size_t bytes, ReadKind kind); + void NotePendingReads(std::span> ranges, ReadKind kind); + // Whether an unexecuted batch (open, or in flight with its fence unsignaled) reads the range in + // place. A hit in an in-flight batch whose fence signaled meanwhile is a miss (one status + // query per hit). `ignoreSignaled` false: every in-flight batch counts, whatever its fence. + bool PendingReadOverlaps(std::uint64_t address, std::size_t bytes, bool ignoreSignaled = true) const; + // The newest batch whose noted read overlaps the range (for the [copyhle] trace and the verify + // switch): its serial (the open batch reports the serial it will get), recording queue, reader + // kind, and whether its fence has signaled. Nullopt: no overlap. + struct PendingReadInfo { + std::uint64_t serial; + std::uint32_t queue; + ReadKind kind; + bool open; + bool signaled; + }; + std::optional DescribePendingRead(std::uint64_t address, std::size_t bytes) const; + // Counters of the read tracking (cumulative, for the [recorder] line): ranges noted, overlap + // queries, hits by reader kind, and hits ignored because the reader's fence had signaled. + struct ReadStatistics { + std::uint64_t noted, queries, staleIgnored; + std::array(ReadKind::Count)> hits; + }; + static ReadStatistics ReadCounts(); // Whether the OPEN (unsubmitted) batch writes the range: a wait on such a range must submit it. bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; // What SyncThrough(address, bytes) would wait for, for the [hooksync] attribution: the newest @@ -104,6 +178,34 @@ public: // Completes batches that already finished; returns whether nothing is in flight. bool Reap(); std::uint64_t Submissions() const { return submissions; } + // Batches submitted and not yet finished (what a full Sync() waits for); under GuestMemory::GpuMutex. + std::size_t InFlightBatches() const { return inFlight.size(); } + // Reaps that retired at least one batch so far (the [recorder] line's 'with work' count). + static std::uint64_t ReapsWithWork(); + + // A store of `bytes` into a host import (a label, or a COPY_DATA/DMA_DATA/DUMP_CONST_RAM store + // of up to 64 KiB; `address` is its guest address) recorded into the open batch: ordered + // after everything recorded before it and visible to the host and to later work. The stores + // of a batch form one run recorded as the batch's last commands at Submit: one barrier + // (everything so far -> transfer and host: every result of the batch reaches host memory + // before any label of it lands, the order the title's pollers rely on), the + // vkCmdUpdateBuffers, one transfer -> host barrier. A later command of the batch that writes + // over a queued store's bytes or reads them in place must find the store recorded (the + // callers that know their ranges call FlushStoresOverlapping, or FlushStores when they do + // not, before recording): the run is then recorded in place, with a trailing barrier toward + // the later work, and the next store starts a new run. Within a run a store contiguous with + // the last queued one joins its vkCmdUpdateBuffer, one inside it replaces its bytes (the + // later store wins, as in program order), and one overlapping a store queued earlier gets a + // transfer-to-transfer barrier first (WAW). Debug aids: APS5_LABEL_RUNS_INLINE=1 records the + // stores at once and closes the run at the next command (Commands()) or at Submit, as + // before; APS5_NO_LABEL_RUNS=1 gives every store barriers of its own. + void RecordStore(VkBuffer buffer, VkDeviceSize offset, std::span bytes, std::uint64_t address); + bool HasQueuedStores() const { return open != nullptr && !open->run.queued.empty(); } + bool QueuedStoreOverlaps(std::uint64_t address, std::size_t bytes) const; + // Whether `overlaps(begin, end)` holds for a queued store's guest range. + bool AnyQueuedStore(const std::function& overlaps) const; + void FlushStores(); + void FlushStoresOverlapping(std::uint64_t address, std::size_t bytes) { if (HasQueuedStores() && QueuedStoreOverlaps(address, bytes)) FlushStores(); } // Pending-label table. A GPU label (RELEASE_MEM/WRITE_DATA recorded as a store into the batch) // is noted per 4-byte dword with the value it will store and a record-order stamp (the driver's @@ -112,22 +214,85 @@ public: // for the GPU: any CPU store the game ordered before that submit call precedes the label in // program order exactly as on hardware, and later recorded work follows the label in queue // order. Entries recorded before the submission (the previous frame's label at the same address) - // are never trusted. Entries leave the table with their batch (a memory bound only). + // are trusted only under the late rule below (APS5_LABEL_TRUST_LATE=0 never trusts them). + // Entries leave the table with their batch (a memory bound only). The note also enters the + // label's range as a pending write of the open batch (the flush hook syncs CPU reads), marked + // as the label's own so it does not count as an overwrite of the entry. void NoteLabel(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue); + // Late rule: an entry whose stamp is not newer than the wait's submission is still the value + // memory will hold, unless a CPU store touched the dword since it was recorded (the table + // cannot see CPU stores; the title recycles label slots by CPU writes) or a later recorded + // non-label GPU write covers it. The table refuses a queued entry, one flagged `overwritten` + // (noteWrite), and one whose record group has not closed (generation 0); the caller checks the + // CPU side with GuestMemory::UnchangedSinceCollected against the hit's generation. Labels are + // noted in groups (a worker's deferred labels recorded back to back, or one immediate label + // or store): CloseLabelGroup, called by the recording thread after the group's last + // MarkWritten, stamps the group's entries with the write tracker's generation at that point; + // per-entry capture would fail, as the next label of the group stamps the shared 64 KiB block + // newer. A generation of 0 leaves the entries unclosed (a failed record), as does a batch + // already submitted at the close (its value may have landed before the generation was taken). + static void CloseLabelGroup(std::uint64_t trackerGeneration); + static bool LateTrust(); + struct LabelHit { + std::uint64_t value; + // The recording queue of the first dword, and its record stamp. + std::uint32_t queue; + std::uint64_t stamp; + // Taken under the late rule: `generation` is the oldest group generation of the dwords. + bool late; + std::uint64_t generation; + }; + // Why a lookup refused a late candidate (an entry with stamp <= afterStamp), for the trace. + // BehindCompletion replaces the other reasons when the refused entry's value reaches memory + // only through a completion action of its batch (AfterCompletions: the GPU has no view of the + // range, or a write-back overwrote the GPU's store): a poller must reap that batch to converge. + enum class LabelRefusal : std::uint8_t { None = 0, TrustOff, Queued, Overwritten, Unclosed, BehindCompletion }; + // Per calling thread (lookups run on the waiting worker): late candidates seen, in both modes, + // and the refusals by reason ([packets] line deltas). + struct LateStatistics { + std::uint64_t candidates, queued, overwritten, unclosed; + }; + static LateStatistics LateCounts(); // The 4- or 8-byte value the table holds for `address` when every dword is present with a stamp - // newer than `afterStamp`; `queue` receives the recording queue of the first dword. - std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; - std::size_t PendingLabels() const { return labels.size(); } - // Whether any tracked label dword lies inside [address, address + bytes): a range query for - // large ranges (a fill of megabytes), where a per-dword lookup would not do. + // newer than `afterStamp`, or trustable under the late rule. + std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal = nullptr) const; + std::size_t PendingLabels() const; + // Whether any tracked label dword (recorded, or queued by a worker and not recorded yet) lies + // inside [address, address + bytes): a range query for large ranges (a fill of megabytes), + // where a per-dword lookup would not do. bool PendingLabelIn(std::uint64_t address, std::size_t bytes) const; // PendingLabel of the active recorder WITHOUT GuestMemory::GpuMutex: the table has a small mutex // of its own (every mutation holds both), so a WAIT_REG_MEM consults it without queueing behind // device work. Nothing is done under the table mutex but the lookup (it never takes the GPU mutex). - static std::optional LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue); + static std::optional LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal = nullptr); + // The value alone, for callers asking whether any label is pending in a dword (afterStamp 0). + static std::optional LookupLabelValue(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp); + // A label a queue worker decoded but has not recorded yet (Driver.cpp DeferredLabels): it + // enters the table with no batch, and only a lookup made on the noting thread (a WAIT_REG_MEM + // of the same queue) takes its value, since that queue's later work follows the label in queue + // order whenever the group is recorded; other queues and the CPU see it once it is recorded. + // The range is also kept per thread, so a CPU access of it by the same worker (the flush hook) + // has the group recorded first through the function SetQueuedLabelRecorder installed. Takes + // the table mutex only. The queued entries live apart from the recorded ones: a recorded + // label of the same dword (an older store still pending on the GPU) keeps serving the other + // queues' waits and the driver's pending-label checks until the queued one is recorded. + // Debug aid: APS5_NO_SEPARATE_QUEUED_LABELS=1 lets a queued entry replace the recorded one. + static void NoteQueuedLabel(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue); + // After the calling worker recorded (or dropped) its queued labels: its ranges are cleared and + // the entries of its dwords still without a batch (stored by the CPU, or dropped) leave the table. + static void ForgetQueuedLabels(); + // The function the flush hook calls, on the accessing thread (which may hold the GPU mutex), + // when an access overlaps one of that thread's queued labels; it records them. Set once. + static void SetQueuedLabelRecorder(void (*recorder)()); // Whether the lock-free pending-write snapshot (open, in-flight and finishing batches) overlaps // the range: false means no recorded work writes it, so a wait on it has nothing to submit. static bool SnapshotWriteOverlaps(std::uint64_t address, std::size_t bytes); + // The snapshot itself (the sorted, merged union of the pending ranges; null when none), for a + // reader that tests many ranges against one loaded snapshot: one atomic shared_ptr load per + // validation instead of one per run, and every test sees the same snapshot (design13 R1's p0). + using WriteRanges = std::vector>; + static std::shared_ptr PendingWriteSnapshot(); + static bool SnapshotOverlaps(const WriteRanges* snapshot, std::uint64_t address, std::size_t bytes); // A label behind pending completions (CPU write-backs the label must not precede): stored to // guest memory by a completion action appended to the open batch, or to the newest in-flight // batch when none is open (no empty submission), so it runs after every earlier write-back in @@ -140,7 +305,13 @@ public: // over a fresh command buffer). Debug aid: APS5_LABEL_STORE_ALWAYS=1 stores unconditionally. void AfterCompletions(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu); // A CPU store of GPU results into guest memory (GuestBufferMemory::WriteBack): recorded so a - // completion label store can tell whether its bytes were overwritten. + // completion label store can tell whether its bytes were overwritten. Under GuestMemory:: + // GpuMutex (every write-back runs inside finish() or a synchronous draw's wait): the first + // write-back overlapping a completion label stored on the GPU is what makes that label's + // completion store necessary, so it is counted into PendingCompletionLabels here rather than + // at registration (APS5_COUNT_ALL_COMPLETION_LABELS=1 counts every one at registration, as + // before). A write-back over any tracked label dword is counted on the [recorder] line (must + // stay 0: it would mean a CPU store landed over a label recorded on the GPU). static void NoteWrittenBack(std::uint64_t address, std::size_t bytes); // Lock-free readers for the queue workers' packet loops and WAIT_REG_MEM polls (values of the @@ -151,8 +322,15 @@ public: // bumped by every note of a pending write into the open batch, so a poller learns that the // producer may have recorded the label it waits for; static std::uint64_t WriteGeneration(); + // bumped right AFTER every store of the pending-write snapshot (the generation above moves + // before the store), so a reader that loads it before and after a set of in-place reads knows + // the snapshot it consulted in between was the one in force for all of them; + static std::uint64_t PublishGeneration(); // labels waiting in completion actions (the CPU must reap their batch for them to land); static std::uint64_t PendingCompletionLabels(); + // write-back completions (OnComplete) of unfinished batches: a reaper retires them when every + // worker sleeps, as for the labels above; + static std::uint64_t PendingWriteBackCompletions(); // dispatches and draws recorded since the last submit (the driver counts them: CountRecordedWork), // so a batch is submitted after a bounded amount of work (APS5_BATCH_CAP). static std::uint64_t RecordedWorkSinceSubmit(); @@ -161,6 +339,9 @@ public: // The recorder of the current device, for code that only has guest addresses (the flush hook) // and for helpers that must not recycle resources the recorded work still uses. static Recorder* Active(); + // Whether the calling thread runs inside a batch's completion action (finish()): a store made + // there must not have a publish recorded for it (UnitShadow), see SyncThrough. + static bool InCompletion(); void Activate(); // APS5_PROFILE_DRAW: syncs by source (0 device idle, 1 CPU access to pending writes, 2 CPU read // after a recorded store, 3 address-based dispatch, 4 other). Contract: the call announces the @@ -181,12 +362,107 @@ public: // when not profiling): a caller reads it around a span of its own work to learn how much of // that span waited for the GPU (a resource build's nested flush-hook waits, a draw's). static double ThreadWaitedMs(); + // The calling thread's flush-hook syncs (SyncThrough) that waited for a target batch whose + // fence had not signaled when the wait began: a read whose bytes are compared around such a + // sync observed the GPU's work; one that found the fence signaled already read the GPU's bytes + // both times and observed nothing (Driver.cpp's dword evidence reads it before and after). + static std::uint64_t ThreadHookWaits(); + // Cumulative counts for the [labels] line (APS5_PROFILE_DRAW): stores recorded (RecordStore), + // the runs they formed (one barrier pair each), stores joined to a pending contiguous one, + // stores that replaced pending bytes, WAW barriers inside runs, joins refused because the + // joined store would overlap one the run already recorded, queued labels noted (of which + // over a recorded entry of the same dword), lookups a queued label satisfied on its own + // queue, flush-hook accesses that had queued labels recorded first, and hook accesses made + // by a completion action whose queued-label record was skipped. + struct StoreStatistics { + std::uint64_t stores, runs, joined, replaced, wawBarriers, joinsRefused, queuedNoted, queuedOverRecorded, queuedHits, queuedHookRecords, queuedHookInCompletion; + // DCC key stores queued, the runs that recorded them (at Submit, or before a writer of a + // queued range), and the queued stores a duplicate range joined. + std::uint64_t keyStores, keyStoreRuns, keyStoreRunsForWriter, keyStoresJoined; + // Store runs recorded at Submit, and in place before a later writer or reader of a queued + // store's bytes (FlushStores). + std::uint64_t runsAtSubmit, runsForced; + }; + static StoreStatistics StoreCounts(); // APS5_PROFILE_GPU=1: GPU time of recorded work by key (a guest program address), from timestamp // queries around each timed range; the totals per key are reported every 10 s. Begin returns the // range index to pass to End, or NoTiming when timing is off or the batch's queries are used up. static constexpr std::uint32_t NoTiming = 0xffffffffu; + static bool GpuTimingEnabled(); std::uint32_t BeginGpuTiming(std::uint64_t key); - void EndGpuTiming(std::uint32_t index); + // `bytes`: what the range moved (a fill's, a copy's), summed per key on the [gputime] line. + void EndGpuTiming(std::uint32_t index, std::uint64_t bytes = 0); + // The key of the whole-batch range (first to last command; reported as "batch" on the [gputime] + // line, apart from the per-program totals). + static constexpr std::uint64_t BatchTimingKey = 0x3; + // The driver's own command classes: their [gputime] ranges take the reserved keys 0x10 + class + // (printed by name, with counts and bytes, per 10 s and per present) and their barriers are + // counted on the [barriers] line (CountBarriers, printed every 10 s under APS5_PROFILE_DRAW + // or APS5_PROFILE_GPU). A class range covers the command with its own barriers (a dispatch's + // barriers are classes of their own, its program range stays keyed by the program), so the + // union of every range of a batch and the batch span differ by what no class times ('untimed'). + enum class CommandClass : std::uint8_t { DispatchLeading = 0, DispatchTrailing, IndirectArguments, LabelRun, Fill, FillClear, Copy, StagingIn, StagingOut, Draw, StorageUpload, StorageWriteBack, DccClear, DccKeyStore, PresentBlit, ShadowPublish, TemplateDataRefresh, Count }; + static constexpr std::uint64_t ClassKey(CommandClass which) { return 0x10 + static_cast(which); } + std::uint32_t BeginGpuTiming(CommandClass which) { return BeginGpuTiming(ClassKey(which)); } + static void CountBarriers(CommandClass which, std::uint32_t count = 1); + // A leading barrier a command left out because the previous trailing barrier covered its + // accesses (see Commands), on the [barriers] line as 'merged'. Debug aid: APS5_FULL_BARRIERS=1 + // (or APS5_NO_BARRIER_ELISION=1) records every leading barrier, as before. + static bool MergeBarriers(); + static void CountMerged(CommandClass which); + // Hazard tracker, counting mode (APS5_BARRIER_VALIDATE=1): every command tells the tracker + // what it reads and writes (guest ranges through host imports, images) with its stage before + // it is recorded; the tracker simulates barriers emitted only on a hazard against the accesses + // since its last simulated barrier (RAW, WAW, WAR, an image written; a command whose ranges are + // unknown, an address-based build or a BDA draw, is a hazard by itself) and counts, per class, + // the leading barriers a hazard-driven recorder would have emitted and skipped, on the + // [barriers] line; the real barriers are recorded as before. APS5_TRACE_BARRIERS= prints + // the first n skipped decisions with their ranges. Costs nothing when off. + struct Access { + std::span> reads; + std::span> writes; + // (image, written) + std::span> images; + VkPipelineStageFlags stages; + bool conservative = false; + }; + static bool BarrierValidate(); + void NoteAccess(CommandClass which, const Access& access); + // A range timed outside the recorder's batches (the presenter's blit, stamped into its own + // pool and read after its fence): enters the class totals like a recorded one. + static void AddGpuTiming(CommandClass which, double nanoseconds, std::uint64_t bytes); + // Presentations, so the [gputime] class totals can be given per present. + static void CountPresent(); + // Presentations so far (never reset): a frame clock for policies that count per frame, such as + // StorageTexture's write-back hysteresis. + static std::uint64_t Presents(); + + // What the presenter learns about finished batches (Driver::Present's [present] line and the + // frame record): finish() writes an entry per submitted batch into a ring of the last 128, + // under GuestMemory::GpuMutex and the ring's own mutex; the readers below take only the ring + // mutex, so the presenter reads them WITHOUT the GpuMutex. gpuStartNs/gpuEndNs are the + // whole-batch stamps (0 without APS5_PROFILE_GPU); `reads` and `readGeneration` (the + // write-watch generation collected over the noted in-place reads at submit) are filled with + // APS5_FLIP_READ_CHECK=1 only. + struct Completed { + std::uint64_t serial = 0; + double gpuStartNs = 0; + double gpuEndNs = 0; + std::chrono::steady_clock::time_point submittedAt{}; + std::chrono::steady_clock::time_point fenceSeenAt{}; + std::uint64_t readGeneration = 0; + std::vector> reads; + }; + // The entries with serial in (afterSerial, throughSerial], oldest first; `missing` counts the + // serials of that range not in the ring (not finished yet, or overwritten by newer batches). + // An entry is overwritten once the batch CompletedRingSize serials later finishes. + static constexpr std::size_t CompletedRingSize = 512; + std::vector CompletedBatches(std::uint64_t afterSerial, std::uint64_t throughSerial, std::size_t& missing) const; + // The newest submitted serial and its vkQueueSubmit time (ring mutex only). + std::uint64_t NewestSubmitted(std::chrono::steady_clock::time_point* submittedAt = nullptr) const; + // In-flight batches whose fence has not signaled (one status query each); under the mutex. + std::size_t UnsignaledBatches() const; + static bool FlipReadCheck(); private: struct Batch { @@ -195,24 +471,151 @@ private: std::vector> kept; std::vector> completions; std::vector> writes; + // In-place reads (see NotePendingRead), dying with the batch: a finished batch's reads are done. + struct Read { + std::uint64_t begin; + std::uint64_t end; + ReadKind kind; + }; + std::vector reads; // Submission number (1-based): identifies a batch after its allocation may have been reused. std::uint64_t serial = 0; bool submitted = false; + // The GpuMutex queue tag of the thread that opened the batch, and when it was submitted + // (the [syncwait] line reports what a wait's target and the batches ahead of it were). + std::uint32_t queue = 0xffffffffu; + std::chrono::steady_clock::time_point submittedAt{}; VkQueryPool queries = VK_NULL_HANDLE; std::vector timedKeys; + std::vector timedBytes; + // The whole-batch timed range (BatchTimingKey) and its stamps once read (see Completed). + std::uint32_t batchTiming = NoTiming; + double gpuStartNs = 0; + double gpuEndNs = 0; + std::uint64_t readGeneration = 0; // Dword addresses this batch noted in the label table (removed when it finishes). std::vector labelDwords; // Labels stored by this batch's completion actions (see AfterCompletions); subtracted from // the lock-free pending count when the batch finishes, whether or not its completions ran. std::uint32_t completionLabelCount = 0; + // Write-back completions (OnComplete), likewise. + std::uint32_t writeBackCompletionCount = 0; + // The ranges of the completion labels also stored on the GPU, and whether each entered the + // pending count (NoteWrittenBack counts one the first time a write-back overlaps it). + struct CompletionLabel { + std::uint64_t begin; + std::uint64_t end; + bool counted; + }; + std::vector completionLabelRanges; + // The store run in progress (RecordStore): whether its leading barrier was recorded and + // the trailing one is still owed, the store not yet recorded (joined by contiguous ones), + // and the [buffer, offset, end) of the stores already recorded (the WAW check). + struct StoreRun { + bool open = false; + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceSize offset = 0; + std::vector bytes; + std::vector> recorded; + // The run's [gputime] range (leading barrier to trailing barrier) and its stored bytes. + std::uint32_t timing = NoTiming; + std::uint64_t storedBytes = 0; + // The stores queued for the batch's run (per-batch runs; see RecordStore), in order. + struct Queued { + VkBuffer buffer; + VkDeviceSize offset; + std::vector bytes; + std::uint64_t address; + }; + std::vector queued; + } run; + // See Commands(): the accesses the last recorded command's trailing barrier covers. + VkAccessFlags coveredAccess = 0; + // The hazard tracker's accesses since its last simulated barrier (see NoteAccess). + struct Tracker { + struct Range { + std::uint64_t begin; + std::uint64_t end; + VkPipelineStageFlags stages; + }; + std::vector reads; + std::vector writes; + struct Image { + VkImage image; + bool written; + VkPipelineStageFlags stages; + }; + std::vector images; + } tracker; + // The render pass a recorded draw left open (see ContinuesRenderPass). + struct RenderPass { + bool open = false; + bool continuable = false; + std::uint64_t key = 0; + std::uint32_t timing = NoTiming; + } renderPass; + // A pass ended in this batch: Submit records the host-read barrier its draws left out. + bool hostReadOwed = false; + // Queued DCC key stores (see QueueKeyStore). + struct KeyStore { + VkBuffer buffer; + VkDeviceSize first; + VkDeviceSize last; + std::shared_ptr seed; + std::uint64_t begin; + std::uint64_t end; + }; + std::vector keyStores; }; struct LabelEntry { std::uint32_t value; std::uint32_t queue; std::uint64_t stamp; + // Null for a queued label (NoteQueuedLabel): not recorded yet, so no batch removes it. const Batch* batch; + // Write-tracker generation at the close of the entry's record group (CloseLabelGroup); + // 0 while the group is open. A recorded non-label GPU write over the dword noted after the + // entry sets `overwritten`: memory will not end with `value`. Both serve the late rule only. + std::uint64_t generation = 0; + bool overwritten = false; + // The value lands in memory only when the batch's completion actions run (LabelRefusal::BehindCompletion). + bool behindCompletion = false; }; void readGpuTiming(Batch& batch); + // BeginGpuTiming on the open batch without Commands() (RecordStore times its own run, which + // Commands() would close). + std::uint32_t beginTiming(std::uint64_t key); + // Vulkan entry points resolved once (constructor): the loader's vkGetDeviceProcAddr is a name + // lookup per call, paid by every reap, store and submit otherwise. APS5_NO_PROC_TABLE=1 leaves + // them null and resolves per call as before; vkWaitSemaphoresKHR stays a lazy lookup (timeline + // semaphores are optional). + PFN_vkGetFenceStatus getFenceStatus = nullptr; + PFN_vkResetFences resetFences = nullptr; + PFN_vkWaitForFences waitForFences = nullptr; + PFN_vkCmdUpdateBuffer cmdUpdateBuffer = nullptr; + PFN_vkBeginCommandBuffer beginCommandBuffer = nullptr; + PFN_vkEndCommandBuffer endCommandBuffer = nullptr; + PFN_vkQueueSubmit queueSubmit = nullptr; + PFN_vkCmdPipelineBarrier cmdPipelineBarrier = nullptr; + template + TFunction function(TFunction resolved, const char* name) const { + return resolved != nullptr ? resolved : context.Function(name); + } + VkResult fenceStatus(VkFence fence) const { return function(getFenceStatus, "vkGetFenceStatus")(context.device, fence); } + void recordBarrier(VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) const; + // Opens a batch when none is open, without closing a store run (Commands() does both). + void ensureOpen(); + // Records the run's pending store (vkCmdUpdateBuffer), if any. + void flushPendingStore(); + // Records the pending store and the run's trailing barrier; the next store starts a new run. + // Per-batch runs: records the queued stores with their barriers (`atSubmit`: the batch-end + // form, toward the host only); returns whether the batch's host-read barrier was included. + bool closeStoreRun(bool atSubmit = false); + // Ends the render pass a draw left open (vkCmdEndRenderPass, the pass's trailing barrier). + void endOpenRenderPass(); + // Records the queued key stores as one run (`forWriter`: before a command writing, reading or + // labelling over one, not at Submit). + void recordKeyStores(bool forWriter); // The unlocked wait of SyncThrough(waitUnlocked): the target batch (submitting the open one when // it is the target), the timeline wait with the mutex released, then the completions up to it. @@ -222,13 +625,22 @@ private: void finish(std::unique_ptr batch, bool wait, int source); void release(Batch& batch) noexcept; static bool overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end); + // The first noted read of `batch` overlapping [address, end), or null. + static const Batch::Read* readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end); + bool signaled(const Batch& batch) const; // Rebuilds the lock-free snapshot of pending writes from open, inFlight and finishing. void publishPendingWrites() const; // Appends one range to the open batch; returns whether the snapshot must be rebuilt for it. - bool noteWrite(std::uint64_t address, std::size_t bytes); + // `ownLabel`: the range is a label's own store (NoteLabel, AfterCompletions), which does not + // overwrite the table entries it covers; any other range flags them (table mutex, briefly). + bool noteWrite(std::uint64_t address, std::size_t bytes, bool ownLabel = false); // Appends one range to `batch` (open or in flight) and publishes the snapshot if needed. - void noteWriteOn(Batch& batch, std::uint64_t address, std::size_t bytes); - void noteLabelOn(Batch& batch, std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue); + void noteWriteOn(Batch& batch, std::uint64_t address, std::size_t bytes, bool ownLabel = false); + void noteLabelOn(Batch& batch, std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool behindCompletion = false); + // Flags the recorded entries of `batch` over [begin, end) as stored by a completion action. + void markBehindCompletion(const Batch& batch, std::uint64_t begin, std::uint64_t end); + // Flags the recorded entries of the dwords overlapping [address, end) as overwritten. + void markOverwritten(std::uint64_t address, std::uint64_t end); // Whether a write-back noted after `sequence` overlapped [begin, end); true when the ring no // longer reaches back to `sequence` (conservative: the store runs as before). bool writtenBackSince(std::uint64_t sequence, std::uint64_t begin, std::uint64_t end); @@ -236,16 +648,26 @@ private: std::deque> writtenBack; std::uint64_t writtenBackSequence = 0; // PendingLabel without the table mutex (the caller holds it, or the GPU mutex). - std::optional lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; + std::optional lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal) const; + // Whether the in-flight batch with that serial has not signaled its fence (false when it is + // not in flight any more): what makes a wait for it a real GPU wait (ThreadHookWaits). + bool unsignaled(std::uint64_t serial) const; Context context; VkSemaphore timeline = VK_NULL_HANDLE; // Identity for a thread that released the mutex around a wait (syncThroughUnlocked): the // recorder may have been torn down meanwhile, so it is looked up by this id, not by pointer. std::uint64_t id = 0; - // Mutated only under both GuestMemory::GpuMutex and the table mutex (Recorder.cpp), so a holder - // of either reads it consistently. - std::unordered_map labels; + // Every access holds the table mutex (Recorder.cpp): the recorded entries are mutated under + // GuestMemory::GpuMutex as well, the queued ones (NoteQueuedLabel) under the table mutex alone. + // Ordered, so a range (a noted write, PendingLabelIn) finds its dwords by lower_bound. + std::map labels; + // The queued entries (batch always null), by dword; a lookup on the noting queue's thread + // prefers them (its newest store in program order), every other lookup sees `labels` alone. + std::map queuedLabels; + // labels.size(), readable without the table mutex: a noted write skips the mutex while the + // table is empty (most of the time between label groups). + std::atomic recordedLabels{0}; std::unique_ptr open; std::deque> inFlight; // Batches popped from inFlight whose completions have not run yet: their writes stay in the @@ -255,6 +677,10 @@ private: // Command buffers and fences of completed batches, reused by later ones (hundreds of batches per // frame would otherwise allocate and free their objects each time). std::vector> spare; + mutable std::mutex completedMutex; + std::array completed; + std::uint64_t newestSubmitted = 0; + std::chrono::steady_clock::time_point newestSubmittedAt{}; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp index 4d4f226f2f..ba43520988 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp @@ -14,7 +14,10 @@ public: Buffer& operator=(const Buffer&) = delete; VkBuffer Handle() const; VkDeviceAddress DeviceAddress() const; + // Host-visible buffers only: a device-local one (properties without HOST_VISIBLE, the staging + // shadows of GuestBufferMemory) has no mapping and its bytes move by GPU copies alone. std::span Bytes(); + bool Mapped() const { return mapping != nullptr; } void Invalidate(); private: @@ -28,6 +31,9 @@ private: std::size_t size; // The VkBuffer's own size, `size` rounded up to its pool class (see BufferPool::Capacity). std::size_t capacity; + // Fully made (or taken from the pool), so release returns it to the pool instead of destroying + // the handles a failed construction left. + bool ready = false; VkDeviceSize allocationBytes = 0; VkBufferUsageFlags usage; VkMemoryPropertyFlags properties; @@ -64,6 +70,9 @@ void CopyBuffer(const Context& context, VkCommandBuffer commands, VkBuffer sourc // Records a global memory barrier. void RecordMemoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess); +// Resolves every DeviceFunctions entry point of the context's device (once, at device setup). +void FillDeviceFunctions(const Context& context, DeviceFunctions& functions); + class RenderTarget { public: RenderTarget(const Context& context, const ColorTarget& target, bool blending); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp index 075dae26f7..540d73af7f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp @@ -3,14 +3,22 @@ #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" #include "Recompiler.hpp" +#include #include #include +#include namespace AgcDriver::Graphics { ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& context, bool hasColorTarget, std::uint8_t colorComponentMapping); ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& shader); -ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData); +// A guest range DecodeVertexStageInfo read (an attribute word, a vertex V#) with the bytes as read: +// the draw cache validates them by value with the stage's capture (design_cpu_final rule RD). +struct DecodeRead { + std::uint64_t address; + std::vector bytes; +}; +ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData, std::vector* reads = nullptr); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 25b3629aa7..2ac3fc2fd8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -24,6 +24,9 @@ class Recorder; // The cached storage image of a surface (render targets use it as their resident image); brought up // to date with guest memory before it is returned. std::shared_ptr CachedStorageSurface(const Context& context, const GuestTextureResource& resource); +// Whether `image` is still the storage cache's image of its surface (what CachedStorageSurface +// would return); a true answer counts as a use for the cache's eviction order, as a lookup would. +bool StorageImageCached(const Context& context, const StorageTexture* image); // Defined in Texture.cpp beside the pending-results registry, for the fast Revalidate below: whether // a storage image other than `except` has results pending in [address, address + bytes). @@ -107,6 +110,19 @@ public: bool HasCopiedWrites() const { return guestMemory.HasCopiedWrites(); } bool HoldsLease() const { return guestMemory.HoldsLease(); } bool WritesOverlap(std::uint64_t address, std::size_t bytes) const { return guestMemory.WritesOverlap(address, bytes); } + // Whether a region the recorded work reads in place through a host import overlaps the range. + bool ReadsOverlap(std::uint64_t address, std::size_t bytes) const; + // The recorder's hazard tracker inputs (Recorder::NoteAccess): the written elements' guest + // ranges, the regions read in place, and the storage images with whether each is written. + const std::vector>& GpuWrites() const { return guestMemory.Writes(); } + std::vector> InPlaceReads() const { return guestMemory.InPlaceReads(); } + std::vector> StorageImages() const; + // Whether a use writes guest memory beyond a draw's attachments (storage images, written or + // copied buffers, an address-based build's unknown writes), or reads `image` (a view of it + // sampled, or the image itself bound): a recorded draw's render pass may only be continued by + // a draw for which neither holds (Draw.cpp). + bool WritesMemory() const; + bool ReadsImage(const StorageTexture* image) const; const std::vector& LayoutKey() const { return layoutKey; } // Debug aid: each bound guest resource with the fraction of sampled bytes that are nonzero. std::string Describe() const; @@ -119,15 +135,52 @@ public: // guest buffer is read in place through a host import. Revalidate brings a reusable object up to // date for another use, or says it cannot be reused any more (a buffer's import went away, or a // texture or storage image the set references is no longer the one its memory maps to). - static std::vector ContentKey(const CompiledShader& shader); + // Without `dataWords` the ShaderData and FlattenedSrt descriptor words stay out of the key + // (their count and size remain): a compute template then serves dispatches whose constants + // differ, and the hit refreshes its data buffers with the dispatch's words (RefreshData). + static std::vector ContentKey(const CompiledShader& shader, bool dataWords = true); + // Records the shader's ShaderData and FlattenedSrt words into this object's data buffers + // (vkCmdUpdateBuffer, a transfer write the caller's pre-dispatch barrier makes visible; a + // buffer already holding the words is left alone). Returns whether anything was recorded. With + // `recorder` the updates are one [gputime] range of class TemplateDataRefresh. + bool RefreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder = nullptr); + // Walks the write watch over every image surface of a completed object (no device lock: the + // ranges are fixed by the build), so a Revalidate's collects on the same worker are memo hits. + void PrecollectSurfaces() const; bool Reusable() const { return reusable; } // `shaders` are the stages the object was built from, in build order (a recorded draw's vertex // and fragment stages, or one compute stage): their bindings are walked like the build did. - bool Revalidate(std::span shaders); - bool Revalidate(const CompiledShader& shader) { return Revalidate(std::span(&shader, 1)); } - // APS5_PROFILE_DRAW: the build's sub-phase times in milliseconds (zero when not profiling), so - // the draw path can total them separately from dispatches. bindingsMs covers the binding plan - // (stage A) and the image lookups (stage B); prepareMs/completeMs are the two stages' totals. + // How a Revalidate proved (or refused) the object, for the [recipe] line: the proof path taken + // and, on a refusal, what failed. + enum class ProofPath : std::size_t { Fast, OwnRefreshed, Full, Count }; + enum class ProofFailure : std::size_t { None, Imports, Evicted, Pending, Changed, Keys, Other, Count }; + struct ProofReport { + ProofPath path = ProofPath::Fast; + ProofFailure failure = ProofFailure::None; + }; + bool Revalidate(std::span shaders, ProofReport* report = nullptr); + bool Revalidate(const CompiledShader& shader, ProofReport* report = nullptr) { return Revalidate(std::span(&shader, 1), report); } + // The proof a recipe hit runs on its template (design_cpu_final M4): Revalidate by that name. + bool ProveCurrent(std::span shaders, ProofReport* report = nullptr) { return Revalidate(shaders, report); } + bool ProveCurrent(const CompiledShader& shader, ProofReport* report = nullptr) { return Revalidate(shader, report); } + // FNV over the ShaderData/FlattenedSrt words, in binding order: of this object's data buffers + // as the recorded work will find them (maintained by addDataBuffer and RefreshData), and of a + // compiled shader's descriptors. A recipe hit refreshes the template iff the two differ. + std::uint64_t DataWordsHash() const { return dataWordsHash; } + static std::uint64_t DataWordsHash(const CompiledShader& shader); + // Whether RefreshData would record anything for `shader` (the per-word compare; verification). + bool DataWordsDiffer(const CompiledShader& shader) const; + // Why the fast proof of a Revalidate left the object to the full walk (the [rescache] + // revalidate line's reasons); Count: it did not. + enum class FastFail : std::size_t { NoRecord, Collect, Pending, Evicted, Changed, Keys, ClearedView, StorageKeys, Count }; + // Why a Pending failure was left to the full walk instead of the own-object refresh (T1, see + // refreshOwnObjects); Count: it was not. + enum class OwnRefreshFallback : std::size_t { Disabled, Snapshot, Keys, ForeignView, SurfaceKey, NotImported, Uncached, Rerun, Count }; + // APS5_PROFILE_DRAW: the build's sub-phases ([resources] totals) and their times in + // milliseconds (zero when not profiling), so the draw path can total them separately from + // dispatches. bindingsMs covers the binding plan (stage A) and the image lookups (stage B); + // prepareMs/completeMs are the two stages' totals. + enum class BuildPhase : std::size_t { Bindings, Precollect, Upload, Descriptors, StageA, Images, Bda, StageB, Count }; struct BuildTiming { double bindingsMs = 0; double uploadMs = 0; @@ -165,6 +218,8 @@ private: ShaderRecompiler::DescriptorRole role = ShaderRecompiler::DescriptorRole::ShaderData; // Guest buffers: whether the shader may store to the range (see addGuestBuffer). bool written = true; + // Data buffers: the words the buffer holds once the recorded work ran (see RefreshData). + std::vector dataWords; }; struct Binding { @@ -190,10 +245,11 @@ private: void buildPrepare(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); void buildComplete(); // APS5_PROFILE_DRAW: closes the current sub-phase of the build into the [resources] totals. - double phase(const char* name); + double phase(BuildPhase which); // `written` is the element's DescriptorBinding::bufferWritten: a read-only element binds the - // same way but is left out of the write set (no write-back, no pending-write note). - std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written); + // same way but is left out of the write set (no write-back, no pending-write note). `atomic` + // is its bufferAtomic (see GuestBufferMemory::AddWritable). + std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic); std::size_t addDataBuffer(std::span words); // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache // locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding). @@ -258,7 +314,39 @@ private: bool valid = false; }; void captureValidation(); - bool fastRevalidate(); + // A surface of this object the fast proof found a foreign image pending over: the element + // (index into textures, or into storageTextures when `storage`) whose own object stands in for + // the full walk's lookup (refreshOwnObjects). `viewUncompressed`: a storage-sourced view under + // uncompressed keys, which the lookup serves by its own source whenever FindPending names it. + // `sourceEligible` (set by refreshOwnObjects on a resolved view): the lookup views the cached + // image of the surface's key, i.e. the view's source, when FindPending names nothing. + struct PendingOverlap { + std::size_t element; + bool storage; + bool viewUncompressed; + bool sourceEligible = false; + }; + // `serialBefore` is StorageTexture::PendingSerial() loaded before any check of this Revalidate; + // `refreshed` the surfaces refreshOwnObjects resolved earlier in the same Revalidate (empty on + // the first run): a foreign image still pending over one of them (the own image's alias, an + // image whose pending units lie outside the surface) is no failure, since the lookup's answer + // for the surface is the own object its Refresh just brought current, whatever the registry + // holds; `reason` receives why the proof failed (Count: it passed), `overlapping` the surfaces + // of a Pending failure in element order (empty otherwise), `accepted` whether a foreign overlap + // was accepted by either rule (the verify switch then runs the full walk beside the proof). + bool fastRevalidate(std::uint64_t serialBefore, std::span refreshed, FastFail& reason, std::vector& overlapping, bool& accepted); + // T1 (design_cpu_final M3): brings the overlapping surfaces' own objects up to date exactly as + // the full walk's lookups would, in the walk's binding order, without the lookups: a storage + // image's Refresh (which stores the foreign images over its surface first), a view's source's + // Refresh when the lookup would refresh the cached image of the view's surface key, nothing + // when FindPending names the source itself (the lookup hits on it). Count, or the fallback + // reason: a surface whose lookup could return another object (a snapshot texture, a view under + // fast-clear keys, a foreign pending image the lookup would view, a surface key mapping to + // another cached image, a surface outside the host imports, an object no longer cached), which + // leaves the whole object to the full walk. Under GuestMemory::GpuMutex, as Revalidate is. + OwnRefreshFallback refreshOwnObjects(std::span shaders, std::span overlapping); + // Today's per-element walk (APS5_NO_EPOCH_REVALIDATE=1). + bool fastRevalidateEach(); Context context; std::vector layoutKey; GuestBufferMemory guestMemory; @@ -285,6 +373,16 @@ private: bool reusable = false; std::vector directRegions; std::vector validatedTextures; + // The pending registry's serial at the last Revalidate that proved this object, taken before + // its checks and kept only when unchanged after them (0: none): while it is still the serial, + // no image other than the object's own sources has results pending over its surfaces and + // regions, so those checks are skipped (see fastRevalidate). + std::uint64_t pendingSerialSeen = 0; + // The import table's identity when the direct regions' serials were last proved. + HostImportsProof importsProof; + // FNV-1a offset basis: the hash of no data buffers (DataWordsHash). + std::uint64_t dataWordsHash = 14695981039346656037ull; + void rehashDataWords(); BuildTiming timing; // Build state carried from stage A to stage B: the bindings in plan order, the image bindings // still to look up (index into `bindings`; the DescriptorBinding lives in the compiled shader), @@ -329,8 +427,19 @@ class ResourceCache { public: using Key = std::vector; std::shared_ptr Find(const Key& key); - void Insert(const Key& key, std::shared_ptr resources); - void Remove(const Key& key); + // `evicted`, when given, receives the objects the insert displaces (the entry replaced under the + // key, the ones over the bound) instead of their being destroyed here: a caller under the GPU + // mutex hands them to the recorder so the destruction runs off the lock (VulkanDevice::dispatch). + void Insert(const Key& key, std::shared_ptr resources, std::vector>* evicted = nullptr); + // With `object`, erases the entry only while it still holds that object (a replacement made + // meanwhile by another worker stays). + void Remove(const Key& key, const ShaderResources* object = nullptr); + // Moves the entry to the front of the LRU (a recipe hit uses its template without Find, and a + // hot template must not age out under the recipes that depend on it); false when no entry. + bool Touch(const Key& key); + // Find and Touch calls so far (the [rescache] line: a recipe hit touches instead of finding). + static std::uint64_t Finds(); + static std::uint64_t Touches(); void Clear(); std::size_t Size() const; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp index b20568e1f2..fb84496c98 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include "Recompiler.hpp" namespace AgcDriver::Graphics { @@ -63,6 +64,64 @@ struct State { ShaderStages DecodeShaderStages(const QueueState& queue); State DecodeState(const QueueState& queue); +// The message DecodeState (or the pixel stage decode after it) would throw for the register rules +// this precheck covers, evaluated without exceptions before the draw is decoded; empty when they +// pass (DecodeState still checks everything). A register a rule needs that is absent is no verdict. +std::string DrawRejection(const QueueState& queue); + +// The recording facade of the draw decoders (design_cpu_final M8, step 8a): every register read +// of DecodeShaderStages, DecodeState, DrawRejection, DecodePixelStageInfo and Driver::draw's +// program prepare goes through NoteRegisterRead, which appends the bank and offset to the calling +// thread's log while one is set (null in production; Driver::draw sets one on a miss under +// APS5_VERIFY_DRAW_RECIPE=1 and checks the log against DrawKeyRegisters). +enum class RegisterBank : std::uint8_t { Context, Shader, UserConfig, Count }; +struct RegisterRead { + RegisterBank bank; + std::uint32_t offset; +}; +std::vector*& RegisterReadLog(); +inline void NoteRegisterRead(RegisterBank bank, std::uint32_t offset) { + if (auto* log = RegisterReadLog()) log->push_back({bank, offset}); +} +inline const char* RegisterBankName(RegisterBank bank) { + return bank == RegisterBank::Context ? "context" : bank == RegisterBank::Shader ? "shader" : "user-config"; +} + +// The frozen table of the registers the draw decoders read, as [first, first + count) per bank in +// bank and offset order: the draw key (Driver::draw) hashes every present register of these +// ranges with its offset, so the decoded state, pixel and program inputs are a pure function of +// the key. A range covers whole register groups (the eight CB_COLOR slots, the 32 user words of +// each bank, the interpolator controls) whether or not a draw's slot count or user count reaches +// them: a stricter key, never a wrong one. A register the decoders read that the table lacks would +// be a wrong hit, which the facade's log finds (DrawKeyCovers) on every verified miss. +struct DrawKeyRange { + RegisterBank bank; + std::uint32_t first; + std::uint32_t count; +}; +inline constexpr std::array DrawKeyRegisters{{ + // PA_SC_SCREEN_SCISSOR, the window offset/scissor and clip rect, the edge rule, the hardware + // screen offset, CB_TARGET_MASK/CB_SHADER_MASK, the generic and viewport 0 scissors, the + // viewport 0 depth clamp, the blend constants, the viewport 0 transform. + {RegisterBank::Context, 0x00c, 2}, {RegisterBank::Context, 0x080, 4}, {RegisterBank::Context, 0x08c, 4}, {RegisterBank::Context, 0x090, 2}, {RegisterBank::Context, 0x094, 2}, {RegisterBank::Context, 0x0b4, 2}, {RegisterBank::Context, 0x105, 4}, {RegisterBank::Context, 0x10f, 6}, + // SPI_PS_INPUT_CNTL_0..31, SPI_PS_INPUT_ENA/ADDR, SPI_PS_IN_CONTROL, SPI_SHADER_POS/Z/COL_FORMAT, + // CB_BLEND0..7_CONTROL, GE_MAX_OUTPUT_PER_SUBGROUP. + {RegisterBank::Context, 0x191, 32}, {RegisterBank::Context, 0x1b3, 2}, {RegisterBank::Context, 0x1b6, 1}, {RegisterBank::Context, 0x1c3, 3}, {RegisterBank::Context, 0x1e0, 8}, {RegisterBank::Context, 0x1ff, 1}, + // DB_DEPTH_CONTROL .. PA_CL_VS_OUT_CNTL, PA_SC_MODE_CNTL_0/1, VGT_GS_MODE, VGT_GS_VERT_ITEMSIZE, + // VGT_SHADER_STAGES_EN/GS_ONCHIP, VGT_TF_PARAM/DB_ALPHA_TO_MASK, PA_SC_AA_CONFIG and + // PA_SU_VTX_CNTL, the sample masks, PA_SC_CONSERVATIVE_RASTERIZATION_CNTL. + {RegisterBank::Context, 0x200, 8}, {RegisterBank::Context, 0x292, 2}, {RegisterBank::Context, 0x29b, 1}, {RegisterBank::Context, 0x2ce, 1}, {RegisterBank::Context, 0x2d5, 2}, {RegisterBank::Context, 0x2db, 2}, {RegisterBank::Context, 0x2f8, 2}, {RegisterBank::Context, 0x30e, 2}, {RegisterBank::Context, 0x313, 1}, + // CB_COLOR0..7_BASE .. DCC_BASE (15 words a slot), CB_COLOR0..7_BASE_EXT, DCC_BASE_EXT, ATTRIB2, ATTRIB3. + {RegisterBank::Context, 0x318, 0x78}, {RegisterBank::Context, 0x390, 8}, {RegisterBank::Context, 0x3a8, 0x18}, + // The pixel program address, RSRC2 and user words; the geometry-back user pointer and program + // address; the vertex/geometry-front RSRC1/RSRC2 and user words; the vertex program address; + // the hull user pointer, program address, RSRC2 and user words; the local program address. + {RegisterBank::Shader, 0x008, 0x24}, {RegisterBank::Shader, 0x082, 2}, {RegisterBank::Shader, 0x088, 2}, {RegisterBank::Shader, 0x08a, 0x22}, {RegisterBank::Shader, 0x0c8, 2}, {RegisterBank::Shader, 0x102, 2}, {RegisterBank::Shader, 0x108, 2}, {RegisterBank::Shader, 0x10b, 0x21}, {RegisterBank::Shader, 0x148, 2}, + // GE_PRIM_TYPE, GE_MULTI_PRIM_IB_RESET_EN, the geometry subgroup sizes. + {RegisterBank::UserConfig, 0x242, 1}, {RegisterBank::UserConfig, 0x24b, 1}, {RegisterBank::UserConfig, 0x25b, 1}, +}}; +// Whether DrawKeyRegisters holds the read. +bool DrawKeyCovers(RegisterRead read); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index b1d9f9353d..3e42c0c42e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -6,9 +6,15 @@ #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include +#include +#include #include #include #include +#include +#include #include namespace AgcDriver::Graphics { @@ -16,6 +22,7 @@ namespace AgcDriver::Graphics { class ResidentColor; class CommandBatch; class StorageTexture; +struct HostImport; // The Vulkan format storage images of a guest format use (sRGB formats store as their UNORM form). // Throws when the format has no storage form; the query is cached per format (a lookup cost a @@ -23,6 +30,9 @@ class StorageTexture; VkFormat StorageFormatForGuest(const Context& context, std::uint32_t guestFormat); // Whether StorageFormatForGuest would succeed, without throwing (unknown guest formats included). bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat); +// Whether a storage image of the guest format takes DCC clear `keys` as a GPU clear (see +// StorageTexture::upload); false for integer formats and non-clear keys. +bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, DccKeys keys); // A sampled texture's own VkImage with its memory, shared with the recorder while a recorded upload // still writes it (see the snapshot constructor), so the texture may go before the batch completes. @@ -58,12 +68,19 @@ public: VkDeviceSize AllocationBytes() const { return allocationBytes; } // Whether this texture is a view of a storage image (no snapshot of its own). bool ViewsStorageImage() const { return storageSource != nullptr; } + const StorageTexture* StorageSource() const { return storageSource.get(); } + const std::shared_ptr& SharedStorageSource() const { return storageSource; } + // The last key scan of the sampled surface (ProvedClearKeys), for surfaces whose keys are not + // the source image's own (snapshots, views over other metadata); one per cache entry, so every + // object binding the texture shares it. Under GuestMemory::GpuMutex only. + DccKeyProof& KeyProof() const { return keyProof; } private: void release() noexcept; // Held by value: cached textures outlive the Context of the draw that created them. Context context; + mutable DccKeyProof keyProof; // The snapshot constructor's image (null for a view of a storage image); `owned` frees it. VkImage image = VK_NULL_HANDLE; std::shared_ptr owned; @@ -101,12 +118,130 @@ public: void Flush(); // Stores every pending image overlapping the range, except `except`; returns whether any was. // Stores into host-imported memory are recorded (not waited for): a CPU reader syncs afterwards. - static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access"); + // Then the unit shadows over the range are published into their imports as `scope` says (see + // UnitShadow.hpp; the reason string picks the publish reason), `published` receiving whether + // any unit was: a reader of the import must sync for a publish as for a store. Only images + // with a pending unit inside the range are stored (APS5_NO_FLUSH_PRETEST=1 lists every + // overlapping pending image, as before). + static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access", PublishScope scope = PublishScope::Whole, bool* published = nullptr); static void FlushAllPending(const char* reason); + // See PendingSerial: a change of a surface's source outside the registry (a unit shadow + // retile) moves it too. + static void BumpPendingSerial(); + // Whether an image with results pending, or one a FlushPending is storing right now, overlaps + // any of the [begin, end) ranges: one acquisition of the registry's mutex. + static bool AnyPendingOverlaps(std::span> ranges); + // Moves with every change of the pending registry (an image marked dirty, stored, flushed or + // dropped; after the store for a flush), bumped under the registry's mutex with the change, so + // a reader that loads it before and after its own reads knows the registry it consulted stayed + // as it was, and one that finds it unchanged since an earlier read knows the registry is what + // it was then. + static std::uint64_t PendingSerial(); + // One registry scan for several ranges (the fast Revalidate): each query's `overlaps` receives + // whether an image other than `except` has results pending, or is being stored right now, over + // [begin, end), and `found` the image FindPending would return for the range (null: none); + // returns whether every query's `pending` image, when given, is still that image. + struct PendingQuery { + std::uint64_t begin; + std::uint64_t end; + const StorageTexture* except; + const StorageTexture* pending; + bool overlaps; + const StorageTexture* found = nullptr; + }; + static bool ScanPending(std::span queries); + // APS5_PROFILE_DRAW, for the flush hook's recorded-store attribution ([hooksync]): the pending + // images overlapping a CPU access, whether a store of them would leave every 64 KiB block of the + // access to the CPU (the write-back keep rule: a block of a pending layer stamped since the + // layer's generation, after an uncached collect; a generation-0 layer is stored whole), and how + // many of them no consumer proved current for DeadImagePresents presents. No state changes. + struct AccessClassification { + std::size_t images = 0; + std::size_t dead = 0; + std::size_t live = 0; + bool allCpuWritten = false; + // Accesses over a megabyte are not scanned. + bool checked = false; + }; + static constexpr std::uint64_t DeadImagePresents = 2; + static void ClassifyAccess(std::uint64_t address, std::size_t bytes, AccessClassification& out); + // The flush hook's skip (C4, Recorder.cpp FlushForAccess): whether every image FlushPending + // would store for the access keeps every 64 KiB block of it (the keep rule of writeBackLayers: + // stamped since the layer's generation, decided after an uncached collect over the access), so + // the access reads the same bytes with or without the store and the images stay pending, dirty + // and at their generation, except the layers a store for the access would leave empty-handed + // (evictStale; `evicted` receives the images that left the registry so). False when no image + // overlaps (`images` receives how many do), for an access over ClassifyLimit, or with a + // generation-0 layer (stored whole). The decision is taken under GuestMemory::GpuMutex (a + // short hold, no GPU wait), the collect before it. + static constexpr std::size_t ClassifyLimit = 1u << 20u; + static bool AccessKeptByCpu(std::uint64_t address, std::size_t bytes, std::size_t* images = nullptr, std::size_t* evicted = nullptr); + // FlushPending's publish of the unit shadows over the range without the image stores (what it + // does when it lists no image, and all an access AccessKeptByCpu approved needs): whether any + // unit was published. + static bool PublishShadowsOnly(std::uint64_t address, std::size_t bytes, PublishScope scope, const char* reason = "memory access"); + // For the [hooksync] recorded-store line: images a FlushPending stored after a hook skip of + // theirs (AccessKeptByCpu), of which by the hook for the same read site as the last skip; 10 s deltas. + struct HookSkipCounts { + std::uint64_t flushedAfterSkip; + std::uint64_t flushedAfterSkipSameSite; + }; + static HookSkipCounts TakeHookSkipCounts(); + // A consumer found the image current and used it (Refresh, the fast revalidation). + void NoteProved() const; + // Whether the storage cache holds this image (set and cleared by the cache under its mutex, the + // clear before the eviction's flush): what a lookup of its surface would return. + bool Cached() const { return cached.load(std::memory_order_acquire); } + void SetCached(bool value) { cached.store(value, std::memory_order_release); } // The pending image whose surface starts at `address` and covers at least `bytes` (a mip chain // contains a descriptor of its first mips), if any; the caller decides whether the geometry fits // (Texture::CanCopyFrom, ResidentPresentable). static std::shared_ptr FindPending(std::uint64_t address, std::uint64_t bytes); + // How a buffer fill of [address, address + bytes) (Driver.cpp's fill HLE) meets the storage + // images alive: none of them; a whole surface (Exact) or one array layer of a surface (Layer), + // `image` and `layer` naming it, whatever other images lie over the range (`others`, of which + // `inside` wholly inside it: stale images of earlier uses of the memory, which the title's + // transient allocator hands out again); otherwise part of one image (the fill inside the + // surface, the surface inside the fill, or straddling), several, or none but exactly one + // surface's DCC metadata. Images the cache let go (see Flush) do not count. + enum class FillCover { None, Exact, Inside, Around, Straddle, Several, Keys, Layer }; + struct FillCoverage { + FillCover cover = FillCover::None; + std::shared_ptr image; + std::uint32_t layer = 0; + std::size_t others = 0; + std::size_t inside = 0; + }; + static FillCoverage ClassifyFill(std::uint64_t address, std::size_t bytes); + // Results pending in images lying wholly inside the range are dead (a fill overwrites every + // byte of them): they are dropped instead of stored. Returns how many images were. + static std::size_t DiscardPendingInside(std::uint64_t address, std::size_t bytes); + // A fill of the whole surface (`layer` WholeImage) or of one array layer with the 16-byte + // `pattern`, done as a GPU clear of the image instead of a store into guest memory: the + // pattern must be one texel of the storage format repeated, exactly representable as a clear + // value, the surface's DCC keys (if any) must read as uncompressed and not be written by + // recorded work, the memory must be host-imported so the write-back that eventually publishes + // the texels is GPU-direct, and for one layer the rest of the surface must be unchanged since + // the image matched it. Records into the active recorder under GuestMemory::GpuMutex and leaves + // the image dirty like a shader write: the guest bytes follow through the deferred write-back, + // the range is stamped written so every other reader of it (other images over the memory, + // captures, snapshots) takes the bytes through the flush hook, and this image alone is current. + // False, naming why in `refusal`, when the fill must be stored as before. + static constexpr std::uint32_t WholeImage = ~0u; + bool FillClear(std::span pattern, std::uint32_t layer, const char*& refusal); + // The cached, live image whose surface is exactly [address, address + bytes), if any (the newest + // of several: see ClassifyFill). + static std::shared_ptr FindLive(std::uint64_t address, std::uint64_t bytes); + // Whether the two surfaces differ in nothing but their address (and DCC metadata). + bool SameSurfaceShape(const StorageTexture& other) const; + // The copy HLE's buffer copy of one whole surface into another (VulkanDevice::CopyBuffer, + // APS5_NO_COPY_ALIAS=1 disables): the source image's content is copied on the device into this + // image, which is then dirty like a shader-written one (the guest bytes follow through the + // deferred write-back; the range is stamped so every other reader takes them through the flush + // hook), and the source's pending results stay pending. The shapes must match, neither surface + // may carry DCC metadata, and this surface must be host-imported (the write-back is GPU-direct). + // False, naming why, when the copy must be a transfer. + bool CopyFrom(StorageTexture& source, const char*& refusal); VkImage Image() const { return image; } const GuestTextureResource& Descriptor() const { return descriptor; } std::uint32_t ImageLayers() const { return geometry.imageLayers; } @@ -116,6 +251,11 @@ public: // validates textures copied from this image. std::uint64_t Version() const { return version; } std::uint64_t Generation() const { return generation; } + // The DCC keys the content was uploaded under and the last scan of the surface's keys + // (ProvedClearKeys): Refresh's own key rule for the fast revalidation. Under + // GuestMemory::GpuMutex only, as Refresh is; never from a build's stage A. + DccKeys UploadedKeys() const { return uploadedKeys; } + DccKeyProof& KeyProof() const { return keyProof; } // Brings the image up to date with guest memory before another use; returns whether its content // was still current (nothing uploaded). // Keeps the image current with guest memory (see GuestMemory::CollectWrites). @@ -123,24 +263,105 @@ public: std::uint64_t GuestBytes() const; private: - std::vector CopyRegions() const; - void upload(); - // Stores the image to guest memory; with `skipWrittenSince` set, 64 KiB blocks the CPU wrote since - // that write generation keep the CPU's bytes. - void writeBack(std::uint64_t skipWrittenSince); - // Pending images whose surface shares a 64 KiB write-stamp block with this one without - // overlapping it, and whose memory is unchanged since their generation (see writeBack), with - // the generation seen at the check. + // The regions of every array layer, or of the tracked layers `layers` selects. + std::vector CopyRegions(const std::vector* layers = nullptr) const; + // Uploads the surface, or only the tracked layers `layers` selects (the direct path; the others + // upload everything). + void upload(const std::vector* layers = nullptr); + // Stores the pending tracked layers overlapping [address, address + bytes) to guest memory; in + // each, 64 KiB blocks the CPU wrote since the layer's generation keep the CPU's bytes. Block + // units asked for in pieces too often are all stored at once for a while (the hysteresis: + // APS5_BLOCK_WRITEBACK_PIECES within APS5_BLOCK_WRITEBACK_FRAMES presents, defaults 4 and 8; + // APS5_BLOCK_WRITEBACK_EACH=1 stores the touched units only). + void writeBack(std::uint64_t address, std::size_t bytes); + void writeBackLayers(const std::vector& layers); + // Tracked units as 64 KiB write-stamp blocks (`blockUnits`: a thin tiled surface at a 64 KiB + // aligned base; APS5_NO_BLOCK_TRACKING=1 tracks array layers as above instead): a fill of one + // layer, a CPU write or another image's store then costs the blocks it touched, moved through + // windows of their (layer, mip) slices (whole tile blocks in whole rows, see + // TextureDetiler::DetileWindow), not the surface. The last unit may be short. + std::uint64_t layerBytes(std::uint32_t layer) const { return std::min(trackedLayerBytes, guestBytes - static_cast(layer) * trackedLayerBytes); } + // Contiguous runs of the selected units, surface-relative [begin, end). + std::vector> unitRuns(const std::vector& units) const; + struct SliceWindow { + std::uint32_t layer; + std::uint32_t level; + // The window's tiled bytes, surface-relative, and the detiler window (mip-relative) that + // moves them out of a linear window of `linearBytes`. + std::uint64_t tiledBegin; + std::uint64_t tiledEnd; + DetileWindow window; + std::uint64_t linearBytes; + // Its tile blocks as buffer-image regions, bufferOffset relative to the linear window. + std::vector regions; + }; + // The windows moving the surface-relative runs: whole tile blocks of thin tiled mips, a tail + // mip whole (its block holds every tail level), in slice order. + std::vector sliceWindows(std::span> runs) const; + // The direct upload and GPU-direct write-back of the runs through their windows (block units); + // both return the bytes moved. The upload reads each unit from its unit shadow while fresh, + // else from the import (`discard`: the image has no layout yet, a whole-surface first upload). + // The write-back retiles into the import's unit shadow where a slab takes the piece + // (`shadowed` receives those guest ranges, `imported` the pieces written to the import). + std::uint64_t uploadWindows(const HostImport& import, std::span> runs, bool discard = false); + std::uint64_t writeBackWindows(const HostImport& import, std::span> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector& shadowed, std::vector>& imported); + // Per-layer validity (thin array surfaces whose layers are 64 KiB-aligned guest slices; every + // other surface is one tracked layer): a layer's generation is the write generation its guest + // bytes were last known to match the image at, and a pending layer holds results (a clear, a + // shader write, a device copy) guest memory has not received. `generation` stays the oldest + // layer generation (the whole-surface fast check), `dirty` whether any layer is pending. + std::uint64_t layerBegin(std::uint32_t layer) const { return descriptor.baseAddress + static_cast(layer) * trackedLayerBytes; } + bool anyLayerPending() const; + void refreshGeneration(); + // Marks `count` tracked layers from `first` pending and registers the image (MarkDirty's + // registration; APS5_EAGER_WRITEBACK=1 stores at once instead). + void markLayersPending(std::uint32_t first, std::uint32_t count); + // Registers or unregisters the image as its pending layers say (after a write-back). + void reconcilePending(); + // Pending images whose surface shares one of the 64 KiB write-stamp blocks firstBlock..lastBlock + // (the span a write-back stamps) without overlapping this one: the tracked layer of theirs + // holding the shared block, when its memory is unchanged since its generation (see writeBack), + // with the generation seen at the check. struct Adjacent { std::shared_ptr texture; + std::uint32_t layer; std::uint64_t generation; }; - std::vector adjacentPendingUnchanged() const; - // Advances the entries still at their seen generation to `now`, the generation this write-back's - // stamps of blocks firstBlock..lastBlock (64 KiB) are below, unless the neighbour's other blocks - // were stamped since the check. + std::vector adjacentPendingUnchanged(std::uint64_t firstBlock, std::uint64_t lastBlock) const; + // Advances the entries' layers still at their seen generation to `now`, the generation this + // write-back's stamps of blocks firstBlock..lastBlock are below, unless the layer's other + // blocks were stamped since the check. static void advanceAdjacent(const std::vector& adjacent, std::uint64_t now, std::uint64_t firstBlock, std::uint64_t lastBlock); + // Per 64 KiB tracker block of the surface (block 0 holds the base address): the generation of + // the tracked layer it belongs to, for GuestMemory::ChangedBlocks. + void blockGenerations(std::vector& generations) const; + // Another image of exactly this surface under another storage format with results pending + // (block units, uncompressed, still cached), if any: Refresh takes its pending units from its + // image on the device (borrowUnits) instead of having them stored and re-read, and the alias + // stays responsible for storing them until this image's own results supersede them + // (markLayersPending). APS5_NO_ALIAS_BORROW=1 stores and re-reads as before. + std::shared_ptr pendingAlias() const; + // The pending images FlushPending(address, bytes) would store (not the one a Refresh validates, + // a pending unit inside the range), kept alive across the caller's use. + static std::vector> overlappingPending(std::uint64_t address, std::size_t bytes); + // Whether every 64 KiB block of [address, address + bytes) inside a pending layer of the images + // is stamped since that layer's generation (the caller collected the range first): the keep + // rule of writeBackLayers over the access. A generation-0 layer is stored whole: false. + static bool blocksKept(std::span> images, std::uint64_t address, std::size_t bytes); + // Under GuestMemory::GpuMutex, after blocksKept approved the access: the layers a write-back for + // it would select (see writeBack; every pending one of an image no consumer proved current for + // DeadImagePresents presents with APS5_STALE_EVICT=1, H3) are left as that write-back would + // leave them when every one of their blocks is stamped since its generation (it would store + // nothing: no results pending, the generation unchanged, `original` no longer vouching). + // Returns whether they were; `dropped` counts the image once it left the registry. + bool evictStale(std::uint64_t address, std::size_t bytes, std::size_t& dropped); + // Whether the results the last hook skip left pending over its access (AccessKeptByCpu) are + // still the pending ones and a write-back for the range would store them. + bool skippedResultsInside(std::uint64_t address, std::size_t bytes) const; + std::uint64_t borrowUnits(StorageTexture& source, const std::vector& units); + void forgetBorrowed(std::uint32_t first, std::uint32_t count); bool overlaps(std::uint64_t address, std::size_t bytes) const; + bool pendingUnitInside(std::uint64_t address, std::size_t bytes) const; VkImageView createView(std::uint32_t mip) const; void release() noexcept; @@ -155,8 +376,27 @@ private: std::vector original; // DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value. DccKeys uploadedKeys = DccKeys::Uncompressed; - // Write generation `original` is known current at. + mutable DccKeyProof keyProof; + // Write generation `original` is known current at (the oldest of layerGeneration). std::uint64_t generation = 0; + std::uint32_t trackedLayers = 1; + std::uint64_t trackedLayerBytes = 0; + std::vector layerGeneration; + std::vector layerPending; + bool blockUnits = false; + // Write-back hysteresis (block units, see writeBack): partial stores in the current window of + // presents, its first present, and the present until which every pending unit is stored. + std::uint32_t partialStores = 0; + std::uint64_t partialWindowStart = 0; + std::uint64_t storeWholeUntil = 0; + // Units holding the alias's results (see pendingAlias), at the alias's version then. + std::weak_ptr borrowedFrom; + std::uint64_t borrowedVersion = 0; + std::vector borrowedUnits; + // Units of this image were borrowed since it was last written: the borrower's proofs + // (the resource cache's serial memo) hold only while this image's version does, so the next + // write here bumps the pending serial although the image was pending already. + bool lent = false; VkImage image = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; VkImageView view = VK_NULL_HANDLE; @@ -170,8 +410,44 @@ private: std::uint64_t version = 0; // `original` holds the guest bytes; false after a GPU-side clear, which never read them. bool originalValid = true; + // The storage cache let the image go (Flush): a lookup makes a new image of the surface, so + // no fill may put results into this one (guarded by the live-image registry lock). + bool released = false; + std::atomic cached{false}; + // The present a consumer last proved the image current at (NoteProved). + mutable std::atomic provedPresent{0}; + // Hook skips of this image since FlushPending last listed it (AccessKeptByCpu), the read site + // of the last one (a GuestMemory::ReadSite), and the first pending layer under its access with + // the version and layer generation seen: a listing counts a flush after a skip only while that + // layer is pending with those results (skippedResultsInside), for TakeHookSkipCounts. + std::atomic hookSkips{0}; + std::atomic hookSkipSite{0}; + std::uint32_t hookSkipLayer = 0; + std::uint64_t hookSkipVersion = 0; + std::uint64_t hookSkipGeneration = 0; + // The present evictStale last walked this image's layers beyond the access (once per present). + std::uint64_t staleCheckPresent = ~0ull; + // The generation at which a one-layer FillClear was refused as 'surface changed': the check + // fails again until `generation` moves, so ClassifyFill skips the layer match meanwhile. + std::uint64_t layerRefusedGeneration = ~0ull; }; +// APS5_PROFILE_DRAW: what the image lookups on this thread did since the last reset, by outcome +// (count and time), so a caller running them under the GPU mutex can name what its hold spent on +// (VulkanDevice's [indirect] line resets and reads it around stage B). The sampled rows are per +// element (a fast miss is followed by a full lookup); the refresh, upload, DCC scan and pending +// flush rows lie inside the storage and sampled rows. +struct LookupOutcomes { + enum Kind : std::size_t { SampledFast, SampledFastMiss, SampledHitView, SampledHitClearedView, SampledHitSnapshot, SampledMadeView, SampledMadeSnapshot, StorageHit, StorageMade, RefreshUnchanged, RefreshCompared, UploadDirect, UploadCpu, UploadClear, DccScan, PendingFlush, Count }; + static const char* Name(Kind kind); + static bool Profiled(); + // Charges the time since `start` to `kind` on this thread and returns now. + static std::chrono::steady_clock::time_point Add(Kind kind, std::chrono::steady_clock::time_point start); + std::array counts{}; + std::array ms{}; +}; +LookupOutcomes& ThreadLookupOutcomes(); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp index 9436f2597f..7ffb40425c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp @@ -11,6 +11,26 @@ namespace AgcDriver::Graphics { + // The part of a mip one Dispatch moves: only elements whose tiled offset (relative to the mip's + // tiled base; the linear offset of a linear surface) lies in [rangeBegin, rangeEnd), with the + // tiled buffer holding the mip from tiledBase and the linear buffer holding `linearBytes` of + // it from linearBase (a window of whole rows: linearBase is a row's start, every element in + // the range lies inside the window; 0 for the rest of the mip). The element rectangle + // [columnBegin, columnEnd) x [rowBegin, rowEnd) (an end of 0: the mip's edge) bounds the + // elements the range can hold: only its workgroups are dispatched, the range check stays the + // guard. The default moves the whole mip out of whole buffers. + struct DetileWindow { + std::uint32_t rangeBegin = 0; + std::uint32_t rangeEnd = 0xffffffffu; + std::uint32_t tiledBase = 0; + std::uint32_t linearBase = 0; + std::uint64_t linearBytes = 0; + std::uint32_t columnBegin = 0; + std::uint32_t columnEnd = 0; + std::uint32_t rowBegin = 0; + std::uint32_t rowEnd = 0; + }; + class TextureDetiler { public: explicit TextureDetiler(const Context& context); @@ -19,8 +39,10 @@ namespace AgcDriver::Graphics { TextureDetiler& operator=(const TextureDetiler&) = delete; // Detiles `source` (tiled) into `destination` (linear), or with `retile` writes linear `source` - // into tiled `destination`; offsets always refer to the respective buffers. - void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false); + // into tiled `destination`; offsets always refer to the respective buffers, and `window` + // restricts the move to part of the mip (a partial upload or write-back, StorageTexture), + // leaving the other elements of both buffers alone. + void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false, const DetileWindow& window = {}); // Recycles the descriptor sets of the previous batch; call before recording a new command batch. void BeginBatch(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp index 3372142915..aad8c0ddc2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp @@ -29,6 +29,10 @@ std::vector ComputeElementMipLayout(TextureTileMode tileMode, std // Thick (3D) SW_4KB_S / SW_64KB_S blocks span several depth slices: {width, height, depth} in elements. std::array ThickBlockExtent(TextureTileMode tileMode, std::uint32_t bytesPerElement); +// Thin (2D) tiled blocks: {block bytes, width, height} in elements for the tile mode and element size +// (the detiler's blockExtent). A mip's tiled bytes are its blocks in row-major order, blocksPerRow per +// row (TileMipLayout), except the tail mips, which share one block. Throws for linear tiling. +std::array ThinBlockLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement); // Single-mip 3D surface. Each depth slice z is detiled from slab z / blockDepth (slabBytes apart) with the // swizzle's slice input set to z, into linear slices sliceLinearBytes apart. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp new file mode 100644 index 0000000000..b69b285fca --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp @@ -0,0 +1,155 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_UNITSHADOW_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_UNITSHADOW_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +struct HostImport; + +// Device-local tiled shadows of host imports, indexed by 64 KiB unit (one unit = one write-tracker +// block; unit index of guest address a inside a shadow = (a - base) / 65536 with base = the import's +// base rounded down to 64 KiB). A storage image's GPU-direct write-back retiles its pending units into +// the shadow instead of the import, and a later upload of any image over those units detiles from the +// shadow: the title's aliased images then exchange their results in device memory, and only a host- +// facing consumer of the bytes (the flush hook, a V# bound in place, indirect arguments, a copy, a +// label or key store over the range, scanout, the import's retirement) publishes them into the +// import with one vkCmdCopyBuffer per slab. +// +// Freshness is derived from the write tracker, never stored: a unit is shadowed ("fresh") while its +// slab bytes were written at a tracker generation no block stamp exceeds (MarkShadowed takes the +// generation the write-back settled at, after its own MarkWritten); every import writer already +// stamps its blocks (labels, fills, copies, key stores, in-place V# writes, CPU stores), so a stale +// unit reads from the import as today. A CPU store stamps only when a collect walks its page, so a +// publish and a seed decision collect the units they judge first (as the write-back's keep decision +// does): the CPU's bytes win the unit. A publish records the copy and marks the unit published (the +// slab and the import agree until the next retile or stamp); a stale unit met by a publish is dropped +// ("stale-dropped": its generation is cleared). A publish never stamps. +// +// Slabs are 8 MiB pieces of a shadow (APS5_UNIT_SHADOW_SLAB_MIB) with their own VkDeviceMemory, +// made on the first retile into them under a budget (APS5_UNIT_SHADOW_MIB, default 1024, LRU +// eviction with a publish of the evicted slab's fresh units). APS5_NO_UNIT_SHADOW=1 makes every +// primitive inert. Every call but AnyShadowedOverlaps runs under GuestMemory::GpuMutex (they record +// or mutate freshness); the registry's own mutex is a leaf (lock order GpuMutex -> Imports/Pending -> +// Shadows -> tracker). +bool UnitShadowEnabled(); + +// None: a device-facing caller (a storage upload reads the slab itself); Whole: a reader of every +// byte of the range; PartialUnits: a writer of the range (only units it covers partly are published, +// so the neighbours' results in the rest of the unit reach the import before the write stamps it). +enum class PublishScope : std::uint8_t { None, Whole, PartialUnits }; +enum class PublishReason : std::uint8_t { Hook, Region, Indirect, CopySource, CopyDestination, Fill, FillClear, Label, Keys, Scanout, Validate, Upload, TailMip, Evict, Retire, Teardown, Count }; +// The reason for a FlushPending reason string ("memory access" -> Hook, ...). +PublishReason PublishReasonFor(const char* flushReason); + +struct ShadowSlab { + ShadowSlab(const Context& context, VkBuffer buffer, VkDeviceMemory memory, std::uint64_t firstUnit, std::uint32_t units); + ShadowSlab(const ShadowSlab&) = delete; + ShadowSlab& operator=(const ShadowSlab&) = delete; + // Destroys the handles: safe on the recorder's release thread (Keep's contract). + ~ShadowSlab(); + Context context; + VkBuffer buffer; + VkDeviceMemory memory; + std::uint64_t firstUnit; + std::uint32_t units; + std::uint64_t lastUse = 0; + // Destinations handed out for a write-back in progress: such a slab is never evicted. + std::atomic pins{0}; +}; + +// Holds one pin of a slab (ShadowDestination::pin). +struct ShadowSlabPin { + explicit ShadowSlabPin(std::shared_ptr slab); + ShadowSlabPin(const ShadowSlabPin&) = delete; + ShadowSlabPin& operator=(const ShadowSlabPin&) = delete; + ~ShadowSlabPin(); + std::shared_ptr slab; +}; + +// A retile piece (guest range) and the slab its copies were recorded into. +struct ShadowedRange { + std::uint64_t begin; + std::uint64_t end; + std::shared_ptr slab; +}; + +// Records vkCmdCopyBuffer(slab -> import) for every fresh unpublished unit the scope selects over +// [address, address + bytes) into the active recorder's open batch (a waited CommandBatch without +// one), with the barriers, the pending-write note and the timing class of a fill; marks them +// published, clears stale ones. Returns the units copied. Inside a completion action a store's +// publish records nothing (the store's own stamp makes the unit stale; a recorded publish would land +// over the store later). +std::size_t PublishShadow(std::uint64_t address, std::size_t bytes, PublishScope scope, PublishReason reason); +// No GpuMutex, no tracker: conservative (a unit with a generation and no publish counts), for +// hook-free readers and a build's stage A. +bool AnyShadowedOverlaps(std::uint64_t address, std::size_t bytes); +bool AnyShadowedOverlaps(std::span> ranges); + +// The sources of a storage upload's surface-relative runs: [begin, end) pieces of them, each read +// from one buffer (a slab while the unit is fresh, else the import) at `offset` (byte x of the run +// is at offset + (x - begin)), split at unit freshness and slab boundaries with one tracker query +// over the span. A tail block (surface-relative, every unit of it) must have one source: a mixed +// one is published (TailMip) and read from the import. +struct ShadowRun { + std::uint64_t begin; + std::uint64_t end; + VkBuffer buffer; + VkDeviceSize offset; + bool shadow; + std::shared_ptr slab; +}; +std::vector ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span> runs, std::span> tailBlocks); + +// The slab destination of a retile copy [begin, end) (guest addresses inside one slab: the caller +// splits at SlabBoundary), making the slab under the budget; nullopt when refused (the caller writes +// the import as before). `seedUnits` names the units the copy covers partly that are not fresh (as +// [begin, end) clipped to the import, collected first): the caller records import -> slab for each +// before its copy. The slab stays pinned against eviction while a copy of the destination lives: +// the caller keeps one until MarkShadowed (an evicted slab would take the retile's copies with it). +struct ShadowDestination { + VkBuffer buffer; + VkDeviceSize offset; + std::shared_ptr slab; + std::vector> seedUnits; + std::shared_ptr pin; +}; +std::optional ShadowDestinationFor(const Context& context, const HostImport& import, std::uint64_t begin, std::uint64_t end); +// A seed copy (import -> slab of a partly covered unit) the caller recorded: counted and traced. +void NoteShadowSeed(std::uint64_t begin, std::uint64_t end); +// The first slab boundary above `address` (a guest address inside the import). +std::uint64_t SlabBoundary(const HostImport& import, std::uint64_t address); +// The slab offset of a guest address for a slab returned by ShadowDestinationFor or ShadowSources. +VkDeviceSize SlabOffset(const HostImport& import, const ShadowSlab& slab, std::uint64_t address); +// After a retile's copies were recorded: the units of `ranges` (guest) hold the newest bytes at +// `generation`, unpublished. A unit whose slab is no longer the one the copies went into is not +// marked (counted `slab lost`). Bumps StorageTexture's pending serial. +void MarkShadowed(const HostImport& import, std::span ranges, std::uint64_t generation); +// retireImport: publishes the fresh units whose memory `registered` still names as a readable +// registered range into the (old) import buffer, hands the slabs to the batch and drops the shadow. +// The import retires because its registration vanished or changed size, so the rest of its memory +// belongs to the title again and is dropped (counted `dropped on retire`); `lost on retire` counts +// the units nothing could record. +void RetireShadow(const Context& context, const HostImport& import, const std::function& registered); +// Teardown: every shadow of the context's device is published; DestroyShadows then drops them. +void PublishAllShadows(const Context& context, PublishReason reason); +void DestroyShadows(VkDevice device); +// The [storage] line's shadow segment (10 s deltas, live slab totals). +std::string ShadowReport(); +// APS5_SHADOW_VERIFY=1: after a shadow-sourced upload, the units read must carry no stamp newer +// than their generation (counted as mismatches otherwise). +bool ShadowVerify(); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp index 30cc45c4a4..65afec2a4e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp @@ -110,6 +110,19 @@ inline VertexInputLayout BuildVertexInputLayout(const Context& context, std::spa return result; } +// The whole byte range a vertex buffer descriptor covers (records * stride, or records when the +// stride is 0): what an indirect draw, whose counts only the GPU knows, copies for the fetch. +inline std::size_t VertexBufferExtent(const ShaderRecompiler::VertexAttribute& attribute) { + const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu; + const auto records = attribute.resource.fields[2]; + Require(attribute.fetchIndex <= 1, "unsupported vertex fetch index"); + const auto bytes = stride == 0 ? static_cast(records) : static_cast(records) * stride; + Require(bytes != 0 && bytes <= std::numeric_limits::max(), "empty or oversized vertex buffer descriptor"); + const auto address = attribute.resource.fields[0] | (static_cast(attribute.resource.fields[1] & 0xffffu) << 32u); + Require(address != 0 && bytes <= std::numeric_limits::max() - address, "invalid vertex buffer address range"); + return static_cast(bytes); +} + inline std::size_t VertexBufferReadSize(const ShaderRecompiler::VertexAttribute& attribute, std::uint32_t maxIndex, std::uint32_t instances, std::uint32_t firstInstance = 0) { Require(instances != 0, "vertex input requires nonzero instance count"); Require(firstInstance <= std::numeric_limits::max() - (instances - 1u), "vertex input instance range overflow"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp index 912df386d2..38b17aeae2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp @@ -44,6 +44,17 @@ layout(push_constant) uniform Push { uint tailY; uint elementBytes; uint slice; + // Only elements whose tiled offset (the linear offset for kLinear) lies in [rangeBegin, + // rangeEnd) are moved; tiledBase and linearBase are subtracted from the tiled and linear + // offsets of those elements (a window of the mip in each buffer; both 0 for the whole mip). + uint rangeBegin; + uint rangeEnd; + uint tiledBase; + uint linearBase; + // The first element column and row of the dispatched grid (a window's rectangle; 0 for the + // whole mip). + uint columnBegin; + uint rowBegin; } params; uint equationBit(uint mask, uvec2 p) { @@ -145,7 +156,7 @@ void copyElement(uint src, uint dst) { } void main() { - uvec2 p = gl_GlobalInvocationID.xy; + uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin); if (p.x >= params.width || p.y >= params.height) { return; } @@ -155,7 +166,14 @@ void main() { if (TILE_FAMILY == 0u) { // kLinear: guest and host layouts match; a straight copy at the // same pitch, no swizzle. - copyElement(params.srcBase + linearOffset, params.dstBase + linearOffset); + if (linearOffset < params.rangeBegin || linearOffset >= params.rangeEnd) { + return; + } + if (RETILE != 0u) { + copyElement(params.srcBase + linearOffset - params.linearBase, params.dstBase + linearOffset - params.tiledBase); + } else { + copyElement(params.srcBase + linearOffset - params.tiledBase, params.dstBase + linearOffset - params.linearBase); + } return; } @@ -169,6 +187,11 @@ void main() { uint blockIndex = block.y * params.blocksPerRow + block.x; uint tiledOffset = blockIndex * BLOCK_BYTES + blockOffset(swizzle); + if (tiledOffset < params.rangeBegin || tiledOffset >= params.rangeEnd) { + return; + } + tiledOffset -= params.tiledBase; + linearOffset -= params.linearBase; if (RETILE != 0u) { copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index f9978ce2c0..1b8b035252 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -29,10 +29,13 @@ namespace { // copies). Hence several recent tables, most recently used first, matched by a word hash before // the byte compare. APS5_BDA_TABLE_CACHE_ENTRIES sets how many (default 8; 1 is the previous // behaviour); APS5_NO_BDA_TABLE_CACHE=1 builds every table. +// A table built for (or found equal to) the cached address space's ranges carries the space's +// serial: a build served by that space alone binds it by the serial, with no hash and no compare. struct TableEntry { std::uint64_t hash; std::vector ranges; std::weak_ptr buffer; + std::uint64_t spaceSerial = 0; }; struct TableCache { @@ -41,8 +44,38 @@ struct TableCache { std::list entries; std::uint64_t hits = 0; std::uint64_t misses = 0; + BdaResources::TableCacheStats classes; }; +bool sameRanges(const std::vector& left, const std::vector& right); + +// APS5_PROFILE_DRAW: why the most recently used entry did not serve this build (see +// TableCacheStats). Under the cache mutex. +void classifyFirstEntry(TableCache& cache, std::uint64_t hash, const std::vector& ranges) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& classes = cache.classes; + if (cache.entries.empty()) { + ++classes.firstEmpty; + return; + } + const auto& first = cache.entries.front(); + if (first.hash == hash) { + if (first.buffer.expired()) ++classes.firstExpired; + else if (!sameRanges(first.ranges, ranges)) ++classes.firstSameHash; + return; + } + if (first.buffer.expired()) { + ++classes.firstExpired; + return; + } + const auto common = std::min(first.ranges.size(), ranges.size()); + std::size_t at = 0; + while (at < common && std::memcmp(&first.ranges[at], &ranges[at], sizeof(ranges[at])) == 0) ++at; + const auto begin = at < ranges.size() ? ranges[at].begin : at < first.ranges.size() ? first.ranges[at].begin : 0; + ++(begin < 0x10000000ull ? classes.firstDiffersLow : classes.firstDiffersHeap); +} + TableCache& Tables() { static TableCache cache; return cache; @@ -93,13 +126,17 @@ BdaResources::BdaResources(const Context& context) { } BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memory) : BdaResources(context) { - auto ranges = memory.AddressRanges(); + const auto cached = memory.CachedAddressTable(); + std::vector built; + if (!cached.has_value()) built = memory.AddressRanges(); + const auto& ranges = cached.has_value() ? *cached->ranges : built; + const auto serial = cached.has_value() ? cached->serial : 0; Require(ranges.size() <= std::numeric_limits::max(), "BDA table range count overflow"); Require(ranges.size() <= (std::numeric_limits::max() - sizeof(ShaderRecompiler::BdaAbi::Header)) / sizeof(ShaderRecompiler::BdaAbi::Range), "BDA table size overflow"); tableBytes = sizeof(ShaderRecompiler::BdaAbi::Header) + ranges.size() * sizeof(ShaderRecompiler::BdaAbi::Range); Require(tableBytes <= context.limits.maxStorageBufferRange && sizeof(ShaderRecompiler::BdaAbi::Fault) <= context.limits.maxStorageBufferRange, "BDA descriptors exceed storage buffer range limit"); auto& cache = Tables(); - const auto hash = tableCacheEnabled() ? hashRanges(ranges) : 0; + std::uint64_t hash = 0; if (tableCacheEnabled()) { std::lock_guard lock(cache.mutex); // Another device's tables are not this one's (their buffers are dead or foreign). @@ -107,6 +144,22 @@ BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memo cache.entries.clear(); cache.device = context.device; } + if (serial != 0) { + for (auto it = cache.entries.begin(); it != cache.entries.end(); ++it) { + if (it->spaceSerial != serial) continue; + if (auto shared = it->buffer.lock(); shared != nullptr) { + ++cache.hits; + ++cache.classes.spaceTables; + table = std::move(shared); + cache.entries.splice(cache.entries.begin(), cache.entries, it); + return; + } + cache.entries.erase(it); + break; + } + } + hash = hashRanges(ranges); + classifyFirstEntry(cache, hash, ranges); for (auto it = cache.entries.begin(); it != cache.entries.end();) { // Compare the hash before taking a strong reference: locking a non-matching entry's // buffer could make this thread its last owner (a concurrent reap releasing it) and @@ -127,6 +180,9 @@ BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memo } if (sameRanges(it->ranges, ranges)) { ++cache.hits; + // An equal table of an earlier space (a rebuild that mapped the same ranges the + // same way): the next build of this space binds it by the serial. + if (serial != 0) it->spaceSerial = serial; table = std::move(shared); cache.entries.splice(cache.entries.begin(), cache.entries, it); return; @@ -145,7 +201,7 @@ BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memo cache.entries.clear(); cache.device = context.device; } - cache.entries.push_front({hash, std::move(ranges), table}); + cache.entries.push_front({hash, cached.has_value() ? ranges : std::move(built), table, serial}); while (cache.entries.size() > tableCacheEntries()) cache.entries.pop_back(); } } @@ -153,7 +209,11 @@ BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memo BdaResources::TableCacheStats BdaResources::TableCacheCounters() { auto& cache = Tables(); std::lock_guard lock(cache.mutex); - return {cache.hits, cache.misses, cache.entries.size()}; + auto stats = cache.classes; + stats.hits = cache.hits; + stats.misses = cache.misses; + stats.held = cache.entries.size(); + return stats; } VkDescriptorBufferInfo BdaResources::Table() const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index b3896cc464..a24303db32 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -20,11 +20,21 @@ bool sharedTiers() { BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")) { smallTier.budget = smallBudget; largeTier.budget = budget; + deviceTier.budget = DeviceBudget(); } BufferPool::~BufferPool() { for (const auto& slot : smallTier.free) destroy(slot.allocation); for (const auto& slot : largeTier.free) destroy(slot.allocation); + for (const auto& slot : deviceTier.free) destroy(slot.allocation); +} + +VkDeviceSize BufferPool::DeviceBudget() { + static const VkDeviceSize deviceBudget = [] { + const char* value = std::getenv("APS5_STAGING_POOL_MIB"); + return (value != nullptr ? std::strtoull(value, nullptr, 10) : 512ull) << 20u; + }(); + return deviceBudget; } void BufferPool::destroy(const BufferAllocation& allocation) noexcept { @@ -41,7 +51,8 @@ std::size_t BufferPool::Capacity(std::size_t bytes) { return std::max(smallest, std::bit_ceil(bytes)); } -BufferPool::Tier& BufferPool::tierFor(std::size_t capacity) { +BufferPool::Tier& BufferPool::tierFor(std::size_t capacity, VkMemoryPropertyFlags properties) { + if ((properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 && (properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0 && DeviceBudget() != 0) return deviceTier; return !sharedTiers() && capacity < classLimit ? smallTier : largeTier; } @@ -55,10 +66,10 @@ std::optional BufferPool::Take(std::size_t bytes, VkBufferUsag const auto now = std::chrono::steady_clock::now(); if (now - lastReport > std::chrono::seconds(10)) { lastReport = now; - std::fprintf(stderr, "[bufferpool] small: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB); large: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB)%s\n", static_cast(smallTier.hits), static_cast(smallTier.misses), static_cast(smallTier.evictions), smallTier.free.size(), smallTier.retainedBytes / 1048576.0, static_cast(largeTier.hits), static_cast(largeTier.misses), static_cast(largeTier.evictions), largeTier.free.size(), largeTier.retainedBytes / 1048576.0, sharedTiers() ? " (shared)" : ""); + std::fprintf(stderr, "[bufferpool] small: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB); large: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB)%s; device: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB of %llu)\n", static_cast(smallTier.hits), static_cast(smallTier.misses), static_cast(smallTier.evictions), smallTier.free.size(), smallTier.retainedBytes / 1048576.0, static_cast(largeTier.hits), static_cast(largeTier.misses), static_cast(largeTier.evictions), largeTier.free.size(), largeTier.retainedBytes / 1048576.0, sharedTiers() ? " (shared)" : "", static_cast(deviceTier.hits), static_cast(deviceTier.misses), static_cast(deviceTier.evictions), deviceTier.free.size(), deviceTier.retainedBytes / 1048576.0, static_cast(deviceTier.budget >> 20u)); } } - auto& tier = tierFor(capacity); + auto& tier = tierFor(capacity, properties); for (auto it = tier.free.begin(); it != tier.free.end(); ++it) { const auto& allocation = it->allocation; if (allocation.bytes != capacity || allocation.usage != usage || allocation.properties != properties) continue; @@ -100,7 +111,7 @@ void BufferPool::Put(const BufferAllocation& allocation) noexcept { std::vector evicted; try { std::lock_guard lock(mutex); - auto& tier = tierFor(allocation.bytes); + auto& tier = tierFor(allocation.bytes, allocation.properties); if (allocation.allocationBytes > tier.budget) { evicted.push_back(allocation); } else { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp index 4693139ac0..a7ba917bab 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" #include #include #include @@ -62,6 +63,18 @@ ScanProfile& Scans() { return profile; } +// ProvedClearKeys outcomes, counted with or without profiling (the tests read them). +struct ProofCounters { + std::atomic proved{0}; + std::atomic scanned{0}; + std::atomic unstable{0}; +}; + +ProofCounters& Proofs() { + static ProofCounters counters; + return counters; +} + bool ScanProfileEnabled() { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; return profile; @@ -77,7 +90,8 @@ void ReportScans(std::chrono::steady_clock::time_point now) { return; } if (nowMs - last < 10000 || !profile.lastReport.compare_exchange_strong(last, nowMs)) return; - std::fprintf(stderr, "[dcc] %llu scans, %.1f MiB scanned, %llu memo hits, %llu flush syncs, %.1f ms; uncompressed keys stored: %llu on the GPU (%llu with an unaligned head or tail), %llu on the CPU\n", static_cast(profile.scans.load()), profile.bytes.load() / 1048576.0, static_cast(profile.memoHits.load()), static_cast(profile.flushSyncs.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.gpuStores.load()), static_cast(profile.gpuSplitStores.load()), static_cast(profile.cpuStores.load())); + const auto& proofs = Proofs(); + std::fprintf(stderr, "[dcc] %llu scans, %.1f MiB scanned, %llu memo hits, %llu flush syncs, %.1f ms; uncompressed keys stored: %llu on the GPU (%llu with an unaligned head or tail), %llu on the CPU; key proofs: %llu proved, %llu scanned, %llu unstable\n", static_cast(profile.scans.load()), profile.bytes.load() / 1048576.0, static_cast(profile.memoHits.load()), static_cast(profile.flushSyncs.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.gpuStores.load()), static_cast(profile.gpuSplitStores.load()), static_cast(profile.cpuStores.load()), static_cast(proofs.proved.load()), static_cast(proofs.scanned.load()), static_cast(proofs.unstable.load())); } void CountScan(std::size_t bytes, std::chrono::steady_clock::time_point start) { @@ -172,6 +186,14 @@ std::optional MemoizedKeys(std::uint64_t begin, std::uint64_t end) { return std::nullopt; } +// Debug aid (APS5_TRACE_DCC_KEYS=1): every uncompressed key store, with the packet that made it. +void TraceKeyStore(const char* path, std::uint64_t begin, std::size_t count) { + static const bool trace = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; + if (!trace) return; + const auto packet = GuestMemory::CurrentPacket(); + std::fprintf(stderr, "[dcc-keys] uncompressed keys stored on the %s: 0x%llx+0x%zx (packet 0x%x queue 0x%x)\n", path, static_cast(begin), count, packet.opcode, packet.queue); +} + // The 0xff keys as a fill recorded into the active recorder's open batch, into the range's host // import (the WriteLabelOnGpu / FillBuffer pattern): ordered behind every earlier recorded read or // write of the range (the title's DCC clear or decompress kernel storing keys through a V#, which the @@ -193,20 +215,14 @@ bool StoreUncompressedOnGpu(const Context& context, Recorder& recorder, std::uin if (fillBegin > first || last > fillEnd) { seed = std::make_shared(context, 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); std::memset(seed->Bytes().data(), 0xff, seed->Bytes().size()); - recorder.Keep(seed); - } - const auto commands = recorder.Commands(); - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); - if (fillEnd > fillBegin) context.Function("vkCmdFillBuffer")(commands, import->buffer, fillBegin, fillEnd - fillBegin, 0xffffffffu); - if (seed != nullptr) { - VkBufferCopy copies[2]; - std::uint32_t copyCount = 0; - if (fillBegin > first) copies[copyCount++] = {0, first, fillBegin - first}; - if (last > fillEnd) copies[copyCount++] = {0, fillEnd, last - fillEnd}; - context.Function("vkCmdCopyBuffer")(commands, seed->Handle(), import->buffer, copyCount, copies); CountStore(Scans().gpuSplitStores); } - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT); + // A unit shadow's results in the key range's edge units reach the import before the fill + // lands over part of them (the stamp below then makes those units stale). + if (AnyShadowedOverlaps(begin, count)) PublishShadow(begin, count, PublishScope::PartialUnits, PublishReason::Keys); + // Queued on the open batch and recorded with the batch's other key stores as one run (or before + // a later command writing the keys; see Recorder::QueueKeyStore). + recorder.QueueKeyStore(import->buffer, first, last, std::move(seed), begin, begin + count); recorder.NotePendingWrite(begin, count); GuestMemory::MarkWritten(begin, count); // After the note: an unlocked memo lookup drops an entry the snapshot does not cover yet. The @@ -219,10 +235,12 @@ bool StoreUncompressedOnGpu(const Context& context, Recorder& recorder, std::uin if (MemoRecorder(memo) == &recorder) memo.entries.push_back({begin, end}); } CountStore(Scans().gpuStores); + TraceKeyStore("gpu", begin, count); return true; } void StoreUncompressedOnCpu(std::uint64_t begin, std::size_t count) { + TraceKeyStore("cpu", begin, count); // The write waits for recorded work over the range (the flush hook), so the bytes are current after. ForgetStores(begin, begin + count); const std::vector uncompressed(count, std::byte{0xff}); @@ -321,11 +339,33 @@ const char* DccKeysName(DccKeys keys) { return "?"; } -DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { +namespace { + +// ReadDccKeys, saying in `memoized` whether the answer came from the pending-store memo rather +// than the bytes. +DccKeys readDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, bool& memoized) { + memoized = false; const auto count = static_cast(surfaceBytes / KeyBytes); if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count)) return DccKeys::Unreadable; // A driver store still pending on the GPU is not in the bytes yet. - if (const auto memoized = MemoizedKeys(metaAddress, metaAddress + count)) return *memoized; + if (const auto memo = MemoizedKeys(metaAddress, metaAddress + count)) { + memoized = true; + return *memo; + } + // Keys under a unit shadow's fresh results (metadata aliased with a surface): under the + // device lock they are published and waited for; a build's unlocked stage answers "unstable" + // so its stage B reads them under the lock. + if (AnyShadowedOverlaps(metaAddress, count)) { + if (!GuestMemory::GpuMutex().HeldByThisThread()) { + memoized = true; + return DccKeys::Mixed; + } + PublishShadow(metaAddress, count, PublishScope::Whole, PublishReason::Keys); + if (auto* recorder = Recorder::Active()) { + Recorder::CountSync(2); + recorder->Sync(); + } + } const auto* keys = reinterpret_cast(metaAddress); const auto first = keys[0]; const auto start = ScanProfileEnabled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -343,6 +383,29 @@ DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { } } +DccKeys textureClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes, bool& memoized) { + memoized = false; + if (resource.dccAddress == 0) return DccKeys::Uncompressed; + const auto keys = readDccKeys(resource.dccAddress, guestBytes, memoized); + if (keys == DccKeys::Uncompressed) return keys; + std::byte probe[16]{}; + if (!IsDccClear(keys) || !FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min(sizeof(probe), BytesPerElement(resource.format))))) { + static std::mutex reportedMutex; + static std::set> reported; + std::lock_guard lock(reportedMutex); + if (reported.size() < 32 && reported.insert({resource.baseAddress, static_cast(keys)}).second) std::fprintf(stderr, "[gpu] texture 0x%llx (format %u) has %s DCC keys at 0x%llx; its texels are read as stored\n", static_cast(resource.baseAddress), resource.format, DccKeysName(keys), static_cast(resource.dccAddress)); + return DccKeys::Uncompressed; + } + return keys; +} + +} + +DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + bool memoized = false; + return readDccKeys(metaAddress, surfaceBytes, memoized); +} + bool IsDccClear(DccKeys keys) { return keys == DccKeys::Clear0000 || keys == DccKeys::Clear0001 || keys == DccKeys::Clear1110 || keys == DccKeys::Clear1111 || keys == DccKeys::ClearRegister; } @@ -409,20 +472,56 @@ bool DccAlphaOnMsb(VkFormat format, std::uint32_t componentSwap) { } DccKeys TextureClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes) { + bool memoized = false; + return textureClearKeys(resource, guestBytes, memoized); +} + +bool KeyFastPath() { + static const bool enabled = std::getenv("APS5_NO_KEY_FAST_PATH") == nullptr; + return enabled; +} + +DccKeys ProvedClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes, DccKeyProof& proof) { if (resource.dccAddress == 0) return DccKeys::Uncompressed; - const auto keys = ReadDccKeys(resource.dccAddress, guestBytes); - if (keys == DccKeys::Uncompressed) return keys; - std::byte probe[16]{}; - if (!IsDccClear(keys) || !FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min(sizeof(probe), BytesPerElement(resource.format))))) { - static std::mutex reportedMutex; - static std::set> reported; - std::lock_guard lock(reportedMutex); - if (reported.size() < 32 && reported.insert({resource.baseAddress, static_cast(keys)}).second) std::fprintf(stderr, "[gpu] texture 0x%llx (format %u) has %s DCC keys at 0x%llx; its texels are read as stored\n", static_cast(resource.baseAddress), resource.format, DccKeysName(keys), static_cast(resource.dccAddress)); - return DccKeys::Uncompressed; + auto& counters = Proofs(); + if (!KeyFastPath()) { + counters.scanned.fetch_add(1, std::memory_order_relaxed); + return TextureClearKeys(resource, guestBytes); } + const auto count = static_cast(guestBytes / KeyBytes); + // The collect precedes the scan: a write landing between them is stamped above `collected`, so + // the proof it would invalidate is never taken as current. + const auto collected = count == 0 ? 0 : GuestMemory::CollectWrites(resource.dccAddress, count); + if (proof.generation != 0 && collected != 0 && GuestMemory::UnchangedSince(resource.dccAddress, count, proof.generation)) { + counters.proved.fetch_add(1, std::memory_order_relaxed); + return proof.keys; + } + bool memoized = false; + const auto keys = textureClearKeys(resource, guestBytes, memoized); + bool stable = collected != 0 && !memoized; + if (stable) { + // After the scan: a write noted since the collect is stamped above it anyway; one noted + // before it may still land after the bytes were read. + if (GuestMemory::GpuMutex().HeldByThisThread()) { + if (auto* recorder = Recorder::Active(); recorder != nullptr) { + const auto info = recorder->DescribePendingWrite(resource.dccAddress, count); + stable = !info.has_value() || info->signaled; + } + } else { + stable = !Recorder::SnapshotWriteOverlaps(resource.dccAddress, count); + } + } + proof = stable ? DccKeyProof{keys, collected} : DccKeyProof{}; + counters.scanned.fetch_add(1, std::memory_order_relaxed); + if (!stable) counters.unstable.fetch_add(1, std::memory_order_relaxed); return keys; } +DccKeyProofCounts KeyProofCounts() { + const auto& counters = Proofs(); + return {counters.proved.load(std::memory_order_relaxed), counters.scanned.load(std::memory_order_relaxed), counters.unstable.load(std::memory_order_relaxed)}; +} + void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes) { // Named for the [hooksync] attribution: the read goes through the flush hook. const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureRead); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index f73afaee7e..ed00f454a2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -13,7 +13,9 @@ #include "Optimization/include/Optimization/ShaderStageInputInfo.hpp" #include #include +#include #include +#include #include #include #include @@ -86,12 +88,12 @@ void imageBarrier(const Context& context, VkCommandBuffer commands, VkImage imag barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; barrier.image = image; barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 0, nullptr, 0, nullptr, 1, &barrier); + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 0, nullptr, 0, nullptr, 1, &barrier); } void memoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { const VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, sourceAccess, destinationAccess}; - context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); } // The color surface as one untailed SW_64KB_R_X mip for the GPU detiler, with tightly packed linear rows. @@ -117,6 +119,8 @@ constexpr std::array DrawPhaseNames{"validate", "vertex // releases an address-based build's lease, which the CPU must not outrun (see `recorded` in Draw). enum SyncReason : std::size_t { SyncNone, SyncNotResident, SyncNoRecorder, SyncDisabled, SyncCopiedWrites, SyncLease, SyncCount }; constexpr std::array SyncReasonNames{"none", "non-resident target", "no recorder", "disabled", "copied writes", "lease"}; +constexpr std::size_t IndirectPathCount = static_cast(IndirectDrawPath::Count); +constexpr std::array IndirectDrawPathNames{"gpu-side", "patched SGPR not folded", "fetch offset unknown", "non-vertex path", "GE_INDX_OFFSET", "draw index", "vertex range too large", "feature gap", "pending image results", "pending label or copied write", "not imported", "disabled"}; // What became of one draw, for the totals. struct DrawOutcome { @@ -124,6 +128,9 @@ struct DrawOutcome { bool recorded = false; bool waited = false; bool completion = false; + // A recorded draw began a render pass of its own, or continued the previous draw's. + bool passBegun = false; + bool passContinued = false; SyncReason reason = SyncNone; // Shader validation memo (see CachedFragmentOutputs). bool validateMemoized = false; @@ -139,6 +146,14 @@ struct DrawOutcome { std::uint64_t targetLookups = 0; std::uint64_t slowLookups = 0; double slowLookupUs = 0; + // The lookups by whether the draw's scissor covers the whole target (an area-scoped proof + // would change nothing for those), the pages their write-watch collects walked and their time. + std::uint64_t fullScissorLookups = 0; + std::uint64_t partialLookups = 0; + std::uint64_t pagesWalked = 0; + double lookupUs = 0; + // How the draw's resources came about (DrawKind), for the per-kind averages. + std::size_t kind = 0; // The flush hook's fence waits made inside this draw (index and vertex reads, the target // lookups' flushes, the build's texture lookups): nested in the draw's GpuMutex hold, so they // are the part of the hold that is waiting rather than working. The draw's own syncs (a @@ -148,8 +163,19 @@ struct DrawOutcome { double ownSyncUs = 0; }; +// The resources of a draw: a recipe hit (reserved for the draw recipe step), a resource-cache +// template hit, a build, or an address-based (BDA) build. +enum DrawKind : std::size_t { KindRecipeHit, KindTemplateHit, KindBuild, KindBda, KindCount }; +constexpr std::array DrawKindNames{"recipe hit", "template hit", "build", "BDA"}; + struct DrawProfile { std::mutex mutex; + std::array kindUs{}; + std::array kindCounts{}; + std::uint64_t fullScissorLookups = 0; + std::uint64_t partialLookups = 0; + std::uint64_t pagesWalked = 0; + double lookupUs = 0; std::array totalsUs{}; // The longest single draw's time per phase, and the longest draw: the [lock] line's 'draw' // hold max (tens of ms against an average well under a millisecond) needs a phase name. @@ -174,6 +200,9 @@ struct DrawProfile { // Recorded draws whose write-back (BDA fault check) runs as a completion action instead of // making the draw synchronous (see APS5_SYNC_COMPLETION_DRAWS). std::uint64_t completion = 0; + // Render passes recorded draws began, and draws that continued the previous draw's pass. + std::uint64_t passesBegun = 0; + std::uint64_t passesContinued = 0; std::array reasons{}; // The CPU inside Graphics::Draw, split by outcome: a synchronous draw costs ten times a recorded // one, so one average would only show the synchronous population. @@ -194,8 +223,20 @@ struct DrawProfile { // Shader validation memo (see CachedFragmentOutputs). std::uint64_t validateHits = 0; std::uint64_t validateMisses = 0; + // Draw recipe outcomes (DrawWithRecipe): hits, and misses by DrawRecipeMiss. + std::uint64_t recipeHits = 0; + std::array(DrawRecipeMiss::Count)> recipeMisses{}; + // Indirect draws by path (IndirectDrawPath), the GPU-side ones whose records were rewritten + // with a constant, and the time of the CPU record reads (their syncs). + std::array indirect{}; + std::uint64_t indirectRewritten = 0; + double indirectReadUs = 0; + // Draw packets that drew nothing, by DrawSkip, and their time. + std::array(DrawSkip::Count)> skips{}; + std::array(DrawSkip::Count)> skipUs{}; std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; +constexpr std::array(DrawSkip::Count)> DrawSkipNames{"nothing to draw", "prechecked", "thrown"}; DrawProfile& Profile() { static DrawProfile profile; @@ -230,19 +271,29 @@ void reportDraw(const std::array& us, const ShaderResources: ++profile.draws; if (outcome.recorded) ++(outcome.waited ? profile.waited : profile.recorded); if (outcome.completion) ++profile.completion; + if (outcome.passBegun) ++profile.passesBegun; + if (outcome.passContinued) ++profile.passesContinued; ++profile.reasons[outcome.reason]; (outcome.recorded ? (outcome.waited ? profile.waitedUs : profile.recordedUs) : profile.synchronousUs) += drawUs; profile.targetLookups += outcome.targetLookups; profile.slowLookups += outcome.slowLookups; profile.slowLookupUs += outcome.slowLookupUs; + profile.fullScissorLookups += outcome.fullScissorLookups; + profile.partialLookups += outcome.partialLookups; + profile.pagesWalked += outcome.pagesWalked; + profile.lookupUs += outcome.lookupUs; + if (outcome.kind < KindCount) { + ++profile.kindCounts[outcome.kind]; + profile.kindUs[outcome.kind] += drawUs; + } if (outcome.validateMemoized) ++(outcome.validateHit ? profile.validateHits : profile.validateMisses); const auto now = std::chrono::steady_clock::now(); if (now - profile.lastReport < std::chrono::seconds(10)) return; profile.lastReport = now; const auto synchronous = profile.draws - profile.recorded - profile.waited; const auto average = [](double total, std::uint64_t count) { return count != 0 ? total / static_cast(count) : 0.0; }; - char line[1536]; - int n = std::snprintf(line, sizeof(line), "[draws] %llu draws over 10 s (%llu recorded avg %.0f us, of them %llu with completion; %llu recorded then waited avg %.0f us; %llu synchronous avg %.0f us; waited or synchronous because:", static_cast(profile.draws), static_cast(profile.recorded), average(profile.recordedUs, profile.recorded), static_cast(profile.completion), static_cast(profile.waited), average(profile.waitedUs, profile.waited), static_cast(synchronous), average(profile.synchronousUs, synchronous)); + char line[2048]; + int n = std::snprintf(line, sizeof(line), "[draws] %llu draws over 10 s (%llu recorded avg %.0f us, of them %llu with completion; %llu recorded then waited avg %.0f us; %llu synchronous avg %.0f us; render passes %llu begun, %llu draws continued one; waited or synchronous because:", static_cast(profile.draws), static_cast(profile.recorded), average(profile.recordedUs, profile.recorded), static_cast(profile.completion), static_cast(profile.waited), average(profile.waitedUs, profile.waited), static_cast(synchronous), average(profile.synchronousUs, synchronous), static_cast(profile.passesBegun), static_cast(profile.passesContinued)); const auto room = [&] { return n > 0 && static_cast(n) < sizeof(line); }; for (std::size_t i = SyncNone + 1; i < SyncCount && room(); ++i) { if (profile.reasons[i] != 0) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %llu", SyncReasonNames[i], static_cast(profile.reasons[i])); @@ -252,7 +303,11 @@ void reportDraw(const std::array& us, const ShaderResources: if (profile.totalsUs[i] <= 0) continue; n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s=%.1fms", DrawPhaseNames[i], profile.totalsUs[i] / 1000.0); if (i == PhaseResources && room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " (bindings %.1f, upload %.1f, descriptors %.1f, other %.1f of which %.1f in %llu address-based builds)", profile.bindingsUs / 1000.0, profile.uploadUs / 1000.0, profile.descriptorsUs / 1000.0, profile.otherUs / 1000.0, profile.addressOtherUs / 1000.0, static_cast(profile.addressBuilds)); - if (i == PhaseReadTarget && room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " (%llu resident lookups, %llu of them >= 1 ms = %.1f)", static_cast(profile.targetLookups), static_cast(profile.slowLookups), profile.slowLookupUs / 1000.0); + if (i == PhaseReadTarget && room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " (%llu resident lookups, %llu of them >= 1 ms = %.1f; target lookups: full-scissor %llu / partial %llu / pages walked %llu / %.0f us)", static_cast(profile.targetLookups), static_cast(profile.slowLookups), profile.slowLookupUs / 1000.0, static_cast(profile.fullScissorLookups), static_cast(profile.partialLookups), static_cast(profile.pagesWalked), profile.lookupUs); + } + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; per kind (avg us, count):"); + for (std::size_t kind = 0; kind < KindCount && room(); ++kind) { + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %.0f (%llu)", DrawKindNames[kind], average(profile.kindUs[kind], profile.kindCounts[kind]), static_cast(profile.kindCounts[kind])); } // The longest draw and the longest single phase of any draw (which phase a long 'draw' hold // was), plus the hook waits nested inside the draws. @@ -262,6 +317,23 @@ void reportDraw(const std::array& us, const ShaderResources: n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %.1f", DrawPhaseNames[i], profile.maxUs[i] / 1000.0); } if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; hook waits inside draws %.1f ms (max %.1f; the draws' own syncs, %.1f ms, not counted)", profile.hookWaitUs / 1000.0, profile.maxHookWaitUs / 1000.0, profile.ownSyncUs / 1000.0); + std::uint64_t indirectCpu = 0; + for (std::size_t i = 1; i < IndirectPathCount; ++i) indirectCpu += profile.indirect[i]; + if ((profile.indirect[0] != 0 || indirectCpu != 0) && room()) { + n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; indirect: gpu-side %llu (%llu rewritten), cpu-side %llu (", static_cast(profile.indirect[0]), static_cast(profile.indirectRewritten), static_cast(indirectCpu)); + for (std::size_t i = 1; i < IndirectPathCount && room(); ++i) { + if (profile.indirect[i] != 0) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %llu", IndirectDrawPathNames[i], static_cast(profile.indirect[i])); + } + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "), argument reads %.1f ms", profile.indirectReadUs / 1000.0); + } + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; skipped packets:"); + for (std::size_t i = 0; i < profile.skips.size() && room(); ++i) { + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %llu in %.1f ms", DrawSkipNames[i], static_cast(profile.skips[i]), profile.skipUs[i] / 1000.0); + } + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; recipe hits %llu, misses by reason:", static_cast(profile.recipeHits)); + for (std::size_t i = 1; i < profile.recipeMisses.size() && room(); ++i) { + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %llu", DrawRecipeMissName(static_cast(i)), static_cast(profile.recipeMisses[i])); + } std::fprintf(stderr, "%s\n", line); std::fprintf(stderr, "[rescache] draws: %llu hits, %llu misses, %llu invalidated, %llu uncacheable; validation memo %llu hits / %llu misses (which key words the misses differ in: the miss churn line)\n", static_cast(profile.cacheHits), static_cast(profile.cacheMisses), static_cast(profile.cacheInvalidated), static_cast(profile.uncacheable), static_cast(profile.validateHits), static_cast(profile.validateMisses)); profile.totalsUs.fill(0); @@ -270,12 +342,24 @@ void reportDraw(const std::array& us, const ShaderResources: profile.bindingsUs = profile.uploadUs = profile.descriptorsUs = profile.otherUs = profile.addressOtherUs = 0; profile.addressBuilds = 0; profile.draws = profile.recorded = profile.waited = profile.completion = 0; + profile.passesBegun = profile.passesContinued = 0; profile.reasons.fill(0); profile.recordedUs = profile.waitedUs = profile.synchronousUs = 0; profile.targetLookups = profile.slowLookups = 0; profile.slowLookupUs = 0; + profile.fullScissorLookups = profile.partialLookups = profile.pagesWalked = 0; + profile.lookupUs = 0; + profile.kindUs.fill(0); + profile.kindCounts.fill(0); profile.cacheHits = profile.cacheMisses = profile.cacheInvalidated = profile.uncacheable = 0; profile.validateHits = profile.validateMisses = 0; + profile.recipeHits = 0; + profile.recipeMisses.fill(0); + profile.indirect.fill(0); + profile.indirectRewritten = 0; + profile.indirectReadUs = 0; + profile.skips.fill(0); + profile.skipUs.fill(0); } void countCache(std::uint64_t DrawProfile::*counter) { @@ -371,7 +455,7 @@ std::set CachedFragmentOutputs(const Context& context, std::span< return found->second; } } - auto outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric); + auto outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing); if (keyed) { std::lock_guard lock(mutex); // A handful of configurations recur; a runaway key space is dropped wholesale. @@ -443,62 +527,126 @@ std::shared_ptr>> DrawCopiedWriters return *writers; } -void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { - PerformanceTimer timing("Graphics.Draw"); - // APS5_PROFILE_DRAW prints the time of each phase of the draw (microseconds) and the [draws] totals. +const char* IndirectDrawPathName(IndirectDrawPath path) { + return IndirectDrawPathNames[static_cast(path)]; +} + +void CountIndirectDraw(IndirectDrawPath path, double readMs, bool rewritten) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - auto phaseStart = std::chrono::steady_clock::now(); + if (!profile) return; + auto& stats = Profile(); + std::lock_guard lock(stats.mutex); + ++stats.indirect[static_cast(path)]; + if (rewritten) ++stats.indirectRewritten; + stats.indirectReadUs += readMs * 1000.0; +} + +void CountDrawSkip(DrawSkip kind, double us) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& stats = Profile(); + std::lock_guard lock(stats.mutex); + ++stats.skips[static_cast(kind)]; + stats.skipUs[static_cast(kind)] += us; +} + + +bool DrawRecipes() { + static const bool noDrawRecipe = std::getenv("APS5_NO_DRAW_RECIPE") != nullptr; + return !noDrawRecipe; +} + +const char* DrawRecipeMissName(DrawRecipeMiss miss) { + constexpr std::array(DrawRecipeMiss::Count)> names{"none", "not recordable", "target gone", "template gone", "objects gone", "proof"}; + return names[static_cast(miss)]; +} + +namespace { + +// APS5_PROFILE_DRAW: the per-phase timers of one draw (DrawPhase), shared by Draw's parts and +// DrawWithRecipe. +struct DrawTimer { + bool profile; + std::chrono::steady_clock::time_point phaseStart; std::array us{}; - const auto phase = [&](DrawPhase which) { + explicit DrawTimer(bool profile) : profile(profile), phaseStart(std::chrono::steady_clock::now()) {} + void phase(DrawPhase which) { if (!profile) return; const auto now = std::chrono::steady_clock::now(); us[which] += std::chrono::duration(now - phaseStart).count(); phaseStart = now; - }; - // APS5_TRACE_DRAWS (together with APS5_PROFILE_DRAW; alone it only enables Driver.cpp's own - // [draw] lines) additionally prints one line per draw with its phases and, for synchronous - // draws, their inputs: per-draw string building at ~1400 draws per 10 s is itself a cost the - // totals must not carry. + } +}; + +// One line per draw with the phases that took time (APS5_TRACE_DRAWS), and the 10 s totals. +// `waitedBefore` is the thread's GPU waits at the draw's start and `ownWaitedMs` the draw's own +// syncs (see DrawOutcome::hookWaitUs). +void reportDrawEnd(const State& state, const DrawTimer& timer, const ShaderResources::BuildTiming* built, DrawOutcome& outcome, double waitedBefore, double ownWaitedMs, const char* suffix) { + if (!timer.profile) return; static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; - DrawOutcome outcome; - const ShaderResources::BuildTiming* built = nullptr; - // The thread's GPU waits so far: the difference at the report, less the draw's own syncs - // (`ownWaitedMs`, sampled around them below), is what this draw waited for through the flush - // hook (nested in the draw's hold, see DrawOutcome::hookWaitUs). - const auto waitedBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; - double ownWaitedMs = 0; - // One line per draw with the phases that took time (tracing), and the 10 s totals. - const auto report = [&](const char* suffix) { - if (!profile) return; - outcome.hookWaitUs = std::max(0.0, Recorder::ThreadWaitedMs() - waitedBefore - ownWaitedMs) * 1000.0; - outcome.ownSyncUs = ownWaitedMs * 1000.0; - if (traceDraws) { - char line[512]; - int n = std::snprintf(line, sizeof(line), "[draw] %ux%u %zu targets%s:", state.renderExtent.width, state.renderExtent.height, state.colors.size(), suffix); - for (std::size_t i = 0; i < PhaseCount && n > 0 && static_cast(n) < sizeof(line); ++i) { - if (us[i] <= 0) continue; - n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s=%.0fus", DrawPhaseNames[i], us[i]); - } - std::fprintf(stderr, "%s\n", line); + outcome.hookWaitUs = std::max(0.0, Recorder::ThreadWaitedMs() - waitedBefore - ownWaitedMs) * 1000.0; + outcome.ownSyncUs = ownWaitedMs * 1000.0; + if (traceDraws) { + char line[512]; + int n = std::snprintf(line, sizeof(line), "[draw] %ux%u %zu targets%s:", state.renderExtent.width, state.renderExtent.height, state.colors.size(), suffix); + for (std::size_t i = 0; i < PhaseCount && n > 0 && static_cast(n) < sizeof(line); ++i) { + if (timer.us[i] <= 0) continue; + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s=%.0fus", DrawPhaseNames[i], timer.us[i]); } - reportDraw(us, built, outcome); - }; + std::fprintf(stderr, "%s\n", line); + } + reportDraw(timer.us, built, outcome); +} + +// The draw's inputs before its resources (prepareDrawInputs): the validated parameters, the index +// buffer copy with its highest index, the vertex buffer copies and their layout, the fragment +// outputs and the pipeline stages. +struct DrawInputs { + // An empty auto draw: nothing to record. + bool nothing = false; + std::uint64_t indexBytes = 0; + std::unique_ptr indices; + std::uint32_t maxIndex = 0; + VertexInputLayout vertexInput; + std::vector> vertexBuffers; + std::vector vertexHandles; + std::vector vertexOffsets; + std::set fragmentOutputs; + VkPipelineStageFlags shaderStages = 0; + std::uint32_t meshGroups = 0; +}; + +// Draw's validation, index and vertex phases. With `recipe` the fragment outputs, the pipeline +// stages and the vertex input layout are the recipe's (derived from the same compiled stages) +// instead of computed. +DrawInputs prepareDrawInputs(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, DrawOutcome& outcome, DrawTimer& timer, const DrawRecipe* recipe) { + DrawInputs inputs; APS5_LOG_OUT_DEBUG("Draw indices=%u instances=%u indexSize=%u flags=%u indexAddress=0x%llx", draw.indexCount, draw.instanceCount, draw.indexSize, draw.flags, static_cast(draw.indexAddress)); APS5_LOG_OUT_DEBUG("State colorTarget=%u render=%ux%u colorAddress=0x%llx colorBytes=%llu colorExtent=%ux%u", state.hasColorTarget ? 1u : 0u, state.renderExtent.width, state.renderExtent.height, static_cast(state.color.address), static_cast(state.color.bytes), state.color.extent.width, state.color.extent.height); APS5_LOG_OUT_DEBUG("Viewport x=%f y=%f w=%f h=%f minDepth=%f maxDepth=%f", state.viewport.x, state.viewport.y, state.viewport.width, state.viewport.height, state.viewport.minDepth, state.viewport.maxDepth); APS5_LOG_OUT_DEBUG("Scissor x=%d y=%d w=%u h=%u topology=%u cullMode=0x%x frontFace=%u", state.scissor.offset.x, state.scissor.offset.y, state.scissor.extent.width, state.scissor.extent.height, static_cast(state.topology), static_cast(state.cullMode), static_cast(state.frontFace)); + // An indirect draw: the counts live in guest memory records (Pm4::DrawParameters::IndirectDraw), + // read by the GPU from the host import or, when the GPU could not see their current bytes, by + // the CPU (Draw's `records`). The driver resolved every non-vertex-path draw before this. + const auto* args = draw.indirect ? &*draw.indirect : nullptr; Require(draw.indexed ? draw.flags == 0 : (draw.flags & ~0x20u) == 0, "draw modifiers are unsupported"); if (draw.indexed) { Require(draw.indexSize == 2 || draw.indexSize == 4, "only uint16 and uint32 index buffers are supported"); - Require(draw.firstVertex == 0 && draw.firstInstance == 0, "indexed draw offsets are unsupported"); } else { Require(draw.indexAddress == 0 && draw.indexSize == 0, "auto draw must not reference an index buffer"); - if (draw.indexCount == 0 || draw.instanceCount == 0) return; - Require(draw.firstVertex <= std::numeric_limits::max() - (draw.indexCount - 1u), "auto draw vertex range overflow"); - Require(draw.firstInstance <= std::numeric_limits::max() - (draw.instanceCount - 1u), "auto draw instance range overflow"); + if (args == nullptr) { + if (draw.indexCount == 0 || draw.instanceCount == 0) { + inputs.nothing = true; + return inputs; + } + Require(draw.firstVertex <= std::numeric_limits::max() - (draw.indexCount - 1u), "auto draw vertex range overflow"); + Require(draw.firstInstance <= std::numeric_limits::max() - (draw.instanceCount - 1u), "auto draw instance range overflow"); + } } - Require(draw.indexCount != 0 && draw.instanceCount != 0, "zero-count indexed draws are unsupported"); - const auto indexBytes = static_cast(draw.indexCount) * draw.indexSize; + if (args == nullptr) Require(draw.indexCount != 0 && draw.instanceCount != 0, "zero-count indexed draws are unsupported"); + else Require(!state.stages.mesh && !state.stages.tessellation && !state.rectList, "indirect draw on a non-vertex path must be resolved by the driver"); + inputs.indexBytes = static_cast(draw.indexCount) * draw.indexSize; + const auto indexBytes = inputs.indexBytes; APS5_LOG_OUT_DEBUG("Index buffer bytes=%llu", static_cast(indexBytes)); Require(indexBytes <= std::numeric_limits::max(), "index buffer size overflow"); if (draw.indexed) GuestMemory::CheckRange(reinterpret_cast(draw.indexAddress), static_cast(indexBytes), draw.indexSize); @@ -506,11 +654,15 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& Require(!draw.indexed || !state.hasColorTarget || draw.indexAddress + indexBytes <= state.color.address || state.color.address + state.color.bytes <= draw.indexAddress, "index buffer aliases the render target"); if (state.rectList) Require(draw.indexCount % 3 == 0, "incomplete rect-list primitive"); APS5_LOG_CHARS_OUT_DEBUG("ValidateShaders"); - const auto fragmentOutputs = CachedFragmentOutputs(context, shaders, state, outcome.validateMemoized, outcome.validateHit); + if (recipe != nullptr) { + inputs.fragmentOutputs = recipe->fragmentOutputs; + inputs.shaderStages = recipe->shaderStages; + } else { + inputs.fragmentOutputs = CachedFragmentOutputs(context, shaders, state, outcome.validateMemoized, outcome.validateHit); + inputs.shaderStages = PipelineStages(shaders); + } APS5_LOG_CHARS_OUT_DEBUG("ValidateShaders OK"); - const auto shaderStages = PipelineStages(shaders); - APS5_LOG_OUT_DEBUG("PipelineStages=0x%x", static_cast(shaderStages)); - std::uint32_t meshGroups = 0; + APS5_LOG_OUT_DEBUG("PipelineStages=0x%x", static_cast(inputs.shaderStages)); if (state.stages.mesh) { Require(draw.firstVertex == 0 && draw.firstInstance == 0, "mesh draw offsets are unsupported"); APS5_LOG_CHARS_OUT_DEBUG("Mesh path"); @@ -520,53 +672,496 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& Require(draw.indexCount >= inputSize && mesh.primitivesPerGroup != 0, "mesh draw contains no complete primitive"); const auto step = mesh.inputPrimitive == 6 ? 1u : inputSize; const auto primitives = (draw.indexCount - inputSize) / step + 1u; - meshGroups = (primitives - 1u) / mesh.primitivesPerGroup + 1u; - APS5_LOG_OUT_DEBUG("Mesh primitives=%u groups=%u", primitives, meshGroups); - Require(meshGroups <= context.meshLimits.maxMeshWorkGroupCount[0] && draw.instanceCount <= context.meshLimits.maxMeshWorkGroupCount[1] && static_cast(meshGroups) * draw.instanceCount <= context.meshLimits.maxMeshWorkGroupTotalCount, "mesh draw exceeds workgroup count limits"); + inputs.meshGroups = (primitives - 1u) / mesh.primitivesPerGroup + 1u; + APS5_LOG_OUT_DEBUG("Mesh primitives=%u groups=%u", primitives, inputs.meshGroups); + Require(inputs.meshGroups <= context.meshLimits.maxMeshWorkGroupCount[0] && draw.instanceCount <= context.meshLimits.maxMeshWorkGroupCount[1] && static_cast(inputs.meshGroups) * draw.instanceCount <= context.meshLimits.maxMeshWorkGroupTotalCount, "mesh draw exceeds workgroup count limits"); } if (state.stages.tessellation) Require(draw.indexCount % state.stages.tessellation->inputControlPoints == 0, "incomplete tessellation patch"); // Viewport and scissor are dynamic pipeline state, so their limits are checked here per draw. ValidateViewport(context, state.viewport); - timing.Mark("validate"); - phase(PhaseValidate); - std::unique_ptr indices; - std::uint32_t maxIndex = draw.indexed ? 0u : draw.firstVertex + draw.indexCount - 1u; + timer.phase(PhaseValidate); + inputs.maxIndex = draw.indexed ? 0u : draw.firstVertex + draw.indexCount - 1u; if (draw.indexed) { - indices = std::make_unique(context, static_cast(indexBytes), VK_BUFFER_USAGE_INDEX_BUFFER_BIT); - GuestMemory::Read(draw.indexAddress, indices->Bytes(), draw.indexSize); + inputs.indices = std::make_unique(context, static_cast(indexBytes), VK_BUFFER_USAGE_INDEX_BUFFER_BIT); + GuestMemory::Read(draw.indexAddress, inputs.indices->Bytes(), draw.indexSize); for (std::size_t offset = 0; offset < indexBytes; offset += draw.indexSize) { std::uint32_t index = 0; if (draw.indexSize == 2) { std::uint16_t value = 0; - std::memcpy(&value, indices->Bytes().data() + offset, sizeof(value)); + std::memcpy(&value, inputs.indices->Bytes().data() + offset, sizeof(value)); index = value; } else { - std::memcpy(&index, indices->Bytes().data() + offset, sizeof(index)); + std::memcpy(&index, inputs.indices->Bytes().data() + offset, sizeof(index)); } Require(index <= context.limits.maxDrawIndexedIndexValue, "index exceeds the device's indexed draw limit"); - maxIndex = std::max(maxIndex, index); + inputs.maxIndex = std::max(inputs.maxIndex, index); } } APS5_LOG_CHARS_OUT_DEBUG("Index validation OK"); - timing.Mark("index_upload"); const auto& attributes = shaders.front().program->vertexAttributes; - // Validates the vertex descriptors; the layout also keys and builds the pipeline below. - const auto vertexInput = BuildVertexInputLayout(context, attributes); - std::vector> vertexBuffers; - std::vector vertexHandles; - std::vector vertexOffsets(attributes.size(), 0); + // Validates the vertex descriptors; the layout also keys and builds the pipeline. + if (recipe != nullptr) inputs.vertexInput = recipe->vertexInput; + else inputs.vertexInput = BuildVertexInputLayout(context, attributes); + inputs.vertexOffsets.assign(attributes.size(), 0); + // An indexed draw's vertex offset moves every fetch: the copy must reach the last one. + if (draw.indexed) { + Require(draw.firstVertex <= std::numeric_limits::max() - inputs.maxIndex, "indexed draw vertex range overflow"); + inputs.maxIndex += draw.firstVertex; + } for (const auto& attribute : attributes) { - const auto bytes = VertexBufferReadSize(attribute, maxIndex, draw.instanceCount, draw.firstInstance); + // An indirect draw's counts are unknown here: the descriptor's whole range is copied. + const auto bytes = args != nullptr ? VertexBufferExtent(attribute) : VertexBufferReadSize(attribute, inputs.maxIndex, draw.instanceCount, draw.firstInstance); const auto& fields = attribute.resource.fields; const auto address = fields[0] | (static_cast(fields[1] & 0xffffu) << 32u); Require(!state.hasColorTarget || address + bytes <= state.color.address || state.color.address + state.color.bytes <= address, "vertex buffer aliases the render target"); GuestMemory::CheckRange(reinterpret_cast(address), bytes, 1); auto buffer = std::make_unique(context, bytes, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); GuestMemory::Read(address, buffer->Bytes(), 1); - vertexHandles.push_back(buffer->Handle()); - vertexBuffers.push_back(std::move(buffer)); + inputs.vertexHandles.push_back(buffer->Handle()); + inputs.vertexBuffers.push_back(std::move(buffer)); } - phase(PhaseVertex); + timer.phase(PhaseVertex); + return inputs; +} + +// The resident-target proof of one attachment (StorageTexture::Refresh: FlushPending, CollectWrites +// over the target's pages, the DCC key scan of TextureClearKeys, then UnchangedSince), with the +// [draws] target-lookup accounting. `lookup` makes (or finds) the image; DrawWithRecipe refreshes +// the stored object instead. +std::shared_ptr refreshResidentTarget(const State& state, const ColorTarget& color, DrawOutcome& outcome, bool profile, const std::function()>& lookup) { + const auto lookupStart = std::chrono::steady_clock::now(); + const auto walkedBefore = profile ? GuestMemory::ThreadCollectedBytes() : 0; + std::shared_ptr resident; + try { + resident = lookup(); + } catch (const std::exception& error) { + static std::mutex reportMutex; + static std::set reported; + std::lock_guard lock(reportMutex); + if (reported.insert(color.address).second) std::fprintf(stderr, "[gpu] color target 0x%llx stays non-resident: %s\n", static_cast(color.address), error.what()); + } + if (profile) { + const auto lookupUs = std::chrono::duration(std::chrono::steady_clock::now() - lookupStart).count(); + ++outcome.targetLookups; + if (lookupUs >= 1000.0) { + ++outcome.slowLookups; + outcome.slowLookupUs += lookupUs; + } + outcome.lookupUs += lookupUs; + outcome.pagesWalked += (GuestMemory::ThreadCollectedBytes() - walkedBefore) / 4096; + const bool fullScissor = state.scissor.offset.x <= 0 && state.scissor.offset.y <= 0 && state.scissor.extent.width >= color.extent.width && state.scissor.extent.height >= color.extent.height; + ++(fullScissor ? outcome.fullScissorLookups : outcome.partialLookups); + } + return resident; +} + +// A draw's resources (resolveDrawResources): the resource-cache template a recordable draw's +// stages repeat, or a fresh build. +struct ResolvedResources { + std::shared_ptr resources; + ResourceCache::Key contentKey; + bool cacheable = false; + const ShaderResources::BuildTiming* built = nullptr; +}; + +ResolvedResources resolveDrawResources(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, std::uint64_t indexBytes, bool recordable, DrawOutcome& outcome, DrawTimer& timer) { + ResolvedResources resolved; + APS5_LOG_CHARS_OUT_DEBUG("Creating ShaderResources"); + // A recordable draw whose stages' compiled content repeats an earlier one binds that build's + // descriptor set when it is still valid (see ResourceCache; the dispatch path does the same). + // Only recordable draws take part: a synchronous draw writes its resources back, which a shared + // object must never do. Revalidate proves a hit by texture identity, which needs the texture + // caches. Push constants still come from this draw's stages. APS5_NO_DRAW_RESOURCE_CACHE=1 + // builds every draw's resources as before. + static const bool noDrawResourceCache = std::getenv("APS5_NO_DRAW_RESOURCE_CACHE") != nullptr; + static const bool noTextureCache = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + // The render target and index ranges stay out of the key (see DrawResourceKey); a hit repeats + // the alias checks instead. Debug aid: APS5_NO_DRAW_KEY_TRIM=1 keys them as before. + static const bool trimKey = std::getenv("APS5_NO_DRAW_KEY_TRIM") == nullptr; + resolved.cacheable = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->variantId != 0; }); + if (resolved.cacheable) { + resolved.contentKey = DrawResourceKey(context, shaders, state.color, draw.indexAddress, indexBytes, !trimKey); + if (auto cached = SharedResourceCache().Find(resolved.contentKey)) { + if (cached->Revalidate(shaders)) { + if (trimKey) CheckBufferAliases(shaders, state.color, draw.indexAddress, indexBytes); + resolved.resources = std::move(cached); + outcome.kind = KindTemplateHit; + countCache(&DrawProfile::cacheHits); + } else { + SharedResourceCache().Remove(resolved.contentKey); + countCache(&DrawProfile::cacheInvalidated); + } + } + } else { + countCache(&DrawProfile::uncacheable); + } + timer.phase(PhaseLookup); + if (resolved.resources == nullptr) { + resolved.resources = std::make_shared(context, shaders, state.color, draw.indexAddress, static_cast(indexBytes), snapshots); + resolved.built = &resolved.resources->Timing(); + outcome.addressBased = resolved.resources->HoldsLease(); + outcome.kind = outcome.addressBased ? KindBda : KindBuild; + if (resolved.cacheable) countCache(&DrawProfile::cacheMisses); + } + timer.phase(PhaseResources); + APS5_LOG_CHARS_OUT_DEBUG("ShaderResources created"); + return resolved; +} + +// A GPU-side indirect draw's records (Draw's recorded path; a recipe draw has none) as recordDraw +// takes them: the arguments, how they are read, their host imports and, on the CPU path, the +// records read. +struct IndirectRecord { + const Pm4::DrawParameters::IndirectDraw* args = nullptr; + IndirectDrawPath path = IndirectDrawPath::Gpu; + const HostImport* argumentImport = nullptr; + const HostImport* countImport = nullptr; + std::span records; + double readMs = 0; +}; + +// The draw commands of one draw: the vertex and index buffer binds, then the direct draw, the +// GPU-side indirect draw from `argumentBuffer` or the CPU-read records with the driver's rules. +void recordDrawCommands(const Context& context, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, const DrawInputs& inputs, const IndirectRecord* indirect, VkBuffer argumentBuffer, VkDeviceSize argumentOffset) { + const auto* args = indirect != nullptr ? indirect->args : nullptr; + if (state.stages.mesh) { + APS5_LOG_OUT_DEBUG("vkCmdDrawMeshTasksEXT groups=%u instances=%u", inputs.meshGroups, draw.instanceCount); + context.Function("vkCmdDrawMeshTasksEXT")(commands, inputs.meshGroups, draw.instanceCount, 1); + return; + } + if (!inputs.vertexHandles.empty()) context.Resolved(&DeviceFunctions::cmdBindVertexBuffers, "vkCmdBindVertexBuffers")(commands, 0, static_cast(inputs.vertexHandles.size()), inputs.vertexHandles.data(), inputs.vertexOffsets.data()); + if (draw.indexed) context.Resolved(&DeviceFunctions::cmdBindIndexBuffer, "vkCmdBindIndexBuffer")(commands, inputs.indices->Handle(), 0, draw.indexSize == 2 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); + if (args == nullptr) { + if (draw.indexed) context.Resolved(&DeviceFunctions::cmdDrawIndexed, "vkCmdDrawIndexed")(commands, draw.indexCount, draw.instanceCount, 0, static_cast(draw.firstVertex), draw.firstInstance); + else context.Resolved(&DeviceFunctions::cmdDraw, "vkCmdDraw")(commands, draw.indexCount, draw.instanceCount, draw.firstVertex, draw.firstInstance); + } else if (indirect->path == IndirectDrawPath::Gpu) { + if (args->countIndirect) { + if (draw.indexed) context.Function("vkCmdDrawIndexedIndirectCountKHR")(commands, argumentBuffer, argumentOffset, indirect->countImport->buffer, args->countAddress - indirect->countImport->base, args->count, args->stride); + else context.Function("vkCmdDrawIndirectCountKHR")(commands, argumentBuffer, argumentOffset, indirect->countImport->buffer, args->countAddress - indirect->countImport->base, args->count, args->stride); + } else if (args->count <= 1 || context.multiDrawIndirect) { + if (draw.indexed) context.Resolved(&DeviceFunctions::cmdDrawIndexedIndirect, "vkCmdDrawIndexedIndirect")(commands, argumentBuffer, argumentOffset, args->count, args->stride); + else context.Resolved(&DeviceFunctions::cmdDrawIndirect, "vkCmdDrawIndirect")(commands, argumentBuffer, argumentOffset, args->count, args->stride); + } else { + for (std::uint32_t record = 0; record < args->count; ++record) { + const auto offset = argumentOffset + static_cast(record) * args->stride; + if (draw.indexed) context.Resolved(&DeviceFunctions::cmdDrawIndexedIndirect, "vkCmdDrawIndexedIndirect")(commands, argumentBuffer, offset, 1, args->stride); + else context.Resolved(&DeviceFunctions::cmdDrawIndirect, "vkCmdDrawIndirect")(commands, argumentBuffer, offset, 1, args->stride); + } + } + } else { + // The CPU-read records with the driver's rules: an InPlace dimension takes the record's + // value, a Constant one (the CP writes nowhere) the constant. + using Rule = Pm4::DrawParameters::IndirectDraw::Rule; + for (const auto& record : indirect->records) { + if (record.count == 0 || record.instances == 0) continue; + const auto firstInstance = args->instanceRule == Rule::InPlace ? record.firstInstance : args->instanceConstant; + if (draw.indexed) { + // The CP clamps the index range to INDEX_BUFFER_SIZE (the bound copy). + if (record.firstVertexOrIndex >= draw.indexCount) continue; + const auto vertexOffset = args->vertexRule == Rule::InPlace ? record.vertexOffset : args->vertexConstant; + context.Resolved(&DeviceFunctions::cmdDrawIndexed, "vkCmdDrawIndexed")(commands, std::min(record.count, draw.indexCount - record.firstVertexOrIndex), record.instances, record.firstVertexOrIndex, static_cast(vertexOffset), firstInstance); + } else { + const auto firstVertex = args->vertexRule == Rule::InPlace ? record.firstVertexOrIndex : args->vertexConstant; + context.Resolved(&DeviceFunctions::cmdDraw, "vkCmdDraw")(commands, record.count, record.instances, firstVertex, firstInstance); + } + } + } +} + +// The GPU-side records of an indirect draw, recorded outside the render pass: the stores that +// produced them (a dispatch in place, a fill, the host) precede the indirect read, as for +// DISPATCH_INDIRECT; a Constant dimension is rewritten in a scratch copy (vkCmdDrawIndirect reads +// the record's dword as the first vertex / instance, and the constant is what the fixed-function +// fetch must start at). The scratch is kept until the batch completed. Returns whether a copy was +// rewritten; `argumentBuffer`/`argumentOffset` receive where the draw reads the records. +bool recordIndirectArguments(const Context& context, VkCommandBuffer commands, Recorder* recorder, bool recorded, const IndirectRecord& indirect, std::unique_ptr& scratch, VkBuffer& argumentBuffer, VkDeviceSize& argumentOffset, const std::function& countBarrier) { + using Rule = Pm4::DrawParameters::IndirectDraw::Rule; + const auto* args = indirect.args; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT); + countBarrier(1); + if (recorded) { + // Read in place from the import when the batch runs (a synchronous draw waits for its own). + recorder->NotePendingRead(args->arguments, static_cast(args->RangeBytes()), Recorder::ReadKind::Indirect); + if (args->countIndirect) recorder->NotePendingRead(args->countAddress, 4, Recorder::ReadKind::Indirect); + } + argumentBuffer = indirect.argumentImport->buffer; + argumentOffset = args->arguments - indirect.argumentImport->base; + const bool rewritten = args->vertexRule == Rule::Constant || args->instanceRule == Rule::Constant; + if (!rewritten) return false; + const auto bytes = static_cast(args->RangeBytes()); + scratch = std::make_unique(context, bytes, VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + CopyBuffer(context, commands, argumentBuffer, argumentOffset, scratch->Handle(), 0, bytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + countBarrier(1); + const auto update = context.Resolved(&DeviceFunctions::cmdUpdateBuffer, "vkCmdUpdateBuffer"); + for (std::uint32_t record = 0; record < args->count; ++record) { + const VkDeviceSize base = static_cast(record) * args->stride; + if (args->vertexRule == Rule::Constant) update(commands, scratch->Handle(), base + args->VertexDwordOffset(), 4, &args->vertexConstant); + if (args->instanceRule == Rule::Constant) update(commands, scratch->Handle(), base + args->InstanceDwordOffset(), 4, &args->instanceConstant); + } + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT); + countBarrier(1); + argumentBuffer = scratch->Handle(); + argumentOffset = 0; + return true; +} + +// Debug aid: APS5_CHECK_INDIRECT_ARGS=1 reads the records of a GPU-side draw back on the CPU once +// its batch completed (the GPU has finished writing them by then) and prints the first 16, counting +// records whose Constant dimension holds a value the hardware would have ignored. Empty otherwise. +std::function indirectRecordCheck(const IndirectRecord* indirect) { + static const bool checkArguments = std::getenv("APS5_CHECK_INDIRECT_ARGS") != nullptr; + if (indirect == nullptr || indirect->args == nullptr || indirect->path != IndirectDrawPath::Gpu || !checkArguments) return {}; + return [args = *indirect->args] { + using Rule = Pm4::DrawParameters::IndirectDraw::Rule; + static std::atomic printed{0}; + static std::atomic ignoredValues{0}; + static std::atomic checked{0}; + try { + for (std::uint32_t record = 0; record < args.count; ++record) { + const auto arguments = Pm4::ReadDrawArguments(args, record); + const bool ignoredVertex = args.vertexRule == Rule::Constant && (args.recordBytes == 20 ? arguments.vertexOffset : arguments.firstVertexOrIndex) != 0; + const bool ignoredInstance = args.instanceRule == Rule::Constant && arguments.firstInstance != 0; + ++checked; + if (ignoredVertex || ignoredInstance) ++ignoredValues; + if (printed.fetch_add(1) < 16) std::fprintf(stderr, "[draw] indirect record 0x%llx: count %u instances %u first %u vertexOffset %u startInstance %u (vertex %s, instance %s)\n", static_cast(args.arguments + static_cast(record) * args.stride), arguments.count, arguments.instances, arguments.firstVertexOrIndex, arguments.vertexOffset, arguments.firstInstance, args.vertexRule == Rule::Constant ? "const: record value ignored" : "in-place", args.instanceRule == Rule::Constant ? "const: record value ignored" : "in-place"); + } + } catch (const std::exception& error) { + std::fprintf(stderr, "[draw] indirect record read-back failed: %s\n", error.what()); + } + if (checked % 64 == 0) std::fprintf(stderr, "[draw] indirect records checked %llu, with a value in a Constant dimension %llu\n", static_cast(checked.load()), static_cast(ignoredValues.load())); + }; +} + +// Everything a recorded draw uses lives until its batch completes. +struct Kept { + std::shared_ptr resources; + std::shared_ptr pipeline; + std::shared_ptr framebuffer; + std::unique_ptr indices; + std::vector> vertexBuffers; + std::vector> targets; + std::unique_ptr scratch; +}; + +// The completion side of a recorded draw: the kept objects, the record check, the lease outcome, +// the GPU write notes, the copied-buffer write-back (listed in DrawCopiedWriters or run as a +// completion action) and the targets marked dirty. +void keepRecordedDraw(Recorder& recorder, const std::shared_ptr& resources, std::shared_ptr pipeline, std::shared_ptr framebuffer, DrawInputs& inputs, std::vector> targets, std::unique_ptr scratch, std::function checkRecords, bool listed, bool completion, const DrawOutcome& outcome) { + auto kept = std::make_shared(); + kept->resources = resources; + kept->pipeline = std::move(pipeline); + kept->framebuffer = std::move(framebuffer); + kept->indices = std::move(inputs.indices); + kept->vertexBuffers = std::move(inputs.vertexBuffers); + kept->scratch = std::move(scratch); + kept->targets = std::move(targets); + const auto& residents = kept->targets; + recorder.Keep(kept); + if (checkRecords) recorder.OnComplete(std::move(checkRecords)); + // Counted for the [address-sync] line: a lease released by the completion (the pin waiter + // finishes the recorder up to the open batch, which holds the kept resources and gets the + // next serial), or waited for by the caller (any wait releases it at once). + if (resources->HoldsLease()) CountLeaseOutcome(outcome.waited, outcome.waited ? 0 : recorder.Submissions() + 1); + // Every use of a (possibly shared) resources object registers its GPU writes anew. + resources->MarkGpuWrites(recorder); + // The write-back (fault check, copied buffers) runs when the batch completed; a fault is + // reported by the recorder ("deferred write-back failed") instead of thrown out of the draw. + if (listed) { + // As VulkanDevice::dispatch lists its copied writers: delisted before the write-back + // (one that fails must not keep indirect dispatches on the CPU), listed after the + // registration (a throw there leaves nothing behind). + auto writers = DrawCopiedWriters(); + recorder.OnComplete([resources, writers] { + writers->erase(std::remove(writers->begin(), writers->end(), resources), writers->end()); + resources->WriteBackBuffers(); + }); + writers->push_back(resources); + } else if (completion) { + recorder.OnComplete([resources] { resources->WriteBackBuffers(); }); + } + for (const auto& resident : residents) { + if (resident != nullptr) resident->MarkDirty(); + } +} + +// What recordDraw records from: the recorder, the resources, the pipeline and framebuffer, the +// resident targets with their views, the indirect records (null for a direct draw), how the +// completion work is handled (`listed`, `completion`, `waited`: see Draw) and, from a recipe, the +// push constant block (else assembled from the stages). +struct RecordedDraw { + Recorder* recorder = nullptr; + std::shared_ptr resources; + std::shared_ptr pipeline; + std::shared_ptr framebuffer; + std::span targetViews; + std::vector> targets; + const IndirectRecord* indirect = nullptr; + bool listed = false; + bool completion = false; + bool waited = false; + const std::array* pushBytes = nullptr; + VkShaderStageFlags pushStages = 0; +}; + +// The record of a draw whose targets are all resident (`lean`: recorded into the device's open +// batch, rendering in the general layout): one memory barrier before the pass (earlier writes to +// the attachments and the draw's inputs visible to it) and one after it, recorded by the recorder +// when the pass ends (Recorder::LeaveRenderPassOpen), so consecutive draws into the same +// attachments with nothing recorded between them share one pass; the host-write and host-read +// barriers are implicit in the submission and the fence. Then the kept objects, the completion +// work and, for a waited-for draw, the recorder sync. +void recordDraw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, DrawInputs& inputs, RecordedDraw& record, DrawOutcome& outcome, DrawTimer& timer, double& ownWaitedMs) { + auto* recorder = record.recorder; + auto& resources = *record.resources; + const auto* args = record.indirect != nullptr ? record.indirect->args : nullptr; + const bool gpuIndirect = args != nullptr && record.indirect->path == IndirectDrawPath::Gpu; + using CommandClass = Recorder::CommandClass; + const auto countBarrier = [&](std::uint32_t count) { Recorder::CountBarriers(CommandClass::Draw, count); }; + // The pass is named by its attachment views (stable while the kept targets live, so unique + // within the open batch) and the extent. The previous draw's pass is continued only when this + // draw neither reads its attachments (a barrier would be owed, which no pass allows) nor + // records anything outside a pass (an indirect draw's argument barrier and scratch copies). + std::uint64_t passKey = 14695981039346656037ull; + const auto mix = [&](std::uint64_t value) { + passKey ^= value; + passKey *= 1099511628211ull; + }; + for (const auto view : record.targetViews) mix(reinterpret_cast(view)); + mix(state.renderExtent.width); + mix(state.renderExtent.height); + const bool readsTarget = std::any_of(record.targets.begin(), record.targets.end(), [&](const std::shared_ptr& target) { return resources.ReadsImage(target.get()); }); + // A queued DCC key store over memory the draw writes or reads in place (unknown for an + // address-based build), or over its GPU-side records, lands before it, as before a dispatch + // (VulkanDevice::dispatch); the flush ends an open pass, so it comes before the decision. + const auto touches = [&](std::uint64_t begin, std::uint64_t end) { + const auto bytes = static_cast(end - begin); + if (resources.WritesOverlap(begin, bytes) || resources.ReadsOverlap(begin, bytes)) return true; + if (!gpuIndirect) return false; + return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end); + }; + if (recorder->HasQueuedKeyStores() && (resources.HoldsLease() || recorder->AnyQueuedKeyStore(touches))) recorder->FlushKeyStores(); + // A queued label store over such memory likewise (Recorder::RecordStore). + if (recorder->HasQueuedStores() && (resources.HoldsLease() || recorder->AnyQueuedStore(touches))) recorder->FlushStores(); + const bool continued = !readsTarget && !gpuIndirect && recorder->ContinuesRenderPass(passKey); + outcome.passContinued = continued; + outcome.passBegun = !continued; + const auto commands = continued ? recorder->CommandsInRenderPass() : recorder->Commands(); + // The draw's [gputime] class range: from its first barrier to the pass's trailing barrier (a + // continued draw lies inside its pass's range). + const auto drawTiming = !continued ? recorder->BeginGpuTiming(CommandClass::Draw) : Recorder::NoTiming; + if (!continued && Recorder::BarrierValidate()) { + auto reads = resources.InPlaceReads(); + if (gpuIndirect) { + reads.emplace_back(args->arguments, args->arguments + args->RangeBytes()); + if (args->countIndirect) reads.emplace_back(args->countAddress, args->countAddress + 4); + } + auto images = resources.StorageImages(); + for (const auto& target : record.targets) images.emplace_back(target->Image(), true); + recorder->NoteAccess(CommandClass::Draw, Recorder::Access{reads, resources.GpuWrites(), images, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | inputs.shaderStages, resources.HoldsLease()}); + } + std::unique_ptr scratch; + VkBuffer argumentBuffer = VK_NULL_HANDLE; + VkDeviceSize argumentOffset = 0; + bool rewritten = false; + if (continued) { + record.pipeline->Continue(commands, state.viewport, state.scissor); + } else { + const VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT}; + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | inputs.shaderStages, 0, 1, &before, 0, nullptr, 0, nullptr); + countBarrier(1); + APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded"); + if (gpuIndirect) rewritten = recordIndirectArguments(context, commands, recorder, true, *record.indirect, scratch, argumentBuffer, argumentOffset, countBarrier); + // Earlier recorded work (dispatches, the previous draw) wrote the images in the general + // layout, which a lean draw renders in: no transitions. + APS5_LOG_OUT_DEBUG("Beginning pipeline renderExtent=%ux%u", state.renderExtent.width, state.renderExtent.height); + record.pipeline->Begin(commands, *record.framebuffer, state.renderExtent, state.viewport, state.scissor); + } + APS5_LOG_CHARS_OUT_DEBUG("Pipeline Begin OK"); + resources.Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout()); + APS5_LOG_CHARS_OUT_DEBUG("Resources bound"); + if (record.pushBytes != nullptr) record.pipeline->PushConstants(commands, record.pushStages, *record.pushBytes); + else record.pipeline->PushConstants(commands, shaders); + APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded"); + recordDrawCommands(context, commands, state, draw, inputs, record.indirect, argumentBuffer, argumentOffset); + if (args != nullptr) CountIndirectDraw(record.indirect->path, record.indirect->readMs, rewritten); + auto checkRecords = indirectRecordCheck(record.indirect); + APS5_LOG_CHARS_OUT_DEBUG("Draw recorded"); + // The pass stays open for the next draw of these attachments; the recorder ends it (and the + // draw class range) before anything else is recorded. A draw that wrote memory owes the next + // one a barrier, so its pass cannot be continued. + recorder->LeaveRenderPassOpen(passKey, drawTiming, !resources.WritesMemory()); + timer.phase(PhaseRecord); + keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome); + timer.phase(PhaseKeep); + if (record.waited) { + // The wait the dispatch path makes for such work (source 3, "address-based"): the + // write-back runs before the next packet, so the CPU never reads copied results or + // touches a leased allocation before they landed. + Recorder::CountSync(3); + const auto ownBefore = timer.profile ? Recorder::ThreadWaitedMs() : 0.0; + recorder->Sync(); + if (timer.profile) ownWaitedMs += Recorder::ThreadWaitedMs() - ownBefore; + timer.phase(PhaseSync); + } +} + +// The state with the outputs the pixel shader lacks masked: Vulkan leaves attachments a pixel +// shader has no output for undefined, so their writes are masked (shaders that only store to +// images or buffers keep their targets as they were). Empty when no mask must change. +std::optional maskedState(const State& state, const std::set& fragmentOutputs) { + std::optional masked; + for (std::size_t index = 0; index < state.blends.size(); ++index) { + if (fragmentOutputs.contains(static_cast(index)) || state.blends[index].colorWriteMask == 0) continue; + if (!masked.has_value()) masked = state; + masked->blends[index].colorWriteMask = 0; + } + return masked; +} + +// Debug aid: APS5_DUMP_TARGETS= saves the first n renders of every color target as a raw file +// (u32 width, u32 height, u32 VkFormat, then tightly packed rows). +int DumpTargetLimit() { + static const int dumpLimit = [] { const char* text = std::getenv("APS5_DUMP_TARGETS"); return text ? std::atoi(text) : 0; }(); + return dumpLimit; +} + +// A draw whose targets are all resident is recorded into the device's open batch like a dispatch: +// the CPU does not wait for it, and the drain before it goes away (queue order and the barriers +// keep it behind earlier recorded work); see `recorded` in Draw for the buffers that still need a +// synchronous draw. Other draws keep their own synchronous batch. Debug aid: APS5_SYNC_DRAWS=1 +// makes every draw synchronous. +bool RecordDraws() { + static const bool recordDraws = std::getenv("APS5_SYNC_DRAWS") == nullptr; + return recordDraws; +} + +} + +void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, std::shared_ptr* recipeOut) { + PerformanceTimer timing("Graphics.Draw"); + // APS5_PROFILE_DRAW prints the time of each phase of the draw (microseconds) and the [draws] totals. + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + DrawTimer timer(profile); + // APS5_TRACE_DRAWS (together with APS5_PROFILE_DRAW; alone it only enables Driver.cpp's own + // [draw] lines) additionally prints one line per draw with its phases and, for synchronous + // draws, their inputs: per-draw string building at ~1400 draws per 10 s is itself a cost the + // totals must not carry. + static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; + DrawOutcome outcome; + const ShaderResources::BuildTiming* built = nullptr; + // The thread's GPU waits so far: the difference at the report, less the draw's own syncs + // (`ownWaitedMs`, sampled around them), is what this draw waited for through the flush hook + // (nested in the draw's hold, see DrawOutcome::hookWaitUs). + const auto waitedBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; + double ownWaitedMs = 0; + const auto report = [&](const char* suffix) { reportDrawEnd(state, timer, built, outcome, waitedBefore, ownWaitedMs, suffix); }; + auto inputs = prepareDrawInputs(context, state, draw, shaders, outcome, timer, nullptr); + if (inputs.nothing) return; + const auto* args = draw.indirect ? &*draw.indirect : nullptr; + const auto indexBytes = inputs.indexBytes; + timing.Mark("validate"); + timing.Mark("index_upload"); // One binding per color attachment. Tiled targets are detiled and retiled on the GPU in video memory; // only changed guest blocks are stored back. struct TargetBinding { @@ -587,9 +1182,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& // APS5_DUMP_TARGETS: the rendered linear pixels, read back for target_
_.raw. std::unique_ptr dump; }; - // Debug aid: APS5_DUMP_TARGETS= saves the first n renders of every color target as a raw file - // (u32 width, u32 height, u32 VkFormat, then tightly packed rows). - static const int dumpLimit = [] { const char* text = std::getenv("APS5_DUMP_TARGETS"); return text ? std::atoi(text) : 0; }(); + const int dumpLimit = DumpTargetLimit(); static std::mutex dumpMutex; static std::map dumped; std::vector targets(state.colors.size()); @@ -602,7 +1195,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& APS5_LOG_OUT_DEBUG("Creating color target %zu address=0x%llx bytes=%llu extent=%ux%u", index, static_cast(color.address), static_cast(color.bytes), color.extent.width, color.extent.height); const ColorTargetLayout colorLayout(color.extent.width, color.extent.height, color.tileMode, color.elementBytes); constexpr VkBufferUsageFlags copies = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; - phase(PhaseSetup); + timer.phase(PhaseSetup); // Debug aid: APS5_NO_RESIDENT_TARGETS=1 copies every target in and out again. static const bool residentTargets = std::getenv("APS5_NO_RESIDENT_TARGETS") == nullptr; if (binding.gpuTiling && residentTargets) { @@ -612,29 +1205,15 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& // (GuestMemory::BumpCollectEpoch: ordering points of the queue, or every packet under // APS5_PACKET_EPOCH=1); the key scan runs on every lookup regardless. Lookups of a // millisecond or more are counted apart; the [texture] line says how many uploaded. - const auto lookupStart = std::chrono::steady_clock::now(); - try { + binding.resident = refreshResidentTarget(state, color, outcome, profile, [&] { auto resident = CachedStorageSurface(context, SurfaceForTarget(color)); Require(resident->Attachable(), "storage format cannot be a color attachment"); Require(resident->GuestBytes() == colorLayout.Bytes(), "resident image layout differs from the color layout"); - binding.resident = std::move(resident); - } catch (const std::exception& error) { - static std::mutex reportMutex; - static std::set reported; - std::lock_guard lock(reportMutex); - if (reported.insert(color.address).second) std::fprintf(stderr, "[gpu] color target 0x%llx stays non-resident: %s\n", static_cast(color.address), error.what()); - } - if (profile) { - const auto lookupUs = std::chrono::duration(std::chrono::steady_clock::now() - lookupStart).count(); - ++outcome.targetLookups; - if (lookupUs >= 1000.0) { - ++outcome.slowLookups; - outcome.slowLookupUs += lookupUs; - } - } + return resident; + }); } if (binding.resident != nullptr) { - phase(PhaseReadTarget); + timer.phase(PhaseReadTarget); targetViews.push_back(binding.resident->AttachmentView(color.format)); continue; } @@ -663,79 +1242,76 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& } } } - phase(PhaseReadTarget); + timer.phase(PhaseReadTarget); binding.target = std::make_unique(context, color, state.blends[index].blendEnable != 0); targetViews.push_back(binding.target->View()); } - phase(PhasePrepare); - // A draw whose targets are all resident is recorded into the device's open batch like a dispatch: - // the CPU does not wait for it, and the drain before it goes away (queue order and the barriers - // below keep it behind earlier recorded work); see `recorded` below for the buffers that still - // need a synchronous draw. Other draws keep their own synchronous batch. Debug aid: - // APS5_SYNC_DRAWS=1 makes every draw synchronous. - static const bool recordDraws = std::getenv("APS5_SYNC_DRAWS") == nullptr; + timer.phase(PhasePrepare); + const bool recordDraws = RecordDraws(); auto* recorder = Recorder::Active(); const bool recordable = recordDraws && recorder != nullptr && dumpLimit == 0 && std::all_of(targets.begin(), targets.end(), [](const TargetBinding& binding) { return binding.resident != nullptr; }); - APS5_LOG_CHARS_OUT_DEBUG("Creating ShaderResources"); - // A recordable draw whose stages' compiled content repeats an earlier one binds that build's - // descriptor set when it is still valid (see ResourceCache; the dispatch path does the same). - // Only recordable draws take part: a synchronous draw writes its resources back, which a shared - // object must never do. Revalidate proves a hit by texture identity, which needs the texture - // caches. Push constants still come from this draw's stages. APS5_NO_DRAW_RESOURCE_CACHE=1 - // builds every draw's resources as before. - static const bool noDrawResourceCache = std::getenv("APS5_NO_DRAW_RESOURCE_CACHE") != nullptr; - static const bool noTextureCache = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; - // The render target and index ranges stay out of the key (see DrawResourceKey); a hit repeats - // the alias checks instead. Debug aid: APS5_NO_DRAW_KEY_TRIM=1 keys them as before. - static const bool trimKey = std::getenv("APS5_NO_DRAW_KEY_TRIM") == nullptr; - const bool cacheable = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->variantId != 0; }); - std::shared_ptr resources; - ResourceCache::Key contentKey; - if (cacheable) { - contentKey = DrawResourceKey(context, shaders, state.color, draw.indexAddress, indexBytes, !trimKey); - if (auto cached = SharedResourceCache().Find(contentKey)) { - if (cached->Revalidate(shaders)) { - if (trimKey) CheckBufferAliases(shaders, state.color, draw.indexAddress, indexBytes); - resources = std::move(cached); - countCache(&DrawProfile::cacheHits); - } else { - SharedResourceCache().Remove(contentKey); - countCache(&DrawProfile::cacheInvalidated); + auto resolved = resolveDrawResources(context, state, draw, shaders, snapshots, indexBytes, recordable, outcome, timer); + auto& resources = resolved.resources; + const auto& contentKey = resolved.contentKey; + const bool cacheable = resolved.cacheable; + built = resolved.built; + // The memory-state decision of an indirect draw, made here after the resource build (which may + // retire imports and wait for recorded work) and before anything is recorded, exactly as + // VulkanDevice::dispatch decides for DISPATCH_INDIRECT: the GPU reads the records in place from + // the host import unless storage-image results are pending over them (flushed, then synced), a + // label or a copied buffer's write-back still has to land on them, or they are not imported; in + // those cases (and with APS5_NO_GPU_INDIRECT_DRAW=1) the CPU reads them through the flush hook + // and records plain draws with the same per-dimension rules. The import pointer stays valid: + // refreshImports runs only under GuestMemory::GpuMutex, held here, and a retired import's + // buffer is kept while the recorder is busy. + IndirectRecord indirect; + indirect.args = args; + std::vector records; + if (args != nullptr) { + static const bool gpuIndirectDraws = std::getenv("APS5_NO_GPU_INDIRECT_DRAW") == nullptr; + const auto decide = [&](std::uint64_t address, std::size_t bytes, const HostImport*& import) { + if (StorageTexture::FlushPending(address, bytes, nullptr, "indirect draw arguments")) { + if (recorder != nullptr) { + Recorder::CountSync(2); + recorder->Sync(); + } + return IndirectDrawPath::PendingImage; + } + if (recorder != nullptr && recorder->PendingLabelIn(address, bytes)) return IndirectDrawPath::PendingLabelOrCopy; + const auto overlaps = [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }; + if (context.copiedWriters != nullptr && std::any_of(context.copiedWriters->begin(), context.copiedWriters->end(), overlaps)) return IndirectDrawPath::PendingLabelOrCopy; + if (std::any_of(DrawCopiedWriters()->begin(), DrawCopiedWriters()->end(), overlaps)) return IndirectDrawPath::PendingLabelOrCopy; + if ((import = HostImportFor(context, address, bytes)) == nullptr) return IndirectDrawPath::NotImported; + return IndirectDrawPath::Gpu; + }; + const auto rangeBytes = args->RangeBytes(); + if (rangeBytes == 0) { + report(" indirect draw without records"); + return; + } + Require(rangeBytes <= std::numeric_limits::max(), "indirect draw record range overflow"); + if (!gpuIndirectDraws) indirect.path = IndirectDrawPath::Disabled; + else indirect.path = decide(args->arguments, static_cast(rangeBytes), indirect.argumentImport); + if (indirect.path == IndirectDrawPath::Gpu && args->countIndirect) { + Require(context.drawIndirectCount, "indirect draw count without VK_KHR_draw_indirect_count must be resolved by the driver"); + indirect.path = decide(args->countAddress, 4, indirect.countImport); + } + if (indirect.path != IndirectDrawPath::Gpu) { + // The reads wait for the producing batch through the flush hook, as the driver's + // resolve did for every indirect dispatch before its GPU path. + const auto readStart = std::chrono::steady_clock::now(); + const auto count = std::min(args->countIndirect ? Pm4::ReadDrawCount(*args) : args->count, args->count); + records.reserve(count); + for (std::uint32_t record = 0; record < count; ++record) records.push_back(Pm4::ReadDrawArguments(*args, record)); + indirect.readMs = std::chrono::duration(std::chrono::steady_clock::now() - readStart).count(); + indirect.records = records; + if (std::none_of(records.begin(), records.end(), [](const Pm4::DrawArguments& record) { return record.count != 0 && record.instances != 0; })) { + CountIndirectDraw(indirect.path, indirect.readMs); + report(" indirect draw with empty records"); + return; } } - } else { - countCache(&DrawProfile::uncacheable); } - phase(PhaseLookup); - if (resources == nullptr) { - resources = std::make_shared(context, shaders, state.color, draw.indexAddress, static_cast(indexBytes), snapshots); - built = &resources->Timing(); - outcome.addressBased = resources->HoldsLease(); - if (cacheable) countCache(&DrawProfile::cacheMisses); - } - phase(PhaseResources); - APS5_LOG_CHARS_OUT_DEBUG("ShaderResources created"); - APS5_LOG_CHARS_OUT_DEBUG("Creating Pipeline"); - // Vulkan leaves attachments a pixel shader has no output for undefined, so their writes are masked - // (shaders that only store to images or buffers keep their targets as they were). The state is - // copied only when a mask must change. - std::optional masked; - for (std::size_t index = 0; index < state.blends.size(); ++index) { - if (fragmentOutputs.contains(static_cast(index)) || state.blends[index].colorWriteMask == 0) continue; - if (!masked.has_value()) masked = state; - masked->blends[index].colorWriteMask = 0; - } - const State& pipelineState = masked.has_value() ? *masked : state; - auto pipeline = CachedPipeline(context, pipelineState, vertexInput, *resources, shaders); - // Resident targets' views are stable while their storage image lives, so the framebuffer is - // reused with the pipeline; a per-draw RenderTarget gets a framebuffer of its own. - std::vector> owners; - owners.reserve(targets.size()); - for (const auto& binding : targets) owners.push_back(binding.resident); - auto framebuffer = pipeline->AcquireFramebuffer(targetViews, owners, state.renderExtent); - phase(PhasePipeline); - APS5_LOG_CHARS_OUT_DEBUG("Pipeline created"); - APS5_LOG_CHARS_OUT_DEBUG("Creating CommandBatch"); // A draw whose only completion work is a BDA fault check (the rect-list stages carry a fault // buffer, so every rect-list draw needed one) is recorded too, with the check as a completion // action like a dispatch's write-back: its own batch would first wait for everything recorded @@ -779,29 +1355,127 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& // hit is reusable by construction, so `recorded` holds for it; one with completion work is // never reusable). if (cacheable && built != nullptr && recorded && resources->Reusable()) SharedResourceCache().Insert(contentKey, resources); + // A recorded draw renders into its resident targets in the general layout (recordDraw). Debug + // aid: APS5_DRAW_TRANSITIONS=1 keeps the per-draw upload and download barriers, the layout + // transitions of every target and one pass per draw, as before. + static const bool drawTransitions = std::getenv("APS5_DRAW_TRANSITIONS") != nullptr; + const bool lean = recorded && !drawTransitions; + APS5_LOG_CHARS_OUT_DEBUG("Creating Pipeline"); + // The state is copied only when a mask must change. + auto masked = maskedState(state, inputs.fragmentOutputs); + const State& pipelineState = masked.has_value() ? *masked : state; + auto pipeline = CachedPipeline(context, pipelineState, inputs.vertexInput, *resources, shaders, lean ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + // Resident targets' views are stable while their storage image lives, so the framebuffer is + // reused with the pipeline; a per-draw RenderTarget gets a framebuffer of its own. + std::vector> owners; + owners.reserve(targets.size()); + for (const auto& binding : targets) owners.push_back(binding.resident); + auto framebuffer = pipeline->AcquireFramebuffer(targetViews, owners, state.renderExtent); + timer.phase(PhasePipeline); + APS5_LOG_CHARS_OUT_DEBUG("Pipeline created"); + if (lean) { + RecordedDraw record; + record.recorder = recorder; + record.resources = resources; + record.pipeline = pipeline; + record.framebuffer = framebuffer; + record.targetViews = targetViews; + record.targets = owners; + record.indirect = args != nullptr ? &indirect : nullptr; + record.listed = listed; + record.completion = completion; + record.waited = outcome.waited; + recordDraw(context, state, draw, shaders, inputs, record, outcome, timer, ownWaitedMs); + // The recipe for the caller's draw-cache entry (design_cpu_final M8, rule R3 for draws): + // only a template the resource cache serves under this content key (reusable: no lease, + // no copied writes, every direct region import- or mirror-served) of a direct draw, so a + // hit's proof is the template's ProveCurrent and nothing needs completion work. + if (recipeOut != nullptr && DrawRecipes() && cacheable && !outcome.waited && args == nullptr && resources->Reusable()) { + auto recipe = std::make_shared(); + recipe->device = context.device; + recipe->templateRef = resources; + recipe->key = contentKey; + recipe->pipeline = pipeline; + recipe->framebuffer = framebuffer; + for (const auto& owner : owners) recipe->targets.emplace_back(owner); + recipe->targetViews = targetViews; + std::uint64_t passKey = 14695981039346656037ull; + for (const auto view : targetViews) passKey = (passKey ^ reinterpret_cast(view)) * 1099511628211ull; + passKey = (passKey ^ state.renderExtent.width) * 1099511628211ull; + passKey = (passKey ^ state.renderExtent.height) * 1099511628211ull; + recipe->passKey = passKey; + recipe->vertexInput = inputs.vertexInput; + recipe->pushStages = PushConstantStages(shaders); + if (recipe->pushStages != 0) recipe->pushBytes = AssemblePushConstants(shaders); + recipe->masked = masked; + recipe->fragmentOutputs = inputs.fragmentOutputs; + recipe->shaderStages = inputs.shaderStages; + *recipeOut = std::move(recipe); + } + timing.Mark("draw_and_resource_release"); + report(outcome.waited ? (outcome.reason == SyncLease ? " recorded then waited (lease)" : " recorded then waited (copied writes)") : completion ? " recorded with completion" : " recorded"); + return; + } + APS5_LOG_CHARS_OUT_DEBUG("Creating CommandBatch"); std::optional batch; if (!recorded) { // The color-target detiles below take their descriptor sets from a fresh batch. if (context.detiler != nullptr) context.detiler->BeginBatch(); batch.emplace(context); } + using CommandClass = Recorder::CommandClass; + const auto countBarrier = [&](std::uint32_t count = 1) { if (recorded) Recorder::CountBarriers(CommandClass::Draw, count); }; + const bool gpuIndirect = args != nullptr && indirect.path == IndirectDrawPath::Gpu; + // A queued DCC key store over memory the draw writes or reads in place (unknown for an + // address-based build), or over its GPU-side records, lands before it, as before a dispatch + // (VulkanDevice::dispatch). + const auto touches = [&](std::uint64_t begin, std::uint64_t end) { + const auto bytes = static_cast(end - begin); + if (resources->WritesOverlap(begin, bytes) || resources->ReadsOverlap(begin, bytes)) return true; + if (!gpuIndirect) return false; + return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end); + }; + if (recorded && recorder->HasQueuedKeyStores() && (resources->HoldsLease() || recorder->AnyQueuedKeyStore(touches))) recorder->FlushKeyStores(); + // A queued label store over such memory likewise (Recorder::RecordStore). + if (recorded && recorder->HasQueuedStores() && (resources->HoldsLease() || recorder->AnyQueuedStore(touches))) recorder->FlushStores(); const auto commands = recorded ? recorder->Commands() : batch->Handle(); APS5_LOG_CHARS_OUT_DEBUG("CommandBatch created"); + // The draw's [gputime] class range: from its first barrier to the download barrier. + const auto drawTiming = recorded ? recorder->BeginGpuTiming(CommandClass::Draw) : Recorder::NoTiming; + if (recorded && Recorder::BarrierValidate()) { + auto reads = resources->InPlaceReads(); + if (gpuIndirect) { + reads.emplace_back(args->arguments, args->arguments + args->RangeBytes()); + if (args->countIndirect) reads.emplace_back(args->countAddress, args->countAddress + 4); + } + auto images = resources->StorageImages(); + for (const auto& binding : targets) { + if (binding.resident != nullptr) images.emplace_back(binding.resident->Image(), true); + } + recorder->NoteAccess(CommandClass::Draw, Recorder::Access{reads, resources->GpuWrites(), images, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | inputs.shaderStages, resources->HoldsLease()}); + } + std::unique_ptr scratch; + VkBuffer argumentBuffer = VK_NULL_HANDLE; + VkDeviceSize argumentOffset = 0; + bool rewritten = false; VkMemoryBarrier upload{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; upload.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; upload.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | shaderStages, 0, 1, &upload, 0, nullptr, 0, nullptr); + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | inputs.shaderStages, 0, 1, &upload, 0, nullptr, 0, nullptr); + countBarrier(); APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded"); + if (gpuIndirect) rewritten = recordIndirectArguments(context, commands, recorder, recorded, indirect, scratch, argumentBuffer, argumentOffset, countBarrier); for (auto& binding : targets) { if (binding.resident != nullptr) { - // Earlier recorded work (dispatches, the previous draw) wrote the image in the general layout. imageBarrier(context, commands, binding.resident->Image(), VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); + countBarrier(); continue; } VkBufferImageCopy copy{}; copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; copy.imageExtent = {binding.color.extent.width, binding.color.extent.height, 1}; const VkBuffer linear = binding.gpuTiling ? binding.linearDevice->Handle() : binding.transfer->Handle(); + countBarrier(binding.gpuTiling ? 4 : 2); if (binding.gpuTiling) { CopyBuffer(context, commands, binding.tiled->Handle(), 0, binding.tiledDevice->Handle(), 0, binding.original.size()); memoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); @@ -809,7 +1483,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& memoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); } imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT); - context.Function("vkCmdCopyBufferToImage")(commands, linear, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); + context.Resolved(&DeviceFunctions::cmdCopyBufferToImage, "vkCmdCopyBufferToImage")(commands, linear, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); } APS5_LOG_OUT_DEBUG("Beginning pipeline renderExtent=%ux%u", state.renderExtent.width, state.renderExtent.height); @@ -819,29 +1493,22 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& APS5_LOG_CHARS_OUT_DEBUG("Resources bound"); pipeline->PushConstants(commands, shaders); APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded"); - if (state.stages.mesh) { - APS5_LOG_OUT_DEBUG("vkCmdDrawMeshTasksEXT groups=%u instances=%u", meshGroups, draw.instanceCount); - context.Function("vkCmdDrawMeshTasksEXT")(commands, meshGroups, draw.instanceCount, 1); - } else { - if (!vertexHandles.empty()) context.Function("vkCmdBindVertexBuffers")(commands, 0, static_cast(vertexHandles.size()), vertexHandles.data(), vertexOffsets.data()); - if (draw.indexed) { - context.Function("vkCmdBindIndexBuffer")(commands, indices->Handle(), 0, draw.indexSize == 2 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); - context.Function("vkCmdDrawIndexed")(commands, draw.indexCount, draw.instanceCount, 0, 0, 0); - } else { - context.Function("vkCmdDraw")(commands, draw.indexCount, draw.instanceCount, draw.firstVertex, draw.firstInstance); - } - } + recordDrawCommands(context, commands, state, draw, inputs, args != nullptr ? &indirect : nullptr, argumentBuffer, argumentOffset); + if (args != nullptr) CountIndirectDraw(indirect.path, indirect.readMs, rewritten); + auto checkRecords = indirectRecordCheck(args != nullptr ? &indirect : nullptr); APS5_LOG_CHARS_OUT_DEBUG("Draw recorded"); - context.Function("vkCmdEndRenderPass")(commands); + context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands); APS5_LOG_CHARS_OUT_DEBUG("Render pass ended"); for (auto& binding : targets) { if (binding.resident != nullptr) { imageBarrier(context, commands, binding.resident->Image(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + countBarrier(); continue; } VkBufferImageCopy copy{}; copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; copy.imageExtent = {binding.color.extent.width, binding.color.extent.height, 1}; + countBarrier(binding.gpuTiling ? 4 : 2); imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); const VkBuffer linear = binding.gpuTiling ? binding.linearDevice->Handle() : binding.transfer->Handle(); VkBufferMemoryBarrier reuse{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; @@ -851,8 +1518,8 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& reuse.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; reuse.buffer = linear; reuse.size = VK_WHOLE_SIZE; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &reuse, 0, nullptr); - context.Function("vkCmdCopyImageToBuffer")(commands, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear, 1, ©); + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &reuse, 0, nullptr); + context.Resolved(&DeviceFunctions::cmdCopyImageToBuffer, "vkCmdCopyImageToBuffer")(commands, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear, 1, ©); if (binding.gpuTiling) { memoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); if (dumpLimit > 0) { @@ -870,51 +1537,15 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& VkMemoryBarrier download{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; download.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT; download.dstAccessMask = VK_ACCESS_HOST_READ_BIT; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | shaderStages, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &download, 0, nullptr, 0, nullptr); + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | inputs.shaderStages, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &download, 0, nullptr, 0, nullptr); + countBarrier(); + if (recorded) recorder->EndGpuTiming(drawTiming); APS5_LOG_CHARS_OUT_DEBUG("Download barrier recorded"); APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait"); - phase(PhaseRecord); + timer.phase(PhaseRecord); if (recorded) { - // Everything the recorded draw uses lives until its batch completes. - struct Kept { - std::shared_ptr resources; - std::shared_ptr pipeline; - std::shared_ptr framebuffer; - std::unique_ptr indices; - std::vector> vertexBuffers; - std::vector> targets; - }; - auto kept = std::make_shared(); - kept->resources = resources; - kept->pipeline = pipeline; - kept->framebuffer = framebuffer; - kept->indices = std::move(indices); - kept->vertexBuffers = std::move(vertexBuffers); - for (auto& binding : targets) kept->targets.push_back(binding.resident); - recorder->Keep(std::move(kept)); - // Counted for the [address-sync] line: a lease released by the completion (the pin waiter - // finishes the recorder up to the open batch, which holds the kept resources and gets the - // next serial), or waited for below (any wait releases it at once). - if (resources->HoldsLease()) CountLeaseOutcome(outcome.waited, outcome.waited ? 0 : recorder->Submissions() + 1); - // Every use of a (possibly shared) resources object registers its GPU writes anew. - resources->MarkGpuWrites(*recorder); - // The write-back (fault check, copied buffers) runs when the batch completed; a fault is - // reported by the recorder ("deferred write-back failed") instead of thrown out of the draw. - if (listed) { - // As VulkanDevice::dispatch lists its copied writers: delisted before the write-back - // (one that fails must not keep indirect dispatches on the CPU), listed after the - // registration (a throw there leaves nothing behind). - auto writers = DrawCopiedWriters(); - recorder->OnComplete([resources, writers] { - writers->erase(std::remove(writers->begin(), writers->end(), resources), writers->end()); - resources->WriteBackBuffers(); - }); - writers->push_back(resources); - } else if (completion) { - recorder->OnComplete([resources] { resources->WriteBackBuffers(); }); - } - for (auto& binding : targets) binding.resident->MarkDirty(); - phase(PhaseKeep); + keepRecordedDraw(*recorder, resources, pipeline, framebuffer, inputs, owners, std::move(scratch), std::move(checkRecords), listed, completion, outcome); + timer.phase(PhaseKeep); if (outcome.waited) { // The wait the dispatch path makes for such work (source 3, "address-based"): the // write-back runs before the next packet, so the CPU never reads copied results or @@ -923,7 +1554,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& const auto ownBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; recorder->Sync(); if (profile) ownWaitedMs += Recorder::ThreadWaitedMs() - ownBefore; - phase(PhaseSync); + timer.phase(PhaseSync); } report(outcome.waited ? (outcome.reason == SyncLease ? " recorded then waited (lease)" : " recorded then waited (copied writes)") : completion ? " recorded with completion" : " recorded"); return; @@ -934,7 +1565,8 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& batch->SubmitAndWait(); if (profile) ownWaitedMs += Recorder::ThreadWaitedMs() - ownBefore; } - phase(PhaseSync); + if (checkRecords) checkRecords(); + timer.phase(PhaseSync); APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait OK"); for (const auto& binding : targets) GuestMemory::CheckRange(reinterpret_cast(binding.color.address), binding.color.bytes, 256, true); for (auto& binding : targets) { @@ -965,7 +1597,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& // The stored texels are the whole target now, so later reads must see them rather than a fast clear. MarkDccUncompressed(binding.color.dccAddress, ColorTargetLayout(binding.color.extent.width, binding.color.extent.height, binding.color.tileMode, binding.color.elementBytes).Bytes()); } - phase(PhaseWriteBack); + timer.phase(PhaseWriteBack); if (profile && traceDraws) { // A synchronous draw's inputs and target sample are described as a rendering debug aid; // the ~0.5 ms that takes (Describe samples every bound range and reads its DCC keys) is @@ -987,7 +1619,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& } std::fprintf(stderr, "[draw] targets:%s\n", list.c_str()); } - phase(PhaseDescribe); + timer.phase(PhaseDescribe); char suffix[160]; std::snprintf(suffix, sizeof(suffix), " synchronous (%s), first 0x%llx format %d (%zu of %zu sampled bytes nonzero)", SyncReasonNames[outcome.reason], static_cast(state.color.address), static_cast(state.color.format), nonzero, sampled); report(suffix); @@ -997,4 +1629,107 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& APS5_LOG_CHARS_OUT_DEBUG("Draw finished"); } +DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, const DrawRecipe& recipe) { + static_cast(snapshots); + PerformanceTimer timing("Graphics.DrawWithRecipe"); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + DrawTimer timer(profile); + DrawRecipeOutcome result; + DrawOutcome outcome; + outcome.kind = KindRecipeHit; + const auto waitedBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; + double ownWaitedMs = 0; + const auto miss = [&](DrawRecipeMiss reason) { + result.miss = reason; + if (profile) { + auto& stats = Profile(); + std::lock_guard lock(stats.mutex); + ++stats.recipeMisses[static_cast(reason)]; + } + return result; + }; + Require(!draw.indirect, "a draw recipe covers direct draws only"); + auto* recorder = Recorder::Active(); + if (!RecordDraws() || recorder == nullptr || DumpTargetLimit() != 0) return miss(DrawRecipeMiss::NotRecordable); + Require(recipe.targets.size() == state.colors.size() && recipe.targetViews.size() == state.colors.size(), "draw recipe targets do not match the state"); + auto inputs = prepareDrawInputs(context, state, draw, shaders, outcome, timer, &recipe); + if (inputs.nothing) { + result.recorded = true; + return result; + } + // The resident-target proof on the stored images (design_cpu_final 3.7): while an image is + // still the storage cache's for its surface (StorageImageCached: the cache entry, removed under + // the cache mutex before an eviction's flush, and the LRU touch a lookup would make), it is + // what the lookup would return, and Refresh brings it up to date exactly as the lookup's does + // (FlushPending, the whole-surface collect and UnchangedSince, the key scan). An image gone + // from the cache is a miss: the ordinary path looks up anew. + std::vector> targets; + targets.reserve(recipe.targets.size()); + for (std::size_t index = 0; index < recipe.targets.size(); ++index) { + timer.phase(PhaseSetup); + auto stored = recipe.targets[index].lock(); + if (stored == nullptr || !StorageImageCached(context, stored.get())) return miss(DrawRecipeMiss::TargetGone); + auto resident = refreshResidentTarget(state, state.colors[index], outcome, profile, [&] { + stored->Refresh(); + return stored; + }); + timer.phase(PhaseReadTarget); + if (resident == nullptr) return miss(DrawRecipeMiss::TargetGone); + targets.push_back(std::move(resident)); + } + timer.phase(PhasePrepare); + // The template's proof (rules R6/R7): ProveCurrent, T1 included, the alias checks the trimmed + // key leaves to a hit repeated; a failure removes the template from the cache (the batch keeps + // it) and the caller rebuilds. + auto resources = recipe.templateRef.lock(); + if (resources == nullptr) return miss(DrawRecipeMiss::TemplateGone); + Require(resources->Reusable(), "draw recipe over a non-reusable template"); + const auto proofStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const bool proved = resources->ProveCurrent(shaders, &result.proof); + if (profile) result.proofUs = std::chrono::duration(std::chrono::steady_clock::now() - proofStart).count(); + if (!proved) { + SharedResourceCache().Remove(recipe.key, resources.get()); + countCache(&DrawProfile::cacheInvalidated); + recorder->Keep(std::move(resources)); + timer.phase(PhaseLookup); + return miss(DrawRecipeMiss::Proof); + } + CheckBufferAliases(shaders, state.color, draw.indexAddress, inputs.indexBytes); + SharedResourceCache().Touch(recipe.key); + countCache(&DrawProfile::cacheHits); + timer.phase(PhaseLookup); + outcome.completion = false; + outcome.recorded = true; + // The pipeline is the recipe's while the pipeline store still holds it (its device's teardown + // or its eviction bound drops it: a miss). The framebuffer is the recipe's while the pipeline's + // list holds it: the views are the stored targets' (stable while they live, and they are the + // same objects), so AcquireFramebuffer only runs when the list evicted it. + auto pipeline = recipe.pipeline.lock(); + if (pipeline == nullptr) return miss(DrawRecipeMiss::ObjectsGone); + auto framebuffer = recipe.framebuffer.lock(); + if (framebuffer == nullptr) framebuffer = pipeline->AcquireFramebuffer(recipe.targetViews, targets, state.renderExtent); + timer.phase(PhasePipeline); + const auto recordStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + RecordedDraw record; + record.recorder = recorder; + record.resources = std::move(resources); + record.pipeline = std::move(pipeline); + record.framebuffer = std::move(framebuffer); + record.targetViews = recipe.targetViews; + record.targets = std::move(targets); + record.pushBytes = &recipe.pushBytes; + record.pushStages = recipe.pushStages; + recordDraw(context, state, draw, shaders, inputs, record, outcome, timer, ownWaitedMs); + if (profile) { + result.recordUs = std::chrono::duration(std::chrono::steady_clock::now() - recordStart).count(); + auto& stats = Profile(); + std::lock_guard lock(stats.mutex); + ++stats.recipeHits; + } + timing.Mark("draw_and_resource_release"); + reportDrawEnd(state, timer, nullptr, outcome, waitedBefore, ownWaitedMs, " recorded from recipe"); + result.recorded = true; + return result; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 3d0c250043..a14325c529 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -2,13 +2,16 @@ #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/CpuTopology.hpp" #ifdef _WIN32 #include #endif #include #include #include +#include #include #include #include @@ -44,8 +47,61 @@ struct ImageMirror { std::uint64_t serial = 0; }; +// The address space. An address-based build maps every readable registered range; building that +// list per build (the lease's ~1200 shared_ptrs, an import lookup per range, the sort and merge, the +// upload loop, the BDA table) repeated the same result while the registry stood still. One space is +// cached per device and taken by every build made while it is current: a pure function of the +// registry generation read BEFORE its lease was acquired (a mutation bumps the generation before it +// releases the registry, so an unchanged generation means the lease is the one a build would +// acquire now), the import registry's epoch (bumped by every retire, so the `direct` pointers of +// its regions stand) and the device. `base` holds the ranges served in place (imports, mirrors), +// sorted and free of per-build state; the ranges copied per build are `copied`, re-added to each +// build's own regions, as are the V#s and snapshots outside the space. The space's lease pins its +// ranges while any build, batch or the cache holds it; the registry's pin waiter (WaitForLeases) +// drops the cache's reference first, lock-free, so a guest free of a range held only by the cache +// costs no GPU wait. APS5_NO_ADDRESS_SPACE_CACHE=1 restores the per-build path. +struct GuestBufferMemory::AddressSpace { + std::uint64_t generation = 0; + std::uint64_t importsEpoch = 0; + VkDevice device = VK_NULL_HANDLE; + // Identity for the life of this space (see CachedAddressTable). + std::uint64_t serial = 0; + GuestAllocations::Lease lease; + std::vector base; + std::vector copied; + // The BDA table entries of `base`, in its order. + std::vector ranges; +}; + namespace { +struct AddressSpaceCache { + std::atomic> current; + std::atomic serials{0}; + // Set by the waiter's drop, cleared by the next publish: the rebuild's reason. + std::atomic droppedByWaiter{false}; + std::atomic hits{0}; + std::atomic rebuiltFirst{0}; + std::atomic rebuiltGeneration{0}; + std::atomic rebuiltEpoch{0}; + std::atomic rebuiltDevice{0}; + std::atomic rebuiltWaiterDrop{0}; + std::atomic unpublished{0}; + std::atomic dissolvedOverlap{0}; + std::atomic dissolvedImports{0}; + std::atomic waiterDrops{0}; +}; + +AddressSpaceCache& Spaces() { + static AddressSpaceCache cache; + return cache; +} + +bool addressSpaceCacheEnabled() { + static const bool disabled = std::getenv("APS5_NO_ADDRESS_SPACE_CACHE") != nullptr; + return !disabled; +} + // Imports persist across draws, keyed by allocation base, and are dropped when their allocation leaves // the registered set; a dropped import is destroyed once the recorded work that may read it completed. struct HostImports { @@ -84,17 +140,23 @@ struct RetiredImport { // Drops an import. Recorded batches (dispatches, recorded draws, GPU label writes) may still read it, // so the objects live until the batch open now completed; with no batches in flight all GPU work that // used it was synchronous and it is destroyed at once. -void retireImport(const Context& context, HostImports& state, std::map::iterator it) { +const GuestAllocations::Range* containingRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end); + +void retireImport(const Context& context, HostImports& state, std::map::iterator it, const GuestAllocations::Lease& lease) { + // Results shadowed for the import reach its (old) buffer first where the memory is still a + // readable registered range (the import retires because its registration vanished or changed + // size); the holder below outlives the batch that copies them. + RetireShadow(context, it->second, [&lease](std::uint64_t begin, std::uint64_t end) { return containingRange(lease, begin, end) != nullptr; }); auto holder = std::make_shared(context, it->second); if (auto* recorder = Recorder::Active(); recorder != nullptr && !recorder->Idle()) recorder->Keep(std::move(holder)); ++state.epoch; state.imports.erase(it); } -const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t base, std::uint64_t bytes) { +const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t base, std::uint64_t bytes, const GuestAllocations::Lease& lease) { if (const auto found = state.imports.find(base); found != state.imports.end()) { if (found->second.bytes == bytes) return &found->second; - retireImport(context, state, found); + retireImport(context, state, found, lease); } const auto alignment = context.hostImportAlignment; if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0 || state.failed.contains(base)) return nullptr; @@ -214,7 +276,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc continue; } const auto next = std::next(it); - retireImport(context, state, it); + retireImport(context, state, it, lease); it = next; } for (auto it = state.failed.begin(); it != state.failed.end();) it = leasedRangeAt(lease, *it) != nullptr ? std::next(it) : state.failed.erase(it); @@ -255,6 +317,38 @@ struct ImageMirrors { std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; +// APS5_PROFILE_DRAW: the parts of one address-based build (see GuestBufferMemory::CountAddressBuild), +// timed on the building thread by AcquireRegistered, summed into the totals with the caller's +// snapshot compares and printed as the [address] line every 10 s. +struct AddressBuildTiming { + double leaseUs = 0; + double importsUs = 0; + double mirrorsUs = 0; + double compareUs = 0; + std::uint64_t blocksCompared = 0; + std::uint64_t blocksCopied = 0; +}; + +AddressBuildTiming& ThreadAddressTiming() { + thread_local AddressBuildTiming timing; + return timing; +} + +struct AddressBuildTotals { + std::mutex mutex; + std::uint64_t builds = 0; + AddressBuildTiming sums; + double snapshotsUs = 0; + BdaResources::TableCacheStats tableSeen; + AddressSpaceStats spaceSeen; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +AddressBuildTotals& AddressBuilds() { + static AddressBuildTotals totals; + return totals; +} + ImageMirrors& Mirrors() { static ImageMirrors mirrors; return mirrors; @@ -338,7 +432,10 @@ private: // not be started would leave it waiting forever. for (unsigned i = 0; i < wanted; ++i) { try { - std::thread([this] { helper(); }).detach(); + std::thread([this] { + CpuTopology::PinHelperThread("mirror refresh"); + helper(); + }).detach(); ++helpers; } catch (const std::system_error& error) { std::fprintf(stderr, "[gpu] mirror refresh helper %u not started: %s\n", i, error.what()); @@ -579,6 +676,21 @@ bool WaitForLeases() noexcept { const auto start = std::chrono::steady_clock::now(); bool synced = false; bool drained = false; + // A range pinned only by the cached address space is released by dropping the cache's + // reference: no GPU wait, no device lock (the mutating thread may be a driver thread holding + // it). The registry rescans; a range still pinned by a build or a batch holding the space comes + // back here and finishes lease batches below. + if (auto dropped = Spaces().current.exchange(nullptr); dropped != nullptr) { + Spaces().droppedByWaiter.store(true, std::memory_order_relaxed); + Spaces().waiterDrops.fetch_add(1, std::memory_order_relaxed); + dropped.reset(); + auto& state = Leases(); + std::lock_guard lock(state.mutex); + ++state.stats.contentionWaits; + ++state.stats.cacheDrops; + state.stats.contentionMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + return true; + } if (GuestMemory::GpuMutex().HeldByThisThread()) { std::this_thread::yield(); } else { @@ -658,7 +770,7 @@ void CountLeaseOutcome(bool synced, std::uint64_t batchSerial) { const auto& stats = state.stats; const auto table = BdaResources::TableCacheCounters(); const auto& snapshots = Snapshots(); - std::fprintf(stderr, "[address-sync] leases: %llu released at completion, %llu synced at once; %llu pin-contention waits by guest threads (%llu finished the lease batch, %llu drained the recorder) %.1f s; BDA table cache %llu hits / %llu misses (%zu tables held); snapshot compares: %llu skipped (live-backed region), %llu made\n", static_cast(stats.deferred), static_cast(stats.synced), static_cast(stats.contentionWaits), static_cast(stats.contentionSyncs), static_cast(stats.contentionDrains), stats.contentionMs / 1000, static_cast(table.hits), static_cast(table.misses), table.held, static_cast(snapshots.skipped.load(std::memory_order_relaxed)), static_cast(snapshots.checked.load(std::memory_order_relaxed))); + std::fprintf(stderr, "[address-sync] leases: %llu released at completion, %llu synced at once; %llu pin-contention waits by guest threads (%llu finished the lease batch, %llu drained the recorder, %llu cache-only drops) %.1f s; BDA table cache %llu hits / %llu misses (%zu tables held); snapshot compares: %llu skipped (live-backed region), %llu made\n", static_cast(stats.deferred), static_cast(stats.synced), static_cast(stats.contentionWaits), static_cast(stats.contentionSyncs), static_cast(stats.contentionDrains), static_cast(stats.cacheDrops), stats.contentionMs / 1000, static_cast(table.hits), static_cast(table.misses), table.held, static_cast(snapshots.skipped.load(std::memory_order_relaxed)), static_cast(snapshots.checked.load(std::memory_order_relaxed))); } LeaseStats LeaseCounters() { @@ -667,6 +779,12 @@ LeaseStats LeaseCounters() { return state.stats; } +AddressSpaceStats AddressSpaceCounters() { + const auto& cache = Spaces(); + const auto load = [](const std::atomic& counter) { return counter.load(std::memory_order_relaxed); }; + return {addressSpaceCacheEnabled(), load(cache.hits), load(cache.rebuiltFirst), load(cache.rebuiltGeneration), load(cache.rebuiltEpoch), load(cache.rebuiltDevice), load(cache.rebuiltWaiterDrop), load(cache.unpublished), load(cache.dissolvedOverlap), load(cache.dissolvedImports), load(cache.waiterDrops)}; +} + const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; auto& state = Imports(); @@ -677,10 +795,16 @@ const HostImport* HostImportFor(const Context& context, std::uint64_t address, s } const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); refreshImports(context, state, lease); - if (const auto* range = containingRange(lease, address, address + bytes)) return importAllocation(context, state, range->address, range->bytes); + if (const auto* range = containingRange(lease, address, address + bytes)) return importAllocation(context, state, range->address, range->bytes, lease); return nullptr; } +bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes) { + if (bytes == 0 || bytes > std::numeric_limits::max() - address) return false; + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + return containingRange(lease, address, address + bytes) != nullptr; +} + bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes) { if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return false; auto& state = Imports(); @@ -696,70 +820,191 @@ bool GuestBufferMemory::WritesOverlap(std::uint64_t address, std::size_t bytes) return std::any_of(writes.begin(), writes.end(), [&](const auto& range) { return address < range.second && range.first < address + bytes; }); } +void GuestBufferMemory::CountAddressBuild(double snapshotsUs) { + auto& totals = AddressBuilds(); + auto& timing = ThreadAddressTiming(); + std::lock_guard lock(totals.mutex); + ++totals.builds; + totals.sums.leaseUs += timing.leaseUs; + totals.sums.importsUs += timing.importsUs; + totals.sums.mirrorsUs += timing.mirrorsUs; + totals.sums.compareUs += timing.compareUs; + totals.sums.blocksCompared += timing.blocksCompared; + totals.sums.blocksCopied += timing.blocksCopied; + totals.snapshotsUs += snapshotsUs; + timing = {}; + const auto now = std::chrono::steady_clock::now(); + if (now - totals.lastReport < std::chrono::seconds(10)) return; + totals.lastReport = now; + const auto per = [&](double us) { return us / static_cast(totals.builds); }; + const auto table = BdaResources::TableCacheCounters(); + const auto& seen = totals.tableSeen; + const auto delta = [](std::uint64_t now, std::uint64_t before) { return static_cast(now - before); }; + const auto space = AddressSpaceCounters(); + const auto& spaceSeen = totals.spaceSeen; + std::fprintf(stderr, "[address] %llu address-based builds (10 s), us per build: lease %.0f, imports pass %.0f, mirror prepare %.0f, compare %.0f (%.0f blocks, %.1f copied), snapshots %.0f; space hits %llu / rebuilds: generation %llu, epoch %llu, device %llu, waiter drop %llu, first %llu; unpublished %llu, dissolved: overlap %llu, imports %llu%s; [bda-table] hits %llu / misses %llu (space tables %llu), first entry: expired %llu, hash differs low %llu / heap %llu, same hash %llu, none held %llu\n", static_cast(totals.builds), per(totals.sums.leaseUs), per(totals.sums.importsUs), per(totals.sums.mirrorsUs), per(totals.sums.compareUs), per(static_cast(totals.sums.blocksCompared)), per(static_cast(totals.sums.blocksCopied)), per(totals.snapshotsUs), delta(space.hits, spaceSeen.hits), delta(space.rebuiltGeneration, spaceSeen.rebuiltGeneration), delta(space.rebuiltEpoch, spaceSeen.rebuiltEpoch), delta(space.rebuiltDevice, spaceSeen.rebuiltDevice), delta(space.rebuiltWaiterDrop, spaceSeen.rebuiltWaiterDrop), delta(space.rebuiltFirst, spaceSeen.rebuiltFirst), delta(space.unpublished, spaceSeen.unpublished), delta(space.dissolvedOverlap, spaceSeen.dissolvedOverlap), delta(space.dissolvedImports, spaceSeen.dissolvedImports), space.enabled ? "" : " (cache off)", delta(table.hits, seen.hits), delta(table.misses, seen.misses), delta(table.spaceTables, seen.spaceTables), delta(table.firstExpired, seen.firstExpired), delta(table.firstDiffersLow, seen.firstDiffersLow), delta(table.firstDiffersHeap, seen.firstDiffersHeap), delta(table.firstSameHash, seen.firstSameHash), delta(table.firstEmpty, seen.firstEmpty)); + totals.tableSeen = table; + totals.spaceSeen = space; + totals.builds = 0; + totals.sums = {}; + totals.snapshotsUs = 0; +} + void GuestBufferMemory::AcquireRegistered() { - Require(!uploaded && regions.empty() && lease.empty(), "guest allocation lease must precede resource registration"); + Require(!uploaded && regions.empty() && lease.empty() && space == nullptr, "guest allocation lease must precede resource registration"); ensurePinWaiter(); - // Pending GPU results in these ranges are stored when Upload prepares each region. - lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& timing = ThreadAddressTiming(); + const auto lap = [&](double& into, std::chrono::steady_clock::time_point& from) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + into += std::chrono::duration(now - from).count(); + from = now; + }; + auto at = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; const bool importable = context.hostImportAlignment != 0; - if (importable) { - auto& state = Imports(); - std::lock_guard lock(state.mutex); - refreshImports(context, state, lease); - // Taken before the pass: the imports it takes are the registry's ones (just reconciled, so - // none is retired by a make below), and a mirror's preparation may sync the recorder, whose - // completions can retire an import already taken; that moves the epoch, and Upload then - // re-checks every pointer instead of trusting it. - importsEpoch = state.epoch; - } - // One pass in the registry's (ascending) order, so the regions come out sorted for AddSnapshot. - // The import lock is taken per range: a mirror's preparation may wait for recorded work, which - // must not happen under it. - regions.reserve(lease.size()); + // APS5_NO_SORTED_SNAPSHOT_LOOKUP=1: AddSnapshot scans the regions as before instead of searching. + static const bool sortedLookup = std::getenv("APS5_NO_SORTED_SNAPSHOT_LOOKUP") == nullptr; std::vector blocks; bool mirrored = false; - for (const auto& range : lease) { - if (!range->readable) continue; - validate(range->address, range->bytes); - Region region{range->address, range->address + range->bytes, range->writable, {}, nullptr}; - const HostImport* entry = nullptr; - if (importable) { + auto& spaces = Spaces(); + // Read before the lease below is acquired: a mutation ending in between leaves the space + // stale by its generation, never newer than it (see AddressSpace). + const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + if (addressSpaceCacheEnabled()) { + auto current = spaces.current.load(); + bool hit = false; + { auto& state = Imports(); std::lock_guard lock(state.mutex); - entry = importAllocation(context, state, range->address, range->bytes); + if (current == nullptr) (spaces.droppedByWaiter.exchange(false, std::memory_order_relaxed) ? spaces.rebuiltWaiterDrop : spaces.rebuiltFirst).fetch_add(1, std::memory_order_relaxed); + else if (current->device != context.device) spaces.rebuiltDevice.fetch_add(1, std::memory_order_relaxed); + else if (current->generation != generation) spaces.rebuiltGeneration.fetch_add(1, std::memory_order_relaxed); + else if (current->importsEpoch != state.epoch) spaces.rebuiltEpoch.fetch_add(1, std::memory_order_relaxed); + else hit = true; } - if (entry != nullptr) { - region.hostBacked = true; - // Reused by Upload while no import was dropped since (see importsEpoch). - region.direct = entry; - regions.push_back(std::move(region)); - continue; - } - if (!range->releasable && mirrorsEnabled()) { - // Main image: served by a persistent mirror (see ImageMirror); the writable ones are - // compared with guest memory below, all at once. - region.mirror = acquireMirror(context, range, blocks); - if (region.mirror != nullptr) { + if (hit) { + spaces.hits.fetch_add(1, std::memory_order_relaxed); + space = std::move(current); + lap(timing.leaseUs, at); + for (const auto& range : space->copied) addCopiedRange(range); + regionsSorted = sortedLookup; + lap(timing.importsUs, at); + // Writable mirrors are compared with guest memory per build, as acquireMirror does for + // an existing mirror. The preparation may wait for recorded work, whose completions can + // retire an import the space's regions point at: UploadFinish re-checks the epoch + // before any of them is dereferenced. + for (const auto& region : space->base) { + if (region.mirror == nullptr) continue; mirrored = true; + if (region.mirror->writable && prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); + } + lap(timing.mirrorsUs, at); + } + } + if (space == nullptr) { + // Pending GPU results in these ranges are stored when Upload prepares each region. + lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + lap(timing.leaseUs, at); + { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + if (importable) refreshImports(context, state, lease); + // Taken before the pass: the imports it takes are the registry's ones (just reconciled, so + // none is retired by a make below), and a mirror's preparation may sync the recorder, whose + // completions can retire an import already taken; that moves the epoch, and Upload then + // re-checks every pointer instead of trusting it. + importsEpoch = state.epoch; + } + lap(timing.importsUs, at); + // One pass in the registry's (ascending) order, so the regions come out sorted for AddSnapshot. + // The import lock is taken per range: a mirror's preparation may wait for recorded work, which + // must not happen under it. + regions.reserve(lease.size()); + for (const auto& range : lease) { + if (!range->readable) continue; + validate(range->address, range->bytes); + Region region{range->address, range->address + range->bytes, range->writable, {}, nullptr}; + const HostImport* entry = nullptr; + if (importable) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + entry = importAllocation(context, state, range->address, range->bytes, lease); + } + lap(timing.importsUs, at); + if (entry != nullptr) { + region.hostBacked = true; + // Reused by Upload while no import was dropped since (see importsEpoch). + region.direct = entry; regions.push_back(std::move(region)); continue; } + if (!range->releasable && mirrorsEnabled()) { + // Main image: served by a persistent mirror (see ImageMirror); the writable ones are + // compared with guest memory below, all at once. + region.mirror = acquireMirror(context, range, blocks); + lap(timing.mirrorsUs, at); + if (region.mirror != nullptr) { + mirrored = true; + regions.push_back(std::move(region)); + continue; + } + } + addCopiedRange({range->address, range->address + range->bytes, range->writable}); } - // Read-only ranges are read from live guest memory at Upload: the lease pins them and the guest - // cannot write them, so that equals a snapshot taken now without the extra copy. Reserved heaps - // are registered whole while the guest commits pages on demand: committed pages only. - region.hostBacked = !range->writable; - auto committed = GuestMemory::DescribeCommitted(range->address, range->bytes, range->writable); - if (!committed.whole) { - region.sparse = true; - region.backed = std::move(committed.ranges); + regionsSorted = sortedLookup; + lap(timing.importsUs, at); + if (addressSpaceCacheEnabled()) { + // Published only while the imports the regions point at stand: the table entries below + // read them, and a retire during the pass (a mirror preparation's sync) has already + // moved the epoch, in which case this build keeps the per-build form and Upload + // re-resolves every pointer as before. + std::vector ranges; + bool publish = true; + { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + if (state.epoch != importsEpoch) { + publish = false; + spaces.unpublished.fetch_add(1, std::memory_order_relaxed); + } else { + for (const auto& region : regions) { + if (region.direct != nullptr || region.mirror != nullptr) ranges.push_back(addressRange(region)); + } + } + } + if (publish) { + auto built = std::make_shared(); + built->generation = generation; + built->importsEpoch = importsEpoch; + built->device = context.device; + built->serial = spaces.serials.fetch_add(1, std::memory_order_relaxed) + 1; + built->lease = std::move(lease); + built->ranges = std::move(ranges); + std::vector extras; + for (auto& region : regions) { + if (region.direct != nullptr || region.mirror != nullptr) { + built->base.push_back(std::move(region)); + } else { + built->copied.push_back({region.begin, region.end, region.writable}); + extras.push_back(std::move(region)); + } + } + regions = std::move(extras); + lease.clear(); + space = std::move(built); + spaces.droppedByWaiter.store(false, std::memory_order_relaxed); + spaces.current.store(space); + } } - regions.push_back(std::move(region)); } - // APS5_NO_SORTED_SNAPSHOT_LOOKUP=1: AddSnapshot scans the regions as before instead of searching. - static const bool sortedLookup = std::getenv("APS5_NO_SORTED_SNAPSHOT_LOOKUP") == nullptr; - regionsSorted = sortedLookup; + const auto copiedBefore = Mirrors().blocksCopied; compareBlocks(blocks); + lap(timing.compareUs, at); + if (profile) { + timing.blocksCompared += blocks.size(); + timing.blocksCopied += Mirrors().blocksCopied - copiedBefore; + } if (mirrorsEnabled()) { if (mirrored) ++Mirrors().builds; sweepMirrors(); @@ -773,11 +1018,89 @@ void GuestBufferMemory::validate(std::uint64_t address, std::size_t bytes) const Require(bytes <= std::numeric_limits::max() - address, "guest memory range overflow"); } -void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes) { +void GuestBufferMemory::addCopiedRange(const CopiedRange& range) { + validate(range.begin, range.end - range.begin); + Region region{range.begin, range.end, range.writable, {}, nullptr}; + // Read-only ranges are read from live guest memory at Upload: the lease pins them and the guest + // cannot write them, so that equals a snapshot taken now without the extra copy. Reserved heaps + // are registered whole while the guest commits pages on demand: committed pages only. + region.hostBacked = !range.writable; + auto committed = GuestMemory::DescribeCommitted(range.begin, range.end - range.begin, range.writable); + if (!committed.whole) { + region.sparse = true; + region.backed = std::move(committed.ranges); + } + regions.push_back(std::move(region)); +} + +GuestBufferMemory::BaseOverlap GuestBufferMemory::baseOverlap(std::uint64_t begin, std::uint64_t end, const Region** owner) const { + if (space == nullptr || space->base.empty()) return BaseOverlap::None; + const auto& base = space->base; + const auto found = std::upper_bound(base.begin(), base.end(), begin, [](std::uint64_t value, const Region& region) { return value < region.begin; }); + if (found != base.begin()) { + const auto& previous = *std::prev(found); + if (begin < previous.end) { + if (end > previous.end) return BaseOverlap::Partial; + if (owner != nullptr) *owner = &previous; + return BaseOverlap::Inside; + } + } + return found != base.end() && found->begin < end ? BaseOverlap::Partial : BaseOverlap::None; +} + +void GuestBufferMemory::dissolveSpace(bool resolve) { + Require(space != nullptr, "no address space to dissolve"); + std::vector merged; + merged.reserve(space->base.size() + regions.size()); + if (prepared) { + // Both sorted and disjoint (UploadPrepare merged this build's regions; the space's are + // registered ranges), so the merge keeps the order Descriptor searches. + std::merge(space->base.begin(), space->base.end(), std::make_move_iterator(regions.begin()), std::make_move_iterator(regions.end()), std::back_inserter(merged), [](const Region& left, const Region& right) { return left.begin < right.begin; }); + } else { + merged.assign(space->base.begin(), space->base.end()); + merged.insert(merged.end(), std::make_move_iterator(regions.begin()), std::make_move_iterator(regions.end())); + regionsSorted = false; + } + if (resolve) { + for (auto& region : merged) { + if (region.direct != nullptr) region.pending = true; + } + } + lease = space->lease; + importsEpoch = space->importsEpoch; + regions = std::move(merged); + space.reset(); +} + +const GuestBufferMemory::Region* GuestBufferMemory::owner(std::uint64_t address) const { + // The region with the greatest begin at or below the address, as the one merged list gave; + // the caller checks the end, as before. + const auto candidate = [address](const std::vector& list) -> const Region* { + const auto found = std::upper_bound(list.begin(), list.end(), address, [](std::uint64_t value, const Region& region) { return value < region.begin; }); + return found == list.begin() ? nullptr : &*std::prev(found); + }; + const auto* own = candidate(regions); + const auto* shared = space != nullptr ? candidate(space->base) : nullptr; + if (own == nullptr || (shared != nullptr && shared->begin > own->begin)) return shared; + return own; +} + +void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic) { validate(address, bytes); + switch (baseOverlap(address, address + bytes, nullptr)) { + case BaseOverlap::Inside: + return; + case BaseOverlap::Partial: + Spaces().dissolvedOverlap.fetch_add(1, std::memory_order_relaxed); + dissolveSpace(false); + break; + case BaseOverlap::None: + break; + } // `writable` here means the bytes come from live guest memory (not a snapshot), whether or not // the shader stores to them; what is written back is decided by Writes() alone. Region region{address, address + bytes, true, {}, nullptr}; + region.atomic = atomic; auto committed = GuestMemory::DescribeCommitted(address, bytes, true); if (!committed.whole) { // A GPU heap bound whole while the guest commits its pages on demand, or a descriptor left @@ -790,8 +1113,8 @@ void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t b regionsSorted = false; } -void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes) { - addDescriptorRegion(address, bytes); +void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes, bool atomic) { + addDescriptorRegion(address, bytes, atomic); writes.emplace_back(address, address + bytes); } @@ -799,7 +1122,7 @@ void GuestBufferMemory::AddReadable(std::uint64_t address, std::size_t bytes) { // Not in `writes`: no reference copy (copyRegion), no write-back, no pending-write note, no // direct-write mark, and UploadPrepare copies it without the device lock. A written descriptor // overlapping the range still covers it through its own Writes() entry. - addDescriptorRegion(address, bytes); + addDescriptorRegion(address, bytes, false); } void GuestBufferMemory::AddSnapshot(const GuestMemorySnapshot& snapshot) { @@ -810,10 +1133,14 @@ void GuestBufferMemory::AddSnapshot(const GuestMemorySnapshot& snapshot) { // ranges), so the one that can contain the snapshot is found by search rather than by scanning // the ~1200 of an address-based build for each captured region. const Region* owner = nullptr; - if (regionsSorted) { + if (baseOverlap(snapshot.address, end, &owner) == BaseOverlap::Partial) { + Spaces().dissolvedOverlap.fetch_add(1, std::memory_order_relaxed); + dissolveSpace(false); + } + if (owner == nullptr && regionsSorted) { const auto found = std::upper_bound(regions.begin(), regions.end(), snapshot.address, [](std::uint64_t value, const Region& region) { return value < region.begin; }); if (found != regions.begin() && end <= std::prev(found)->end) owner = &*std::prev(found); - } else { + } else if (owner == nullptr) { for (const auto& region : regions) { if (region.begin <= snapshot.address && end <= region.end) { owner = ®ion; @@ -836,7 +1163,9 @@ void GuestBufferMemory::AddSnapshot(const GuestMemorySnapshot& snapshot) { static const bool checkLive = std::getenv("APS5_NO_SNAPSHOT_CHECK") == nullptr; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; const bool live = owner->writable || owner->hostBacked || owner->mirror != nullptr; - if (live && !checkLive) { + // A live region over a unit shadow's fresh results reads the import's stale bytes here + // (the publish happens when the region is bound, under the device lock): not compared. + if (live && (!checkLive || AnyShadowedOverlaps(snapshot.address, snapshot.bytes.size()))) { if (profile) Snapshots().skipped.fetch_add(1, std::memory_order_relaxed); return; } @@ -894,7 +1223,69 @@ VkBufferUsageFlags gpuCopyUsage(bool addressable) { return VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); } -// APS5_PROFILE_DRAW: the GPU copies, reported in the [buffers] uploads line (cumulative). +// Device-local staging. A written V# element bound in place makes the shader store into host +// memory over PCIe: the froxel culling kernel's ~248K scattered 4-byte stores took 2.8 ms per +// dispatch and the histogram kernels' ~60K atomics on a 1 KB bin table 1.4 ms, on a GPU that is +// otherwise idle. Instead an element the recompiler marks written (DescriptorBinding::bufferWritten, +// within a size window) or atomic (bufferAtomic, up to its own cap) that lies inside a host import +// takes the gpuCopy path above with a DEVICE_LOCAL buffer: copied in from the import before the +// work (never skipped for a write-only element: the copy-back is the whole written range, so every +// byte the shader leaves alone must hold the import's bytes), bound in the import's place, and +// copied back by RecordCopyBacks after the work, with the pending-write note and the direct-write +// marks of an in-place write. Only builds whose every use records the work and calls +// RecordCopyBacks take it (dispatches, see AllowDeviceStaging; a synchronous draw would store the +// staging bytes from the CPU, which a device-local buffer cannot serve), and never address-based +// ones (their pointers read the import in place, and AddressRanges needs a device address). The +// shadow is per use, not resident: a reused build records its copy-in anew (RecordStagingCopies). +// Accepted trade-off, as for the misaligned copies above: a CPU write into a staged range between +// the copy-in and the copy-back is rolled back by the copy-back (the whole written element range, +// since store extents are not known), for the batch's latency; the staging window keeps the +// exposure to the elements named on the [buffers] staging line and APS5_TRACE_STAGING. +// APS5_NO_WRITTEN_SHADOW=1 and APS5_NO_ATOMIC_STAGING=1 bind those elements in place as before; +// APS5_WRITTEN_SHADOW_MIN_KIB / APS5_WRITTEN_SHADOW_MAX_KIB (16 / 2048) bound the written window +// (a kernel streaming once through a large buffer, the engine's memcpy kernel over 4-8 MiB video +// frames, would only gain the two copies), APS5_ATOMIC_STAGE_MAX_KIB (1024) the atomic one. +bool writtenShadowEnabled() { + static const bool disabled = std::getenv("APS5_NO_WRITTEN_SHADOW") != nullptr; + return !disabled; +} + +bool atomicStagingEnabled() { + static const bool disabled = std::getenv("APS5_NO_ATOMIC_STAGING") != nullptr; + return !disabled; +} + +std::uint64_t kibSetting(const char* name, std::uint64_t defaultKib) { + const char* value = std::getenv(name); + return (value != nullptr ? std::strtoull(value, nullptr, 10) : defaultKib) << 10u; +} + +std::uint64_t writtenShadowMin() { + static const std::uint64_t bytes = kibSetting("APS5_WRITTEN_SHADOW_MIN_KIB", 16); + return bytes; +} + +std::uint64_t writtenShadowMax() { + static const std::uint64_t bytes = kibSetting("APS5_WRITTEN_SHADOW_MAX_KIB", 2048); + return bytes; +} + +std::uint64_t atomicStageMax() { + static const std::uint64_t bytes = kibSetting("APS5_ATOMIC_STAGE_MAX_KIB", 1024); + return bytes; +} + +VkMemoryPropertyFlags copyBufferProperties(bool deviceLocal) { + return deviceLocal ? VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); +} + +// [gputime] classes (APS5_PROFILE_GPU) of a build's GPU copies, barriers included: the copy-ins +// of its regions and their copy-backs (staged or host shadows alike). +constexpr auto StagingCopyInKey = Recorder::CommandClass::StagingIn; +constexpr auto StagingCopyBackKey = Recorder::CommandClass::StagingOut; + +// APS5_PROFILE_DRAW: the GPU copies, reported in the [buffers] uploads line (cumulative), and the +// device-local staging counts, reported as [buffers] staging every 10 s (see reportStaging). struct CopyStats { std::atomic gpuCopies{0}; std::atomic gpuCopyBytes{0}; @@ -903,6 +1294,17 @@ struct CopyStats { // Written gpuCopy regions of a use that recorded no copy-back (a synchronous draw): stored // from the staging buffer by the CPU in WriteBack. std::atomic stagingStores{0}; + // Staged regions copied in (every use), those with an atomic element, those of a reused build, + // the bytes copied in and back, staged regions whose use recorded a copy-in but no copy-back + // (counted when the build is reused or destroyed), and shadows the device refused. + std::atomic staged{0}; + std::atomic stagedAtomic{0}; + std::atomic stagedReused{0}; + std::atomic stagedInBytes{0}; + std::atomic stagedOutBytes{0}; + std::atomic stagedLost{0}; + std::atomic stagingRefused{0}; + std::atomic lastStagingReport{0}; }; CopyStats& Copies() { @@ -910,6 +1312,58 @@ CopyStats& Copies() { return stats; } +// A shadow the device cannot provide (device memory or the allocation count exhausted: every +// shadow is its own VkDeviceMemory, retained by cached builds) is no error: nullptr, and the +// region takes the path it would take without staging (Region::unstaged). +std::shared_ptr stagingBuffer(const Context& context, std::size_t bytes, VkBufferUsageFlags usage) { + try { + return std::make_shared(context, bytes, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + } catch (const std::exception& error) { + static std::atomic reported{0}; + if (reported.fetch_add(1, std::memory_order_relaxed) < 4) std::fprintf(stderr, "[buffers] staging shadow of %zu bytes refused: %s\n", bytes, error.what()); + Copies().stagingRefused.fetch_add(1, std::memory_order_relaxed); + return nullptr; + } +} + +// APS5_TRACE_STAGING=1: every distinct staged range once, with its size and whether an atomic +// element lies in it, to see which elements the staging window admits at a stage. +void traceStaged(std::uint64_t begin, std::uint64_t end, bool atomic) { + static const bool trace = std::getenv("APS5_TRACE_STAGING") != nullptr; + if (!trace) return; + static std::mutex mutex; + static std::set> seen; + std::lock_guard lock(mutex); + if (!seen.insert({begin, end}).second) return; + std::fprintf(stderr, "[staging] 0x%llx+0x%llx (%.1f KiB)%s\n", static_cast(begin), static_cast(end - begin), (end - begin) / 1024.0, atomic ? " atomic" : ""); +} + +void reportStaging() { + auto& stats = Copies(); + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = stats.lastStagingReport.load(); + if (nowMs - last < 10000 || !stats.lastStagingReport.compare_exchange_strong(last, nowMs)) return; + // One reporter per period (the exchange above), so the previous totals need no lock. + static std::uint64_t lastStaged = 0, lastIn = 0, lastOut = 0; + const auto staged = stats.staged.load(); + const auto in = stats.stagedInBytes.load(); + const auto out = stats.stagedOutBytes.load(); + std::fprintf(stderr, "[buffers] staging: %llu regions staged device-local (%llu with atomics, %llu of reused builds), %.0f KiB copied in, %.0f KiB copied back, %llu without copy-back, %llu shadows refused; last 10 s: %llu regions, %.0f KiB in, %.0f KiB back\n", static_cast(staged), static_cast(stats.stagedAtomic.load()), static_cast(stats.stagedReused.load()), in / 1024.0, out / 1024.0, static_cast(stats.stagedLost.load()), static_cast(stats.stagingRefused.load()), static_cast(staged - lastStaged), (in - lastIn) / 1024.0, (out - lastOut) / 1024.0); + lastStaged = staged; + lastIn = in; + lastOut = out; +} + +} + +GuestBufferMemory::~GuestBufferMemory() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + // A staged region whose last use recorded its copy-in but no copy-back (the use threw before + // RecordCopyBacks, or never dispatched): whatever the shader stored stayed in the shadow. + for (const auto& region : regions) { + if (region.gpuCopy && region.deviceLocal && !region.copiedBack) Copies().stagedLost.fetch_add(1, std::memory_order_relaxed); + } } bool GuestBufferMemory::gpuCopyEligible(const Region& region) const { @@ -917,6 +1371,14 @@ bool GuestBufferMemory::gpuCopyEligible(const Region& region) const { return region.end - region.begin <= gpuCopyLimit(); } +bool GuestBufferMemory::stagingEligible(const Region& region, bool addressable) const { + if (!stagingAllowed || addressable || region.unstaged || !gpuCopiesEnabled() || region.sparse || region.mirror != nullptr) return false; + const auto bytes = region.end - region.begin; + if (!WritesOverlap(region.begin, static_cast(bytes))) return false; + if (region.atomic && atomicStagingEnabled() && bytes <= atomicStageMax()) return true; + return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= writtenShadowMax(); +} + void GuestBufferMemory::UploadPrepare(bool addressable) { Require(!prepared && !uploaded, "guest memory was already uploaded"); prepared = true; @@ -953,6 +1415,7 @@ void GuestBufferMemory::UploadPrepare(bool addressable) { previous.hostBacked = true; } mergeBacked(previous, region); + previous.atomic = previous.atomic || region.atomic; // A range starting before the mirror (only possible after the swap) keeps its prefix; the // earlier merged region ends at or before it, so the merged list stays sorted. previous.begin = std::min(previous.begin, region.begin); @@ -1021,12 +1484,30 @@ void GuestBufferMemory::UploadPrepare(bool addressable) { // Misaligned inside an import: copied, whatever the registry does meanwhile. By the GPU // when eligible (see Region::gpuCopy): UploadFinish records the copy from the import it // confirms then (the pointer is provisional, as for an aligned region); the buffer the - // copy fills is made here, without the device lock, and serves the CPU fallback too. - if ((region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment != 0) { - if (!gpuCopyEligible(region)) continue; + // copy fills is made here, without the device lock, and serves the CPU fallback too. A + // staged region (see stagingEligible) takes the same path, aligned or not, with a + // device-local buffer that the CPU fallback replaces (none without a recorder to + // record the copies: UploadFinish then binds the region in place). + bool staged = Recorder::Active() != nullptr && stagingEligible(region, addressable); + const bool misaligned = (region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment != 0; + if (staged) { const auto allocateStart = std::chrono::steady_clock::now(); - region.buffer = std::make_shared(context, static_cast(region.end - region.begin), gpuCopyUsage(addressable)); + region.buffer = stagingBuffer(context, static_cast(region.end - region.begin), gpuCopyUsage(addressable)); if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); + if (region.buffer == nullptr) { + // No shadow to be had: in place when aligned, else the host copy below. + region.unstaged = true; + staged = false; + } + } + if (misaligned || staged) { + if (!staged) { + if (!gpuCopyEligible(region)) continue; + const auto allocateStart = std::chrono::steady_clock::now(); + region.buffer = std::make_shared(context, static_cast(region.end - region.begin), gpuCopyUsage(addressable), copyBufferProperties(false)); + if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); + } + region.deviceLocal = staged; region.direct = entry; region.pending = true; continue; @@ -1063,8 +1544,10 @@ void GuestBufferMemory::UploadFinish(bool addressable) { // keep managing memory. GuestAllocations::Lease acquired; const auto importRanges = [&]() -> const GuestAllocations::Lease& { - if (lease.empty() && acquired.empty()) acquired = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); - return lease.empty() ? acquired : lease; + if (!lease.empty()) return lease; + if (space != nullptr) return space->lease; + if (acquired.empty()) acquired = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + return acquired; }; // Sub-range mirrors were found by UploadPrepare, possibly long before this lock was taken: one // whose Range went meanwhile (the guest reprotected its image) is dropped as findMirror would @@ -1084,10 +1567,29 @@ void GuestBufferMemory::UploadFinish(bool addressable) { ++Mirrors().subranges; } bool importsRefreshed = false; + if (space != nullptr && context.hostImportAlignment != 0) { + // The space's regions were confirmed against the imports when it was built; one check that + // none was retired since (a mirror refresh or a flush between the stages can reconcile + // them). If one was, every region takes the per-build path below, which re-resolves each + // pointer instead of trusting it. + auto& state = Imports(); + std::lock_guard lock(state.mutex); + if (importsStale(context, state)) refreshImports(context, state, importRanges()); + importsRefreshed = true; + if (state.epoch != space->importsEpoch) { + Spaces().dissolvedImports.fetch_add(1, std::memory_order_relaxed); + dissolveSpace(true); + } + } // Regions the GPU copies out of their import (see Region::gpuCopy), recorded together below. // No recorder (tests) keeps them on the CPU path. auto* recorder = Recorder::Active(); std::vector gpuCopies; + // One clock pair around the loop (an address-based build has ~1200 regions): the [buffers] + // 'import lookup' is the loop less the mirror refreshes and the CPU copies, timed apart. + const auto loopStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + std::uint64_t refreshUs = 0; + std::uint64_t copyUs = 0; for (auto& region : regions) { if (!region.pending) continue; region.pending = false; @@ -1096,14 +1598,13 @@ void GuestBufferMemory::UploadFinish(bool addressable) { // A writable sub-range mirror: refreshed here, where waiting for recorded work is allowed. // Read-only mirrors are never written back; a writable one is diffed for this sub-range // at write-back. - const auto refreshStart = std::chrono::steady_clock::now(); + const auto refreshStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; refreshMirror(*region.mirror, region.begin, bytes); - if (profile) readUs.fetch_add(microsecondsSince(refreshStart), std::memory_order_relaxed); + if (profile) refreshUs += microsecondsSince(refreshStart); continue; } bool copyOnGpu = false; if (context.hostImportAlignment != 0) { - const auto lookupStart = std::chrono::steady_clock::now(); auto& state = Imports(); std::lock_guard lock(state.mutex); // Imports are made on demand for the registered range around each region. @@ -1117,23 +1618,28 @@ void GuestBufferMemory::UploadFinish(bool addressable) { region.direct = nullptr; if (entry == nullptr) entry = findImport(state, region.begin, region.end); if (entry == nullptr) { - if (const auto* range = containingRange(importRanges(), region.begin, region.end)) entry = importAllocation(context, state, range->address, range->bytes); + if (const auto* range = containingRange(importRanges(), region.begin, region.end)) entry = importAllocation(context, state, range->address, range->bytes, importRanges()); } - if (entry != nullptr && (region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment == 0) { + // A staged region (see stagingEligible) is copied out of the import even when aligned; + // without a recorder to record the copies it binds in place like any other. + const bool staged = recorder != nullptr && stagingEligible(region, addressable); + if (entry != nullptr && !staged && (region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment == 0) { region.direct = entry; region.snapshot.clear(); // A buffer UploadPrepare made for a GPU copy is not needed: the import serves the // region in place (Descriptor prefers `direct`), and a kept buffer would count as copied. region.buffer.reset(); - } else if (entry != nullptr && recorder != nullptr && gpuCopyEligible(region)) { - // Misaligned in the import: the GPU copies the sub-range out of it. The handle and - // base are taken now, under the import lock: a reconcile by the flush below could - // retire the entry (the batch opened below keeps a retired import's buffer alive). + region.deviceLocal = false; + } else if (entry != nullptr && recorder != nullptr && (staged || gpuCopyEligible(region))) { + // Misaligned in the import (or staged): the GPU copies the sub-range out of it. The + // handle and base are taken now, under the import lock: a reconcile by the flush + // below could retire the entry (the batch opened below keeps a retired import's + // buffer alive). copyOnGpu = true; + region.deviceLocal = staged; region.copySource = entry->buffer; region.copySourceBase = entry->base; } - if (profile) importUs.fetch_add(microsecondsSince(lookupStart), std::memory_order_relaxed); } if (region.direct != nullptr) { // The GPU reads this range in place, so storage results pending in it must be stored first @@ -1156,10 +1662,21 @@ void GuestBufferMemory::UploadFinish(bool addressable) { gpuCopies.push_back(®ion); continue; } + if (region.deviceLocal) { + // The staging shadow UploadPrepare made cannot take the CPU copy (no host mapping): the + // region falls back to a host buffer of its own. + region.buffer.reset(); + region.deviceLocal = false; + } + const auto copyStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; copyRegion(region, addressable); + if (profile) copyUs += microsecondsSince(copyStart); } if (!gpuCopies.empty()) recordGpuCopies(gpuCopies, addressable); if (!profile) return; + const auto loopUs = microsecondsSince(loopStart); + readUs.fetch_add(refreshUs, std::memory_order_relaxed); + importUs.fetch_add(loopUs > refreshUs + copyUs ? loopUs - refreshUs - copyUs : 0, std::memory_order_relaxed); const auto uploads = uploadsProfiled.fetch_add(1, std::memory_order_relaxed) + 1; if (uploads % 2000 == 0) { const auto& copies = Copies(); @@ -1252,35 +1769,70 @@ void GuestBufferMemory::recordGpuCopies(std::span copies, bool ad static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; auto* recorder = Recorder::Active(); Require(recorder != nullptr, "GPU buffer copies need an active recorder"); + // A queued DCC key store or label store over a copied range lands before the copy reads it. + for (const auto* region : copies) { + recorder->FlushKeyStoresOverlapping(region->begin, static_cast(region->end - region->begin)); + recorder->FlushStoresOverlapping(region->begin, static_cast(region->end - region->begin)); + } const auto commands = recorder->Commands(); + const auto timing = recorder->BeginGpuTiming(StagingCopyInKey); + if (Recorder::BarrierValidate()) { + std::vector> reads; + for (const auto* region : copies) reads.emplace_back(region->begin, region->end); + recorder->NoteAccess(Recorder::CommandClass::StagingIn, Recorder::Access{reads, {}, {}, VK_PIPELINE_STAGE_TRANSFER_BIT}); + } // Every earlier recorded store into the imports (dispatches in place, fills, GPU labels, // GPU-direct write-backs and the flushes just recorded) and the host's own writes precede the // copies. A CPU write made after the batch is submitted is the game's race, as on hardware. RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + Recorder::CountBarriers(Recorder::CommandClass::StagingIn); + bool stagedAny = false; + std::uint64_t copiedBytes = 0; for (auto* region : copies) { const auto bytes = region->end - region->begin; + copiedBytes += bytes; if (region->buffer == nullptr) { // Not made by UploadPrepare (the registry was stale then): made here, under the lock. const auto allocateStart = std::chrono::steady_clock::now(); - region->buffer = std::make_shared(context, static_cast(bytes), gpuCopyUsage(addressable)); + if (region->deviceLocal) region->buffer = stagingBuffer(context, static_cast(bytes), gpuCopyUsage(addressable)); + if (region->buffer == nullptr) { + // Or no shadow to be had: a host copy serves the region instead, aligned or not. + region->deviceLocal = false; + region->unstaged = true; + region->buffer = std::make_shared(context, static_cast(bytes), gpuCopyUsage(addressable), copyBufferProperties(false)); + } if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); } CopyBuffer(context, commands, region->copySource, region->begin - region->copySourceBase, region->buffer->Handle(), 0, bytes); + // The copy reads the import in place when the batch runs: a CPU store into the range before + // then would be copied instead of the bytes recorded here. + recorder->NotePendingRead(region->begin, static_cast(bytes), Recorder::ReadKind::GpuCopy); // Kept by the batch at record time, as every other recorded target is: the caller keeps // the resources only after descriptor and pipeline work that may throw, and a released // buffer would go back to the pool (reused or destroyed) under this copy command. The // source is a live import or one retireImport already handed to this batch. recorder->Keep(region->buffer); region->snapshot.clear(); + if (region->deviceLocal) traceStaged(region->begin, region->end, region->atomic); if (profile) { Copies().gpuCopies.fetch_add(1, std::memory_order_relaxed); Copies().gpuCopyBytes.fetch_add(bytes, std::memory_order_relaxed); + if (region->deviceLocal) { + stagedAny = true; + Copies().staged.fetch_add(1, std::memory_order_relaxed); + Copies().stagedInBytes.fetch_add(bytes, std::memory_order_relaxed); + if (region->atomic) Copies().stagedAtomic.fetch_add(1, std::memory_order_relaxed); + } } } + if (stagedAny) reportStaging(); // The copied bytes are visible to the shaders bound to the buffers, which may also store into // them. Every stage: dispatches and draws share this path, and draws run mesh, task and // tessellation shaders too (the batch's own barriers around dispatches are as wide). RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + Recorder::CountBarriers(Recorder::CommandClass::StagingIn); + recorder->MarkCovered(VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + recorder->EndGpuTiming(timing, copiedBytes); } void GuestBufferMemory::RecordCopyBacks(Recorder& recorder) { @@ -1290,6 +1842,8 @@ void GuestBufferMemory::RecordCopyBacks(Recorder& recorder) { std::vector> merged; bool recording = false; VkCommandBuffer commands = VK_NULL_HANDLE; + auto timing = Recorder::NoTiming; + std::uint64_t copiedBytes = 0; for (auto& region : regions) { if (!region.gpuCopy || region.copiedBack) continue; if (merged.empty() && !writes.empty()) { @@ -1306,17 +1860,21 @@ void GuestBufferMemory::RecordCopyBacks(Recorder& recorder) { if (from >= to) continue; if (!recording) { commands = recorder.Commands(); + timing = recorder.BeginGpuTiming(StagingCopyBackKey); // The shader's stores into the buffer (whichever stage made them) precede the copy. RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + Recorder::CountBarriers(Recorder::CommandClass::StagingOut); recording = true; } // Exactly the sub-range the shader may write, in place in the import: what the shader // would have stored there itself had the range been bindable. CopyBuffer(context, commands, region.buffer->Handle(), from - region.begin, region.copySource, from - region.copySourceBase, to - from); + copiedBytes += to - from; recorder.Keep(region.buffer); if (profile) { Copies().gpuCopyBacks.fetch_add(1, std::memory_order_relaxed); Copies().gpuCopyBackBytes.fetch_add(to - from, std::memory_order_relaxed); + if (region.deviceLocal) Copies().stagedOutBytes.fetch_add(to - from, std::memory_order_relaxed); } } // Only once its copies are recorded: a throw above leaves the region counted by @@ -1325,18 +1883,23 @@ void GuestBufferMemory::RecordCopyBacks(Recorder& recorder) { region.copiedBack = true; } if (!recording) return; + if (Recorder::BarrierValidate()) recorder.NoteAccess(Recorder::CommandClass::StagingOut, Recorder::Access{{}, merged, {}, VK_PIPELINE_STAGE_TRANSFER_BIT}); // As a GPU label store or fill: visible to everything recorded after (shaders, transfers, an // indirect dispatch's arguments) and to the host once the batch completed; the caller's // pending-write note makes CPU readers wait for that. - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT); + constexpr VkAccessFlags copiedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, copiedAccess); + Recorder::CountBarriers(Recorder::CommandClass::StagingOut); + recorder.MarkCovered(copiedAccess); + recorder.EndGpuTiming(timing, copiedBytes); } VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std::size_t bytes) const { Require(uploaded && !committed, "guest GPU memory is not available"); Require(bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest buffer view"); - const auto found = std::upper_bound(regions.begin(), regions.end(), address, [](std::uint64_t value, const Region& region) { return value < region.begin; }); - Require(found != regions.begin(), "guest buffer has no GPU owner"); - const auto& region = *std::prev(found); + const auto* found = owner(address); + Require(found != nullptr, "guest buffer has no GPU owner"); + const auto& region = *found; Require(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr), "guest buffer view exceeds its GPU owner"); const auto base = region.direct != nullptr ? region.direct->base : region.mirror != nullptr ? region.mirror->base : region.begin; const auto offset = address - base; @@ -1346,27 +1909,44 @@ VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std: return {handle, offset, bytes}; } +ShaderRecompiler::BdaAbi::Range GuestBufferMemory::addressRange(const Region& region) { + Require(region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr, "incomplete guest GPU upload"); + const auto address = region.direct != nullptr ? region.direct->address + (region.begin - region.direct->base) : region.mirror != nullptr ? region.mirror->buffer->DeviceAddress() + (region.begin - region.mirror->base) : region.buffer->DeviceAddress(); + Require(region.end - region.begin <= std::numeric_limits::max() - address, "GPU address range overflow"); + return {region.begin, region.end, address, ShaderRecompiler::BdaAbi::Read, 0}; +} + std::vector GuestBufferMemory::AddressRanges() const { Require(uploaded && !committed, "guest GPU address ranges are not available"); std::vector result; - for (const auto& region : regions) { - Require(region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr, "incomplete guest GPU upload"); - const auto address = region.direct != nullptr ? region.direct->address + (region.begin - region.direct->base) : region.mirror != nullptr ? region.mirror->buffer->DeviceAddress() + (region.begin - region.mirror->base) : region.buffer->DeviceAddress(); - Require(region.end - region.begin <= std::numeric_limits::max() - address, "GPU address range overflow"); - result.push_back({region.begin, region.end, address, ShaderRecompiler::BdaAbi::Read, 0}); - } - return result; + result.reserve(regions.size() + (space != nullptr ? space->ranges.size() : 0)); + for (const auto& region : regions) result.push_back(addressRange(region)); + if (space == nullptr) return result; + // The table is searched by address (the recompiler's lookup bisects it): both lists are sorted + // and disjoint, so a merge keeps it so. + std::vector merged; + merged.reserve(result.size() + space->ranges.size()); + std::merge(space->ranges.begin(), space->ranges.end(), result.begin(), result.end(), std::back_inserter(merged), [](const auto& left, const auto& right) { return left.begin < right.begin; }); + return merged; +} + +std::optional GuestBufferMemory::CachedAddressTable() const { + if (space == nullptr || !regions.empty()) return std::nullopt; + Require(uploaded && !committed, "guest GPU address ranges are not available"); + return CachedTable{space->serial, &space->ranges}; } bool GuestBufferMemory::HasCopiedWrites() const { for (const auto& [begin, end] : writes) { - const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); - if (found == regions.begin()) continue; - const auto& region = *std::prev(found); + const auto* found = owner(begin); + if (found == nullptr) continue; + const auto& region = *found; if (region.direct != nullptr) continue; // A GPU copy whose written sub-ranges were copied back by the GPU lands like a direct // write; until RecordCopyBacks ran it still counts (a use without it stores in WriteBack). - if (region.gpuCopy && region.copiedBack) continue; + // A staged copy never counts: its use always records the copy-back (AllowDeviceStaging), + // and nothing could be stored from it by the CPU. + if (region.gpuCopy && (region.copiedBack || region.deviceLocal)) continue; // Writes into a read-only image mirror are never stored (its pages could not take them). if (region.mirror != nullptr && !region.mirror->writable) continue; return true; @@ -1376,9 +1956,9 @@ bool GuestBufferMemory::HasCopiedWrites() const { void GuestBufferMemory::MarkDirectWrites() const { for (const auto& [begin, end] : writes) { - const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); - if (found == regions.begin()) continue; - const auto& region = *std::prev(found); + const auto* found = owner(begin); + if (found == nullptr) continue; + const auto& region = *found; if (region.direct != nullptr || (region.gpuCopy && region.copiedBack)) GuestMemory::MarkWritten(begin, static_cast(end - begin)); } } @@ -1394,11 +1974,11 @@ void GuestBufferMemory::WriteBack() { // The CPU keeps running while the GPU works, so storing whole ranges would roll back its writes to // bytes the shader never touched. Only runs that differ from the uploaded copy are stored. for (const auto& [begin, end] : merged) { - const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); - Require(found != regions.begin(), "write-back range has no GPU owner"); - const auto& region = *std::prev(found); - // Every branch below but the direct and copied-back ones stores from the CPU. - if (region.direct == nullptr && !(region.gpuCopy && region.copiedBack) && !(region.mirror != nullptr && !region.mirror->writable)) Recorder::NoteWrittenBack(begin, static_cast(end - begin)); + const auto* found = owner(begin); + Require(found != nullptr, "write-back range has no GPU owner"); + const auto& region = *found; + // Every branch below but the direct, copied-back and staged ones stores from the CPU. + if (region.direct == nullptr && !(region.gpuCopy && (region.copiedBack || region.deviceLocal)) && !(region.mirror != nullptr && !region.mirror->writable)) Recorder::NoteWrittenBack(begin, static_cast(end - begin)); if (region.direct != nullptr) { // The GPU wrote imported guest memory directly; page write watching did not see it. GuestMemory::MarkWritten(begin, static_cast(end - begin)); @@ -1425,10 +2005,16 @@ void GuestBufferMemory::WriteBack() { GuestMemory::MarkWritten(begin, static_cast(end - begin)); continue; } + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (region.deviceLocal) { + // A staged region's use recorded no copy-back (it threw before RecordCopyBacks): + // the results stay in the shadow, which has no mapping to store from; counted + // once, when the build is destroyed or reused. + continue; + } // No copy-back was recorded (a synchronous draw): the staging bytes of the written // sub-range are stored whole, which is what the copy-back would have done. There is no // reference copy to diff against (the copy in never passed through the CPU). - static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; GuestMemory::Write(begin, region.buffer->Bytes().subspan(static_cast(begin - region.begin), static_cast(end - begin))); if (profile) Copies().stagingStores.fetch_add(1, std::memory_order_relaxed); continue; @@ -1455,21 +2041,74 @@ void GuestBufferMemory::WriteBack() { } committed = true; lease.clear(); + space.reset(); } std::optional>> GuestBufferMemory::DirectRegions() const { if (!uploaded || committed) return std::nullopt; std::vector> result; + if (space != nullptr) { + for (const auto& region : space->base) { + if (region.direct == nullptr && (region.mirror == nullptr || region.mirror->writable)) return std::nullopt; + result.emplace_back(region.begin, region.end); + } + } for (const auto& region : regions) { // A read-only mirror is as fixed as an import: its bytes and VkBuffer never change while its // Range exists (see ImageMirrorSerial). A writable one is refreshed and written back per build. + // A staged region is keyed by its import like one bound in place: the import is the source + // of its copy-in and the destination of its copy-back, both recorded per use. const bool fixedMirror = region.mirror != nullptr && !region.mirror->writable; - if (region.direct == nullptr && !fixedMirror) return std::nullopt; + const bool staged = region.gpuCopy && region.deviceLocal; + if (region.direct == nullptr && !fixedMirror && !staged) return std::nullopt; + if (staged) { + // A later use copies from `copySource` again, so it must still be the import serving + // the range (the flush before the copy-in can retire and remake imports): the serial + // the caller records names the current import, and its buffer must be that one. + auto& state = Imports(); + std::lock_guard lock(state.mutex); + const auto* entry = state.device == context.device ? findImport(state, region.begin, region.end) : nullptr; + if (entry == nullptr || entry->buffer != region.copySource) return std::nullopt; + } result.emplace_back(region.begin, region.end); } return result; } +std::vector> GuestBufferMemory::InPlaceReads() const { + std::vector> result; + if (!uploaded || committed) return result; + if (space != nullptr) { + for (const auto& region : space->base) { + if (region.direct != nullptr) result.emplace_back(region.begin, region.end); + } + } + for (const auto& region : regions) { + if (region.direct != nullptr) result.emplace_back(region.begin, region.end); + } + return result; +} + +void GuestBufferMemory::RecordStagingCopies(Recorder& recorder) { + if (!uploaded || committed) return; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + std::vector copies; + for (auto& region : regions) { + if (!region.gpuCopy || !region.deviceLocal) continue; + // The previous use's copy-back was recorded by its MarkGpuWrites (or never, when that use + // threw first: counted); this use records its own after its work, from the shadow this + // copy-in refills. The import is the one the region was taken from (the caller checked its + // serial), so the copy source still stands. + if (!region.copiedBack && profile) Copies().stagedLost.fetch_add(1, std::memory_order_relaxed); + region.copiedBack = false; + copies.push_back(®ion); + } + if (copies.empty()) return; + Require(Recorder::Active() == &recorder, "staging copies recorded into a recorder that is not the device's"); + if (profile) Copies().stagedReused.fetch_add(copies.size(), std::memory_order_relaxed); + recordGpuCopies(copies, false); +} + std::uint64_t ImageMirrorSerial(const Context& context, std::uint64_t address, std::size_t bytes) { if (bytes == 0 || bytes > std::numeric_limits::max() - address || !mirrorsEnabled()) return 0; // A replaced Range expires the mirror (findMirror), and a replacement mirror has a new serial. @@ -1496,4 +2135,18 @@ std::uint64_t HostImportSerial(const Context& context, std::uint64_t address, st return entry.serial; } +bool HostImportsUnchanged(const Context& context, const HostImportsProof& proof) { + if (proof.device == VK_NULL_HANDLE || proof.device != context.device) return false; + auto& state = Imports(); + std::lock_guard lock(state.mutex); + return state.device == proof.device && state.epoch == proof.epoch && state.refreshedGeneration == proof.refreshedGeneration && !importsStale(context, state); +} + +HostImportsProof HostImportsIdentity(const Context& context) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + if (importsStale(context, state)) return {}; + return {state.device, state.epoch, state.refreshedGeneration}; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 517072b9a8..af75014dd1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -41,7 +41,7 @@ void ValidateViewport(const Context& context, const VkViewport& viewport) { Require(std::min(viewport.y, viewport.y + viewport.height) >= context.limits.viewportBoundsRange[0] && std::max(viewport.y, viewport.y + viewport.height) <= context.limits.viewportBoundsRange[1], "viewport Y exceeds device bounds"); } -Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders) : context(context), _modules(shaders.size()), attachments(state.colors.size()) { +Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), attachments(state.colors.size()) { // A cached pipeline may outlive its device's teardown (see ClearCachedPipelines); it must not keep // the buffer pool, which is reset with the device, alive past it. this->context.bufferPool.reset(); @@ -96,10 +96,10 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput color.storeOp = VK_ATTACHMENT_STORE_OP_STORE; color.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; color.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; - color.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - color.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + color.initialLayout = attachmentLayout; + color.finalLayout = attachmentLayout; colors.push_back(color); - references.push_back({index, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}); + references.push_back({index, attachmentLayout}); } VkSubpassDescription subpass{}; subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; @@ -244,17 +244,26 @@ void Pipeline::Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, V begin.renderPass = renderPass; begin.framebuffer = framebuffer.Handle(); begin.renderArea = {{0, 0}, extent}; - context.Function("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE); - context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); - context.Function("vkCmdSetViewport")(commands, 0, 1, &viewport); - context.Function("vkCmdSetScissor")(commands, 0, 1, &scissor); + context.Resolved(&DeviceFunctions::cmdBeginRenderPass, "vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE); + Continue(commands, viewport, scissor); +} + +void Pipeline::Continue(VkCommandBuffer commands, const VkViewport& viewport, const VkRect2D& scissor) const { + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + context.Resolved(&DeviceFunctions::cmdSetViewport, "vkCmdSetViewport")(commands, 0, 1, &viewport); + context.Resolved(&DeviceFunctions::cmdSetScissor, "vkCmdSetScissor")(commands, 0, 1, &scissor); } void Pipeline::PushConstants(VkCommandBuffer commands, std::span shaders) const { const auto stages = PushConstantStages(shaders); if (stages == 0) return; const auto bytes = AssemblePushConstants(shaders); - context.Function("vkCmdPushConstants")(commands, layout, stages, 0, PipelinePushConstantBytes, bytes.data()); + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, stages, 0, PipelinePushConstantBytes, bytes.data()); +} + +void Pipeline::PushConstants(VkCommandBuffer commands, VkShaderStageFlags stages, std::span bytes) const { + if (stages == 0) return; + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, stages, 0, PipelinePushConstantBytes, bytes.data()); } namespace { @@ -270,10 +279,11 @@ void append(std::vector& key, const TValue& value) { // (the recompiler could not identify it, so nothing else may share its pipeline). The rect-list // control and evaluation stages are generated from the vertex and fragment results, which the key // already names, so they carry no id of their own. -std::vector pipelineKey(const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span shaders) { +std::vector pipelineKey(const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) { using Stage = ShaderRecompiler::ShaderStage; std::vector key; append(key, context.device); + append(key, attachmentLayout); append(key, shaders.size()); for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); @@ -401,16 +411,16 @@ void reportPipelines(PipelineStore& store) { } -std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders) { +std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) { static const bool disabled = std::getenv("APS5_NO_PIPELINE_CACHE") != nullptr; - if (disabled) return std::make_shared(context, state, vertexInput, resources, shaders); + if (disabled) return std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); auto& store = Pipelines(); std::lock_guard lock(store.mutex); reportPipelines(store); - const auto key = pipelineKey(context, state, vertexInput, resources, shaders); + const auto key = pipelineKey(context, state, vertexInput, resources, shaders, attachmentLayout); if (key.empty()) { ++store.uncached; - return std::make_shared(context, state, vertexInput, resources, shaders); + return std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); } const auto hash = hashKey(key); if (const auto found = store.index.find(hash); found != store.index.end()) { @@ -427,7 +437,7 @@ std::shared_ptr CachedPipeline(const Context& context, const State& st } else { // A different configuration with the same hash keeps the resident entry; this one stays private. ++store.uncached; - return std::make_shared(context, state, vertexInput, resources, shaders); + return std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); } } ++store.misses; @@ -436,7 +446,7 @@ std::shared_ptr CachedPipeline(const Context& context, const State& st for (auto it = store.entries.begin(); it != store.entries.end();) { it = alive(*it, context) ? std::next(it) : abandon(store, it); } - auto pipeline = std::make_shared(context, state, vertexInput, resources, shaders); + auto pipeline = std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); store.entries.push_back({context.device, context.bufferPool, hash, key, pipeline}); store.index[hash] = std::prev(store.entries.end()); constexpr std::size_t bound = 256; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 938c438b0c..8c11f3ccd7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -1,7 +1,10 @@ #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4Opcodes.hpp" +#include "prx/libc/include/CpuTopology.hpp" #include #include #include @@ -145,6 +148,9 @@ std::string ThreadSyncReport() { } // Unlocked timeline waits (WaitSerial): count and time, for the [recorder] line. std::atomic unlockedWaits{0}, unlockedWaitedUs{0}; +// vkQueueSubmit calls and their time (APS5_PROFILE_DRAW; Submit runs under the GpuMutex). +std::uint64_t submitCount = 0; +double submitUs = 0, submitMaxUs = 0; // Threads inside a timeline wait with the GpuMutex released (WaitSerial, syncThroughUnlocked): a // recorder's teardown, after its Sync() made every wait satisfiable, waits until they left the // semaphore before destroying it. Counted per recorder id (a small fixed array; the waiter cannot @@ -167,6 +173,23 @@ bool RecorderAlive(std::uint64_t id) { // Completion actions in progress on this thread (finish()): a pending-write sync the flush hook // makes from inside one (a copied buffer's write-back store) is skipped, see SyncThrough. thread_local int completionDepth = 0; +// This thread's hook syncs that waited for an unsignaled target (Recorder::ThreadHookWaits). +thread_local std::uint64_t hookRealWaits = 0; + +} + +bool Recorder::InCompletion() { + return completionDepth != 0; +} + +namespace { + +// APS5_NO_HOOK_COMPLETION_GUARD=1: the flush hook records the accessing worker's queued labels +// even from inside a completion action, as before (see FlushForAccess). +bool HookCompletionGuard() { + static const bool guard = std::getenv("APS5_NO_HOOK_COMPLETION_GUARD") == nullptr; + return guard; +} // APS5_COMPLETION_STORE_SYNC=1: a store made by a completion action waits for later batches that // note its range, as before (the wait runs under whichever hold reaped the batch). @@ -181,6 +204,24 @@ bool HookLockedWait() { return locked; } +// APS5_HOOK_FULL_SYNC=1: the flush hook's recorded-store wait is a full Sync() under the GpuMutex +// (every batch in flight) as before, instead of SyncThrough's targeted, unlocked wait (H1). +bool HookFullSync() { + static const bool hookFullSync = std::getenv("APS5_HOOK_FULL_SYNC") != nullptr; + return hookFullSync; +} + +// APS5_HOOK_FLUSH_CPU_BLOCKS=0: an access whose every 64 KiB block the CPU wrote since the +// overlapping pending images' generations flushes them as before, instead of the C4 skip (H2: +// StorageTexture::AccessKeptByCpu). +bool HookFlushCpuBlocks() { + static const bool hookFlushCpuBlocks = [] { + const char* text = std::getenv("APS5_HOOK_FLUSH_CPU_BLOCKS"); + return text != nullptr && text[0] == '0'; + }(); + return hookFlushCpuBlocks; +} + // What the holds spend on the recorder (APS5_PROFILE_DRAW, all under the GpuMutex, printed at the // end of the [recorder] line): completions and their time (of which releasing the kept objects), // pending-write syncs from inside completions (skipped, or waited with the kill switch), reaps and @@ -203,6 +244,9 @@ struct HoldCounters { // remaining costs, GPU latency and hold contention, stay apart. double hookUnlockedWaitMs = 0; double hookRelockMs = 0; + // The same by sync source (1 pending write, 2 recorded store). + std::array hookUnlockedWaitsBySource{}; + std::array hookUnlockedWaitMsBySource{}; std::uint64_t hookUnlockedTornDown = 0; // Genuine nested hooks (depth >= 2 outside a completion action) that had to wait locked. std::uint64_t hookLockedWaits = 0; @@ -311,6 +355,7 @@ ReleaseQueue& ReleaseThreadQueue() { } void ReleaseThreadMain() { + CpuTopology::PinHelperThread("release thread"); auto& queue = ReleaseThreadQueue(); std::unique_lock lock(queue.mutex); for (;;) { @@ -397,10 +442,29 @@ void ReleaseDeferredKeeps() { constexpr std::int64_t NoPendingLabel = std::numeric_limits::min(); std::atomic pendingLabelSince{NoPendingLabel}; std::atomic writeGeneration{0}; +std::atomic publishGeneration{0}; std::atomic completionLabels{0}; +std::atomic writeBackCompletions{0}; std::atomic completionStoresSkipped{0}; std::atomic completionStoresRun{0}; +// Write-backs noted over a tracked label dword (NoteWrittenBack): a CPU store over a GPU label. +std::atomic writeBacksOverLabels{0}; +// Completion labels that entered the pending count at a write-back (see NoteWrittenBack). +std::atomic completionLabelsCountedLate{0}; std::atomic workSinceSubmit{0}; + +// APS5_COUNT_ALL_COMPLETION_LABELS=1: a completion label also stored on the GPU counts as pending +// from its registration, as before (see Recorder::AfterCompletions). +bool CountAllCompletionLabels() { + static const bool all = std::getenv("APS5_COUNT_ALL_COMPLETION_LABELS") != nullptr; + return all; +} + +// APS5_NO_PROC_TABLE=1: the recorder resolves its Vulkan entry points per call, as before. +bool ProcTable() { + static const bool table = std::getenv("APS5_NO_PROC_TABLE") == nullptr; + return table; +} // The pending-label table's own mutex: every mutation of a recorder's `labels` holds it (under the // GPU mutex), so a WAIT_REG_MEM looks a label up with this small lock alone (Recorder::LookupLabel) // instead of queueing behind a dispatch. `labelTableOwner` is the active recorder while it lives, @@ -408,6 +472,87 @@ std::atomic workSinceSubmit{0}; // under this mutex takes the GPU mutex (no lock-order cycle). std::mutex labelTableMutex; Recorder* labelTableOwner = nullptr; +// The ranges of the calling worker's queued labels (NoteQueuedLabel, cleared by +// ForgetQueuedLabels): the flush hook, on the same thread, has them recorded before an access that +// overlaps one, through the function the driver installed. +thread_local std::vector> queuedLabelRanges; +std::atomic queuedLabelRecorder{nullptr}; +// The dwords (with their stamps) this thread noted since its last CloseLabelGroup (see Recorder::CloseLabelGroup). +thread_local std::vector> labelGroupDwords; +// Late-rule counters (Recorder::LateCounts), per thread: lookups run on the waiting worker. +thread_local Recorder::LateStatistics lateCounts{}; + +// APS5_LABEL_TRUST_LATE=1 trusts entries with stamp <= afterStamp under the late rule; off by +// default until the frame-rate collapse it produced at the video stage (lt_on: 119 -> 25 presents +// per 10 s with 0 late-trusted waits) is understood. +bool LateTrustEnabled() { + static const bool enabled = [] { + const char* value = std::getenv("APS5_LABEL_TRUST_LATE"); + return value != nullptr && std::strcmp(value, "0") != 0; + }(); + return enabled; +} +// Store-run and queued-label counters (Recorder::StoreCounts), relaxed: they are only reported. +std::atomic storeCount{0}, storeRuns{0}, storesJoined{0}, storesReplaced{0}, storeWawBarriers{0}, storeJoinsRefused{0}; +std::atomic keyStoreCount{0}, keyStoreRuns{0}, keyStoreRunsForWriter{0}, keyStoresJoined{0}; + +// Debug aid: APS5_DCC_KEYS_EACH=1 records every DCC key store at once (see Recorder::QueueKeyStore). +bool KeyStoresEach() { + static const bool each = std::getenv("APS5_DCC_KEYS_EACH") != nullptr; + return each; +} +std::atomic queuedLabelsNoted{0}, queuedLabelsOverRecorded{0}, queuedLabelHits{0}, queuedLabelHookRecords{0}, queuedLabelHookInCompletion{0}; +// Read-tracking counters (Recorder::ReadCounts), relaxed: they are only reported. +constexpr std::size_t ReadKinds = static_cast(Recorder::ReadKind::Count); +std::atomic readsNoted{0}, readQueries{0}, readStaleIgnored{0}; +std::atomic readHits[ReadKinds]{}; + +bool ReadTrackingEnabled() { + // APS5_COPY_READ_TRACKING=0: no in-place reads are noted; the copy HLE's CPU path then requires + // an idle recorder (VulkanDevice::CopyBuffer), never less. + static const bool enabled = [] { + const char* value = std::getenv("APS5_COPY_READ_TRACKING"); + return value == nullptr || std::strcmp(value, "0") != 0; + }(); + return enabled; +} + +bool LabelRunsEnabled() { + // Debug aid: APS5_NO_LABEL_RUNS=1 records every store between a barrier pair of its own + // (APS5_NO_LABEL_BATCHING=1 turns off every label-batching change of Driver.cpp too). + static const bool enabled = std::getenv("APS5_NO_LABEL_RUNS") == nullptr && std::getenv("APS5_NO_LABEL_BATCHING") == nullptr; + return enabled; +} + +// One store run per batch, recorded at Submit (see Recorder::RecordStore). Debug aid: +// APS5_LABEL_RUNS_INLINE=1 records the stores at once and closes the run at the next command. +bool LabelRunsPerBatch() { + static const bool perBatch = LabelRunsEnabled() && std::getenv("APS5_LABEL_RUNS_INLINE") == nullptr; + return perBatch; +} +std::atomic storeRunsAtSubmit{0}, storeRunsForced{0}; + +// APS5_NO_JOIN_WAW_CHECK=1: a contiguous store joins the pending one without the check against +// the run's recorded stores, as before (see RecordStore). +bool JoinWawCheck() { + static const bool check = std::getenv("APS5_NO_JOIN_WAW_CHECK") == nullptr; + return check; +} + +// APS5_NO_SEPARATE_QUEUED_LABELS=1: a queued label enters the recorded table and replaces a +// recorded entry of its dword, as before (see NoteQueuedLabel). +bool SeparateQueuedLabels() { + static const bool separate = std::getenv("APS5_NO_SEPARATE_QUEUED_LABELS") == nullptr; + return separate; +} + +bool QueuedLabelOverlaps(std::uint64_t address, std::size_t bytes) { + const auto end = address + bytes; + for (const auto& [begin, finish] : queuedLabelRanges) { + if (address < finish && begin < end) return true; + } + return false; +} // Flush hook statistics: calls, calls whose snapshot check overlapped (the GpuMutex was taken), // targeted syncs and the batches they left in flight. std::atomic hookCalls{0}, hookLocks{0}, targetedSyncs{0}, batchesLeftInFlight{0}; @@ -416,7 +561,7 @@ std::atomic hookCalls{0}, hookLocks{0}, targetedSyncs{0}, batches std::uint64_t snapshotCovered = 0, snapshotRebuilds = 0; double snapshotRebuildMs = 0; -using WriteRanges = std::vector>; +using WriteRanges = Recorder::WriteRanges; // The sorted, merged union of the guest ranges every unfinished batch will write. Rebuilt under // GuestMemory::GpuMutex (the only writer) and read by the flush hook without it: the lock is the // graphics worker's main stall, so a no-overlap access must not wait for another queue's device @@ -431,15 +576,19 @@ bool HookSnapshotEnabled() { return enabled; } -bool SnapshotOverlaps(std::uint64_t address, std::size_t bytes) { - if (bytes == 0) return false; - const auto snapshot = pendingWrites.load(std::memory_order_acquire); - if (snapshot == nullptr || snapshot->empty()) return false; +bool SnapshotOverlaps(const WriteRanges* snapshot, std::uint64_t address, std::size_t bytes) { + if (bytes == 0 || snapshot == nullptr || snapshot->empty()) return false; // Merged ranges are ordered by both bounds: the first one ending past the access decides. const auto it = std::partition_point(snapshot->begin(), snapshot->end(), [&](const auto& range) { return range.second <= address; }); return it != snapshot->end() && it->first < address + bytes; } +bool SnapshotOverlaps(std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return false; + const auto snapshot = pendingWrites.load(std::memory_order_acquire); + return SnapshotOverlaps(snapshot.get(), address, bytes); +} + // Whether one merged range of the snapshot contains [address, end) entirely. bool SnapshotCovers(std::uint64_t address, std::uint64_t end) { const auto snapshot = pendingWrites.load(std::memory_order_acquire); @@ -651,11 +800,157 @@ private: std::chrono::steady_clock::time_point start; }; +// Attribution of the hook's recorded-store syncs (source 2: a pending image's write-back or a unit +// publish was recorded for the access and SyncThrough waits for it), the [hooksync] recorded-store +// line every 10 s under the same switch as HookSyncScope: per (queue, packet, read site), the syncs, +// their wait, the batches in flight at the sync, the C4 classification of the access (every 64 KiB +// block CPU-written since the overlapping images' generations: the store would not change a byte +// the CPU reads), whether the bytes changed over the wait, whether the images overlapped had a +// consumer proof within StorageTexture::DeadImagePresents presents, and the accesses the C4 skip +// took instead of a flush (with the images that left the registry there as an empty write-back +// would have left them). The skips are counted without the GpuMutex (their own mutex); the syncs +// under it. +struct RecordedStoreTotals { + std::uint64_t count = 0; + double waitedMs = 0; + std::uint64_t batchesFinished = 0; + std::uint64_t cpuWrittenYes = 0, cpuWrittenNo = 0, unclassified = 0, publishOnly = 0; + std::uint64_t changed = 0, unchanged = 0, unchecked = 0; + std::uint64_t imagesDead = 0, imagesLive = 0; + std::uint64_t skipped = 0, flushed = 0, evicted = 0; +}; + +struct RecordedStoreStats { + std::mutex mutex; + std::map byKey; + RecordedStoreTotals totals; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +RecordedStoreStats& RecordedStoreSyncs() { + static RecordedStoreStats stats; + return stats; +} + +void ReportRecordedStoreSyncs(RecordedStoreStats& stats) { + std::vector> hot; + for (const auto& [key, totals] : stats.byKey) hot.emplace_back(&key, &totals); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return std::tie(a.second->waitedMs, a.second->skipped) > std::tie(b.second->waitedMs, b.second->skipped); }); + const auto count = [](std::uint64_t value) { return static_cast(value); }; + const auto& t = stats.totals; + const auto skips = StorageTexture::TakeHookSkipCounts(); + char text[640]; + std::snprintf(text, sizeof(text), "[hooksync] recorded-store syncs (10 s): %llu waited %.0f ms, %llu batches in flight at them; skipped (cpu-written blocks) %llu / flushed %llu / evicted stale %llu; flushed later after a skip %llu (by the same site %llu); C4 every block CPU-written: yes %llu / no %llu / unclassified %llu (publish only %llu); bytes after the wait: changed %llu, unchanged %llu, unchecked %llu; images overlapped: dead %llu / live %llu (no consumer proof for %llu presents); by queue packet site (count/ms, skipped, cpu-written yes/no, changed/unchanged, dead/live):", count(t.count), t.waitedMs, count(t.batchesFinished), count(t.skipped), count(t.flushed), count(t.evicted), count(skips.flushedAfterSkip), count(skips.flushedAfterSkipSameSite), count(t.cpuWrittenYes), count(t.cpuWrittenNo), count(t.unclassified), count(t.publishOnly), count(t.changed), count(t.unchanged), count(t.unchecked), count(t.imagesDead), count(t.imagesLive), count(StorageTexture::DeadImagePresents)); + std::string report = text; + for (std::size_t i = 0; i < hot.size() && i < 10; ++i) { + const auto& key = *hot[i].first; + const auto& k = *hot[i].second; + std::snprintf(text, sizeof(text), " [0x%x %s %s +0x%llx/+0x%llx/+0x%llx: %llu/%.0fms k%llu y%llu/n%llu c%llu/u%llu d%llu/l%llu]", key.queue, PacketName(key.opcode).c_str(), GuestMemory::ReadSiteName(key.site), key.frames[0], key.frames[1], key.frames[2], count(k.count), k.waitedMs, count(k.skipped), count(k.cpuWrittenYes), count(k.cpuWrittenNo), count(k.changed), count(k.unchanged), count(k.imagesDead), count(k.imagesLive)); + report += text; + } + std::fprintf(stderr, "%s\n", report.c_str()); + stats.byKey.clear(); + stats.totals = {}; +} + +// The key of the access the hook handles now: always inlined into FlushForAccess so that frame 0 +// is the hook's caller for the skip counters and the sync scope alike, whatever the inliner does +// with the scope's constructor. +[[gnu::always_inline]] inline HookSyncKey RecordedStoreKey() { + const auto packet = GuestMemory::CurrentPacket(); + HookSyncKey key{packet.queue, packet.opcode, GuestMemory::CurrentReadSite(), {}}; + GuestMemory::CaptureCallerOffsets(key.frames, 1); + return key; +} + +void CountRecordedStoreSkip(const HookSyncKey& key, std::size_t evicted) { + auto& stats = RecordedStoreSyncs(); + std::lock_guard lock(stats.mutex); + for (auto* totals : {&stats.byKey[key], &stats.totals}) { + ++totals->skipped; + totals->evicted += evicted; + } + const auto now = std::chrono::steady_clock::now(); + if (now - stats.lastReport > std::chrono::seconds(10)) { + stats.lastReport = now; + ReportRecordedStoreSyncs(stats); + } +} + +class RecordedStoreSyncScope { +public: + RecordedStoreSyncScope(const Recorder& recorder, std::uint64_t address, std::size_t bytes, const StorageTexture::AccessClassification& classification, bool stored, const HookSyncKey& key) : enabled(HookSyncProfiled()), address(address), bytes(bytes), classification(classification), stored(stored), key(key) { + if (!enabled) return; + batchesInFlight = recorder.InFlightBatches(); + if (key.site != GuestMemory::ReadSite::Store && bytes <= CompareLimit && GuestMemory::Accessible(reinterpret_cast(address), bytes)) { + before.resize(bytes); + std::memcpy(before.data(), reinterpret_cast(address), bytes); + } + start = std::chrono::steady_clock::now(); + } + ~RecordedStoreSyncScope() { + if (!enabled) return; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - start).count(); + auto& stats = RecordedStoreSyncs(); + std::lock_guard lock(stats.mutex); + const auto add = [&](RecordedStoreTotals& totals) { + ++totals.count; + totals.waitedMs += ms; + totals.batchesFinished += batchesInFlight; + if (stored) ++totals.flushed; + if (classification.images == 0) ++totals.publishOnly; + else if (!classification.checked) ++totals.unclassified; + else ++(classification.allCpuWritten ? totals.cpuWrittenYes : totals.cpuWrittenNo); + totals.imagesDead += classification.dead; + totals.imagesLive += classification.live; + if (before.empty() || !GuestMemory::Accessible(reinterpret_cast(address), bytes)) ++totals.unchecked; + else if (std::memcmp(before.data(), reinterpret_cast(address), bytes) != 0) ++totals.changed; + else ++totals.unchanged; + }; + add(stats.byKey[key]); + add(stats.totals); + if (now - stats.lastReport > std::chrono::seconds(10)) { + stats.lastReport = now; + ReportRecordedStoreSyncs(stats); + } + } + RecordedStoreSyncScope(const RecordedStoreSyncScope&) = delete; + RecordedStoreSyncScope& operator=(const RecordedStoreSyncScope&) = delete; + +private: + static constexpr std::size_t CompareLimit = 65536; + bool enabled; + std::uint64_t address; + std::size_t bytes; + StorageTexture::AccessClassification classification; + bool stored; + HookSyncKey key; + std::size_t batchesInFlight = 0; + std::vector before; + std::chrono::steady_clock::time_point start; +}; + // GuestMemory flush hook: a CPU access to memory that recorded GPU work will write waits for that // work first; then storage image results pending for the range are stored. The recorder is only // dereferenced under the GpuMutex (it is destroyed with its device under that lock). void FlushForAccess(std::uint64_t address, std::size_t bytes) { hookCalls.fetch_add(1, std::memory_order_relaxed); + // A label this worker queued but has not recorded writes the range: in queue order the label + // precedes this access, so the group is recorded first (it then notes the range as a pending + // write of the open batch, and the check below syncs on it like any recorded store). Not from + // inside a completion action (a write-back store made while a batch is reaped): the record + // would reap nested in the reap in progress, landing later batches' write-backs before the + // rest of this one's, and this queue's queued label follows every batch of its own in flight + // anyway, so the write-back landing first is the order the hardware gives. + if (!queuedLabelRanges.empty() && QueuedLabelOverlaps(address, bytes)) { + if (completionDepth != 0 && HookCompletionGuard()) { + queuedLabelHookInCompletion.fetch_add(1, std::memory_order_relaxed); + } else if (auto* record = queuedLabelRecorder.load(std::memory_order_acquire); record != nullptr) { + queuedLabelHookRecords.fetch_add(1, std::memory_order_relaxed); + record(); + } + } if (!HookSnapshotEnabled() || SnapshotOverlaps(address, bytes)) { hookLocks.fetch_add(1, std::memory_order_relaxed); GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); @@ -668,13 +963,43 @@ void FlushForAccess(std::uint64_t address, std::size_t bytes) { recorder->SyncThrough(address, bytes, true); } } - if (StorageTexture::FlushPending(address, bytes)) { - // Stores into imported memory were only recorded; the CPU is about to read them. + // A reader takes every shadowed unit of the range; a store only the units it covers partly + // (its own stamp makes the others stale). Inside a completion action a store has no publish + // recorded: the batch is not waited for, and the copy would land over the store later. + const auto site = GuestMemory::CurrentReadSite(); + const auto scope = site != GuestMemory::ReadSite::Store ? PublishScope::Whole : completionDepth != 0 ? PublishScope::None : PublishScope::PartialUnits; + bool published = false; + // The classification is made before the store (the store changes what it measures); profile only. + StorageTexture::AccessClassification classification{}; + if (HookSyncProfiled()) StorageTexture::ClassifyAccess(address, bytes, classification); + // C4: an access whose every 64 KiB block the CPU wrote since the overlapping images' generations + // reads the same bytes with or without their store (the store keeps those blocks), so the + // images stay pending (short of the layers the store would have left empty-handed anyway) and + // only the unit shadows over the range are published; with no image over the range that + // publish is all FlushPending would do (one registry scan, not two). + bool stored = false; + std::size_t images = 0, evicted = 0; + const bool kept = !HookFlushCpuBlocks() && StorageTexture::AccessKeptByCpu(address, bytes, &images, &evicted); + if (kept || (images == 0 && !HookFlushCpuBlocks())) { + published = StorageTexture::PublishShadowsOnly(address, bytes, scope); + if (kept && HookSyncProfiled()) CountRecordedStoreSkip(RecordedStoreKey(), evicted); + } else { + stored = StorageTexture::FlushPending(address, bytes, nullptr, "memory access", scope, &published); + } + if (stored || published) { + // Stores into imported memory were only recorded; the CPU is about to read them. The wait + // targets the batch holding them (the open one: SyncThrough submits it) and runs with the + // mutex released when this acquisition is the outermost; a nested hook waits locked through + // that batch ([lock] 'locked GPU waits'). Inside a completion action the store must land + // before the completing batch's CPU write-back (an older result under a newer one), which + // only the full Sync() gives there. GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); std::lock_guard gpu(GuestMemory::GpuMutex()); if (auto* recorder = Recorder::Active(); recorder != nullptr) { + const RecordedStoreSyncScope attribution(*recorder, address, bytes, classification, stored, HookSyncProfiled() ? RecordedStoreKey() : HookSyncKey{}); Recorder::CountSync(2); - recorder->Sync(); + if (HookFullSync() || completionDepth != 0) recorder->Sync(); + else recorder->SyncThrough(address, bytes, true); } } } @@ -688,6 +1013,16 @@ Recorder::Recorder(const Context& context, bool timelineSemaphores) : context(co } // Idempotent: the same hook for every recorder (the deferred lists are per thread, not per recorder). GuestMemory::SetGpuUnlockHook(&ReleaseDeferredKeeps); + if (ProcTable() && context.deviceProc != nullptr) { + getFenceStatus = context.Function("vkGetFenceStatus"); + resetFences = context.Function("vkResetFences"); + waitForFences = context.Function("vkWaitForFences"); + cmdUpdateBuffer = context.Function("vkCmdUpdateBuffer"); + beginCommandBuffer = context.Function("vkBeginCommandBuffer"); + endCommandBuffer = context.Function("vkEndCommandBuffer"); + queueSubmit = context.Function("vkQueueSubmit"); + cmdPipelineBarrier = context.Function("vkCmdPipelineBarrier"); + } if (!timelineSemaphores) return; // The timeline starts at 0 and every Submit signals its serial (1, 2, ...): a value that only // grows, so a thread waiting for it outside the mutex can never observe a reset or a reused @@ -713,8 +1048,10 @@ Recorder::~Recorder() { if (activeRecorder == this) { activeRecorder = nullptr; pendingWrites.store(nullptr, std::memory_order_release); + publishGeneration.fetch_add(1, std::memory_order_release); pendingLabelSince.store(NoPendingLabel, std::memory_order_release); completionLabels.store(0, std::memory_order_release); + writeBackCompletions.store(0, std::memory_order_release); workSinceSubmit.store(0, std::memory_order_release); } { @@ -769,10 +1106,18 @@ std::uint64_t Recorder::WriteGeneration() { return writeGeneration.load(std::memory_order_acquire); } +std::uint64_t Recorder::PublishGeneration() { + return publishGeneration.load(std::memory_order_acquire); +} + std::uint64_t Recorder::PendingCompletionLabels() { return completionLabels.load(std::memory_order_acquire); } +std::uint64_t Recorder::PendingWriteBackCompletions() { + return writeBackCompletions.load(std::memory_order_acquire); +} + std::uint64_t Recorder::RecordedWorkSinceSubmit() { return workSinceSubmit.load(std::memory_order_relaxed); } @@ -794,14 +1139,106 @@ void Recorder::Activate() { GuestMemory::SetFlushHook(&FlushForAccess); } -std::optional Recorder::LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) { +std::optional Recorder::LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal) { std::lock_guard tableLock(labelTableMutex); if (labelTableOwner == nullptr) return std::nullopt; - return labelTableOwner->lookupLabel(address, bytes, afterStamp, queue); + return labelTableOwner->lookupLabel(address, bytes, afterStamp, refusal); +} + +std::optional Recorder::LookupLabelValue(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp) { + const auto hit = LookupLabel(address, bytes, afterStamp); + if (!hit.has_value()) return std::nullopt; + return hit->value; +} + +bool Recorder::LateTrust() { + return LateTrustEnabled(); +} + +Recorder::LateStatistics Recorder::LateCounts() { + return lateCounts; +} + +void Recorder::CloseLabelGroup(std::uint64_t trackerGeneration) { + if (labelGroupDwords.empty()) return; + auto group = std::move(labelGroupDwords); + labelGroupDwords.clear(); + if (trackerGeneration == 0) return; + std::lock_guard tableLock(labelTableMutex); + if (labelTableOwner == nullptr) return; + auto& recorded = labelTableOwner->labels; + for (const auto& [dword, stamp] : group) { + // Only an entry still ours (stamps are unique): a later note of the dword has its own group. + // And only while its batch is unsubmitted: a submitted one (a drain or APS5_LABEL_SUBMIT_NOW + // inside the group) may have landed the value already, and a CPU store between that and + // this close is not newer than the generation. + const auto found = recorded.find(dword); + if (found != recorded.end() && found->second.stamp == stamp && found->second.batch != nullptr && !found->second.batch->submitted) found->second.generation = trackerGeneration; + } + // A poller that refused the group as unclosed re-consults the table on the next generation. + writeGeneration.fetch_add(1, std::memory_order_release); +} + +void Recorder::NoteQueuedLabel(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue) { + if (bytes.size() < 4 || address % 4 != 0) return; + // The range before the table: the hook covers the group whether or not a table exists to + // enter it in (no owner: the callback records it outright, by a CPU store when need be). + queuedLabelRanges.emplace_back(address, address + bytes.size()); + queuedLabelsNoted.fetch_add(1, std::memory_order_relaxed); + std::lock_guard tableLock(labelTableMutex); + if (labelTableOwner == nullptr) return; + auto& recorded = labelTableOwner->labels; + auto& table = SeparateQueuedLabels() ? labelTableOwner->queuedLabels : recorded; + for (std::size_t offset = 0; offset + 4 <= bytes.size(); offset += 4) { + std::uint32_t value = 0; + std::memcpy(&value, bytes.data() + offset, 4); + if (const auto found = recorded.find(address + offset); found != recorded.end() && found->second.batch != nullptr) queuedLabelsOverRecorded.fetch_add(1, std::memory_order_relaxed); + table.insert_or_assign(address + offset, LabelEntry{value, queue, stamp, nullptr}); + } + labelTableOwner->recordedLabels.store(recorded.size(), std::memory_order_relaxed); +} + +void Recorder::ForgetQueuedLabels() { + if (queuedLabelRanges.empty()) return; + auto ranges = std::move(queuedLabelRanges); + queuedLabelRanges.clear(); + std::lock_guard tableLock(labelTableMutex); + if (labelTableOwner == nullptr) return; + auto& recorded = labelTableOwner->labels; + auto& queued = labelTableOwner->queuedLabels; + const auto tag = GuestMemory::GpuLockThreadTag(); + for (const auto& [begin, end] : ranges) { + for (auto dword = begin; dword + 4 <= end; dword += 4) { + if (const auto own = queued.find(dword); own != queued.end() && own->second.queue == tag) queued.erase(own); + const auto found = recorded.find(dword); + if (found != recorded.end() && found->second.batch == nullptr) recorded.erase(found); + } + } + labelTableOwner->recordedLabels.store(recorded.size(), std::memory_order_relaxed); +} + +void Recorder::SetQueuedLabelRecorder(void (*recorder)()) { + queuedLabelRecorder.store(recorder, std::memory_order_release); +} + +Recorder::StoreStatistics Recorder::StoreCounts() { + return StoreStatistics{storeCount.load(std::memory_order_relaxed), storeRuns.load(std::memory_order_relaxed), storesJoined.load(std::memory_order_relaxed), storesReplaced.load(std::memory_order_relaxed), storeWawBarriers.load(std::memory_order_relaxed), storeJoinsRefused.load(std::memory_order_relaxed), queuedLabelsNoted.load(std::memory_order_relaxed), queuedLabelsOverRecorded.load(std::memory_order_relaxed), queuedLabelHits.load(std::memory_order_relaxed), queuedLabelHookRecords.load(std::memory_order_relaxed), queuedLabelHookInCompletion.load(std::memory_order_relaxed), keyStoreCount.load(std::memory_order_relaxed), keyStoreRuns.load(std::memory_order_relaxed), keyStoreRunsForWriter.load(std::memory_order_relaxed), keyStoresJoined.load(std::memory_order_relaxed), storeRunsAtSubmit.load(std::memory_order_relaxed), storeRunsForced.load(std::memory_order_relaxed)}; +} + +std::uint64_t Recorder::ThreadHookWaits() { + return hookRealWaits; } bool Recorder::SnapshotWriteOverlaps(std::uint64_t address, std::size_t bytes) { - return SnapshotOverlaps(address, bytes); + return AgcDriver::Graphics::SnapshotOverlaps(address, bytes); +} + +std::shared_ptr Recorder::PendingWriteSnapshot() { + return pendingWrites.load(std::memory_order_acquire); +} + +bool Recorder::SnapshotOverlaps(const WriteRanges* snapshot, std::uint64_t address, std::size_t bytes) { + return AgcDriver::Graphics::SnapshotOverlaps(snapshot, address, bytes); } void Recorder::CountSync(int source, const void* site) { @@ -819,7 +1256,125 @@ double Recorder::ThreadWaitedMs() { return threadWaitedMs; } -VkCommandBuffer Recorder::Commands() { +std::uint64_t Recorder::ReapsWithWork() { + return holdCounters.reapsWithWork; +} + +VkCommandBuffer Recorder::Commands(VkAccessFlags* coveredAccess) { + ensureOpen(); + // Work recorded after a draw's render pass or an inline store run must see their writes: the + // pass's end and the run's trailing barrier go in first (a per-batch run waits for Submit, or + // for a caller whose ranges overlap a queued store: FlushStores). + if (open->renderPass.open) endOpenRenderPass(); + if (open->run.open && !LabelRunsPerBatch()) closeStoreRun(); + if (coveredAccess != nullptr) *coveredAccess = open->coveredAccess; + open->coveredAccess = 0; + return open->commands; +} + +void Recorder::MarkCovered(VkAccessFlags access) { + if (open != nullptr) open->coveredAccess = access; +} + +bool Recorder::ContinuesRenderPass(std::uint64_t key) const { + return open != nullptr && open->renderPass.open && open->renderPass.continuable && open->renderPass.key == key; +} + +VkCommandBuffer Recorder::CommandsInRenderPass() { + Require(open != nullptr && open->renderPass.open, "no render pass is open in the recorder"); + return open->commands; +} + +void Recorder::LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable) { + Require(open != nullptr, "no batch is open for the render pass"); + auto& pass = open->renderPass; + if (!pass.open) pass.timing = timing; + pass.open = true; + pass.key = key; + pass.continuable = continuable; +} + +void Recorder::endOpenRenderPass() { + auto& pass = open->renderPass; + context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(open->commands); + // The pass's attachment and shader writes are visible to everything recorded after it (the + // host sees them at the batch's fence). + recordBarrier(open->commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + CountBarriers(CommandClass::Draw); + EndGpuTiming(pass.timing); + open->coveredAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + open->hostReadOwed = true; + pass = {}; +} + +void Recorder::QueueKeyStore(VkBuffer buffer, VkDeviceSize first, VkDeviceSize last, std::shared_ptr seed, std::uint64_t begin, std::uint64_t end) { + ensureOpen(); + // A queued label store over the keys lands first (program order: the key store is later). + FlushStoresOverlapping(begin, static_cast(end - begin)); + keyStoreCount.fetch_add(1, std::memory_order_relaxed); + if (seed != nullptr) open->kept.push_back(seed); + // The same range queued twice lands once: nothing wrote over it in between, or the writer + // would have recorded the first store first. + for (const auto& store : open->keyStores) { + if (store.buffer == buffer && store.first == first && store.last == last) { + keyStoresJoined.fetch_add(1, std::memory_order_relaxed); + return; + } + } + open->keyStores.push_back({buffer, first, last, std::move(seed), begin, end}); + if (KeyStoresEach()) recordKeyStores(false); +} + +bool Recorder::QueuedKeyStoreOverlaps(std::uint64_t address, std::size_t bytes) const { + if (open == nullptr || bytes == 0) return false; + const auto end = address + bytes; + return std::any_of(open->keyStores.begin(), open->keyStores.end(), [&](const Batch::KeyStore& store) { return address < store.end && store.begin < end; }); +} + +bool Recorder::AnyQueuedKeyStore(const std::function& overlaps) const { + if (open == nullptr) return false; + return std::any_of(open->keyStores.begin(), open->keyStores.end(), [&](const Batch::KeyStore& store) { return overlaps(store.begin, store.end); }); +} + +void Recorder::FlushKeyStores() { + if (open == nullptr || open->keyStores.empty()) return; + recordKeyStores(true); +} + +void Recorder::recordKeyStores(bool forWriter) { + auto stores = std::move(open->keyStores); + open->keyStores.clear(); + if (stores.empty()) return; + const auto commands = Commands(); + const auto timing = beginTiming(ClassKey(CommandClass::DccKeyStore)); + // Ordered behind every earlier recorded read or write of the ranges (the title's DCC clear or + // decompress kernel storing keys through a V#), and visible to the work after and to the host. + recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + const auto fill = context.Resolved(&DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer"); + const auto copy = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); + std::uint64_t bytes = 0; + for (const auto& store : stores) { + const VkDeviceSize fillBegin = std::min((store.first + 3) & ~VkDeviceSize{3}, store.last); + const VkDeviceSize fillEnd = std::max(store.last & ~VkDeviceSize{3}, fillBegin); + if (fillEnd > fillBegin) fill(commands, store.buffer, fillBegin, fillEnd - fillBegin, 0xffffffffu); + if (store.seed != nullptr) { + VkBufferCopy copies[2]; + std::uint32_t copyCount = 0; + if (fillBegin > store.first) copies[copyCount++] = {0, store.first, fillBegin - store.first}; + if (store.last > fillEnd) copies[copyCount++] = {0, fillEnd, store.last - fillEnd}; + if (copyCount != 0) copy(commands, static_cast(store.seed.get())->Handle(), store.buffer, copyCount, copies); + } + bytes += store.last - store.first; + } + recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT); + CountBarriers(CommandClass::DccKeyStore, 2); + EndGpuTiming(timing, bytes); + open->coveredAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT; + keyStoreRuns.fetch_add(1, std::memory_order_relaxed); + if (forWriter) keyStoreRunsForWriter.fetch_add(1, std::memory_order_relaxed); +} + +void Recorder::ensureOpen() { GuestMemory::AssertGpuLockHeld("Recorder::Commands"); if (open == nullptr) { auto batch = std::make_unique(); @@ -839,30 +1394,366 @@ VkCommandBuffer Recorder::Commands() { } VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - Check(context.Function("vkBeginCommandBuffer")(batch->commands, &begin), "vkBeginCommandBuffer recorder"); + Check(function(beginCommandBuffer, "vkBeginCommandBuffer")(batch->commands, &begin), "vkBeginCommandBuffer recorder"); } catch (...) { release(*batch); throw; } + batch->queue = GuestMemory::GpuLockThreadTag(); open = std::move(batch); + // The whole-batch range: its start stamp waits for the previous batches like any + // bottom-of-pipe stamp, so it marks when this batch's execution began. + open->batchTiming = BeginGpuTiming(BatchTimingKey); } - return open->commands; +} + +void Recorder::RecordStore(VkBuffer buffer, VkDeviceSize offset, std::span bytes, std::uint64_t address) { + if (bytes.empty()) return; + ensureOpen(); + auto& run = open->run; + const VkDeviceSize end = offset + bytes.size(); + storeCount.fetch_add(1, std::memory_order_relaxed); + if (LabelRunsPerBatch()) { + // Queued for the batch's run: a store inside the last queued one replaces its bytes, a + // contiguous one joins it (a joined range overlapping an earlier queued store takes the + // WAW barrier when the run is recorded, so no join is refused). + if (run.queued.empty()) storeRuns.fetch_add(1, std::memory_order_relaxed); + else if (auto& last = run.queued.back(); last.buffer == buffer) { + const VkDeviceSize lastEnd = last.offset + last.bytes.size(); + if (offset >= last.offset && end <= lastEnd) { + std::memcpy(last.bytes.data() + (offset - last.offset), bytes.data(), bytes.size()); + storesReplaced.fetch_add(1, std::memory_order_relaxed); + return; + } + if (offset == lastEnd && last.bytes.size() + bytes.size() <= 65536) { + last.bytes.insert(last.bytes.end(), bytes.begin(), bytes.end()); + storesJoined.fetch_add(1, std::memory_order_relaxed); + return; + } + } + run.queued.push_back({buffer, offset, std::vector(bytes.begin(), bytes.end()), address}); + return; + } + // A label says the work before it is done: the queued key stores (results of that work) land + // ahead of every store of the run. + if (!open->keyStores.empty()) recordKeyStores(true); + if (open->renderPass.open) endOpenRenderPass(); + open->coveredAccess = 0; + // Two transfers to the same bytes have no order of their own within the run. + const auto overlapsRecorded = [&] { return std::any_of(run.recorded.begin(), run.recorded.end(), [&](const auto& store) { return std::get<0>(store) == buffer && offset < std::get<2>(store) && std::get<1>(store) < end; }); }; + if (!run.open) { + run.timing = beginTiming(ClassKey(CommandClass::LabelRun)); + run.storedBytes = 0; + // The run's leading barrier: every earlier write of the range (a shader's, a transfer's, a + // color attachment's) lands before the stores of the run. + recordBarrier(open->commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + CountBarriers(CommandClass::LabelRun); + run.open = true; + storeRuns.fetch_add(1, std::memory_order_relaxed); + } else if (run.buffer == buffer && !run.bytes.empty()) { + const VkDeviceSize pendingEnd = run.offset + run.bytes.size(); + if (offset >= run.offset && end <= pendingEnd) { + std::memcpy(run.bytes.data() + (offset - run.offset), bytes.data(), bytes.size()); + storesReplaced.fetch_add(1, std::memory_order_relaxed); + return; + } + if (offset == pendingEnd && run.bytes.size() + bytes.size() <= 65536) { + // A joined store shares the pending vkCmdUpdateBuffer, which no barrier precedes: one + // that overlaps a store the run already recorded takes the WAW barrier below instead. + if (!JoinWawCheck() || !overlapsRecorded()) { + run.bytes.insert(run.bytes.end(), bytes.begin(), bytes.end()); + storesJoined.fetch_add(1, std::memory_order_relaxed); + return; + } + storeJoinsRefused.fetch_add(1, std::memory_order_relaxed); + } + } + flushPendingStore(); + const bool waw = overlapsRecorded(); + if (waw) { + recordBarrier(open->commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + CountBarriers(CommandClass::LabelRun); + storeWawBarriers.fetch_add(1, std::memory_order_relaxed); + } + run.buffer = buffer; + run.offset = offset; + run.bytes.assign(bytes.begin(), bytes.end()); + if (!LabelRunsEnabled()) closeStoreRun(); +} + +void Recorder::recordBarrier(VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) const { + if (cmdPipelineBarrier == nullptr) { + RecordMemoryBarrier(context, commands, sourceStage, destinationStage, sourceAccess, destinationAccess); + return; + } + const VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, sourceAccess, destinationAccess}; + cmdPipelineBarrier(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + +void Recorder::flushPendingStore() { + auto& run = open->run; + if (run.bytes.empty()) return; + function(cmdUpdateBuffer, "vkCmdUpdateBuffer")(open->commands, run.buffer, run.offset, run.bytes.size(), run.bytes.data()); + run.recorded.emplace_back(run.buffer, run.offset, run.offset + run.bytes.size()); + run.storedBytes += run.bytes.size(); + run.bytes.clear(); +} + +bool Recorder::closeStoreRun(bool atSubmit) { + auto& run = open->run; + if (LabelRunsPerBatch()) { + auto stores = std::move(run.queued); + run.queued.clear(); + if (stores.empty()) return false; + // A label says the work before it is done: the queued key stores (results of that work) + // land ahead of every store of the run, and the pass a draw left open ends. + if (!open->keyStores.empty()) recordKeyStores(true); + if (open->renderPass.open) endOpenRenderPass(); + const auto commands = open->commands; + run.timing = beginTiming(ClassKey(CommandClass::LabelRun)); + if (BarrierValidate()) { + std::vector> ranges; + for (const auto& store : stores) ranges.emplace_back(store.address, store.address + store.bytes.size()); + NoteAccess(CommandClass::LabelRun, Access{{}, ranges, {}, VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + // At Submit every result of the batch reaches the host before the stores (the title's + // pollers read results after the label); in place, later work sees the stores as well. + if (atSubmit) recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT); + else recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + std::uint32_t barriers = 2; + std::uint64_t storedBytes = 0; + const auto update = function(cmdUpdateBuffer, "vkCmdUpdateBuffer"); + for (const auto& store : stores) { + const VkDeviceSize end = store.offset + store.bytes.size(); + // Two transfers to the same bytes have no order of their own within the run. + if (std::any_of(run.recorded.begin(), run.recorded.end(), [&](const auto& earlier) { return std::get<0>(earlier) == store.buffer && store.offset < std::get<2>(earlier) && std::get<1>(earlier) < end; })) { + recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + ++barriers; + storeWawBarriers.fetch_add(1, std::memory_order_relaxed); + } + update(commands, store.buffer, store.offset, store.bytes.size(), store.bytes.data()); + run.recorded.emplace_back(store.buffer, store.offset, end); + storedBytes += store.bytes.size(); + } + if (atSubmit) { + recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + open->coveredAccess = 0; + storeRunsAtSubmit.fetch_add(1, std::memory_order_relaxed); + } else { + recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + open->coveredAccess = VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + storeRunsForced.fetch_add(1, std::memory_order_relaxed); + } + CountBarriers(CommandClass::LabelRun, barriers); + EndGpuTiming(run.timing, storedBytes); + run.timing = NoTiming; + run.recorded.clear(); + return atSubmit; + } + flushPendingStore(); + // The run's stores are visible to the host and to everything recorded after them. + recordBarrier(open->commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + CountBarriers(CommandClass::LabelRun); + EndGpuTiming(run.timing, run.storedBytes); + run.timing = NoTiming; + open->coveredAccess = VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + run.open = false; + run.buffer = VK_NULL_HANDLE; + run.recorded.clear(); + return false; +} + +bool Recorder::QueuedStoreOverlaps(std::uint64_t address, std::size_t bytes) const { + if (open == nullptr || bytes == 0) return false; + const auto end = address + bytes; + return std::any_of(open->run.queued.begin(), open->run.queued.end(), [&](const Batch::StoreRun::Queued& store) { return address < store.address + store.bytes.size() && store.address < end; }); +} + +bool Recorder::AnyQueuedStore(const std::function& overlaps) const { + if (open == nullptr) return false; + return std::any_of(open->run.queued.begin(), open->run.queued.end(), [&](const Batch::StoreRun::Queued& store) { return overlaps(store.address, store.address + store.bytes.size()); }); +} + +void Recorder::FlushStores() { + if (open == nullptr || open->run.queued.empty()) return; + closeStoreRun(false); } namespace { constexpr std::uint32_t MaxTimedRanges = 512; +constexpr std::size_t CommandClasses = static_cast(Recorder::CommandClass::Count); +constexpr const char* CommandClassNames[CommandClasses] = {"dispatch-lead", "dispatch-trail", "indirect-args", "label-run", "fill", "fill-clear", "copy", "staging-in", "staging-out", "draw", "storage-upload", "storage-writeback", "dcc-clear", "dcc-keys", "present-blit", "shadow-publish", "template-refresh"}; +// [barriers] counters by class (relaxed: only reported) and the [gputime] totals, which the +// presenter's thread adds to without the GPU mutex (AddGpuTiming). +std::atomic classBarriers[CommandClasses]{}; +std::atomic classMerged[CommandClasses]{}; +// The hazard tracker's counts (APS5_BARRIER_VALIDATE=1): simulated barriers by class and by +// hazard kind, leading barriers it would have skipped by class, batch-end barriers, and the +// APS5_TRACE_BARRIERS budget left. +constexpr std::size_t HazardKinds = 5; +constexpr const char* HazardNames[HazardKinds] = {"raw", "waw", "war", "image", "conservative"}; +std::atomic validateEmitted[CommandClasses]{}, validateSkipped[CommandClasses]{}, validateKinds[HazardKinds]{}; +std::atomic validateBatchEnds{0}; +std::atomic traceBarriersLeft{[] { + const char* text = std::getenv("APS5_TRACE_BARRIERS"); + return text != nullptr ? static_cast(std::strtoll(text, nullptr, 10)) : std::int64_t{0}; +}()}; +std::atomic timingPresents{0}; +std::atomic presentSerial{0}; +std::atomic timingDropped{0}; +struct TimingTotals { std::uint64_t count = 0; double ms = 0; std::uint64_t bytes = 0; }; +std::mutex timingMutex; +std::map timingByKey; +double timingProgramMs = 0, timingClassMs = 0, timingUnionMs = 0, timingBatchMs = 0; +std::uint64_t timingBatches = 0; -bool GpuTimingEnabled() { +bool DrawOrGpuProfiled() { + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr || Recorder::GpuTimingEnabled(); + return profiled; +} + +void reportBarriers() { + if (!DrawOrGpuProfiled()) return; + static auto lastReport = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::uint64_t total = 0, merged = 0; + std::string line, mergedLine; + for (std::size_t i = 0; i < CommandClasses; ++i) { + const auto count = classBarriers[i].exchange(0, std::memory_order_relaxed); + total += count; + char text[64]; + std::snprintf(text, sizeof(text), " %s %llu", CommandClassNames[i], static_cast(count)); + line += text; + const auto mergedCount = classMerged[i].exchange(0, std::memory_order_relaxed); + if (mergedCount == 0) continue; + merged += mergedCount; + std::snprintf(text, sizeof(text), " %s %llu", CommandClassNames[i], static_cast(mergedCount)); + mergedLine += text; + } + std::fprintf(stderr, "[barriers] %llu recorded (10 s) by class:%s; merged %llu:%s", static_cast(total), line.c_str(), static_cast(merged), mergedLine.c_str()); + if (Recorder::BarrierValidate()) { + std::uint64_t emitted = 0, skipped = 0; + std::string classes, kinds; + for (std::size_t i = 0; i < CommandClasses; ++i) { + const auto emittedCount = validateEmitted[i].exchange(0, std::memory_order_relaxed); + const auto skippedCount = validateSkipped[i].exchange(0, std::memory_order_relaxed); + emitted += emittedCount; + skipped += skippedCount; + if (emittedCount == 0 && skippedCount == 0) continue; + char text[96]; + std::snprintf(text, sizeof(text), " %s %llu/%llu", CommandClassNames[i], static_cast(emittedCount), static_cast(skippedCount)); + classes += text; + } + for (std::size_t i = 0; i < HazardKinds; ++i) { + char text[64]; + std::snprintf(text, sizeof(text), " %s %llu", HazardNames[i], static_cast(validateKinds[i].exchange(0, std::memory_order_relaxed))); + kinds += text; + } + std::fprintf(stderr, "; validate: would emit %llu (%s) + %llu batch ends, would skip %llu; emitted/skipped by class:%s", static_cast(emitted), kinds.c_str() + 1, static_cast(validateBatchEnds.exchange(0, std::memory_order_relaxed)), static_cast(skipped), classes.c_str()); + } + std::fputc('\n', stderr); +} + +} + +bool Recorder::GpuTimingEnabled() { static const bool enabled = std::getenv("APS5_PROFILE_GPU") != nullptr; return enabled; } +void Recorder::CountBarriers(CommandClass which, std::uint32_t count) { + if (!DrawOrGpuProfiled()) return; + classBarriers[static_cast(which)].fetch_add(count, std::memory_order_relaxed); +} + +bool Recorder::MergeBarriers() { + static const bool merge = std::getenv("APS5_FULL_BARRIERS") == nullptr && std::getenv("APS5_NO_BARRIER_ELISION") == nullptr; + return merge; +} + +void Recorder::CountMerged(CommandClass which) { + if (!DrawOrGpuProfiled()) return; + classMerged[static_cast(which)].fetch_add(1, std::memory_order_relaxed); +} + +bool Recorder::BarrierValidate() { + static const bool validate = std::getenv("APS5_BARRIER_VALIDATE") != nullptr; + return validate; +} + +void Recorder::NoteAccess(CommandClass which, const Access& access) { + if (!BarrierValidate()) return; + ensureOpen(); + auto& tracker = open->tracker; + const auto overlapsAny = [](const auto& ranges, const std::pair& range) { + return std::any_of(ranges.begin(), ranges.end(), [&](const auto& other) { return range.first < other.end && other.begin < range.second; }); + }; + std::size_t kind = HazardKinds; + if (access.conservative) kind = 4; + for (const auto& range : access.reads) { + if (kind != HazardKinds) break; + if (overlapsAny(tracker.writes, range)) kind = 0; + } + for (const auto& range : access.writes) { + if (kind != HazardKinds) break; + if (overlapsAny(tracker.writes, range)) kind = 1; + else if (overlapsAny(tracker.reads, range)) kind = 2; + } + for (const auto& [image, written] : access.images) { + if (kind != HazardKinds) break; + if (std::any_of(tracker.images.begin(), tracker.images.end(), [&](const auto& seen) { return seen.image == image && (seen.written || written); })) kind = 3; + } + const auto index = static_cast(which); + if (kind != HazardKinds) { + validateEmitted[index].fetch_add(1, std::memory_order_relaxed); + validateKinds[kind].fetch_add(1, std::memory_order_relaxed); + tracker.reads.clear(); + tracker.writes.clear(); + tracker.images.clear(); + } else { + validateSkipped[index].fetch_add(1, std::memory_order_relaxed); + if (traceBarriersLeft.load(std::memory_order_relaxed) > 0 && traceBarriersLeft.fetch_sub(1, std::memory_order_relaxed) > 0) { + const auto first = [](const auto& ranges) { return ranges.empty() ? std::pair{0, 0} : std::pair{ranges.front().first, ranges.front().second}; }; + const auto reads = first(access.reads); + const auto writes = first(access.writes); + std::fprintf(stderr, "[barriers] would skip %s: reads %zu (first 0x%llx+0x%llx), writes %zu (first 0x%llx+0x%llx), images %zu; since the last barrier: %zu reads, %zu writes, %zu images\n", CommandClassNames[index], access.reads.size(), static_cast(reads.first), static_cast(reads.second - reads.first), access.writes.size(), static_cast(writes.first), static_cast(writes.second - writes.first), access.images.size(), tracker.reads.size(), tracker.writes.size(), tracker.images.size()); + } + } + for (const auto& [begin, end] : access.reads) tracker.reads.push_back({begin, end, access.stages}); + for (const auto& [begin, end] : access.writes) tracker.writes.push_back({begin, end, access.stages}); + for (const auto& [image, written] : access.images) tracker.images.push_back({image, written, access.stages}); +} + +void Recorder::CountPresent() { + timingPresents.fetch_add(1, std::memory_order_relaxed); + presentSerial.fetch_add(1, std::memory_order_relaxed); +} + +std::uint64_t Recorder::Presents() { + return presentSerial.load(std::memory_order_relaxed); +} + +void Recorder::AddGpuTiming(CommandClass which, double nanoseconds, std::uint64_t bytes) { + std::lock_guard lock(timingMutex); + auto& totals = timingByKey[ClassKey(which)]; + ++totals.count; + totals.ms += nanoseconds / 1e6; + totals.bytes += bytes; + timingClassMs += nanoseconds / 1e6; } std::uint32_t Recorder::BeginGpuTiming(std::uint64_t key) { if (!GpuTimingEnabled()) return NoTiming; - const auto commands = Commands(); + Commands(); + return beginTiming(key); +} + +std::uint32_t Recorder::beginTiming(std::uint64_t key) { + if (!GpuTimingEnabled()) return NoTiming; + const auto commands = open->commands; if (open->queries == VK_NULL_HANDLE) { VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; info.queryType = VK_QUERY_TYPE_TIMESTAMP; @@ -873,17 +1764,22 @@ std::uint32_t Recorder::BeginGpuTiming(std::uint64_t key) { } context.Function("vkCmdResetQueryPool")(commands, open->queries, 0, info.queryCount); } - if (open->timedKeys.size() >= MaxTimedRanges) return NoTiming; + if (open->timedKeys.size() >= MaxTimedRanges) { + timingDropped.fetch_add(1, std::memory_order_relaxed); + return NoTiming; + } const auto index = static_cast(open->timedKeys.size()); open->timedKeys.push_back(key); + open->timedBytes.push_back(0); // Both stamps wait for everything before them to complete, so the range is the timed work alone // (a top-of-pipe stamp is not held back by the barrier that precedes the work). context.Function("vkCmdWriteTimestamp")(commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, open->queries, index * 2); return index; } -void Recorder::EndGpuTiming(std::uint32_t index) { +void Recorder::EndGpuTiming(std::uint32_t index, std::uint64_t bytes) { if (index == NoTiming || open == nullptr || open->queries == VK_NULL_HANDLE) return; + if (index < open->timedBytes.size()) open->timedBytes[index] += bytes; context.Function("vkCmdWriteTimestamp")(open->commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, open->queries, index * 2 + 1); } @@ -892,47 +1788,113 @@ void Recorder::readGpuTiming(Batch& batch) { std::vector stamps(batch.timedKeys.size() * 2); const auto result = context.Function("vkGetQueryPoolResults")(context.device, batch.queries, 0, static_cast(stamps.size()), stamps.size() * sizeof(std::uint64_t), stamps.data(), sizeof(std::uint64_t), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); if (result != VK_SUCCESS) return; - struct Totals { std::uint64_t count = 0; double ms = 0; }; - static std::map byKey; - static double totalMs = 0; - static std::uint64_t batches = 0; static auto lastReport = std::chrono::steady_clock::now(); + const auto period = context.limits.timestampPeriod; + // The union of the timed ranges (class ranges nest program ranges: a copy's transfer inside + // its class range) against the batch span gives what no range covers. + std::vector> intervals; + intervals.reserve(batch.timedKeys.size()); + std::unique_lock lock(timingMutex); for (std::size_t i = 0; i < batch.timedKeys.size(); ++i) { - const auto ns = static_cast(stamps[i * 2 + 1] - stamps[i * 2]) * context.limits.timestampPeriod; - auto& totals = byKey[batch.timedKeys[i]]; + const auto ns = static_cast(stamps[i * 2 + 1] - stamps[i * 2]) * period; + if (i == batch.batchTiming) { + // The whole batch: apart from the per-program totals it contains. + batch.gpuStartNs = static_cast(stamps[i * 2]) * period; + batch.gpuEndNs = static_cast(stamps[i * 2 + 1]) * period; + timingBatchMs += ns / 1e6; + continue; + } + const auto key = batch.timedKeys[i]; + auto& totals = timingByKey[key]; ++totals.count; totals.ms += ns / 1e6; - totalMs += ns / 1e6; + totals.bytes += batch.timedBytes[i]; + (key >= ClassKey(CommandClass::DispatchLeading) && key < ClassKey(CommandClass::Count) ? timingClassMs : timingProgramMs) += ns / 1e6; + intervals.emplace_back(stamps[i * 2], stamps[i * 2 + 1]); } - ++batches; + std::sort(intervals.begin(), intervals.end()); + std::uint64_t coveredTicks = 0, unionEnd = 0; + for (const auto& [begin, end] : intervals) { + const auto from = std::max(begin, unionEnd); + if (end > from) coveredTicks += end - from; + unionEnd = std::max(unionEnd, end); + } + timingUnionMs += static_cast(coveredTicks) * period / 1e6; + ++timingBatches; const auto now = std::chrono::steady_clock::now(); if (now - lastReport < std::chrono::seconds(10)) return; lastReport = now; - std::vector> hot(byKey.begin(), byKey.end()); + const auto presents = timingPresents.exchange(0, std::memory_order_relaxed); + const double perPresent = presents != 0 ? 1.0 / static_cast(presents) : 0.0; + std::vector> hot; + std::string classes; + for (const auto& [key, totals] : timingByKey) { + if (key >= ClassKey(CommandClass::DispatchLeading) && key < ClassKey(CommandClass::Count)) { + char text[160]; + std::snprintf(text, sizeof(text), " %s x%llu %.1fms %.1fMiB (per present x%.1f %.2fms %.2fMiB)", CommandClassNames[key - ClassKey(CommandClass::DispatchLeading)], static_cast(totals.count), totals.ms, totals.bytes / 1048576.0, static_cast(totals.count) * perPresent, totals.ms * perPresent, totals.bytes / 1048576.0 * perPresent); + classes += text; + } else { + hot.emplace_back(key, totals); + } + } std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.ms > b.second.ms; }); - std::fprintf(stderr, "[gputime] %.0f ms of GPU time in %llu batches over 10 s; by program:", totalMs, static_cast(batches)); + // The first field stays the program sum: the measure scripts match on it. + std::fprintf(stderr, "[gputime] %.0f ms of GPU time in %llu batches over 10 s (batch %.0f ms first to last command; classes %.0f ms, all ranges %.0f ms, untimed %.0f ms outside every range; %llu presents; %llu ranges dropped at the %u cap); by program:", timingProgramMs, static_cast(timingBatches), timingBatchMs, timingClassMs, timingUnionMs, timingBatchMs - timingUnionMs, static_cast(presents), static_cast(timingDropped.exchange(0, std::memory_order_relaxed)), MaxTimedRanges); for (std::size_t i = 0; i < hot.size() && i < 12; ++i) std::fprintf(stderr, " 0x%llx x%llu %.0fms", static_cast(hot[i].first), static_cast(hot[i].second.count), hot[i].second.ms); - std::fprintf(stderr, "\n"); - byKey.clear(); - totalMs = 0; - batches = 0; + std::fprintf(stderr, "; by class:%s\n", classes.c_str()); + timingByKey.clear(); + timingProgramMs = timingClassMs = timingUnionMs = timingBatchMs = 0; + timingBatches = 0; + lock.unlock(); + reportBarriers(); +} + +bool Recorder::FlipReadCheck() { + static const bool enabled = std::getenv("APS5_FLIP_READ_CHECK") != nullptr; + return enabled; +} + +std::vector Recorder::CompletedBatches(std::uint64_t afterSerial, std::uint64_t throughSerial, std::size_t& missing) const { + std::vector found; + missing = 0; + if (throughSerial <= afterSerial) return found; + std::lock_guard lock(completedMutex); + for (auto serial = afterSerial + 1; serial <= throughSerial; ++serial) { + const auto& entry = completed[serial % completed.size()]; + if (entry.serial == serial) found.push_back(entry); + else ++missing; + } + return found; +} + +std::uint64_t Recorder::NewestSubmitted(std::chrono::steady_clock::time_point* submittedAt) const { + std::lock_guard lock(completedMutex); + if (submittedAt != nullptr) *submittedAt = newestSubmittedAt; + return newestSubmitted; +} + +std::size_t Recorder::UnsignaledBatches() const { + return static_cast(std::count_if(inFlight.begin(), inFlight.end(), [&](const auto& batch) { return !signaled(*batch); })); } void Recorder::Keep(std::shared_ptr object) { - Commands(); + ensureOpen(); open->kept.push_back(std::move(object)); } void Recorder::OnComplete(std::function action) { - Commands(); + ensureOpen(); open->completions.push_back(std::move(action)); + ++open->writeBackCompletionCount; + writeBackCompletions.fetch_add(1, std::memory_order_acq_rel); } -bool Recorder::noteWrite(std::uint64_t address, std::size_t bytes) { +bool Recorder::noteWrite(std::uint64_t address, std::size_t bytes, bool ownLabel) { if (bytes == 0) return false; - Commands(); + ensureOpen(); const auto end = address + bytes; open->writes.emplace_back(address, end); + if (!ownLabel) markOverwritten(address, end); // A poller waiting on this range learns that the open batch may now hold its producer. if (activeRecorder == this) writeGeneration.fetch_add(1, std::memory_order_release); // Consecutive dispatches write the same output buffers: a range the snapshot already contains @@ -944,21 +1906,31 @@ bool Recorder::noteWrite(std::uint64_t address, std::size_t bytes) { return true; } -void Recorder::noteWriteOn(Batch& batch, std::uint64_t address, std::size_t bytes) { +void Recorder::noteWriteOn(Batch& batch, std::uint64_t address, std::size_t bytes, bool ownLabel) { if (bytes == 0) return; if (&batch == open.get()) { - NotePendingWrite(address, bytes); + if (!noteWrite(address, bytes, ownLabel)) return; + publishPendingWrites(); + std::atomic_thread_fence(std::memory_order_seq_cst); return; } // An in-flight batch: its range joins the snapshot at once (the completion that stores it runs // when the batch finishes, and the hook must sync for a CPU read until then). The generation // moves as well, so a poller re-consults the label table for a completion label. batch.writes.emplace_back(address, address + bytes); + if (!ownLabel) markOverwritten(address, address + bytes); if (activeRecorder == this) writeGeneration.fetch_add(1, std::memory_order_release); if (!SnapshotCovers(address, address + bytes)) publishPendingWrites(); std::atomic_thread_fence(std::memory_order_seq_cst); } +void Recorder::markOverwritten(std::uint64_t address, std::uint64_t end) { + if (recordedLabels.load(std::memory_order_relaxed) == 0) return; + std::lock_guard tableLock(labelTableMutex); + // A dword overlaps the range when it starts before the end and ends after the start. + for (auto it = labels.lower_bound(address >= 3 ? address - 3 : 0); it != labels.end() && it->first < end; ++it) it->second.overwritten = true; +} + void Recorder::NotePendingWrite(std::uint64_t address, std::size_t bytes) { if (!noteWrite(address, bytes)) return; publishPendingWrites(); @@ -994,6 +1966,7 @@ void Recorder::publishPendingWrites() const { } merged->resize(out); pendingWrites.store(std::move(merged), std::memory_order_release); + publishGeneration.fetch_add(1, std::memory_order_release); ++snapshotRebuilds; if (DrawProfiled()) snapshotRebuildMs += std::chrono::duration(std::chrono::steady_clock::now() - started).count(); } @@ -1019,6 +1992,89 @@ bool Recorder::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const return bytes != 0 && open != nullptr && overlaps(*open, address, address + bytes); } +bool Recorder::signaled(const Batch& batch) const { + return batch.submitted && fenceStatus(batch.fence) == VK_SUCCESS; +} + +bool Recorder::PendingWriteSettled(std::uint64_t address, std::size_t bytes) const { + if (bytes == 0) return true; + const auto end = address + bytes; + if (open != nullptr && overlaps(*open, address, end)) return false; + for (const auto& batch : inFlight) { + if (overlaps(*batch, address, end) && !signaled(*batch)) return false; + } + return true; +} + +bool Recorder::ReadTracking() { + return ReadTrackingEnabled(); +} + +void Recorder::NotePendingRead(std::uint64_t address, std::size_t bytes, ReadKind kind) { + if (bytes == 0 || !ReadTrackingEnabled()) return; + ensureOpen(); + open->reads.push_back({address, address + bytes, kind}); + readsNoted.fetch_add(1, std::memory_order_relaxed); +} + +void Recorder::NotePendingReads(std::span> ranges, ReadKind kind) { + if (ranges.empty() || !ReadTrackingEnabled()) return; + ensureOpen(); + for (const auto& [begin, end] : ranges) { + if (end > begin) open->reads.push_back({begin, end, kind}); + } + readsNoted.fetch_add(ranges.size(), std::memory_order_relaxed); +} + +const Recorder::Batch::Read* Recorder::readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end) { + for (const auto& read : batch.reads) { + if (address < read.end && read.begin < end) return &read; + } + return nullptr; +} + +bool Recorder::PendingReadOverlaps(std::uint64_t address, std::size_t bytes, bool ignoreSignaled) const { + if (bytes == 0) return false; + readQueries.fetch_add(1, std::memory_order_relaxed); + const auto end = address + bytes; + if (open != nullptr) { + if (const auto* read = readOverlap(*open, address, end)) { + readHits[static_cast(read->kind)].fetch_add(1, std::memory_order_relaxed); + return true; + } + } + // Newest first: the batch most likely still running decides without a status query per batch. + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) { + const auto* read = readOverlap(**it, address, end); + if (read == nullptr) continue; + if (ignoreSignaled && signaled(**it)) { + readStaleIgnored.fetch_add(1, std::memory_order_relaxed); + continue; + } + readHits[static_cast(read->kind)].fetch_add(1, std::memory_order_relaxed); + return true; + } + return false; +} + +std::optional Recorder::DescribePendingRead(std::uint64_t address, std::size_t bytes) const { + if (bytes == 0) return std::nullopt; + const auto end = address + bytes; + if (open != nullptr) { + if (const auto* read = readOverlap(*open, address, end)) return PendingReadInfo{submissions + 1, open->queue, read->kind, true, false}; + } + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) { + if (const auto* read = readOverlap(**it, address, end)) return PendingReadInfo{(*it)->serial, (*it)->queue, read->kind, false, signaled(**it)}; + } + return std::nullopt; +} + +Recorder::ReadStatistics Recorder::ReadCounts() { + ReadStatistics counts{readsNoted.load(std::memory_order_relaxed), readQueries.load(std::memory_order_relaxed), readStaleIgnored.load(std::memory_order_relaxed), {}}; + for (std::size_t kind = 0; kind < ReadKinds; ++kind) counts.hits[kind] = readHits[kind].load(std::memory_order_relaxed); + return counts; +} + std::optional Recorder::DescribePendingWrite(std::uint64_t address, std::size_t bytes) const { if (bytes == 0) return std::nullopt; const auto end = address + bytes; @@ -1043,7 +2099,7 @@ std::optional Recorder::DescribePendingWrite(std::ui const auto* range = firstOverlap(**it); if (range == nullptr) continue; // In flight means submitted, so the fence is live; signaled = the GPU already ran it. - const bool signaled = context.Function("vkGetFenceStatus")(context.device, (*it)->fence) == VK_SUCCESS; + const bool signaled = fenceStatus((*it)->fence) == VK_SUCCESS; if (syncAll) return PendingWriteInfo{submissions + (open != nullptr ? 1 : 0), open != nullptr, signaled, range->first, range->second, allBatches}; return PendingWriteInfo{(*it)->serial, false, signaled, range->first, range->second, finished}; } @@ -1058,7 +2114,7 @@ bool Recorder::HasCompletions() const { return false; } -void Recorder::noteLabelOn(Batch& batch, std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue) { +void Recorder::noteLabelOn(Batch& batch, std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool behindCompletion) { // One entry per dword; a label overlapping older entries (a 4-byte store inside an 8-byte one or // the reverse) replaces exactly the dwords it stores, so a lookup composes what memory will hold. std::lock_guard tableLock(labelTableMutex); @@ -1066,45 +2122,109 @@ void Recorder::noteLabelOn(Batch& batch, std::uint64_t address, std::spanfirst < end; ++it) { + if (it->second.batch == &batch) it->second.behindCompletion = true; } } void Recorder::NoteLabel(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue) { - Commands(); + ensureOpen(); + // The table entry before the write note: the note bumps the write generation a poller + // watches, and a poller that sees the bump then finds the label without the GPU mutex. noteLabelOn(*open, address, bytes, stamp, queue); if (activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) { pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release); } + noteWriteOn(*open, address, bytes.size(), true); } -std::optional Recorder::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { +std::optional Recorder::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal) const { // Under the GPU mutex; the table mutex is taken too so this one path serves both lock regimes. std::lock_guard tableLock(labelTableMutex); - return lookupLabel(address, bytes, afterStamp, queue); + return lookupLabel(address, bytes, afterStamp, refusal); +} + +std::size_t Recorder::PendingLabels() const { + std::lock_guard tableLock(labelTableMutex); + return labels.size(); } bool Recorder::PendingLabelIn(std::uint64_t address, std::size_t bytes) const { std::lock_guard tableLock(labelTableMutex); - if (labels.empty() || bytes == 0) return false; + if (bytes == 0) return false; const auto end = address + bytes; - for (const auto& [dword, entry] : labels) { - if (dword >= address && dword < end) return true; - } + if (const auto first = labels.lower_bound(address); first != labels.end() && first->first < end) return true; + // A queued label of another queue is unordered against the caller on hardware (nothing that + // queue recorded could have satisfied a wait yet); it is still reported, since the table mutex + // is held anyway and a caller deciding a CPU store wants the conservative answer. + if (const auto first = queuedLabels.lower_bound(address); first != queuedLabels.end() && first->first < end) return true; return false; } -std::optional Recorder::lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { - if (labels.empty() || (bytes != 4 && bytes != 8) || address % 4 != 0) return std::nullopt; - std::uint64_t value = 0; +std::optional Recorder::lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal) const { + if (refusal != nullptr) *refusal = LabelRefusal::None; + if ((labels.empty() && queuedLabels.empty()) || (bytes != 4 && bytes != 8) || address % 4 != 0) return std::nullopt; + LabelHit hit{0, 0, 0, false, std::numeric_limits::max()}; + bool queued = false; + bool candidate = false; + const auto tag = GuestMemory::GpuLockThreadTag(); + const auto refuse = [&](LabelRefusal reason, std::uint64_t LateStatistics::*counter, const LabelEntry& entry) { + if (refusal != nullptr) *refusal = entry.behindCompletion ? LabelRefusal::BehindCompletion : reason; + if (counter != nullptr) ++(lateCounts.*counter); + return std::nullopt; + }; for (std::size_t offset = 0; offset < bytes; offset += 4) { - const auto found = labels.find(address + offset); - if (found == labels.end() || found->second.stamp <= afterStamp) return std::nullopt; - value |= static_cast(found->second.value) << (offset * 8u); - if (offset == 0) queue = found->second.queue; + const auto dword = address + offset; + const LabelEntry* entry = nullptr; + // A queued label is only good for its own queue's wait (see NoteQueuedLabel), where it is + // that queue's newest store of the dword; everyone else sees the recorded entry. + if (const auto own = queuedLabels.find(dword); own != queuedLabels.end() && own->second.queue == tag) { + entry = &own->second; + queued = true; + } else if (const auto found = labels.find(dword); found != labels.end()) { + entry = &found->second; + if (entry->batch == nullptr) { + if (entry->queue != tag) return std::nullopt; + queued = true; + } + } + if (entry == nullptr) return std::nullopt; + if (entry->stamp <= afterStamp) { + // The late rule (see NoteLabel): counted in both modes, so a control run shows the population. + if (!candidate) ++lateCounts.candidates; + candidate = true; + if (!LateTrustEnabled()) return refuse(LabelRefusal::TrustOff, nullptr, *entry); + if (entry->batch == nullptr) return refuse(LabelRefusal::Queued, &LateStatistics::queued, *entry); + if (entry->overwritten) return refuse(LabelRefusal::Overwritten, &LateStatistics::overwritten, *entry); + if (entry->generation == 0) return refuse(LabelRefusal::Unclosed, &LateStatistics::unclosed, *entry); + hit.late = true; + hit.generation = std::min(hit.generation, entry->generation); + } + hit.value |= static_cast(entry->value) << (offset * 8u); + if (offset == 0) { + hit.queue = entry->queue; + hit.stamp = entry->stamp; + } } - return value; + if (!hit.late) hit.generation = 0; + if (queued) queuedLabelHits.fetch_add(1, std::memory_order_relaxed); + return hit; +} + +bool Recorder::unsignaled(std::uint64_t serial) const { + for (const auto& batch : inFlight) { + if (batch->serial == serial) return fenceStatus(batch->fence) != VK_SUCCESS; + } + return false; } void Recorder::AfterCompletions(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu) { @@ -1132,12 +2252,19 @@ void Recorder::AfterCompletions(std::uint64_t address, std::spanwrittenBackMutex); - recorder->writtenBack.push_back({++recorder->writtenBackSequence, address, address + bytes}); - while (recorder->writtenBack.size() > 16384) recorder->writtenBack.pop_front(); + GuestMemory::AssertGpuLockHeld("Recorder::NoteWrittenBack"); + const auto end = address + bytes; + { + std::lock_guard ringLock(recorder->writtenBackMutex); + recorder->writtenBack.push_back({++recorder->writtenBackSequence, address, end}); + while (recorder->writtenBack.size() > 16384) recorder->writtenBack.pop_front(); + } + if (recorder->recordedLabels.load(std::memory_order_relaxed) != 0 && recorder->PendingLabelIn(address, bytes)) writeBacksOverLabels.fetch_add(1, std::memory_order_relaxed); + // The GPU-stored completion labels this write-back overlaps now need their completion store, + // so they enter the pending count (once each; finish() subtracts the batch's count). Batches + // in `finishing` run their own completions in the same finish(), so only open and in-flight + // ones are scanned. + const auto count = [&](Batch& batch) { + for (auto& label : batch.completionLabelRanges) { + if (label.counted || label.begin >= end || address >= label.end) continue; + label.counted = true; + ++batch.completionLabelCount; + completionLabels.fetch_add(1, std::memory_order_acq_rel); + completionLabelsCountedLate.fetch_add(1, std::memory_order_relaxed); + recorder->markBehindCompletion(batch, label.begin, label.end); + } + }; + if (recorder->open != nullptr) count(*recorder->open); + for (auto& batch : recorder->inFlight) count(*batch); } bool Recorder::writtenBackSince(std::uint64_t sequence, std::uint64_t begin, std::uint64_t end) { @@ -1169,8 +2317,25 @@ void Recorder::Submit() { if (activeRecorder == this) workSinceSubmit.store(0, std::memory_order_relaxed); if (open == nullptr) return; GuestMemory::AssertGpuLockHeld("Recorder::Submit"); + if (!open->keyStores.empty()) recordKeyStores(false); + if (open->renderPass.open) endOpenRenderPass(); + const bool hostReadCovered = (open->run.open || !open->run.queued.empty()) && closeStoreRun(true); + if (BarrierValidate()) validateBatchEnds.fetch_add(1, std::memory_order_relaxed); + if (open->hostReadOwed && !hostReadCovered) { + // A fence makes nothing visible to the host: a lean draw's writes (no download barrier of + // its own) get the batch's one here. + recordBarrier(open->commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + CountBarriers(CommandClass::Draw); + } + EndGpuTiming(open->batchTiming); + if (!GpuTimingEnabled()) reportBarriers(); + if (FlipReadCheck()) { + // A CPU write into a noted read after this collect (and so possibly before the GPU read + // it) fails UnchangedSince at the presenter's check (VulkanDevice RetirePresents). + for (const auto& read : open->reads) open->readGeneration = std::max(open->readGeneration, GuestMemory::CollectWrites(read.begin, read.end - read.begin)); + } auto batch = std::move(open); - Check(context.Function("vkEndCommandBuffer")(batch->commands), "vkEndCommandBuffer recorder"); + Check(function(endCommandBuffer, "vkEndCommandBuffer")(batch->commands), "vkEndCommandBuffer recorder"); VkSubmitInfo submission{VK_STRUCTURE_TYPE_SUBMIT_INFO}; submission.commandBufferCount = 1; submission.pCommandBuffers = &batch->commands; @@ -1184,9 +2349,23 @@ void Recorder::Submit() { submission.signalSemaphoreCount = 1; submission.pSignalSemaphores = &timeline; } - Check(context.Function("vkQueueSubmit")(context.queue, 1, &submission, batch->fence), "vkQueueSubmit recorder"); + const auto submitStart = std::chrono::steady_clock::now(); + Check(function(queueSubmit, "vkQueueSubmit")(context.queue, 1, &submission, batch->fence), "vkQueueSubmit recorder"); batch->submitted = true; batch->serial = ++submissions; + batch->submittedAt = std::chrono::steady_clock::now(); + if (DrawProfiled()) { + const auto us = std::chrono::duration(batch->submittedAt - submitStart).count(); + ++submitCount; + submitUs += us; + submitMaxUs = std::max(submitMaxUs, us); + if (auto* frame = PerformanceContext::Current()) frame->Add(frame->Get("Recorder", "submit"), batch->submittedAt - submitStart); + } + { + std::lock_guard lock(completedMutex); + newestSubmitted = batch->serial; + newestSubmittedAt = batch->submittedAt; + } inFlight.push_back(std::move(batch)); if (activeRecorder == this) pendingLabelSince.store(NoPendingLabel, std::memory_order_release); } @@ -1320,6 +2499,8 @@ void Recorder::SyncThrough(std::uint64_t address, std::size_t bytes, bool waitUn if (!SyncThroughEnabled() || (open != nullptr && overlaps(*open, address, end))) { // The announced site (or this call's caller) names the sync, not this function. if (announcedSite == nullptr) announcedSite = __builtin_return_address(0); + // The open batch's work has not run; without one the newest in flight is the target. + if (open != nullptr || (!inFlight.empty() && unsignaled(inFlight.back()->serial))) ++hookRealWaits; Sync(); return; } @@ -1333,6 +2514,7 @@ void Recorder::SyncThrough(std::uint64_t address, std::size_t bytes, bool waitUn } } if (targetSerial == 0) return; + if (unsignaled(targetSerial)) ++hookRealWaits; targetedSyncs.fetch_add(1, std::memory_order_relaxed); const RestoreSyncSite restoreSite{BeginSyncSite(source, site != nullptr ? site : __builtin_return_address(0))}; // Batches are in flight in serial order. A completion's own guest access can sync re-entrantly @@ -1346,6 +2528,63 @@ void Recorder::SyncThrough(std::uint64_t address, std::size_t bytes, bool waitUn batchesLeftInFlight.fetch_add(inFlight.size(), std::memory_order_relaxed); } +namespace { + +// What the unlocked hook waits waited for, per read site (the [syncwait] line every 10 s under +// APS5_PROFILE_DRAW): the batches up to the target, how many of them the GPU had not finished when +// the wait began, and whether the target was the newest submission. APS5_TRACE_CAPSYNC=1 prints +// every such wait ([capsync] sync lines). +struct SyncWaitSite { + std::uint64_t count = 0; + std::uint64_t batchesToTarget = 0; + std::uint64_t unsignaledAtStart = 0; + std::uint64_t newest = 0; + double waitedMs = 0; + // The target batch's submit -> wait-end latency: what the wait's length is made of when the + // batches ahead of it carry little GPU work (submission latency, not throughput). + double targetAgeMs = 0; +}; + +struct SyncWaitStats { + std::array(GuestMemory::ReadSite::Count)> bySite{}; + // Batches up to the targets, by the queue that opened them: whether a wait for one queue's + // producer is really a wait for the other queues' batches ahead of it on the one timeline. + std::map aheadByQueue; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +SyncWaitStats& SyncWaits() { + static SyncWaitStats stats; + return stats; +} + +bool TraceCapSync() { + static const bool trace = std::getenv("APS5_TRACE_CAPSYNC") != nullptr; + return trace; +} + +void ReportSyncWaits(SyncWaitStats& stats) { + std::string report = "[syncwait] unlocked hook waits by read site (10 s; count/GPU wait, avg batches up to the target, avg of them unsignaled at the start, target was the newest submission, avg target submit-to-signal ms):"; + for (std::size_t site = 0; site < stats.bySite.size(); ++site) { + const auto& entry = stats.bySite[site]; + if (entry.count == 0) continue; + char text[160]; + std::snprintf(text, sizeof(text), " %s %llu/%.0fms %.1f %.1f %.0f%% %.1f", GuestMemory::ReadSiteName(static_cast(site)), static_cast(entry.count), entry.waitedMs, static_cast(entry.batchesToTarget) / entry.count, static_cast(entry.unsignaledAtStart) / entry.count, entry.newest * 100.0 / entry.count, entry.targetAgeMs / entry.count); + report += text; + } + report += "; batches ahead by queue:"; + for (const auto& [queue, count] : stats.aheadByQueue) { + char text[48]; + if (queue == 0xffffffffu) std::snprintf(text, sizeof(text), " untagged %llu", static_cast(count)); + else std::snprintf(text, sizeof(text), " 0x%x %llu", queue, static_cast(count)); + report += text; + } + std::fprintf(stderr, "%s\n", report.c_str()); + stats = SyncWaitStats{}; +} + +} + bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int source, const void* site) { // Only for the outermost acquisition: a nested hook (inside a dispatch's hold) must not give // up the caller's mutex, and a thread that waits here holds nothing the relock could invert @@ -1369,6 +2608,7 @@ bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int announcedSource = 4; announcedSite = nullptr; if (targetSerial == 0) return true; + if (unsignaled(targetSerial)) ++hookRealWaits; // Everything the wait needs is copied out: the recorder may be torn down while the mutex is // released (its destructor completes every batch and waits for the semaphore to be left, see // WaitTimeline), after which `this` must not be touched. The id re-check below relies on the @@ -1382,7 +2622,32 @@ bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int // appended, never removed, so the index stays valid across the release). const RestoreSyncSite restoreSite{BeginSyncSite(source, site)}; const bool profile = DrawProfiled(); - const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const bool trace = TraceCapSync(); + const auto readSite = GuestMemory::CurrentReadSite(); + const auto newestSerial = submissions; + std::size_t batchesToTarget = 0, unsignaledAtStart = 0, targetRanges = 0; + std::pair hit{0, 0}; + std::chrono::steady_clock::time_point targetSubmittedAt{}; + std::map aheadByQueue; + if (profile || trace) { + const auto status = context.Function("vkGetFenceStatus"); + for (const auto& batch : inFlight) { + if (batch->serial > targetSerial) break; + ++batchesToTarget; + ++aheadByQueue[batch->queue]; + if (status(context.device, batch->fence) != VK_SUCCESS) ++unsignaledAtStart; + if (batch->serial != targetSerial) continue; + targetSubmittedAt = batch->submittedAt; + targetRanges = batch->writes.size(); + for (const auto& range : batch->writes) { + if (address < range.second && range.first < end) { + hit = range; + break; + } + } + } + } + const auto start = profile || trace ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; auto waited = start; auto& mutex = GuestMemory::GpuMutex(); VkResult result = VK_SUCCESS; @@ -1392,12 +2657,12 @@ bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int mutex.unlock(); result = WaitTimeline(device, semaphore, waitSemaphores, targetSerial); // Read before the relock: the GPU wait, not the wait for the mutex behind it. - if (profile) waited = std::chrono::steady_clock::now(); + if (profile || trace) waited = std::chrono::steady_clock::now(); } GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); mutex.lock(); + const auto ms = std::chrono::duration(waited - start).count(); if (profile) { - const auto ms = std::chrono::duration(waited - start).count(); // The GPU wait is charged like a locked wait, so the per-source, per-thread and per-site // totals of the [recorder] line keep their meaning; the unlocked count says how many were // not held. The relock's wait for the mutex is kept apart (the [lock] line also counts it @@ -1407,7 +2672,28 @@ bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int CountSiteWait(ms); ++holdCounters.hookUnlockedWaits; holdCounters.hookUnlockedWaitMs += ms; + if (source >= 0 && source < 5) { + ++holdCounters.hookUnlockedWaitsBySource[source]; + holdCounters.hookUnlockedWaitMsBySource[source] += ms; + } holdCounters.hookRelockMs += std::chrono::duration(std::chrono::steady_clock::now() - waited).count(); + auto& stats = SyncWaits(); + auto& siteStats = stats.bySite[static_cast(readSite)]; + ++siteStats.count; + siteStats.waitedMs += ms; + siteStats.batchesToTarget += batchesToTarget; + siteStats.unsignaledAtStart += unsignaledAtStart; + if (targetSerial == newestSerial) ++siteStats.newest; + if (targetSubmittedAt != std::chrono::steady_clock::time_point{}) siteStats.targetAgeMs += std::chrono::duration(waited - targetSubmittedAt).count(); + for (const auto& [queue, count] : aheadByQueue) stats.aheadByQueue[queue] += count; + if (std::chrono::steady_clock::now() - stats.lastReport > std::chrono::seconds(10)) ReportSyncWaits(stats); + } + if (trace) { + static std::atomic lines{0}; + if (lines.fetch_add(1) < 600) { + const auto packet = GuestMemory::CurrentPacket(); + std::fprintf(stderr, "[capsync] sync q0x%x %s %s 0x%llx+0x%llx: target %llu (newest %llu), %zu batches up to it (%zu unsignaled at the start), target noted %zu ranges, hit 0x%llx+0x%llx, GPU wait %.1f ms\n", packet.queue, PacketName(packet.opcode).c_str(), GuestMemory::ReadSiteName(readSite), static_cast(address), static_cast(end - address), static_cast(targetSerial), static_cast(newestSerial), batchesToTarget, unsignaledAtStart, targetRanges, static_cast(hit.first), static_cast(hit.second - hit.first), ms); + } } if (!RecorderAlive(myId)) { // Torn down meanwhile: its destructor's Sync() ran every completion, nothing is pending. @@ -1430,12 +2716,11 @@ bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int } bool Recorder::Reap() { - const auto status = context.Function("vkGetFenceStatus"); const bool profile = DrawProfiled(); const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; std::uint64_t retired = 0; while (!inFlight.empty()) { - if (status(context.device, inFlight.front()->fence) != VK_SUCCESS) break; + if (fenceStatus(inFlight.front()->fence) != VK_SUCCESS) break; auto batch = std::move(inFlight.front()); inFlight.pop_front(); finish(std::move(batch), false, 4); @@ -1460,6 +2745,7 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source) { list.erase(std::remove(list.begin(), list.end(), batch), list.end()); // The batch's completion labels have landed, or never will (fence failure below). if (batch->completionLabelCount != 0) completionLabels.fetch_sub(batch->completionLabelCount, std::memory_order_acq_rel); + if (batch->writeBackCompletionCount != 0) writeBackCompletions.fetch_sub(batch->writeBackCompletionCount, std::memory_order_acq_rel); // This also runs while the fence-failure throw unwinds: an allocation failure in the // rebuild keeps the previous snapshot, which is a superset (a stale entry only causes a // false hit, never a miss) rather than terminating. @@ -1477,11 +2763,15 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source) { static std::uint64_t waits = 0; static auto lastReport = std::chrono::steady_clock::now(); const auto waitStart = std::chrono::steady_clock::now(); + // A fence still unsignaled here makes this a real GPU wait under the mutex (finish runs + // under it): the [lock] line's 'locked GPU waits'. + const bool signaledAtStart = !profile || context.Function("vkGetFenceStatus")(context.device, batch->fence) == VK_SUCCESS; struct Report { bool enabled; int source; std::chrono::steady_clock::time_point start; const Recorder& recorder; + bool signaledAtStart; ~Report() { if (!enabled) return; const auto now = std::chrono::steady_clock::now(); @@ -1491,15 +2781,19 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source) { ++waits; CountThreadSync(source, ms); CountSiteWait(ms); + if (!signaledAtStart) GuestMemory::NoteLockedGpuWait(ms); if (now - lastReport > std::chrono::seconds(10)) { lastReport = now; const auto& h = holdCounters; - std::fprintf(stderr, "[recorder] %llu syncs waited %.1f s for the GPU in total (sources, count/wait: idle %llu/%.1fs, pending write %llu/%.1fs, recorded store %llu/%.1fs, address-based %llu/%.1fs, other %llu/%.1fs); hook %llu calls, %llu locked; %llu targeted syncs left %llu batches in flight; snapshot %llu rebuilds %.0f ms, %llu notes covered; %llu unlocked timeline waits %.1f s; %llu submissions, %zu label entries, %llu completion labels pending; fence waits by thread (count/wait):%s; top sync sites (source@caller syncs/batches/wait):%s; under holds (cumulative): %llu completions ran %.0f ms (kept objects released %.0f ms), pending-write syncs from completions: %llu skipped, %llu waited %.0f ms; %llu reaps (%llu with work) retired %llu batches in %.0f ms; hook waits unlocked %llu / %.0f ms GPU + %.0f ms relock (%llu found the recorder torn down), locked %llu; deferred releases: on the release thread %llu batches (%llu objects) in %.0f ms, inline %llu batches (%llu objects) in %.0f ms (%llu batches over the queue bound of %zu), queue max %llu batches, %llu pending\n",static_cast(waits), waitedMs / 1000, static_cast(syncCounts[0]), syncWaitedMs[0] / 1000, static_cast(syncCounts[1]), syncWaitedMs[1] / 1000, static_cast(syncCounts[2]), syncWaitedMs[2] / 1000, static_cast(syncCounts[3]), syncWaitedMs[3] / 1000, static_cast(syncCounts[4]), syncWaitedMs[4] / 1000, static_cast(hookCalls.load()), static_cast(hookLocks.load()), static_cast(targetedSyncs.load()), static_cast(batchesLeftInFlight.load()), static_cast(snapshotRebuilds), snapshotRebuildMs, static_cast(snapshotCovered), static_cast(unlockedWaits.load()), unlockedWaitedUs.load() / 1e6, static_cast(recorder.submissions), recorder.labels.size(), static_cast(completionLabels.load()), ThreadSyncReport().c_str(), SyncSiteReport().c_str(), static_cast(h.completions), h.completionMs, h.keptReleaseMs, static_cast(h.completionSyncsSkipped), static_cast(h.completionSyncsWaited), h.completionSyncWaitMs, static_cast(h.reaps), static_cast(h.reapsWithWork), static_cast(h.reapBatches), h.reapMs, static_cast(h.hookUnlockedWaits), h.hookUnlockedWaitMs, h.hookRelockMs, static_cast(h.hookUnlockedTornDown), static_cast(h.hookLockedWaits), static_cast(threadReleases.load()), static_cast(threadObjects.load()), threadReleaseUs.load() / 1000.0, static_cast(inlineReleases.load()), static_cast(inlineObjects.load()), inlineReleaseUs.load() / 1000.0, static_cast(inlineOverBound.load()), ReleaseQueueBound(), static_cast(releaseQueueMax.load()), static_cast(deferredPending.load())); - std::fprintf(stderr, "[recorder] completion label stores: %llu run, %llu skipped (no CPU write-back overlapped them)\n", static_cast(completionStoresRun.load()), static_cast(completionStoresSkipped.load())); + std::fprintf(stderr, "[recorder] %llu syncs waited %.1f s for the GPU in total (sources, count/wait: idle %llu/%.1fs, pending write %llu/%.1fs, recorded store %llu/%.1fs, address-based %llu/%.1fs, other %llu/%.1fs); hook %llu calls, %llu locked; %llu targeted syncs left %llu batches in flight; snapshot %llu rebuilds %.0f ms, %llu notes covered; %llu unlocked timeline waits %.1f s; %llu submissions, %zu label entries, %llu completion labels pending; fence waits by thread (count/wait):%s; top sync sites (source@caller syncs/batches/wait):%s; under holds (cumulative): %llu completions ran %.0f ms (kept objects released %.0f ms), pending-write syncs from completions: %llu skipped, %llu waited %.0f ms; %llu reaps (%llu with work) retired %llu batches in %.0f ms; hook waits unlocked %llu / %.0f ms GPU (pending write %llu / %.0f ms, recorded store %llu / %.0f ms) + %.0f ms relock (%llu found the recorder torn down), locked %llu; deferred releases: on the release thread %llu batches (%llu objects) in %.0f ms, inline %llu batches (%llu objects) in %.0f ms (%llu batches over the queue bound of %zu), queue max %llu batches, %llu pending\n",static_cast(waits), waitedMs / 1000, static_cast(syncCounts[0]), syncWaitedMs[0] / 1000, static_cast(syncCounts[1]), syncWaitedMs[1] / 1000, static_cast(syncCounts[2]), syncWaitedMs[2] / 1000, static_cast(syncCounts[3]), syncWaitedMs[3] / 1000, static_cast(syncCounts[4]), syncWaitedMs[4] / 1000, static_cast(hookCalls.load()), static_cast(hookLocks.load()), static_cast(targetedSyncs.load()), static_cast(batchesLeftInFlight.load()), static_cast(snapshotRebuilds), snapshotRebuildMs, static_cast(snapshotCovered), static_cast(unlockedWaits.load()), unlockedWaitedUs.load() / 1e6, static_cast(recorder.submissions), recorder.PendingLabels(), static_cast(completionLabels.load()), ThreadSyncReport().c_str(), SyncSiteReport().c_str(), static_cast(h.completions), h.completionMs, h.keptReleaseMs, static_cast(h.completionSyncsSkipped), static_cast(h.completionSyncsWaited), h.completionSyncWaitMs, static_cast(h.reaps), static_cast(h.reapsWithWork), static_cast(h.reapBatches), h.reapMs, static_cast(h.hookUnlockedWaits), h.hookUnlockedWaitMs, static_cast(h.hookUnlockedWaitsBySource[1]), h.hookUnlockedWaitMsBySource[1], static_cast(h.hookUnlockedWaitsBySource[2]), h.hookUnlockedWaitMsBySource[2], h.hookRelockMs, static_cast(h.hookUnlockedTornDown), static_cast(h.hookLockedWaits), static_cast(threadReleases.load()), static_cast(threadObjects.load()), threadReleaseUs.load() / 1000.0, static_cast(inlineReleases.load()), static_cast(inlineObjects.load()), inlineReleaseUs.load() / 1000.0, static_cast(inlineOverBound.load()), ReleaseQueueBound(), static_cast(releaseQueueMax.load()), static_cast(deferredPending.load())); + std::fprintf(stderr, "[recorder] completion label stores: %llu run, %llu skipped (no CPU write-back overlapped them); %llu counted pending at a write-back; %llu write-backs over a tracked label; %llu write-back completions pending\n", static_cast(completionStoresRun.load()), static_cast(completionStoresSkipped.load()), static_cast(completionLabelsCountedLate.load()), static_cast(writeBacksOverLabels.load()), static_cast(writeBackCompletions.load())); + std::fprintf(stderr, "[recorder] submits %llu, vkQueueSubmit mean %.1f us, max %.1f us\n", static_cast(submitCount), submitCount != 0 ? submitUs / static_cast(submitCount) : 0.0, submitMaxUs); + const auto reads = Recorder::ReadCounts(); + std::fprintf(stderr, "[recorder] in-place reads: %llu noted, %llu queries, hits by reader: dispatch element %llu, gpu copy %llu, address-based %llu, indirect %llu, storage upload %llu, copy source %llu; %llu hits on signaled batches ignored\n", static_cast(reads.noted), static_cast(reads.queries), static_cast(reads.hits[0]), static_cast(reads.hits[1]), static_cast(reads.hits[2]), static_cast(reads.hits[3]), static_cast(reads.hits[4]), static_cast(reads.hits[5]), static_cast(reads.staleIgnored)); } } - } report{profile, source, waitStart, *this}; - const auto waitFences = context.Function("vkWaitForFences"); + } report{profile, source, waitStart, *this, signaledAtStart}; + const auto waitFences = function(waitForFences, "vkWaitForFences"); auto result = waitFences(context.device, 1, &batch->fence, VK_TRUE, 5'000'000'000ull); for (int waited = 5; result == VK_TIMEOUT; waited += 5) { // A batch still running after 5 s is reported (every 5 s) so a GPU-side hang is visible. @@ -1516,6 +2810,20 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source) { } } readGpuTiming(*batch); + if (batch->serial != 0) { + std::lock_guard ringLock(completedMutex); + auto& entry = completed[batch->serial % completed.size()]; + entry.serial = batch->serial; + entry.gpuStartNs = batch->gpuStartNs; + entry.gpuEndNs = batch->gpuEndNs; + entry.submittedAt = batch->submittedAt; + entry.fenceSeenAt = std::chrono::steady_clock::now(); + entry.readGeneration = batch->readGeneration; + entry.reads.clear(); + if (FlipReadCheck()) { + for (const auto& read : batch->reads) entry.reads.emplace_back(read.begin, read.end); + } + } // Completions store GPU results to guest memory; a failing one is reported, the rest still run. // Timed (APS5_PROFILE_DRAW) with the release of the kept objects: both run under whichever hold // reaped or synced the batch. Inside them the flush hook makes no pending-write wait (SyncThrough). @@ -1567,6 +2875,7 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source) { const auto found = labels.find(dword); if (found != labels.end() && found->second.batch == batch.get()) labels.erase(found); } + recordedLabels.store(labels.size(), std::memory_order_relaxed); } batch->labelDwords.clear(); release(*batch); @@ -1577,7 +2886,7 @@ void Recorder::release(Batch& batch) noexcept { batch.queries = VK_NULL_HANDLE; // A completed (or never submitted) batch's objects are kept for reuse: the fence is signaled or // untouched, so resetting it cannot block, and the command buffer is no longer pending. - if (batch.commands != VK_NULL_HANDLE && batch.fence != VK_NULL_HANDLE && spare.size() < 64 && context.Function("vkResetFences")(context.device, 1, &batch.fence) == VK_SUCCESS) { + if (batch.commands != VK_NULL_HANDLE && batch.fence != VK_NULL_HANDLE && spare.size() < 64 && function(resetFences, "vkResetFences")(context.device, 1, &batch.fence) == VK_SUCCESS) { spare.emplace_back(batch.commands, batch.fence); batch.commands = VK_NULL_HANDLE; batch.fence = VK_NULL_HANDLE; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index a8ad8e72dc..f39491ce02 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -18,6 +18,7 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag mapping = allocation->mapping; deviceAddress = allocation->address; allocationBytes = allocation->allocationBytes; + ready = true; return; } try { @@ -48,7 +49,8 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory buffer"); Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory"); initializeAddress(usage); - Check(context.Function("vkMapMemory")(context.device, memory, 0, VK_WHOLE_SIZE, 0, &mapping), "vkMapMemory"); + if ((properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) Check(context.Function("vkMapMemory")(context.device, memory, 0, VK_WHOLE_SIZE, 0, &mapping), "vkMapMemory"); + ready = true; } catch (...) { release(); throw; @@ -60,7 +62,7 @@ Buffer::~Buffer() { } void Buffer::release() noexcept { - if (mapping && buffer && memory && cache) { + if (ready && cache) { cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, capacity, usage, properties}); return; } @@ -74,6 +76,7 @@ VkBuffer Buffer::Handle() const { } std::span Buffer::Bytes() { + Require(mapping != nullptr, "device-local buffer has no host mapping"); return {static_cast(mapping), size}; } @@ -136,12 +139,40 @@ std::size_t DeviceBuffer::Size() const { void CopyBuffer(const Context& context, VkCommandBuffer commands, VkBuffer source, VkDeviceSize sourceOffset, VkBuffer destination, VkDeviceSize destinationOffset, VkDeviceSize bytes) { const VkBufferCopy region{sourceOffset, destinationOffset, bytes}; - context.Function("vkCmdCopyBuffer")(commands, source, destination, 1, ®ion); + context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, destination, 1, ®ion); } void RecordMemoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { const VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, sourceAccess, destinationAccess}; - context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + +void FillDeviceFunctions(const Context& context, DeviceFunctions& functions) { + functions.cmdPipelineBarrier = context.Function("vkCmdPipelineBarrier"); + functions.cmdCopyBuffer = context.Function("vkCmdCopyBuffer"); + functions.cmdUpdateBuffer = context.Function("vkCmdUpdateBuffer"); + functions.cmdFillBuffer = context.Function("vkCmdFillBuffer"); + functions.cmdBindPipeline = context.Function("vkCmdBindPipeline"); + functions.cmdBindDescriptorSets = context.Function("vkCmdBindDescriptorSets"); + functions.cmdPushConstants = context.Function("vkCmdPushConstants"); + functions.cmdDispatch = context.Function("vkCmdDispatch"); + functions.cmdDispatchIndirect = context.Function("vkCmdDispatchIndirect"); + functions.cmdBeginRenderPass = context.Function("vkCmdBeginRenderPass"); + functions.cmdEndRenderPass = context.Function("vkCmdEndRenderPass"); + functions.cmdSetViewport = context.Function("vkCmdSetViewport"); + functions.cmdSetScissor = context.Function("vkCmdSetScissor"); + functions.cmdBindVertexBuffers = context.Function("vkCmdBindVertexBuffers"); + functions.cmdBindIndexBuffer = context.Function("vkCmdBindIndexBuffer"); + functions.cmdDraw = context.Function("vkCmdDraw"); + functions.cmdDrawIndexed = context.Function("vkCmdDrawIndexed"); + functions.cmdDrawIndirect = context.Function("vkCmdDrawIndirect"); + functions.cmdDrawIndexedIndirect = context.Function("vkCmdDrawIndexedIndirect"); + functions.cmdCopyBufferToImage = context.Function("vkCmdCopyBufferToImage"); + functions.cmdCopyImageToBuffer = context.Function("vkCmdCopyImageToBuffer"); + functions.cmdClearColorImage = context.Function("vkCmdClearColorImage"); + functions.updateDescriptorSets = context.Function("vkUpdateDescriptorSets"); + functions.allocateDescriptorSets = context.Function("vkAllocateDescriptorSets"); + functions.getFenceStatus = context.Function("vkGetFenceStatus"); } RenderTarget::RenderTarget(const Context& context, const ColorTarget& target, bool blending) : context(context) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp index 2615c772de..8077e8882c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp @@ -1,4 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include +#include namespace AgcDriver::Graphics { @@ -22,8 +24,11 @@ void ShaderResources::prepareAddressBindings(std::span sha } if (usesBda) { Require(context.bufferDeviceAddress, "buffer device address is not enabled"); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; guestMemory.AcquireRegistered(); + const auto snapshotsStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; for (const auto& snapshot : snapshots) guestMemory.AddSnapshot(snapshot); + if (profile) GuestBufferMemory::CountAddressBuild(std::chrono::duration(std::chrono::steady_clock::now() - snapshotsStart).count()); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp index b5663629f1..e69fe0fb83 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "SceShaders.hpp" #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include #include @@ -25,7 +26,8 @@ constexpr std::uint32_t spiPsInControl = 0x1B6; constexpr std::uint32_t dbShaderControl = 0x203; constexpr std::uint32_t spiShaderColFormat = 0x1C5; -std::uint32_t read(const Registers& registers, std::uint32_t offset) { +std::uint32_t read(const Registers& registers, std::uint32_t offset, RegisterBank bank) { + NoteRegisterRead(bank, offset); const auto it = registers.find(offset); if (it == registers.end()) { char text[64]; @@ -60,13 +62,13 @@ template void _readHeaderArray(std::span header, s } ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& shader) { - const auto numThreadX = read(shader, computeNumThreadX); - const auto numThreadY = read(shader, computeNumThreadY); - const auto numThreadZ = read(shader, computeNumThreadZ); + const auto numThreadX = read(shader, computeNumThreadX, RegisterBank::Shader); + const auto numThreadY = read(shader, computeNumThreadY, RegisterBank::Shader); + const auto numThreadZ = read(shader, computeNumThreadZ, RegisterBank::Shader); if (numThreadX == 0 || numThreadY == 0 || numThreadZ == 0) { throw std::runtime_error("AGC graphics: COMPUTE_NUM_THREAD_X/Y/Z must be nonzero"); } - const auto rsrc2 = read(shader, computePgmRsrc2); + const auto rsrc2 = read(shader, computePgmRsrc2, RegisterBank::Shader); if ((rsrc2 & 0x1u) != 0) { throw std::runtime_error("AGC graphics: COMPUTE_PGM_RSRC2.SCRATCH_EN is unsupported"); } @@ -83,13 +85,13 @@ ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& } ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& context, bool hasColorTarget, std::uint8_t colorComponentMapping) { - const auto inControl = read(context, spiPsInControl); + const auto inControl = read(context, spiPsInControl, RegisterBank::Context); const auto inputNum = inControl & 0x3Fu; if (inputNum > 32u) { throw std::runtime_error("AGC graphics: SPI_PS_IN_CONTROL input count exceeds 32"); } - const auto ena = read(context, spiPsInputEna); - const auto addr = read(context, spiPsInputAddr); + const auto ena = read(context, spiPsInputEna, RegisterBank::Context); + const auto addr = read(context, spiPsInputAddr, RegisterBank::Context); const auto activeInputs = ena & addr; constexpr std::uint32_t knownMask = 0x1u | 0x2u | 0x10u | 0x20u | 0x100u | 0x200u | 0x400u | 0x800u | 0x1000u | 0x2000u; if ((activeInputs & ~knownMask) != 0) { @@ -97,15 +99,15 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con } std::array interpolatorSettings{}; for (std::uint32_t i = 0; i < inputNum; ++i) { - interpolatorSettings[i] = read(context, spiPsInputCntl0 + i); + interpolatorSettings[i] = read(context, spiPsInputCntl0 + i, RegisterBank::Context); } - const auto shaderControl = read(context, dbShaderControl); + const auto shaderControl = read(context, dbShaderControl, RegisterBank::Context); // Bits 9 and 11 are EXEC_ON_HIER_FAIL and ALPHA_TO_MASK_DISABLE on GFX10; neither changes what the // recompiled pixel shader computes. if (((shaderControl >> 13u) & 0x3u) != 0) { throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.CONSERVATIVE_Z_EXPORT is unsupported"); } - const auto colFormat = read(context, spiShaderColFormat); + const auto colFormat = read(context, spiShaderColFormat, RegisterBank::Context); std::array targetOutputMode{}; for (std::uint32_t i = 0; i < 8u; ++i) { targetOutputMode[i] = static_cast((colFormat >> (4u * i)) & 0xFu); @@ -144,7 +146,7 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con }; } -ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData) { +ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData, std::vector* reads) { if (header.size() < sizeof(Shader)) throw std::runtime_error("AGC graphics: shader header is smaller than the fixed AGC header"); Shader shader; std::memcpy(&shader, header.data(), sizeof(Shader)); @@ -182,7 +184,9 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span attribWordBytes{}; - AgcDriver::GuestMemory::Read(attribTableAddr + static_cast(semantic.semantic) * 4u, attribWordBytes, 4); + const auto attribWordAddress = attribTableAddr + static_cast(semantic.semantic) * 4u; + AgcDriver::GuestMemory::Read(attribWordAddress, attribWordBytes, 4); + if (reads != nullptr) reads->push_back({attribWordAddress, {attribWordBytes.begin(), attribWordBytes.end()}}); std::uint32_t attribWord; std::memcpy(&attribWord, attribWordBytes.data(), 4); const auto index = attribWord & 0x1fu; @@ -191,7 +195,9 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span> 26u) & 0x1u; if (index >= ShaderRecompiler::ShaderVertexStageInfo::MaxResources) throw std::runtime_error("AGC graphics: vertex buffer index exceeds the supported domain"); std::array sharpBytes{}; - AgcDriver::GuestMemory::Read(bufferTableAddr + static_cast(index) * 16u, sharpBytes, 4); + const auto sharpAddress = bufferTableAddr + static_cast(index) * 16u; + AgcDriver::GuestMemory::Read(sharpAddress, sharpBytes, 4); + if (reads != nullptr) reads->push_back({sharpAddress, {sharpBytes.begin(), sharpBytes.end()}}); std::array sharp{}; std::memcpy(sharp.data(), sharpBytes.data(), 16); if (info.resourcesNum >= ShaderRecompiler::ShaderVertexStageInfo::MaxResources) throw std::runtime_error("AGC graphics: vertex resource count exceeds the supported domain"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 637d3ce4f7..ec222e29af 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -145,6 +145,9 @@ struct TextureCounters { std::atomic fastMisses{0}; std::atomic storageHits{0}; std::atomic storageCreated{0}; + // Storage images a Revalidate refreshed directly instead of through the lookups (T1, see + // ShaderResources::refreshOwnObjects). + std::atomic ownRefreshes{0}; std::atomic lastReport{0}; }; @@ -161,7 +164,7 @@ void reportTextureCounters() { auto last = counters.lastReport.load(); if (nowMs - last < 10000 || !counters.lastReport.compare_exchange_strong(last, nowMs)) return; const auto count = [](const std::atomic& value) { return static_cast(value.load(std::memory_order_relaxed)); }; - std::fprintf(stderr, "[textures] sampled created: %llu from storage images, %llu snapshots (%llu snapshot reads over recorded writes inside cachedTexture, %llu storage-path fallbacks); stage-A records %llu: %llu fast hits, %llu full lookups; storage images %llu hits, %llu created\n", count(counters.fromStorage), count(counters.snapshots), count(counters.pendingReads), count(counters.storageFallbacks), count(counters.records), count(counters.fastHits), count(counters.fastMisses), count(counters.storageHits), count(counters.storageCreated)); + std::fprintf(stderr, "[textures] sampled created: %llu from storage images, %llu snapshots (%llu snapshot reads over recorded writes inside cachedTexture, %llu storage-path fallbacks); stage-A records %llu: %llu fast hits, %llu full lookups; storage images %llu hits, %llu created, %llu own-object refreshes\n", count(counters.fromStorage), count(counters.snapshots), count(counters.pendingReads), count(counters.storageFallbacks), count(counters.records), count(counters.fastHits), count(counters.fastMisses), count(counters.storageHits), count(counters.storageCreated), count(counters.ownRefreshes)); } // What the sampled-texture lookups on this thread proved their returned objects current against, @@ -206,6 +209,13 @@ bool SampledFromStorageEligible(const Context& context, const GuestTextureResour return SampledFromStorageEligible(context, resource.format, resource.baseAddress, guestBytes); } +// Whether a sampled texture over a fast-cleared, storage-eligible surface views the cleared storage +// image (see cachedTexture). APS5_NO_CLEARED_VIEW=1 keeps such surfaces snapshots, as before. +bool ClearedViewEnabled() { + static const bool disabled = std::getenv("APS5_NO_CLEARED_VIEW") != nullptr; + return !disabled; +} + // Surfaces whose storage image could not be made (no writable committed pages, say): remembered // so the sampled lookup does not throw and fall back on every use. A failure is keyed by the surface // (address and size: the heap reuses addresses) and holds only while the guest mappings are what @@ -251,6 +261,14 @@ std::shared_ptr sampledStorageSource(const Context& context, con // `guestBytes` is the surface size when the caller described the surface already (0: described here). std::shared_ptr cachedTexture(const Context& context, std::span words, const GuestTextureResource& resource, VkComponentMapping components, std::uint64_t guestBytes = 0) { static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + const bool profile = LookupOutcomes::Profiled(); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto scanKeys = [&] { + const auto scanStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto keys = TextureClearKeys(resource, guestBytes); + if (profile && resource.dccAddress != 0) LookupOutcomes::Add(LookupOutcomes::DccScan, scanStart); + return keys; + }; if (guestBytes == 0) guestBytes = DescribeSurface(resource).guestBytes; auto& counters = TextureCounts(); const auto address = resource.baseAddress; @@ -263,12 +281,27 @@ std::shared_ptr cachedTexture(const Context& context, std::span keys; if (source == nullptr && SampledFromStorageEligible(context, resource, guestBytes)) { - keys = TextureClearKeys(resource, guestBytes); - if (*keys == DccKeys::Uncompressed) source = sampledStorageSource(context, resource, guestBytes); + keys = scanKeys(); + if (*keys == DccKeys::Uncompressed) { + source = sampledStorageSource(context, resource, guestBytes); + } else if (ClearedViewEnabled() && StorageClearAvailable(context, resource.format, *keys)) { + // A fast-cleared surface is viewed as well, through its image cleared on the GPU (the + // lookup's Refresh, see StorageTexture::upload), when the image's own descriptor names + // the same DCC metadata so the refresh sees the keys. A snapshot of the clear texels + // (filled and uploaded on the CPU, then replaced by a view once results are pending: two + // 4K uploads per clear at the movie stage) serves the surface otherwise, as before. + auto candidate = sampledStorageSource(context, resource, guestBytes); + if (candidate != nullptr && candidate->Descriptor().dccAddress == resource.dccAddress) source = std::move(candidate); + } + // The lookup's Refresh may have flushed another image's results over the memory (which + // marks the keys uncompressed): a surface that stays a snapshot re-reads them behind that + // flush, as it does behind its own. APS5_NO_KEYS_RESCAN=1 keeps the keys scanned above. + static const bool rescan = std::getenv("APS5_NO_KEYS_RESCAN") == nullptr; + if (rescan && source == nullptr) keys.reset(); } // Pages of the surface written since they were last collected are stamped now, before any // UnchangedSince (here and on a cache hit) looks at them: the checks only compare stamped blocks, @@ -282,11 +315,13 @@ std::shared_ptr cachedTexture(const Context& context, std::span(context, source, resource, components); std::vector snapshot(bytes); @@ -303,9 +338,11 @@ std::shared_ptr cachedTexture(const Context& context, std::spansource) && (source != nullptr || *keys == DccKeys::Uncompressed)) { if (!GuestMemory::UnchangedSince(address, bytes, it->source->Generation())) it->source->Refresh(); + it->keys = *keys; touchTexture(cache, it); logLookup({it->texture.get(), resource, guestBytes, *keys, 0, it->source.get()}); reportTextureCounters(); + if (profile) LookupOutcomes::Add(*keys != DccKeys::Uncompressed ? LookupOutcomes::SampledHitClearedView : LookupOutcomes::SampledHitView, start); return it->texture; } } else if (source == nullptr && it->bytes.size() == guestBytes && it->keys == *keys) { @@ -322,6 +359,7 @@ std::shared_ptr cachedTexture(const Context& context, std::spantexture.get(), resource, guestBytes, *keys, generation, nullptr}); reportTextureCounters(); + if (profile) LookupOutcomes::Add(LookupOutcomes::SampledHitSnapshot, start); return it->texture; } } @@ -355,6 +393,7 @@ std::shared_ptr cachedTexture(const Context& context, std::span::iterator findStorageByImage(StorageTextureCache // Evicts an entry: its pending results go to guest memory first (the image may die with the entry). void evictStorage(StorageTextureCache& cache, std::list::iterator it) { + it->texture->SetCached(false); it->texture->Flush(); cache.bytes -= it->texture->GuestBytes(); cache.index.erase(it->key); @@ -435,6 +475,8 @@ std::shared_ptr cachedStorageTexture(const Context& context, std static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; if (disabled) return std::make_shared(context, *context.detiler, resource, mip); static_cast(words); + const bool profile = LookupOutcomes::Profiled(); + auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; auto& counters = TextureCounts(); const StorageKey key{context.device, SurfaceKey(context, resource)}; auto& cache = StorageTextures(); @@ -443,14 +485,21 @@ std::shared_ptr cachedStorageTexture(const Context& context, std it->texture->Refresh(); cache.entries.splice(cache.entries.begin(), cache.entries, it); counters.storageHits.fetch_add(1, std::memory_order_relaxed); + if (profile) LookupOutcomes::Add(LookupOutcomes::StorageHit, start); return it->texture; } // Results other images hold over this memory reach it before the new image reads it: a hit's // Refresh flushes them, the constructor's upload does not, and a GPU-direct upload reads the // import buffer without the flush hook. The flush is recorded ahead of the upload in the batch. if (guestBytes == 0) guestBytes = DescribeSurface(resource).guestBytes; - StorageTexture::FlushPending(resource.baseAddress, static_cast(guestBytes), nullptr, "storage image creation"); + if (StorageTexture::FlushPending(resource.baseAddress, static_cast(guestBytes), nullptr, "storage image creation", PublishScope::None) && profile) start = LookupOutcomes::Add(LookupOutcomes::PendingFlush, start); CachedStorageTexture entry{key, mip, std::make_shared(context, *context.detiler, resource, mip)}; + // The constructor's upload may have recorded into the open batch (a GPU clear, a direct + // detile) before the image could keep itself (no weak_from_this yet): the batch keeps it here, + // so an eviction or a failed view before it ran cannot destroy a referenced image. + // APS5_NO_KEEP_NEW_STORAGE=1 leaves the image to its cache entry alone, as before. + static const bool keepNew = std::getenv("APS5_NO_KEEP_NEW_STORAGE") == nullptr; + if (auto* recorder = Recorder::Active(); keepNew && recorder != nullptr && GuestMemory::GpuMutex().HeldByThisThread() && recorder->Recording()) recorder->Keep(entry.texture); constexpr std::uint64_t budget = 2048ull << 20u; while (!cache.entries.empty() && cache.bytes + entry.texture->GuestBytes() > budget) evictStorage(cache, std::prev(cache.entries.end())); cache.bytes += entry.texture->GuestBytes(); @@ -458,14 +507,18 @@ std::shared_ptr cachedStorageTexture(const Context& context, std cache.entries.push_front(std::move(entry)); cache.index[key] = cache.entries.begin(); cache.byImage[texture.get()] = cache.entries.begin(); + texture->SetCached(true); counters.storageCreated.fetch_add(1, std::memory_order_relaxed); + if (profile) LookupOutcomes::Add(LookupOutcomes::StorageMade, start); return texture; } +} + // Whether `image` is still the storage cache's image of its surface, i.e. what a lookup would return // (an evicted image next to a newer one of the same memory must not be reused: two images of one -// surface would hold results). A use through the fast Revalidate counts as a use for the cache's -// eviction order, as a lookup would. +// surface would hold results). A use through the fast Revalidate or a draw recipe's target proof +// counts as a use for the cache's eviction order, as a lookup would. bool StorageImageCached(const Context& context, const StorageTexture* image) { auto& cache = StorageTextures(); std::lock_guard lock(cache.mutex); @@ -475,6 +528,21 @@ bool StorageImageCached(const Context& context, const StorageTexture* image) { return true; } +namespace { + +// StorageImageCached for several images under one acquisition of the cache mutex. +bool StorageImagesCached(const Context& context, std::span images) { + if (images.empty()) return true; + auto& cache = StorageTextures(); + std::lock_guard lock(cache.mutex); + for (const auto* image : images) { + const auto it = findStorageByImage(cache, context.device, image); + if (it == cache.entries.end()) return false; + cache.entries.splice(cache.entries.begin(), cache.entries, it); + } + return true; +} + } std::shared_ptr CachedStorageSurface(const Context& context, const GuestTextureResource& resource) { @@ -548,6 +616,9 @@ bool UsesAddressTables(const CompiledShader& compute) { ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred) : context(context), guestMemory(context), deferredCompute(compute), deferredSnapshots(snapshots) { Require(compute.stage == ShaderRecompiler::ShaderStage::Compute, "compute resources require a compute shader"); + // Every use of a compute build is a recorded dispatch that calls MarkGpuWrites, which staged + // buffers need (a synchronous draw's use would not). + guestMemory.AllowDeviceStaging(); const std::span shaders(&deferredCompute, 1); if (!deferred) { prepareAddressBindings(shaders, snapshots); @@ -592,11 +663,16 @@ void ShaderResources::build(std::span shaders, const Color namespace { -// APS5_PROFILE_DRAW: the [resources] phase totals, under a mutex because stage A of several builds -// runs at once. +// APS5_PROFILE_DRAW: the [resources] phase totals. Each thread accumulates its builds' phases in +// arrays of its own and merges them into the shared totals every 1000 of its builds (and when it +// ends), so no build takes the shared mutex per phase; the totals lag by up to 999 builds per +// worker. +constexpr std::size_t BuildPhaseCount = static_cast(ShaderResources::BuildPhase::Count); +constexpr std::array BuildPhaseNames{"bindings", "precollect", "guest memory upload", "descriptors", "stage A", "images", "bda", "stage B"}; + struct BuildProfile { std::mutex mutex; - std::map phaseTotals; + std::array ms{}; std::uint64_t builds = 0; }; @@ -610,10 +686,31 @@ bool BuildProfiled() { return profile; } -void addBuildPhase(const char* name, double ms) { - auto& profile = Builds(); - std::lock_guard lock(profile.mutex); - profile.phaseTotals[name] += ms; +struct ThreadBuildProfile { + std::array ms{}; + std::uint64_t builds = 0; + ~ThreadBuildProfile() { merge(); } + void merge() { + auto& profile = Builds(); + std::lock_guard lock(profile.mutex); + for (std::size_t i = 0; i < BuildPhaseCount; ++i) profile.ms[i] += ms[i]; + profile.builds += builds; + ms = {}; + if (builds == 0) return; + builds = 0; + std::string report; + for (std::size_t i = 0; i < BuildPhaseCount; ++i) report += " " + std::string(BuildPhaseNames[i]) + "=" + std::to_string(static_cast(profile.ms[i])) + "ms"; + std::fprintf(stderr, "[resources] %llu builds, phase totals:%s\n", static_cast(profile.builds), report.c_str()); + } +}; + +ThreadBuildProfile& ThreadBuilds() { + thread_local ThreadBuildProfile profile; + return profile; +} + +void addBuildPhase(ShaderResources::BuildPhase which, double ms) { + ThreadBuilds().ms[static_cast(which)] += ms; } // APS5_PROFILE_DRAW: guest buffer elements bound by descriptor (addGuestBuffer) and how many the @@ -633,12 +730,12 @@ BufferWriteCounters& BufferWrites() { } -double ShaderResources::phase(const char* name) { +double ShaderResources::phase(BuildPhase which) { if (!BuildProfiled()) return 0.0; const auto now = std::chrono::steady_clock::now(); const auto ms = std::chrono::duration(now - phaseStart).count(); - addBuildPhase(name, ms); - if (ms > 50) std::fprintf(stderr, "[resources] %s took %.0f ms (%zu textures, %zu storage images, %zu buffers, bda %d)\n", name, ms, textures.size(), storageTextures.size(), allocations.size(), usesBda ? 1 : 0); + addBuildPhase(which, ms); + if (ms > 50) std::fprintf(stderr, "[resources] %s took %.0f ms (%zu textures, %zu storage images, %zu buffers, bda %d)\n", BuildPhaseNames[static_cast(which)], ms, textures.size(), storageTextures.size(), allocations.size(), usesBda ? 1 : 0); phaseStart = now; return ms; } @@ -647,13 +744,8 @@ void ShaderResources::buildPrepare(std::span shaders, cons const auto stageStart = std::chrono::steady_clock::now(); phaseStart = stageStart; if (BuildProfiled()) { - auto& profile = Builds(); - std::lock_guard lock(profile.mutex); - if (++profile.builds % 1000 == 0) { - std::string report; - for (const auto& [name, ms] : profile.phaseTotals) report += " " + name + "=" + std::to_string(static_cast(ms)) + "ms"; - std::fprintf(stderr, "[resources] %llu builds, phase totals:%s\n", static_cast(profile.builds), report.c_str()); - } + auto& profile = ThreadBuilds(); + if (++profile.builds % 1000 == 0) profile.merge(); } try { Require(!shaders.empty() && context.limits.maxBoundDescriptorSets >= 1, "shader descriptor set exceeds device limits"); @@ -686,7 +778,8 @@ void ShaderResources::buildPrepare(std::span shaders, cons // An element the recompiler did not classify (a producer without the // vector) counts as written, like imageWritten below. const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; - item.allocations.push_back(addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written)); + const bool atomic = element < binding.bufferAtomic.size() && binding.bufferAtomic[element]; + item.allocations.push_back(addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written, atomic)); } } else if (addressRole) { item.allocations.push_back(allocations.size()); @@ -700,12 +793,12 @@ void ShaderResources::buildPrepare(std::span shaders, cons } } Require(storageBuffers <= context.limits.maxDescriptorSetStorageBuffers, "pipeline descriptors exceed device limits"); - timing.bindingsMs = phase("bindings"); + timing.bindingsMs = phase(BuildPhase::Bindings); // For every build, locked ones included: their stage B then takes the fast path too, and the // collects cost the same wherever they run. - if (precollectImages()) phase("precollect"); + if (precollectImages()) phase(BuildPhase::Precollect); guestMemory.UploadPrepare(usesBda); - timing.uploadMs = phase("guest memory upload"); + timing.uploadMs = phase(BuildPhase::Upload); std::vector description; for (const auto& binding : bindings) { description.push_back(binding.layout); @@ -748,10 +841,10 @@ void ShaderResources::buildPrepare(std::span shaders, cons Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &_set), "vkAllocateDescriptorSets"); } } - timing.descriptorsMs = phase("descriptors"); + timing.descriptorsMs = phase(BuildPhase::Descriptors); if (BuildProfiled()) { timing.prepareMs = std::chrono::duration(std::chrono::steady_clock::now() - stageStart).count(); - addBuildPhase("stage A", timing.prepareMs); + addBuildPhase(BuildPhase::StageA, timing.prepareMs); } } catch (...) { release(); @@ -774,12 +867,12 @@ void ShaderResources::buildComplete() { imageRecords.shrink_to_fit(); nextImageRecord = 0; Require(textures.size() == plannedSampledImages && storageTextures.size() == plannedStorageImages, "image lookups disagree with the descriptor plan"); - timing.bindingsMs += phase("images"); + timing.bindingsMs += phase(BuildPhase::Images); guestMemory.UploadFinish(usesBda); - timing.uploadMs += phase("guest memory upload"); + timing.uploadMs += phase(BuildPhase::Upload); if (usesBda) bda = std::make_unique(context, guestMemory); else if (usesFaultBuffer) bda = std::make_unique(context); - phase("bda"); + phase(BuildPhase::Bda); if (_set != VK_NULL_HANDLE) { // One update call for the whole set: the info arrays are sized up front so every write's // pointer into them stays valid until the call. @@ -822,14 +915,14 @@ void ShaderResources::buildComplete() { } writes.push_back(write); } - context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); } - timing.descriptorsMs += phase("descriptors"); + timing.descriptorsMs += phase(BuildPhase::Descriptors); noteReusable(); completed = true; if (BuildProfiled()) { timing.completeMs = std::chrono::duration(std::chrono::steady_clock::now() - stageStart).count(); - addBuildPhase("stage B", timing.completeMs); + addBuildPhase(BuildPhase::StageB, timing.completeMs); reportDescriptorCaches(); } } catch (...) { @@ -840,13 +933,45 @@ void ShaderResources::buildComplete() { // Whether this build can serve later dispatches of the same content (see ContentKey): nothing to do // once the GPU completed (no BDA, no copied written buffers, no lease) and every guest buffer bound -// in place through a host import whose identity is recorded for Revalidate. +// in place through a host import (or staged in device memory from one, see +// GuestBufferMemory::DirectRegions) whose identity is recorded for Revalidate. +namespace { + +// APS5_NO_TEMPLATE_DATA_REFRESH=1: compute keys keep the ShaderData/FlattenedSrt words and no +// template refreshes its data buffers (see ShaderResources::ContentKey). +bool TemplateDataRefresh() { + static const bool enabled = std::getenv("APS5_NO_TEMPLATE_DATA_REFRESH") == nullptr; + return enabled; +} + +bool DataRole(ShaderRecompiler::DescriptorRole role) { + return role == ShaderRecompiler::DescriptorRole::ShaderData || role == ShaderRecompiler::DescriptorRole::FlattenedSrt; +} + +// The largest data buffer vkCmdUpdateBuffer refreshes; a template with a bigger one is not reused. +constexpr std::size_t MaxRefreshBytes = 65536; + +// FNV-1a over one data buffer's words, its count first (DataWordsHash). +constexpr std::uint64_t FnvOffset = 14695981039346656037ull; +constexpr std::uint64_t FnvPrime = 1099511628211ull; +void mixDataWords(std::uint64_t& hash, std::span words) { + hash = (hash ^ static_cast(words.size())) * FnvPrime; + for (const auto word : words) hash = (hash ^ word) * FnvPrime; +} + +// ResourceCache::Find and Touch calls (the [rescache] line). +std::atomic resourceCacheFinds{0}; +std::atomic resourceCacheTouches{0}; + +} + void ShaderResources::noteReusable() { // Taken whether or not the object turns out reusable: the lookups' records are this build's. captureValidation(); reusable = false; directRegions.clear(); if (NeedsCompletion() || HoldsLease()) return; + if (TemplateDataRefresh() && std::any_of(allocations.begin(), allocations.end(), [](const Allocation& allocation) { return allocation.buffer != nullptr && !allocation.guest && allocation.size > MaxRefreshBytes; })) return; const auto regions = guestMemory.DirectRegions(); if (!regions.has_value()) return; for (const auto& [begin, end] : *regions) { @@ -876,11 +1001,12 @@ void ShaderResources::reportDescriptorCaches() const { std::fprintf(stderr, "[descriptors] layouts %llu hits / %llu created, sets %llu from %llu pools, samplers %llu hits / %llu created\n", static_cast(descriptors.layoutHits), static_cast(descriptors.layoutMisses), static_cast(descriptors.sets), static_cast(descriptors.pools), static_cast(samplerHits), static_cast(samplerMisses)); } -std::vector ShaderResources::ContentKey(const CompiledShader& shader) { +std::vector ShaderResources::ContentKey(const CompiledShader& shader, bool dataWords) { Require(shader.program != nullptr, "missing compiled shader"); const auto& program = *shader.program; std::vector key; key.reserve(8 + program.bindings.size() * 12); + key.push_back(dataWords ? 1u : 0u); key.push_back(static_cast(shader.stage)); key.push_back(static_cast(program.variantId)); key.push_back(static_cast(program.variantId >> 32u)); @@ -898,7 +1024,7 @@ std::vector ShaderResources::ContentKey(const CompiledShader& sha }; for (const auto& binding : program.bindings) { key.insert(key.end(), {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(binding.guestDescriptor.size())}); - key.insert(key.end(), binding.guestDescriptor.begin(), binding.guestDescriptor.end()); + if (dataWords || !DataRole(binding.role)) key.insert(key.end(), binding.guestDescriptor.begin(), binding.guestDescriptor.end()); packBits(binding.imageWritten); packBits(binding.samplerDepthCompare); // Read-only elements are bound without a write set: an object built for one written set @@ -913,12 +1039,48 @@ namespace { // APS5_PROFILE_DRAW: Revalidate outcomes and time, printed as [rescache] every 10 s next to the // device's hit/miss line: failed = the object could not be reused (whichever image path it took); // of the reused ones, fast = every image proved current from stamps, full = the lookups were repeated. +// Why the fast path left an object to the full walk (the "fast-fail by reason" counts). +using FastFail = ShaderResources::FastFail; +constexpr std::array(FastFail::Count)> FastFailNames{"no record", "collect", "pending image", "evicted image", "memory changed", "keys", "cleared view", "storage keys"}; +using OwnRefreshFallback = ShaderResources::OwnRefreshFallback; +constexpr std::array(OwnRefreshFallback::Count)> OwnRefreshFallbackNames{"disabled", "snapshot texture", "cleared view", "foreign view", "surface key", "not imported", "uncached", "re-run failed"}; + struct RevalidateProfile { std::atomic calls{0}; std::atomic nanoseconds{0}; std::atomic fast{0}; std::atomic full{0}; std::atomic failed{0}; + std::array, static_cast(FastFail::Count)> fastFails{}; + // Epoch gate: registry scans skipped because the pending serial was unchanged, and import + // serial loops skipped because the table's identity was unchanged. + std::atomic serialSkips{0}; + std::atomic importSkips{0}; + // Elements with DCC keys the fast path accepted through the images' key proofs. + std::atomic keysProven{0}; + std::atomic storageKeysProven{0}; + // The 'pending image' fast-fails by what overlapped: a surface with an own object (the query + // excepts it, so the overlapping image is foreign to the surface) or a snapshot texture (no own + // object); and the fails whose overlapping surfaces all have an own object (the T1 trim of + // design_cpu_final applies: flush the foreign image, refresh the own object, no full walk). + std::atomic pendingForeign{0}; + std::atomic pendingSnapshot{0}; + std::atomic pendingT1Eligible{0}; + // The full walks by the reason of the fast proof that preceded them (fastFails also counts the + // Pending failures the own-object refresh resolved), the Pending failures it resolved (with the + // objects it refreshed: storage images, view sources) and those it left to the walk by reason; + // view surfaces a foreign image overlapped that the proof accepted because FindPending names + // their own source; surfaces a foreign image was still pending over after their own object's + // refresh (its alias, a straddling image) that the re-run accepted; proofs the verify switch + // checked against the full walk. + std::array, static_cast(FastFail::Count)> fullByReason{}; + std::atomic ownRefreshed{0}; + std::atomic ownStorageRefreshes{0}; + std::atomic ownViewRefreshes{0}; + std::array, static_cast(OwnRefreshFallback::Count)> ownFallbacks{}; + std::atomic ownSourceViews{0}; + std::atomic refreshedOverlaps{0}; + std::atomic proofsVerified{0}; std::atomic lastReport{0}; }; @@ -927,6 +1089,65 @@ RevalidateProfile& Revalidations() { return profile; } +void countFastFail(FastFail reason) { + if (BuildProfiled()) Revalidations().fastFails[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); +} + +void countPendingFail(std::span pending) { + if (!BuildProfiled()) return; + auto& profile = Revalidations(); + bool eligible = true; + for (const auto& query : pending) { + if (!query.overlaps) continue; + if (query.except != nullptr) { + profile.pendingForeign.fetch_add(1, std::memory_order_relaxed); + } else { + profile.pendingSnapshot.fetch_add(1, std::memory_order_relaxed); + eligible = false; + } + } + if (eligible) profile.pendingT1Eligible.fetch_add(1, std::memory_order_relaxed); +} + +void countKeysProven(bool storage) { + if (BuildProfiled()) (storage ? Revalidations().storageKeysProven : Revalidations().keysProven).fetch_add(1, std::memory_order_relaxed); +} + +void countFullWalk(FastFail reason) { + if (BuildProfiled() && reason != FastFail::Count) Revalidations().fullByReason[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); +} + +void countOwnRefreshFallback(OwnRefreshFallback reason) { + if (BuildProfiled()) Revalidations().ownFallbacks[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); +} + +void countOwnRefresh(bool storage) { + TextureCounts().ownRefreshes.fetch_add(1, std::memory_order_relaxed); + if (BuildProfiled()) (storage ? Revalidations().ownStorageRefreshes : Revalidations().ownViewRefreshes).fetch_add(1, std::memory_order_relaxed); +} + +// APS5_NO_OWN_IMAGE_REFRESH=1: a Pending failure of the fast proof takes the full walk, as before +// the own-object refresh (T1). +bool OwnImageRefreshEnabled() { + static const bool disabled = std::getenv("APS5_NO_OWN_IMAGE_REFRESH") != nullptr; + return !disabled; +} + +// APS5_VERIFY_PROOFS=1: after every own-object refresh the full walk runs beside the proof and the +// process aborts when it would return another object or upload again (a different decision). +bool VerifyProofs() { + static const bool enabled = std::getenv("APS5_VERIFY_PROOFS") != nullptr; + return enabled; +} + +// Whether a sampled surface's keys are the very keys of the image the view follows (the same +// metadata, extent, format and alpha placement: the same scan), so the image's proof serves it. +bool SameKeySurface(const StorageTexture* source, const GuestTextureResource& resource, std::uint64_t guestBytes) { + if (source == nullptr) return false; + const auto& own = source->Descriptor(); + return own.dccAddress == resource.dccAddress && source->GuestBytes() == guestBytes && own.format == resource.format && own.dccAlphaOnMsb == resource.dccAlphaOnMsb; +} + void countRevalidate(bool fast, bool ok, std::chrono::steady_clock::time_point start) { auto& profile = Revalidations(); const auto now = std::chrono::steady_clock::now(); @@ -936,7 +1157,26 @@ void countRevalidate(bool fast, bool ok, std::chrono::steady_clock::time_point s const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); auto last = profile.lastReport.load(); if (nowMs - last < 10000 || !profile.lastReport.compare_exchange_strong(last, nowMs)) return; - std::fprintf(stderr, "[rescache] revalidate %llu calls %.1f ms: fast %llu, full %llu, failed %llu\n", static_cast(profile.calls.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.fast.load()), static_cast(profile.full.load()), static_cast(profile.failed.load())); + const auto byReason = [](const auto& counts, const auto& names) { + std::string reasons; + for (std::size_t i = 0; i < names.size(); ++i) { + const auto count = counts[i].load(std::memory_order_relaxed); + if (count == 0) continue; + reasons += " " + std::string(names[i]) + " " + std::to_string(count); + } + return reasons; + }; + const auto reasons = byReason(profile.fastFails, FastFailNames); + const auto fullReasons = byReason(profile.fullByReason, FastFailNames); + const auto fallbacks = byReason(profile.ownFallbacks, OwnRefreshFallbackNames); + std::fprintf(stderr, "[rescache] revalidate %llu calls %.1f ms: fast %llu, full %llu, failed %llu; fast-fail by reason:%s; full walks: T1 refreshed %llu (%llu storage images, %llu view sources, %llu views served by their own pending source, %llu overlaps left by the refresh accepted), full by reason:%s, T1 fallback by reason:%s, proofs verified %llu; fast-fail pending image: own 0 (the query excepts the own object), foreign %llu, snapshot %llu, T1 eligible %llu; keys proven: %llu sampled, %llu storage; epoch gate: %llu registry scans skipped, %llu import loops skipped\n", static_cast(profile.calls.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.fast.load()), static_cast(profile.full.load()), static_cast(profile.failed.load()), reasons.c_str(), static_cast(profile.ownRefreshed.load()), static_cast(profile.ownStorageRefreshes.load()), static_cast(profile.ownViewRefreshes.load()), static_cast(profile.ownSourceViews.load()), static_cast(profile.refreshedOverlaps.load()), fullReasons.c_str(), fallbacks.c_str(), static_cast(profile.proofsVerified.load()), static_cast(profile.pendingForeign.load()), static_cast(profile.pendingSnapshot.load()), static_cast(profile.pendingT1Eligible.load()), static_cast(profile.keysProven.load()), static_cast(profile.storageKeysProven.load()), static_cast(profile.serialSkips.load()), static_cast(profile.importSkips.load())); +} + +// APS5_NO_EPOCH_REVALIDATE=1: the per-element registry, cache and stamp checks and the per-region +// flush and import lookups of every Revalidate, as before the epoch gate. +bool EpochRevalidate() { + static const bool enabled = std::getenv("APS5_NO_EPOCH_REVALIDATE") == nullptr; + return enabled; } } @@ -967,7 +1207,154 @@ void ShaderResources::captureValidation() { // and the DCC keys must be what the content was made under whenever the surface has any (a fast // clear touches only the keys). Anything else, including a failed collect (memory outside the arena, // uncommitted pages), is left to the full walk. -bool ShaderResources::fastRevalidate() { +// The epoch gate: while the pending registry's serial is what this object's last proof saw +// (pendingSerialSeen), the registry is what it was then (every change bumps the serial under the +// registry mutex), so the pending-image checks are skipped; otherwise one scan answers them for +// every surface at once. The stamp checks go through one tracker lock, the cache checks through +// the images' cached flags and one touch of the cache. +bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::span refreshed, FastFail& reason, std::vector& overlapping, bool& accepted) { + reason = FastFail::Count; + overlapping.clear(); + accepted = false; + if (!EpochRevalidate()) return fastRevalidateEach(); + const auto fail = [&reason](FastFail why) { + reason = why; + countFastFail(why); + return false; + }; + if (validatedTextures.size() != textures.size()) return fail(FastFail::NoRecord); + const bool unchanged = pendingSerialSeen != 0 && pendingSerialSeen == serialBefore; + const bool keyProofs = KeyFastPath(); + thread_local std::vector queries; + thread_local std::vector pending; + // The element each pending query stands for, in query order. + thread_local std::vector owners; + thread_local std::vector images; + thread_local std::vector scannedKeys; + queries.clear(); + pending.clear(); + owners.clear(); + images.clear(); + scannedKeys.assign(textures.size(), DccKeys::Uncompressed); + const auto query = [&](std::uint64_t begin, std::uint64_t end, const StorageTexture* except, const StorageTexture* identity, PendingOverlap owner) { + pending.push_back({begin, end, except, identity, false}); + owners.push_back(owner); + }; + for (std::size_t i = 0; i < textures.size(); ++i) { + auto& surface = validatedTextures[i]; + if (!surface.valid) return fail(FastFail::NoRecord); + const auto address = surface.resource.baseAddress; + const auto bytes = static_cast(surface.bytes); + surface.collected = GuestMemory::CollectWrites(address, bytes); + if (surface.collected == 0) return fail(FastFail::Collect); + const auto* source = surface.source; + if (!keyProofs) { + // Without proofs the keys are compared with the record's (a cleared view's identity + // below is the record's too): the scan repeats on every call, as before. + if (surface.resource.dccAddress != 0 && TextureClearKeys(surface.resource, surface.bytes) != surface.keys) return fail(FastFail::Keys); + scannedKeys[i] = surface.keys; + if (!unchanged) query(address, address + bytes, source, source != nullptr && surface.keys != DccKeys::Uncompressed ? source : nullptr, {i, false, source != nullptr && surface.keys == DccKeys::Uncompressed}); + } else if (source != nullptr) { + // The view follows its image, so it stays valid while the image would Refresh as + // unchanged: the image's own keys must be what its content was uploaded under (its + // proof answers without a scan while they are unstamped), and its memory unchanged + // since its generation (the query below). The surface's own keys come from the same + // proof when they are the image's, else from the texture's. + const auto& own = source->Descriptor(); + const auto sourceKeys = own.dccAddress != 0 ? ProvedClearKeys(own, source->GuestBytes(), source->KeyProof()) : DccKeys::Uncompressed; + if (sourceKeys != source->UploadedKeys()) return fail(FastFail::Keys); + const auto keys = surface.resource.dccAddress == 0 ? DccKeys::Uncompressed : SameKeySurface(source, surface.resource, surface.bytes) ? sourceKeys : ProvedClearKeys(surface.resource, surface.bytes, textures[i]->KeyProof()); + scannedKeys[i] = keys; + // A view of a fast-cleared surface stays one only while its image still has results + // pending over it (cachedTexture's hit rule), unless the image's own descriptor names + // the surface's metadata (the lookup views the cleared image then). The identity is + // asked whatever the registry's serial did: the keys may have moved since the last + // proof while the registry did not. + if (keys != DccKeys::Uncompressed && !(own.dccAddress == surface.resource.dccAddress && ClearedViewEnabled() && StorageClearAvailable(context, surface.resource.format, keys))) query(address, address + bytes, source, source, {i, false, false}); + else if (!unchanged) query(address, address + bytes, source, nullptr, {i, false, keys == DccKeys::Uncompressed}); + if (surface.resource.dccAddress != 0 || own.dccAddress != 0) countKeysProven(false); + } else { + // A snapshot holds its clear texels or the guest bytes: the keys must be what it was made under. + const auto keys = surface.resource.dccAddress != 0 ? ProvedClearKeys(surface.resource, surface.bytes, textures[i]->KeyProof()) : DccKeys::Uncompressed; + if (keys != surface.keys) return fail(FastFail::Keys); + scannedKeys[i] = keys; + if (!unchanged) query(address, address + bytes, nullptr, nullptr, {i, false, false}); + if (surface.resource.dccAddress != 0) countKeysProven(false); + } + if (source != nullptr) { + if (!source->Cached()) return fail(FastFail::Evicted); + queries.push_back({address, bytes, source->Generation()}); + images.push_back(source); + } else { + queries.push_back({address, bytes, surface.generation}); + } + } + for (std::size_t i = 0; i < storageTextures.size(); ++i) { + if (i != 0 && storageTextures[i] == storageTextures[i - 1]) continue; + const auto* image = storageTextures[i].get(); + if (image == nullptr) return fail(FastFail::NoRecord); + const auto& own = image->Descriptor(); + const auto address = own.baseAddress; + const auto bytes = static_cast(image->GuestBytes()); + if (GuestMemory::CollectWrites(address, bytes) == 0) return fail(FastFail::Collect); + if (own.dccAddress != 0) { + // Refresh's unchanged branch (its key compare against the keys the content was + // uploaded under, through the image's proof; the memory query below), minus its + // byte-compare fallback; the image's generation is left where it is. + if (!keyProofs || ProvedClearKeys(own, bytes, image->KeyProof()) != image->UploadedKeys()) return fail(FastFail::StorageKeys); + countKeysProven(true); + } + if (!unchanged) query(address, address + bytes, image, nullptr, {i, true, false}); + if (!image->Cached()) return fail(FastFail::Evicted); + queries.push_back({address, bytes, image->Generation()}); + images.push_back(image); + } + if (!pending.empty()) { + if (!StorageTexture::ScanPending(pending)) return fail(FastFail::ClearedView); + // A foreign image over a view under uncompressed keys is no failure while FindPending + // names the view's own source: the lookup views the source (cachedTexture's hit rule) + // and touches nothing else. A surface whose own object T1 refreshed in this Revalidate + // (`refreshed`) is no failure either: a storage element's lookup is that Refresh and + // returns the image whatever stays registered over it; a view's lookup hits on the source + // when FindPending names it, and views the cached image of its key (the source, proved by + // refreshOwnObjects) when FindPending names nothing. Every other overlap is the full + // walk's, or T1's (Revalidate). + const auto refreshedOwner = [&](const PendingOverlap& owner) { + return std::find_if(refreshed.begin(), refreshed.end(), [&](const PendingOverlap& entry) { return entry.element == owner.element && entry.storage == owner.storage; }); + }; + for (std::size_t k = 0; k < pending.size(); ++k) { + if (!pending[k].overlaps) continue; + if (OwnImageRefreshEnabled() && owners[k].viewUncompressed && pending[k].found == pending[k].except) { + if (BuildProfiled()) Revalidations().ownSourceViews.fetch_add(1, std::memory_order_relaxed); + accepted = true; + continue; + } + if (const auto entry = refreshedOwner(owners[k]); entry != refreshed.end() && (entry->storage || pending[k].found == pending[k].except || (pending[k].found == nullptr && entry->sourceEligible))) { + if (BuildProfiled()) Revalidations().refreshedOverlaps.fetch_add(1, std::memory_order_relaxed); + accepted = true; + continue; + } + overlapping.push_back(owners[k]); + } + if (!overlapping.empty()) { + countPendingFail(pending); + return fail(FastFail::Pending); + } + } else if (unchanged && BuildProfiled()) { + Revalidations().serialSkips.fetch_add(1, std::memory_order_relaxed); + } + if (!GuestMemory::UnchangedSinceAll(queries)) return fail(FastFail::Changed); + if (!StorageImagesCached(context, images)) return fail(FastFail::Evicted); + for (const auto* image : images) image->NoteProved(); + for (std::size_t i = 0; i < validatedTextures.size(); ++i) { + auto& surface = validatedTextures[i]; + if (surface.source == nullptr) surface.generation = surface.collected; + surface.keys = scannedKeys[i]; + } + return true; +} + +bool ShaderResources::fastRevalidateEach() { if (validatedTextures.size() != textures.size()) return false; for (std::size_t i = 0; i < textures.size(); ++i) { auto& surface = validatedTextures[i]; @@ -1020,27 +1407,101 @@ bool ShaderResources::fastRevalidate() { return true; } -bool ShaderResources::Revalidate(std::span shaders) { +ShaderResources::OwnRefreshFallback ShaderResources::refreshOwnObjects(std::span shaders, std::span overlapping) { + const auto listed = [&](std::size_t element, bool storage) { + return std::find_if(overlapping.begin(), overlapping.end(), [&](const PendingOverlap& overlap) { return overlap.element == element && overlap.storage == storage; }); + }; + // A sampled view's lookup (cachedTexture): the pending image FindPending serves the surface + // by, if it fits the view, else the cached storage image of the surface's key, refreshed; the + // view stays the lookup's answer only when that image is its own source. `sourceEligible` is + // recorded whichever branch answers: a later refresh of this call may flush the source, and + // the re-run then judges the surface by the other branch. + const auto refreshView = [&](std::size_t i, PendingOverlap& overlap) { + const auto& surface = validatedTextures[i]; + const auto& source = textures[i]->SharedStorageSource(); + if (source == nullptr || source.get() != surface.source) return OwnRefreshFallback::Snapshot; + if (!source->Cached()) return OwnRefreshFallback::Uncached; + const auto address = surface.resource.baseAddress; + const auto bytes = static_cast(surface.bytes); + const bool imported = SampledFromStorageEligible(context, surface.resource, surface.bytes); + overlap.sourceEligible = imported && SurfaceKey(context, source->Descriptor()) == SurfaceKey(context, surface.resource); + auto found = StorageTexture::FindPending(address, surface.bytes); + if (found != nullptr && !Texture::CanCopyFrom(*found, surface.resource)) found.reset(); + if (found != nullptr) { + if (found != source) return OwnRefreshFallback::ForeignView; + if (!GuestMemory::UnchangedSince(address, bytes, source->Generation())) { + countOwnRefresh(false); + source->Refresh(); + } + return OwnRefreshFallback::Count; + } + if (!imported) return OwnRefreshFallback::NotImported; + if (!overlap.sourceEligible) return OwnRefreshFallback::SurfaceKey; + countOwnRefresh(false); + // The cache's image of the key is this source while it is cached (an entry is replaced + // only after its image left the cache), so the lookup would refresh and return it. + source->Refresh(); + return OwnRefreshFallback::Count; + }; + // A storage element's lookup (cachedStorageTexture): the cached image of its surface key, + // refreshed; the key is the element's words', which mapped to this image at the build. + const auto refreshStorage = [&](std::size_t i) { + auto& image = storageTextures[i]; + if (image == nullptr || !image->Cached()) return OwnRefreshFallback::Uncached; + countOwnRefresh(true); + image->Refresh(); + return OwnRefreshFallback::Count; + }; + std::size_t textureIndex = 0; + std::size_t storageIndex = 0; + for (const auto& shader : shaders) { + if (shader.program == nullptr) return OwnRefreshFallback::Uncached; + for (const auto& binding : shader.program->bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; + if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { + for (std::uint32_t element = 0; element < binding.count; ++element, ++textureIndex) { + if (textureIndex >= textures.size()) return OwnRefreshFallback::Uncached; + const auto overlap = listed(textureIndex, false); + if (overlap == overlapping.end()) continue; + if (const auto fallback = refreshView(textureIndex, *overlap); fallback != OwnRefreshFallback::Count) return fallback; + } + } else if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { + for (std::uint32_t element = 0; element < binding.count; ++element, ++storageIndex) { + if (storageIndex >= storageTextures.size()) return OwnRefreshFallback::Uncached; + if (listed(storageIndex, true) == overlapping.end()) continue; + if (const auto fallback = refreshStorage(storageIndex); fallback != OwnRefreshFallback::Count) return fallback; + } + } + } + } + return OwnRefreshFallback::Count; +} + +bool ShaderResources::Revalidate(std::span shaders, ProofReport* report) { + if (report != nullptr) *report = {ProofPath::Full, ProofFailure::Other}; if (!reusable || shaders.empty()) return false; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; // APS5_NO_FAST_REVALIDATE=1 always repeats the lookups. static const bool noFast = std::getenv("APS5_NO_FAST_REVALIDATE") != nullptr; const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; - const auto finish = [&](bool fast, bool ok) { + const auto finish = [&](bool fast, bool ok, ProofFailure failure = ProofFailure::Other) { if (profile) countRevalidate(fast, ok, start); + if (report != nullptr) report->failure = ok ? ProofFailure::None : failure; return ok; }; + // Loaded before any check (the memo rule of fastRevalidate): kept as the memo only when the + // registry did not move through the whole proof, flushes of this call included. + const auto serialBefore = StorageTexture::PendingSerial(); // (1) Textures and storage images, from stamps when every element allows it (fastRevalidate), // else by the same lookups as the build (which refresh or replace them as guest memory changed): // they must hand back the very objects the set's views belong to. The build appended them stage // by stage, binding by binding, so the walk repeats that order. - const bool fast = !noFast && fastRevalidate(); - if (!fast) { + const auto fullWalk = [&] { lookupLog.clear(); std::size_t textureIndex = 0; std::size_t storageIndex = 0; for (const auto& shader : shaders) { - if (shader.program == nullptr) return finish(false, false); + if (shader.program == nullptr) return false; for (const auto& binding : shader.program->bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { @@ -1049,38 +1510,141 @@ bool ShaderResources::Revalidate(std::span shaders) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); const auto resource = DecodeTextureResource(words); const VkComponentMapping components{ComponentSwizzleFor(resource.dstSelX), ComponentSwizzleFor(resource.dstSelY), ComponentSwizzleFor(resource.dstSelZ), ComponentSwizzleFor(resource.dstSelW)}; - if (textureIndex >= textures.size() || cachedTexture(context, words, resource, components) != textures[textureIndex]) return finish(false, false); + if (textureIndex >= textures.size() || cachedTexture(context, words, resource, components) != textures[textureIndex]) return false; ++textureIndex; } } else if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); - if (storageIndex >= storageTextures.size()) return finish(false, false); + if (storageIndex >= storageTextures.size()) return false; std::shared_ptr expected; if (SameAsPreviousStorageElement(binding, element) && StorageDedupeEnabled()) expected = storageTextures[storageIndex - 1]; else expected = cachedStorageTexture(context, words, DecodeTextureResource(words), storageMips[storageIndex]); - if (expected != storageTextures[storageIndex]) return finish(false, false); + if (expected != storageTextures[storageIndex]) return false; ++storageIndex; } } } } - if (textureIndex != textures.size() || storageIndex != storageTextures.size()) return finish(false, false); + if (textureIndex != textures.size() || storageIndex != storageTextures.size()) return false; // The walk proved every object current again: the records move to what it proved, or one // spurious stamp (a label sharing a 64 KiB block with a surface's edge) would keep this // object on the full walk for good. captureValidation(); + return true; + }; + thread_local std::vector overlapping; + thread_local std::vector refreshed; + refreshed.clear(); + FastFail reason = FastFail::Count; + bool accepted = false; + bool fast = !noFast && fastRevalidate(serialBefore, refreshed, reason, overlapping, accepted); + // T1 (design_cpu_final M3, rule RT1): a Pending failure whose overlapping images are foreign + // to the surfaces is resolved by the own objects' refresh (what the walk's lookups would do to + // them) and the fast proof run again, which is then authoritative (it accepts what stays + // pending over the refreshed surfaces, see fastRevalidate); a fallback, or a second failure, + // takes the full walk as before. + bool ownRefreshed = false; + if (!fast && !overlapping.empty()) { + auto fallback = OwnRefreshFallback::Count; + if (!OwnImageRefreshEnabled()) fallback = OwnRefreshFallback::Disabled; + else { + refreshed = overlapping; + fallback = refreshOwnObjects(shaders, refreshed); + } + if (fallback == OwnRefreshFallback::Count) { + fast = fastRevalidate(serialBefore, refreshed, reason, overlapping, accepted); + if (fast) ownRefreshed = true; + else fallback = OwnRefreshFallback::Rerun; + } + if (fallback != OwnRefreshFallback::Count) countOwnRefreshFallback(fallback); + } + if (report != nullptr) report->path = !fast ? ProofPath::Full : ownRefreshed ? ProofPath::OwnRefreshed : ProofPath::Fast; + if (!fast) { + countFullWalk(reason); + if (!fullWalk()) { + const auto failure = [&] { + switch (reason) { + case FastFail::Pending: return ProofFailure::Pending; + case FastFail::Evicted: return ProofFailure::Evicted; + case FastFail::Changed: return ProofFailure::Changed; + case FastFail::Keys: case FastFail::ClearedView: case FastFail::StorageKeys: return ProofFailure::Keys; + default: return ProofFailure::Other; + } + }(); + return finish(false, false, failure); + } + } else if (ownRefreshed || accepted) { + if (profile && ownRefreshed) Revalidations().ownRefreshed.fetch_add(1, std::memory_order_relaxed); + if (VerifyProofs()) { + // The walk after the refresh, or beside a proof that accepted a foreign overlap, must + // find every object in place and current: another object, or an upload (a moved + // content version), is a decision the proof got wrong. + thread_local std::vector> versions; + versions.clear(); + for (const auto& surface : validatedTextures) { + if (surface.source != nullptr) versions.emplace_back(surface.source, surface.source->Version()); + } + for (const auto& image : storageTextures) versions.emplace_back(image.get(), image->Version()); + const bool same = fullWalk(); + const auto moved = std::find_if(versions.begin(), versions.end(), [](const auto& entry) { return entry.first->Version() != entry.second; }); + if (!same || moved != versions.end()) { + std::fprintf(stderr, "[rescache] APS5_VERIFY_PROOFS: the T1 proof (%s) disagrees with the full walk (%s)\n", ownRefreshed ? "own-object refresh" : "accepted overlap", !same ? "another object" : "an upload"); + std::fflush(stderr); + std::abort(); + } + if (profile) Revalidations().proofsVerified.fetch_add(1, std::memory_order_relaxed); + } } // (2) The imported buffers the set reads in place: results of storage images pending in them go // to guest memory first (as an upload does), then each import must still be the one the set was // written against. This comes last because the lookups and flushes above can reconcile imports - // themselves; nothing else touches them between here and the dispatch being recorded. - for (const auto& region : directRegions) StorageTexture::FlushPending(region.begin, static_cast(region.end - region.begin), nullptr, "imported buffer region"); - for (const auto& region : directRegions) { - auto serial = HostImportSerial(context, region.begin, static_cast(region.end - region.begin), true); - if (serial == 0) serial = ImageMirrorSerial(context, region.begin, static_cast(region.end - region.begin)); - if (serial != region.serial) return finish(fast, false); + // themselves; nothing else touches them between here and the dispatch being recorded. Under the + // epoch gate the flush runs only for regions one registry scan finds pending images over (none + // while the serial is the memo's), and the serial loop only while the import table's identity + // moved since the last proof (a retire bumps its epoch, a registry change its generation). + const auto serialLoop = [&] { + for (const auto& region : directRegions) { + auto serial = HostImportSerial(context, region.begin, static_cast(region.end - region.begin), true); + if (serial == 0) serial = ImageMirrorSerial(context, region.begin, static_cast(region.end - region.begin)); + if (serial != region.serial) return false; + } + return true; + }; + if (EpochRevalidate()) { + if (!directRegions.empty() && !(pendingSerialSeen != 0 && pendingSerialSeen == StorageTexture::PendingSerial())) { + thread_local std::vector regions; + regions.clear(); + for (const auto& region : directRegions) regions.push_back({region.begin, region.end, nullptr, nullptr, false}); + StorageTexture::ScanPending(regions); + // A region over a unit shadow's fresh results (a retile bumped the serial) is + // published by the flush, as one over a pending image is stored. + for (const auto& region : regions) { + if (region.overlaps || AnyShadowedOverlaps(region.begin, static_cast(region.end - region.begin))) StorageTexture::FlushPending(region.begin, static_cast(region.end - region.begin), nullptr, "imported buffer region"); + } + } + if (HostImportsUnchanged(context, importsProof)) { + if (profile) Revalidations().importSkips.fetch_add(1, std::memory_order_relaxed); + } else { + if (!serialLoop()) return finish(fast, false, ProofFailure::Imports); + importsProof = HostImportsIdentity(context); + } + } else { + for (const auto& region : directRegions) StorageTexture::FlushPending(region.begin, static_cast(region.end - region.begin), nullptr, "imported buffer region"); + if (!serialLoop()) return finish(fast, false, ProofFailure::Imports); } + // (3) Buffers staged in device memory (GuestBufferMemory::AllowDeviceStaging) are copied in + // from their imports anew for this use, after the flushes above and before the work is + // recorded; a failure to record leaves the object unusable for this dispatch, not the batch. + if (auto* recorder = Recorder::Active(); recorder != nullptr) { + try { + guestMemory.RecordStagingCopies(*recorder); + } catch (const std::exception& error) { + std::fprintf(stderr, "[resources] staging copies of a reused build failed: %s\n", error.what()); + return finish(fast, false); + } + } + pendingSerialSeen = EpochRevalidate() && StorageTexture::PendingSerial() == serialBefore ? serialBefore : 0; return finish(fast, true); } @@ -1090,9 +1654,7 @@ namespace { // seen for the same shader variant(s) and the first differing word is charged to its binding's role // (and to its index within the V# or T# element), printed as [rescache] every 10 s: it says whether // V# bases of ring allocations, SRT words or image descriptors churn. -struct ChurnProfile { - std::mutex mutex; - std::unordered_map lastByVariant; +struct ChurnCounts { std::array roles{}; std::array bufferWords{}; std::array imageWords{}; @@ -1100,6 +1662,14 @@ struct ChurnProfile { std::uint64_t sameKey = 0; std::uint64_t layout = 0; std::uint64_t drawWords = 0; +}; + +// Dispatch keys and draw keys are counted apart: what churns a draw's key decides the draw steps. +struct ChurnProfile { + std::mutex mutex; + std::unordered_map lastByVariant; + ChurnCounts dispatch; + ChurnCounts draws; std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; @@ -1110,13 +1680,14 @@ ChurnProfile& Churn() { // Charges word `diff` of `key` to the binding of the ContentKey that starts at `at` (the layout of // ShaderResources::ContentKey), moving `at` past that key; false when `diff` lies beyond it. -bool chargeShaderKey(ChurnProfile& profile, const ResourceCache::Key& key, std::size_t& at, std::size_t diff) { - if (at + 4 > key.size() || diff < at + 4) { +bool chargeShaderKey(ChurnCounts& profile, const ResourceCache::Key& key, std::size_t& at, std::size_t diff) { + if (at + 5 > key.size() || diff < at + 5) { ++profile.layout; return true; } - const auto bindings = key[at + 3]; - at += 4; + const bool dataWords = key[at] != 0; + const auto bindings = key[at + 4]; + at += 5; for (std::uint32_t binding = 0; binding < bindings; ++binding) { if (at + 8 > key.size() || diff < at + 8) { ++profile.layout; @@ -1124,7 +1695,7 @@ bool chargeShaderKey(ChurnProfile& profile, const ResourceCache::Key& key, std:: } const auto kind = key[at]; const auto role = key[at + 1]; - const auto descriptorWords = key[at + 7]; + const auto descriptorWords = dataWords || !DataRole(static_cast(role)) ? key[at + 7] : 0u; at += 8; if (diff < at + descriptorWords) { if (role < profile.roles.size()) ++profile.roles[role]; @@ -1154,6 +1725,7 @@ bool chargeShaderKey(ChurnProfile& profile, const ResourceCache::Key& key, std:: } std::shared_ptr ResourceCache::Find(const Key& key) { + resourceCacheFinds.fetch_add(1, std::memory_order_relaxed); std::lock_guard lock(mutex); const auto found = index.find(key); if (found == index.end()) { @@ -1166,25 +1738,27 @@ std::shared_ptr ResourceCache::Find(const Key& key) { void ResourceCache::noteMiss(const Key& key) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - if (!profile || key.size() < 4) return; - // The variant(s) the key belongs to: a compute key starts with the stage and its variant, a draw - // key with a marker, the device and its stages' keys (Draw.cpp DrawResourceKey). + if (!profile || key.size() < 5) return; + // The variant(s) the key belongs to: a compute key starts with the data-words flag, the stage + // and its variant, a draw key with a marker, the device and its stages' keys (Draw.cpp + // DrawResourceKey). const bool draw = key[0] == 0xffffffffu; std::uint64_t variants = 0; if (draw) { std::size_t at = 4; - for (std::uint32_t stage = 0; stage < key[3] && at + 3 < key.size(); ++stage) { - variants = variants * 1000003ull ^ (key[at + 2] | (static_cast(key[at + 3]) << 32u)); + for (std::uint32_t stage = 0; stage < key[3] && at + 4 < key.size(); ++stage) { + variants = variants * 1000003ull ^ (key[at + 3] | (static_cast(key[at + 4]) << 32u)); at += 1 + key[at]; } } else { - variants = key[1] | (static_cast(key[2]) << 32u); + variants = key[2] | (static_cast(key[3]) << 32u); } auto& churn = Churn(); std::lock_guard lock(churn.mutex); + auto& counts = draw ? churn.draws : churn.dispatch; const auto found = churn.lastByVariant.find(variants); if (found == churn.lastByVariant.end()) { - ++churn.firstSeen; + ++counts.firstSeen; churn.lastByVariant.emplace(variants, key); } else { const auto& previous = found->second; @@ -1192,46 +1766,54 @@ void ResourceCache::noteMiss(const Key& key) { std::size_t diff = 0; while (diff < common && key[diff] == previous[diff]) ++diff; if (diff == common && key.size() == previous.size()) { - ++churn.sameKey; + ++counts.sameKey; } else if (draw) { std::size_t at = 4; bool charged = false; for (std::uint32_t stage = 0; stage < key[3] && at < key.size() && !charged; ++stage) { ++at; - charged = chargeShaderKey(churn, key, at, diff); + charged = chargeShaderKey(counts, key, at, diff); } - if (!charged) ++churn.drawWords; + if (!charged) ++counts.drawWords; } else { std::size_t at = 0; - if (!chargeShaderKey(churn, key, at, diff)) ++churn.layout; + if (!chargeShaderKey(counts, key, at, diff)) ++counts.layout; } found->second = key; } const auto now = std::chrono::steady_clock::now(); if (now - churn.lastReport < std::chrono::seconds(10)) return; churn.lastReport = now; - std::string line = "[rescache] miss churn, first differing word by role:"; - char item[256]; - for (std::size_t role = 0; role < churn.roles.size(); ++role) { - if (churn.roles[role] == 0) continue; - std::snprintf(item, sizeof(item), " %s %llu", roleName(static_cast(role)), static_cast(churn.roles[role])); - line += item; - if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestBuffers)) { - std::snprintf(item, sizeof(item), " (V# word %llu/%llu/%llu/%llu)", static_cast(churn.bufferWords[0]), static_cast(churn.bufferWords[1]), static_cast(churn.bufferWords[2]), static_cast(churn.bufferWords[3])); - line += item; - } else if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestImages)) { - std::snprintf(item, sizeof(item), " (T# word %llu/%llu/%llu/%llu/%llu/%llu/%llu/%llu)", static_cast(churn.imageWords[0]), static_cast(churn.imageWords[1]), static_cast(churn.imageWords[2]), static_cast(churn.imageWords[3]), static_cast(churn.imageWords[4]), static_cast(churn.imageWords[5]), static_cast(churn.imageWords[6]), static_cast(churn.imageWords[7])); + const auto report = [](const char* what, const ChurnCounts& churn) { + std::string line = std::string("[rescache] miss churn (") + what + "), first differing word by role:"; + char item[256]; + for (std::size_t role = 0; role < churn.roles.size(); ++role) { + if (churn.roles[role] == 0) continue; + std::snprintf(item, sizeof(item), " %s %llu", roleName(static_cast(role)), static_cast(churn.roles[role])); line += item; + if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestBuffers)) { + std::snprintf(item, sizeof(item), " (V# word %llu/%llu/%llu/%llu)", static_cast(churn.bufferWords[0]), static_cast(churn.bufferWords[1]), static_cast(churn.bufferWords[2]), static_cast(churn.bufferWords[3])); + line += item; + } else if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestImages)) { + std::snprintf(item, sizeof(item), " (T# word %llu/%llu/%llu/%llu/%llu/%llu/%llu/%llu)", static_cast(churn.imageWords[0]), static_cast(churn.imageWords[1]), static_cast(churn.imageWords[2]), static_cast(churn.imageWords[3]), static_cast(churn.imageWords[4]), static_cast(churn.imageWords[5]), static_cast(churn.imageWords[6]), static_cast(churn.imageWords[7])); + line += item; + } } - } - std::snprintf(item, sizeof(item), "; layout %llu, draw target/index %llu, same key %llu (not inserted or evicted), first seen %llu", static_cast(churn.layout), static_cast(churn.drawWords), static_cast(churn.sameKey), static_cast(churn.firstSeen)); - line += item; - std::fprintf(stderr, "%s\n", line.c_str()); + std::snprintf(item, sizeof(item), "; layout %llu, draw target/index %llu, same key %llu (not inserted or evicted), first seen %llu", static_cast(churn.layout), static_cast(churn.drawWords), static_cast(churn.sameKey), static_cast(churn.firstSeen)); + line += item; + std::fprintf(stderr, "%s\n", line.c_str()); + }; + report("dispatch", churn.dispatch); + report("draws", churn.draws); } -void ResourceCache::Insert(const Key& key, std::shared_ptr resources) { +void ResourceCache::Insert(const Key& key, std::shared_ptr resources, std::vector>* evicted) { std::lock_guard lock(mutex); - erase(key); + if (const auto found = index.find(key); found != index.end()) { + if (evicted != nullptr) evicted->push_back(std::move(found->second->second)); + entries.erase(found->second); + index.erase(found); + } entries.emplace_front(key, std::move(resources)); index.emplace(key, entries.begin()); // Entries pin their textures and storage images past the texture caches' budgets, so the bound @@ -1243,16 +1825,38 @@ void ResourceCache::Insert(const Key& key, std::shared_ptr reso return static_cast(parsed != 0 ? parsed : 1024ull); }(); while (entries.size() > capacity) { + if (evicted != nullptr) evicted->push_back(std::move(entries.back().second)); index.erase(entries.back().first); entries.pop_back(); } } -void ResourceCache::Remove(const Key& key) { +void ResourceCache::Remove(const Key& key, const ShaderResources* object) { std::lock_guard lock(mutex); + if (object != nullptr) { + const auto found = index.find(key); + if (found == index.end() || found->second->second.get() != object) return; + } erase(key); } +bool ResourceCache::Touch(const Key& key) { + resourceCacheTouches.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(mutex); + const auto found = index.find(key); + if (found == index.end()) return false; + entries.splice(entries.begin(), entries, found->second); + return true; +} + +std::uint64_t ResourceCache::Finds() { + return resourceCacheFinds.load(std::memory_order_relaxed); +} + +std::uint64_t ResourceCache::Touches() { + return resourceCacheTouches.load(std::memory_order_relaxed); +} + void ResourceCache::Clear() { std::lock_guard lock(mutex); index.clear(); @@ -1312,7 +1916,7 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l if (capacity == ChainPoolSizes.end() || size.descriptorCount > capacity->descriptorCount) return {}; } std::lock_guard lock(mutex); - const auto allocate = context.Function("vkAllocateDescriptorSets"); + const auto allocate = context.Resolved(&DeviceFunctions::allocateDescriptorSets, "vkAllocateDescriptorSets"); VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; allocation.descriptorSetCount = 1; allocation.pSetLayouts = &layout; @@ -1355,7 +1959,7 @@ DescriptorCache::Stats DescriptorCache::Counters() const { return stats; } -std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written) { +std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic) { Require(words.size() == 4, "buffer descriptor must contain four DWORDs"); Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set"); const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; @@ -1374,7 +1978,7 @@ std::size_t ShaderResources::addGuestBuffer(std::span words // Unconditional (also for read-only elements): Draw.cpp's CheckBufferAliases repeats this check // on every resource-cache hit, and a draw must fail its hit exactly when it fails its build. Require(!overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer"); - if (written) guestMemory.AddWritable(address, size); + if (written) guestMemory.AddWritable(address, size, atomic); else { guestMemory.AddReadable(address, size); ++readOnlyBuffers; @@ -1450,12 +2054,80 @@ std::string ShaderResources::Describe() const { std::size_t ShaderResources::addDataBuffer(std::span words) { const auto size = words.size() * sizeof(std::uint32_t); Require(size <= context.limits.maxStorageBufferRange, "shader data buffer exceeds descriptor range limit"); - auto buffer = std::make_unique(context, size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + const bool refreshable = TemplateDataRefresh() && size <= MaxRefreshBytes; + auto buffer = std::make_unique(context, size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (refreshable ? VK_BUFFER_USAGE_TRANSFER_DST_BIT : 0u)); std::memcpy(buffer->Bytes().data(), words.data(), size); - allocations.push_back({0, size, false, std::move(buffer)}); + Allocation allocation{0, size, false, std::move(buffer)}; + if (refreshable) allocation.dataWords.assign(words.begin(), words.end()); + allocations.push_back(std::move(allocation)); + mixDataWords(dataWordsHash, allocations.back().dataWords); return allocations.size() - 1; } +void ShaderResources::rehashDataWords() { + dataWordsHash = FnvOffset; + // Data buffers are the non-guest allocations with a buffer (an address-role allocation has + // none), appended in binding order by buildPrepare: the order DataWordsHash(shader) hashes. + for (const auto& allocation : allocations) { + if (!allocation.guest && allocation.buffer != nullptr) mixDataWords(dataWordsHash, allocation.dataWords); + } +} + +std::uint64_t ShaderResources::DataWordsHash(const CompiledShader& shader) { + Require(shader.program != nullptr, "missing compiled shader"); + std::uint64_t hash = FnvOffset; + for (const auto& binding : shader.program->bindings) { + if (DataRole(binding.role)) mixDataWords(hash, binding.guestDescriptor); + } + return hash; +} + +bool ShaderResources::DataWordsDiffer(const CompiledShader& shader) const { + Require(shader.program != nullptr, "missing compiled shader"); + const auto& program = *shader.program; + if (program.bindings.size() != bindings.size()) return true; + for (std::size_t index = 0; index < program.bindings.size(); ++index) { + const auto& binding = program.bindings[index]; + if (!DataRole(binding.role)) continue; + if (bindings[index].allocations.size() != 1) return true; + const auto& allocation = allocations[bindings[index].allocations.front()]; + if (allocation.dataWords.size() != binding.guestDescriptor.size() || !std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) return true; + } + return false; +} + +bool ShaderResources::RefreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder) { + Require(shader.program != nullptr, "missing compiled shader"); + const auto& program = *shader.program; + Require(program.bindings.size() == bindings.size(), "template bindings disagree with the shader"); + bool recorded = false; + // One [gputime] range of class TemplateDataRefresh around the updates, begun at the first one + // (a refresh that finds every word equal records nothing and times nothing). + auto timing = Recorder::NoTiming; + std::uint64_t refreshedBytes = 0; + for (std::size_t index = 0; index < program.bindings.size(); ++index) { + const auto& binding = program.bindings[index]; + if (!DataRole(binding.role)) continue; + Require(bindings[index].allocations.size() == 1, "data binding without its buffer"); + auto& allocation = allocations[bindings[index].allocations.front()]; + const auto size = binding.guestDescriptor.size() * sizeof(std::uint32_t); + Require(allocation.buffer != nullptr && !allocation.guest && allocation.size == size && size <= MaxRefreshBytes, "template data buffer cannot take the dispatch's words"); + if (allocation.dataWords.size() == binding.guestDescriptor.size() && std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) continue; + if (recorder != nullptr && timing == Recorder::NoTiming) timing = recorder->BeginGpuTiming(Recorder::CommandClass::TemplateDataRefresh); + context.Resolved(&DeviceFunctions::cmdUpdateBuffer, "vkCmdUpdateBuffer")(commands, allocation.buffer->Handle(), 0, size, binding.guestDescriptor.data()); + allocation.dataWords.assign(binding.guestDescriptor.begin(), binding.guestDescriptor.end()); + refreshedBytes += size; + recorded = true; + } + if (timing != Recorder::NoTiming) recorder->EndGpuTiming(timing, refreshedBytes); + if (recorded) rehashDataWords(); + return recorded; +} + +void ShaderResources::PrecollectSurfaces() const { + for (const auto& range : describedRanges) GuestMemory::CollectWrites(range.address, static_cast(range.bytes)); +} + void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags) { Require(binding.count != 0, "empty descriptor binding"); if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { @@ -1583,6 +2255,14 @@ bool ShaderResources::precollectImages() { std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) { if (record.texture == nullptr) return nullptr; + struct Outcome { + bool profile; + std::chrono::steady_clock::time_point start; + bool hit = false; + ~Outcome() { + if (profile) LookupOutcomes::Add(hit ? LookupOutcomes::SampledFast : LookupOutcomes::SampledFastMiss, start); + } + } outcome{LookupOutcomes::Profiled(), LookupOutcomes::Profiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}}; // Exactly the "unchanged" branches of cachedTexture, from write stamps, the pending-results // registry and the DCC keys, the way fastRevalidate proves a built object current: the collect // first (a memo hit: stage A walked the range), then no other image may have results pending @@ -1596,7 +2276,13 @@ std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) if (GuestMemory::CollectWrites(address, bytes) == 0) return nullptr; if (PendingStorageOverlaps(address, bytes, record.source.get())) return nullptr; auto keys = record.keys; - if (record.resource.dccAddress != 0) keys = TextureClearKeys(record.resource, record.guestBytes); + if (record.resource.dccAddress != 0) { + const auto scanStart = outcome.profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + // Through the proof of the image whose keys they are (the view's source when the surface + // is the image's, else the texture's): a scan only when the key range was stamped. + keys = ProvedClearKeys(record.resource, record.guestBytes, SameKeySurface(record.source.get(), record.resource, record.guestBytes) ? record.source->KeyProof() : record.texture->KeyProof()); + if (outcome.profile) LookupOutcomes::Add(LookupOutcomes::DccScan, scanStart); + } // The keys the entry was made under (a view of a pending render target was made under its clear // keys and stays valid while they are unchanged); a change is the full lookup's to judge. if (keys != record.entryKeys) return nullptr; @@ -1617,6 +2303,7 @@ std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) if (it->source == nullptr) it->generation = record.generation; touchTexture(cache, it); logLookup({record.texture.get(), record.resource, record.guestBytes, keys, record.source != nullptr ? 0 : record.generation, record.source.get()}); + outcome.hit = true; return record.texture; } @@ -1744,7 +2431,7 @@ VkDescriptorSetLayout ShaderResources::Layout() const { void ShaderResources::Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout) const { if (_set == VK_NULL_HANDLE) return; - context.Function("vkCmdBindDescriptorSets")(commands, bindPoint, layout, 0, 1, &_set, 0, nullptr); + context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, bindPoint, layout, 0, 1, &_set, 0, nullptr); } namespace { @@ -1764,6 +2451,10 @@ void ShaderResources::WriteBack() { } void ShaderResources::MarkGpuWrites(Recorder& recorder) { + // The ranges this use reads in place through their host imports (read-only and written elements + // alike, and an address-based build's whole leased heaps), before the writes: a CPU store into + // one of them (the copy HLE) must not land before the recorded work read it. + recorder.NotePendingReads(guestMemory.InPlaceReads(), guestMemory.HoldsLease() ? Recorder::ReadKind::AddressBased : Recorder::ReadKind::DispatchElement); if (SkipWriteBack()) return; for (std::size_t index = 0; index < storageTextures.size(); ++index) { if (storageWritten[index]) storageTextures[index]->MarkDirty(); @@ -1795,4 +2486,32 @@ void ShaderResources::WriteBackBuffers() { guestMemory.WriteBack(); } +bool ShaderResources::WritesMemory() const { + return HoldsLease() || NeedsCompletion() || !guestMemory.Writes().empty() || std::any_of(storageWritten.begin(), storageWritten.end(), [](bool written) { return written; }); +} + +bool ShaderResources::ReadsOverlap(std::uint64_t address, std::size_t bytes) const { + const auto reads = guestMemory.InPlaceReads(); + return std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return address < range.second && range.first < address + bytes; }); +} + +std::vector> ShaderResources::StorageImages() const { + std::vector> images; + for (std::size_t index = 0; index < storageTextures.size(); ++index) { + if (storageTextures[index] != nullptr) images.emplace_back(storageTextures[index]->Image(), storageWritten[index]); + } + for (const auto& texture : textures) { + if (texture != nullptr && texture->StorageSource() != nullptr) images.emplace_back(texture->StorageSource()->Image(), false); + } + return images; +} + +bool ShaderResources::ReadsImage(const StorageTexture* image) const { + if (image == nullptr) return false; + for (const auto& texture : textures) { + if (texture != nullptr && texture->StorageSource() == image) return true; + } + return std::any_of(storageTextures.begin(), storageTextures.end(), [&](const auto& storage) { return storage.get() == image; }); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index 1eb231c1aa..08f5495226 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -123,7 +123,7 @@ struct Module { } }; -Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric) { +Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing) { using Stage = ShaderRecompiler::ShaderStage; Require(compiled.program != nullptr, "missing compiled shader"); const auto& shader = *compiled.program; @@ -212,7 +212,17 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi capability == spv::CapabilityInt16 || capability == spv::CapabilityFloat16 || capability == spv::CapabilityStorageBuffer8BitAccess || - capability == spv::CapabilityPhysicalStorageBufferAddresses; + capability == spv::CapabilityPhysicalStorageBufferAddresses || + capability == spv::CapabilitySampledImageArrayDynamicIndexing || + capability == spv::CapabilityStorageImageArrayDynamicIndexing; + + // Bindless image tables index their slots non-uniformly in graphics stages + // (VK_EXT_descriptor_indexing, enabled by the device setup when available). + const bool isDescriptorIndexingCapability = + descriptorIndexing && + (capability == spv::CapabilityShaderNonUniform || + capability == spv::CapabilitySampledImageArrayNonUniformIndexing || + capability == spv::CapabilityStorageImageArrayNonUniformIndexing); Require( isBaseCapability || @@ -221,7 +231,8 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi isBdaCapability || isTessellationCapability || isMeshCapability || - isFeatureCapability, + isFeatureCapability || + isDescriptorIndexingCapability, std::string("SPIR-V requires unsupported device capability ") + std::to_string(static_cast(capability))); @@ -238,7 +249,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(fragment && fragmentShaderBarycentric, "SPV_KHR_fragment_shader_barycentric requires enabled fragmentShaderBarycentric in a fragment shader"); break; } - Require(extension == "SPV_KHR_float_controls" || (mesh && extension == "SPV_EXT_mesh_shader") || (shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && (extension == "SPV_KHR_physical_storage_buffer" || extension == "SPV_KHR_8bit_storage")), "unsupported SPIR-V extension"); + Require(extension == "SPV_KHR_float_controls" || (mesh && extension == "SPV_EXT_mesh_shader") || (descriptorIndexing && extension == "SPV_EXT_descriptor_indexing") || (shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && (extension == "SPV_KHR_physical_storage_buffer" || extension == "SPV_KHR_8bit_storage")), "unsupported SPIR-V extension"); break; } case spv::OpDecorateId: @@ -474,7 +485,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi } -std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric) { +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing) { using Stage = ShaderRecompiler::ShaderStage; const bool tessellation = state.stages.path == ShaderPath::Tessellation; const bool mesh = state.stages.path == ShaderPath::Geometry; @@ -490,7 +501,7 @@ std::set ValidateShaders(std::span shaders, Require(shaders[i].program != nullptr, "missing compiled shader"); Require(shaders[i].stage == expected, "graphics stage order disagrees"); for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == 0, "graphics resource uses a descriptor set other than zero"); - const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric); + const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric, descriptorIndexing); if (i != 0) { for (const auto& [location, signature] : current.inputs) { const auto output = previous.outputs.find(location); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index 5b71e9fae4..8d214688f1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -6,36 +6,92 @@ #include "prx/libc/include/General.hpp" #include #include +#include #include +#include #include #include namespace AgcDriver::Graphics { namespace { -std::uint32_t read(const Registers& registers, std::uint32_t offset, const char* bank = "context") { +std::uint32_t read(const Registers& registers, std::uint32_t offset, RegisterBank bank = RegisterBank::Context) { + NoteRegisterRead(bank, offset); const auto it = registers.find(offset); if (it == registers.end()) { std::ostringstream message; - message << "missing register in " << bank << " bank at DWORD 0x" << std::hex << offset << " (" << std::dec << offset << ')'; + message << "missing register in " << RegisterBankName(bank) << " bank at DWORD 0x" << std::hex << offset << " (" << std::dec << offset << ')'; throw std::runtime_error("AGC graphics: " + message.str()); } return it->second; } +// A register that may be absent (the decoders' find/contains), through the facade. +Registers::const_iterator find(const Registers& registers, std::uint32_t offset, RegisterBank bank = RegisterBank::Context) { + NoteRegisterRead(bank, offset); + return registers.find(offset); +} + float readFloat(const Registers& registers, std::uint32_t offset) { const auto value = std::bit_cast(read(registers, offset)); Require(std::isfinite(value), "non-finite register at DWORD " + std::to_string(offset)); return value; } -void zero(const Registers& registers, std::uint32_t offset, std::uint32_t mask, const char* name, const char* bank = "context") { +std::string zeroMessage(std::uint32_t offset, std::uint32_t value, const char* name) { + char detail[64]; + std::snprintf(detail, sizeof(detail), " (register 0x%x = 0x%08x)", offset, value); + return std::string(name) + " is unsupported" + detail; +} + +void zero(const Registers& registers, std::uint32_t offset, std::uint32_t mask, const char* name, RegisterBank bank = RegisterBank::Context) { const auto value = read(registers, offset, bank); - if ((value & mask) != 0) { - char detail[64]; - std::snprintf(detail, sizeof(detail), " (register 0x%x = 0x%08x)", offset, value); - throw std::runtime_error(std::string(name) + " is unsupported" + detail); - } + if ((value & mask) != 0) throw std::runtime_error(zeroMessage(offset, value, name)); +} + +std::string vteMessage(std::uint32_t viewportControl) { + std::ostringstream message; + message << "AGC graphics: PA_CL_VTE_CNTL=0x" << std::hex << viewportControl << ": expected 0x43f for homogeneous positions and all viewport transforms; pre-divided coordinates, reciprocal W or disabled transforms are unsupported"; + return message.str(); +} + +// The register rules DecodeState and DrawRejection share (the precheck must reject exactly what +// DecodeState would): the masks whose set bits are unsupported, and the depth-control verdict. +// Render target index, viewport index and the misc export vector that carries them are accepted but +// not routed: color targets are single-layer, so layered draws land in layer 0. +constexpr std::uint32_t LayerExports = (1u << 18u) | (1u << 19u) | (1u << 21u) | (1u << 24u); +constexpr std::uint32_t DepthControlMask = ~0x007007f0u; +// EXEC_ON_HIER_FAIL / EXEC_ON_NOOP / EXEC_IF_OVERLAPPED (bits 9, 10, 17) only force the pixel shader +// to run, which it always does here. +constexpr std::uint32_t ShaderControlMask = ~(0x00009870u | 0x00020600u); +constexpr std::uint32_t AlphaToCoverageMask = ~0x0001ff00u; +constexpr std::uint32_t ScanModeMask = ~2u; +constexpr std::uint32_t ScanControlMask = ~0x06003fffu; +constexpr std::uint32_t ScreenOffsetMask = ~0x01ff01ffu; +// Bits 26/27 (ZCLIP_NEAR/FAR_DISABLE) become depth clamping; bit 19 selects the [0, 1] clip space. +constexpr std::uint32_t ClipControlMask = ~(0x80000u | 0x0c000000u); + +// Debug aid: APS5_IGNORE_DEPTH_TEST=1 renders depth- and stencil-tested draws without a depth +// target as if their tests always passed (wrong occlusion, but the draws run), so stages that +// depth-test everything can exercise the draw paths before depth targets exist. +bool IgnoreDepthTest() { + static const bool ignore = std::getenv("APS5_IGNORE_DEPTH_TEST") != nullptr; + return ignore; +} + +// DB_DEPTH_CONTROL: depth and stencil tests whose function is ALWAYS and that write no depth cannot +// change color output, so they are accepted without a depth target (stencil writes are dropped). +bool depthPassThrough(std::uint32_t depthControl) { + const bool stencil = (depthControl & 1u) != 0; + const bool depth = (depthControl & 2u) != 0; + const bool depthWrite = (depthControl & 4u) != 0; + const bool passThroughDepth = !depth || (!depthWrite && ((depthControl >> 4u) & 7u) == 7u); + const bool passThroughStencil = !stencil || (((depthControl >> 8u) & 7u) == 7u && ((depthControl & 0x80u) == 0 || ((depthControl >> 20u) & 7u) == 7u)); + return (stencil || depth) && passThroughDepth && passThroughStencil; +} + +bool colorControlSupported(std::uint32_t colorControl, bool hasColorTarget) { + return colorControl == 0xcc0010u || (!hasColorTarget && (colorControl & ~0x70u) == 0xcc0000u); } VkBlendFactor blendFactor(std::uint32_t value) { @@ -167,12 +223,15 @@ void intersect(VkRect2D& result, const Registers& registers, std::uint32_t offse ShaderStages DecodeShaderStages(const QueueState& queue) { const auto value = read(queue.context, 0x2d5); - std::ostringstream prefix; - prefix << "VGT_SHADER_STAGES_EN=0x" << std::hex << value << ": "; - const auto validate = [&](bool condition, const char* reason) { Require(condition, prefix.str() + reason); }; + const auto validate = [&](bool condition, const char* reason) { + if (condition) return; + std::ostringstream prefix; + prefix << "VGT_SHADER_STAGES_EN=0x" << std::hex << value << ": " << reason; + Require(false, prefix.str()); + }; validate((value & 0xfc000000u) == 0, "reserved stage bits are set"); validate((value & 3u) != 3u && ((value >> 3u) & 3u) != 3u && ((value >> 6u) & 3u) != 3u, "reserved LS_EN, ES_EN or VS_EN encoding"); - const auto primitive = read(queue.userConfig, 0x242, "user-config"); + const auto primitive = read(queue.userConfig, 0x242, RegisterBank::UserConfig); const bool tessellation = primitive == 9; const bool geometry = (value & 0x20u) != 0; validate(tessellation == ((value & 4u) != 0), "Patch topology and HS_EN disagree"); @@ -194,7 +253,7 @@ ShaderStages DecodeShaderStages(const QueueState& queue) { result.tessellation = tess; } else { validate((value & ~0x0047ec30u) == 0, "unsupported geometry routing, fast launch or wave-ID state"); - const auto group = read(queue.userConfig, 0x25b, "user-config"); + const auto group = read(queue.userConfig, 0x25b, RegisterBank::UserConfig); const auto vertices = (group >> 9u) & 0x1ffu; const auto primitives = group & 0x1ffu; const auto maxVertices = read(queue.context, 0x1ff); @@ -205,8 +264,8 @@ ShaderStages DecodeShaderStages(const QueueState& queue) { const auto inputStep = primitive == 6 ? 1u : inputSize; const auto groupPrimitives = std::min({primitives, (vertices - inputSize) / inputStep + 1u, maxVertices / verticesPerPrimitive}); validate(groupPrimitives != 0, "geometry subgroup contains no primitives"); - const auto resources = read(queue.shader, 0x8b, "shader"); - validate(((read(queue.shader, 0x8a, "shader") >> 29u) & 3u) == 3 && ((resources >> 16u) & 3u) == 3, "unsupported geometry VGPR allocation"); + const auto resources = read(queue.shader, 0x8b, RegisterBank::Shader); + validate(((read(queue.shader, 0x8a, RegisterBank::Shader) >> 29u) & 3u) == 3 && ((resources >> 16u) & 3u) == 3, "unsupported geometry VGPR allocation"); result.mesh = ShaderRecompiler::MeshConfiguration{primitive, groupPrimitives, (groupPrimitives - 1u) * inputStep + inputSize, maxVertices, primitives * (verticesPerPrimitive - 2u), ((maxVertices + result.vertexWaveSize - 1u) / result.vertexWaveSize) * result.vertexWaveSize, ((resources >> 19u) & 0xffu) * 128u, 0}; } return result; @@ -216,7 +275,7 @@ State DecodeState(const QueueState& queue) { const auto& cx = queue.context; State result{}; result.stages = DecodeShaderStages(queue); - const auto primitive = read(queue.userConfig, 0x242, "user-config"); + const auto primitive = read(queue.userConfig, 0x242, RegisterBank::UserConfig); APS5_LOG_OUT_DEBUG("DecodeState primitive=%u path=%u vertexWave=%u", primitive, static_cast(result.stages.path), result.stages.vertexWaveSize); switch (primitive) { case 1: Require(result.stages.mesh.has_value(), "point-list vertex rendering requires point-size output support"); result.topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; break; @@ -234,60 +293,49 @@ State DecodeState(const QueueState& queue) { default: throw std::runtime_error("AGC graphics: unsupported primitive type " + std::to_string(primitive)); } APS5_LOG_OUT_DEBUG("Topology=%u", static_cast(result.topology)); - zero(queue.userConfig, 0x24b, ~0u, "primitive restart (GE_MULTI_PRIM_IB_RESET_EN)", "user-config"); - // Render target index, viewport index and the misc export vector that carries them are accepted but - // not routed: color targets are single-layer, so layered draws land in layer 0. - constexpr std::uint32_t layerExports = (1u << 18u) | (1u << 19u) | (1u << 21u) | (1u << 24u); - if ((read(cx, 0x207, "context") & layerExports) != 0) { + zero(queue.userConfig, 0x24b, ~0u, "primitive restart (GE_MULTI_PRIM_IB_RESET_EN)", RegisterBank::UserConfig); + if ((read(cx, 0x207) & LayerExports) != 0) { static bool reported = false; if (!reported) { reported = true; - std::fprintf(stderr, "[gpu] layer/viewport index vertex exports are ignored (PA_CL_VS_OUT_CNTL=0x%08x)\n", read(cx, 0x207, "context")); + std::fprintf(stderr, "[gpu] layer/viewport index vertex exports are ignored (PA_CL_VS_OUT_CNTL=0x%08x)\n", read(cx, 0x207)); } } - zero(cx, 0x207, ~layerExports, "clip distances, layer, viewport or auxiliary vertex exports"); - // DB_DEPTH_CONTROL: depth and stencil tests whose function is ALWAYS and that write no depth cannot - // change color output, so they are accepted without a depth target (stencil writes are dropped). + zero(cx, 0x207, ~LayerExports, "clip distances, layer, viewport or auxiliary vertex exports"); { - const auto depthControl = read(cx, 0x200, "context"); - const bool stencil = (depthControl & 1u) != 0; - const bool depth = (depthControl & 2u) != 0; - const bool depthWrite = (depthControl & 4u) != 0; - const bool passThroughDepth = !depth || (!depthWrite && ((depthControl >> 4u) & 7u) == 7u); - const bool passThroughStencil = !stencil || (((depthControl >> 8u) & 7u) == 7u && ((depthControl & 0x80u) == 0 || ((depthControl >> 20u) & 7u) == 7u)); - if ((stencil || depth) && passThroughDepth && passThroughStencil) { + const auto depthControl = read(cx, 0x200); + if (depthPassThrough(depthControl)) { static bool reported = false; if (!reported) { reported = true; std::fprintf(stderr, "[gpu] always-pass depth/stencil state is rendered without a depth target (DB_DEPTH_CONTROL=0x%08x)\n", depthControl); } + } else if (IgnoreDepthTest()) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] depth/stencil tests are ignored (APS5_IGNORE_DEPTH_TEST; DB_DEPTH_CONTROL=0x%08x)\n", depthControl); + } } else { - zero(cx, 0x200, ~0x007007f0u, "depth, stencil or conditional color writes"); + zero(cx, 0x200, DepthControlMask, "depth, stencil or conditional color writes"); } Require((depthControl & 0xc0000008u) == 0, "depth bounds or depth-conditional color writes are unsupported"); } - // EXEC_ON_HIER_FAIL / EXEC_ON_NOOP / EXEC_IF_OVERLAPPED (bits 9, 10, 17) only force the pixel shader - // to run, which it always does here. - zero(cx, 0x203, ~(0x00009870u | 0x00020600u), "depth export, shader coverage or ordered fragment execution"); - zero(cx, 0x2dc, ~0x0001ff00u, "alpha-to-coverage"); + zero(cx, 0x203, ShaderControlMask, "depth export, shader coverage or ordered fragment execution"); + zero(cx, 0x2dc, AlphaToCoverageMask, "alpha-to-coverage"); zero(cx, 0x2f8, ~0u, "multisampling or coverage conversion"); - zero(cx, 0x292, ~2u, "scan conversion mode"); - zero(cx, 0x293, ~0x06003fffu, "sample iteration, primitive discard or out-of-order rasterization"); + zero(cx, 0x292, ScanModeMask, "scan conversion mode"); + zero(cx, 0x293, ScanControlMask, "sample iteration, primitive discard or out-of-order rasterization"); zero(cx, 0x80, ~0u, "window offset"); - zero(cx, 0x8d, ~0x01ff01ffu, "reserved PA_SU_HARDWARE_SCREEN_OFFSET bits"); + zero(cx, 0x8d, ScreenOffsetMask, "reserved PA_SU_HARDWARE_SCREEN_OFFSET bits"); Require(read(cx, 0x83) == 0xffffu, "clip rectangles are unsupported"); Require((read(cx, 0x8c) & 0xfu) == 0xau, "nonstandard triangle edge rules are unsupported"); Require(read(cx, 0x2f9) == 0x2du, "nonstandard pixel center or vertex quantization is unsupported"); Require(read(cx, 0x313) == 0x6000u, "conservative rasterization is unsupported"); Require(read(cx, 0x30e) == 0xffffffffu && read(cx, 0x30f) == 0xffffffffu, "sample masks are unsupported"); const auto viewportControl = read(cx, 0x206); - if (viewportControl != 0x43fu) { - std::ostringstream message; - message << "AGC graphics: PA_CL_VTE_CNTL=0x" << std::hex << viewportControl << ": expected 0x43f for homogeneous positions and all viewport transforms; pre-divided coordinates, reciprocal W or disabled transforms are unsupported"; - throw std::runtime_error(message.str()); - } - // Bits 26/27 (ZCLIP_NEAR/FAR_DISABLE) become depth clamping; bit 19 selects the [0, 1] clip space. - zero(cx, 0x204, ~(0x80000u | 0x0c000000u), "unsupported PA_CL_CLIP_CNTL flags"); + if (viewportControl != 0x43fu) throw std::runtime_error(vteMessage(viewportControl)); + zero(cx, 0x204, ClipControlMask, "unsupported PA_CL_CLIP_CNTL flags"); result.depthClamp = (read(cx, 0x204) & 0x0c000000u) != 0; result.negativeOneToOne = (read(cx, 0x204) & 0x80000u) == 0; const auto raster = read(cx, 0x205); @@ -320,7 +368,7 @@ State DecodeState(const QueueState& queue) { APS5_LOG_OUT_DEBUG("hasColorTarget=%u slots=%u", result.hasColorTarget ? 1u : 0u, slotCount); // CB_COLOR_CONTROL mode 0 disables color writes, which only matters when a target is written. - Require(read(cx, 0x202) == 0xcc0010u || (!result.hasColorTarget && (read(cx, 0x202) & ~0x70u) == 0xcc0000u), "only normal color rendering with copy ROP is supported"); + Require(colorControlSupported(read(cx, 0x202), result.hasColorTarget), "only normal color rendering with copy ROP is supported"); zero(cx, 0x1c4, ~0u, "depth or sample-mask export"); const auto exportFormat = read(cx, 0x1c5); APS5_LOG_OUT_DEBUG("Export format=%u", exportFormat); @@ -374,7 +422,7 @@ State DecodeState(const QueueState& queue) { color.componentMapping = 0xe4u; if ((info & 0x10000000u) != 0) { if (maxMip == 0) { - const auto dccHigh = cx.find(0x3a8 + slot); + const auto dccHigh = find(cx, 0x3a8 + slot); color.dccAddress = ((dccHigh == cx.end() ? 0ull : static_cast(dccHigh->second & 0xffu)) << 40u) | (static_cast(read(cx, 0x325 + stride)) << 8u); color.dccAlphaOnMsb = DccAlphaOnMsb(color.format, swap); } else { @@ -450,4 +498,72 @@ State DecodeState(const QueueState& queue) { return result; } +std::string DrawRejection(const QueueState& queue) { + const auto& cx = queue.context; + // A register a rule needs that is absent gives no verdict here: DecodeState reports it. + const auto value = [&](const Registers& registers, std::uint32_t offset, std::uint32_t& out) { + const auto it = find(registers, offset, ®isters == &queue.userConfig ? RegisterBank::UserConfig : RegisterBank::Context); + if (it == registers.end()) return false; + out = it->second; + return true; + }; + std::uint32_t word = 0; + const auto nonzero = [&](const Registers& registers, std::uint32_t offset, std::uint32_t mask, const char* name) { + return value(registers, offset, word) && (word & mask) != 0 ? zeroMessage(offset, word, name) : std::string(); + }; + const auto require = [&](bool condition, const char* reason) { + return condition ? std::string() : "AGC graphics: " + std::string(reason); + }; + if (auto reason = nonzero(queue.userConfig, 0x24b, ~0u, "primitive restart (GE_MULTI_PRIM_IB_RESET_EN)"); !reason.empty()) return reason; + if (auto reason = nonzero(cx, 0x207, ~LayerExports, "clip distances, layer, viewport or auxiliary vertex exports"); !reason.empty()) return reason; + if (value(cx, 0x200, word)) { + if (!depthPassThrough(word) && !IgnoreDepthTest() && (word & DepthControlMask) != 0) return zeroMessage(0x200, word, "depth, stencil or conditional color writes"); + if (auto reason = require((word & 0xc0000008u) == 0, "depth bounds or depth-conditional color writes are unsupported"); !reason.empty()) return reason; + } + if (auto reason = nonzero(cx, 0x203, ShaderControlMask, "depth export, shader coverage or ordered fragment execution"); !reason.empty()) return reason; + if (auto reason = nonzero(cx, 0x2dc, AlphaToCoverageMask, "alpha-to-coverage"); !reason.empty()) return reason; + if (auto reason = nonzero(cx, 0x2f8, ~0u, "multisampling or coverage conversion"); !reason.empty()) return reason; + if (auto reason = nonzero(cx, 0x292, ScanModeMask, "scan conversion mode"); !reason.empty()) return reason; + if (auto reason = nonzero(cx, 0x293, ScanControlMask, "sample iteration, primitive discard or out-of-order rasterization"); !reason.empty()) return reason; + if (auto reason = nonzero(cx, 0x80, ~0u, "window offset"); !reason.empty()) return reason; + if (auto reason = nonzero(cx, 0x8d, ScreenOffsetMask, "reserved PA_SU_HARDWARE_SCREEN_OFFSET bits"); !reason.empty()) return reason; + if (value(cx, 0x83, word) && word != 0xffffu) return require(false, "clip rectangles are unsupported"); + if (value(cx, 0x8c, word) && (word & 0xfu) != 0xau) return require(false, "nonstandard triangle edge rules are unsupported"); + if (value(cx, 0x2f9, word) && word != 0x2du) return require(false, "nonstandard pixel center or vertex quantization is unsupported"); + if (value(cx, 0x313, word) && word != 0x6000u) return require(false, "conservative rasterization is unsupported"); + std::uint32_t other = 0; + if (value(cx, 0x30e, word) && value(cx, 0x30f, other) && (word != 0xffffffffu || other != 0xffffffffu)) return require(false, "sample masks are unsupported"); + if (value(cx, 0x206, word) && word != 0x43fu) return vteMessage(word); + if (auto reason = nonzero(cx, 0x204, ClipControlMask, "unsupported PA_CL_CLIP_CNTL flags"); !reason.empty()) return reason; + std::uint32_t targetMask = 0, shaderMask = 0; + if (value(cx, 0x8e, targetMask) && value(cx, 0x8f, shaderMask) && value(cx, 0x202, word) && !colorControlSupported(word, (targetMask & shaderMask) != 0)) return require(false, "only normal color rendering with copy ROP is supported"); + if (auto reason = nonzero(cx, 0x1c4, ~0u, "depth or sample-mask export"); !reason.empty()) return reason; + // The pixel stage decode (ShaderInputState.cpp) reads these after DecodeState and the program + // prepare; a bank without them fails there with this message. + for (const auto offset : {0x1b3u, 0x1b4u, 0x1c5u}) { + if (find(cx, offset) != cx.end()) continue; + char text[64]; + std::snprintf(text, sizeof(text), "AGC graphics: missing register at DWORD 0x%x", offset); + return text; + } + return {}; +} + +std::vector*& RegisterReadLog() { + static thread_local std::vector* log = nullptr; + return log; +} + +bool DrawKeyCovers(RegisterRead read) { + // One bit per register offset of each bank, set from the table once. + static const auto covered = [] { + std::array, static_cast(RegisterBank::Count)> bits{}; + for (const auto& range : DrawKeyRegisters) { + for (std::uint32_t i = 0; i < range.count; ++i) bits[static_cast(range.bank)].set(range.first + i); + } + return bits; + }(); + return read.bank < RegisterBank::Count && read.offset < 1024 && covered[static_cast(read.bank)].test(read.offset); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 890e9dbcfb..cacb96d29b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -7,9 +7,13 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include +#include +#include #include +#include #include #include #include @@ -22,7 +26,9 @@ #include #include #include +#include #include +#include namespace AgcDriver::Graphics { @@ -41,6 +47,10 @@ struct TextureProfile { std::uint64_t storageDirectWriteBacks = 0; // Sampled-texture uploads recorded into the open batch instead of waited for. std::uint64_t recordedUploads = 0; + // GPU clears of fast-cleared storage images recorded into the open batch, and waited for in a + // batch of their own (APS5_NO_RECORDED_CLEAR=1, or no recorder). + std::uint64_t storageRecordedClears = 0; + std::uint64_t storageWaitedClears = 0; }; TextureProfile& Profile() { @@ -48,6 +58,128 @@ TextureProfile& Profile() { return profile; } +// APS5_PROFILE_DRAW, every 10 s on the [storage] line: storage image bytes uploaded by path +// (recorded DCC clear, direct detile from the import, CPU) and written back by the reason of the +// FlushPending that forced the write-back (the other callers name themselves: "refresh", "cache +// eviction", "explicit"). `flushReason` is set by the caller around its writeBack calls. +thread_local const char* flushReason = nullptr; +// Likewise what made an upload necessary: "first" (a new image), "keys" (the DCC keys changed), +// "cpu" (a CPU store stamped a changed unit), "flushed" (another image's results were stored over +// the surface by this refresh), "store" (a driver store: a fill, a copy, a label, an earlier +// write-back), "untracked" (no write stamps for the surface: every refresh re-uploads), "alias" +// (units taken from an alias image on the device); set by the constructor and Refresh around +// upload() and borrowUnits(). +thread_local const char* uploadReason = "first"; +// Block-unit traffic, per 10 s like the rest of the [storage] line: partial uploads and write-backs +// (count and bytes), write-backs that stored nothing (every selected block CPU-written), partial +// selections widened to every pending unit (the hysteresis in writeBack), units dropped as dead +// (DiscardPendingInside) and units an alias's own results superseded (markLayersPending); relaxed, +// reported only. +std::atomic partialUploads{0}, partialUploadBytes{0}, partialWriteBacks{0}, partialWriteBackBytes{0}, emptyWriteBacks{0}, coalescedWriteBacks{0}, unitsDropped{0}, unitsSuperseded{0}; +// FlushPending listings skipped because no pending unit of the image lay inside the range, images +// whose selected pending units were dropped because their memory is no longer registered (see +// writeBackLayers), and pending units a DCC clear -> uncompressed key flip kept as the texels +// (Refresh). +std::atomic pretestSkipped{0}, unregisteredDropped{0}, keyFlipKept{0}; +// Images stored by a FlushPending after a hook skip of theirs (AccessKeptByCpu), of which by the +// hook for the read site of the last skip; images evictStale dropped (the [hooksync] and [storage] lines). +std::atomic flushedAfterSkip{0}, flushedAfterSkipSameSite{0}, staleEvicted{0}; + +struct StorageTraffic { + std::mutex mutex; + std::map> writeBacks; + std::array, 4> uploads{}; + std::map> uploadReasons; + std::uint64_t directWriteBacks = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +StorageTraffic& Traffic() { + static StorageTraffic traffic; + return traffic; +} + +void reportStorageTraffic(StorageTraffic& traffic) { + const auto now = std::chrono::steady_clock::now(); + if (now - traffic.lastReport < std::chrono::seconds(10)) return; + traffic.lastReport = now; + static constexpr const char* uploadNames[4] = {"clear", "direct", "cpu", "alias"}; + std::string line; + char text[96]; + for (std::size_t i = 0; i < 4; ++i) { + std::snprintf(text, sizeof(text), " %s %llu/%.1f", uploadNames[i], static_cast(traffic.uploads[i].first), traffic.uploads[i].second / 1048576.0); + line += text; + traffic.uploads[i] = {}; + } + line += "; by reason:"; + for (const auto& [reason, totals] : traffic.uploadReasons) { + std::snprintf(text, sizeof(text), " %llu/%.1f", static_cast(totals.first), totals.second / 1048576.0); + line += " " + reason + text; + } + line += "; write-backs by reason (count/MiB):"; + for (const auto& [reason, totals] : traffic.writeBacks) { + std::snprintf(text, sizeof(text), " %llu/%.1f", static_cast(totals.first), totals.second / 1048576.0); + line += " " + reason + text; + } + const auto take = [](std::atomic& counter) { return static_cast(counter.exchange(0, std::memory_order_relaxed)); }; + const auto partialUploadCount = take(partialUploads), partialWriteBackCount = take(partialWriteBacks); + const auto uploadMiB = take(partialUploadBytes) / 1048576.0, writeBackMiB = take(partialWriteBackBytes) / 1048576.0; + std::fprintf(stderr, "[storage] uploads by path (count/MiB, 10 s):%s; %llu write-backs GPU-direct, %llu stored nothing; block units: partial uploads %llu/%.1f, partial write-backs %llu/%.1f, %llu widened to every pending unit, units dropped %llu, superseded %llu, dropped unregistered %llu, kept over a key flip %llu%s, pretest skipped %llu, evicted stale %llu\n", line.c_str(), static_cast(traffic.directWriteBacks), take(emptyWriteBacks), partialUploadCount, uploadMiB, partialWriteBackCount, writeBackMiB, take(coalescedWriteBacks), take(unitsDropped), take(unitsSuperseded), take(unregisteredDropped), take(keyFlipKept), ShadowReport().c_str(), take(pretestSkipped), take(staleEvicted)); + traffic.writeBacks.clear(); + traffic.uploadReasons.clear(); + traffic.directWriteBacks = 0; +} + +void countStorageUpload(std::size_t path, std::uint64_t bytes) { + if (!LookupOutcomes::Profiled()) return; + auto& traffic = Traffic(); + std::lock_guard lock(traffic.mutex); + ++traffic.uploads[path].first; + traffic.uploads[path].second += bytes; + auto& reason = traffic.uploadReasons[uploadReason != nullptr ? uploadReason : "other"]; + ++reason.first; + reason.second += bytes; + reportStorageTraffic(traffic); +} + +void countStorageWriteBack(std::uint64_t bytes, bool direct) { + if (!LookupOutcomes::Profiled()) return; + auto& traffic = Traffic(); + std::lock_guard lock(traffic.mutex); + auto& totals = traffic.writeBacks[flushReason != nullptr ? flushReason : "other"]; + ++totals.first; + totals.second += bytes; + if (direct) ++traffic.directWriteBacks; + reportStorageTraffic(traffic); +} + +// Every storage image alive, for the fill HLE's cover check (StorageTexture::ClassifyFill): the +// storage cache indexes surfaces by key, not by address range. Its mutex is a leaf. +struct LiveImages { + std::mutex mutex; + std::vector textures; +}; + +LiveImages& Live() { + static LiveImages live; + return live; +} + +// Per-layer validity of array surfaces (see StorageTexture::layerBegin): a fill of one layer clears +// it on the image and only that layer's guest bytes go stale, so the other layers' fills, uploads +// and write-backs stay separate. APS5_NO_LAYER_CLEAR=1 tracks every surface as one layer, and a +// one-layer FillClear then demands the rest of the surface unchanged, as before. +bool LayerTrackingEnabled() { + static const bool disabled = std::getenv("APS5_NO_LAYER_CLEAR") != nullptr; + return !disabled; +} + +// APS5_NO_BLOCK_TRACKING=1 tracks array layers (or the whole surface) instead of 64 KiB blocks. +bool BlockTrackingEnabled() { + static const bool disabled = std::getenv("APS5_NO_BLOCK_TRACKING") != nullptr; + return !disabled; +} + struct PhaseTimer { std::chrono::steady_clock::time_point last = std::chrono::steady_clock::now(); double lap() { @@ -446,6 +578,11 @@ VkFormat StorageFormatForGuest(const Context& context, std::uint32_t guestFormat return StorageFormatFor(context, ResolveTextureFormat(guestFormat)); } +bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, DccKeys keys) { + VkClearColorValue clear{}; + return IsDccClear(keys) && StorageFormatAvailable(context, guestFormat) && ClearColorFor(StorageFormatForGuest(context, guestFormat), keys, clear); +} + StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, std::uint32_t mipLevel) : context(context), detiler(detiler), descriptor(descriptor) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; PhaseTimer timer; @@ -463,6 +600,15 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, guestBytes = geometry.guestBytes; // Storage images in heaps the guest commits on demand only read and store committed pages. Require(!GuestMemory::CommittedRanges(descriptor.baseAddress, static_cast(guestBytes), true).empty(), "storage texture has no committed guest pages"); + // Write stamps are per 64 KiB block, so only 64 KiB-aligned layer slices can be told apart. + const bool layered = geometry.layers >= 2 && !geometry.thick && geometry.imageDepth == 1 && geometry.layers == geometry.imageLayers && geometry.layerBytes * geometry.layers == guestBytes && geometry.layerBytes % 65536 == 0 && descriptor.baseAddress % 65536 == 0; + // Tile blocks divide the stamp blocks, so at an aligned base every unit is whole tile blocks + // of its (layer, mip) slices (sliceWindows). + blockUnits = LayerTrackingEnabled() && BlockTrackingEnabled() && !geometry.thick && descriptor.tileMode != TextureTileMode::kLinear && descriptor.baseAddress % 65536 == 0; + trackedLayers = blockUnits ? static_cast((guestBytes + 65535) / 65536) : layered && LayerTrackingEnabled() ? geometry.layers : 1u; + trackedLayerBytes = blockUnits ? 65536 : guestBytes / trackedLayers; + layerGeneration.assign(trackedLayers, 0); + layerPending.assign(trackedLayers, false); sliceLinearBytes = geometry.sliceLinearBytes; const auto linearBytes = sliceLinearBytes * arrayLayers; @@ -493,14 +639,20 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory storage texture"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage"); + uploadReason = "first"; upload(); defaultMip = mipLevel; view = createView(mipLevel); + { + auto& live = Live(); + std::lock_guard lock(live.mutex); + live.textures.push_back(this); + } if (profile) { auto& totals = Profile(); totals.storageCreate += timer.lap(); totals.storageBytes += guestBytes; - if (++totals.storageCount % 50 == 0) std::fprintf(stderr, "[texture] %llu storage images (%.0f MiB): create %.0f ms, write-back %.0f ms (alloc %.0f, host copy %.0f, gpu %.0f, store %.0f), %llu reused, %llu direct uploads, %llu direct write-backs\n", static_cast(totals.storageCount), totals.storageBytes / 1048576.0, totals.storageCreate, totals.storageWriteBack, totals.storageAlloc, totals.storageHostCopy, totals.storageGpu, totals.storageStore, static_cast(totals.storageReused), static_cast(totals.storageDirectUploads), static_cast(totals.storageDirectWriteBacks)); + if (++totals.storageCount % 50 == 0) std::fprintf(stderr, "[texture] %llu storage images (%.0f MiB): create %.0f ms, write-back %.0f ms (alloc %.0f, host copy %.0f, gpu %.0f, store %.0f), %llu reused, %llu direct uploads, %llu direct write-backs, clears %llu recorded / %llu waited\n", static_cast(totals.storageCount), totals.storageBytes / 1048576.0, totals.storageCreate, totals.storageWriteBack, totals.storageAlloc, totals.storageHostCopy, totals.storageGpu, totals.storageStore, static_cast(totals.storageReused), static_cast(totals.storageDirectUploads), static_cast(totals.storageDirectWriteBacks), static_cast(totals.storageRecordedClears), static_cast(totals.storageWaitedClears)); } } catch (...) { release(); @@ -523,6 +675,158 @@ PendingWrites& Pending() { return pending; } +// See StorageTexture::PendingSerial: bumped after every mutation of the registry above. +std::atomic pendingSerial{0}; + +// The texel layout a clear value is built from (FillClearColor): channels in memory order with +// their bit offset and width inside the element and the clear component they feed (R, G, B, A). +enum class ClearKind { Unorm, Snorm, Uint, Sint, Float, UFloat }; + +struct ClearChannel { + std::uint32_t offset; + std::uint32_t bits; + std::uint32_t component; +}; + +struct ClearLayout { + ClearKind kind; + std::uint32_t channels; + ClearChannel channel[4]; +}; + +bool ClearLayoutFor(VkFormat format, ClearLayout& layout) { + const auto uniform = [&](ClearKind kind, std::uint32_t count, std::uint32_t width, bool reversed = false) { + layout.kind = kind; + layout.channels = count; + for (std::uint32_t i = 0; i < count; ++i) layout.channel[i] = {i * width, width, reversed && i < 3 ? 2u - i : i}; + }; + switch (format) { + case VK_FORMAT_R8_UNORM: uniform(ClearKind::Unorm, 1, 8); return true; + case VK_FORMAT_R8_SNORM: uniform(ClearKind::Snorm, 1, 8); return true; + case VK_FORMAT_R8_UINT: uniform(ClearKind::Uint, 1, 8); return true; + case VK_FORMAT_R8_SINT: uniform(ClearKind::Sint, 1, 8); return true; + case VK_FORMAT_R8G8_UNORM: uniform(ClearKind::Unorm, 2, 8); return true; + case VK_FORMAT_R8G8_SNORM: uniform(ClearKind::Snorm, 2, 8); return true; + case VK_FORMAT_R8G8_UINT: uniform(ClearKind::Uint, 2, 8); return true; + case VK_FORMAT_R8G8_SINT: uniform(ClearKind::Sint, 2, 8); return true; + case VK_FORMAT_R8G8B8A8_UNORM: case VK_FORMAT_A8B8G8R8_UNORM_PACK32: uniform(ClearKind::Unorm, 4, 8); return true; + case VK_FORMAT_R8G8B8A8_SNORM: case VK_FORMAT_A8B8G8R8_SNORM_PACK32: uniform(ClearKind::Snorm, 4, 8); return true; + case VK_FORMAT_R8G8B8A8_UINT: case VK_FORMAT_A8B8G8R8_UINT_PACK32: uniform(ClearKind::Uint, 4, 8); return true; + case VK_FORMAT_R8G8B8A8_SINT: case VK_FORMAT_A8B8G8R8_SINT_PACK32: uniform(ClearKind::Sint, 4, 8); return true; + case VK_FORMAT_B8G8R8A8_UNORM: uniform(ClearKind::Unorm, 4, 8, true); return true; + case VK_FORMAT_B8G8R8A8_SNORM: uniform(ClearKind::Snorm, 4, 8, true); return true; + case VK_FORMAT_B8G8R8A8_UINT: uniform(ClearKind::Uint, 4, 8, true); return true; + case VK_FORMAT_B8G8R8A8_SINT: uniform(ClearKind::Sint, 4, 8, true); return true; + case VK_FORMAT_R16_UNORM: uniform(ClearKind::Unorm, 1, 16); return true; + case VK_FORMAT_R16_SNORM: uniform(ClearKind::Snorm, 1, 16); return true; + case VK_FORMAT_R16_UINT: uniform(ClearKind::Uint, 1, 16); return true; + case VK_FORMAT_R16_SINT: uniform(ClearKind::Sint, 1, 16); return true; + case VK_FORMAT_R16_SFLOAT: uniform(ClearKind::Float, 1, 16); return true; + case VK_FORMAT_R16G16_UNORM: uniform(ClearKind::Unorm, 2, 16); return true; + case VK_FORMAT_R16G16_SNORM: uniform(ClearKind::Snorm, 2, 16); return true; + case VK_FORMAT_R16G16_UINT: uniform(ClearKind::Uint, 2, 16); return true; + case VK_FORMAT_R16G16_SINT: uniform(ClearKind::Sint, 2, 16); return true; + case VK_FORMAT_R16G16_SFLOAT: uniform(ClearKind::Float, 2, 16); return true; + case VK_FORMAT_R16G16B16A16_UNORM: uniform(ClearKind::Unorm, 4, 16); return true; + case VK_FORMAT_R16G16B16A16_SNORM: uniform(ClearKind::Snorm, 4, 16); return true; + case VK_FORMAT_R16G16B16A16_UINT: uniform(ClearKind::Uint, 4, 16); return true; + case VK_FORMAT_R16G16B16A16_SINT: uniform(ClearKind::Sint, 4, 16); return true; + case VK_FORMAT_R16G16B16A16_SFLOAT: uniform(ClearKind::Float, 4, 16); return true; + case VK_FORMAT_R32_UINT: uniform(ClearKind::Uint, 1, 32); return true; + case VK_FORMAT_R32_SINT: uniform(ClearKind::Sint, 1, 32); return true; + case VK_FORMAT_R32_SFLOAT: uniform(ClearKind::Float, 1, 32); return true; + case VK_FORMAT_R32G32_UINT: uniform(ClearKind::Uint, 2, 32); return true; + case VK_FORMAT_R32G32_SINT: uniform(ClearKind::Sint, 2, 32); return true; + case VK_FORMAT_R32G32_SFLOAT: uniform(ClearKind::Float, 2, 32); return true; + case VK_FORMAT_R32G32B32A32_UINT: uniform(ClearKind::Uint, 4, 32); return true; + case VK_FORMAT_R32G32B32A32_SINT: uniform(ClearKind::Sint, 4, 32); return true; + case VK_FORMAT_R32G32B32A32_SFLOAT: uniform(ClearKind::Float, 4, 32); return true; + case VK_FORMAT_A2B10G10R10_UNORM_PACK32: layout = {ClearKind::Unorm, 4, {{0, 10, 0}, {10, 10, 1}, {20, 10, 2}, {30, 2, 3}}}; return true; + case VK_FORMAT_A2B10G10R10_UINT_PACK32: layout = {ClearKind::Uint, 4, {{0, 10, 0}, {10, 10, 1}, {20, 10, 2}, {30, 2, 3}}}; return true; + case VK_FORMAT_A2R10G10B10_UNORM_PACK32: layout = {ClearKind::Unorm, 4, {{0, 10, 2}, {10, 10, 1}, {20, 10, 0}, {30, 2, 3}}}; return true; + case VK_FORMAT_A2R10G10B10_UINT_PACK32: layout = {ClearKind::Uint, 4, {{0, 10, 2}, {10, 10, 1}, {20, 10, 0}, {30, 2, 3}}}; return true; + case VK_FORMAT_B10G11R11_UFLOAT_PACK32: layout = {ClearKind::UFloat, 3, {{0, 11, 0}, {11, 11, 1}, {22, 10, 2}}}; return true; + default: return false; + } +} + +// A float of `bits` (16: IEEE half; 11 and 10: the unsigned 5-bit-exponent floats of +// B10G11R11) as a float32, or nullopt for a NaN, a denormal or (unsigned) an infinity. +std::optional SmallFloat(std::uint32_t code, std::uint32_t bits) { + const std::uint32_t mantissaBits = bits == 16 ? 10u : bits - 5u; + const bool negative = bits == 16 && (code >> 15u) != 0; + const std::uint32_t exponent = (code >> mantissaBits) & 0x1fu; + const std::uint32_t mantissa = code & ((1u << mantissaBits) - 1u); + if (exponent == 31u) { + if (mantissa != 0 || bits != 16) return std::nullopt; + return negative ? -std::numeric_limits::infinity() : std::numeric_limits::infinity(); + } + if (exponent == 0) { + if (mantissa != 0) return std::nullopt; + return negative ? -0.0f : 0.0f; + } + const float value = std::ldexp(1.0f + static_cast(mantissa) / static_cast(1u << mantissaBits), static_cast(exponent) - 15); + return negative ? -value : value; +} + +// One texel of `format` (`elementBytes` wide) taken from a 16-byte fill pattern as the clear value +// that stores exactly those bits, when the pattern is that texel repeated: integer formats take the +// codes as they are, normalized ones the code over the channel's maximum (the conversion back rounds +// to the same code), float ones the decoded value. Codes a clear might store differently are +// refused: NaNs and denormals (an implementation may canonicalize or flush them) and the most +// negative SNORM code (stored as its neighbour, the same -1.0). +bool FillClearColor(VkFormat format, std::uint32_t elementBytes, std::span pattern, VkClearColorValue& clear) { + ClearLayout layout{}; + if (elementBytes == 0 || elementBytes > 16 || 16 % elementBytes != 0 || !ClearLayoutFor(format, layout)) return false; + std::array bytes{}; + std::memcpy(bytes.data(), pattern.data(), bytes.size()); + for (std::size_t at = elementBytes; at < bytes.size(); at += elementBytes) { + if (std::memcmp(bytes.data(), bytes.data() + at, elementBytes) != 0) return false; + } + for (std::uint32_t i = 0; i < layout.channels; ++i) { + const auto& channel = layout.channel[i]; + const auto word = pattern[channel.offset / 32u]; + const auto code = channel.bits == 32 ? word : (word >> (channel.offset % 32u)) & ((1u << channel.bits) - 1u); + const auto signedCode = static_cast(code << (32u - channel.bits)) >> (32u - channel.bits); + switch (layout.kind) { + case ClearKind::Unorm: + clear.float32[channel.component] = static_cast(code) / static_cast((1u << channel.bits) - 1u); + break; + case ClearKind::Snorm: { + const auto maximum = static_cast((1u << (channel.bits - 1u)) - 1u); + if (signedCode < -maximum) return false; + clear.float32[channel.component] = static_cast(signedCode) / static_cast(maximum); + break; + } + case ClearKind::Uint: + clear.uint32[channel.component] = code; + break; + case ClearKind::Sint: + clear.int32[channel.component] = signedCode; + break; + case ClearKind::Float: + if (channel.bits == 32) { + const auto exponent = (code >> 23u) & 0xffu; + const auto mantissa = code & 0x7fffffu; + if ((exponent == 0xffu && mantissa != 0) || (exponent == 0 && mantissa != 0)) return false; + clear.float32[channel.component] = std::bit_cast(code); + } else { + const auto value = SmallFloat(code, channel.bits); + if (!value) return false; + clear.float32[channel.component] = *value; + } + break; + case ClearKind::UFloat: { + const auto value = SmallFloat(code, channel.bits); + if (!value) return false; + clear.float32[channel.component] = *value; + break; + } + } + } + return true; +} + // The image being validated by Refresh: its own pending results are what the next dispatch wants, // so the comparison with guest memory must not flush them. thread_local const StorageTexture* refreshing = nullptr; @@ -586,31 +890,173 @@ VkImageView StorageTexture::View(std::uint32_t mip) { return created; } +const char* LookupOutcomes::Name(Kind kind) { + static constexpr const char* names[Count] = {"sampled fast hit", "sampled fast miss", "sampled hit view", "sampled hit cleared view", "sampled hit snapshot", "sampled made view", "sampled made snapshot", "storage hit", "storage made", "refresh unchanged", "refresh compared", "upload direct", "upload cpu", "upload clear", "dcc scan", "pending flush"}; + return kind < Count ? names[kind] : "?"; +} + +bool LookupOutcomes::Profiled() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + +std::chrono::steady_clock::time_point LookupOutcomes::Add(Kind kind, std::chrono::steady_clock::time_point start) { + const auto now = std::chrono::steady_clock::now(); + auto& outcomes = ThreadLookupOutcomes(); + ++outcomes.counts[kind]; + outcomes.ms[kind] += std::chrono::duration(now - start).count(); + return now; +} + +LookupOutcomes& ThreadLookupOutcomes() { + thread_local LookupOutcomes outcomes; + return outcomes; +} + bool StorageTexture::Refresh() { - // Results of other images pending in this memory must reach it first; this image's own pending - // results stay on the GPU, where the next dispatch wants them. - FlushPending(descriptor.baseAddress, static_cast(guestBytes), this, "storage refresh"); + const bool profile = LookupOutcomes::Profiled(); + auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + struct Exempt { + const StorageTexture* previous; + ~Exempt() { refreshing = previous; } + } exempt{refreshing}; + // This image's own pending results stay on the GPU, where the next dispatch wants them (the + // flush below and the compare through the hook skip it). + refreshing = this; + NoteProved(); + // Results of other images pending in this memory must reach it first, except an alias's: its + // units are taken on the device below (borrowUnits), so it stays pending. The keys read for + // that decision are read again after the flush, which may store keys itself. The exemption + // holds only when every pending unit of the alias is borrowable (tracked, unstamped since its + // generation, no results of this image's own there) or dead to its own write-back (tracked and + // stamped); an untracked unit is stored whole by that write-back, and a unit pending in both + // images takes the old order (store, then this image re-uploads it). + auto alias = pendingAlias(); + if (alias != nullptr && !(ProvedClearKeys(descriptor, guestBytes, keyProof) == DccKeys::Uncompressed && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr)) alias = nullptr; + if (alias != nullptr) { + std::vector aliasStamped(trackedLayers); + GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), alias->layerGeneration, aliasStamped); + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if (!alias->layerPending[unit]) continue; + if (alias->layerGeneration[unit] == 0 || (aliasStamped[unit] == 0 && layerPending[unit])) { + alias = nullptr; + break; + } + } + } + // No publish: the upload below reads the stored units from the unit shadow itself. + const bool flushed = FlushPending(descriptor.baseAddress, static_cast(guestBytes), alias.get(), "storage refresh", PublishScope::None); + if (profile && flushed) start = LookupOutcomes::Add(LookupOutcomes::PendingFlush, start); // `original` holds the guest bytes the image was last uploaded from or written back as; while the // guest memory and the DCC keys still match, the image content is current. Pages nobody wrote // since `generation` need no comparison. const auto current = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); - bool unchanged = false; + // Per tracked layer: changed since its generation (a keys change makes every layer stale). + std::vector changed(trackedLayers, false); + // Per 64 KiB block of the surface (blockGenerations): stamped since its layer's generation, and + // by a CPU store; scanned once when anything changed. + std::vector generations; + std::vector stampedBlocks, cpuBlocks; + // Units taken from the alias's image instead of guest memory. + std::vector borrowed; + bool anyBorrowed = false; + bool unchanged = true; + bool stamped = true; + bool keysChanged = false; + bool cpuWrote = false; + bool direct = false; + DccKeys keys = uploadedKeys; { - struct Exempt { - const StorageTexture* previous; - ~Exempt() { refreshing = previous; } - } exempt{refreshing}; - refreshing = this; const auto equalsOriginal = [&] { // Named for the [hooksync] attribution: the compare goes through the flush hook. const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureCompare); return GuestMemory::EqualsCommitted(descriptor.baseAddress, original); }; - unchanged = TextureClearKeys(descriptor, guestBytes) == uploadedKeys && (GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast(guestBytes), generation) || (originalValid && equalsOriginal())); + const auto scanBlocks = [&] { + blockGenerations(generations); + stampedBlocks.assign(generations.size(), 0); + cpuBlocks.assign(generations.size(), 0); + GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), generations, stampedBlocks, cpuBlocks); + }; + const auto keysStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + keys = ProvedClearKeys(descriptor, guestBytes, keyProof); + if (profile && descriptor.dccAddress != 0) LookupOutcomes::Add(LookupOutcomes::DccScan, keysStart); + if (keys != uploadedKeys) { + changed.assign(trackedLayers, true); + unchanged = false; + keysChanged = true; + // Which units the CPU stamped still decides what the drop rule below may drop. + if (blockUnits) scanBlocks(); + } else if (blockUnits) { + if (!GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast(guestBytes), generation)) { + // One stamp scan for every unit (each is one stamp block); the unchanged units move + // to the current generation, so the whole-surface check answers next time. + scanBlocks(); + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if (stampedBlocks[unit] == 0) { + layerGeneration[unit] = current; + continue; + } + changed[unit] = true; + unchanged = false; + } + refreshGeneration(); + } + } else { + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (GuestMemory::UnchangedSince(layerBegin(layer), static_cast(layerBytes(layer)), layerGeneration[layer])) continue; + changed[layer] = true; + unchanged = false; + } + if (!unchanged) scanBlocks(); + } + // "cpu" names a CPU store in both models; a block without a generation (untracked) says + // nothing about who wrote it. + for (std::size_t k = 0; k < cpuBlocks.size(); ++k) { + if (cpuBlocks[k] != 0 && generations[k] != 0) cpuWrote = true; + } + // Unchanged bytes rescue an upload only when nothing else moved: with the keys changed + // the same bytes read differently, and a changed unit holding pending results must take + // the drop rule below (the image holds the results, `original` what the CPU wrote back). + bool pendingChanged = false; + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (changed[layer] && layerPending[layer]) pendingChanged = true; + } + if (!unchanged && !keysChanged && !pendingChanged && originalValid && equalsOriginal()) { + unchanged = true; + stamped = false; + } + // Only the direct path uploads the selected layers alone (see upload); the others replace + // the whole image, so every pending layer's results are stored first. + direct = keys == DccKeys::Uncompressed && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr; + if (alias != nullptr && direct) { + // The alias's pending units that nothing stamped since its generation (a CPU store or a + // driver store there wins over its results, as at its own write-back) are taken from + // its image: every one of them, since its write-back would have stamped them all. Units + // already holding them at the same alias version stay as they are. + const bool sameBorrow = borrowedFrom.lock() == alias && borrowedVersion == alias->version && borrowedUnits.size() == trackedLayers; + std::vector aliasStamped(trackedLayers); + GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), alias->layerGeneration, aliasStamped); + borrowed.assign(trackedLayers, false); + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if (!alias->layerPending[unit] || aliasStamped[unit] != 0 || alias->layerGeneration[unit] == 0 || layerPending[unit]) continue; + if (sameBorrow && borrowedUnits[unit]) continue; + borrowed[unit] = true; + anyBorrowed = true; + changed[unit] = false; + unchanged = false; + } + if (!sameBorrow) { + borrowedFrom.reset(); + borrowedUnits.clear(); + } + } } if (unchanged) { ++Profile().storageReused; - generation = current; + layerGeneration.assign(trackedLayers, current); + refreshGeneration(); + if (profile) LookupOutcomes::Add(stamped ? LookupOutcomes::RefreshUnchanged : LookupOutcomes::RefreshCompared, start); return true; } // Debug aid: APS5_TRACE_UPLOAD names why a storage image is uploaded again. @@ -619,39 +1065,150 @@ bool StorageTexture::Refresh() { const auto keys = TextureClearKeys(descriptor, guestBytes); std::fprintf(stderr, "[upload] 0x%llx+0x%llx %ux%u mips %u: %s (keys %s -> %s, originalValid %d, dirty %d)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), descriptor.width, descriptor.height, descriptor.mipCount, keys != uploadedKeys ? "DCC keys changed" : "guest memory changed", DccKeysName(uploadedKeys), DccKeysName(keys), originalValid ? 1 : 0, dirty ? 1 : 0); } + // Changed layers with results pending: the CPU wrote this memory while GPU results were + // pending, so as with an immediate write-back followed by the CPU write, its blocks win and the + // results land everywhere else in those layers; the unchanged pending layers keep their + // results on the GPU when the direct path leaves them alone, and are stored otherwise. The + // image stays registered throughout (the store clears what it stored). + std::vector stored(trackedLayers, false); bool pendingResults = false; - { - auto& pending = Pending(); - std::lock_guard lock(pending.mutex); - if (dirty) { - dirty = false; - std::erase(pending.textures, this); - pendingResults = true; + bool anyStored = false; + bool dropped = false; + // A unit is one 64 KiB block, which the CPU's bytes win whole once a store stamped it: + // nothing of its results is stored, and the unit re-uploads from memory below. A keys change + // alone (the title marking the surface uncompressed behind a pass) leaves unstamped units + // with their results, which are stored first, as the layer model does; new keys that are a + // clear code make every result dead on hardware too. Without a generation (no tracking) a + // unit's stamps say nothing, so it is stored. + const auto droppable = [&](std::uint32_t unit) { + if (IsDccClear(keys)) return true; + return layerGeneration[unit] != 0 && unit < stampedBlocks.size() && stampedBlocks[unit] != 0; + }; + // A clear code -> uncompressed flip on an image with results pending: an unstamped pending + // unit's results ARE the uncompressed texels (nothing wrote its memory since), so it keeps + // them instead of being dropped and re-read from stale guest bytes; a stamped one takes the + // drop rule. Debug aid: APS5_KEYFLIP_DROP=1 drops as before. + static const bool keyFlipDrop = std::getenv("APS5_KEYFLIP_DROP") != nullptr; + static const bool traceKeys = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; + const bool keyFlip = keysChanged && blockUnits && !keyFlipDrop && IsDccClear(uploadedKeys) && keys == DccKeys::Uncompressed && stampedBlocks.size() >= trackedLayers; + if (traceKeys && keysChanged) { + std::size_t pendingUnits = 0, stampedUnits = 0; + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if (layerPending[unit]) ++pendingUnits; + if (unit < cpuBlocks.size() && cpuBlocks[unit] != 0 && generations[unit] != 0) ++stampedUnits; } + std::fprintf(stderr, "[dcc-keys] refresh 0x%llx+0x%llx keys %s -> %s (dirty %d, %zu pending units, %zu cpu-stamped units%s)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), DccKeysName(uploadedKeys), DccKeysName(keys), dirty ? 1 : 0, pendingUnits, stampedUnits, keyFlip ? ", flip keeps unstamped results" : ""); + } + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!layerPending[layer]) continue; + if (keyFlip && layerGeneration[layer] != 0 && stampedBlocks[layer] == 0) { + changed[layer] = false; + keyFlipKept.fetch_add(1, std::memory_order_relaxed); + if (direct) continue; + stored[layer] = true; + anyStored = true; + continue; + } + if (changed[layer]) { + pendingResults = true; + if (blockUnits && droppable(layer)) { + layerPending[layer] = false; + dropped = true; + continue; + } + } else if (direct) { + continue; + } + stored[layer] = true; + anyStored = true; } if (pendingResults) { - // The CPU wrote this memory while GPU results were pending: as with an immediate write-back - // followed by the CPU write, its blocks win and the results land everywhere else. static std::atomic reports{0}; if (reports.fetch_add(1) < 8) std::fprintf(stderr, "[gpu] storage image 0x%llx: guest memory changed while GPU results were pending; keeping the CPU's blocks\n", static_cast(descriptor.baseAddress)); - writeBack(generation); } - upload(); + if (anyStored) { + const auto previous = std::exchange(flushReason, "refresh"); + writeBackLayers(stored); + flushReason = previous; + } else if (dropped) { + reconcilePending(); + } + if (anyBorrowed) { + uploadReason = "alias"; + borrowUnits(*alias, borrowed); + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if (borrowed[unit]) layerGeneration[unit] = current; + } + refreshGeneration(); + } + if (std::any_of(changed.begin(), changed.end(), [](bool selected) { return selected; })) { + uploadReason = keysChanged ? "keys" : cpuWrote ? "cpu" : generation == 0 ? "untracked" : flushed ? "flushed" : "store"; + upload(&changed); + } else { + uploadedKeys = keys; + } return false; } -void StorageTexture::upload() { +namespace { + +// Debug aid (APS5_TRACE_DCC_KEYS=1): who stores uncompressed keys over a surface (recommendation +// 2's tracer): the write-back path, the surface and its key range. +void traceKeyStore(const char* path, const GuestTextureResource& descriptor, std::uint64_t guestBytes) { + static const bool trace = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; + if (!trace || descriptor.dccAddress == 0) return; + const auto packet = GuestMemory::CurrentPacket(); + std::fprintf(stderr, "[dcc-keys] uncompressed store by %s (%s) for surface 0x%llx+0x%llx: keys 0x%llx+0x%llx (packet 0x%x queue 0x%x)\n", path, flushReason != nullptr ? flushReason : "?", static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(descriptor.dccAddress), static_cast(guestBytes / 256), packet.opcode, packet.queue); +} + +} + +void StorageTexture::upload(const std::vector* layers) { + const bool profile = LookupOutcomes::Profiled(); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; const auto elementBytes = BytesPerElement(descriptor.format); const auto linearBytes = sliceLinearBytes * arrayLayers; original.resize(static_cast(guestBytes)); - generation = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); - uploadedKeys = TextureClearKeys(descriptor, guestBytes); + // A layer selection applies to the direct path alone (the others upload everything); it names + // array layers, which the tracked layers are when there are several. + if (layers != nullptr && ((!blockUnits && trackedLayers != arrayLayers) || std::all_of(layers->begin(), layers->end(), [](bool selected) { return selected; }))) layers = nullptr; + const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + const auto stampLayers = [&](bool selectedOnly) { + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!selectedOnly || layers == nullptr || (*layers)[layer]) layerGeneration[layer] = now; + } + refreshGeneration(); + }; + uploadedKeys = ProvedClearKeys(descriptor, guestBytes, keyProof); VkClearColorValue clearValue{}; if (IsDccClear(uploadedKeys) && ClearColorFor(storageFormat, uploadedKeys, clearValue)) { // A fast-cleared surface is cleared on the GPU; its texel memory is neither read nor filled. + // The clear is recorded into the open batch like a direct upload (the image kept by it): a + // batch of its own submitted the recorder's work first and waited for all of it, 25-40 ms + // under the GPU mutex at the movie stage. APS5_NO_RECORDED_CLEAR=1 waits as before. originalValid = false; - CommandBatch batch(context); - const auto commands = batch.Handle(); + forgetBorrowed(0, trackedLayers); + stampLayers(false); + static const bool recordClear = std::getenv("APS5_NO_RECORDED_CLEAR") == nullptr; + auto* recorder = recordClear ? Recorder::Active() : nullptr; + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + auto timing = Recorder::NoTiming; + if (recorder != nullptr) { + commands = recorder->Commands(); + timing = recorder->BeginGpuTiming(Recorder::CommandClass::DccClear); + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + ++Profile().storageRecordedClears; + Recorder::CountBarriers(Recorder::CommandClass::DccClear, 2); + if (Recorder::BarrierValidate()) { + const std::pair cleared{image, true}; + recorder->NoteAccess(Recorder::CommandClass::DccClear, Recorder::Access{{}, {}, std::span(&cleared, 1), VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + ++Profile().storageWaitedClears; + } VkImageMemoryBarrier toClear{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; toClear.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; toClear.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; @@ -669,24 +1226,74 @@ void StorageTexture::upload() { toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); - batch.SubmitAndWait(); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, guestBytes); + countStorageUpload(0, guestBytes); ++version; + if (profile) LookupOutcomes::Add(LookupOutcomes::UploadClear, start); return; } if (const auto* import = uploadedKeys == DccKeys::Uncompressed ? HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) : nullptr) { // The surface lives in host-imported memory: the detiler reads it in place, no guest bytes // are copied, and write tracking alone validates the image (a change re-runs this). originalValid = false; + stampLayers(true); + // A whole-surface upload of a block-unit image goes through the windows too when a unit + // shadow holds part of the surface (the detile then reads the slabs; a new image has no + // layout yet); a partial one always does. + if (blockUnits && (layers != nullptr || AnyShadowedOverlaps(descriptor.baseAddress, static_cast(guestBytes)))) { + std::vector> runs; + if (layers != nullptr) { + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if ((*layers)[unit]) forgetBorrowed(unit, 1); + } + runs = unitRuns(*layers); + } else { + forgetBorrowed(0, trackedLayers); + runs.emplace_back(0, guestBytes); + } + const auto uploadedBytes = uploadWindows(*import, runs, layers == nullptr && version == 0); + countStorageUpload(1, uploadedBytes); + if (layers != nullptr) { + partialUploads.fetch_add(1, std::memory_order_relaxed); + partialUploadBytes.fetch_add(uploadedBytes, std::memory_order_relaxed); + } + ++Profile().storageDirectUploads; + ++version; + if (profile) LookupOutcomes::Add(LookupOutcomes::UploadDirect, start); + return; + } + // Units another image's results shadow reach the import before the detile reads it in place. + PublishShadow(descriptor.baseAddress, static_cast(guestBytes), PublishScope::Whole, PublishReason::Upload); + forgetBorrowed(0, trackedLayers); + std::uint64_t uploadedBytes = layers == nullptr ? guestBytes : 0; + if (layers != nullptr) { + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if ((*layers)[layer]) uploadedBytes += trackedLayerBytes; + } + } auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; VkCommandBuffer commands = VK_NULL_HANDLE; + auto timing = Recorder::NoTiming; if (recorder != nullptr) { + // A queued label store in the surface lands before the detile reads it. + recorder->FlushStoresOverlapping(descriptor.baseAddress, static_cast(guestBytes)); commands = recorder->Commands(); + timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); recorder->Keep(linear); // The image itself must outlive the recorded copy: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + // The detile reads the tiled bytes from the import when the batch runs. + recorder->NotePendingRead(descriptor.baseAddress, static_cast(guestBytes), Recorder::ReadKind::StorageUpload); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3); + if (Recorder::BarrierValidate()) { + const std::pair read{descriptor.baseAddress, descriptor.baseAddress + guestBytes}; + const std::pair written{image, true}; + recorder->NoteAccess(Recorder::CommandClass::StorageUpload, Recorder::Access{std::span(&read, 1), {}, std::span(&written, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); + } } else { batch = std::make_unique(context); commands = batch->Handle(); @@ -694,21 +1301,24 @@ void StorageTexture::upload() { const auto importOffset = descriptor.baseAddress - import->base; RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (layers != nullptr && !(*layers)[layer]) continue; for (const auto& mip : mips) { detiler.Dispatch(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick); } } const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.srcAccessMask = layers != nullptr ? VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT : 0u; toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - toTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + // Some layers keep their content: the others are replaced, not the whole image. + toTransfer.oldLayout = layers != nullptr ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED; toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toTransfer.image = image; toTransfer.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); - const auto regions = CopyRegions(); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); + const auto regions = CopyRegions(layers); context.Function("vkCmdCopyBufferToImage")(commands, linear->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); VkImageMemoryBarrier toGeneral = toTransfer; toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; @@ -717,11 +1327,16 @@ void StorageTexture::upload() { toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, uploadedBytes); + countStorageUpload(1, uploadedBytes); ++Profile().storageDirectUploads; ++version; + if (profile) LookupOutcomes::Add(LookupOutcomes::UploadDirect, start); return; } originalValid = true; + forgetBorrowed(0, trackedLayers); + stampLayers(false); GuestMemory::ReadCommitted(descriptor.baseAddress, original); { Buffer staging(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); @@ -762,12 +1377,407 @@ void StorageTexture::upload() { batch.SubmitAndWait(); APS5_LOG_CHARS_OUT("StorageTexture upload done"); } + countStorageUpload(2, guestBytes); ++version; + if (profile) LookupOutcomes::Add(LookupOutcomes::UploadCpu, start); } -std::vector StorageTexture::CopyRegions() const { +std::vector> StorageTexture::unitRuns(const std::vector& units) const { + std::vector> runs; + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if (!units[unit]) continue; + const auto begin = static_cast(unit) * trackedLayerBytes; + const auto end = begin + layerBytes(unit); + if (!runs.empty() && runs.back().second == begin) runs.back().second = end; + else runs.emplace_back(begin, end); + } + return runs; +} + +std::vector StorageTexture::sliceWindows(std::span> runs) const { + std::vector windows; + const auto elementBytes = BytesPerElement(descriptor.format); + const auto block = ThinBlockLayout(descriptor.tileMode, elementBytes); + for (const auto& [runBegin, runEnd] : runs) { + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) { + const auto& mip = mips[level]; + const auto sliceBegin = geometry.GuestLayerOffset(layer) + mip.tiledOffset; + const auto sliceEnd = sliceBegin + mip.tiledSize; + if (runEnd <= sliceBegin || sliceEnd <= runBegin) continue; + SliceWindow window{layer, level, 0, 0, {}, 0, {}}; + const auto pitch = static_cast(mip.pitchBytes / elementBytes * BlockWidth(descriptor.format)); + if (mip.tail) { + // The tail block holds every tail level: each is moved whole. + window.tiledBegin = sliceBegin; + window.tiledEnd = sliceEnd; + window.window = {0, static_cast(mip.tiledSize), 0, 0, mip.linearSize}; + window.linearBytes = mip.linearSize; + VkBufferImageCopy region{}; + region.bufferRowLength = pitch; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, geometry.CopyLayer(layer), 1}; + region.imageOffset = {0, 0, geometry.CopyDepth(layer)}; + region.imageExtent = {mip.width, mip.height, 1u}; + window.regions.push_back(region); + windows.push_back(std::move(window)); + continue; + } + const auto low = std::max(runBegin, sliceBegin) - sliceBegin; + const auto high = std::min(runEnd, sliceEnd) - sliceBegin; + Require(low % block[0] == 0 && high % block[0] == 0, "storage image window is not tile-block aligned"); + const auto firstBlock = low / block[0]; + const auto lastBlock = (high - 1) / block[0]; + const auto firstRow = static_cast(firstBlock / mip.blocksPerRow); + const auto lastRow = static_cast(lastBlock / mip.blocksPerRow); + const auto rowBegin = firstRow * block[2]; + const auto rowEnd = std::min(mip.height, (lastRow + 1u) * block[2]); + window.tiledBegin = sliceBegin + low; + window.tiledEnd = sliceBegin + high; + window.linearBytes = static_cast(rowEnd - rowBegin) * mip.pitchBytes; + window.window = {static_cast(low), static_cast(high), static_cast(low), static_cast(rowBegin) * mip.pitchBytes, window.linearBytes}; + // The dispatched rectangle: the block rows, and inside one block row the block span. + window.window.rowBegin = rowBegin; + window.window.rowEnd = rowEnd; + if (firstRow == lastRow) { + window.window.columnBegin = static_cast(firstBlock - static_cast(firstRow) * mip.blocksPerRow) * block[1]; + window.window.columnEnd = std::min(mip.width, static_cast(lastBlock - static_cast(firstRow) * mip.blocksPerRow + 1u) * block[1]); + } + // One region per block row; whole rows merge (a run's middle rows are whole). + for (auto row = firstRow; row <= lastRow; ++row) { + const auto rowFirst = std::max(firstBlock, static_cast(row) * mip.blocksPerRow); + const auto rowLast = std::min(lastBlock, static_cast(row + 1u) * mip.blocksPerRow - 1u); + const auto x0 = static_cast(rowFirst - static_cast(row) * mip.blocksPerRow) * block[1]; + const auto x1 = std::min(mip.width, static_cast(rowLast - static_cast(row) * mip.blocksPerRow + 1u) * block[1]); + const auto y0 = row * block[2]; + const auto y1 = std::min(mip.height, (row + 1u) * block[2]); + if (x0 >= x1 || y0 >= y1) continue; + if (!window.regions.empty()) { + auto& previous = window.regions.back(); + const bool wholeRows = previous.imageOffset.x == 0 && previous.imageExtent.width == mip.width && x0 == 0 && x1 == mip.width; + if (wholeRows && static_cast(previous.imageOffset.y) + previous.imageExtent.height == y0) { + previous.imageExtent.height += y1 - y0; + continue; + } + } + VkBufferImageCopy region{}; + region.bufferOffset = static_cast(y0 - rowBegin) * mip.pitchBytes + static_cast(x0) * elementBytes; + region.bufferRowLength = pitch; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, geometry.CopyLayer(layer), 1}; + region.imageOffset = {static_cast(x0), static_cast(y0), geometry.CopyDepth(layer)}; + region.imageExtent = {x1 - x0, y1 - y0, 1u}; + window.regions.push_back(region); + } + windows.push_back(std::move(window)); + } + } + } + return windows; +} + +std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span> runs, bool discard) { + const auto elementBytes = BytesPerElement(descriptor.format); + // Each run's pieces by source (a unit shadow's slab while fresh, else the import), the tail + // blocks moved whole by one window each; the windows are built per piece. + std::vector> tailBlocks; + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) { + const auto& mip = mips[level]; + if (!mip.tail) continue; + const auto sliceBegin = geometry.GuestLayerOffset(layer) + mip.tiledOffset; + if (tailBlocks.empty() || tailBlocks.back().first != sliceBegin) tailBlocks.emplace_back(sliceBegin, sliceBegin + mip.tiledSize); + } + } + const auto sources = ShadowSources(context, import, descriptor.baseAddress, runs, tailBlocks); + std::vector> pieces; + bool allShadow = true; + for (const auto& source : sources) { + pieces.emplace_back(source.begin, source.end); + if (!source.shadow) allShadow = false; + } + if (allShadow && !sources.empty() && (std::strcmp(uploadReason, "flushed") == 0 || std::strcmp(uploadReason, "store") == 0)) uploadReason = "shadow"; + auto windows = sliceWindows(pieces); + Require(!windows.empty(), "storage image upload selects no unit"); + const auto sourceOf = [&](const SliceWindow& window) -> const ShadowRun& { + for (const auto& source : sources) { + if (source.begin <= window.tiledBegin && window.tiledBegin < source.end) return source; + } + throw std::runtime_error("AGC graphics: storage image window has no source"); + }; + // One linear buffer for every window, each at a 256-byte position (the detiler's descriptor + // offsets are aligned by the dispatch). + std::vector positions; + std::uint64_t linearTotal = 0, uploadedBytes = 0; + for (const auto& window : windows) { + positions.push_back(linearTotal); + linearTotal += (window.linearBytes + 255) & ~std::uint64_t{255}; + } + for (const auto& [begin, end] : runs) uploadedBytes += end - begin; + auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + detiler.BeginBatch(); + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + auto timing = Recorder::NoTiming; + if (recorder != nullptr) { + // The import pieces are read in place when the batch runs; the slab pieces are the + // device's own. + std::vector> reads; + for (const auto& source : sources) { + if (!source.shadow) reads.emplace_back(descriptor.baseAddress + source.begin, descriptor.baseAddress + source.end); + } + // A queued label store in a run lands before the detile reads it. + for (const auto& [begin, end] : reads) recorder->FlushStoresOverlapping(begin, static_cast(end - begin)); + commands = recorder->Commands(); + timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); + recorder->Keep(linear); + for (const auto& source : sources) { + if (source.slab != nullptr) recorder->Keep(source.slab); + } + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + if (!reads.empty()) recorder->NotePendingReads(reads, Recorder::ReadKind::StorageUpload); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3); + if (Recorder::BarrierValidate()) { + const std::pair written{image, true}; + recorder->NoteAccess(Recorder::CommandClass::StorageUpload, Recorder::Access{reads, {}, std::span(&written, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } + const auto importOffset = descriptor.baseAddress - import.base; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + std::vector regions; + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + const auto& mip = mips[window.level]; + const auto& source = sourceOf(window); + auto detile = window.window; + if (source.shadow) { + // The slab holds the piece from the window's first tiled byte: the window's tiled base + // is its range begin (sliceWindows builds it so), read from the piece's slab offset. + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, source.buffer, source.offset + (window.tiledBegin - source.begin), linear->Handle(), positions[i], mip, false, window.layer, false, detile); + } else { + // The import holds the whole mip: the window's tiled base is the mip's. + detile.tiledBase = 0; + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, import.buffer, importOffset + geometry.GuestLayerOffset(window.layer) + mip.tiledOffset, linear->Handle(), positions[i], mip, false, window.layer, false, detile); + } + for (auto region : window.regions) { + region.bufferOffset += positions[i]; + regions.push_back(region); + } + } + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.srcAccessMask = discard ? 0u : VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + // The other blocks keep their content (a new image, every block of which is written, has none). + toTransfer.oldLayout = discard ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_GENERAL; + toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = image; + toTransfer.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); + context.Function("vkCmdCopyBufferToImage")(commands, linear->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + VkImageMemoryBarrier toGeneral = toTransfer; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, uploadedBytes); + return uploadedBytes; +} + +std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::span> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector& shadowed, std::vector>& imported) { + const auto elementBytes = BytesPerElement(descriptor.format); + std::vector> runs; + std::uint64_t storedBytes = 0; + for (const auto& [from, to] : keep) { + runs.emplace_back(from - descriptor.baseAddress, to - descriptor.baseAddress); + storedBytes += to - from; + } + if (runs.empty()) return 0; + auto windows = sliceWindows(runs); + // The linear rows and the tiled bytes of every window, each at a 256-byte position; the tail + // levels of a layer retile into one scratch region (their block is shared). + std::vector linearPositions, scratchPositions; + std::uint64_t linearTotal = 0, scratchTotal = 0; + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + linearPositions.push_back(linearTotal); + linearTotal += (window.linearBytes + 255) & ~std::uint64_t{255}; + std::size_t shared = 0; + while (shared < i && (windows[shared].tiledBegin != window.tiledBegin || windows[shared].tiledEnd != window.tiledEnd)) ++shared; + if (shared < i) { + scratchPositions.push_back(scratchPositions[shared]); + continue; + } + scratchPositions.push_back(scratchTotal); + scratchTotal += (window.tiledEnd - window.tiledBegin + 255) & ~std::uint64_t{255}; + } + // The kept ranges inside each window, from the window's scratch (a shared tail region once) + // to their destination: the import's unit shadow where a slab takes the piece (split at slab + // boundaries; units a piece covers partly and does not already hold fresh are seeded from the + // import first), else the import as before. Decided before anything is recorded: making a + // slab may evict another with a publish of its own (a chosen slab stays pinned by its + // destination until the pieces are marked, so no later choice evicts it). + struct SlabPieces { + std::shared_ptr slab; + std::shared_ptr pin; + std::vector copies; + }; + std::vector slabPieces; + std::vector importCopies; + std::vector seeds; + const auto addPiece = [&](std::uint64_t scratchOffset, std::uint64_t guestBegin, std::uint64_t guestEnd) { + while (guestBegin < guestEnd) { + const auto pieceEnd = UnitShadowEnabled() ? std::min(guestEnd, SlabBoundary(import, guestBegin)) : guestEnd; + const auto destination = ShadowDestinationFor(context, import, guestBegin, pieceEnd); + if (destination.has_value()) { + if (slabPieces.empty() || slabPieces.back().slab != destination->slab) slabPieces.push_back({destination->slab, destination->pin, {}}); + slabPieces.back().copies.push_back({scratchOffset, destination->offset, pieceEnd - guestBegin}); + for (const auto& [seedBegin, seedEnd] : destination->seedUnits) { + if (std::none_of(seeds.begin(), seeds.end(), [&](const ShadowedRange& seed) { return seed.begin == seedBegin; })) seeds.push_back({seedBegin, seedEnd, destination->slab}); + } + if (!shadowed.empty() && shadowed.back().end == guestBegin && shadowed.back().slab == destination->slab) shadowed.back().end = pieceEnd; + else shadowed.push_back({guestBegin, pieceEnd, destination->slab}); + } else { + importCopies.push_back({scratchOffset, guestBegin - import.base, pieceEnd - guestBegin}); + if (!imported.empty() && imported.back().second == guestBegin) imported.back().second = pieceEnd; + else imported.emplace_back(guestBegin, pieceEnd); + } + scratchOffset += pieceEnd - guestBegin; + guestBegin = pieceEnd; + } + }; + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + // The tail levels follow each other, sharing the previous one's region. + if (i != 0 && scratchPositions[i] == scratchPositions[i - 1]) continue; + for (const auto& [from, to] : runs) { + const auto begin = std::max(from, window.tiledBegin); + const auto end = std::min(to, window.tiledEnd); + if (begin >= end) continue; + addPiece(scratchPositions[i] + (begin - window.tiledBegin), descriptor.baseAddress + begin, descriptor.baseAddress + end); + } + } + std::sort(seeds.begin(), seeds.end(), [](const ShadowedRange& a, const ShadowedRange& b) { return a.begin < b.begin; }); + // A queued label or key store inside a seeded unit lands before the seed copies the import. + auto flushBegin = firstStored; + auto flushEnd = lastStored; + for (const auto& seed : seeds) { + flushBegin = std::min(flushBegin, seed.begin); + flushEnd = std::max(flushEnd, seed.end); + } + auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + auto tiledScratch = std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + detiler.BeginBatch(); + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + VkAccessFlags covered = 0; + auto timing = Recorder::NoTiming; + if (recorder != nullptr) { + recorder->FlushKeyStoresOverlapping(flushBegin, static_cast(flushEnd - flushBegin)); + recorder->FlushStoresOverlapping(flushBegin, static_cast(flushEnd - flushBegin)); + commands = recorder->Commands(&covered); + timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); + recorder->Keep(linear); + recorder->Keep(tiledScratch); + for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4); + if (Recorder::BarrierValidate()) { + const std::pair read{image, false}; + std::vector> seedReads; + for (const auto& seed : seeds) seedReads.emplace_back(seed.begin, seed.end); + recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{seedReads, imported, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } + VkImageMemoryBarrier toSource{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toSource.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toSource.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toSource.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.image = image; + toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + if (!seeds.empty()) { + // The seeded units' import bytes (every earlier writer of them, host stores included) + // precede the seed copies; the previous command's trailing barrier may have covered that. + constexpr VkAccessFlags transferAccess = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + if (recorder != nullptr && (covered & transferAccess) == transferAccess && Recorder::MergeBarriers()) { + Recorder::CountMerged(Recorder::CommandClass::StorageWriteBack); + } else { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, transferAccess); + Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack); + } + for (const auto& [begin, end, slab] : seeds) { + // The piece's own slab (a seeded unit is one the piece covers partly). + const VkBufferCopy seed{begin - import.base, SlabOffset(import, *slab, begin), end - begin}; + context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, import.buffer, slab->buffer, 1, &seed); + NoteShadowSeed(begin, end); + } + } + std::vector regions; + for (std::size_t i = 0; i < windows.size(); ++i) { + for (auto region : windows[i].regions) { + region.bufferOffset += linearPositions[i]; + regions.push_back(region); + } + } + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, window.window); + } + { + // The retiled scratch (and a seed's slab bytes, which the scratch copies overwrite in + // part) precede the copies into the slabs and the import. + const VkMemoryBarrier scratchDone{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &scratchDone, 0, nullptr, 0, nullptr); + const auto copyBuffer = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); + for (const auto& pieces : slabPieces) copyBuffer(commands, tiledScratch->Handle(), pieces.slab->buffer, static_cast(pieces.copies.size()), pieces.copies.data()); + if (!importCopies.empty()) copyBuffer(commands, tiledScratch->Handle(), import.buffer, static_cast(importCopies.size()), importCopies.data()); + } + VkImageMemoryBarrier backToGeneral = toSource; + backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + backToGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + backToGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + backToGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + // Transfer writes are in the destination mask too: a following publish or detile of the slabs + // merges its leading barrier. + constexpr VkAccessFlags storedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT; + const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, storedAccess}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral); + if (batch) { + batch->SubmitAndWait(); + } else { + recorder->EndGpuTiming(timing, storedBytes); + recorder->MarkCovered(storedAccess); + // Only the pieces written into the import are stores a CPU reader must wait for; the + // shadowed ones reach it through a publish, which notes its own. + if (!imported.empty()) recorder->NotePendingWrites(imported); + } + partialWriteBacks.fetch_add(1, std::memory_order_relaxed); + partialWriteBackBytes.fetch_add(storedBytes, std::memory_order_relaxed); + return storedBytes; +} + +std::vector StorageTexture::CopyRegions(const std::vector* layers) const { std::vector regions; for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (layers != nullptr && !(*layers)[layer]) continue; for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) { const auto& mip = mips[level]; VkBufferImageCopy region{}; @@ -787,7 +1797,35 @@ bool StorageTexture::overlaps(std::uint64_t address, std::size_t bytes) const { } void StorageTexture::MarkDirty() { + markLayersPending(0, trackedLayers); +} + +bool StorageTexture::anyLayerPending() const { + return std::any_of(layerPending.begin(), layerPending.end(), [](bool pending) { return pending; }); +} + +void StorageTexture::refreshGeneration() { + generation = *std::min_element(layerGeneration.begin(), layerGeneration.end()); +} + +void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count) { static const bool eager = std::getenv("APS5_EAGER_WRITEBACK") != nullptr || std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + for (std::uint32_t layer = first; layer < first + count; ++layer) layerPending[layer] = true; + // Results of this image now cover the alias's results borrowed into these units: this image + // stores them, the alias no longer has to (its results, unchanged since the borrow, are in + // this image's content). An alias written since keeps its own newer results pending. + if (auto source = borrowedFrom.lock(); source != nullptr && borrowedUnits.size() == trackedLayers) { + bool superseded = false; + for (std::uint32_t unit = first; unit < first + count; ++unit) { + if (!borrowedUnits[unit]) continue; + borrowedUnits[unit] = false; + if (source->version != borrowedVersion || !source->layerPending[unit]) continue; + source->layerPending[unit] = false; + unitsSuperseded.fetch_add(1, std::memory_order_relaxed); + superseded = true; + } + if (superseded) source->reconcilePending(); + } if (eager) { WriteBack(); return; @@ -804,24 +1842,64 @@ void StorageTexture::MarkDirty() { static_cast(hooked); } ++version; - if (dirty) return; + const bool wasLent = std::exchange(lent, false); + if (dirty) { + if (wasLent) BumpPendingSerial(); + return; + } dirty = true; pending.textures.push_back(this); + BumpPendingSerial(); +} + +void StorageTexture::reconcilePending() { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + const bool any = anyLayerPending(); + if (any == dirty) return; + dirty = any; + if (any) pending.textures.push_back(this); + else std::erase(pending.textures, this); + BumpPendingSerial(); } void StorageTexture::Flush() { + { + auto& live = Live(); + std::lock_guard lock(live.mutex); + released = true; + } { auto& pending = Pending(); std::lock_guard lock(pending.mutex); if (!dirty) return; dirty = false; std::erase(pending.textures, this); + BumpPendingSerial(); } std::lock_guard gpu(GuestMemory::GpuMutex()); - writeBack(generation); + const auto previous = std::exchange(flushReason, "cache eviction"); + writeBack(descriptor.baseAddress, static_cast(guestBytes)); + flushReason = previous; } -bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason) { +// Whether a pending unit of the image lies inside the range (writeBack's own selection): an image +// listed by FlushPending without one stored nothing and only cost the GPU mutex. +bool StorageTexture::pendingUnitInside(std::uint64_t address, std::size_t bytes) const { + static const bool pretest = std::getenv("APS5_NO_FLUSH_PRETEST") == nullptr; + if (!pretest) return true; + const bool whole = descriptor.dccAddress != 0 && uploadedKeys != DccKeys::Uncompressed; + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!layerPending[layer]) continue; + if (whole) return true; + const auto begin = layerBegin(layer); + if (address < begin + layerBytes(layer) && begin < address + bytes) return true; + } + return false; +} + +bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason, PublishScope scope, bool* published) { + if (published != nullptr) *published = false; // The images stay alive across the scan: their last owner may be a batch's kept list, which // another thread releases outside the GPU mutex once the batch completed. std::vector> flush; @@ -831,7 +1909,16 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons for (auto it = pending.textures.begin(); it != pending.textures.end();) { auto* texture = *it; if (texture != except && texture != refreshing && texture->overlaps(address, bytes)) { + if (!texture->pendingUnitInside(address, bytes)) { + pretestSkipped.fetch_add(1, std::memory_order_relaxed); + ++it; + continue; + } texture->dirty = false; + if (const auto skips = texture->hookSkips.exchange(0, std::memory_order_relaxed); skips != 0 && texture->skippedResultsInside(address, bytes)) { + flushedAfterSkip.fetch_add(1, std::memory_order_relaxed); + if (PublishReasonFor(reason) == PublishReason::Hook && texture->hookSkipSite.load(std::memory_order_relaxed) == static_cast(GuestMemory::CurrentReadSite())) flushedAfterSkipSameSite.fetch_add(1, std::memory_order_relaxed); + } if (auto alive = texture->weak_from_this().lock()) flush.push_back(std::move(alive)); it = pending.textures.erase(it); } else { @@ -840,14 +1927,22 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons } // Still pending for the adjacency rule of writeBack until each one is stored. for (const auto& texture : flush) pending.flushing.push_back(texture.get()); + if (!flush.empty()) BumpPendingSerial(); + } + if (flush.empty()) { + // Units shadowed over the range (another image's results retiled into the import's unit + // shadow, or an earlier flush's) still reach the import as the scope asks. + const bool units = PublishShadowsOnly(address, bytes, scope, reason); + if (published != nullptr) *published = units; + return false; } - if (flush.empty()) return false; struct Unregister { const std::vector>& flush; ~Unregister() { auto& pending = Pending(); std::lock_guard lock(pending.mutex); for (const auto& texture : flush) std::erase(pending.flushing, texture.get()); + BumpPendingSerial(); } } unregister{flush}; // Debug aid: APS5_TRACE_FLUSH names what forces pending results to guest memory. @@ -857,17 +1952,33 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons } std::lock_guard gpu(GuestMemory::GpuMutex()); std::exception_ptr failure; + const auto previousReason = std::exchange(flushReason, reason); for (const auto& texture : flush) { try { - texture->writeBack(texture->generation); + // Only the pending layers the access overlaps are stored; the others stay pending (the + // store re-registers the image for them). + texture->writeBack(address, bytes); } catch (...) { if (!failure) failure = std::current_exception(); } } + flushReason = previousReason; if (failure) std::rethrow_exception(failure); + // The stores above went into the unit shadows where slabs took them: the consumer reads the + // import, so they are published under the same hold, after every store. + if (scope != PublishScope::None) { + const auto units = PublishShadow(address, bytes, scope, PublishReasonFor(reason)); + if (published != nullptr) *published = units != 0; + } return true; } +bool StorageTexture::PublishShadowsOnly(std::uint64_t address, std::size_t bytes, PublishScope scope, const char* reason) { + if (scope == PublishScope::None || !AnyShadowedOverlaps(address, bytes)) return false; + std::lock_guard gpu(GuestMemory::GpuMutex()); + return PublishShadow(address, bytes, scope, PublishReasonFor(reason)) != 0; +} + void StorageTexture::FlushAllPending(const char* reason) { static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason)); } @@ -895,35 +2006,637 @@ bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const Stor return false; } +bool StorageTexture::AnyPendingOverlaps(std::span> ranges) { + if (ranges.empty()) return false; + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + const auto overlapsAny = [&](const StorageTexture* texture) { + for (const auto& [begin, end] : ranges) { + if (end > begin && texture->overlaps(begin, static_cast(end - begin))) return true; + } + return false; + }; + for (const auto* texture : pending.textures) { + if (overlapsAny(texture)) return true; + } + for (const auto* texture : pending.flushing) { + if (overlapsAny(texture)) return true; + } + return false; +} + +std::uint64_t StorageTexture::PendingSerial() { + return pendingSerial.load(std::memory_order_acquire); +} + +void StorageTexture::NoteProved() const { + provedPresent.store(Recorder::Presents(), std::memory_order_relaxed); +} + +std::vector> StorageTexture::overlappingPending(std::uint64_t address, std::size_t bytes) { + // FlushPending's listing; the images stay alive across the caller's scan as there. + std::vector> overlapping; + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (auto* texture : pending.textures) { + if (texture == refreshing || !texture->overlaps(address, bytes) || !texture->pendingUnitInside(address, bytes)) continue; + if (auto alive = texture->weak_from_this().lock()) overlapping.push_back(std::move(alive)); + } + return overlapping; +} + +bool StorageTexture::blocksKept(std::span> images, std::uint64_t address, std::size_t bytes) { + // The stamps are read outside the registry mutex (the tracker lock is never taken under it); + // per pending layer, as the keep decision of writeBackLayers reads them. + constexpr std::uint64_t block = 65536; + const auto accessEnd = address + bytes; + for (const auto& texture : images) { + for (std::uint32_t layer = 0; layer < texture->trackedLayers; ++layer) { + if (!texture->layerPending[layer]) continue; + const auto begin = std::max(address, texture->layerBegin(layer)); + const auto end = std::min(accessEnd, texture->layerBegin(layer) + texture->layerBytes(layer)); + if (end <= begin) continue; + if (texture->layerGeneration[layer] == 0) return false; + for (auto at = begin & ~(block - 1); at < end; at += block) { + const auto from = std::max(at, begin); + const auto to = std::min(at + block, end); + if (GuestMemory::UnchangedSince(from, static_cast(to - from), texture->layerGeneration[layer])) return false; + } + } + } + return true; +} + +void StorageTexture::ClassifyAccess(std::uint64_t address, std::size_t bytes, AccessClassification& out) { + out = {}; + const auto overlapping = overlappingPending(address, bytes); + if (overlapping.empty()) return; + out.images = overlapping.size(); + const auto present = Recorder::Presents(); + for (const auto& texture : overlapping) { + if (texture->provedPresent.load(std::memory_order_relaxed) + DeadImagePresents <= present) ++out.dead; + else ++out.live; + } + if (bytes > ClassifyLimit) return; + out.checked = true; + GuestMemory::CollectWritesUncached(address, bytes); + out.allCpuWritten = blocksKept(overlapping, address, bytes); +} + +bool StorageTexture::AccessKeptByCpu(std::uint64_t address, std::size_t bytes, std::size_t* images, std::size_t* evicted) { + if (evicted != nullptr) *evicted = 0; + const auto overlapping = overlappingPending(address, bytes); + if (images != nullptr) *images = overlapping.size(); + if (overlapping.empty() || bytes > ClassifyLimit) return false; + // The keep decision of a store sees every CPU write to the range up to now (an uncached + // collect, as writeBackLayers makes it: a memo hit could leave the store's own faults for the + // next walk); so must this one. + GuestMemory::CollectWritesUncached(address, bytes); + // A neighbour's write-back stamps the 64 KiB block its surface shares with one of these images + // and then advances that image's layer generation past the stamp (advanceAdjacent), both under + // the GpuMutex; read between the two, the stamp would pass for a CPU write and the access + // would read the image's stale bytes in the block. Under the mutex the layer state is what a + // store there would see, and the stamps only grow. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); + std::lock_guard gpu(GuestMemory::GpuMutex()); + if (!blocksKept(overlapping, address, bytes)) return false; + const auto site = static_cast(GuestMemory::CurrentReadSite()); + std::size_t dropped = 0; + for (const auto& texture : overlapping) { + if (texture->evictStale(address, bytes, dropped)) continue; + texture->hookSkips.fetch_add(1, std::memory_order_relaxed); + texture->hookSkipSite.store(site, std::memory_order_relaxed); + for (std::uint32_t layer = 0; layer < texture->trackedLayers; ++layer) { + if (!texture->layerPending[layer] || address >= texture->layerBegin(layer) + texture->layerBytes(layer) || texture->layerBegin(layer) >= address + bytes) continue; + texture->hookSkipLayer = layer; + texture->hookSkipVersion = texture->version; + texture->hookSkipGeneration = texture->layerGeneration[layer]; + break; + } + } + if (evicted != nullptr) *evicted = dropped; + return true; +} + +bool StorageTexture::evictStale(std::uint64_t address, std::size_t bytes, std::size_t& dropped) { + static const bool staleEvict = std::getenv("APS5_STALE_EVICT") != nullptr; + constexpr std::uint64_t block = 65536; + // writeBack's selection for the access: the pending layers it overlaps, every one under a + // clear code (see there), or every one of a dead image (H3). + const bool whole = descriptor.dccAddress != 0 && uploadedKeys != DccKeys::Uncompressed; + const bool all = whole || (staleEvict && provedPresent.load(std::memory_order_relaxed) + DeadImagePresents <= Recorder::Presents()); + const auto accessEnd = address + bytes; + const auto selected = [&](std::uint32_t layer) { + return layerPending[layer] && (all || (address < layerBegin(layer) + layerBytes(layer) && layerBegin(layer) < accessEnd)); + }; + std::uint64_t first = descriptor.baseAddress + guestBytes; + std::uint64_t last = descriptor.baseAddress; + bool beyond = false; + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!selected(layer)) continue; + if (layerGeneration[layer] == 0) return false; + const auto begin = layerBegin(layer); + const auto end = begin + layerBytes(layer); + first = std::min(first, begin); + last = std::max(last, end); + if (begin < (address & ~(block - 1)) || end > ((accessEnd + block - 1) & ~(block - 1))) beyond = true; + } + if (last <= first) return false; + if (beyond) { + // Blocks outside the access's own were not collected by this call: the walk of the layers' + // span (a write-back's own) runs once per present per image. + const auto present = Recorder::Presents(); + if (staleCheckPresent == present) return false; + staleCheckPresent = present; + GuestMemory::CollectWritesUncached(first, static_cast(last - first)); + } + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!selected(layer)) continue; + const auto begin = layerBegin(layer); + const auto end = begin + layerBytes(layer); + for (auto at = begin & ~(block - 1); at < end; at += block) { + const auto from = std::max(at, begin); + if (GuestMemory::UnchangedSince(from, static_cast(std::min(at + block, end) - from), layerGeneration[layer])) return false; + } + } + // As the empty write-back leaves the layers: no results pending, the generation unchanged (the + // stamps make their next Refresh re-upload), `original` no longer vouching for the bytes. + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (selected(layer)) layerPending[layer] = false; + } + originalValid = false; + if (!anyLayerPending()) { + ++dropped; + staleEvicted.fetch_add(1, std::memory_order_relaxed); + } + reconcilePending(); + return true; +} + +bool StorageTexture::skippedResultsInside(std::uint64_t address, std::size_t bytes) const { + const auto layer = hookSkipLayer; + if (layer >= trackedLayers || !layerPending[layer] || version != hookSkipVersion || layerGeneration[layer] != hookSkipGeneration) return false; + const bool whole = descriptor.dccAddress != 0 && uploadedKeys != DccKeys::Uncompressed; + return whole || (address < layerBegin(layer) + layerBytes(layer) && layerBegin(layer) < address + bytes); +} + +StorageTexture::HookSkipCounts StorageTexture::TakeHookSkipCounts() { + return {flushedAfterSkip.exchange(0, std::memory_order_relaxed), flushedAfterSkipSameSite.exchange(0, std::memory_order_relaxed)}; +} + +void StorageTexture::BumpPendingSerial() { + pendingSerial.fetch_add(1, std::memory_order_release); +} + +bool StorageTexture::ScanPending(std::span queries) { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + bool identities = true; + for (auto& query : queries) { + query.overlaps = false; + query.found = nullptr; + if (query.end <= query.begin) continue; + const auto bytes = static_cast(query.end - query.begin); + for (const auto* texture : pending.textures) { + if (query.found == nullptr && texture->descriptor.baseAddress == query.begin && texture->guestBytes >= bytes) query.found = texture; + if (texture != query.except && texture->overlaps(query.begin, bytes)) query.overlaps = true; + } + for (const auto* texture : pending.flushing) { + if (texture != query.except && texture->overlaps(query.begin, bytes)) query.overlaps = true; + } + if (query.pending != nullptr && query.found != query.pending) identities = false; + } + return identities; +} + +std::size_t StorageTexture::DiscardPendingInside(std::uint64_t address, std::size_t bytes) { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + std::size_t discarded = 0; + const auto end = address + bytes; + for (auto it = pending.textures.begin(); it != pending.textures.end();) { + auto* texture = *it; + const auto begin = texture->descriptor.baseAddress; + if (texture != refreshing && begin >= address && begin + texture->guestBytes <= end) { + // The content no longer matches guest memory anywhere: the next use re-uploads once the + // fill stamped the range, never from a matching `original` (the fill may not have + // touched the bytes a CPU-path upload copied). + texture->dirty = false; + texture->layerPending.assign(texture->trackedLayers, false); + texture->originalValid = false; + it = pending.textures.erase(it); + ++discarded; + } else if (texture != refreshing && texture->blockUnits && texture->overlaps(address, bytes)) { + // Its units wholly inside the range are dead too (re-uploaded from the range's bytes at + // the next use, never from a matching `original`: the dropped units hold results the + // bytes never received); the others stay pending. + bool dropped = false; + for (std::uint32_t unit = 0; unit < texture->trackedLayers; ++unit) { + const auto unitBegin = texture->layerBegin(unit); + if (!texture->layerPending[unit] || unitBegin < address || unitBegin + texture->layerBytes(unit) > end) continue; + texture->layerPending[unit] = false; + unitsDropped.fetch_add(1, std::memory_order_relaxed); + dropped = true; + } + if (dropped) { + texture->originalValid = false; + ++discarded; + } + if (dropped && !texture->anyLayerPending()) { + texture->dirty = false; + it = pending.textures.erase(it); + continue; + } + ++it; + } else { + ++it; + } + } + if (discarded != 0) BumpPendingSerial(); + return discarded; +} + +StorageTexture::FillCoverage StorageTexture::ClassifyFill(std::uint64_t address, std::size_t bytes) { + FillCoverage coverage; + auto& live = Live(); + std::lock_guard lock(live.mutex); + std::vector overlapping; + for (auto* texture : live.textures) { + if (!texture->released && texture->guestBytes != 0 && texture->overlaps(address, bytes)) overlapping.push_back(texture); + } + const auto end = address + bytes; + StorageTexture* covered = nullptr; + // The newest of several exact matches (the registry is in construction order): the same + // surface under two storage formats keeps two images, and the older one may be bound by + // nothing any more, so clearing it would only revive it for a write-back. + for (auto* texture : overlapping) { + if (texture->descriptor.baseAddress == address && texture->guestBytes == bytes) { + covered = texture; + coverage.cover = FillCover::Exact; + coverage.layer = WholeImage; + } + } + // APS5_NO_LAYER_REFUSAL_MEMO=1: match a layer of an image whose surface check failed at its + // current generation anyway (the whole-surface collect that check costs then runs per fill). + static const bool layerMemo = std::getenv("APS5_NO_LAYER_REFUSAL_MEMO") == nullptr; + for (auto* texture : overlapping) { + if (covered != nullptr) break; + // One array layer of a thin surface: its guest bytes are one contiguous slice. + const auto& geometry = texture->geometry; + const auto base = texture->descriptor.baseAddress; + if (geometry.layers < 2 || geometry.thick || geometry.imageDepth != 1 || geometry.layers != geometry.imageLayers || geometry.layerBytes != bytes || geometry.layerBytes * geometry.layers != texture->guestBytes || address < base) continue; + if (layerMemo && texture->layerRefusedGeneration == texture->generation) continue; + const auto offset = address - base; + if (offset % bytes != 0 || offset / bytes >= geometry.layers) continue; + covered = texture; + coverage.cover = FillCover::Layer; + coverage.layer = static_cast(offset / bytes); + } + if (covered != nullptr) { + coverage.image = covered->weak_from_this().lock(); + if (coverage.image == nullptr) { + coverage.cover = FillCover::None; + } else { + for (const auto* texture : overlapping) { + if (texture == covered) continue; + ++coverage.others; + if (texture->descriptor.baseAddress >= address && texture->descriptor.baseAddress + texture->guestBytes <= end) ++coverage.inside; + } + } + } else if (overlapping.empty()) { + constexpr std::uint64_t keyBytes = 256; + for (const auto* texture : live.textures) { + if (!texture->released && texture->descriptor.dccAddress == address && texture->guestBytes / keyBytes == bytes) coverage.cover = FillCover::Keys; + } + } else if (overlapping.size() > 1) { + coverage.cover = FillCover::Several; + } else { + auto* single = overlapping.front(); + const auto begin = single->descriptor.baseAddress; + const auto stop = begin + single->guestBytes; + if (address >= begin && end <= stop) coverage.cover = FillCover::Inside; + else if (address <= begin && end >= stop) coverage.cover = FillCover::Around; + else coverage.cover = FillCover::Straddle; + } + // Debug aid: APS5_TRACE_FILL_COVER=1 prints the first fills that meet images without covering + // one of them (or a layer of one), with the images' surfaces, so a wider conversion can be + // designed from them. + static const bool trace = std::getenv("APS5_TRACE_FILL_COVER") != nullptr; + static int traced = 0; + const auto cover = coverage.cover; + if (trace && (cover == FillCover::Inside || cover == FillCover::Around || cover == FillCover::Straddle || cover == FillCover::Several) && traced < 96) { + ++traced; + std::fprintf(stderr, "[fill-cover] fill 0x%llx+0x%zx meets %zu image(s):", static_cast(address), bytes, overlapping.size()); + for (const auto* texture : overlapping) { + const auto& d = texture->descriptor; + std::fprintf(stderr, " [0x%llx+0x%llx %ux%u mips %u layers %u dim %d tile %d format %u dcc 0x%llx%s]", static_cast(d.baseAddress), static_cast(texture->guestBytes), d.width, d.height, d.mipCount, texture->geometry.imageLayers, static_cast(d.dimension), static_cast(d.tileMode), d.format, static_cast(d.dccAddress), texture->dirty ? " dirty" : ""); + } + std::fprintf(stderr, "\n"); + } + return coverage; +} + +bool StorageTexture::FillClear(std::span pattern, std::uint32_t layer, const char*& refusal) { + // Debug aid (APS5_TRACE_FILL_COVER=1): the first layer covers refused, with the surface, the + // pattern and the generation the refusal was memoized at (ClassifyFill's layerRefusedGeneration). + static const bool trace = std::getenv("APS5_TRACE_FILL_COVER") != nullptr; + struct TraceLayerRefusal { + const StorageTexture& texture; + std::span pattern; + std::uint32_t layer; + const char*& refusal; + bool cleared = false; + ~TraceLayerRefusal() { + static std::atomic traced{0}; + if (cleared || !trace || layer == WholeImage || refusal == nullptr || traced.fetch_add(1) >= 64) return; + const auto& d = texture.descriptor; + std::fprintf(stderr, "[fill-cover] layer %u of 0x%llx+0x%llx (%ux%u mips %u layers %u, %llu bytes per layer, format %u tile %d dcc 0x%llx%s) refused: %s; pattern %08x %08x %08x %08x; generation %llu\n", layer, static_cast(d.baseAddress), static_cast(texture.guestBytes), d.width, d.height, d.mipCount, texture.geometry.imageLayers, static_cast(texture.geometry.layerBytes), d.format, static_cast(d.tileMode), static_cast(d.dccAddress), texture.dirty ? ", dirty" : "", refusal, pattern[0], pattern[1], pattern[2], pattern[3], static_cast(texture.generation)); + } + } traceRefusal{*this, pattern, layer, refusal}; + VkClearColorValue clearValue{}; + if (!FillClearColor(storageFormat, BytesPerElement(descriptor.format), pattern, clearValue)) { + refusal = "pattern"; + return false; + } + if (layer != WholeImage && (layer >= geometry.imageLayers || geometry.thick || geometry.imageDepth != 1)) { + refusal = "layer"; + return false; + } + if (descriptor.dccAddress != 0) { + // Under a clear code reads see that value whatever the texels hold, and keys a recorded + // kernel still writes are not in the bytes yet: only keys that read as uncompressed now let + // the image stand for the texels. + if (Recorder::SnapshotWriteOverlaps(descriptor.dccAddress, static_cast(guestBytes / 256))) { + refusal = "keys pending"; + return false; + } + if (TextureClearKeys(descriptor, guestBytes) != DccKeys::Uncompressed) { + refusal = "keys"; + return false; + } + } + if (HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) == nullptr) { + refusal = "not imported"; + return false; + } + auto* recorder = Recorder::Active(); + if (recorder == nullptr) { + refusal = "no recorder"; + return false; + } + const auto begin = layer == WholeImage ? descriptor.baseAddress : descriptor.baseAddress + geometry.GuestLayerOffset(layer); + const auto bytes = static_cast(layer == WholeImage ? guestBytes : geometry.layerBytes); + // A tracked layer answers for itself (its own generation and pending flag): the rest of the + // surface may have changed under the image, which its own layers' next use uploads. + if (layer != WholeImage && trackedLayers == 1) { + // The rest of the surface keeps the image's content, which must still be what guest memory + // holds: the one generation the write-back keeps CPU-written blocks after moves to now. A + // surface that changed under the image (another image's results stored into its memory, a + // fill of another layer stored as bytes) is not refreshed here: the images so filled at the + // movie stage are stale ones of an earlier use of the memory, and reviving one every frame + // (a 60 MiB write-back and re-upload inside the fill) cost what the conversion gained. + // Uncached, as writeBack's keep decision walks: a memo hit of this submission's epoch + // would leave pages the game dirtied since that walk unconsumed, and the check below + // would pass over them. + GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); + const auto before = static_cast(begin - descriptor.baseAddress); + const auto after = static_cast(descriptor.baseAddress + guestBytes - (begin + bytes)); + if ((before != 0 && !GuestMemory::UnchangedSince(descriptor.baseAddress, before, generation)) || (after != 0 && !GuestMemory::UnchangedSince(begin + bytes, after, generation))) { + refusal = "surface changed"; + layerRefusedGeneration = generation; + return false; + } + } + if (layer != WholeImage && blockUnits && ((begin - descriptor.baseAddress) % trackedLayerBytes != 0 || bytes % trackedLayerBytes != 0)) { + refusal = "layer alignment"; + return false; + } + // Other images over the range hold content the fill supersedes: stamped below, they re-upload + // at their next use (behind this image's write-back), and never from a matching `original`, + // since the fill leaves the bytes a CPU-path upload copied as they are. + bool aliasedKeys = false; + { + auto& live = Live(); + std::lock_guard lock(live.mutex); + for (auto* texture : live.textures) { + if (texture == this || texture->released || !texture->overlaps(begin, bytes)) continue; + texture->originalValid = false; + if (texture->descriptor.dccAddress != 0) aliasedKeys = true; + } + } + // Debug aid (APS5_TRACE_FILL_COVER=1): the first clears of a surface with DCC keys (its own or + // another image's over the same memory), with the pattern: a title that fills the keys to a + // clear code right after makes the clear moot for that image (its next lookup re-uploads + // under the code) unless the pattern is that code's. + static std::atomic traced{0}; + if (trace && (descriptor.dccAddress != 0 || aliasedKeys) && traced.fetch_add(1) < 16) std::fprintf(stderr, "[fill-cover] clear of surface 0x%llx+0x%llx (format %u, keys 0x%llx%s): pattern %08x %08x %08x %08x\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), descriptor.format, static_cast(descriptor.dccAddress), aliasedKeys ? ", another image over the memory has keys" : "", pattern[0], pattern[1], pattern[2], pattern[3]); + // A neighbour's results shadowed in the range's edge units reach the import before the stamp + // below makes those units stale (nothing for a 64 KiB-multiple surface). + PublishShadow(begin, bytes, PublishScope::PartialUnits, PublishReason::FillClear); + GuestMemory::MarkWritten(begin, bytes); + const auto commands = recorder->Commands(); + const auto timing = recorder->BeginGpuTiming(Recorder::CommandClass::FillClear); + Recorder::CountBarriers(Recorder::CommandClass::FillClear, 2); + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + if (Recorder::BarrierValidate()) { + const std::pair cleared{image, true}; + recorder->NoteAccess(Recorder::CommandClass::FillClear, Recorder::Access{{}, {}, std::span(&cleared, 1), VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + VkImageMemoryBarrier toClear{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toClear.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toClear.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + // A whole image is discarded into the clear; one layer keeps the others' content. + toClear.oldLayout = layer == WholeImage ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_GENERAL; + toClear.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toClear.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.image = image; + toClear.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, layer == WholeImage ? 0u : layer, layer == WholeImage ? geometry.imageLayers : 1u}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toClear); + context.Function("vkCmdClearColorImage")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &clearValue, 1, &toClear.subresourceRange); + VkImageMemoryBarrier toGeneral = toClear; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + recorder->EndGpuTiming(timing, bytes); + traceRefusal.cleared = true; + // The image holds the fill and guest memory what it held: the texels reach it at the + // write-back, which keeps the 64 KiB blocks the CPU writes from now on (a CPU write before the + // fill is overwritten by it in program order; the stamps above are below this generation), and + // the keys were read uncompressed. The collect is uncached (see writeBack's keep decision): a + // memo hit of this submission's epoch would not consume pages the game dirtied since that + // walk, the write-back's own walk would stamp them above this generation and keep their + // blocks, and the fill's pattern would be lost in them. + originalValid = false; + uploadedKeys = DccKeys::Uncompressed; + if (layer != WholeImage && trackedLayers > 1) { + // The cleared layer's units alone: their stamps above lie below this generation, the other + // units keep theirs (their pages are consumed by their own collects). + const auto first = blockUnits ? static_cast((begin - descriptor.baseAddress) / trackedLayerBytes) : layer; + const auto count = blockUnits ? static_cast(bytes / trackedLayerBytes) : 1u; + const auto now = GuestMemory::CollectWritesUncached(begin, bytes); + for (auto unit = first; unit < first + count; ++unit) layerGeneration[unit] = now; + refreshGeneration(); + forgetBorrowed(first, count); + markLayersPending(first, count); + return true; + } + layerGeneration.assign(trackedLayers, GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes))); + refreshGeneration(); + forgetBorrowed(0, trackedLayers); + MarkDirty(); + return true; +} + void StorageTexture::WriteBack() { - writeBack(generation); + const auto previous = std::exchange(flushReason, "explicit"); + writeBack(descriptor.baseAddress, static_cast(guestBytes)); + flushReason = previous; +} + +std::shared_ptr StorageTexture::FindLive(std::uint64_t address, std::uint64_t bytes) { + auto& live = Live(); + std::lock_guard lock(live.mutex); + for (auto it = live.textures.rbegin(); it != live.textures.rend(); ++it) { + auto* texture = *it; + if (texture->released || !texture->Cached() || texture->descriptor.baseAddress != address || texture->guestBytes != bytes) continue; + return texture->weak_from_this().lock(); + } + return nullptr; +} + +bool StorageTexture::SameSurfaceShape(const StorageTexture& other) const { + const auto& mine = descriptor; + const auto& theirs = other.descriptor; + return guestBytes == other.guestBytes && mine.width == theirs.width && mine.height == theirs.height && mine.depthOrLastArray == theirs.depthOrLastArray && mine.baseArray == theirs.baseArray && mine.mipCount == theirs.mipCount && mine.tileMode == theirs.tileMode && mine.dimension == theirs.dimension && mine.format == theirs.format && storageFormat == other.storageFormat && geometry.imageLayers == other.geometry.imageLayers && geometry.imageDepth == other.geometry.imageDepth; +} + +bool StorageTexture::CopyFrom(StorageTexture& source, const char*& refusal) { + if (&source == this || !SameSurfaceShape(source)) { + refusal = "shape"; + return false; + } + if (descriptor.dccAddress != 0 || source.descriptor.dccAddress != 0) { + refusal = "keys"; + return false; + } + if (released || source.released || !Cached() || !source.Cached()) { + refusal = "released"; + return false; + } + if (HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) == nullptr) { + refusal = "not imported"; + return false; + } + auto* recorder = Recorder::Active(); + if (recorder == nullptr) { + refusal = "no recorder"; + return false; + } + // The source image must hold the surface: a stale one uploads first (and other images' + // results pending over its memory land first, as the transfer's flush would have them). + source.Refresh(); + // Results pending over the destination range from before the copy are dead inside it and land + // first around it; other images over the range hold content the copy supersedes (they + // re-upload at their next use, behind this image's write-back). + static_cast(DiscardPendingInside(descriptor.baseAddress, static_cast(guestBytes))); + FlushPending(descriptor.baseAddress, static_cast(guestBytes), this, "buffer copy destination", PublishScope::PartialUnits); + { + auto& live = Live(); + std::lock_guard lock(live.mutex); + for (auto* texture : live.textures) { + if (texture != this && !texture->released && texture->overlaps(descriptor.baseAddress, static_cast(guestBytes))) texture->originalValid = false; + } + } + GuestMemory::MarkWritten(descriptor.baseAddress, static_cast(guestBytes)); + const auto commands = recorder->Commands(); + const auto timing = recorder->BeginGpuTiming(Recorder::CommandClass::Copy); + Recorder::CountBarriers(Recorder::CommandClass::Copy, 2); + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + if (auto other = source.weak_from_this().lock()) recorder->Keep(std::move(other)); + if (Recorder::BarrierValidate()) { + const std::array, 2> images{{{source.image, false}, {image, true}}}; + recorder->NoteAccess(Recorder::CommandClass::Copy, Recorder::Access{{}, {}, images, VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + VkImageMemoryBarrier barriers[2]{}; + for (auto& barrier : barriers) { + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + } + barriers[0].srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + barriers[0].dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barriers[0].image = source.image; + barriers[1].srcAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + barriers[1].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barriers[1].image = image; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); + std::vector regions; + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) { + VkImageCopy region{}; + region.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, geometry.CopyLayer(layer), 1}; + region.srcOffset = {0, 0, geometry.CopyDepth(layer)}; + region.dstSubresource = region.srcSubresource; + region.dstOffset = region.srcOffset; + region.extent = {std::max(descriptor.width >> level, 1u), std::max(descriptor.height >> level, 1u), 1u}; + regions.push_back(region); + } + } + context.Function("vkCmdCopyImage")(commands, source.image, VK_IMAGE_LAYOUT_GENERAL, image, VK_IMAGE_LAYOUT_GENERAL, static_cast(regions.size()), regions.data()); + VkImageMemoryBarrier copied = barriers[1]; + copied.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + copied.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &copied); + recorder->EndGpuTiming(timing, guestBytes); + // As after a clear: the image holds the surface, guest memory follows at the write-back (which + // keeps the 64 KiB blocks the CPU writes from now on; the collect is uncached for the reason + // given in FillClear). + originalValid = false; + uploadedKeys = DccKeys::Uncompressed; + layerGeneration.assign(trackedLayers, GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes))); + refreshGeneration(); + forgetBorrowed(0, trackedLayers); + MarkDirty(); + return true; } // See AdjacentGenerationEnabled above. -std::vector StorageTexture::adjacentPendingUnchanged() const { +std::vector StorageTexture::adjacentPendingUnchanged(std::uint64_t firstBlock, std::uint64_t lastBlock) const { std::vector adjacent; if (!AdjacentGenerationEnabled() || guestBytes == 0) return adjacent; constexpr std::uint64_t block = 65536; - const auto firstBlock = descriptor.baseAddress / block; - const auto lastBlock = (descriptor.baseAddress + guestBytes - 1) / block; { auto& pending = Pending(); std::lock_guard lock(pending.mutex); const auto consider = [&](StorageTexture* texture) { if (texture == this || texture->guestBytes == 0 || texture->overlaps(descriptor.baseAddress, static_cast(guestBytes))) return; const auto begin = texture->descriptor.baseAddress; - if (begin / block > lastBlock || (begin + texture->guestBytes - 1) / block < firstBlock) return; - if (auto alive = texture->weak_from_this().lock()) adjacent.push_back({std::move(alive), texture->generation}); + const auto neighbourFirst = begin / block; + const auto neighbourLast = (begin + texture->guestBytes - 1) / block; + if (neighbourFirst > lastBlock || neighbourLast < firstBlock) return; + // Surfaces that do not overlap share at most one block: the neighbour's tracked layer + // holding it is what the stamps touch. + const auto shared = std::max(neighbourFirst, firstBlock); + const auto layer = static_cast((std::max(shared * block, begin) - begin) / texture->trackedLayerBytes); + if (auto alive = texture->weak_from_this().lock()) adjacent.push_back({std::move(alive), layer, texture->layerGeneration[layer]}); }; for (auto* texture : pending.textures) consider(texture); for (auto* texture : pending.flushing) consider(texture); } - // Only an image whose memory has not changed since its generation can be advanced: the collect + // Only a layer whose memory has not changed since its generation can be advanced: the collect // stamps the CPU's writes so far, and a later one lands at a newer generation either way. std::erase_if(adjacent, [](const Adjacent& entry) { const auto& texture = entry.texture; - GuestMemory::CollectWritesUncached(texture->descriptor.baseAddress, static_cast(texture->guestBytes)); - return !GuestMemory::UnchangedSince(texture->descriptor.baseAddress, static_cast(texture->guestBytes), texture->generation); + const auto begin = texture->layerBegin(entry.layer); + const auto bytes = static_cast(texture->layerBytes(entry.layer)); + GuestMemory::CollectWritesUncached(begin, bytes); + return !GuestMemory::UnchangedSince(begin, bytes, entry.generation); }); return adjacent; } @@ -931,27 +2644,177 @@ std::vector StorageTexture::adjacentPendingUnchanged() void StorageTexture::advanceAdjacent(const std::vector& adjacent, std::uint64_t now, std::uint64_t firstBlock, std::uint64_t lastBlock) { static const bool trace = std::getenv("APS5_TRACE_FLUSH") != nullptr; constexpr std::uint64_t block = 65536; - for (const auto& [texture, seen] : adjacent) { - // An image stored or refreshed meanwhile set its own generation (an old one on purpose when + for (const auto& [texture, layer, seen] : adjacent) { + // A layer stored or refreshed meanwhile set its own generation (an old one on purpose when // it kept blocks for the CPU): only the value seen at the check is advanced. - if (texture->generation != seen || now <= seen) continue; + if (texture->layerGeneration[layer] != seen || now <= seen) continue; // A CPU write to the neighbour landing since the check may have been stamped by another // thread's walk of its range at a value in (seen, now]; the advance would hide it. Its blocks // outside this write-back's span carry no stamp of ours, so they are re-checked against the // seen generation (a stamp scan, no walk). A write inside the shared block itself in that // window is indistinguishable from this write-back's stamp, as it is for the block rule. - const auto begin = texture->descriptor.baseAddress; - const auto end = begin + texture->guestBytes; + const auto begin = texture->layerBegin(layer); + const auto end = begin + texture->layerBytes(layer); const auto beforeEnd = std::min(end, firstBlock * block); const auto afterBegin = std::max(begin, (lastBlock + 1) * block); if (begin < beforeEnd && !GuestMemory::UnchangedSince(begin, static_cast(beforeEnd - begin), seen)) continue; if (afterBegin < end && !GuestMemory::UnchangedSince(afterBegin, static_cast(end - afterBegin), seen)) continue; - if (trace) std::fprintf(stderr, "[flush] adjacent pending image 0x%llx+0x%llx advanced past a write-back's stamps (generation %llu -> %llu)\n", static_cast(begin), static_cast(texture->guestBytes), static_cast(seen), static_cast(now)); - texture->generation = now; + if (trace) std::fprintf(stderr, "[flush] adjacent pending image 0x%llx+0x%llx layer %u advanced past a write-back's stamps (generation %llu -> %llu)\n", static_cast(texture->descriptor.baseAddress), static_cast(texture->guestBytes), layer, static_cast(seen), static_cast(now)); + texture->layerGeneration[layer] = now; + texture->refreshGeneration(); } } -void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { +void StorageTexture::blockGenerations(std::vector& generations) const { + constexpr std::uint64_t block = 65536; + const auto spanBegin = descriptor.baseAddress & ~(block - 1); + generations.assign(static_cast((descriptor.baseAddress + guestBytes - spanBegin + block - 1) / block), 0); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + const auto end = layerBegin(layer) + layerBytes(layer); + for (auto at = layerBegin(layer) & ~(block - 1); at < end; at += block) generations[static_cast((at - spanBegin) / block)] = layerGeneration[layer]; + } +} + +std::shared_ptr StorageTexture::pendingAlias() const { + static const bool disabled = std::getenv("APS5_NO_ALIAS_BORROW") != nullptr; + if (disabled || !blockUnits) return nullptr; + std::vector> candidates; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (auto* texture : pending.textures) { + if (texture == this || texture->descriptor.baseAddress != descriptor.baseAddress || texture->guestBytes != guestBytes || !texture->blockUnits) continue; + if (auto alive = texture->weak_from_this().lock()) candidates.push_back(std::move(alive)); + } + } + for (const auto& candidate : candidates) { + const auto& mine = descriptor; + const auto& theirs = candidate->descriptor; + // The same texels under another format of the same size: the device copy moves them bit + // for bit, as the guest bytes would. + const bool sameShape = mine.width == theirs.width && mine.height == theirs.height && mine.depthOrLastArray == theirs.depthOrLastArray && mine.baseArray == theirs.baseArray && mine.mipCount == theirs.mipCount && mine.tileMode == theirs.tileMode && mine.dimension == theirs.dimension && BytesPerElement(mine.format) == BytesPerElement(theirs.format) && geometry.imageLayers == candidate->geometry.imageLayers && geometry.imageDepth == candidate->geometry.imageDepth && trackedLayers == candidate->trackedLayers; + if (sameShape && candidate->Cached() && candidate->uploadedKeys == DccKeys::Uncompressed) return candidate; + } + return nullptr; +} + +void StorageTexture::forgetBorrowed(std::uint32_t first, std::uint32_t count) { + if (borrowedUnits.empty()) return; + for (auto unit = first; unit < first + count && unit < borrowedUnits.size(); ++unit) borrowedUnits[unit] = false; + if (std::none_of(borrowedUnits.begin(), borrowedUnits.end(), [](bool held) { return held; })) { + borrowedUnits.clear(); + borrowedFrom.reset(); + } +} + +std::uint64_t StorageTexture::borrowUnits(StorageTexture& source, const std::vector& units) { + const auto runs = unitRuns(units); + const auto windows = sliceWindows(runs); + Require(!windows.empty(), "storage image borrows no unit"); + std::vector regions; + for (const auto& window : windows) { + for (const auto& region : window.regions) regions.push_back({region.imageSubresource, region.imageOffset, region.imageSubresource, region.imageOffset, region.imageExtent}); + } + std::uint64_t bytes = 0; + for (const auto& [begin, end] : runs) bytes += end - begin; + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + auto timing = Recorder::NoTiming; + if (recorder != nullptr) { + commands = recorder->Commands(); + timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + if (auto other = source.weak_from_this().lock()) recorder->Keep(std::move(other)); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 2); + if (Recorder::BarrierValidate()) { + const std::array, 2> images{{{source.image, false}, {image, true}}}; + recorder->NoteAccess(Recorder::CommandClass::StorageUpload, Recorder::Access{{}, {}, images, VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } + VkImageMemoryBarrier barriers[2]{}; + for (auto& barrier : barriers) { + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + } + barriers[0].srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + barriers[0].dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barriers[0].image = source.image; + barriers[1].srcAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + barriers[1].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barriers[1].image = image; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); + context.Function("vkCmdCopyImage")(commands, source.image, VK_IMAGE_LAYOUT_GENERAL, image, VK_IMAGE_LAYOUT_GENERAL, static_cast(regions.size()), regions.data()); + VkImageMemoryBarrier copied = barriers[1]; + copied.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + copied.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &copied); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, bytes); + countStorageUpload(3, bytes); + if (borrowedFrom.lock() != source.weak_from_this().lock() || borrowedUnits.size() != trackedLayers) { + borrowedUnits.assign(trackedLayers, false); + borrowedFrom = source.weak_from_this(); + } + borrowedVersion = source.version; + source.lent = true; + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if (units[unit]) borrowedUnits[unit] = true; + } + ++version; + return bytes; +} + +void StorageTexture::writeBack(std::uint64_t address, std::size_t bytes) { + // Block units are stored per access; an image whose units keep being asked for in pieces (a + // consumer touching it through many small ranges, each piece re-arming the pending memos of + // the next dispatch) stores every pending unit once more than `pieces` partial stores fall + // within `window` presents, and keeps doing so for another window. Off by default: the title's + // aliased images are stored in pieces by design (every piece is another image's hand-over) and + // widening them stored whole images again (movie stage: 39 vs 35 ms of GPU per present, 188 vs + // 214 presents per 10 s); APS5_BLOCK_WRITEBACK_WIDEN=1 enables the widening, and + // APS5_BLOCK_WRITEBACK_EACH=1 stores the touched units only, every time. + static const bool each = std::getenv("APS5_BLOCK_WRITEBACK_EACH") != nullptr || std::getenv("APS5_BLOCK_WRITEBACK_WIDEN") == nullptr; + static const std::uint32_t pieces = [] { const char* text = std::getenv("APS5_BLOCK_WRITEBACK_PIECES"); return text != nullptr ? static_cast(std::strtoul(text, nullptr, 10)) : 4u; }(); + static const std::uint64_t window = [] { const char* text = std::getenv("APS5_BLOCK_WRITEBACK_FRAMES"); return text != nullptr ? std::strtoull(text, nullptr, 10) : 8ull; }(); + std::vector layers(trackedLayers, false); + std::size_t selected = 0, pendingCount = 0; + // Under a clear code the texels outside the stored units would read as the clear while the + // keys, marked uncompressed for the whole surface, say texels: such an image stores whole. + const bool whole = descriptor.dccAddress != 0 && uploadedKeys != DccKeys::Uncompressed; + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!layerPending[layer]) continue; + ++pendingCount; + layers[layer] = whole || (address < layerBegin(layer) + layerBytes(layer) && layerBegin(layer) < address + bytes); + if (layers[layer]) ++selected; + } + if (selected == 0) { + reconcilePending(); + return; + } + if (blockUnits && !each && selected < pendingCount) { + const auto present = Recorder::Presents(); + if (present - partialWindowStart >= window) { + partialWindowStart = present; + partialStores = 0; + } + if (present < storeWholeUntil || ++partialStores > pieces) { + storeWholeUntil = present + window; + layers = layerPending; + coalescedWriteBacks.fetch_add(1, std::memory_order_relaxed); + } + } + writeBackLayers(layers); +} + +void StorageTexture::writeBackLayers(const std::vector& layers) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; PhaseTimer timer; struct Account { @@ -959,49 +2822,132 @@ void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { PhaseTimer& timer; ~Account() { if (enabled) Profile().storageWriteBack += timer.lap(); } } account{profile, timer}; + // Whatever the outcome, the registration follows the pending flags (a throw leaves the + // layers pending). + struct Reconcile { + StorageTexture& texture; + ~Reconcile() { texture.reconcilePending(); } + } reconcile{*this}; const auto elementBytes = BytesPerElement(descriptor.format); - // 64 KiB blocks the CPU wrote since the image was last in sync keep the CPU's bytes: the game may - // have reused the memory for something else entirely (see skipWrittenSince). Zero means no - // tracking, and everything is stored. + // 64 KiB blocks the CPU wrote since the layer was last in sync keep the CPU's bytes: the game + // may have reused the memory for something else entirely (see layerGeneration). A generation of + // zero means no tracking, and the layer is stored whole. std::vector> keep; + std::vector skippedLayer(trackedLayers, false); bool skippedAny = false; std::size_t skipped = 0; - { - const auto begin = descriptor.baseAddress; - const auto end = begin + guestBytes; - if (skipWrittenSince != 0) { - constexpr std::uint64_t block = 65536; - // The keep-blocks decision must see every CPU write up to now, so it bypasses the - // per-packet collect memo (a Refresh collect earlier in the same packet would satisfy it). - GuestMemory::CollectWritesUncached(begin, static_cast(guestBytes)); - for (auto at = begin & ~(block - 1); at < end; at += block) { - const auto from = std::max(at, begin); - const auto to = std::min(at + block, end); - if (from >= to) continue; - if (!GuestMemory::UnchangedSince(from, static_cast(to - from), skipWrittenSince)) { - skippedAny = true; - ++skipped; - continue; - } - if (!keep.empty() && keep.back().second == from) keep.back().second = to; - else keep.emplace_back(from, to); - } - } else { + std::uint64_t firstStored = descriptor.baseAddress + guestBytes; + std::uint64_t lastStored = descriptor.baseAddress; + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!layers[layer]) continue; + firstStored = std::min(firstStored, layerBegin(layer)); + lastStored = std::max(lastStored, layerBegin(layer) + layerBytes(layer)); + } + if (lastStored <= firstStored) return; + constexpr std::uint64_t block = 65536; + const auto spanBegin = firstStored & ~(block - 1); + const auto spanBlocks = static_cast((lastStored - spanBegin + block - 1) / block); + // The keep-blocks decision must see every CPU write to the stored span up to now, so it + // bypasses the per-packet collect memo (a Refresh collect earlier in the same packet would + // satisfy it); the unselected layers keep their own generations and are collected by their + // own refreshes. One tracker lock then reads the span's stamps against the layers' generations. + GuestMemory::CollectWritesUncached(firstStored, static_cast(lastStored - firstStored)); + std::vector generations(spanBlocks, 0); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!layers[layer]) continue; + const auto end = layerBegin(layer) + layerBytes(layer); + for (auto at = layerBegin(layer) & ~(block - 1); at < end; at += block) generations[static_cast((at - spanBegin) / block)] = layerGeneration[layer]; + } + std::vector changedBlocks(spanBlocks); + GuestMemory::ChangedBlocks(firstStored, static_cast(lastStored - firstStored), generations, changedBlocks); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!layers[layer]) continue; + const auto begin = layerBegin(layer); + const auto end = begin + layerBytes(layer); + if (layerGeneration[layer] == 0) { keep.emplace_back(begin, end); + continue; + } + for (auto at = begin & ~(block - 1); at < end; at += block) { + const auto from = std::max(at, begin); + const auto to = std::min(at + block, end); + if (from >= to) continue; + if (changedBlocks[static_cast((at - spanBegin) / block)] != 0) { + skippedAny = true; + skippedLayer[layer] = true; + ++skipped; + continue; + } + if (!keep.empty() && keep.back().second == from) keep.back().second = to; + else keep.emplace_back(from, to); } } // Debug aid: APS5_TRACE_FLUSH also names the blocks a write-back leaves to the CPU. static const bool traceKept = std::getenv("APS5_TRACE_FLUSH") != nullptr; if (traceKept && skippedAny) { const auto first = keep.empty() ? descriptor.baseAddress + guestBytes : keep.front().first; - std::fprintf(stderr, "[flush] image 0x%llx+0x%llx keeps %zu CPU-written 64 KiB blocks (first stored byte at +0x%llx, %zu ranges stored, generation %llu)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), skipped, static_cast(first - descriptor.baseAddress), keep.size(), static_cast(skipWrittenSince)); + std::fprintf(stderr, "[flush] image 0x%llx+0x%llx keeps %zu CPU-written 64 KiB blocks (first stored byte at +0x%llx, %zu ranges stored, generation %llu)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), skipped, static_cast(first - descriptor.baseAddress), keep.size(), static_cast(generation)); } // Taken before this write-back stamps anything (see advanceAdjacent), with the 64 KiB block span // its stamps cover. - const auto adjacent = adjacentPendingUnchanged(); - const auto firstBlock = descriptor.baseAddress / 65536; - const auto lastBlock = guestBytes == 0 ? firstBlock : (descriptor.baseAddress + guestBytes - 1) / 65536; + const auto firstBlock = firstStored / block; + const auto lastBlock = (lastStored - 1) / block; + const auto adjacent = adjacentPendingUnchanged(firstBlock, lastBlock); + // After the store: the stored layers are in sync at a fresh generation unless blocks were kept + // for the CPU (then the layer is stale there, keeps its old generation and re-uploads at its + // next use); their results are no longer pending either way. The collect covers the stored + // span, which holds every stored layer's pages. Pieces retiled into the import's unit shadow + // (`shadowed`, with its import) are fresh at the generation taken after this write-back's + // own stamps, so every later import writer stamps newer. + std::vector shadowed; + std::vector> imported; + const HostImport* shadowImport = nullptr; + const auto settle = [&](bool memoizedCollect) { + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (layers[layer]) layerPending[layer] = false; + } + const bool allKept = std::none_of(skippedLayer.begin(), skippedLayer.end(), [](bool kept) { return kept; }); + if (allKept || !adjacent.empty()) { + const auto now = memoizedCollect ? GuestMemory::CollectWrites(firstStored, static_cast(lastStored - firstStored)) : GuestMemory::CollectWritesUncached(firstStored, static_cast(lastStored - firstStored)); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (layers[layer] && !skippedLayer[layer]) layerGeneration[layer] = now; + } + refreshGeneration(); + advanceAdjacent(adjacent, now, firstBlock, lastBlock); + if (!shadowed.empty()) MarkShadowed(*shadowImport, shadowed, now); + } else if (!shadowed.empty()) { + MarkShadowed(*shadowImport, shadowed, GuestMemory::TrackerGeneration()); + } + }; + if (keep.empty()) { + // Every selected block was written by the CPU since the layer's generation: nothing to + // store, the keys stay as they are, and the image is stale there (re-uploaded at its next + // use, never from a matching `original`). + emptyWriteBacks.fetch_add(1, std::memory_order_relaxed); + originalValid = false; + settle(true); + return; + } if (const auto* import = HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes))) { + if (blockUnits) { + // The kept blocks alone pass through the retiler (windows of their slices). + shadowImport = import; + const auto storedBytes = writeBackWindows(*import, keep, firstStored, lastStored, shadowed, imported); + countStorageWriteBack(storedBytes, true); + if (profile) Profile().storageGpu += timer.lap(); + originalValid = false; + traceKeyStore("block write-back", descriptor, guestBytes); + MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + uploadedKeys = DccKeys::Uncompressed; + for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); + settle(true); + ++Profile().storageDirectWriteBacks; + return; + } + // The whole-layer copies below overwrite the interior units of the span; the edge units + // keep a neighbour's shadowed results in the rest of their bytes, which reach the import + // first (this write-back's stamps then make those units stale). + PublishShadow(firstStored, static_cast(lastStored - firstStored), PublishScope::PartialUnits, PublishReason::Upload); // The surface lives in host-imported memory: the retiler writes into device scratch and the // untouched blocks are copied into the imported bytes in place, recorded behind the work that // produced the image; nothing crosses to the CPU. @@ -1011,17 +2957,31 @@ void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { auto* recorder = Recorder::Active(); std::unique_ptr batch; VkCommandBuffer commands = VK_NULL_HANDLE; + auto timing = Recorder::NoTiming; + std::uint64_t storedBytes = 0; + for (const auto& [from, to] : keep) storedBytes += to - from; if (recorder != nullptr) { + recorder->FlushKeyStoresOverlapping(firstStored, static_cast(lastStored - firstStored)); + recorder->FlushStoresOverlapping(firstStored, static_cast(lastStored - firstStored)); commands = recorder->Commands(); + timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); recorder->Keep(linear); recorder->Keep(tiledScratch); // The image itself must outlive the recorded retile: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4); + if (Recorder::BarrierValidate()) { + const std::pair read{image, false}; + recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{{}, keep, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); + } } else { batch = std::make_unique(context); commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import->base; + // The tracked layers are the array layers when there are several; one tracked layer means + // the whole surface. + const auto* storedLayers = trackedLayers == arrayLayers && trackedLayers > 1 ? &layers : nullptr; VkImageMemoryBarrier toSource{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; toSource.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; @@ -1032,12 +2992,13 @@ void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { toSource.image = image; toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); - const auto regions = CopyRegions(); + const auto regions = CopyRegions(storedLayers); context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (storedLayers != nullptr && !(*storedLayers)[layer]) continue; for (const auto& mip : mips) { detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiledScratch->Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick); } @@ -1056,36 +3017,52 @@ void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { backToGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT}; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral); - if (batch) batch->SubmitAndWait(); - else recorder->NotePendingWrite(descriptor.baseAddress, static_cast(guestBytes)); + if (batch) { + batch->SubmitAndWait(); + } else { + recorder->EndGpuTiming(timing, storedBytes); + recorder->MarkShaderReadsCovered(); + recorder->NotePendingWrite(firstStored, static_cast(lastStored - firstStored)); + } + countStorageWriteBack(storedBytes, true); if (profile) Profile().storageGpu += timer.lap(); - // The guest bytes now differ from `original`; other caches of the range see the write. With - // every block stored the image is current at a fresh generation; with blocks kept for the CPU - // the image is stale there, so the old generation stays and the next use re-uploads. + // The guest bytes now differ from `original`; other caches of the range see the write. originalValid = false; // The keys go the same way as the texels: a fill recorded behind the retile when the // metadata is host-imported (no CPU wait for the title's key-writing kernels), else a CPU // store (APS5_CPU_DCC_KEYS=1 keeps the CPU store; see DccMetadata.hpp). + traceKeyStore("layer write-back", descriptor, guestBytes); MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); uploadedKeys = DccKeys::Uncompressed; for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); - if (!skippedAny || !adjacent.empty()) { - // The walk covers this surface's pages only: an adjacent image's later CPU write is - // stamped newer than this value when its own range is collected. No CPU wrote the pages - // here (MarkWritten made the only stamps), so the memoized collect is exact. - const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); - if (!skippedAny) generation = now; - advanceAdjacent(adjacent, now, firstBlock, lastBlock); - } + // The walk covers this surface's pages only: an adjacent image's later CPU write is stamped + // newer than this value when its own range is collected. No CPU wrote the pages here + // (MarkWritten made the only stamps), so the memoized collect is exact. + settle(true); ++Profile().storageDirectWriteBacks; return; } - if (!originalValid) { - // The image was cleared on the GPU without reading the guest bytes; the store below compares - // against them, so read them now. - GuestMemory::ReadCommitted(descriptor.baseAddress, original); - originalValid = true; + // A surface whose memory is no longer registered (freed, or re-registered under another + // allocation: no readable registered range contains it) has nothing to receive its results; + // the round trip through the CPU below would store stale texels over whatever the memory + // holds now. Its selected units are dropped instead. APS5_NO_UNREGISTERED_DROP=1 stores. + static const bool unregisteredDrop = std::getenv("APS5_NO_UNREGISTERED_DROP") == nullptr; + if (unregisteredDrop && !RegisteredReadableCovers(descriptor.baseAddress, static_cast(guestBytes))) { + static std::atomic reports{0}; + if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx+0x%llx: memory no longer registered; %zu pending ranges dropped\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), keep.size()); + unregisteredDropped.fetch_add(1, std::memory_order_relaxed); + originalValid = false; + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (layers[layer]) layerPending[layer] = false; + } + return; } + // The store below compares against the guest bytes, read now when `original` does not hold + // them (a GPU clear, a dropped or kept unit). Reading them says nothing about the image: a + // stale unit (dropped, or kept for the CPU) still differs from them, so `original` vouches for + // the content again only where it did before, or once every unit is stored. + const bool wasValid = originalValid; + if (!wasValid) GuestMemory::ReadCommitted(descriptor.baseAddress, original); DeviceBuffer linear(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); DeviceBuffer tiled(context, original.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); Buffer host(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); @@ -1139,6 +3116,7 @@ void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { APS5_LOG_CHARS_OUT("StorageTexture writeback submit"); batch.SubmitAndWait(); APS5_LOG_CHARS_OUT("StorageTexture writeback done"); + countStorageWriteBack(guestBytes, false); if (profile) Profile().storageGpu += timer.lap(); if (dump) { char name[64]; @@ -1150,45 +3128,37 @@ void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { std::fclose(file); } } - // Store only blocks that changed so concurrent CPU writes to untouched texels survive. + // Store only the kept ranges' blocks that changed, so concurrent CPU writes to untouched texels + // survive; `original` follows for the stored layers (the others' guest bytes are unchanged). const auto current = host.Bytes(); - if (skipWrittenSince == 0) { - GuestMemory::WriteChangedCommitted(descriptor.baseAddress, current, original); - } else { - constexpr std::uint64_t block = 65536; - const auto begin = descriptor.baseAddress; - const auto end = begin + guestBytes; - for (auto at = begin & ~(block - 1); at < end; at += block) { - const auto from = std::max(at, begin); - const auto to = std::min(at + block, end); - if (from >= to || !GuestMemory::UnchangedSince(from, static_cast(to - from), skipWrittenSince)) continue; - const auto offset = static_cast(from - begin); - const auto length = static_cast(to - from); - GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span(original).subspan(offset, length)); - } + for (const auto& [from, to] : keep) { + const auto offset = static_cast(from - descriptor.baseAddress); + const auto length = static_cast(to - from); + GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span(original).subspan(offset, length)); + std::memcpy(original.data() + offset, current.data() + offset, length); } - std::memcpy(original.data(), current.data(), original.size()); // The texels now hold the whole image, so later reads must see them rather than a fast clear // (the keys may be host-imported although the texels were not: then a recorded fill, else a CPU store). + traceKeyStore("cpu write-back", descriptor, guestBytes); MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); uploadedKeys = DccKeys::Uncompressed; // The store above is the only write to these pages, so `original` is current at a fresh - // generation, unless blocks were kept for the CPU: then the image is stale there and the old - // generation stays so the next use re-uploads. - if (skippedAny) originalValid = false; - if (!skippedAny || !adjacent.empty()) { - // As in the GPU-direct path, but the store's memcpy dirtied this surface's pages in the write - // watch: a memoized collect would leave them for the next walk to stamp newer than this value - // (the shared edge block included, undoing the advance), so the walk is made here and - // consumes them. - const auto now = GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); - if (!skippedAny) generation = now; - advanceAdjacent(adjacent, now, firstBlock, lastBlock); - } + // generation, unless blocks were kept for the CPU: then the image is stale there. + originalValid = !skippedAny && (wasValid || std::all_of(layers.begin(), layers.end(), [](bool selected) { return selected; })); + // As in the GPU-direct path, but the store's memcpy dirtied this surface's pages in the write + // watch: a memoized collect would leave them for the next walk to stamp newer than this value + // (the shared edge block included, undoing the advance), so the walk is made here and + // consumes them. + settle(false); if (profile) Profile().storageStore += timer.lap(); } StorageTexture::~StorageTexture() { + { + auto& live = Live(); + std::lock_guard lock(live.mutex); + std::erase(live.textures, this); + } // Cache eviction flushes first; anything still pending here is being torn down with the device. { auto& pending = Pending(); @@ -1196,6 +3166,7 @@ StorageTexture::~StorageTexture() { if (dirty) { dirty = false; std::erase(pending.textures, this); + BumpPendingSerial(); static std::atomic reports{0}; if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx destroyed with GPU results pending\n", static_cast(descriptor.baseAddress)); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index b6bcab80d2..21f6ff3e9a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -25,6 +25,12 @@ struct Push { std::uint32_t tailY; std::uint32_t elementBytes; std::uint32_t slice; + std::uint32_t rangeBegin; + std::uint32_t rangeEnd; + std::uint32_t tiledBase; + std::uint32_t linearBase; + std::uint32_t columnBegin; + std::uint32_t rowBegin; }; std::uint32_t BlockBytesFor(TextureTileMode tileMode) { @@ -138,11 +144,15 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem return result; } -void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile, std::uint32_t slice, bool thick) { +void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile, std::uint32_t slice, bool thick, const DetileWindow& window) { Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); + Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty"); Require(source != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "texture detiling requires source and destination buffers"); Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); + const auto columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width; + const auto rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height; + Require(window.columnBegin < columnEnd && window.rowBegin < rowEnd, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height)); const auto target = pipeline(tileMode, elementBytes, retile, thick); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment; @@ -150,8 +160,12 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode const auto sourceBase = sourceOffset - sourceDescriptorOffset; const auto destinationBase = destinationOffset - destinationDescriptorOffset; Require(sourceBase <= UINT32_MAX && destinationBase <= UINT32_MAX, "texture detiling buffer offset exceeds addressable range"); - const auto sourceRange = (sourceBase + (retile ? layout.linearSize : layout.tiledSize) + 3) / 4 * 4; - const auto destinationRange = (destinationBase + (retile ? layout.tiledSize : layout.linearSize) + 3) / 4 * 4; + // A window covers the mip's bytes from its base to the range's end (tiled) or the last row the + // range can touch (linear). + const std::uint64_t tiledBytes = window.rangeEnd == 0xffffffffu ? layout.tiledSize : std::min(layout.tiledSize, window.rangeEnd) - window.tiledBase; + const std::uint64_t linearBytes = window.linearBytes != 0 ? window.linearBytes : layout.linearSize - std::min(window.linearBase, layout.linearSize); + const auto sourceRange = (sourceBase + (retile ? linearBytes : tiledBytes) + 3) / 4 * 4; + const auto destinationRange = (destinationBase + (retile ? tiledBytes : linearBytes) + 3) / 4 * 4; Require(sourceRange <= context.limits.maxStorageBufferRange && destinationRange <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits"); const auto set = allocateSet(); const VkDescriptorBufferInfo sourceInfo{source, sourceDescriptorOffset, sourceRange}; @@ -184,9 +198,15 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode push.tailY = layout.tailY; push.elementBytes = elementBytes; push.slice = slice; + push.rangeBegin = window.rangeBegin; + push.rangeEnd = window.rangeEnd; + push.tiledBase = window.tiledBase; + push.linearBase = window.linearBase; + push.columnBegin = window.columnBegin; + push.rowBegin = window.rowBegin; context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); - const auto groupsX = (layout.width + 7u) / 8u; - const auto groupsY = (layout.height + 7u) / 8u; + const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; + const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; context.Function("vkCmdDispatch")(commands, groupsX, groupsY, 1); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp index 9afbe81677..49af94ea11 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp @@ -249,6 +249,11 @@ std::array ThickBlockExtent(TextureTileMode tileMode, std::uin } } +std::array ThinBlockLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement) { + const auto block = GetBlockLayout(tileMode, bytesPerElement); + return {block.blockSize, block.blockWidth, block.blockHeight}; +} + ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth) { Require(width != 0 && height != 0 && depth != 0, "cannot compute layout for a zero-sized 3D texture"); const auto bytesPerElement = BytesPerElement(format); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp new file mode 100644 index 0000000000..7e3661a1d2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp @@ -0,0 +1,848 @@ +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +namespace { + +constexpr std::uint64_t UnitBytes = 65536; +constexpr std::size_t ReasonCount = static_cast(PublishReason::Count); +constexpr const char* ReasonNames[ReasonCount] = {"hook", "region", "indirect", "copy-src", "copy-dst", "fill", "fill-clear", "label", "keys", "scanout", "validate", "upload", "tail", "evict", "retire", "teardown"}; + +std::uint64_t BudgetBytes() { + static const std::uint64_t budget = [] { + const char* text = std::getenv("APS5_UNIT_SHADOW_MIB"); + return (text != nullptr ? std::strtoull(text, nullptr, 10) : 1024ull) << 20u; + }(); + return budget; +} + +std::uint32_t SlabUnits() { + static const std::uint32_t units = [] { + const char* text = std::getenv("APS5_UNIT_SHADOW_SLAB_MIB"); + const auto mib = text != nullptr ? std::strtoull(text, nullptr, 10) : 8ull; + return static_cast(std::clamp(mib, 1, 1024) * 16); + }(); + return units; +} + +// APS5_TRACE_SHADOW=: the first n publishes, the first 64 seeds, every evict and retire. +std::atomic tracePublishesLeft{[] { + const char* text = std::getenv("APS5_TRACE_SHADOW"); + return text != nullptr ? static_cast(std::strtoll(text, nullptr, 10)) : std::int64_t{0}; +}()}; +std::atomic traceSeedsLeft{64}; + +bool TraceEnabled() { + static const bool enabled = std::getenv("APS5_TRACE_SHADOW") != nullptr; + return enabled; +} + +struct Statistics { + std::atomic retiled{0}, retiledBytes{0}, refused{0}, refusedBytes{0}, seeds{0}, seedBytes{0}; + std::atomic detiledShadow{0}, detiledShadowBytes{0}, detiledImport{0}, detiledImportBytes{0}, tailPublishes{0}; + std::array, ReasonCount> published{}, publishedBytes{}; + std::atomic staleDroppedCpu{0}, staleDroppedDriver{0}, skippedInCompletion{0}, evictions{0}, lostOnRetire{0}, droppedOnRetire{0}, slabLost{0}, verified{0}, mismatches{0}; +}; + +Statistics& Stats() { + static Statistics statistics; + return statistics; +} + +struct UnitShadow { + Context context; + // The import's range and buffer (taken at creation: a publish under the import table's mutex + // must not look them up), and the unit base (the import's base rounded down to a unit). + std::uint64_t importBase = 0; + std::uint64_t importBytes = 0; + VkBuffer importBuffer = VK_NULL_HANDLE; + std::uint64_t base = 0; + // Per unit: 0 = never shadowed, else the tracker generation the slab bytes are current at; + // whether a slab -> import copy was recorded at that generation. + std::vector generation; + std::vector published; + // Slab k covers units [k * SlabUnits(), (k + 1) * SlabUnits()), made on demand; a failed + // allocation is not retried before the noted present. + std::vector> slabs; + std::vector slabFailedUntil; + std::uint32_t liveUnits = 0; + + std::uint64_t Units() const { return generation.size(); } + std::uint64_t UnitBegin(std::uint64_t unit) const { return base + unit * UnitBytes; } + std::uint64_t UnitOf(std::uint64_t address) const { return (address - base) / UnitBytes; } + std::uint64_t ImportEnd() const { return importBase + importBytes; } + // The unit's bytes inside the import. + std::pair UnitRange(std::uint64_t unit) const { return {std::max(UnitBegin(unit), importBase), std::min(UnitBegin(unit + 1), ImportEnd())}; } + bool Live(std::uint64_t unit) const { return generation[unit] != 0 && published[unit] == 0; } + void Recount() { + std::uint32_t live = 0; + for (std::uint64_t unit = 0; unit < Units(); ++unit) live += Live(unit) ? 1u : 0u; + liveUnits = live; + } +}; + +struct Shadows { + std::mutex mutex; + std::map> byBase; + std::uint64_t liveBytes = 0; + std::uint64_t peakBytes = 0; + std::uint32_t liveSlabs = 0; + // Counted for the [storage] segment: shadows retired and the slab bytes they returned. + std::uint64_t retired = 0; +}; + +Shadows& Registry() { + static Shadows shadows; + return shadows; +} + +VkDeviceSize SlabBytes(const ShadowSlab& slab) { + return static_cast(slab.units) * UnitBytes; +} + +// The shadows overlapping [address, end): imports never overlap, so at most a few. +std::vector> overlapping(const Shadows& registry, std::uint64_t address, std::uint64_t end) { + std::vector> result; + if (end <= address || registry.byBase.empty()) return result; + auto it = registry.byBase.upper_bound(address); + if (it != registry.byBase.begin()) --it; + for (; it != registry.byBase.end() && it->first < end; ++it) { + const auto& shadow = it->second; + if (address < shadow->ImportEnd() && shadow->importBase < end) result.push_back(shadow); + } + return result; +} + +std::shared_ptr find(const Shadows& registry, const HostImport& import) { + const auto found = registry.byBase.find(import.base); + if (found == registry.byBase.end()) return nullptr; + const auto& shadow = found->second; + return shadow->importBuffer == import.buffer && shadow->importBytes == import.bytes ? shadow : nullptr; +} + +std::shared_ptr findOrCreate(Shadows& registry, const Context& context, const HostImport& import) { + if (auto shadow = find(registry, import); shadow != nullptr && shadow->context.device == context.device) return shadow; + // A stale entry (an import replaced without a retire, a replaced device): dropped; its slabs + // live on wherever a batch kept them. + if (const auto found = registry.byBase.find(import.base); found != registry.byBase.end()) { + for (const auto& slab : found->second->slabs) { + if (slab == nullptr) continue; + registry.liveBytes -= SlabBytes(*slab); + --registry.liveSlabs; + } + registry.byBase.erase(found); + } + auto shadow = std::make_shared(); + shadow->context = context; + shadow->importBase = import.base; + shadow->importBytes = import.bytes; + shadow->importBuffer = import.buffer; + shadow->base = import.base & ~(UnitBytes - 1); + const auto units = (import.base + import.bytes - shadow->base + UnitBytes - 1) / UnitBytes; + shadow->generation.assign(static_cast(units), 0); + shadow->published.assign(static_cast(units), 0); + const auto slabs = (units + SlabUnits() - 1) / SlabUnits(); + shadow->slabs.assign(static_cast(slabs), nullptr); + shadow->slabFailedUntil.assign(static_cast(slabs), 0); + registry.byBase.emplace(import.base, shadow); + return shadow; +} + +// One tracker query over units [first, last]: `changed[k]` (k = unit - first) whether a block +// stamp exceeds the unit's generation (a generation of 0 reads as changed), `cpu[k]` whether a +// collect made it (a CPU store). +void changedUnits(const UnitShadow& shadow, std::uint64_t first, std::uint64_t last, std::vector& changed, std::vector& cpu) { + const auto count = static_cast(last - first + 1); + changed.assign(count, 1); + cpu.assign(count, 1); + const auto begin = std::max(shadow.UnitBegin(first), shadow.importBase); + const auto end = std::min(shadow.UnitBegin(last + 1), shadow.ImportEnd()); + if (end <= begin) return; + GuestMemory::ChangedBlocks(begin, static_cast(end - begin), std::span(shadow.generation).subspan(static_cast(first), count), changed, cpu); +} + +std::shared_ptr makeSlab(const Context& context, std::uint64_t firstUnit, std::uint32_t units) { + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + info.size = static_cast(units) * UnitBytes; + info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + VkBuffer buffer = VK_NULL_HANDLE; + if (context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer) != VK_SUCCESS) return nullptr; + VkMemoryRequirements requirements{}; + context.Function("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + VkDeviceMemory memory = VK_NULL_HANDLE; + try { + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + if (context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory) != VK_SUCCESS) memory = VK_NULL_HANDLE; + if (memory != VK_NULL_HANDLE && context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0) != VK_SUCCESS) { + context.Function("vkFreeMemory")(context.device, memory, nullptr); + memory = VK_NULL_HANDLE; + } + } catch (const std::exception&) { + memory = VK_NULL_HANDLE; + } + if (memory == VK_NULL_HANDLE) { + context.Function("vkDestroyBuffer")(context.device, buffer, nullptr); + return nullptr; + } + return std::make_shared(context, buffer, memory, firstUnit, units); +} + +struct SlabCopies { + std::shared_ptr slab; + std::vector regions; +}; + +// Records the copies as one command group into the active recorder's open batch (a waited batch +// without one). `ranges` are the guest ranges copied, for the note and the hazard tracker. +void recordCopies(const UnitShadow& shadow, const std::vector& copies, const std::vector>& ranges, std::uint64_t bytes) { + const auto& context = shadow.context; + using CommandClass = Recorder::CommandClass; + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + VkAccessFlags covered = 0; + auto timing = Recorder::NoTiming; + if (recorder != nullptr) { + commands = recorder->Commands(&covered); + timing = recorder->BeginGpuTiming(CommandClass::ShadowPublish); + if (Recorder::BarrierValidate()) recorder->NoteAccess(CommandClass::ShadowPublish, Recorder::Access{{}, ranges, {}, VK_PIPELINE_STAGE_TRANSFER_BIT}); + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } + // The retile that filled the slab (its trailing barrier covers transfer reads and writes) and + // every earlier import writer precede the copies; the CopyBuffer merge rule. + constexpr VkAccessFlags transferAccess = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + if (recorder != nullptr && (covered & transferAccess) == transferAccess && Recorder::MergeBarriers()) { + Recorder::CountMerged(CommandClass::ShadowPublish); + } else { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, transferAccess); + Recorder::CountBarriers(CommandClass::ShadowPublish); + } + const auto copyBuffer = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); + for (const auto& entry : copies) copyBuffer(commands, entry.slab->buffer, shadow.importBuffer, static_cast(entry.regions.size()), entry.regions.data()); + // The published bytes are visible to everything recorded after and to the host, as after a fill. + constexpr VkAccessFlags publishedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, publishedAccess); + Recorder::CountBarriers(CommandClass::ShadowPublish); + if (batch) { + batch->SubmitAndWait(); + return; + } + recorder->EndGpuTiming(timing, bytes); + recorder->MarkCovered(publishedAccess); + for (const auto& entry : copies) recorder->Keep(entry.slab); + // CPU readers reach the import's bytes only after the hook's SyncThrough waits by this note. + recorder->NotePendingWrites(ranges); +} + +// APS5_SHADOW_VERIFY=1: the published bytes read back from the slabs equal the guest bytes. +void verifyPublished(const UnitShadow& shadow, const std::vector& copies) { + const auto& context = shadow.context; + if (auto* recorder = Recorder::Active()) { + Recorder::CountSync(4); + recorder->Sync(); + } + for (const auto& entry : copies) { + for (const auto& region : entry.regions) { + Buffer host(context, static_cast(region.size), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + CommandBatch batch(context); + CopyBuffer(context, batch.Handle(), entry.slab->buffer, region.srcOffset, host.Handle(), 0, region.size); + RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + batch.SubmitAndWait(); + host.Invalidate(); + const auto address = shadow.importBase + region.dstOffset; + Stats().verified.fetch_add(1, std::memory_order_relaxed); + if (!GuestMemory::Accessible(reinterpret_cast(address), static_cast(region.size)) || std::memcmp(reinterpret_cast(address), host.Bytes().data(), static_cast(region.size)) != 0) { + Stats().mismatches.fetch_add(1, std::memory_order_relaxed); + static std::atomic reports{0}; + if (reports.fetch_add(1) < 8) std::fprintf(stderr, "[shadow] verify: published bytes differ from the slab at 0x%llx+0x%llx\n", static_cast(address), static_cast(region.size)); + } + } + } +} + +// The publish of one shadow: `selected[k]` (k = unit - first) names the units the scope chose. +std::size_t publishUnits(const std::shared_ptr& shadow, std::uint64_t first, std::uint64_t last, const std::vector& selected, PublishReason reason) { + auto& registry = Registry(); + std::vector copies; + std::vector> ranges; + std::uint64_t bytes = 0; + std::size_t units = 0; + { + // The stamps judged below see every CPU store into the live selected units up to now: a + // store no collect walked yet would read as fresh and the copy would land over it. Every + // caller holds the GpuMutex, so the live set is stable across the walk. + std::uint64_t walkFirst = last + 1, walkLast = first; + { + std::lock_guard lock(registry.mutex); + for (auto unit = first; unit <= last; ++unit) { + if (!selected[static_cast(unit - first)] || !shadow->Live(unit)) continue; + walkFirst = std::min(walkFirst, unit); + walkLast = std::max(walkLast, unit); + } + } + if (walkFirst > walkLast) return 0; + const auto walkBegin = std::max(shadow->UnitBegin(walkFirst), shadow->importBase); + const auto walkEnd = std::min(shadow->UnitBegin(walkLast + 1), shadow->ImportEnd()); + if (walkEnd > walkBegin) GuestMemory::CollectWritesUncached(walkBegin, static_cast(walkEnd - walkBegin)); + } + { + std::lock_guard lock(registry.mutex); + std::vector changed, cpu; + changedUnits(*shadow, first, last, changed, cpu); + for (auto unit = first; unit <= last; ++unit) { + const auto k = static_cast(unit - first); + if (!selected[k] || !shadow->Live(unit)) continue; + if (changed[k] != 0) { + // Newer bytes reached the import since the retile: the slab's are dead. + shadow->generation[unit] = 0; + (cpu[k] != 0 ? Stats().staleDroppedCpu : Stats().staleDroppedDriver).fetch_add(1, std::memory_order_relaxed); + continue; + } + const auto slabIndex = static_cast(unit / SlabUnits()); + const auto& slab = shadow->slabs[slabIndex]; + if (slab == nullptr) { + shadow->generation[unit] = 0; + continue; + } + const auto [begin, end] = shadow->UnitRange(unit); + if (end <= begin) continue; + const VkBufferCopy region{(unit - slab->firstUnit) * UnitBytes + (begin - shadow->UnitBegin(unit)), begin - shadow->importBase, end - begin}; + if (!copies.empty() && copies.back().slab == slab && copies.back().regions.back().dstOffset + copies.back().regions.back().size == region.dstOffset) { + copies.back().regions.back().size += region.size; + ranges.back().second = end; + } else { + if (copies.empty() || copies.back().slab != slab) copies.push_back({slab, {}}); + copies.back().regions.push_back(region); + ranges.emplace_back(begin, end); + } + shadow->published[unit] = 1; + slab->lastUse = Recorder::Presents(); + bytes += end - begin; + ++units; + } + shadow->Recount(); + } + if (units == 0) return 0; + Stats().published[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); + Stats().publishedBytes[static_cast(reason)].fetch_add(bytes, std::memory_order_relaxed); + if (tracePublishesLeft.load(std::memory_order_relaxed) > 0 && tracePublishesLeft.fetch_sub(1, std::memory_order_relaxed) > 0) { + const auto packet = GuestMemory::CurrentPacket(); + std::fprintf(stderr, "[shadow] publish %s: 0x%llx+0x%llx, %zu units, %.2f MiB, %zu slabs (packet 0x%x queue 0x%x)\n", ReasonNames[static_cast(reason)], static_cast(ranges.front().first), static_cast(ranges.back().second - ranges.front().first), units, bytes / 1048576.0, copies.size(), packet.opcode, packet.queue); + } + recordCopies(*shadow, copies, ranges, bytes); + if (ShadowVerify()) verifyPublished(*shadow, copies); + return units; +} + +std::size_t publishRange(const std::shared_ptr& shadow, std::uint64_t address, std::uint64_t end, PublishScope scope, PublishReason reason) { + const auto begin = std::max(address, shadow->importBase); + const auto stop = std::min(end, shadow->ImportEnd()); + if (stop <= begin || shadow->liveUnits == 0) return 0; + const auto first = shadow->UnitOf(begin); + const auto last = shadow->UnitOf(stop - 1); + std::vector selected(static_cast(last - first + 1), 0); + bool any = false; + for (auto unit = first; unit <= last; ++unit) { + const auto [unitBegin, unitEnd] = shadow->UnitRange(unit); + const bool whole = address <= unitBegin && end >= unitEnd; + if (scope == PublishScope::PartialUnits && whole) continue; + selected[static_cast(unit - first)] = 1; + any = true; + } + return any ? publishUnits(shadow, first, last, selected, reason) : 0; +} + +// Frees the least recently used unpinned slab of any shadow after publishing its fresh units; false +// when nothing can be freed (a pinned slab is a destination of the write-back in progress: its +// copies are recorded into it after this decision). +bool evictOne(Shadows& registry) { + std::shared_ptr victim; + std::size_t victimSlab = 0; + { + std::lock_guard lock(registry.mutex); + std::uint64_t oldest = ~0ull; + for (const auto& [base, shadow] : registry.byBase) { + for (std::size_t k = 0; k < shadow->slabs.size(); ++k) { + const auto& slab = shadow->slabs[k]; + if (slab != nullptr && slab->pins.load(std::memory_order_relaxed) == 0 && slab->lastUse < oldest) { + oldest = slab->lastUse; + victim = shadow; + victimSlab = k; + } + } + } + } + if (victim == nullptr) return false; + const auto firstUnit = static_cast(victimSlab) * SlabUnits(); + const auto lastUnit = std::min(firstUnit + SlabUnits(), victim->Units()) - 1; + const auto published = publishRange(victim, victim->UnitBegin(firstUnit), victim->UnitBegin(lastUnit + 1), PublishScope::Whole, PublishReason::Evict); + std::lock_guard lock(registry.mutex); + auto& slab = victim->slabs[victimSlab]; + if (slab == nullptr) return true; + if (TraceEnabled()) std::fprintf(stderr, "[shadow] evict slab %zu of import 0x%llx (%.1f MiB, %zu units published)\n", victimSlab, static_cast(victim->importBase), SlabBytes(*slab) / 1048576.0, published); + // Its units read from the import from now on (the publish put their bytes there). + for (auto unit = firstUnit; unit <= lastUnit; ++unit) { + victim->generation[unit] = 0; + victim->published[unit] = 0; + } + victim->Recount(); + registry.liveBytes -= SlabBytes(*slab); + --registry.liveSlabs; + slab.reset(); + Stats().evictions.fetch_add(1, std::memory_order_relaxed); + return true; +} + +} + +bool UnitShadowEnabled() { + static const bool disabled = std::getenv("APS5_NO_UNIT_SHADOW") != nullptr; + return !disabled; +} + +bool ShadowVerify() { + static const bool verify = std::getenv("APS5_SHADOW_VERIFY") != nullptr; + return verify; +} + +void NoteShadowSeed(std::uint64_t begin, std::uint64_t end) { + Stats().seeds.fetch_add(1, std::memory_order_relaxed); + Stats().seedBytes.fetch_add(end - begin, std::memory_order_relaxed); + if (traceSeedsLeft.load(std::memory_order_relaxed) > 0 && traceSeedsLeft.fetch_sub(1, std::memory_order_relaxed) > 0) std::fprintf(stderr, "[shadow] seed 0x%llx+0x%llx from the import\n", static_cast(begin), static_cast(end - begin)); +} + +PublishReason PublishReasonFor(const char* flushReason) { + const auto is = [&](const char* name) { return flushReason != nullptr && std::strcmp(flushReason, name) == 0; }; + if (is("memory access")) return GuestMemory::CurrentReadSite() == GuestMemory::ReadSite::Scanout ? PublishReason::Scanout : PublishReason::Hook; + if (is("imported buffer region") || is("copied buffer region") || is("sampled texture")) return PublishReason::Region; + if (is("indirect dispatch arguments") || is("indirect draw arguments")) return PublishReason::Indirect; + if (is("buffer copy source")) return PublishReason::CopySource; + if (is("buffer copy destination")) return PublishReason::CopyDestination; + if (is("buffer fill")) return PublishReason::Fill; + if (is("packet store")) return PublishReason::Label; + if (is("buffer copy count") || is("dispatch-cache entry")) return PublishReason::Validate; + if (is("storage image creation") || is("storage refresh")) return PublishReason::Upload; + return PublishReason::Hook; +} + +ShadowSlab::ShadowSlab(const Context& context, VkBuffer buffer, VkDeviceMemory memory, std::uint64_t firstUnit, std::uint32_t units) : context(context), buffer(buffer), memory(memory), firstUnit(firstUnit), units(units) {} + +ShadowSlab::~ShadowSlab() { + if (buffer != VK_NULL_HANDLE) context.Function("vkDestroyBuffer")(context.device, buffer, nullptr); + if (memory != VK_NULL_HANDLE) context.Function("vkFreeMemory")(context.device, memory, nullptr); +} + +ShadowSlabPin::ShadowSlabPin(std::shared_ptr slab) : slab(std::move(slab)) { + this->slab->pins.fetch_add(1, std::memory_order_relaxed); +} + +ShadowSlabPin::~ShadowSlabPin() { + slab->pins.fetch_sub(1, std::memory_order_relaxed); +} + +std::size_t PublishShadow(std::uint64_t address, std::size_t bytes, PublishScope scope, PublishReason reason) { + if (!UnitShadowEnabled() || scope == PublishScope::None || bytes == 0) return 0; + if (Recorder::InCompletion() && GuestMemory::CurrentReadSite() == GuestMemory::ReadSite::Store) { + // The store's own stamp makes the unit stale; a publish recorded now would land over the + // store when the batch runs (the completion is not waited for). + Stats().skippedInCompletion.fetch_add(1, std::memory_order_relaxed); + return 0; + } + const auto end = bytes > std::numeric_limits::max() - address ? std::numeric_limits::max() : address + bytes; + std::vector> shadows; + { + auto& registry = Registry(); + std::lock_guard lock(registry.mutex); + shadows = overlapping(registry, address, end); + } + std::size_t units = 0; + for (const auto& shadow : shadows) units += publishRange(shadow, address, end, scope, reason); + return units; +} + +bool AnyShadowedOverlaps(std::uint64_t address, std::size_t bytes) { + if (!UnitShadowEnabled() || bytes == 0) return false; + const auto end = bytes > std::numeric_limits::max() - address ? std::numeric_limits::max() : address + bytes; + auto& registry = Registry(); + std::lock_guard lock(registry.mutex); + for (const auto& shadow : overlapping(registry, address, end)) { + if (shadow->liveUnits == 0) continue; + const auto begin = std::max(address, shadow->importBase); + const auto stop = std::min(end, shadow->ImportEnd()); + for (auto unit = shadow->UnitOf(begin); unit <= shadow->UnitOf(stop - 1); ++unit) { + if (shadow->Live(unit)) return true; + } + } + return false; +} + +bool AnyShadowedOverlaps(std::span> ranges) { + if (!UnitShadowEnabled()) return false; + for (const auto& [begin, end] : ranges) { + if (end > begin && AnyShadowedOverlaps(begin, static_cast(end - begin))) return true; + } + return false; +} + +std::uint64_t SlabBoundary(const HostImport& import, std::uint64_t address) { + const auto base = import.base & ~(UnitBytes - 1); + const auto unit = (address - base) / UnitBytes; + return base + (unit / SlabUnits() + 1) * SlabUnits() * UnitBytes; +} + +VkDeviceSize SlabOffset(const HostImport& import, const ShadowSlab& slab, std::uint64_t address) { + const auto base = import.base & ~(UnitBytes - 1); + return (address - base) - slab.firstUnit * UnitBytes; +} + +std::optional ShadowDestinationFor(const Context& context, const HostImport& import, std::uint64_t begin, std::uint64_t end) { + if (!UnitShadowEnabled() || end <= begin || begin < import.base || end > import.base + import.bytes) return std::nullopt; + auto& registry = Registry(); + std::shared_ptr shadow; + std::size_t slabIndex = 0; + std::uint64_t first = 0, last = 0; + bool missing = false; + { + std::lock_guard lock(registry.mutex); + shadow = findOrCreate(registry, context, import); + first = shadow->UnitOf(begin); + last = shadow->UnitOf(end - 1); + slabIndex = static_cast(first / SlabUnits()); + if (last / SlabUnits() != slabIndex) return std::nullopt; + missing = shadow->slabs[slabIndex] == nullptr; + if (missing && shadow->slabFailedUntil[slabIndex] > Recorder::Presents()) return std::nullopt; + } + if (missing) { + const auto firstUnit = static_cast(slabIndex) * SlabUnits(); + const auto units = static_cast(std::min(SlabUnits(), shadow->Units() - firstUnit)); + const auto needed = static_cast(units) * UnitBytes; + for (;;) { + { + std::lock_guard lock(registry.mutex); + if (registry.liveBytes + needed <= BudgetBytes()) break; + } + if (!evictOne(registry)) { + Stats().refused.fetch_add(1, std::memory_order_relaxed); + Stats().refusedBytes.fetch_add(end - begin, std::memory_order_relaxed); + return std::nullopt; + } + } + auto slab = makeSlab(context, firstUnit, units); + std::lock_guard lock(registry.mutex); + if (slab == nullptr) { + shadow->slabFailedUntil[slabIndex] = Recorder::Presents() + 1; + Stats().refused.fetch_add(1, std::memory_order_relaxed); + Stats().refusedBytes.fetch_add(end - begin, std::memory_order_relaxed); + return std::nullopt; + } + if (shadow->slabs[slabIndex] == nullptr) { + shadow->slabs[slabIndex] = slab; + registry.liveBytes += SlabBytes(*slab); + registry.peakBytes = std::max(registry.peakBytes, registry.liveBytes); + ++registry.liveSlabs; + } + } + // A unit the copy covers partly keeps its other bytes from the slab when fresh, else from a + // seed of the import: the stamps must see the CPU's stores into it first (the write-back's own + // walk covers the stored span only, not the unit's remainder outside the surface). + for (auto unit = first; unit <= last; unit = last) { + const auto [unitBegin, unitEnd] = shadow->UnitRange(unit); + if (!(begin <= unitBegin && end >= unitEnd) && unitEnd > unitBegin) GuestMemory::CollectWritesUncached(unitBegin, static_cast(unitEnd - unitBegin)); + if (unit == last) break; + } + std::lock_guard lock(registry.mutex); + const auto& slab = shadow->slabs[slabIndex]; + if (slab == nullptr) return std::nullopt; + slab->lastUse = Recorder::Presents(); + ShadowDestination destination{slab->buffer, SlabOffset(import, *slab, begin), slab, {}, std::make_shared(slab)}; + std::vector changed, cpu; + bool queried = false; + for (auto unit = first; unit <= last; ++unit) { + const auto [unitBegin, unitEnd] = shadow->UnitRange(unit); + if (begin <= unitBegin && end >= unitEnd) continue; + if (!queried) { + changedUnits(*shadow, first, last, changed, cpu); + queried = true; + } + const auto k = static_cast(unit - first); + if (shadow->generation[unit] != 0 && changed[k] == 0) continue; + destination.seedUnits.emplace_back(unitBegin, unitEnd); + } + return destination; +} + +std::vector ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span> runs, std::span> tailBlocks) { + std::vector result; + if (runs.empty()) return result; + const auto importPiece = [&](std::uint64_t begin, std::uint64_t end) { + result.push_back({begin, end, import.buffer, surfaceBase + begin - import.base, false, nullptr}); + Stats().detiledImportBytes.fetch_add(end - begin, std::memory_order_relaxed); + }; + std::shared_ptr shadow; + if (UnitShadowEnabled()) { + auto& registry = Registry(); + std::lock_guard lock(registry.mutex); + shadow = find(registry, import); + if (shadow != nullptr && (shadow->context.device != context.device || (shadow->liveUnits == 0 && std::none_of(shadow->generation.begin(), shadow->generation.end(), [](std::uint64_t generation) { return generation != 0; })))) shadow = nullptr; + } + if (shadow == nullptr) { + for (const auto& [begin, end] : runs) importPiece(begin, end); + Stats().detiledImport.fetch_add(1, std::memory_order_relaxed); + return result; + } + const auto first = shadow->UnitOf(surfaceBase + runs.front().first); + const auto last = shadow->UnitOf(surfaceBase + runs.back().second - 1); + std::vector fresh(static_cast(last - first + 1), 0); + std::vector> slabs(shadow->slabs.size()); + { + auto& registry = Registry(); + std::lock_guard lock(registry.mutex); + std::vector changed, cpu; + changedUnits(*shadow, first, last, changed, cpu); + for (auto unit = first; unit <= last; ++unit) { + const auto k = static_cast(unit - first); + const auto& slab = shadow->slabs[static_cast(unit / SlabUnits())]; + fresh[k] = shadow->generation[unit] != 0 && changed[k] == 0 && slab != nullptr ? 1 : 0; + } + slabs = shadow->slabs; + } + // A tail block is moved whole by one window: its units take one source. + for (const auto& [tailBegin, tailEnd] : tailBlocks) { + if (tailEnd <= tailBegin) continue; + const auto tailFirst = shadow->UnitOf(surfaceBase + tailBegin); + const auto tailLast = shadow->UnitOf(surfaceBase + tailEnd - 1); + if (tailFirst < first || tailLast > last) continue; + bool anyFresh = false, anyImport = false; + for (auto unit = tailFirst; unit <= tailLast; ++unit) (fresh[static_cast(unit - first)] != 0 ? anyFresh : anyImport) = true; + if (!anyFresh || !anyImport) continue; + PublishShadow(surfaceBase + tailBegin, static_cast(tailEnd - tailBegin), PublishScope::Whole, PublishReason::TailMip); + Stats().tailPublishes.fetch_add(1, std::memory_order_relaxed); + for (auto unit = tailFirst; unit <= tailLast; ++unit) fresh[static_cast(unit - first)] = 0; + } + std::uint64_t shadowBytes = 0, importBytes = 0; + const auto present = Recorder::Presents(); + for (const auto& [runBegin, runEnd] : runs) { + if (runEnd <= runBegin) continue; + const auto guestBegin = surfaceBase + runBegin; + const auto guestEnd = surfaceBase + runEnd; + for (auto unit = shadow->UnitOf(guestBegin); unit <= shadow->UnitOf(guestEnd - 1); ++unit) { + const auto pieceBegin = std::max(guestBegin, shadow->UnitBegin(unit)) - surfaceBase; + const auto pieceEnd = std::min(guestEnd, shadow->UnitBegin(unit + 1)) - surfaceBase; + if (pieceEnd <= pieceBegin) continue; + const bool isFresh = fresh[static_cast(unit - first)] != 0; + const auto& slab = isFresh ? slabs[static_cast(unit / SlabUnits())] : nullptr; + if (!result.empty() && result.back().end == pieceBegin && result.back().shadow == isFresh && result.back().slab == slab) { + result.back().end = pieceEnd; + } else if (isFresh) { + slab->lastUse = present; + result.push_back({pieceBegin, pieceEnd, slab->buffer, SlabOffset(import, *slab, surfaceBase + pieceBegin), true, slab}); + } else { + result.push_back({pieceBegin, pieceEnd, import.buffer, surfaceBase + pieceBegin - import.base, false, nullptr}); + } + (isFresh ? shadowBytes : importBytes) += pieceEnd - pieceBegin; + } + } + if (shadowBytes != 0) { + Stats().detiledShadow.fetch_add(1, std::memory_order_relaxed); + Stats().detiledShadowBytes.fetch_add(shadowBytes, std::memory_order_relaxed); + } + if (importBytes != 0) { + Stats().detiledImport.fetch_add(1, std::memory_order_relaxed); + Stats().detiledImportBytes.fetch_add(importBytes, std::memory_order_relaxed); + } + if (ShadowVerify()) { + std::vector changed, cpu; + std::lock_guard lock(Registry().mutex); + changedUnits(*shadow, first, last, changed, cpu); + for (const auto& run : result) { + if (!run.shadow) continue; + for (auto unit = shadow->UnitOf(surfaceBase + run.begin); unit <= shadow->UnitOf(surfaceBase + run.end - 1); ++unit) { + Stats().verified.fetch_add(1, std::memory_order_relaxed); + if (changed[static_cast(unit - first)] != 0) Stats().mismatches.fetch_add(1, std::memory_order_relaxed); + } + } + } + return result; +} + +void MarkShadowed(const HostImport& import, std::span ranges, std::uint64_t generation) { + if (!UnitShadowEnabled() || ranges.empty() || generation == 0) return; + auto& registry = Registry(); + std::uint64_t bytes = 0; + std::uint64_t lost = 0; + { + std::lock_guard lock(registry.mutex); + const auto shadow = find(registry, import); + if (shadow == nullptr) return; + const auto present = Recorder::Presents(); + for (const auto& [begin, end, slab] : ranges) { + if (end <= begin) continue; + bytes += end - begin; + for (auto unit = shadow->UnitOf(begin); unit <= shadow->UnitOf(end - 1); ++unit) { + const auto& current = shadow->slabs[static_cast(unit / SlabUnits())]; + if (current == nullptr || current != slab) { + // The copies went into a slab the shadow no longer holds: the import keeps + // whatever it had, as after a refused piece. + ++lost; + continue; + } + shadow->generation[unit] = generation; + shadow->published[unit] = 0; + current->lastUse = present; + } + } + shadow->Recount(); + } + if (lost != 0) { + Stats().slabLost.fetch_add(lost, std::memory_order_relaxed); + static std::atomic reports{0}; + if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[shadow] %llu retiled units of import 0x%llx lost their slab before they were marked\n", static_cast(lost), static_cast(import.base)); + } + Stats().retiled.fetch_add(1, std::memory_order_relaxed); + Stats().retiledBytes.fetch_add(bytes, std::memory_order_relaxed); + // The resource cache's pending memos (fastRevalidate) must see the change of the surface's + // source, as they see a write-back. + StorageTexture::BumpPendingSerial(); +} + +void RetireShadow(const Context& context, const HostImport& import, const std::function& registered) { + if (!UnitShadowEnabled()) return; + auto& registry = Registry(); + std::shared_ptr shadow; + std::vector selected; + bool any = false; + std::uint64_t dropped = 0; + { + std::lock_guard lock(registry.mutex); + shadow = find(registry, import); + if (shadow == nullptr) return; + // The memory the title took back holds its bytes now: only a unit still inside a readable + // registered range receives its results. + selected.assign(static_cast(shadow->Units()), 0); + for (std::uint64_t unit = 0; unit < shadow->Units(); ++unit) { + if (!shadow->Live(unit)) continue; + const auto [begin, end] = shadow->UnitRange(unit); + if (end > begin && registered(begin, end)) { + selected[static_cast(unit)] = 1; + any = true; + } else { + shadow->generation[unit] = 0; + ++dropped; + } + } + shadow->Recount(); + } + if (dropped != 0) Stats().droppedOnRetire.fetch_add(dropped, std::memory_order_relaxed); + std::size_t published = 0; + if (any) { + if (GuestMemory::GpuMutex().HeldByThisThread() && shadow->context.device == context.device) { + published = publishUnits(shadow, 0, shadow->Units() - 1, selected, PublishReason::Retire); + } else { + Stats().lostOnRetire.fetch_add(shadow->liveUnits, std::memory_order_relaxed); + static std::atomic reports{0}; + if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[shadow] import 0x%llx+0x%llx retired outside the device lock: %u shadowed units lost\n", static_cast(import.base), static_cast(import.bytes), shadow->liveUnits); + } + } + std::lock_guard lock(registry.mutex); + const auto found = registry.byBase.find(import.base); + if (found == registry.byBase.end() || found->second != shadow) return; + for (const auto& slab : shadow->slabs) { + if (slab == nullptr) continue; + registry.liveBytes -= SlabBytes(*slab); + --registry.liveSlabs; + } + ++registry.retired; + if (TraceEnabled()) std::fprintf(stderr, "[shadow] retire import 0x%llx+0x%llx: %zu units published, %llu dropped (memory no longer registered)\n", static_cast(import.base), static_cast(import.bytes), published, static_cast(dropped)); + registry.byBase.erase(found); +} + +void PublishAllShadows(const Context& context, PublishReason reason) { + if (!UnitShadowEnabled()) return; + std::vector> shadows; + { + auto& registry = Registry(); + std::lock_guard lock(registry.mutex); + for (const auto& [base, shadow] : registry.byBase) { + if (shadow->context.device == context.device) shadows.push_back(shadow); + } + } + for (const auto& shadow : shadows) publishRange(shadow, shadow->importBase, shadow->ImportEnd(), PublishScope::Whole, reason); +} + +void DestroyShadows(VkDevice device) { + auto& registry = Registry(); + std::lock_guard lock(registry.mutex); + for (auto it = registry.byBase.begin(); it != registry.byBase.end();) { + if (it->second->context.device != device) { + ++it; + continue; + } + for (const auto& slab : it->second->slabs) { + if (slab == nullptr) continue; + registry.liveBytes -= SlabBytes(*slab); + --registry.liveSlabs; + } + it = registry.byBase.erase(it); + } +} + +std::string ShadowReport() { + auto& statistics = Stats(); + const auto take = [](std::atomic& counter) { return static_cast(counter.exchange(0, std::memory_order_relaxed)); }; + const auto mib = [](unsigned long long bytes) { return bytes / 1048576.0; }; + char text[256]; + std::string line; + if (!UnitShadowEnabled()) return "; shadow: off"; + std::snprintf(text, sizeof(text), "; shadow: retiled %llu/%.1f (refused %llu/%.1f to import), seeds %llu/%.1f, detiled from shadow %llu/%.1f, from import %llu/%.1f, tail publishes %llu, published", take(statistics.retiled), mib(take(statistics.retiledBytes)), take(statistics.refused), mib(take(statistics.refusedBytes)), take(statistics.seeds), mib(take(statistics.seedBytes)), take(statistics.detiledShadow), mib(take(statistics.detiledShadowBytes)), take(statistics.detiledImport), mib(take(statistics.detiledImportBytes)), take(statistics.tailPublishes)); + line += text; + unsigned long long publishedTotal = 0, publishedBytesTotal = 0; + std::string reasons; + for (std::size_t i = 0; i < ReasonCount; ++i) { + const auto count = take(statistics.published[i]); + const auto bytes = take(statistics.publishedBytes[i]); + publishedTotal += count; + publishedBytesTotal += bytes; + if (count == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu/%.1f", ReasonNames[i], count, mib(bytes)); + reasons += text; + } + const auto cpuDropped = take(statistics.staleDroppedCpu), driverDropped = take(statistics.staleDroppedDriver); + unsigned long long liveSlabs = 0, liveBytes = 0, peakBytes = 0, retired = 0; + { + auto& registry = Registry(); + std::lock_guard lock(registry.mutex); + liveSlabs = registry.liveSlabs; + liveBytes = registry.liveBytes; + peakBytes = registry.peakBytes; + retired = registry.retired; + } + std::snprintf(text, sizeof(text), " %llu/%.1f by reason {%s }, stale-dropped %llu (cpu %llu, driver %llu), skipped-in-completion %llu, slabs %llu live %.1f MiB / peak %.1f MiB, evictions %llu, slab lost %llu, retired %llu, dropped on retire %llu, lost on retire %llu", publishedTotal, mib(publishedBytesTotal), reasons.c_str(), cpuDropped + driverDropped, cpuDropped, driverDropped, take(statistics.skippedInCompletion), liveSlabs, mib(liveBytes), mib(peakBytes), take(statistics.evictions), take(statistics.slabLost), retired, take(statistics.droppedOnRetire), take(statistics.lostOnRetire)); + line += text; + if (ShadowVerify()) { + std::snprintf(text, sizeof(text), ", verified %llu, mismatches %llu", take(statistics.verified), take(statistics.mismatches)); + line += text; + } + return line; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index 6e136b79ec..d5c87f22ec 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -1,5 +1,8 @@ #include "BdaTests.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestHeap.hpp" +#include #include #include #include @@ -118,4 +121,36 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { reject([&] { unaligned.Descriptor(address + 4, 4); }, "offset alignment"); reject([&] { unaligned.Descriptor(address + sizeof(guest), 4); }, "exceeds its GPU owner"); } + { + // The cached address space: a second build in an unchanged registry takes the first one's + // space, and a guest free of a range pinned only by the cache goes through the pin waiter's + // drop (no GPU work to wait for) and empties the registry of it. + void* block = GuestHeap::GuestHeapAllocate_nid_postfix(64); + const auto blockAddress = reinterpret_cast(block); + std::memset(block, 0x5a, 64); + const auto registered = [&] { + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + return std::any_of(lease.begin(), lease.end(), [&](const auto& range) { return range->address == blockAddress; }); + }; + Require(registered(), "guest heap block is not registered"); + const auto before = AddressSpaceCounters(); + for (int build = 0; build < 2; ++build) { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + Require(leased.HoldsLease(), "address-based build holds no lease"); + leased.Upload(true); + const auto view = leased.Descriptor(blockAddress, 64); + Require(view.range == 64, "leased block has no descriptor"); + const auto ranges = leased.AddressRanges(); + Require(std::any_of(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == blockAddress && range.end == blockAddress + 64; }), "leased block is missing from the BDA table"); + leased.WriteBack(); + Require(!leased.HoldsLease(), "write-back kept the lease"); + } + const auto after = AddressSpaceCounters(); + if (after.enabled) Require(after.hits == before.hits + 1 && after.rebuiltFirst + after.rebuiltGeneration + after.rebuiltWaiterDrop + after.rebuiltEpoch + after.rebuiltDevice == before.rebuiltFirst + before.rebuiltGeneration + before.rebuiltWaiterDrop + before.rebuiltEpoch + before.rebuiltDevice + 1, "second build did not take the cached address space"); + GuestHeap::GuestHeapFree_nid_postfix(block); + Require(!registered(), "freed guest heap block remains registered"); + const auto dropped = AddressSpaceCounters(); + if (dropped.enabled) Require(dropped.waiterDrops == after.waiterDrops + 1 && LeaseCounters().cacheDrops == dropped.waiterDrops, "the free did not drop the cached address space"); + } } diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index 9554c7b760..a96f0b446f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include #include #include @@ -114,6 +115,23 @@ void stateTests() { queue = makeState(); queue.context[0x10f] = 0x7fc00000; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "non-finite"); + // The register facade: every register the decoders read is in DrawKeyRegisters (a wrong hit of + // the draw key otherwise), and nothing is recorded without a log pointer. + queue = makeState(); + std::vector log; + AgcDriver::Graphics::RegisterReadLog() = &log; + queue.context[0x1b3] = 2; + queue.context[0x1b4] = 2; + state = AgcDriver::Graphics::DecodeState(queue); + Require(AgcDriver::Graphics::DrawRejection(queue).empty(), "precheck rejected the reference state"); + static_cast(AgcDriver::Graphics::DecodePixelStageInfo(queue.context, state.hasColorTarget, state.color.componentMapping)); + AgcDriver::Graphics::RegisterReadLog() = nullptr; + Require(!log.empty(), "the register facade recorded nothing"); + for (const auto read : log) Require(AgcDriver::Graphics::DrawKeyCovers(read), "DrawKeyRegisters lacks a register the decoders read: " + std::string(AgcDriver::Graphics::RegisterBankName(read.bank)) + " " + std::to_string(read.offset)); + Require(!AgcDriver::Graphics::DrawKeyCovers({AgcDriver::Graphics::RegisterBank::Context, 0x100}) && AgcDriver::Graphics::DrawKeyCovers({AgcDriver::Graphics::RegisterBank::Shader, 0xab}) && !AgcDriver::Graphics::DrawKeyCovers({AgcDriver::Graphics::RegisterBank::Shader, 0xac}), "DrawKeyRegisters coverage changed"); + log.clear(); + static_cast(AgcDriver::Graphics::DecodeState(queue)); + Require(log.empty(), "the register facade recorded without a log"); } void hardwareScreenOffsetTests() { @@ -430,8 +448,10 @@ VKAPI_ATTR void VKAPI_CALL mockDestroyShaderModule(VkDevice, VkShaderModule, con VKAPI_ATTR VkResult VKAPI_CALL mockCreateComputePipelines(VkDevice, VkPipelineCache, std::uint32_t count, const VkComputePipelineCreateInfo* infos, const VkAllocationCallbacks*, VkPipeline* pipelines) { Require(count == 1, "mock expects exactly one compute pipeline per call"); Require(infos[0].stage.pSpecializationInfo != nullptr, "compute pipeline must provide specialization data"); + // The detiler's constants 0-2 (element size, block size, addressing family) are what the tests + // check; the swizzle equation and block extent follow them. std::array values{}; - Require(infos[0].stage.pSpecializationInfo->dataSize == sizeof(values), "compute pipeline specialization data has an unexpected size"); + Require(infos[0].stage.pSpecializationInfo->dataSize >= sizeof(values), "compute pipeline specialization data has an unexpected size"); std::memcpy(values.data(), infos[0].stage.pSpecializationInfo->pData, sizeof(values)); *pipelines = makeHandle(); mock.pipelineSpecializations.push_back(values); diff --git a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp index 75629c05f3..057638e04a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp @@ -189,6 +189,61 @@ void testIndexedDraw() { expectFailure([&] { AgcDriver::Pm4::ResolveDraw(packet, state); }, "guest"); } +void testIndirectDraw() { + check(AgcDriver::Pm4::AccessesMemory(0xc0032400u) && AgcDriver::Pm4::AccessesMemory(0xc0083800u), "indirect draws must synchronize guest memory"); + check(AgcDriver::Pm4::UnsupportedReason(0xc0032400u).empty() && AgcDriver::Pm4::UnsupportedReason(0xc0082c00u).empty(), "indirect draws are rejected"); + AgcDriver::QueueState state; + state.userConfig[0x24a] = 5; + const auto packet = makePacket(0x24, {0, 0x280, 0x8e, 2}); + AgcDriver::Pm4::Validate(packet, 0); + expectFailure([&] { AgcDriver::Pm4::Validate(packet, 0x20); }, "compute queue"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x24, {2, 0x280, 0x8e, 2}), 0); }, "misaligned indirect draw offset"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x24, {0, 0x10280, 0x8e, 2}), 0); }, "start-index location"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x24, {0, 0x200, 0x8e, 2}), 0); }, "register location"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x24, {0, 0x280, 0x8e, 0}), 0); }, "initiator"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x25, {0, 0x280, 0x8e, 2}), 0); }, "initiator"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x24, {0, 0x280, 0x8e}), 0); }, "packet size"); + AgcDriver::Pm4::Validate(makePacket(0x2c, {0, 0x8c, 0x280, 0x8d | (1u << 31u), 3, 0, 0, 16, 2}), 0); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x2c, {0, 0x280, 0x8e, 0x280 | (1u << 27u), 3, 0, 0, 16, 2}), 0); }, "control bits"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x2c, {0, 0x280, 0x8e, 0x280, 3, 0, 0, 12, 2}), 0); }, "stride"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x38, {0, 0x280, 0x8e, 0x280, 3, 0, 0, 16, 0}), 0); }, "stride"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x2c, {0, 0x280, 0x8e, 0x280 | (1u << 30u), 3, 0, 0, 16, 2}), 0); }, "count address"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x2c, {0, 0x280, 0x8e, 0x280, 3, 0x1000, 0, 16, 2}), 0); }, "count address"); + expectFailure([&] { AgcDriver::Pm4::ResolveDraw(packet, state); }, "base has not been set"); + execute(state, makePacket(0x11, {1, 0x5d87fc40, 0x10})); + auto draw = AgcDriver::Pm4::ResolveDraw(packet, state); + check(draw.indirect.has_value() && draw.indirect->arguments == 0x105d87fc40ull && draw.indirect->opcode == 0x24 && draw.indirect->recordBytes == 16 && draw.indirect->stride == 16 && draw.indirect->count == 1 && !draw.indirect->countIndirect, "indirect draw arguments mismatch"); + check(draw.indirect->baseVertexLocation == 0x280 && draw.indirect->startInstanceLocation == 0x8e && draw.indirect->drawIndexLocation == 0x280 && !draw.indirect->drawIndexEnabled && draw.indirect->indxOffset == 5 && draw.firstVertex == 5 && !draw.indexed && draw.indexCount == 0 && draw.instanceCount == 0, "indirect draw locations mismatch"); + check(draw.indirect->RangeBytes() == 16 && draw.indirect->VertexDwordOffset() == 8 && draw.indirect->InstanceDwordOffset() == 12, "indirect draw record geometry mismatch"); + check(AgcDriver::Pm4::ResolveDraw(makePacket(0x24, {0x60, 0x280, 0x8e, 2}), state).indirect->arguments == 0x105d87fca0ull, "indirect draw offset not applied"); + const auto multi = AgcDriver::Pm4::ResolveDraw(makePacket(0x2c, {0x20, 0x8c, 0x280, 0x8d | (1u << 31u), 3, 0, 0, 32, 0x22}), state); + check(multi.indirect->count == 3 && multi.indirect->stride == 32 && multi.indirect->drawIndexEnabled && multi.indirect->drawIndexLocation == 0x8d && !multi.indirect->countIndirect && multi.flags == 0x20 && multi.indirect->RangeBytes() == 80, "indirect multi-draw mismatch"); + expectFailure([&] { AgcDriver::Pm4::ResolveDraw(makePacket(0x25, {0, 0x8c, 0x280, 0}), state); }, "index base"); + alignas(4) std::array indices{}; + state.indexBase = reinterpret_cast(indices.data()); + state.indexType = 0; + expectFailure([&] { AgcDriver::Pm4::ResolveDraw(makePacket(0x25, {0, 0x8c, 0x280, 0}), state); }, "INDEX_BUFFER_SIZE"); + state.indexBufferSize = 8; + const auto indexed = AgcDriver::Pm4::ResolveDraw(makePacket(0x25, {0, 0x8c, 0x280, 0}), state); + check(indexed.indexed && indexed.indexAddress == state.indexBase && indexed.indexCount == 8 && indexed.indexSize == 2 && indexed.indirect->recordBytes == 20 && indexed.indirect->stride == 20 && indexed.firstVertex == 0 && indexed.indirect->indxOffset == 0, "indexed indirect draw mismatch"); + check(indexed.indirect->VertexDwordOffset() == 12 && indexed.indirect->InstanceDwordOffset() == 16, "indexed record geometry mismatch"); + alignas(16) std::array records{3, 2, 7, 9, 4, 1, 5, 6, 0, 0}; + state.drawIndirectBase = reinterpret_cast(records.data()); + const auto local = AgcDriver::Pm4::ResolveDraw(makePacket(0x2c, {0, 0x280, 0x280, 0x280, 2, 0, 0, 16, 2}), state); + const auto first = AgcDriver::Pm4::ReadDrawArguments(*local.indirect, 0); + const auto second = AgcDriver::Pm4::ReadDrawArguments(*local.indirect, 1); + check(first.count == 3 && first.instances == 2 && first.firstVertexOrIndex == 7 && first.vertexOffset == 0 && first.firstInstance == 9, "non-indexed record layout mismatch"); + check(second.count == 4 && second.instances == 1 && second.firstVertexOrIndex == 5 && second.firstInstance == 6, "second record mismatch"); + expectFailure([&] { AgcDriver::Pm4::ReadDrawArguments(*local.indirect, 2); }, "record index"); + records = {3, 2, 7, 9, 11, 0, 0, 0, 0, 0}; + const auto indexedRecord = AgcDriver::Pm4::ReadDrawArguments(*AgcDriver::Pm4::ResolveDraw(makePacket(0x25, {0, 0x8c, 0x280, 0}), state).indirect, 0); + check(indexedRecord.count == 3 && indexedRecord.instances == 2 && indexedRecord.firstVertexOrIndex == 7 && indexedRecord.vertexOffset == 9 && indexedRecord.firstInstance == 11, "indexed record layout mismatch"); + alignas(4) std::uint32_t countValue = 2; + const auto counted = AgcDriver::Pm4::ResolveDraw(makePacket(0x2c, {0, 0x280, 0x280, 0x280 | (1u << 30u), 5, low(&countValue), high(&countValue), 16, 2}), state); + check(counted.indirect->countIndirect && counted.indirect->count == 5 && counted.indirect->countAddress == reinterpret_cast(&countValue) && AgcDriver::Pm4::ReadDrawCount(*counted.indirect) == 2, "indirect draw count mismatch"); + expectFailure([&] { AgcDriver::Pm4::ReadDrawCount(*local.indirect); }, "count address"); +} + void testMemory() { AgcDriver::QueueState state; std::array data{0, 0, 0, 0}; @@ -410,6 +465,7 @@ int main(int argc, char** argv) { testContextAndBases(); testIndexedDraw(); testAutoDraw(); + testIndirectDraw(); testMemory(); testCopies(); testMemorySynchronization(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp new file mode 100644 index 0000000000..881487a71d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -0,0 +1,945 @@ +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; +using AgcDriver::GuestMemory::GpuMutex; + +// A compute-capable device with host imports (VK_EXT_external_memory_host) when the host offers +// them, as the driver creates its own; the recorder's batches need a real queue and real fences. +class Device { +public: + Device() { +#ifdef _WIN32 + library = SDL_LoadObject("vulkan-1.dll"); +#else + library = SDL_LoadObject("libvulkan.so.1"); +#endif + Require(library != nullptr, "cannot load Vulkan"); + try { + instanceProc = reinterpret_cast(SDL_LoadFunction(library, "vkGetInstanceProcAddr")); + Require(instanceProc != nullptr, "missing Vulkan instance resolver"); + VkApplicationInfo application{VK_STRUCTURE_TYPE_APPLICATION_INFO}; + application.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo info{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + info.pApplicationInfo = &application; + Check(function("vkCreateInstance")(&info, nullptr, &instance), "vkCreateInstance"); + std::uint32_t count = 0; + const auto enumerate = function("vkEnumeratePhysicalDevices"); + Check(enumerate(instance, &count, nullptr), "vkEnumeratePhysicalDevices"); + Require(count != 0, "no Vulkan device"); + std::vector devices(count); + Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); + context.physical = devices.front(); + const auto extensions = function("vkEnumerateDeviceExtensionProperties"); + Check(extensions(context.physical, nullptr, &count, nullptr), "vkEnumerateDeviceExtensionProperties"); + std::vector available(count); + Check(extensions(context.physical, nullptr, &count, available.data()), "vkEnumerateDeviceExtensionProperties"); + const auto hasExtension = [&](const char* name) { + for (const auto& extension : available) { + if (std::strcmp(extension.extensionName, name) == 0) return true; + } + return false; + }; + auto bytes = AgcDriver::QueryBdaByteFeatures(context.physical, function("vkGetPhysicalDeviceFeatures2"), available); + auto address = AgcDriver::QueryBdaFeatures(context.physical, function("vkGetPhysicalDeviceFeatures2"), available); + const auto queues = function("vkGetPhysicalDeviceQueueFamilyProperties"); + queues(context.physical, &count, nullptr); + std::vector families(count); + queues(context.physical, &count, families.data()); + std::uint32_t family = 0; + while (family < count && (families[family].queueFlags & VK_QUEUE_COMPUTE_BIT) == 0) ++family; + Require(family < count, "no Vulkan compute queue"); + const float priority = 1; + VkDeviceQueueCreateInfo queue{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + queue.queueFamilyIndex = family; + queue.queueCount = 1; + queue.pQueuePriorities = &priority; + VkPhysicalDeviceFeatures enabled{}; + enabled.shaderInt64 = VK_TRUE; + address.pNext = &bytes; + std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; + if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME)) { + extensionsEnabled.push_back(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); + VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; + function("vkGetPhysicalDeviceProperties2")(context.physical, &properties); + context.hostImportAlignment = hostProperties.minImportedHostPointerAlignment; + } + VkDeviceCreateInfo device{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, &address}; + device.queueCreateInfoCount = 1; + device.pQueueCreateInfos = &queue; + device.enabledExtensionCount = static_cast(extensionsEnabled.size()); + device.ppEnabledExtensionNames = extensionsEnabled.data(); + device.pEnabledFeatures = &enabled; + Check(function("vkCreateDevice")(context.physical, &device, nullptr, &context.device), "vkCreateDevice"); + context.deviceProc = function("vkGetDeviceProcAddr"); + function("vkGetPhysicalDeviceMemoryProperties")(context.physical, &context.memory); + VkPhysicalDeviceProperties properties{}; + function("vkGetPhysicalDeviceProperties")(context.physical, &properties); + context.limits = properties.limits; + context.bufferDeviceAddress = true; + context.Function("vkGetDeviceQueue")(context.device, family, 0, &context.queue); + VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.queueFamilyIndex = family; + Check(context.Function("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); + } catch (...) { + release(); + throw; + } + } + + ~Device() { release(); } + const Context& GetContext() const { return context; } + void WaitQueue() const { Check(context.Function("vkQueueWaitIdle")(context.queue), "vkQueueWaitIdle"); } + +private: + template + TFunction function(const char* name) const { + const auto result = reinterpret_cast(instanceProc(instance, name)); + Require(result != nullptr, name); + return result; + } + + void release() noexcept { + if (context.pool != VK_NULL_HANDLE) context.Function("vkDestroyCommandPool")(context.device, context.pool, nullptr); + context.bufferPool.reset(); + if (context.device != VK_NULL_HANDLE) function("vkDestroyDevice")(context.device, nullptr); + if (instance != VK_NULL_HANDLE) function("vkDestroyInstance")(instance, nullptr); + if (library != nullptr) SDL_UnloadObject(library); + } + + void* library = nullptr; + PFN_vkGetInstanceProcAddr instanceProc = nullptr; + VkInstance instance = VK_NULL_HANDLE; + Context context{}; +}; + +using Kind = Recorder::ReadKind; + +void readTrackingTests(const Device& device, Recorder& recorder) { + Require(Recorder::ReadTracking(), "read tracking is off (APS5_COPY_READ_TRACKING=0 set?)"); + Require(recorder.Idle() && !recorder.PendingReadOverlaps(0x10000, 16), "a fresh recorder reports a read"); + recorder.NotePendingRead(0x10000, 0x100, Kind::DispatchElement); + Require(recorder.Recording(), "a read note did not open a batch"); + Require(recorder.PendingReadOverlaps(0x10080, 4) && recorder.PendingReadOverlaps(0xff00, 0x101) && recorder.PendingReadOverlaps(0x100ff, 1), "an open batch's read is not seen"); + Require(!recorder.PendingReadOverlaps(0x10100, 4) && !recorder.PendingReadOverlaps(0xff00, 0x100) && !recorder.PendingReadOverlaps(0x10000, 0), "a disjoint range is reported as read"); + const auto open = recorder.DescribePendingRead(0x10000, 4); + Require(open.has_value() && open->open && !open->signaled && open->kind == Kind::DispatchElement && open->serial == recorder.Submissions() + 1, "the open batch's read is described wrongly"); + recorder.Submit(); + Require(!recorder.Recording() && !recorder.Idle(), "the batch is not in flight after Submit"); + Require(recorder.PendingReadOverlaps(0x10000, 4, false), "the raw scan does not see the in-flight batch's read"); + // An empty batch completes at once: once its fence signaled, its read is no reader any more, + // although the batch stays in flight until it is reaped. + device.WaitQueue(); + Require(!recorder.PendingReadOverlaps(0x10000, 4), "a signaled batch's read still refuses"); + Require(!recorder.Idle(), "an overlap query reaped the batch"); + const auto signaled = recorder.DescribePendingRead(0x10000, 4); + Require(signaled.has_value() && !signaled->open && signaled->signaled && signaled->serial == recorder.Submissions(), "the signaled batch's read is described wrongly"); + const auto counts = Recorder::ReadCounts(); + Require(counts.noted == 1 && counts.staleIgnored >= 1 && counts.hits[static_cast(Kind::DispatchElement)] >= 3 && counts.queries >= 6, "read counters are off"); + recorder.Sync(); + Require(recorder.Idle() && !recorder.PendingReadOverlaps(0x10000, 4, false) && !recorder.DescribePendingRead(0x10000, 4).has_value(), "a read outlived its batch"); + const std::vector> ranges{{0x20000, 0x21000}, {0x30000, 0x30010}, {0x40000, 0x40000}}; + recorder.NotePendingReads(ranges, Kind::AddressBased); + Require(recorder.PendingReadOverlaps(0x20fff, 1) && recorder.PendingReadOverlaps(0x30000, 16) && !recorder.PendingReadOverlaps(0x21000, 16) && !recorder.PendingReadOverlaps(0x40000, 16), "noted ranges are not seen as reads"); + const auto batch = recorder.DescribePendingRead(0x30008, 4); + Require(batch.has_value() && batch->kind == Kind::AddressBased && batch->open, "a noted range has the wrong reader kind"); + Require(Recorder::ReadCounts().hits[static_cast(Kind::AddressBased)] >= 2, "address-based hits are not counted"); + recorder.Sync(); + Require(recorder.Idle() && !recorder.PendingReadOverlaps(0x20000, 0x1000, false), "noted ranges outlived their batch"); +} + +void writeSettledTests(const Device& device, Recorder& recorder) { + Require(recorder.PendingWriteSettled(0x50000, 0x100), "an unwritten range is not settled"); + recorder.NotePendingWrite(0x50000, 0x100); + Require(recorder.PendingWriteOverlaps(0x50000, 0x100) && !recorder.PendingWriteSettled(0x50000, 0x100) && recorder.PendingWriteSettled(0x50100, 0x100), "an open batch's write counts as settled"); + recorder.Submit(); + device.WaitQueue(); + Require(recorder.PendingWriteOverlaps(0x50000, 0x100) && recorder.PendingWriteSettled(0x50000, 0x100), "a signaled batch's write is not settled"); + recorder.NotePendingWrite(0x50080, 0x10); + Require(!recorder.PendingWriteSettled(0x50000, 0x100) && recorder.PendingWriteSettled(0x50000, 0x80), "a later open write over the range still counts as settled"); + recorder.Sync(); + Require(recorder.Idle() && !recorder.PendingWriteOverlaps(0x50000, 0x100), "writes outlived their batches"); +} + +// Completion counting: a write-back completion (OnComplete) is pending until its batch finished; +// a completion label the GPU also stored (AfterCompletions storedOnGpu) is pending only once a CPU +// write-back overlapped it (NoteWrittenBack, once per label), one the GPU has no view of from its +// registration; both counts return to 0 when the batch finishes. +void completionCountTests(const Device& device, Recorder& recorder) { + alignas(64) static std::uint32_t memory[64]; + const auto base = reinterpret_cast(memory); + const std::array value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; + Require(recorder.Idle() && Recorder::PendingCompletionLabels() == 0 && Recorder::PendingWriteBackCompletions() == 0, "a fresh recorder reports pending completions"); + int ran = 0; + recorder.OnComplete([&] { ++ran; }); + Require(Recorder::PendingWriteBackCompletions() == 1 && recorder.HasCompletions(), "a write-back completion is not pending"); + // APS5_COUNT_ALL_COMPLETION_LABELS=1 (or APS5_LABEL_STORE_ALWAYS=1) counts the GPU-stored + // labels at registration, as before; the write-back then changes nothing. + const bool countAll = std::getenv("APS5_COUNT_ALL_COMPLETION_LABELS") != nullptr || std::getenv("APS5_LABEL_STORE_ALWAYS") != nullptr; + const std::uint64_t registered = countAll ? 2 : 0; + recorder.AfterCompletions(base, value, 3, 0, true); + recorder.AfterCompletions(base + 8, value, 4, 0, true); + Require(Recorder::PendingCompletionLabels() == registered, "a GPU-stored completion label counts before any write-back"); + recorder.AfterCompletions(base + 0x40, value, 5, 0, false); + Require(Recorder::PendingCompletionLabels() == registered + 1, "a label the GPU has no view of is not pending"); + // A late lookup (stamp <= afterStamp) of a label whose value reaches memory only through the + // batch's completion action is refused as BehindCompletion (a poller reaps for it); a GPU-stored + // one keeps the ordinary reason (unclosed group, or trust off) until a write-back overlaps it. + using Refusal = Recorder::LabelRefusal; + const Refusal ordinary = Recorder::LateTrust() ? Refusal::Unclosed : Refusal::TrustOff; + Refusal refusal{}; + Require(!Recorder::LookupLabel(base + 0x40, 4, 5, &refusal).has_value() && refusal == Refusal::BehindCompletion, "a not-imported completion label is not refused as behind a completion"); + Require(!Recorder::LookupLabel(base, 4, 3, &refusal).has_value() && refusal == (countAll ? Refusal::BehindCompletion : ordinary), "a GPU-stored label is refused as behind a completion before any write-back"); + Require(Recorder::LookupLabel(base, 4, 2, &refusal).has_value() && refusal == Refusal::None, "an early lookup of a completion label is refused"); + Recorder::NoteWrittenBack(base + 0x80, 0x40); + Require(Recorder::PendingCompletionLabels() == registered + 1, "a disjoint write-back counted a label"); + Recorder::NoteWrittenBack(base, 4); + Recorder::NoteWrittenBack(base + 2, 4); + const std::uint64_t counted = countAll ? 3 : 2; + Require(Recorder::PendingCompletionLabels() == counted, "a write-back over a GPU-stored label did not count it exactly once"); + Require(!Recorder::LookupLabel(base, 4, 3, &refusal).has_value() && refusal == Refusal::BehindCompletion, "a written-back GPU-stored label is not refused as behind a completion"); + Require(!Recorder::LookupLabel(base + 8, 4, 4, &refusal).has_value() && refusal == (countAll ? Refusal::BehindCompletion : ordinary), "an untouched GPU-stored label became behind a completion"); + recorder.Submit(); + device.WaitQueue(); + Require(Recorder::PendingCompletionLabels() == counted && Recorder::PendingWriteBackCompletions() == 1, "counts dropped before the batch finished"); + memory[0] = 7; + memory[2] = 7; + memory[16] = 7; + recorder.Sync(); + Require(ran == 1 && Recorder::PendingCompletionLabels() == 0 && Recorder::PendingWriteBackCompletions() == 0 && recorder.Idle(), "counts did not return to 0 at finish"); + const bool storeAlways = std::getenv("APS5_LABEL_STORE_ALWAYS") != nullptr; + Require(memory[0] == 1 && memory[2] == (storeAlways ? 1u : 7u) && memory[16] == 1, "completion stores ran for the wrong labels (overlapped and not-imported ones store, the untouched GPU-stored one skips)"); +} + +void labelTests(Recorder& recorder) { + const std::array value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; + Require(!recorder.PendingLabelIn(0x60000, 0x100), "an empty table reports a label"); + // A label this worker queued but has not recorded yet is inside the range for a CPU store decision. + Recorder::NoteQueuedLabel(0x60010, value, 1, AgcDriver::GuestMemory::GpuLockThreadTag()); + Require(recorder.PendingLabelIn(0x60010, 4) && recorder.PendingLabelIn(0x60000, 0x100) && !recorder.PendingLabelIn(0x60014, 4) && !recorder.PendingLabelIn(0x60000, 0x10), "a queued label is not seen by PendingLabelIn"); + Recorder::ForgetQueuedLabels(); + Require(!recorder.PendingLabelIn(0x60000, 0x100), "a forgotten queued label is still pending"); + recorder.NoteLabel(0x60020, value, 2, 0); + Require(recorder.PendingLabelIn(0x60020, 4) && recorder.PendingLabelIn(0x60000, 0x100) && !recorder.PendingLabelIn(0x60000, 0x20), "a recorded label is not seen by PendingLabelIn"); + recorder.Sync(); + Require(!recorder.PendingLabelIn(0x60000, 0x100), "a recorded label outlived its batch"); +} + +// The late rule (Recorder::NoteLabel): an entry with stamp <= afterStamp is served once its group +// closed, unless a non-label write covered it, it is queued, or APS5_LABEL_TRUST_LATE=0 (run the +// binary with that set for the switch-off case). +void lateLabelTests(Recorder& recorder) { + using Refusal = Recorder::LabelRefusal; + const std::array one{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; + const std::array two{std::byte{2}, std::byte{0}, std::byte{0}, std::byte{0}}; + const std::array pair{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}, std::byte{2}, std::byte{0}, std::byte{0}, std::byte{0}}; + const auto tag = AgcDriver::GuestMemory::GpuLockThreadTag(); + constexpr std::uint64_t a = 0x70000, b = 0x70100, c = 0x70200, d = 0x70300; + Refusal refusal{}; + const auto before = Recorder::LateCounts(); + recorder.NoteLabel(a, one, 10, 0); + const auto early = Recorder::LookupLabel(a, 4, 5, &refusal); + Require(early.has_value() && !early->late && early->value == 1 && early->queue == 0 && early->stamp == 10 && early->generation == 0 && refusal == Refusal::None, "an early lookup is wrong"); + auto late = Recorder::LookupLabel(a, 4, 10, &refusal); + if (!Recorder::LateTrust()) { + Require(!late.has_value() && refusal == Refusal::TrustOff, "(1) a late lookup is not refused with the switch off"); + Require(Recorder::LateCounts().candidates == before.candidates + 1, "(1) the late candidate is not counted with the switch off"); + Recorder::CloseLabelGroup(AgcDriver::GuestMemory::TrackerGeneration()); + Require(!Recorder::LookupLabel(a, 4, 10, &refusal).has_value() && refusal == Refusal::TrustOff, "(1) a closed group is served with the switch off"); + Require(Recorder::LookupLabel(a, 4, 5).has_value(), "(1) the early lookup broke with the switch off"); + recorder.Sync(); + Require(!Recorder::LookupLabel(a, 4, 5).has_value(), "(7) the entry outlived its batch"); + std::cout << "late label tests ran with APS5_LABEL_TRUST_LATE=0\n"; + return; + } + Require(!late.has_value() && refusal == Refusal::Unclosed, "(2) an unclosed group is served"); + Require(Recorder::LateCounts().candidates == before.candidates + 1 && Recorder::LateCounts().unclosed == before.unclosed + 1, "(2) the unclosed refusal is not counted"); + const auto generation = AgcDriver::GuestMemory::TrackerGeneration(); + Recorder::CloseLabelGroup(generation); + late = Recorder::LookupLabel(a, 4, 10, &refusal); + Require(late.has_value() && late->late && late->value == 1 && late->generation == generation && late->stamp == 10 && refusal == Refusal::None, "(2) a closed group is not served"); + Require(Recorder::LookupLabel(a, 4, 9)->late == false, "(2) an early lookup became late"); + // (3) A disjoint write leaves the entry alone; an overlapping non-label write refuses the late + // lookup only (the early one still composes what memory will hold in order). + recorder.NotePendingWrite(a + 4, 16); + Require(Recorder::LookupLabel(a, 4, 10).has_value(), "(3) a disjoint write flagged the entry"); + recorder.NotePendingWrite(a - 8, 12); + late = Recorder::LookupLabel(a, 4, 10, &refusal); + Require(!late.has_value() && refusal == Refusal::Overwritten, "(3) an overwritten entry is served late"); + Require(Recorder::LookupLabel(a, 4, 5).has_value(), "(3) an overwrite refused the early lookup"); + // The label's own note (inside NoteLabel) is no overwrite: an 8-byte label closed as a group. + recorder.NoteLabel(b, pair, 20, 0); + Recorder::CloseLabelGroup(AgcDriver::GuestMemory::TrackerGeneration()); + const auto wide = Recorder::LookupLabel(b, 8, 20, &refusal); + Require(wide.has_value() && wide->late && wide->value == 0x200000001ull && refusal == Refusal::None, "(3) the label's own note refused it"); + // (4) A later label on the dword replaces the entry: unclosed until its own group closes. + recorder.NoteLabel(a, two, 30, 0); + Require(!Recorder::LookupLabel(a, 4, 40, &refusal).has_value() && refusal == Refusal::Unclosed, "(4) a replaced entry kept the old close or flag"); + const auto replaced = Recorder::LookupLabel(a, 4, 25); + Require(replaced.has_value() && !replaced->late && replaced->value == 2 && replaced->stamp == 30, "(4) the replaced entry's early lookup is wrong"); + const auto generation2 = AgcDriver::GuestMemory::TrackerGeneration(); + Recorder::CloseLabelGroup(generation2); + late = Recorder::LookupLabel(a, 4, 40, &refusal); + Require(late.has_value() && late->late && late->value == 2 && late->generation == generation2, "(4) the replaced entry is not late-trustable after its close"); + // (5) Queued entries: another queue's is never seen, this queue's is early-only. + const std::uint32_t other = tag == 7 ? 8 : 7; + Recorder::NoteQueuedLabel(d, one, 50, other); + Require(!Recorder::LookupLabel(d, 4, 60, &refusal).has_value() && refusal == Refusal::None && !Recorder::LookupLabel(d, 4, 40).has_value(), "(5) another queue's queued label is served"); + Recorder::NoteQueuedLabel(d, one, 51, tag); + Recorder::ForgetQueuedLabels(); + Recorder::NoteQueuedLabel(c, one, 50, tag); + Require(Recorder::LookupLabel(c, 4, 40).has_value() && !Recorder::LookupLabel(c, 4, 40)->late, "(5) this queue's queued label is not served early"); + Require(!Recorder::LookupLabel(c, 4, 60, &refusal).has_value() && refusal == Refusal::Queued, "(5) a queued entry is late-trusted"); + Recorder::CloseLabelGroup(AgcDriver::GuestMemory::TrackerGeneration()); + Require(!Recorder::LookupLabel(c, 4, 60, &refusal).has_value() && refusal == Refusal::Queued, "(5) a close closed a queued entry"); + Recorder::ForgetQueuedLabels(); + Require(!Recorder::LookupLabel(c, 4, 40).has_value(), "(5) a forgotten queued entry is still served"); + const auto after = Recorder::LateCounts(); + Require(after.queued == before.queued + 2 && after.overwritten == before.overwritten + 1 && after.unclosed == before.unclosed + 2 && after.candidates >= before.candidates + 6, "late counters are off"); + // (7) Entries leave the table with their batch. + recorder.Sync(); + Require(!Recorder::LookupLabel(a, 4, 5).has_value() && !Recorder::LookupLabel(b, 8, 5).has_value() && !recorder.PendingLabelIn(0x70000, 0x400), "(7) entries outlived their batch"); +} + +// (6) GuestMemory::UnchangedSinceCollected: false outside the watched arena, and around a +// MarkWritten or a CPU write of the block inside it. +void unchangedSinceTests() { + using namespace AgcDriver::GuestMemory; + static std::uint32_t outside[16]; + Require(!UnchangedSinceCollected(reinterpret_cast(outside), 4, TrackerGeneration()), "(6) a range outside the arena is unchanged"); + if (!GuestArena::GuestArenaAvailable_nid_postfix()) { + std::cout << "guest arena unavailable: UnchangedSinceCollected not tested inside it\n"; + return; + } + constexpr std::size_t bytes = 65536; + void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes); +#ifdef _WIN32 + Require(VirtualAlloc(block, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the arena block"); +#endif + auto* words = static_cast(block); + const auto address = reinterpret_cast(block); + words[0] = 0; + if (!GuestArena::GuestArenaWriteWatched_nid_postfix()) { + Require(!UnchangedSinceCollected(address, 4, TrackerGeneration()), "(6) an unwatched arena reports unchanged"); + std::cout << "arena not write-watched: UnchangedSinceCollected is always false\n"; + return; + } + CollectWritesUncached(address, 4); + const auto generation = TrackerGeneration(); + Require(UnchangedSinceCollected(address, 4, generation), "(6) an untouched range is not unchanged"); + Require(UnchangedSinceCollected(address, 4, generation), "(6) the check itself dirtied the range"); + Require(!UnchangedSinceCollected(address, 4, 0), "(6) generation 0 is unchanged"); + MarkWritten(address, 4); + Require(UnchangedSinceCollected(address, 4, generation), "(6) a MarkWritten of the block (a GPU label record) refuses"); + Require(!UnchangedSince(address, 4, generation), "(6) a MarkWritten of the block is not seen by UnchangedSince"); + const auto generation2 = TrackerGeneration(); + Require(UnchangedSinceCollected(address, 4, generation2), "(6) unchanged after the stamp fails"); + words[1] = 1; + Require(!UnchangedSinceCollected(address, 4, generation2), "(6) a CPU write to the page is not seen"); + Require(UnchangedSinceCollected(address, 4, TrackerGeneration()), "(6) the collect did not reset the page"); + const auto generation3 = TrackerGeneration(); + words[1] = 2; + CollectWritesUncached(address, 4); + Require(!UnchangedSinceCollected(address, 4, generation3), "(6) a CPU write collected by another caller is not seen"); +#ifdef _WIN32 + VirtualFree(block, bytes, MEM_DECOMMIT); +#endif + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); +} + +// (9) ProvedClearKeys: a surface's DCC keys are scanned once and answered from the proof after, +// until the key range is stamped (a GPU key store's MarkWritten) or the CPU writes it; a scan made +// while recorded work still writes the keys is not kept. With APS5_NO_KEY_FAST_PATH=1 every call +// scans and no proof is stored. +void keyProofTests(const Device& device, Recorder& recorder) { + using namespace AgcDriver::GuestMemory; + if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { + std::cout << "guest arena unavailable or not write-watched: key proofs not tested\n"; + return; + } + constexpr std::size_t bytes = 65536; + void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes); +#ifdef _WIN32 + Require(VirtualAlloc(block, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the arena block"); +#endif + struct Release { + void* block; + ~Release() { +#ifdef _WIN32 + VirtualFree(block, bytes, MEM_DECOMMIT); +#endif + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); + } + } release{block}; + constexpr std::size_t keyCount = 1024; + constexpr std::uint64_t surfaceBytes = keyCount * 256; + auto* keys = static_cast(block); + std::memset(keys, 0x00, keyCount); + const auto address = reinterpret_cast(block); + GuestTextureResource resource{}; + resource.baseAddress = address + 8192; + resource.width = 256; + resource.height = 256; + resource.mipCount = 1; + resource.tileMode = TextureTileMode::kLinear; + resource.dimension = TextureDimension::k2D; + resource.format = 56; + resource.dccAddress = address; + DccKeyProof proof; + auto before = KeyProofCounts(); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0000, "(9) 0x00 keys are not a 0000 clear"); + auto after = KeyProofCounts(); + Require(after.scanned == before.scanned + 1 && after.proved == before.proved, "(9) the first call did not scan"); + if (!KeyFastPath()) { + Require(proof.generation == 0, "(9) APS5_NO_KEY_FAST_PATH stored a proof"); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0000 && proof.generation == 0, "(9) APS5_NO_KEY_FAST_PATH proved keys"); + std::cout << "key fast path off: every call scans\n"; + return; + } + Require(proof.generation != 0 && proof.keys == DccKeys::Clear0000 && after.unstable == before.unstable, "(9) a stable scan left no proof"); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0000, "(9) the proof answers other keys"); + before = KeyProofCounts(); + Require(before.proved == after.proved + 1 && before.scanned == after.scanned, "(9) the second call scanned"); + const auto generation = proof.generation; + MarkWritten(address, keyCount); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0000, "(9) unchanged bytes read differently"); + after = KeyProofCounts(); + Require(after.scanned == before.scanned + 1 && after.proved == before.proved && proof.generation > generation, "(9) a stamped key range was proved"); + std::memset(keys, 0x40, keyCount); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0001, "(9) a CPU write of the keys was not seen"); + before = KeyProofCounts(); + Require(before.scanned == after.scanned + 1 && proof.keys == DccKeys::Clear0001, "(9) the CPU write did not make a scan"); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0001 && KeyProofCounts().proved == before.proved + 1, "(9) the new keys were not proved"); + // A recorded write over the keys (noted and stamped as the driver's key stores are): the scans + // are not kept until its batch signaled. + recorder.NotePendingWrite(address, keyCount); + MarkWritten(address, keyCount); + before = KeyProofCounts(); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0001 && proof.generation == 0, "(9) a scan under a pending write was kept"); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0001 && proof.generation == 0, "(9) a second scan under a pending write was kept"); + after = KeyProofCounts(); + Require(after.scanned == before.scanned + 2 && after.unstable == before.unstable + 2 && after.proved == before.proved, "(9) unstable scans were miscounted"); + recorder.Submit(); + device.WaitQueue(); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0001 && proof.generation != 0, "(9) a signaled write kept the scan unstable"); + recorder.Sync(); + before = KeyProofCounts(); + Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0001 && KeyProofCounts().proved == before.proved + 1, "(9) the proof did not hold after the batch finished"); +} + +// (8) The group close against a collect in progress: a CPU store made after the close must refuse +// the entry although another thread's resetting walk of the label's page (over a large range) may +// absorb the store and stamp the block with the generation it took before the close. And an entry +// whose batch was submitted before the close stays unclosed. +void closeRaceTests(const Device& device, Recorder& recorder) { + using namespace AgcDriver::GuestMemory; + using Refusal = Recorder::LabelRefusal; + const std::array one{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; + Refusal refusal{}; + if (!Recorder::LateTrust()) return; + recorder.NoteLabel(0x80000, one, 70, 0); + recorder.Submit(); + Recorder::CloseLabelGroup(TrackerGeneration()); + Require(!Recorder::LookupLabel(0x80000, 4, 70, &refusal).has_value() && refusal == Refusal::Unclosed, "(8) a submitted batch's entry was closed"); + device.WaitQueue(); + recorder.Sync(); + if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) return; + constexpr std::size_t bytes = 1u << 20; + void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536); +#ifdef _WIN32 + Require(VirtualAlloc(block, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the arena block"); +#endif + const auto base = reinterpret_cast(block); + // Late in the range, so the walker is usually still ahead of the page when the store lands. + const auto label = base + 4096 * 250; + auto* word = reinterpret_cast(label); + *word = 0; + CollectWritesUncached(base, bytes); + // Joined on every exit, so a failed Require reports instead of terminating on the thread. + struct Walker { + std::atomic stop{false}; + std::thread thread; + ~Walker() { + stop.store(true, std::memory_order_relaxed); + if (thread.joinable()) thread.join(); + } + } walker; + walker.thread = std::thread([&] { + while (!walker.stop.load(std::memory_order_relaxed)) CollectWritesUncached(base, bytes); + }); + for (std::uint32_t i = 0; i < 4000; ++i) { + const std::uint64_t stamp = 100 + i; + recorder.NoteLabel(label, one, stamp, 0); + Recorder::CloseLabelGroup(TrackerGeneration()); + *word = i + 2; + const auto hit = Recorder::LookupLabel(label, 4, stamp, &refusal); + Require(hit.has_value() && hit->late && hit->generation != 0, "(8) the closed entry is not served late"); + Require(!UnchangedSinceCollected(label, 4, hit->generation), "(8) a store after the close is hidden by a racing collect"); + } + walker.stop.store(true, std::memory_order_relaxed); + walker.thread.join(); + recorder.Sync(); + Require(!Recorder::LookupLabel(label, 4, 5).has_value(), "(8) the entry outlived its batch"); +#ifdef _WIN32 + VirtualFree(block, bytes, MEM_DECOMMIT); +#endif + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); +} + +// A resource build over host-imported guest memory notes its in-place reads when its writes are +// marked, so a CPU copy into that memory is refused until the batch ran. +// Per-batch store runs (Recorder::RecordStore): the stores of a batch queue until Submit, a +// later writer or reader of a queued store's bytes has the run recorded in place first, and the +// stores land in the import's memory once the batch ran. With APS5_LABEL_RUNS_INLINE=1 or +// APS5_NO_LABEL_RUNS=1 (run the binary with either set) nothing queues and the stores land alike. +void storeRunTests(const Device& device, Recorder& recorder) { + const auto& context = device.GetContext(); + if (context.hostImportAlignment == 0) { + std::cout << "host imports unavailable: store runs not tested\n"; + return; + } + constexpr std::size_t bytes = 65536; +#ifdef _WIN32 + void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + void* block = std::aligned_alloc(65536, bytes); +#endif + Require(block != nullptr, "cannot allocate the store test block"); + std::memset(block, 0, bytes); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + const auto* import = HostImportFor(context, address, bytes); + if (import == nullptr) { + std::cout << "host import of the store test block refused: store runs not tested\n"; + return; + } + const bool perBatch = std::getenv("APS5_LABEL_RUNS_INLINE") == nullptr && std::getenv("APS5_NO_LABEL_RUNS") == nullptr && std::getenv("APS5_NO_LABEL_BATCHING") == nullptr; + auto* words = static_cast(block); + const auto store = [&](std::size_t word, std::uint32_t value) { + const std::array value4{std::byte{static_cast(value)}, std::byte{static_cast(value >> 8u)}, std::byte{static_cast(value >> 16u)}, std::byte{static_cast(value >> 24u)}}; + recorder.RecordStore(import->buffer, address + word * 4 - import->base, value4, address + word * 4); + }; + Require(recorder.Idle() && !recorder.HasQueuedStores(), "a fresh recorder has queued stores"); + store(0, 1); + store(1, 2); + store(4, 3); + store(4, 4); + Require(recorder.Recording(), "a store did not open a batch"); + Require(recorder.HasQueuedStores() == perBatch, "per-batch runs do not queue (or inline runs do)"); + Require(recorder.QueuedStoreOverlaps(address, 8) == perBatch && recorder.QueuedStoreOverlaps(address + 16, 4) == perBatch && !recorder.QueuedStoreOverlaps(address + 8, 8), "queued store ranges are wrong"); + const auto before = Recorder::StoreCounts(); + // A reader or writer of untouched bytes forces nothing; one over a queued store records the run. + recorder.FlushStoresOverlapping(address + 8, 8); + Require(recorder.HasQueuedStores() == perBatch, "a disjoint range flushed the run"); + recorder.FlushStoresOverlapping(address + 16, 4); + Require(!recorder.HasQueuedStores() && Recorder::StoreCounts().runsForced == before.runsForced + (perBatch ? 1 : 0), "an overlapping writer did not record the run in place"); + store(2, 5); + Require(recorder.HasQueuedStores() == perBatch, "a store after a forced run did not start a new run"); + recorder.Submit(); + Require(!recorder.HasQueuedStores() && Recorder::StoreCounts().runsAtSubmit == before.runsAtSubmit + (perBatch ? 1 : 0), "Submit did not record the run"); + recorder.Sync(); + Require(words[0] == 1 && words[1] == 2 && words[2] == 5 && words[4] == 4 && words[3] == 0, "the stores did not land (or the later store of a dword lost)"); + const auto after = Recorder::StoreCounts(); + Require(after.stores == before.stores + 1 && after.replaced == before.replaced + 0 && after.joined == before.joined + 0, "store counters are off"); + // The covered-access mask: reported once by the next Commands() call, then cleared. + VkAccessFlags covered = 0xffffffffu; + recorder.Commands(&covered); + Require(covered == 0, "a new batch starts covered"); + recorder.MarkCovered(VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + recorder.Commands(&covered); + Require(covered == (VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT), "the covered mask is not reported"); + recorder.Commands(&covered); + Require(covered == 0, "the covered mask survived a Commands() call"); + recorder.Sync(); + // The hazard tracker (counting mode only): accesses are noted without effect on the batch. + const std::pair range{address, address + 64}; + recorder.NoteAccess(Recorder::CommandClass::Fill, Recorder::Access{{}, std::span(&range, 1), {}, VK_PIPELINE_STAGE_TRANSFER_BIT}); + recorder.NoteAccess(Recorder::CommandClass::DispatchLeading, Recorder::Access{std::span(&range, 1), {}, {}, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT}); + Require(recorder.Recording() == Recorder::BarrierValidate(), "a noted access opened a batch with the tracker off (or none with it on)"); + recorder.Sync(); + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); +} + +void resourceReadTests(const Device& device, Recorder& recorder) { + const auto& context = device.GetContext(); + if (context.hostImportAlignment == 0) { + std::cout << "host imports unavailable: resource read notes not tested\n"; + return; + } + constexpr std::size_t bytes = 65536; +#ifdef _WIN32 + void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + void* block = std::aligned_alloc(65536, bytes); +#endif + Require(block != nullptr, "cannot allocate the test block"); + std::memset(block, 0x11, bytes); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + if (HostImportFor(context, address, bytes) == nullptr) { + std::cout << "host import of the test block refused: resource read notes not tested\n"; + return; + } + const auto element = address + 4096; + constexpr std::size_t elementBytes = 1024; + { + GuestBufferMemory memory(context); + memory.AddReadable(element, elementBytes); + memory.Upload(false); + const auto reads = memory.InPlaceReads(); + Require(reads.size() == 1 && reads[0].first == element && reads[0].second == element + elementBytes, "an imported region is not an in-place read"); + Require(memory.Writes().empty(), "a readable element counts as written"); + } + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding binding; + binding.kind = ShaderRecompiler::DescriptorKind::StorageBuffer; + binding.role = ShaderRecompiler::DescriptorRole::GuestBuffers; + binding.descriptorSet = 0; + binding.binding = 0; + binding.count = 1; + binding.guestDescriptor = {static_cast(element), static_cast(element >> 32u) & 0xffffu, elementBytes, 0x31000000u}; + binding.bufferWritten = {false}; + program.bindings.push_back(binding); + const CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &program, 0}; + { + ShaderResources resources(context, compute); + Require(recorder.Idle() && !recorder.PendingReadOverlaps(element, 16), "the build itself noted a read"); + resources.MarkGpuWrites(recorder); + Require(recorder.PendingReadOverlaps(element, 16) && recorder.PendingReadOverlaps(element + elementBytes - 4, 4) && !recorder.PendingReadOverlaps(element + elementBytes, 16), "the build's in-place read was not noted"); + Require(!recorder.PendingWriteOverlaps(element, elementBytes), "a read-only element was noted as written"); + const auto info = recorder.DescribePendingRead(element, 16); + Require(info.has_value() && info->kind == Kind::DispatchElement && info->open, "the build's read has the wrong kind"); + recorder.Submit(); + device.WaitQueue(); + Require(!recorder.PendingReadOverlaps(element, 16), "the build's read refuses after its batch ran"); + recorder.Sync(); + } + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + // The import is retired by the next reconcile; the block itself is left to the process. + HostImportFor(context, address, bytes); +} + +// Unit shadows (UnitShadow.hpp) over a host import of write-watched arena memory: a retile piece's +// slab destination and its seeds, freshness from the tracker (a publish never stamps, a CPU write +// makes the unit stale), the scopes, the slab boundary, and the retire publish. With +// APS5_UNIT_SHADOW_MIB=8 (one slab) the second slab's allocation evicts the first with a publish; +// with APS5_NO_UNIT_SHADOW=1 every primitive is inert. +void unitShadowTests(const Device& device, Recorder& recorder) { + using namespace AgcDriver::GuestMemory; + const auto& context = device.GetContext(); + if (!UnitShadowEnabled()) { + Require(!AnyShadowedOverlaps(0x10000, 16) && PublishShadow(0x10000, 16, PublishScope::Whole, PublishReason::Hook) == 0, "unit shadows are off but not inert"); + std::cout << "unit shadows off (APS5_NO_UNIT_SHADOW): primitives inert\n"; + return; + } + if (context.hostImportAlignment == 0 || !GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { + std::cout << "host imports or the write-watched arena unavailable: unit shadows not tested\n"; + return; + } + constexpr std::uint64_t unit = 65536; + // Three slabs at the default 8 MiB: units 0..127, 128..255, 256..271. + constexpr std::size_t bytes = (17u << 20u); + void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536); + Require(block != nullptr, "cannot allocate the shadow test block"); +#ifdef _WIN32 + Require(VirtualAlloc(block, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the shadow test block"); +#endif + struct Release { + void* block; + ~Release() { +#ifdef _WIN32 + VirtualFree(block, bytes, MEM_DECOMMIT); +#endif + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); + } + } release{block}; + std::memset(block, 0x11, bytes); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + struct Unregister { + void* block; + bool armed = true; + ~Unregister() { + if (!armed) return; + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + } unregister{block}; + const auto* import = HostImportFor(context, address, bytes); + if (import == nullptr) { + std::cout << "host import of the shadow test block refused: unit shadows not tested\n"; + return; + } + Require(import->base == address && import->bytes == bytes, "the import does not cover the block"); + CollectWritesUncached(address, bytes); + const auto* words = static_cast(block); + const auto unit3 = address + 3 * unit; + const auto unit4 = unit3 + unit; + const auto unit5 = unit4 + unit; + // A half-unit piece seeds its unit from the import; a whole-unit piece does not; both land in + // one slab at one offset. + auto half = ShadowDestinationFor(context, *import, unit3, unit3 + unit / 2); + Require(half.has_value() && half->seedUnits.size() == 1 && half->seedUnits[0].first == unit3 && half->seedUnits[0].second == unit4, "a half-unit retile piece does not seed its unit"); + auto whole = ShadowDestinationFor(context, *import, unit3, unit4); + Require(whole.has_value() && whole->seedUnits.empty() && whole->slab == half->slab && whole->offset == half->offset && whole->buffer == half->buffer, "a whole-unit retile piece seeds, or lands elsewhere"); + Require(!AnyShadowedOverlaps(unit3, unit), "a destination alone counts as shadowed"); + const auto firstSlab = half->slab; + Require(firstSlab->pins.load() == 2 && half->pin != nullptr, "destinations do not pin their slab"); + // The retile as writeBackWindows records it: the seed, then the piece's bytes over its half. + const auto copyBuffer = context.Function("vkCmdCopyBuffer"); + const auto retile = [&](const ShadowDestination& destination, std::uint64_t begin, std::uint64_t length, std::uint8_t value, bool seed) { + auto pattern = std::make_shared(context, static_cast(length), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memset(pattern->Bytes().data(), value, pattern->Bytes().size()); + const auto commands = recorder.Commands(); + recorder.Keep(pattern); + recorder.Keep(destination.slab); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + if (seed) { + const VkBufferCopy seedCopy{begin - import->base, SlabOffset(*import, *destination.slab, begin), unit}; + copyBuffer(commands, import->buffer, destination.buffer, 1, &seedCopy); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + } + const VkBufferCopy piece{0, SlabOffset(*import, *destination.slab, begin), length}; + copyBuffer(commands, pattern->Handle(), destination.buffer, 1, &piece); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + recorder.MarkCovered(VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + const ShadowedRange range{begin & ~(unit - 1), (begin & ~(unit - 1)) + unit, destination.slab}; + MarkShadowed(*import, std::span(&range, 1), TrackerGeneration()); + }; + retile(*half, unit3, unit / 2, 0x22, true); + Require(AnyShadowedOverlaps(unit3, 1) && AnyShadowedOverlaps(unit4 - 1, 1) && AnyShadowedOverlaps(unit3 + 1000, 64), "a fresh unit is not seen as shadowed"); + Require(!AnyShadowedOverlaps(unit3 - 1, 1) && !AnyShadowedOverlaps(unit4, unit) && !AnyShadowedOverlaps(address, 3 * unit), "a range outside the unit is seen as shadowed"); + { + // An upload's runs over units 2..4 split at unit 3's freshness; the fresh piece reads the slab. + const std::pair run{2 * unit, 5 * unit}; + const auto sources = ShadowSources(context, *import, address, std::span(&run, 1), {}); + Require(sources.size() == 3, "an upload run is not split at the fresh unit"); + Require(!sources[0].shadow && sources[0].begin == 2 * unit && sources[0].end == 3 * unit && sources[0].buffer == import->buffer && sources[0].offset == 2 * unit, "the import piece before the fresh unit is wrong"); + Require(sources[1].shadow && sources[1].begin == 3 * unit && sources[1].end == 4 * unit && sources[1].buffer == half->buffer && sources[1].offset == half->offset && sources[1].slab == half->slab, "the fresh unit's piece does not read the slab"); + Require(!sources[2].shadow && sources[2].begin == 4 * unit && sources[2].end == 5 * unit, "the import piece after the fresh unit is wrong"); + } + // A whole publish: the untouched half keeps the seed (the import's bytes), the other the + // retile's; the copy stamps nothing, and a second publish copies nothing. + const auto pre = CollectWritesUncached(address, bytes); + Require(PublishShadow(unit3, unit, PublishScope::Whole, PublishReason::Hook) == 1, "the whole publish did not copy the unit"); + Require(recorder.PendingWriteOverlaps(unit3, unit), "the publish noted no pending write"); + recorder.Sync(); + Require(words[3 * unit] == 0x22 && words[3 * unit + unit / 2 - 1] == 0x22 && words[3 * unit + unit / 2] == 0x11 && words[4 * unit - 1] == 0x11 && words[3 * unit - 1] == 0x11 && words[4 * unit] == 0x11, "the published bytes are wrong"); + Require(!AnyShadowedOverlaps(unit3, unit), "a published unit still counts as shadowed"); + Require(PublishShadow(unit3, unit, PublishScope::Whole, PublishReason::Hook) == 0, "a second publish copied the unit again"); + Require(UnchangedSince(unit3, unit, pre), "the publish stamped the unit"); + { + // The published unit still reads from the slab (both hold the bytes). + const std::pair run{3 * unit, 4 * unit}; + const auto sources = ShadowSources(context, *import, address, std::span(&run, 1), {}); + Require(sources.size() == 1 && sources[0].shadow, "a published unit does not read from the slab"); + } + // Partial-unit scope over unit 3 (partly, fresh again) and unit 4 (wholly, fresh): unit 3 only. + retile(*whole, unit3, unit, 0x33, false); + auto four = ShadowDestinationFor(context, *import, unit4, unit5); + Require(four.has_value() && four->seedUnits.empty() && four->slab == half->slab, "unit 4's destination is not in the same slab"); + retile(*four, unit4, unit, 0x44, false); + Require(AnyShadowedOverlaps(unit3, unit) && AnyShadowedOverlaps(unit4, unit), "re-marked units are not shadowed"); + Require(PublishShadow(unit3 + 100, static_cast(2 * unit - 100), PublishScope::PartialUnits, PublishReason::Fill) == 1, "the partial publish did not copy exactly the partly covered unit"); + Require(!AnyShadowedOverlaps(unit3, unit) && AnyShadowedOverlaps(unit4, unit), "the partial publish copied the wrong unit"); + recorder.Sync(); + Require(words[3 * unit] == 0x33 && words[4 * unit - 1] == 0x33 && words[4 * unit] == 0x11, "the partial publish's bytes are wrong"); + // A CPU write into a page of unit 4 makes it stale: the publish collects the unit itself (no + // walk saw the store before it), copies nothing and drops it. + static_cast(CollectWritesUncached(unit4, unit)); + *static_cast(static_cast(static_cast(block) + 4 * unit + 4096)) = 0x55; + Require(PublishShadow(unit4, unit, PublishScope::Whole, PublishReason::Hook) == 0, "a stale unit was published"); + Require(!AnyShadowedOverlaps(unit4, unit), "a stale unit still counts as shadowed after its publish"); + recorder.Sync(); + Require(words[4 * unit + 4096] == 0x55 && words[4 * unit] == 0x11, "a stale unit's publish overwrote the CPU's bytes"); + // The slab boundary: a piece over units 127 and 128 is refused, each alone lands in its slab. + const auto unit127 = address + 127 * unit; + const auto unit128 = unit127 + unit; + Require(SlabBoundary(*import, unit127) == unit128 && SlabBoundary(*import, address) == unit128 && SlabBoundary(*import, unit128) == unit128 + 128 * unit, "the slab boundary is wrong"); + Require(!ShadowDestinationFor(context, *import, unit127, unit128 + unit).has_value(), "a piece across a slab boundary was accepted"); + auto low = ShadowDestinationFor(context, *import, unit127, unit128); + Require(low.has_value() && low->slab == firstSlab && low->offset == 127 * unit, "unit 127 does not land in the first slab"); + retile(*low, unit127, unit, 0x66, false); + auto high = ShadowDestinationFor(context, *import, unit128, unit128 + unit); + const char* budgetText = std::getenv("APS5_UNIT_SHADOW_MIB"); + const auto budgetMiB = budgetText != nullptr ? std::strtoull(budgetText, nullptr, 10) : 1024ull; + if (budgetMiB < 16) { + // One slab fits: while destinations pin the first slab the second is refused (a + // write-back in progress must keep its slab); with the pins released, the second slab's + // allocation evicts the first after publishing its fresh unit 127 (the eviction publish is + // recorded, so the bytes land at the sync). + Require(!high.has_value(), "a pinned slab was evicted for a second slab"); + half.reset(); + whole.reset(); + four.reset(); + low.reset(); + Require(firstSlab->pins.load() == 0, "a released destination left its pin"); + high = ShadowDestinationFor(context, *import, unit128, unit128 + unit); + Require(high.has_value() && high->slab != firstSlab && high->offset == 0, "unit 128 does not land in a second slab under the one-slab budget"); + recorder.Sync(); + Require(words[127 * unit] == 0x66 && !AnyShadowedOverlaps(unit127, unit), "the evicted slab's fresh unit was not published"); + std::cout << "unit shadow eviction under APS5_UNIT_SHADOW_MIB=" << budgetMiB << " verified\n"; + } else { + Require(high.has_value() && high->slab != firstSlab && high->offset == 0 && AnyShadowedOverlaps(unit127, unit), "unit 128 does not land in a second slab beside the first"); + half.reset(); + whole.reset(); + four.reset(); + low.reset(); + } + retile(*high, unit128, unit, 0x77, false); + Require(AnyShadowedOverlaps(unit128, unit), "the second slab's unit is not shadowed"); + high.reset(); + // Retire through the registry: the block is re-registered as its first five units, so the next + // lookup reconciles and retires the import; the fresh units still inside a registered range + // are published into its (kept) buffer, the rest (memory the title took back) are dropped. + // Under the one-slab budget unit 5's slab evicts the second one first (unit 128 published). + retile(*ShadowDestinationFor(context, *import, unit5, unit5 + unit), unit5, unit, 0x88, false); + retile(*ShadowDestinationFor(context, *import, address + 2 * unit, unit3), address + 2 * unit, unit, 0x99, false); + Require(AnyShadowedOverlaps(unit5, unit) && AnyShadowedOverlaps(address + 2 * unit, unit), "units 2 and 5 are not shadowed before the retire"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + mutation.Add(block, static_cast(5 * unit), true, true); + } + Require(HostImportFor(context, address, bytes) == nullptr, "a shrunk registration still imports the old range"); + Require(!AnyShadowedOverlaps(address, bytes), "the retired import's shadow survived"); + recorder.Sync(); + Require(words[2 * unit] == 0x99 && words[3 * unit - 1] == 0x99, "the retire did not publish the unit still registered"); + Require(words[5 * unit] == 0x11 && words[5 * unit + unit - 1] == 0x11, "the retire published a unit whose memory is no longer registered"); + if (budgetMiB >= 16) Require(words[127 * unit] == 0x11 && words[128 * unit] == 0x11, "the retire published the second slab's units outside the registration"); + else Require(words[128 * unit] == 0x77, "the eviction before the retire did not publish unit 128"); +} + +// The data word positions of a dispatch-cache variant (Driver.cpp's data-only hits): leaves +// located among the runs' words, aliased, unaligned, out-of-run and mismatched ones skipped. +void dataWordPositionsTests() { + using AgcDriver::DataWordPositions; + const std::vector> runs{{0x1000, 0x1010}, {0x2000, 0x2008}}; + const std::vector words{1, 2, 3, 4, 5, 6}; + const std::vector flattened{2, 6, 3}; + std::vector positions, slots; + { + const std::vector> leaves{{1, 0x2004}, {0, 0x1004}, {2, 0x1008}}; + const std::vector otherReads{0x1008}; + const auto counts = DataWordPositions(runs, leaves, otherReads, words, flattened, positions, slots); + Require(positions == std::vector{1, 5} && slots == std::vector{0, 1}, "data word positions are wrong"); + Require(counts.aliased == 1 && counts.unmapped == 0 && counts.mismatched == 0, "an aliased leaf was not counted"); + } + { + const std::vector> leaves{{0, 0x1006}, {1, 0x3000}, {1, 0x2008}, {2, 0x1000}}; + const auto counts = DataWordPositions(runs, leaves, {}, words, flattened, positions, slots); + Require(positions.empty() && slots.empty(), "a skipped leaf produced a position"); + Require(counts.unmapped == 3 && counts.mismatched == 1 && counts.aliased == 0, "skipped leaves were counted wrongly"); + } + { + // Two pure slots over one dword: both positions kept, in position order. + const std::vector> leaves{{2, 0x1008}, {1, 0x2004}, {0, 0x1004}}; + static_cast(DataWordPositions(runs, leaves, {}, words, flattened, positions, slots)); + Require(positions == std::vector{1, 2, 5} && slots == std::vector{0, 2, 1}, "positions are not sorted"); + } +} + +// A template's data buffers refreshed by words from a patched compiled result (a data-only hit) +// and back: DataWordsHash() follows the buffers exactly, so a later recipe hit's hash compare +// (RecordedDispatch::DataRefresh::Hash) decides correctly in both directions. +void dataRefreshTests(const Device& device, Recorder& recorder) { + const auto& context = device.GetContext(); + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding binding; + binding.kind = ShaderRecompiler::DescriptorKind::StorageBuffer; + binding.role = ShaderRecompiler::DescriptorRole::FlattenedSrt; + binding.descriptorSet = 0; + binding.binding = 0; + binding.count = 1; + binding.guestDescriptor = {1, 2, 3, 4}; + program.bindings.push_back(binding); + auto patched = program; + patched.bindings[0].guestDescriptor[2] = 0x33; + const CompiledShader original{ShaderRecompiler::ShaderStage::Compute, &program, 0}; + const CompiledShader live{ShaderRecompiler::ShaderStage::Compute, &patched, 0}; + ShaderResources resources(context, original); + Require(resources.DataWordsHash() == ShaderResources::DataWordsHash(original) && resources.DataWordsHash() != ShaderResources::DataWordsHash(live), "a fresh template's data hash is not its words'"); + Require(!resources.DataWordsDiffer(original) && resources.DataWordsDiffer(live), "the per-word compare disagrees with the words"); + Require(resources.RefreshData(recorder.Commands(), live, &recorder), "a refresh with different words recorded nothing"); + Require(resources.DataWordsHash() == ShaderResources::DataWordsHash(live) && !resources.DataWordsDiffer(live), "the refreshed template's hash is not the patched words'"); + Require(resources.RefreshData(recorder.Commands(), original, &recorder), "the refresh back recorded nothing"); + Require(resources.DataWordsHash() == ShaderResources::DataWordsHash(original), "the refreshed template's hash is not the original words'"); + Require(!resources.RefreshData(recorder.Commands(), original, &recorder), "a refresh with equal words recorded"); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); +} + +} + +int main() { + try { + Device device; + std::lock_guard gpu(GpuMutex()); + Recorder recorder(device.GetContext()); + recorder.Activate(); + readTrackingTests(device, recorder); + writeSettledTests(device, recorder); + completionCountTests(device, recorder); + labelTests(recorder); + lateLabelTests(recorder); + unchangedSinceTests(); + closeRaceTests(device, recorder); + keyProofTests(device, recorder); + resourceReadTests(device, recorder); + storeRunTests(device, recorder); + unitShadowTests(device, recorder); + dataWordPositionsTests(); + dataRefreshTests(device, recorder); + std::cout << "Recorder read tracking and label tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 0cae0fd698..030579c1c7 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1,15 +1,21 @@ #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" #include "ControlFlow/RequestSerializer.hpp" #include "Optimization/RequestMemoryView.hpp" +#include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" +#include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp" #if ANYPS5_ENABLE_SPIRV_TOOLS #include "SpirvBackend/SpirvOptimizer.hpp" #endif +#include #include #include #include +#include #include #include +#include +#include namespace { @@ -66,12 +72,302 @@ void verifyRegisterSources() { require(!EquivalentValue(plan, &samplerRegister, &firstVector), "different register types were merged"); } +// The pure flat slots of a hand-built plan (Detail::ComputePureFlatSlots): a slot is pure unless +// a descriptor dword, a condition, a uniform value or another slot's address cone reaches it. +void verifyPureFlatSlots() { + using namespace ShaderRecompiler; + std::vector> values; + std::uint32_t ids = 0; + const auto make = [&](IrOpcode opcode, IrType type) -> IrValue& { + values.push_back(std::make_unique(opcode, type, ids++)); + return *values.back(); + }; + const auto constant = [&](std::uint32_t value) -> IrValue& { + auto& immediate = make(IrOpcode::Void, IrType::U32); + immediate.SetImmediateU32(value); + return immediate; + }; + auto& resource = make(IrOpcode::GetSrtResource, IrType::SrtResource); + const auto userData = [&](std::uint32_t index) -> IrValue& { + auto& reg = make(IrOpcode::Void, IrType::ScalarReg); + reg.SetRegister({RegisterBank::Scalar, index}); + auto& read = make(IrOpcode::GetUserData, IrType::U32); + read.AddArgument(®); + return read; + }; + const auto handle = [&](IrValue& low, IrValue& high) -> IrValue& { + auto& composed = make(IrOpcode::CompositeConstructU64, IrType::U64); + composed.AddArgument(&low); + composed.AddArgument(&high); + return composed; + }; + const auto rawRead = [&](IrValue& address, std::uint32_t offset) -> IrValue& { + auto& read = make(IrOpcode::LoadAddressU32, IrType::U32); + read.AddArgument(&address); + read.AddArgument(&constant(offset)); + return read; + }; + const auto readConst = [&](std::uint32_t slot) -> IrValue& { + auto& read = make(IrOpcode::ReadConst, IrType::U32); + read.AddArgument(&resource); + read.AddArgument(&constant(slot)); + return read; + }; + // Slots: A (0) reads through a pointer B (1) read from user data; C (2) and D (3) read from + // user-data handles; a descriptor source consumes C. + auto& b = rawRead(handle(userData(0), userData(1)), 0); + auto& c = rawRead(handle(userData(2), userData(3)), 4); + auto& a = rawRead(handle(readConst(1), userData(4)), 8); + auto& d = rawRead(handle(userData(5), userData(6)), 12); + IrResourcePlan plan; + plan.srtPlanComplete = true; + plan.resourceTrackingComplete = true; + plan.srtReads = {{&a, 0}, {&b, 1}, {&c, 2}, {&d, 3}}; + DescriptorSource source; + source.dwordCount = 1; + source.dwords[0] = &readConst(2); + plan.descriptorSources.push_back(source); + using Pure = std::vector; + require(Detail::ComputePureFlatSlots(plan) == Pure{1, 0, 0, 1}, "pure flat slots: the address cone or the descriptor source was not excluded"); + plan.controlFlow.push_back({&readConst(3), {}, {}}); + require(Detail::ComputePureFlatSlots(plan) == Pure{1, 0, 0, 0}, "pure flat slots: a control-flow condition was not excluded"); + plan.controlFlow.clear(); + auto& phi = make(IrOpcode::Phi, IrType::U32); + phi.AddArgument(&readConst(0)); + phi.AddArgument(&constant(0)); + plan.descriptorSources[0].dwords[0] = φ + require(Detail::ComputePureFlatSlots(plan) == Pure{0, 0, 1, 1}, "pure flat slots: a phi argument was not excluded"); + plan.descriptorSources[0].dwords[0] = &b; + require(Detail::ComputePureFlatSlots(plan) == Pure{1, 0, 1, 1}, "pure flat slots: a raw read named directly was not excluded"); + plan.descriptorSources[0].dwords[0] = &readConst(2); + plan.uniformFill.fill.kind = UniformFillKind::Buffer; + plan.uniformFill.fill.words = 1; + plan.uniformFill.values[0] = &readConst(3); + require(Detail::ComputePureFlatSlots(plan) == Pure{1, 0, 0, 0}, "pure flat slots: a uniform-fill value was not excluded"); + plan.uniformFill = {}; + plan.descriptorSources[0].indirectImage = DescriptorSource::IndirectImage{}; + require(Detail::ComputePureFlatSlots(plan) == Pure{0, 0, 0, 0}, "pure flat slots: an indirect image did not disqualify the plan"); + plan.descriptorSources[0].indirectImage.reset(); + plan.requiresSpecializationMemory = true; + require(Detail::ComputePureFlatSlots(plan) == Pure{0, 0, 0, 0}, "pure flat slots: specialization memory did not disqualify the plan"); + plan.requiresSpecializationMemory = false; + plan.srtPlanComplete = false; + require(Detail::ComputePureFlatSlots(plan) == Pure{0, 0, 0, 0}, "pure flat slots: an incomplete plan was classified"); +} + +// A compute program sampling a T# loaded from a table buffer at a runtime key (a bindless image +// table): mode M enumerates the keys from the material records, mode T binds the whole table. +void verifyBindlessTable() { + using namespace ShaderRecompiler; + constexpr std::uint32_t Format8888UNorm = 56; + constexpr std::uint32_t Type2D = 9; + const std::uint32_t slots = ResourceMaterializer::BindlessSlots(); + + struct alignas(256) Texture { std::array bytes{}; }; + static Texture textures[2]; + std::array, 4> heap{}; + const auto makeTexture = [&](std::uint32_t entry, const Texture& texture) { + const auto base = static_cast(reinterpret_cast(texture.bytes.data())); + heap[entry] = {static_cast(base >> 8u), static_cast((base >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (3u << 30u), 3u << 14u, 0xfacu | (Type2D << 28u), 0u, 0u, 0u, 0u}; + }; + makeTexture(0, textures[0]); + makeTexture(1, textures[1]); + heap[3] = heap[0]; + std::array, 3> materials{{{0u, 1u, 0u, 0u}, {0u, 0u, 0u, 0u}, {0u, 3u, 0u, 0u}}}; + std::array output{}; + const auto bufferDescriptor = [](const void* base, std::uint32_t stride, std::uint32_t records) { + const auto address = static_cast(reinterpret_cast(base)); + return std::array{static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), records, 0xfacu}; + }; + std::array srt{}; + const auto fillSrt = [&](std::uint32_t heapRecords) { + const auto heapV = bufferDescriptor(heap.data(), 32u, heapRecords); + const auto materialV = bufferDescriptor(materials.data(), 16u, 3u); + const auto outputV = bufferDescriptor(output.data(), 0u, 16u); + std::copy(heapV.begin(), heapV.end(), srt.begin()); + srt[4] = 0u; srt[5] = 0u; srt[6] = 0u; srt[7] = 0u; + std::copy(materialV.begin(), materialV.end(), srt.begin() + 8); + std::copy(outputV.begin(), outputV.end(), srt.begin() + 12); + }; + fillSrt(4u); + + // s_load_dwordx4 x4 (heap V#, S#, material V#, output V#); v_readfirstlane_b32 s16, v0; + // s_mul_i32 s16, s16, 16; s_buffer_load_dword s16, s[12:15], s16 offset:4; s_lshl_b32 s16, s16, 5; + // s_buffer_load_dwordx8 s[20:27], s[4:7], s16; image_sample_lz v[0:3], v[0:1], s[20:27], s[8:11]; + // buffer_store_dword v0, off, s[28:31], 0; s_endpgm. + const std::vector materialCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x93109010u, 0xf4200406u, 0x20000004u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; + // The same without the material read: the key is the wave's first lane id. + const std::vector wholeCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; + const auto srtAddress = static_cast(reinterpret_cast(srt.data())); + const std::array userData{static_cast(srtAddress), static_cast(srtAddress >> 32u)}; + const std::array capabilities{29u}; + // A wave64 workgroup on a 32-wide host is held by one subgroup (two lanes per invocation). + const auto makeRequest = [&](const std::vector& code) { + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.supportedCapabilities = capabilities; + request.target.fragmentShaderBarycentricEnabled = false; + request.layout.pushConstantSizeBytes = 128; + return request; + }; + const auto covered = [](const std::vector& regions, const void* pointer, std::size_t bytes) { + const auto address = static_cast(reinterpret_cast(pointer)); + return std::any_of(regions.begin(), regions.end(), [&](const MemoryRegion& region) { return address >= region.guestAddress && address + bytes <= region.guestAddress + region.bytes.size(); }); + }; + const auto mappingOf = [&](const ResourceSnapshot& snapshot) { + require(snapshot.flattenedSrt.size() >= 1u + 2u * slots, "bindless: the mapping block is missing from the flattened SRT"); + return std::vector(snapshot.flattenedSrt.end() - static_cast(1u + 2u * slots), snapshot.flattenedSrt.end()); + }; + const auto tableRoot = [&](const ResourceCapture& capture, std::uint32_t direct) { + require(capture.specialization.images.size() == direct + slots - 1u, "bindless: the specialization does not hold the table slots"); + require(capture.snapshot.images.size() == direct + slots - 1u, "bindless: the snapshot does not hold the table slots"); + std::uint32_t root = ImageResource::NoIndirectImage; + for (std::uint32_t i = 0; i < direct; i++) { + if (capture.specialization.images[i].indirectRoot == i) root = i; + } + require(root != ImageResource::NoIndirectImage, "bindless: no table root"); + for (std::uint32_t i = direct; i < capture.specialization.images.size(); i++) require(capture.specialization.images[i].indirectRoot == root, "bindless: an extra image is not the root's slot"); + return root; + }; + + auto request = makeRequest(materialCode); + const auto plan = GetResourcePlan(request); + std::size_t tables = 0; + for (const auto& source : plan->descriptorSources) { + if (!source.indirectImage.has_value()) continue; + ++tables; + const auto& table = *source.indirectImage; + require(table.hasMaterial && table.selectorStride == 16u && table.selectorOffset == 4u && table.entryOffset == 0u, "bindless: the material pattern was not recorded"); + } + require(tables == 1, "bindless: the table source was not planned"); + for (const auto& image : plan->info.images) require(image.indirectSearchIterations == 0u, "bindless: the plan carries a search depth"); + const auto direct = static_cast(plan->info.images.size()); + + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(request); + const auto root = tableRoot(*capture, direct); + require(capture->snapshot.images[root].dwords == heap[0] && capture->snapshot.images[direct].dwords == heap[1] && capture->snapshot.images[direct + 1u].dwords == heap[3], "bindless: the slots do not hold the keyed entries"); + for (std::uint32_t i = direct + 2u; i < capture->snapshot.images.size(); i++) require(capture->snapshot.images[i].dwords == heap[0], "bindless: a pad slot is not a copy of slot 0"); + const auto mapping = mappingOf(capture->snapshot); + require(std::vector(mapping.begin(), mapping.begin() + 7) == std::vector{3u, 0u, 0u, 1u, 1u, 3u, 2u}, "bindless: the (key, slot) mapping is wrong"); + require(capture->specialization.images[root].indirectMappingOffset + mapping.size() == capture->snapshot.flattenedSrt.size(), "bindless: the mapping offset does not name the block"); + auto regions = memory.Regions(); + for (const auto& material : materials) require(covered(regions, &material[1], sizeof(std::uint32_t)), "bindless: a material key was not captured"); + for (const auto entry : {0u, 1u, 3u}) require(covered(regions, heap[entry].data(), 32u), "bindless: a table entry was not captured"); + request.context.memory = regions; + const auto compiled = Recompile(request, *capture); + bool sampled = false; + bool flattened = false; + for (const auto& binding : compiled->bindings) { + if (binding.role == DescriptorRole::FlattenedSrt) flattened = true; + if (binding.kind != DescriptorKind::SampledImage) continue; + sampled = true; + require(binding.count == direct + slots - 1u && binding.guestDescriptor.size() == 8u * binding.count, "bindless: the sampled image binding does not hold the table slots"); + require(std::none_of(binding.imageWritten.begin(), binding.imageWritten.end(), [](bool written) { return written; }), "bindless: a table slot is marked written"); + } + require(sampled && flattened, "bindless: the bindings lack the image array or the flattened SRT"); + struct SpirvScan { + bool dynamicIndexing = false; + bool shaderNonUniform = false; + bool nonUniform = false; + bool switched = false; + }; + const auto scan = [&](const std::vector& words) { + SpirvScan result; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + require(count != 0 && count <= words.size() - cursor, "bindless: truncated SPIR-V instruction"); + const auto op = words[cursor] & 0xffffu; + if (op == 17u && words[cursor + 1] == 29u) result.dynamicIndexing = true; + if (op == 17u && words[cursor + 1] == 5301u) result.shaderNonUniform = true; + if (op == 71u && words[cursor + 2] == 5300u) result.nonUniform = true; + if (op == 251u) result.switched = true; + cursor += count; + } + return result; + }; + const auto uniform = scan(compiled->spirv); + require(uniform.dynamicIndexing && !uniform.switched, "bindless: the SPIR-V does not index the image array dynamically"); + require(!uniform.shaderNonUniform && !uniform.nonUniform, "bindless: a single-subgroup workgroup was decorated NonUniform"); +#if ANYPS5_ENABLE_SPIRV_TOOLS + static_cast(ValidateAndOptimizeSpirv(compiled->spirv, request.target.vulkanVersion, request.target.spirvVersion)); +#endif + + // A wave64 workgroup kept at one lane per invocation (a 64-wide host) spans two subgroups, so + // the slot needs NonUniform: rejected without the descriptor indexing capabilities, decorated + // with them. + auto split = request; + split.target.subgroupSize = 64; + AgcDriver::ShaderMemory splitMemory({}); + const auto splitCapture = splitMemory.Capture(split); + expectFailure([&] { static_cast(Recompile(split, *splitCapture)); }, "not uniform over the workgroup", "bindless: a split wave indexed the image array as uniform"); + const std::array indexingCapabilities{29u, 5301u, 5307u}; + const std::array indexingExtensions{"SPV_EXT_descriptor_indexing"}; + split.target.supportedCapabilities = indexingCapabilities; + split.target.supportedExtensions = indexingExtensions; + AgcDriver::ShaderMemory indexingMemory({}); + const auto indexingCapture = indexingMemory.Capture(split); + const auto splitScan = scan(Recompile(split, *indexingCapture)->spirv); + require(splitScan.dynamicIndexing && splitScan.shaderNonUniform && splitScan.nonUniform, "bindless: a split wave's slot is not decorated NonUniform"); + + // A key past the table (the no-texture marker 0xffffffff among them) and a key naming a null + // entry are left out of the mapping, so they sample zeros; the variant is the same. + for (const auto unmapped : {9u, 0xffffffffu}) { + materials[2][1] = unmapped; + AgcDriver::ShaderMemory rangeMemory({}); + const auto rangeCapture = rangeMemory.Capture(request); + const auto rangeMapping = mappingOf(rangeCapture->snapshot); + require(std::vector(rangeMapping.begin(), rangeMapping.begin() + 5) == std::vector{2u, 0u, 0u, 1u, 1u}, "bindless: an out-of-range key was kept"); + request.context.memory = rangeMemory.Regions(); + require(Recompile(request, *rangeCapture)->variantId == compiled->variantId, "bindless: the keys changed the variant"); + } + materials[2][1] = 2u; + AgcDriver::ShaderMemory nullMemory({}); + const auto nullCapture = nullMemory.Capture(request); + const auto nullMapping = mappingOf(nullCapture->snapshot); + require(std::vector(nullMapping.begin(), nullMapping.begin() + 5) == std::vector{2u, 0u, 0u, 1u, 1u}, "bindless: a null entry's key was mapped"); + require(nullCapture->snapshot.images[direct + 1u].dwords == heap[0], "bindless: a null entry's slot is not the pad"); + materials[2][1] = 3u; + + // Mode T: every entry keeps its slot; the null entry's slot holds the pad and its key is + // left out of the mapping. + auto whole = makeRequest(wholeCode); + const auto wholePlan = GetResourcePlan(whole); + for (const auto& source : wholePlan->descriptorSources) { + if (source.indirectImage.has_value()) require(!source.indirectImage->hasMaterial, "bindless: a material pattern was recorded without one"); + } + AgcDriver::ShaderMemory wholeMemory({}); + const auto wholeCapture = wholeMemory.Capture(whole); + const auto wholeDirect = static_cast(wholePlan->info.images.size()); + const auto wholeRoot = tableRoot(*wholeCapture, wholeDirect); + const auto wholeMapping = mappingOf(wholeCapture->snapshot); + require(std::vector(wholeMapping.begin(), wholeMapping.begin() + 7) == std::vector{3u, 0u, 0u, 1u, 1u, 3u, 3u}, "bindless: mode T is not the identity mapping"); + require(wholeCapture->snapshot.images[wholeRoot].dwords == heap[0] && wholeCapture->snapshot.images[wholeDirect].dwords == heap[1] && wholeCapture->snapshot.images[wholeDirect + 1u].dwords == heap[0] && wholeCapture->snapshot.images[wholeDirect + 2u].dwords == heap[3], "bindless: mode T slots are wrong"); + whole.context.memory = wholeMemory.Regions(); + require(!Recompile(whole, *wholeCapture)->spirv.empty(), "bindless: mode T did not compile"); + + // A table wider than the slots without a material pattern is rejected. + fillSrt(100u); + AgcDriver::ShaderMemory wideMemory({}); + expectFailure([&] { static_cast(wideMemory.Capture(whole)); }, "bindless image table has 100 entries", "bindless: a wide table was bound"); + fillSrt(4u); +} + } int main() { try { using namespace ShaderRecompiler; verifyRegisterSources(); + verifyPureFlatSlots(); + verifyBindlessTable(); #if ANYPS5_ENABLE_SPIRV_TOOLS const std::vector minimalSpirv{ 0x07230203u, 0x00010000u, 0u, 5u, 0u, @@ -110,9 +406,24 @@ int main() { expectFailure([&] { static_cast(Recompile(request)); }, "SrtWalker::EvaluateRuntimeSources", "missing snapshot unexpectedly read live memory"); AgcDriver::ShaderMemory memory({}); - memory.Capture(request); + const auto capture = memory.Capture(request); + { + // The payload slot is consumed by the export alone: pure, its leaf traced at the + // payload's address; the table pointer's words (the payload's address cone) are among + // the other reads. + const auto& pure = capture->plan->pureFlatSlots; + require(std::count(pure.begin(), pure.end(), std::uint8_t{1}) == 1, "the payload slot is not the one pure flat slot"); + const auto& trace = capture->readTrace; + require(trace.leaves.size() == 1 && trace.leaves[0].second == reinterpret_cast(&payload), "the pure slot's leaf was not traced at the payload"); + const auto slot = trace.leaves[0].first; + require(slot < pure.size() && pure[slot] != 0 && capture->snapshot.flattenedSrt.at(slot) == payload, "the traced leaf is not the pure slot"); + require(std::find(trace.otherReads.begin(), trace.otherReads.end(), address) != trace.otherReads.end(), "the table pointer read was not traced among the other reads"); + require(std::find(trace.otherReads.begin(), trace.otherReads.end(), reinterpret_cast(&payload)) == trace.otherReads.end(), "the payload counts as a walk read"); + } auto regions = memory.Regions(); - require(regions.size() == 3, "nested pointer reads were not captured"); + std::size_t capturedBytes = 0; + for (const auto& region : regions) capturedBytes += region.bytes.size(); + require(capturedBytes == sizeof(table) + sizeof(payload), "nested pointer reads were not captured"); request.context.memory = regions; const auto first = Recompile(request); require(!first.spirv.empty(), "empty compiled shader"); @@ -157,7 +468,7 @@ int main() { #endif payload = 0x40000000u; AgcDriver::ShaderMemory updatedMemory({}); - updatedMemory.Capture(request); + const auto updatedCapture = updatedMemory.Capture(request); const auto updatedRegions = updatedMemory.Regions(); auto updated = request; updated.context.memory = updatedRegions; @@ -168,6 +479,37 @@ int main() { bool changedData = updatedCached.pushConstants != first.pushConstants; for (std::size_t i = 0; i < first.bindings.size(); ++i) changedData = changedData || updatedCached.bindings.at(i).guestDescriptor != first.bindings[i].guestDescriptor; require(changedData, "cache hit retained stale shader data"); + { + // Two captures differing only in the payload (the pure slot) recompile to the same + // variant, with bindings equal apart from that slot's FlattenedSrt word. + payload = 0x40400000u; + AgcDriver::ShaderMemory changedMemory({}); + const auto changedCapture = changedMemory.Capture(request); + auto changed = updated; + changed.useCache = true; + changed.context.memory = changedMemory.Regions(); + auto before = updated; + before.useCache = true; + before.context.memory = updatedRegions; + const auto first = Recompile(before, *updatedCapture); + const auto second = Recompile(changed, *changedCapture); + require(first->variantId == second->variantId, "a pure slot's value changed the variant"); + require(first->bindings.size() == second->bindings.size(), "a pure slot's value changed the bindings"); + const auto slot = changedCapture->readTrace.leaves.at(0).first; + for (std::size_t i = 0; i < first->bindings.size(); ++i) { + const auto& left = first->bindings[i]; + const auto& right = second->bindings[i]; + if (left.role != DescriptorRole::FlattenedSrt) { + require(left.guestDescriptor == right.guestDescriptor, "a pure slot's value changed a non-data binding"); + continue; + } + require(left.guestDescriptor.size() == right.guestDescriptor.size() && left.guestDescriptor.at(slot) == 0x40000000u && right.guestDescriptor.at(slot) == 0x40400000u, "the FlattenedSrt binding does not carry the pure slot's value"); + for (std::size_t j = 0; j < left.guestDescriptor.size(); ++j) { + if (j != slot) require(left.guestDescriptor[j] == right.guestDescriptor[j], "the FlattenedSrt binding differs beyond the pure slot"); + } + } + payload = 0x40000000u; + } auto concurrent = request; concurrent.layout.pushConstantSizeBytes = 60; std::array, 4> concurrentResults; diff --git a/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp index 2a695b1e4f..5b4779f8a1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp @@ -19,7 +19,13 @@ struct Push { std::uint32_t tailX; std::uint32_t tailY; std::uint32_t elementBytes; - std::uint32_t arrayLayer; + std::uint32_t slice; + std::uint32_t rangeBegin; + std::uint32_t rangeEnd; + std::uint32_t tiledBase; + std::uint32_t linearBase; + std::uint32_t columnBegin; + std::uint32_t rowBegin; }; Push decodePush(const std::vector& bytes) { @@ -66,12 +72,14 @@ void RunTextureDetilerTests(const Context& context, const TextureDetilerTestAcce detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 20, destination, 40, layout, 0); auto capture = access.lastDispatch(); Require(capture.groupsX == 3 && capture.groupsY == 2 && capture.groupsZ == 1, "dispatch group counts were computed incorrectly"); - const auto push = decodePush(capture.pushConstants); + auto push = decodePush(capture.pushConstants); Require(push.srcBase == 4 && push.dstBase == 8, "push constant buffer bases must account for storage buffer offset alignment"); Require(push.width == 20 && push.height == 12, "push constant dimensions changed"); Require(push.pitchBytes == 96 && push.blocksPerRow == 5, "push constant row layout changed"); Require(push.tail == 0 && push.tailX == 0 && push.tailY == 0, "push constant tail fields must reflect a non-tail mip"); Require(push.elementBytes == 4, "push constant element size changed"); + Require(push.rangeBegin == 0 && push.rangeEnd == 0xffffffffu && push.tiledBase == 0 && push.linearBase == 0, "a whole-mip dispatch must move every element out of whole buffers"); + Require(push.columnBegin == 0 && push.rowBegin == 0, "a whole-mip dispatch starts at the mip's origin"); Require(capture.sourceBuffer == source && capture.destinationBuffer == destination, "dispatch bound the wrong source or destination buffer"); Require(capture.sourceOffset == 16 && capture.sourceRange == 68, "source descriptor offset or range computed incorrectly"); Require(capture.destinationOffset == 32 && capture.destinationRange == 56, "destination descriptor offset or range computed incorrectly"); @@ -79,22 +87,79 @@ void RunTextureDetilerTests(const Context& context, const TextureDetilerTestAcce const auto pipelinesAfterFirst = access.pipelineCount(); Require(pipelinesAfterFirst == 1, "the first dispatch must create exactly one compute pipeline"); + // A window inside the mip: tiled bytes [16, 48) of a mip whose tiled buffer starts at the + // mip's byte 16 and whose linear buffer holds one 96-byte row from linear byte 96; the grid + // covers only the window's rectangle (columns 8-16 of row 1: one workgroup) and the buffer + // ranges only the window's bytes, in both directions. + { + DetileWindow window{16, 48, 16, 96, 96}; + window.columnBegin = 8; + window.columnEnd = 16; + window.rowBegin = 1; + window.rowEnd = 2; + detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 20, destination, 40, layout, false, 0, false, window); + capture = access.lastDispatch(); + Require(capture.groupsX == 1 && capture.groupsY == 1 && capture.groupsZ == 1, "a windowed detile must dispatch only the window's workgroups"); + push = decodePush(capture.pushConstants); + Require(push.rangeBegin == 16 && push.rangeEnd == 48 && push.tiledBase == 16 && push.linearBase == 96, "windowed detile push constants must carry the window"); + Require(push.columnBegin == 8 && push.rowBegin == 1, "windowed detile push constants must carry the grid origin"); + Require(push.width == 20 && push.height == 12, "a window must not change the mip's dimensions"); + Require(capture.sourceOffset == 16 && capture.sourceRange == 4 + 32, "a windowed detile reads the window's tiled bytes only"); + Require(capture.destinationOffset == 32 && capture.destinationRange == 8 + 96, "a windowed detile writes the window's linear rows only"); + + detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 20, destination, 40, layout, true, 0, false, window); + capture = access.lastDispatch(); + Require(capture.groupsX == 1 && capture.groupsY == 1 && capture.groupsZ == 1, "a windowed retile must dispatch only the window's workgroups"); + push = decodePush(capture.pushConstants); + Require(push.rangeBegin == 16 && push.rangeEnd == 48 && push.tiledBase == 16 && push.linearBase == 96, "windowed retile push constants must carry the window"); + Require(push.columnBegin == 8 && push.rowBegin == 1, "windowed retile push constants must carry the grid origin"); + Require(capture.sourceOffset == 16 && capture.sourceRange == 4 + 96, "a windowed retile reads the window's linear rows only"); + Require(capture.destinationOffset == 32 && capture.destinationRange == 8 + 32, "a windowed retile writes the window's tiled bytes only"); + Require(access.pipelineCount() == pipelinesAfterFirst + 1, "a retile pipeline is separate from the detile pipeline"); + + // The unit-shadow form of the same window (StorageTexture::uploadWindows): the tiled bytes + // come from a sub-buffer holding the window from its first byte (a slab at offset 1000), + // so the tiled base is the range's begin and the descriptor covers the window's bytes + // only; the push constants and the grid are the import form's. + detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 1000, destination, 40, layout, false, 0, false, window); + capture = access.lastDispatch(); + Require(capture.groupsX == 1 && capture.groupsY == 1 && capture.groupsZ == 1, "a slab-sourced detile must dispatch only the window's workgroups"); + push = decodePush(capture.pushConstants); + Require(push.rangeBegin == 16 && push.rangeEnd == 48 && push.tiledBase == 16 && push.linearBase == 96, "slab-sourced detile push constants must carry the window"); + Require(push.columnBegin == 8 && push.rowBegin == 1, "slab-sourced detile push constants must carry the grid origin"); + Require(push.srcBase == 8 && capture.sourceOffset == 992 && capture.sourceRange == 8 + 32, "a slab-sourced detile reads the window's tiled bytes at the slab offset"); + Require(capture.destinationOffset == 32 && capture.destinationRange == 8 + 96, "a slab-sourced detile writes the same linear rows as the import form"); + + // Whole block rows: the columns default to the mip's width, the rows to the window's. + DetileWindow rows{0, 64, 0, 0, 48}; + rows.rowBegin = 8; + detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, layout, false, 0, false, rows); + capture = access.lastDispatch(); + Require(capture.groupsX == 3 && capture.groupsY == 1, "a row window must dispatch whole rows from its first row"); + reject([&] { + DetileWindow outside{0, 64, 0, 0, 0}; + outside.rowBegin = 12; + detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, layout, false, 0, false, outside); + }, "window lies outside the mip"); + } + const auto pipelinesAfterWindow = access.pipelineCount(); + detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, makeLayout(8, 8, 32, 2, 32, 32), 0); - Require(access.pipelineCount() == pipelinesAfterFirst, "dispatching with the same tile mode and element size must reuse the cached pipeline"); + Require(access.pipelineCount() == pipelinesAfterWindow, "dispatching with the same tile mode and element size must reuse the cached pipeline"); detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 8, source, 0, destination, 0, makeLayout(8, 8, 32, 2, 32, 32), 0); - Require(access.pipelineCount() == pipelinesAfterFirst + 1, "a different element size must create a new compute pipeline"); + Require(access.pipelineCount() == pipelinesAfterWindow + 1, "a different element size must create a new compute pipeline"); detiler.Dispatch(commands, TextureTileMode::kLinear, 4, source, 0, destination, 0, makeLayout(8, 8, 32, 2, 32, 32), 0); - Require(access.pipelineCount() == pipelinesAfterFirst + 2, "a different tile mode must create a new compute pipeline"); + Require(access.pipelineCount() == pipelinesAfterWindow + 2, "a different tile mode must create a new compute pipeline"); detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, makeLayout(8, 8, 32, 2, 32, 32), 0); - Require(access.pipelineCount() == pipelinesAfterFirst + 2, "reusing an earlier tile mode and element size must not create another pipeline"); + Require(access.pipelineCount() == pipelinesAfterWindow + 2, "reusing an earlier tile mode and element size must not create another pipeline"); - detiler.Dispatch(commands, TextureTileMode::RenderTarget64KB, 4, source, 0, destination, 0, layout, 13); + detiler.Dispatch(commands, TextureTileMode::RenderTarget64KB, 4, source, 0, destination, 0, layout, false, 13); capture = access.lastDispatch(); - Require(decodePush(capture.pushConstants).arrayLayer == 13, "render target detiling must preserve the absolute array layer for XOR addressing"); - Require(access.pipelineCount() == pipelinesAfterFirst + 3, "render target detiling must use a separate pipeline"); + Require(decodePush(capture.pushConstants).slice == 13, "render target detiling must preserve the absolute array layer for XOR addressing"); + Require(access.pipelineCount() == pipelinesAfterWindow + 3, "render target detiling must use a separate pipeline"); const auto specialization = access.lastSpecialization(); Require(specialization[0] == 4 && specialization[1] == 65536 && specialization[2] == 2, "render target detiling must select its own swizzle family"); @@ -107,4 +172,5 @@ void RunTextureDetilerTests(const Context& context, const TextureDetilerTestAcce reject([&] { detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, makeLayout(20, 12, 96, 5, 64, 0), 0); }, "non-empty mip layout"); reject([&] { detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 3, source, 0, destination, 0, layout, 0); }, "unsupported element size"); reject([&] { detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 20, destination, 40, makeLayout(20, 12, 96, 5, 1024, 48), 0); }, "buffer range exceeds device limits"); + reject([&] { detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, layout, false, 0, false, DetileWindow{48, 48, 0, 0, 0}); }, "window is empty"); } diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 60325e1735..e9f394071b 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -104,6 +104,7 @@ function(add_shader_recompiler target runtimeTarget) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SrtWalker/SrtDiagnostics.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SrtWalker/SrtEvaluator.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SrtWalker/SrtPlanBuilder.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SrtWalker/SrtRuntimeValidator.cpp diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index c11a390c83..1e6fc2025b 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -33,6 +33,27 @@ public: append(key, DebugProbeActive()); } + // A hash over every field Build appends except the code, the target and the probe flag: the + // key of a source memo whose owner fixes the code (a registered shader at an offset) and the + // device itself, and which is bypassed while the probe is active. + static std::uint64_t ContextHash(const RecompileRequest& request) { + thread_local std::vector key; + key.clear(); + append(key, request.shader.stage); + append(key, request.context.waveSize); + append(key, request.context.userDataBaseRegister); + append(key, request.context.userData.size()); + append(key, request.context.compute); + append(key, request.context.pixel); + append(key, request.context.vertex); + std::uint64_t hash = 0xcbf29ce484222325ull; + for (const auto value : key) { + hash ^= value; + hash *= 0x100000001b3ull; + } + return hash; + } + // A 64-bit hash of the code, two dwords per step; collisions are resolved by comparing the code. static std::uint64_t HashCode(std::span code) { std::uint64_t hash = 0x9e3779b97f4a7c15ull ^ (static_cast(code.size()) * 0x100000001b3ull); diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index be890c2d4b..5447bef4e5 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -20,12 +20,18 @@ struct BlockInfo { }; struct DescriptorSource { + // A T# loaded from a table buffer (`heapSource`) at `entryOffset + key * 32`, the key a + // wave-uniform runtime value (handle argument `keyArg`). With `hasMaterial` the key is itself + // `M[readfirstlane(i) * selectorStride + selectorOffset]` over `materialSource`, so the key + // set can be enumerated from the material records (see ResourceMaterializer). struct IndirectImage { std::uint32_t materialSource = 0; std::uint32_t heapSource = 0; std::uint32_t selectorStride = 0; std::uint32_t selectorOffset = 0; std::uint32_t keyArg = 0; + std::uint32_t entryOffset = 0; + bool hasMaterial = false; bool operator==(const IndirectImage& other) const = default; }; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp index b7b82ba3f6..b8111b9c65 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp @@ -61,6 +61,9 @@ struct IrResourcePlan { std::vector materializationSources; std::vector srtReads; std::vector cleanFlatSlots; + // One byte per srtReads slot, 1 when the CPU walk never consumes the slot's value (see + // Detail::ComputePureFlatSlots): a driver may reuse a capture whose words differ only there. + std::vector pureFlatSlots; bool requiresSpecializationMemory = false; bool srtPlanComplete = false; bool resourceTrackingComplete = false; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp index ec6d4359c8..39228dc3be 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp @@ -26,6 +26,10 @@ struct ShaderInfo { std::array vertexFetchComponents {}; std::int32_t vertexOffsetSgpr = -1; std::int32_t instanceOffsetSgpr = -1; + bool vertexOffsetShared = false; + bool instanceOffsetShared = false; + bool vertexOffsetConflict = false; + bool instanceOffsetConflict = false; bool hasBitwiseXor = false; bool usesDma = false; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index 50108dcfd2..ce0db25cb5 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -257,6 +257,9 @@ enum class IrOpcode : std::uint16_t { LoadAddressU8, LoadAddressU16, LoadAddressU32, + LoadAddressU32x2, + LoadAddressU32x3, + LoadAddressU32x4, StoreAddressU8, StoreAddressU16, StoreAddressU32, @@ -350,6 +353,7 @@ enum class ImageResourceClass { None, Sampled, Storage }; struct AddressOpcodeInfo { AddressAccess access = AddressAccess::None; std::uint32_t dataBits = 0; + std::uint32_t components = 1; }; struct ImageOpcodeInfo { diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp index 58097fda7b..e7aa605044 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp @@ -298,6 +298,9 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("LoadAddressU8", U8, AddressResource, U32, U32, U1), makeMeta("LoadAddressU16", U16, AddressResource, U32, U32, U1), makeMeta("LoadAddressU32", U32, AddressResource, U32, U32, U1), + makeMeta("LoadAddressU32x2", U32x2, AddressResource, U32, U32, U1), + makeMeta("LoadAddressU32x3", U32x3, AddressResource, U32, U32, U1), + makeMeta("LoadAddressU32x4", U32x4, AddressResource, U32, U32, U1), makeMeta("StoreAddressU8", Void, AddressResource, U32, U32, U8, U1), makeMeta("StoreAddressU16", Void, AddressResource, U32, U32, U16, U1), makeMeta("StoreAddressU32", Void, AddressResource, U32, U32, U32, U1), @@ -531,6 +534,12 @@ AddressOpcodeInfo AddressOpcodeInfoOf(IrOpcode opcode) { return {AddressAccess::Read, 16u}; case IrOpcode::LoadAddressU32: return {AddressAccess::Read, 32u}; + case IrOpcode::LoadAddressU32x2: + return {AddressAccess::Read, 32u, 2u}; + case IrOpcode::LoadAddressU32x3: + return {AddressAccess::Read, 32u, 3u}; + case IrOpcode::LoadAddressU32x4: + return {AddressAccess::Read, 32u, 4u}; case IrOpcode::StoreAddressU8: return {AddressAccess::Write, 8u}; case IrOpcode::StoreAddressU16: diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp index febd9143ab..ea86054cfb 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp @@ -509,7 +509,8 @@ void ValidateProgram(const IrProgram& program, bool requireSsa) { } const bool scalarAddress = memory.kind == ResourceKind::ScalarAddress; const bool validGroupWidth = scalarAddress ? (memory.componentCount == 1u || memory.componentCount == 2u || memory.componentCount == 4u || memory.componentCount == 8u || memory.componentCount == 16u) : (memory.componentCount >= 1u && memory.componentCount <= 4u); - if (memory.dataBits != addressInfo.dataBits || memory.dataDwords != 1u || !validGroupWidth || memory.componentIndex >= memory.componentCount || memory.sampler != 0u) { + const bool validWideLayout = addressInfo.components == 1u || (memory.componentIndex == 0u && memory.componentCount == addressInfo.components); + if (memory.dataBits != addressInfo.dataBits || memory.dataDwords != addressInfo.components || !validGroupWidth || !validWideLayout || memory.componentIndex >= memory.componentCount || memory.sampler != 0u) { fail(std::string(IrOpcodeName(inst->Opcode())) + " has inconsistent address-memory metadata"); } } diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index b732a6fea3..762ac4cf5f 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -40,12 +40,20 @@ struct ResourceSpecialization { std::vector boundDescriptors; }; +// Why a bindless image table (a T# loaded from a table buffer at a runtime key) was not bound; +// counted on the [bindless] line (APS5_PROFILE_DRAW). +enum class BindlessRejection { Capacity, MaterialScan, NoEntry, Storage, NonUniform, ImageSlots, Count }; + class ResourceMaterializer { public: void Apply(IrProgram& program, const ResourceSpecialization& specialization) const; [[nodiscard]] IrResourcePlan ExtractPlan(const IrProgram& program) const; void Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization) const; - + // APS5_PROFILE_DRAW: the time Materialize spent building specializations, over every call. + static std::uint64_t SpecializationNanoseconds(); + // The slots every bindless image table binds (APS5_BINDLESS_SLOTS, default 16, 1..48). + static std::uint32_t BindlessSlots(); + static void CountBindlessRejection(BindlessRejection reason); }; } diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index e22701682b..7e56f67e47 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -4,6 +4,7 @@ #include "Recompiler.hpp" #include "IntermediateRepresentation/IrProgram.hpp" #include "Optimization/ResourceMaterializer.hpp" +#include #include namespace ShaderRecompiler { @@ -20,9 +21,28 @@ struct ResourceCapture { ResourceSpecialization specialization; // The cache entry the plan belongs to; null when the request bypasses the cache. std::shared_ptr source; + // The walk's read addresses when the plan has pure flat slots (the leaf of each pure slot, + // and every other read sorted and deduplicated); empty otherwise. + SrtReadTrace readTrace; + // APS5_PROFILE_DRAW: what CaptureResources spent resolving the source (the stage inputs, the + // key over the code, the plan) before the walk. + std::uint64_t sourceNanoseconds = 0; }; [[nodiscard]] std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime); +// The resolved source of a request (its cache entry with the plan built), for a driver that +// memoizes it per registered shader: ResolveSource is what CaptureResources does before the walk +// (the stage input validation, the key over the code, the lookup, the first-sight plan build), and +// the overload below captures over a handle without repeating it. A handle stays valid for every +// request with the same code and the same cache key fields (RecompileCacheKey::ContextHash plus +// the target); the vertex stages' input validation is repeated per capture because it reads V# +// fields the key does not cover. Null for a request that bypasses the cache (useCache false). +struct SourceHandle { + std::shared_ptr source; +}; +[[nodiscard]] std::shared_ptr ResolveSource(const RecompileRequest& request); +[[nodiscard]] std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime, const SourceHandle& handle); + } #endif diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp index a61feb3cfa..f5f5a40703 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp @@ -5,18 +5,31 @@ #include #include +#include #include namespace ShaderRecompiler { using SrtMemoryReader = bool (*)(void* userData, std::uint64_t address, std::uint32_t* value); +// The guest addresses a walk dereferenced (Detail::Evaluator::EvaluateRawRead): the leaf read of +// each pure flat slot (IrResourcePlan::pureFlatSlots) as (flat offset, address), set by the +// flattened loop through `leaf`/`leafSlot` around that slot's evaluation, and every other raw read +// address. +struct SrtReadTrace { + const IrValue* leaf = nullptr; + std::uint32_t leafSlot = 0; + std::vector> leaves; + std::vector otherReads; +}; + struct SrtRuntime { std::span userData; std::uint64_t shaderBase = 0; SrtMemoryReader readMemory = nullptr; void* userContext = nullptr; SrtMemoryReader readSpecializationMemory = nullptr; + SrtReadTrace* readTrace = nullptr; }; enum class RuntimeValueType { diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp new file mode 100644 index 0000000000..f1f806f084 --- /dev/null +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp @@ -0,0 +1,22 @@ +#ifndef CORE_SHADER_RECOMPILIER_OPTIMIZATION_SRTWALKER_SRTFLATSLOTCLASSES_HPP +#define CORE_SHADER_RECOMPILIER_OPTIMIZATION_SRTWALKER_SRTFLATSLOTCLASSES_HPP + +#include "IntermediateRepresentation/IrProgram.hpp" + +#include +#include + +namespace ShaderRecompiler::Detail { + +// One byte per srtReads slot, 1 when the slot is pure: its value is consumed by nothing the CPU +// walk evaluates (no descriptor source dword, no control-flow condition, no uniform-fill value, no +// address argument of any raw read reaches it through the plan's use-def edges), so the shader +// reads it only through the FlattenedSrt binding at run time. A plan the walk cannot classify +// statically (an incomplete SRT plan, specialization memory, an indirect image, an invalid uniform +// fill) has no pure slot. Reachability over-approximates what the evaluators consume, so an +// unknown opcode only makes more slots impure. +std::vector ComputePureFlatSlots(const IrResourcePlan& plan); + +} + +#endif diff --git a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp index 8859a50df7..362fe9cc4a 100644 --- a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp +++ b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp @@ -109,7 +109,7 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin } std::array, ImageBindingCount> imageGroups; - for (std::uint32_t i = 0; i < info.images.size(); i++) { + const auto place = [&](std::uint32_t i) { const DescriptorBindingKind kind = DescriptorBindingForImage(info.images[i]); const std::uint32_t group = ImageBindingIndex(kind); if (group >= imageGroups.size()) { @@ -123,6 +123,24 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin std::to_string(info.images[i].mipCount)); } resources.insert(resources.end(), count, i); + }; + // A bindless table's slots follow their root as consecutive elements: the SPIR-V indexes the + // binding with element(root) + slot. + for (std::uint32_t i = 0; i < info.images.size(); i++) { + const auto root = info.images[i].indirectRoot; + if (root != ImageResource::NoIndirectImage && root != i) { + continue; + } + place(i); + if (root != i) { + continue; + } + for (const auto slot : info.images[i].indirectResources) { + if (slot >= info.images.size() || info.images[slot].indirectRoot != i) { + fail("shader binding layout failed: image " + std::to_string(i) + " has an inconsistent table slot"); + } + if (slot != i) place(slot); + } } for (std::uint32_t i = 0; i < imageGroups.size(); i++) { if (!imageGroups[i].empty()) { diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 9bc93845a1..37ec5bea26 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -1,13 +1,23 @@ #include "Optimization/ResourceMaterializer.hpp" +#include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp" #include "IntermediateRepresentation/IrBuilder.hpp" #include "Optimization/ShaderStageInputInfo.hpp" #include "RdnaDecoder/RdnaDescriptorFormat.hpp" #include #include +#include +#include +#include #include +#include +#include #include +#include +#include +#include #include #include +#include #include #include @@ -17,6 +27,13 @@ namespace { constexpr std::uint32_t NoRemap = std::numeric_limits::max(); +std::atomic specializationNanoseconds{0}; + +bool MaterializeProfiled() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + struct DecodedImage { IrTextureNumericClass numericClass = IrTextureNumericClass::Unsupported; RdnaImageDimension dimension = RdnaImageDimension::Unknown; @@ -109,10 +126,6 @@ std::uint32_t storageMipCount(const ImageResource& base, const DescriptorValue& return mipBase <= mipLast ? mipLast - mipBase + 1u : 0u; } -bool sameImageShape(const ResourceSpecialization::Image& left, const ResourceSpecialization::Image& right) { - return left.numericClass == right.numericClass && left.dimension == right.dimension && left.mipCount == right.mipCount && left.conversionFormat == right.conversionFormat && left.shaderSwizzle == right.shaderSwizzle && left.cube == right.cube && left.fmask == right.fmask; -} - DecodedImage decodeImageDescriptor(const DescriptorValue& descriptor, const ImageResource& base) { DecodedImage decoded; decoded.mipCount = storageMipCount(base, descriptor); @@ -165,53 +178,230 @@ bool requiresPointSampler(const ResourceSpecialization::Image& image) { return image.numericClass == IrTextureNumericClass::Sint || image.conversionFormat != IrBufferFormat::Invalid; } -void resolveIndirectImage(const IrResourcePlan& plan, std::uint32_t imageIndex, const DescriptorSource::IndirectImage& indirect, const SrtRuntime& runtime, SrtWalker& walker, DescriptorValue& resolved, std::vector& candidates) { - const auto& image = plan.info.images.at(imageIndex); - if (image.indirectSearchIterations == 0u) { - throw std::runtime_error("indirect image has no candidate range"); - } - DescriptorValue materialValue; - walker.EvaluateDescriptorSource(plan, indirect.materialSource, runtime, materialValue); - const ShaderBufferResource material = decodeBufferDescriptor(materialValue); - if (material.GetSize() == 0u) { - throw std::runtime_error("indirect image material buffer is empty"); - } - DescriptorValue heapValue; - walker.EvaluateDescriptorSource(plan, indirect.heapSource, runtime, heapValue); - const ShaderBufferResource heap = decodeBufferDescriptor(heapValue); - DescriptorValue keyValue; - walker.EvaluateDescriptorSource(plan, indirect.keyArg, runtime, keyValue); - if (keyValue.dwordCount == 0u) { - throw std::runtime_error("indirect image selector has no value"); - } - if (runtime.readMemory == nullptr) { - throw std::runtime_error("indirect image resolution requires runtime memory access"); - } - candidates.clear(); - candidates.reserve(image.indirectSearchIterations); - const std::uint64_t heapBase = heap.Base48(); - for (std::uint32_t iteration = 0; iteration < image.indirectSearchIterations; iteration++) { - DescriptorValue candidate; - candidate.dwordCount = 8u; - const std::uint64_t address = heapBase + indirect.selectorOffset + static_cast(iteration) * indirect.selectorStride; - for (std::uint32_t dword = 0; dword < 8u; dword++) { - if (!runtime.readMemory(runtime.userContext, address + dword * sizeof(std::uint32_t), &candidate.dwords[dword])) { - throw std::runtime_error("failed to read indirect image candidate from memory"); - } - } - if (nullImageDescriptor(candidate)) { - candidate.dwords.fill(0u); - } - candidates.push_back(candidate); - } - const auto key = keyValue.dwords[0]; - if (key >= candidates.size()) { - throw std::runtime_error("indirect image selector is out of range"); - } - resolved = candidates[key]; +constexpr std::uint32_t TableEntryBytes = 32; + +// Bindless image tables: the most material records mode M reads to enumerate a table's keys +// (APS5_BINDLESS_MATERIAL_SCAN, default 256); beyond it the whole table is bound instead. +std::uint32_t MaterialScanLimit() { + static const std::uint32_t limit = [] { + const char* text = std::getenv("APS5_BINDLESS_MATERIAL_SCAN"); + return text != nullptr ? static_cast(std::strtoul(text, nullptr, 0)) : 256u; + }(); + return limit; } -void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, SrtWalker& walker, ResourceSnapshot& snapshot, std::vector>& indirectCandidates) { +bool BindlessTraced() { + static const bool traced = std::getenv("APS5_TRACE_BINDLESS") != nullptr; + return traced; +} + +struct BindlessCounters { + std::atomic tablesMaterial{0}; + std::atomic tablesWhole{0}; + std::atomic keys{0}; + std::atomic paddedNull{0}; + std::atomic paddedShape{0}; + std::atomic paddedConversion{0}; + std::atomic outOfRange{0}; + std::array, static_cast(BindlessRejection::Count)> rejected{}; + std::atomic lastReport{0}; +}; + +BindlessCounters& bindlessCounters() { + static BindlessCounters counters; + return counters; +} + +[[noreturn]] void rejectTable(BindlessRejection reason, const std::string& message) { + ResourceMaterializer::CountBindlessRejection(reason); + throw std::runtime_error(message); +} + +void reportBindless() { + if (!MaterializeProfiled()) return; + auto& counters = bindlessCounters(); + const auto now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(std::memory_order_relaxed); + if (last == 0) { + counters.lastReport.compare_exchange_strong(last, now, std::memory_order_relaxed); + return; + } + if (now - last < 10'000'000'000ll || !counters.lastReport.compare_exchange_strong(last, now, std::memory_order_relaxed)) return; + const auto material = counters.tablesMaterial.exchange(0, std::memory_order_relaxed); + const auto whole = counters.tablesWhole.exchange(0, std::memory_order_relaxed); + const auto keys = counters.keys.exchange(0, std::memory_order_relaxed); + std::array(BindlessRejection::Count)> rejected{}; + std::uint64_t rejections = 0; + for (std::size_t i = 0; i < rejected.size(); i++) { + rejected[i] = counters.rejected[i].exchange(0, std::memory_order_relaxed); + rejections += rejected[i]; + } + if (material + whole + rejections == 0) return; + const auto tables = material + whole; + std::fprintf(stderr, "[bindless] (10 s): tables bound %llu (mode M %llu, mode T %llu), slots %u, keys avg %.1f, entries unmapped (sample zeros): null/invalid %llu, shape %llu, conversion %llu, out of range %llu; rejected: capacity %llu, material scan %llu, no entry %llu, storage %llu, non-uniform %llu, image slots %llu\n", + static_cast(tables), static_cast(material), static_cast(whole), ResourceMaterializer::BindlessSlots(), tables != 0 ? static_cast(keys) / static_cast(tables) : 0.0, + static_cast(counters.paddedNull.exchange(0, std::memory_order_relaxed)), static_cast(counters.paddedShape.exchange(0, std::memory_order_relaxed)), static_cast(counters.paddedConversion.exchange(0, std::memory_order_relaxed)), static_cast(counters.outOfRange.exchange(0, std::memory_order_relaxed)), + static_cast(rejected[0]), static_cast(rejected[1]), static_cast(rejected[2]), static_cast(rejected[3]), static_cast(rejected[4]), static_cast(rejected[5])); +} + +// A bindless image table's bound slots and its (key, slot) mapping, keys ascending. Slots the +// mapping does not name (past the keys, or an entry that is null, invalid or of another shape) +// hold a copy of the first usable entry: a key the mapping lacks samples zeros in the shader, as +// a null T# does on hardware. Empty when the image is no table root. +struct TableResolution { + std::vector slots; + std::vector> mapping; +}; + +struct TableTrace { + bool material = false; + std::uint32_t entries = 0; + std::uint32_t materialEntries = 0; + std::uint32_t keys = 0; + + bool operator==(const TableTrace& other) const = default; +}; + +void traceTable(const IrResourcePlan& plan, const ImageResource& image, const DescriptorSource::IndirectImage& table, std::uint64_t heapBase, std::uint64_t materialBase, const TableTrace& trace) { + static std::mutex mutex; + static std::map, TableTrace> seen; + std::lock_guard lock(mutex); + auto& last = seen[{plan.shaderHash, image.firstUsePc}]; + if (last == trace) return; + last = trace; + std::fprintf(stderr, "[bindless] table at pc 0x%x of shader %llx: heap V# base 0x%llx entries %u, material V# base 0x%llx entries %u stride 0x%x offset 0x%x, mode %c, %u keys\n", image.firstUsePc, static_cast(plan.shaderHash), static_cast(heapBase), trace.entries, static_cast(materialBase), trace.materialEntries, table.selectorStride, table.selectorOffset, trace.material ? 'M' : 'T', trace.keys); +} + +// The slots of the table image `imageIndex` (see TableResolution). Mode M enumerates the keys the +// dispatch can reach from the material records; mode T binds the whole table when it fits. +void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, const DescriptorSource::IndirectImage& table, const SrtRuntime& runtime, SrtWalker& walker, DescriptorValue& resolved, TableResolution& resolution) { + const auto& image = plan.info.images.at(imageIndex); + if (runtime.readMemory == nullptr) { + throw std::runtime_error("bindless image table resolution requires runtime memory access"); + } + if (image.resourceClass != ImageResourceClass::Sampled) { + rejectTable(BindlessRejection::Storage, "bindless storage image tables are unsupported"); + } + const auto slots = ResourceMaterializer::BindlessSlots(); + DescriptorValue heapValue; + walker.EvaluateDescriptorSource(plan, table.heapSource, runtime, heapValue); + const ShaderBufferResource heap = decodeBufferDescriptor(heapValue); + const std::uint64_t heapSize = heap.GetSize(); + const auto entries = heapSize > table.entryOffset ? static_cast(std::min((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits::max())) : 0u; + const auto readWord = [&](std::uint64_t address, std::uint32_t& word) { + if (!runtime.readMemory(runtime.userContext, address, &word)) { + throw std::runtime_error("failed to read a bindless image table from memory"); + } + }; + + auto& counters = bindlessCounters(); + std::vector keys; + bool materialMode = false; + std::uint32_t materialEntries = 0; + std::uint64_t materialBase = 0; + std::uint32_t outOfRange = 0; + if (table.hasMaterial && table.selectorStride != 0u) { + DescriptorValue materialValue; + walker.EvaluateDescriptorSource(plan, table.materialSource, runtime, materialValue); + const ShaderBufferResource material = decodeBufferDescriptor(materialValue); + materialBase = material.Base48(); + const std::uint64_t materialSize = material.GetSize(); + materialEntries = static_cast(std::min(materialSize / table.selectorStride, std::numeric_limits::max())); + if (materialEntries <= MaterialScanLimit()) { + materialMode = true; + for (std::uint32_t record = 0; record < materialEntries; record++) { + const std::uint64_t offset = static_cast(record) * table.selectorStride + table.selectorOffset; + if (offset + sizeof(std::uint32_t) > materialSize) break; + std::uint32_t key = 0; + readWord(materialBase + offset, key); + if (key >= entries) { + outOfRange++; + continue; + } + keys.push_back(key); + } + std::sort(keys.begin(), keys.end()); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); + if (keys.size() > slots) { + rejectTable(BindlessRejection::Capacity, "bindless image table: " + std::to_string(keys.size()) + " distinct keys exceed the " + std::to_string(slots) + " slots"); + } + } + } + if (!materialMode) { + if (entries > slots) { + rejectTable(table.hasMaterial ? BindlessRejection::MaterialScan : BindlessRejection::Capacity, "bindless image table has " + std::to_string(entries) + " entries (" + std::to_string(materialEntries) + " materials), limit " + std::to_string(slots)); + } + keys.resize(entries); + for (std::uint32_t key = 0; key < entries; key++) keys[key] = key; + } + counters.outOfRange.fetch_add(outOfRange, std::memory_order_relaxed); + if (keys.empty()) { + rejectTable(BindlessRejection::NoEntry, "bindless image table selects no entry"); + } + + const std::uint64_t heapBase = heap.Base48(); + std::vector candidates(keys.size()); + std::vector valid(keys.size(), 0u); + std::optional shape; + std::uint32_t paddedNull = 0; + std::uint32_t paddedShape = 0; + std::uint32_t paddedConversion = 0; + for (std::size_t i = 0; i < keys.size(); i++) { + auto& candidate = candidates[i]; + candidate.dwordCount = 8u; + const std::uint64_t address = heapBase + table.entryOffset + static_cast(keys[i]) * TableEntryBytes; + for (std::uint32_t dword = 0; dword < 8u; dword++) { + readWord(address + dword * sizeof(std::uint32_t), candidate.dwords[dword]); + } + DecodedImage decoded; + bool usable = !nullImageDescriptor(candidate) && validImageDescriptor(candidate, image.r128); + if (usable) { + try { + decoded = decodeImageDescriptor(candidate, image); + } catch (const std::exception&) { + usable = false; + } + } + if (!usable) { + paddedNull++; + continue; + } + if (decoded.conversionFormat != IrBufferFormat::Invalid || decoded.fmask) { + paddedConversion++; + continue; + } + if (shape.has_value() && (decoded.numericClass != shape->numericClass || decoded.dimension != shape->dimension || decoded.cube != shape->cube)) { + paddedShape++; + continue; + } + if (!shape.has_value()) shape = decoded; + valid[i] = 1u; + } + if (!shape.has_value()) { + rejectTable(BindlessRejection::NoEntry, "bindless image table has no valid entry"); + } + const auto pad = candidates[static_cast(std::find(valid.begin(), valid.end(), std::uint8_t{1}) - valid.begin())]; + resolution.mapping.clear(); + for (std::size_t i = 0; i < keys.size(); i++) { + if (valid[i] == 0u) { + candidates[i] = pad; + continue; + } + resolution.mapping.emplace_back(keys[i], static_cast(i)); + } + resolution.slots = std::move(candidates); + resolution.slots.resize(slots, pad); + resolved = resolution.slots[0]; + + (materialMode ? counters.tablesMaterial : counters.tablesWhole).fetch_add(1, std::memory_order_relaxed); + counters.keys.fetch_add(resolution.mapping.size(), std::memory_order_relaxed); + counters.paddedNull.fetch_add(paddedNull, std::memory_order_relaxed); + counters.paddedShape.fetch_add(paddedShape, std::memory_order_relaxed); + counters.paddedConversion.fetch_add(paddedConversion, std::memory_order_relaxed); + if (BindlessTraced()) traceTable(plan, image, table, heapBase, materialBase, {materialMode, entries, materialEntries, static_cast(resolution.mapping.size())}); +} + +void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, SrtWalker& walker, ResourceSnapshot& snapshot, std::vector& tables) { snapshot = ResourceSnapshot{}; if (plan.uniformFill.fill.kind != UniformFillKind::None) { const auto words = plan.uniformFill.fill.words; @@ -245,19 +435,29 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, cursor += plan.info.buffers.size(); snapshot.images.resize(plan.info.images.size()); - indirectCandidates.assign(plan.info.images.size(), {}); + tables.assign(plan.info.images.size(), {}); + std::uint32_t activeTables = 0; for (std::uint32_t i = 0; i < plan.info.images.size(); i++) { const auto& image = plan.info.images[i]; if (image.source >= plan.descriptorSources.size()) { throw std::runtime_error("image resource references an unknown descriptor source"); } + const bool active = image.source >= activeSources.size() || activeSources[image.source] != 0u; + if (plan.descriptorSources[image.source].indirectImage.has_value() && active) activeTables++; + } + const auto tableSlots = ResourceMaterializer::BindlessSlots(); + if (activeTables != 0u && plan.info.images.size() + static_cast(tableSlots - 1u) * activeTables > ShaderInfo::MaxImages) { + rejectTable(BindlessRejection::ImageSlots, "bindless image tables need " + std::to_string(plan.info.images.size() + static_cast(tableSlots - 1u) * activeTables) + " image slots, limit " + std::to_string(ShaderInfo::MaxImages)); + } + for (std::uint32_t i = 0; i < plan.info.images.size(); i++) { + const auto& image = plan.info.images[i]; const auto& source = plan.descriptorSources[image.source]; if (source.indirectImage.has_value()) { if (image.source < activeSources.size() && activeSources[image.source] == 0u) { snapshot.images[i].dwordCount = 8u; continue; } - resolveIndirectImage(plan, i, *source.indirectImage, runtime, walker, snapshot.images[i], indirectCandidates[i]); + resolveTableImage(plan, i, *source.indirectImage, runtime, walker, snapshot.images[i], tables[i]); continue; } if (cursor >= values.size()) { @@ -280,7 +480,7 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, snapshot.samplers.assign(values.begin() + cursor, values.begin() + cursor + plan.info.samplers.size()); } -void buildResourceSpecialization(const IrResourcePlan& plan, ResourceSnapshot& snapshot, std::vector>& indirectCandidates, ResourceSpecialization& specialization) { +void buildResourceSpecialization(const IrResourcePlan& plan, ResourceSnapshot& snapshot, const std::vector& tables, ResourceSpecialization& specialization) { ResourceSpecialization result; result.buffers.reserve(plan.info.buffers.size()); for (std::uint32_t i = 0; i < plan.info.buffers.size(); i++) { @@ -325,43 +525,32 @@ void buildResourceSpecialization(const IrResourcePlan& plan, ResourceSnapshot& s result.images.push_back(entry); } + // A table root is followed by its slots 1..C-1 as extra images of the root's shape; the + // (key, slot) mapping the SPIR-V selector searches is appended to the flattened SRT in a + // block of fixed size, so the offsets (part of the specialization) never depend on the keys. for (std::uint32_t i = 0; i < plan.info.images.size(); i++) { - auto& candidates = indirectCandidates[i]; - if (candidates.empty()) { + const auto& table = tables[i]; + if (table.slots.empty()) { continue; } - const auto& image = plan.info.images[i]; const auto mappingOffset = static_cast(snapshot.flattenedSrt.size()); - snapshot.flattenedSrt.push_back(static_cast(candidates.size())); - for (const auto& candidate : candidates) { - const DecodedImage decodedCandidate = decodeImageDescriptor(candidate, image); - ResourceSpecialization::Image entry; - entry.numericClass = decodedCandidate.numericClass; - entry.dimension = decodedCandidate.dimension; - entry.mipCount = decodedCandidate.mipCount; - entry.conversionFormat = decodedCandidate.conversionFormat; - entry.shaderSwizzle = decodedCandidate.shaderSwizzle; - entry.indirectRoot = i; - entry.indirectMappingOffset = 0u; - entry.indirectSearchIterations = 0u; - entry.cube = decodedCandidate.cube; - entry.fmask = decodedCandidate.fmask; - std::uint32_t targetIndex = NoRemap; - for (std::uint32_t search = i; search < result.images.size(); search++) { - if (sameImageShape(result.images[search], entry)) { - targetIndex = search; - break; - } - } - if (targetIndex == NoRemap) { - result.images.push_back(entry); - targetIndex = static_cast(result.images.size() - 1); - } - snapshot.flattenedSrt.push_back(targetIndex); + snapshot.flattenedSrt.push_back(static_cast(table.mapping.size())); + for (const auto& [key, slot] : table.mapping) { + snapshot.flattenedSrt.push_back(key); + snapshot.flattenedSrt.push_back(slot); + } + snapshot.flattenedSrt.resize(mappingOffset + 1u + 2u * table.slots.size(), 0u); + auto root = result.images[i]; + root.indirectRoot = i; + root.indirectMappingOffset = mappingOffset; + root.indirectSearchIterations = static_cast(std::bit_width(table.slots.size())); + result.images[i] = root; + root.indirectMappingOffset = 0u; + root.indirectSearchIterations = 0u; + for (std::uint32_t slot = 1; slot < table.slots.size(); slot++) { + result.images.push_back(root); + snapshot.images.push_back(table.slots[slot]); } - result.images[i].indirectRoot = i; - result.images[i].indirectMappingOffset = mappingOffset; - result.images[i].indirectSearchIterations = static_cast(candidates.size()); } result.boundDescriptors.clear(); @@ -629,6 +818,7 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const } } for (const auto& sampler : plan.info.samplers) addSource(sampler.source); + plan.pureFlatSlots = Detail::ComputePureFlatSlots(plan); return plan; } @@ -642,12 +832,37 @@ void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtR } SrtWalker walker; ResourceSnapshot nextSnapshot; - std::vector> indirectCandidates; - materializeSnapshot(plan, runtime, walker, nextSnapshot, indirectCandidates); + std::vector tables; + try { + materializeSnapshot(plan, runtime, walker, nextSnapshot, tables); + } catch (...) { + reportBindless(); + throw; + } ResourceSpecialization nextSpecialization; - buildResourceSpecialization(plan, nextSnapshot, indirectCandidates, nextSpecialization); + const auto started = MaterializeProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + buildResourceSpecialization(plan, nextSnapshot, tables, nextSpecialization); + if (MaterializeProfiled()) specializationNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); snapshot = std::move(nextSnapshot); specialization = std::move(nextSpecialization); + reportBindless(); +} + +std::uint64_t ResourceMaterializer::SpecializationNanoseconds() { + return specializationNanoseconds.load(std::memory_order_relaxed); +} + +std::uint32_t ResourceMaterializer::BindlessSlots() { + static const std::uint32_t slots = [] { + const char* text = std::getenv("APS5_BINDLESS_SLOTS"); + const auto value = text != nullptr ? std::strtoul(text, nullptr, 0) : 16ul; + return static_cast(std::clamp(value, 1ul, 48ul)); + }(); + return slots; +} + +void ResourceMaterializer::CountBindlessRejection(BindlessRejection reason) { + if (reason < BindlessRejection::Count) bindlessCounters().rejected[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); } bool ResourceSpecialization::Buffer::operator==(const Buffer& other) const { diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 6e11c7f2b8..933ad2454b 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -1,5 +1,6 @@ #include "Optimization/ResourceTracker.hpp" #include "Optimization/SrtWalker.hpp" +#include "Optimization/ResourceMaterializer.hpp" #include "IntermediateRepresentation/IrBuilder.hpp" #include @@ -125,7 +126,7 @@ public: for (const auto& plan : m_indirectImages) { plan.handle->ReplaceArgument(0, plan.key); for (std::uint32_t dword = 0; dword < 4u; dword++) { - plan.handle->ReplaceArgument(dword + 1u, plan.roots[dword + 4u]); + plan.handle->ReplaceArgument(dword + 1u, plan.roots[dword]); } for (std::uint32_t dword = 5u; dword < plan.roots.size(); dword++) { plan.handle->ReplaceArgument(dword, plan.key); @@ -323,7 +324,49 @@ private: return stride != 0u && selector->Opcode() == IrOpcode::ReadFirstLane; } - bool TryMakeIndirectImage(IrValue& handle, IndirectImagePlan& plan) { + // The table entry offset `key * 32` (a shift or a multiply), optionally plus an immediate. + static bool MatchTableOffset(IrValue* value, IrValue*& key, std::uint32_t& entryOffset) { + value = value->Resolve(); + entryOffset = 0; + if (value->Opcode() == IrOpcode::IAdd32 && value->ArgumentCount() == 2u) { + std::uint32_t immediate = 0; + if (immediateU32(value->Argument(0), immediate)) { + value = value->Argument(1)->Resolve(); + } else if (immediateU32(value->Argument(1), immediate)) { + value = value->Argument(0)->Resolve(); + } else { + return false; + } + entryOffset = immediate; + } + if (value->ArgumentCount() != 2u) { + return false; + } + std::uint32_t scale = 0; + if (value->Opcode() == IrOpcode::ShiftLeftLogical32) { + if (!immediateU32(value->Argument(1), scale) || scale != 5u) { + return false; + } + key = value->Argument(0)->Resolve(); + return true; + } + if (value->Opcode() != IrOpcode::IMul32) { + return false; + } + if (immediateU32(value->Argument(1), scale)) { + key = value->Argument(0)->Resolve(); + } else if (immediateU32(value->Argument(0), scale)) { + key = value->Argument(1)->Resolve(); + } else { + return false; + } + return scale == 32u; + } + + // A T# whose eight dwords are scalar reads of one V# (the table) at `entryOffset + key * 32` + // with a wave-uniform runtime key: a bindless image table. The key stays an ordinary value + // (the SPIR-V selects the bound slot from it); the eight reads become planning-only. + bool TryMakeTableImage(IrValue& handle, IndirectImagePlan& plan) { if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u) { return false; } @@ -331,11 +374,17 @@ private: std::array heapReads {}; IrValue* heapHandle = nullptr; IrValue* heapOffset = nullptr; + std::uint32_t immediateOffset = 0; for (std::uint32_t dword = 0; dword < heapReads.size(); dword++) { heapReads[dword] = handle.Argument(dword)->Resolve(); std::uint32_t memoryIndex = 0; const MemoryInfo* memory = ScalarReadMemory(*heapReads[dword], memoryIndex); - if (memory == nullptr || memory->offset != dword * sizeof(std::uint32_t) || !MemoryIndexBelongsTo(memoryIndex, *heapReads[dword])) { + if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *heapReads[dword])) { + return false; + } + if (dword == 0u) { + immediateOffset = memory->offset; + } else if (memory->offset != immediateOffset + dword * sizeof(std::uint32_t)) { return false; } IrValue* currentHandle = heapReads[dword]->Argument(0)->Resolve(); @@ -352,56 +401,57 @@ private: plan.reads[dword] = heapReads[dword]; } - IrValue* shift = heapOffset; - std::uint32_t shiftAmount = 0; - if (shift->Opcode() != IrOpcode::ShiftLeftLogical32 || shift->ArgumentCount() != 2u || !immediateU32(shift->Argument(1), shiftAmount) || shiftAmount != 5u) { + IrValue* key = nullptr; + std::uint32_t entryOffset = 0; + if (!MatchTableOffset(heapOffset, key, entryOffset) || key->Type() != IrType::U32) { return false; } - IrValue* materialRead = shift->Argument(0)->Resolve(); - std::uint32_t materialMemoryIndex = 0; - const MemoryInfo* materialMemory = ScalarReadMemory(*materialRead, materialMemoryIndex); - if (materialMemory == nullptr || materialMemory->offset != 0u || !MemoryIndexBelongsTo(materialMemoryIndex, *materialRead)) { - return false; - } - IrValue* materialHandle = materialRead->Argument(0)->Resolve(); + entryOffset += immediateOffset; - IrValue* selector = nullptr; - std::uint32_t selectorStride = 0; - std::uint32_t selectorOffset = 0; - if (!MatchMaterialOffset(materialRead->Argument(1), selector, selectorStride, selectorOffset)) { - return false; - } - - const std::array materialUsers {shift}; - std::array heapUsers {}; - std::copy(heapReads.begin(), heapReads.end(), heapUsers.begin()); const std::array imageUsers {&handle}; - if (!usesOnly(*materialRead, materialUsers) || !usesOnly(*shift, heapUsers)) { - return false; - } for (const auto* read : heapReads) { if (!usesOnly(*read, imageUsers)) { return false; } } - DescriptorSource materialSource; DescriptorSource heapSource; - std::uint32_t materialSourceIndex = 0; std::uint32_t heapSourceIndex = 0; - if (!MakeRuntimeBufferSource(*materialHandle, materialSourceIndex, materialSource) || !MakeRuntimeBufferSource(*heapHandle, heapSourceIndex, heapSource)) { + if (!MakeRuntimeBufferSource(*heapHandle, heapSourceIndex, heapSource)) { return false; } + DescriptorSource::IndirectImage table; + table.heapSource = heapSourceIndex; + table.materialSource = heapSourceIndex; + table.entryOffset = entryOffset; + DescriptorSource materialSource = heapSource; + std::uint32_t materialMemoryIndex = 0; + const MemoryInfo* materialMemory = ScalarReadMemory(*key, materialMemoryIndex); + if (materialMemory != nullptr && MemoryIndexBelongsTo(materialMemoryIndex, *key)) { + IrValue* selector = nullptr; + std::uint32_t selectorStride = 0; + std::uint32_t selectorOffset = 0; + DescriptorSource candidateSource; + std::uint32_t candidateIndex = 0; + if (MatchMaterialOffset(key->Argument(1), selector, selectorStride, selectorOffset) && MakeRuntimeBufferSource(*key->Argument(0)->Resolve(), candidateIndex, candidateSource)) { + table.hasMaterial = true; + table.materialSource = candidateIndex; + table.selectorStride = selectorStride; + table.selectorOffset = selectorOffset + materialMemory->offset; + materialSource = candidateSource; + } + } + DescriptorSource imageSource; imageSource.dwordCount = 8u; - std::copy(materialSource.dwords.begin(), materialSource.dwords.begin() + 4u, imageSource.dwords.begin()); - std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin() + 4u); - imageSource.indirectImage = DescriptorSource::IndirectImage {materialSourceIndex, heapSourceIndex, selectorStride, selectorOffset, 0u}; + std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin()); + std::copy(materialSource.dwords.begin(), materialSource.dwords.begin() + 4u, imageSource.dwords.begin() + 4u); + imageSource.indirectImage = table; plan.handle = &handle; plan.source = InternSource(imageSource); - plan.key = materialRead; + plan.key = key; plan.roots = imageSource.dwords; return true; } @@ -420,17 +470,30 @@ private: } void PlanIndirectImages() { + // Kill switch: APS5_NO_BINDLESS_IMAGES=1 leaves table-loaded T#s unplanned (GetHandle then + // rejects them as before). + static const bool enabled = std::getenv("APS5_NO_BINDLESS_IMAGES") == nullptr; + if (!enabled) { + return; + } for (auto& block : m_program.Blocks()) { for (IrValue* inst : block->Instructions()) { - if (ImageOpcodeInfoOf(inst->Opcode()).access == ImageAccess::None || inst->ArgumentCount() == 0u) { + const auto imageInfo = ImageOpcodeInfoOf(inst->Opcode()); + if (imageInfo.access == ImageAccess::None || inst->ArgumentCount() == 0u) { continue; } IrValue* handle = inst->Argument(0)->Resolve(); - if (FindIndirectImage(*handle) != nullptr) { + const IndirectImagePlan* planned = FindIndirectImage(*handle); + IndirectImagePlan plan; + if (planned == nullptr && !TryMakeTableImage(*handle, plan)) { continue; } - IndirectImagePlan plan; - if (TryMakeIndirectImage(*handle, plan)) { + // A store through a table would mark every bound slot pending write-back. + if (imageInfo.resourceClass == ImageResourceClass::Storage) { + ResourceMaterializer::CountBindlessRejection(BindlessRejection::Storage); + fail("bindless storage image tables are unsupported"); + } + if (planned == nullptr) { m_indirectImages.push_back(std::move(plan)); } } diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp index 89b1a36703..ee1e7c6d63 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp @@ -118,7 +118,18 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span= flattened.size() || !selected.Evaluate(read.value, flattened[read.flatOffset])) { + // A pure slot's raw read is reachable from no root, so it was not evaluated (nor + // cached) before this loop: its dereference happens here, once, and is recorded as + // the slot's leaf; reads nested in its address cone land among the other reads. + auto* trace = runtime.readTrace; + const bool pure = trace != nullptr && read.flatOffset < program.pureFlatSlots.size() && program.pureFlatSlots[read.flatOffset] != 0u; + if (pure) { + trace->leaf = read.value->Resolve(); + trace->leafSlot = read.flatOffset; + } + const bool evaluated = read.flatOffset < flattened.size() && selected.Evaluate(read.value, flattened[read.flatOffset]); + if (pure) trace->leaf = nullptr; + if (!evaluated) { return Fail(std::string(clean ? "clean " : "") + "SRT read at flat offset " + std::to_string(read.flatOffset) + ": " + DescribeValue(read.value, 4)); } } diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp index 2785f182e3..c7377f1017 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp @@ -153,6 +153,10 @@ bool Evaluator::EvaluateRawRead(IrValue& inst, std::uint64_t& result) { return false; } } + if (auto* trace = _runtime.readTrace; trace != nullptr) { + if (&inst == trace->leaf) trace->leaves.emplace_back(trace->leafSlot, address); + else trace->otherReads.push_back(address); + } std::uint32_t word = 0; if (_runtime.readMemory != nullptr) { if (!_runtime.readMemory(_runtime.userContext, address, &word)) { diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp new file mode 100644 index 0000000000..d0b787b7ff --- /dev/null +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp @@ -0,0 +1,64 @@ +#include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp" + +#include +#include +#include + +namespace ShaderRecompiler::Detail { + +std::vector ComputePureFlatSlots(const IrResourcePlan& plan) { + std::vector pure(plan.srtReads.size(), 1u); + if (pure.empty()) return pure; + const bool disqualified = !plan.srtPlanComplete || plan.requiresSpecializationMemory + || std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.indirectImage.has_value(); }) + || (plan.uniformFill.fill.kind != UniformFillKind::None && plan.uniformFill.fill.words > plan.uniformFill.values.size()); + if (disqualified) { + std::fill(pure.begin(), pure.end(), 0u); + return pure; + } + // The raw read node of each slot: a descriptor source dword may point at it directly (the + // tracker resolved the handle argument before the plan builder patched the operands). + std::unordered_map readSlots; + for (std::uint32_t slot = 0; slot < plan.srtReads.size(); ++slot) { + if (plan.srtReads[slot].value != nullptr) readSlots.emplace(plan.srtReads[slot].value->Resolve(), slot); + } + std::unordered_set visited; + std::vector pending; + const auto mark = [&](IrValue* root) { + if (root == nullptr) return; + pending.push_back(root); + while (!pending.empty()) { + IrValue* value = pending.back()->Resolve(); + pending.pop_back(); + if (!visited.insert(value).second) continue; + if (value->Opcode() == IrOpcode::ReadConst && value->ArgumentCount() >= 2 && value->Argument(1) != nullptr) { + const IrValue* slot = value->Argument(1)->Resolve(); + if (slot->HasImmediate() && slot->Type() == IrType::U32 && slot->ImmediateU32() < pure.size()) { + const auto index = slot->ImmediateU32(); + pure[index] = 0u; + pending.push_back(plan.srtReads[index].value); + } + } + if (const auto found = readSlots.find(value); found != readSlots.end()) pure[found->second] = 0u; + for (std::size_t i = 0; i < value->ArgumentCount(); ++i) { + if (value->Argument(i) != nullptr) pending.push_back(value->Argument(i)); + } + } + }; + for (const auto& source : plan.descriptorSources) { + for (std::uint32_t i = 0; i < source.dwordCount && i < source.dwords.size(); ++i) mark(source.dwords[i]); + } + for (const auto& block : plan.controlFlow) mark(block.condition); + if (plan.uniformFill.fill.kind != UniformFillKind::None) { + for (std::uint32_t i = 0; i < plan.uniformFill.fill.words; ++i) mark(plan.uniformFill.values[i]); + } + // The address cones: the arguments of every slot's raw read, never the read itself. + for (const auto& read : plan.srtReads) { + if (read.value == nullptr) continue; + IrValue* raw = read.value->Resolve(); + for (std::size_t i = 0; i < raw->ArgumentCount(); ++i) mark(raw->Argument(i)); + } + return pure; +} + +} diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 52be8f178d..f147573661 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -1,9 +1,11 @@ #include "Recompiler.hpp" #include #include +#include #include #include #include "CacheKey.hpp" +#include #include #include #include @@ -166,6 +168,15 @@ struct CompiledVariant { RecompileResult result; }; +// A materialized result of one variant over one snapshot (Recompile(request, capture)): the +// shared immutable object every later capture that reproduces the snapshot receives, so Populate +// and the per-request copy run once per distinct snapshot. +struct ResultMemoEntry { + std::uint64_t variantId; + std::uint64_t hash; + std::shared_ptr result; +}; + struct SourceEntry { std::mutex mutex; // The code the entry was built for: the key carries only a hash of it, so a candidate entry is @@ -173,7 +184,14 @@ struct SourceEntry { // points into a registration the driver may replace while the entry lives on. std::vector code; std::shared_ptr plan; + // A plan build that threw (an unsupported resource chain or control flow) is remembered and + // rethrown: the front end ran every pass before failing, ~13 ms per dispatch of a shader the + // title issues every frame (0x1048947300 at the intro video). APS5_NO_FAILURE_MEMO=1 rebuilds. + std::exception_ptr planFailure; std::vector> variants; + // The result memo, most recently used first, at most ResultMemoEntries (under mutex). + std::list memo; + std::unordered_map::iterator> memoIndex; }; namespace { @@ -241,7 +259,14 @@ std::shared_ptr getSource(const RecompileRequest& request) { { std::lock_guard lock(source->mutex); if (source->plan == nullptr) { - source->plan = makeResourcePlan(request); + static const bool memoFailures = std::getenv("APS5_NO_FAILURE_MEMO") == nullptr; + if (memoFailures && source->planFailure) std::rethrow_exception(source->planFailure); + try { + source->plan = makeResourcePlan(request); + } catch (...) { + if (memoFailures) source->planFailure = std::current_exception(); + throw; + } } } return source; @@ -286,6 +311,10 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra result.bdaAbiVersion = program.Info().usesDma ? request.target.bdaAbiVersion : 0u; result.vertexOffsetSgpr = program.Info().vertexOffsetSgpr; result.instanceOffsetSgpr = program.Info().instanceOffsetSgpr; + result.vertexOffsetShared = program.Info().vertexOffsetShared; + result.instanceOffsetShared = program.Info().instanceOffsetShared; + result.vertexOffsetConflict = program.Info().vertexOffsetConflict; + result.instanceOffsetConflict = program.Info().instanceOffsetConflict; for (const auto& output : program.Info().outputs) { if (output.kind == StageOutputKind::Parameter) result.parameterExports.push_back(output.location); } @@ -370,19 +399,166 @@ RecompileResult RecompileImpl(const RecompileRequest& request) { return materializeVariant(*source, request, snapshot, specialization); } +// APS5_NO_RESULT_MEMO=1: every Recompile(request, capture) materializes its own result as before. +bool ResultMemo() { + static const bool resultMemo = std::getenv("APS5_NO_RESULT_MEMO") == nullptr; + return resultMemo; +} + +constexpr std::size_t ResultMemoEntries = 256; + +struct ResultMemoCounters { + std::atomic hits{0}, misses{0}, evictions{0}, populateNanoseconds{0}; + std::atomic lastReport{0}; +}; + +ResultMemoCounters& resultMemoCounters() { + static ResultMemoCounters counters; + return counters; +} + +void reportResultMemo() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& counters = resultMemoCounters(); + const auto now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(std::memory_order_relaxed); + if (last == 0) { + counters.lastReport.compare_exchange_strong(last, now, std::memory_order_relaxed); + return; + } + if (now - last < 10'000'000'000ll || !counters.lastReport.compare_exchange_strong(last, now, std::memory_order_relaxed)) return; + const auto hits = counters.hits.exchange(0, std::memory_order_relaxed); + const auto misses = counters.misses.exchange(0, std::memory_order_relaxed); + const auto evictions = counters.evictions.exchange(0, std::memory_order_relaxed); + const auto populate = counters.populateNanoseconds.exchange(0, std::memory_order_relaxed); + std::fprintf(stderr, "[recompile] result memo (10 s): %llu hits, %llu misses (%.1f%% hits), Populate %.1f us per miss / %.1f ms in total, %llu evictions\n", static_cast(hits), static_cast(misses), hits + misses != 0 ? 100.0 * static_cast(hits) / static_cast(hits + misses) : 0.0, misses != 0 ? static_cast(populate) / 1000.0 / static_cast(misses) : 0.0, static_cast(populate) / 1e6, static_cast(evictions)); +} + +// Everything materializeResult reads besides the variant: the snapshot (the descriptor words, the +// flattened SRT, the user data, the uniform fill) and, for the vertex family, the V# table the +// attributes are resolved from. +std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapshot& snapshot) { + std::uint64_t hash = 0xcbf29ce484222325ull; + const auto mix = [&](std::uint64_t value) { + hash ^= value; + hash *= 0x100000001b3ull; + }; + const auto mixWords = [&](std::span words) { + mix(words.size()); + for (const auto word : words) mix(word); + }; + const auto mixDescriptors = [&](const std::vector& values) { + mix(values.size()); + for (const auto& value : values) { + mix(value.dwordCount); + for (std::uint32_t i = 0; i < value.dwordCount && i < value.dwords.size(); ++i) mix(value.dwords[i]); + } + }; + mixDescriptors(snapshot.buffers); + mixDescriptors(snapshot.images); + mixDescriptors(snapshot.samplers); + mixWords(snapshot.flattenedSrt); + mixWords(snapshot.userData); + mix(static_cast(snapshot.uniformFill.kind)); + mix(snapshot.uniformFill.resource); + for (const auto stride : snapshot.uniformFill.groupStride) mix(stride); + mix(snapshot.uniformFill.words); + mix(snapshot.uniformFill.value); + if (request.context.vertex) { + const auto& vertex = *request.context.vertex; + const auto count = std::min(vertex.resourcesNum, ShaderVertexStageInfo::MaxResources); + mix(count); + for (std::uint32_t i = 0; i < count; ++i) { + for (const auto field : vertex.resources[i].fields) mix(field); + } + } else { + mix(1ull << 32u); + } + return hash; +} + +// The memo'd result of `source`'s variant for the snapshot (design13 R5): a hit returns the shared +// object, a miss materializes outside the source mutex and inserts (a concurrent miss's object is +// as good). `memoHit` reports the hit. +std::shared_ptr materializeMemoized(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, const ResourceSpecialization& specialization, bool* memoHit) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + std::shared_ptr variant; + bool cacheHit = false; + const auto hash = snapshotHash(request, snapshot); + std::uint64_t index = 0; + auto& counters = resultMemoCounters(); + { + std::lock_guard lock(source.mutex); + for (const auto& candidate : source.variants) { + if (sameLayout(candidate->layout, request.layout) && candidate->specialization == specialization) { + variant = candidate; + cacheHit = true; + break; + } + } + if (variant == nullptr) { + auto program = PrepareResourceProgram(request); + variant = std::make_shared(compileVariant(request, std::move(program), snapshot, specialization)); + source.variants.push_back(variant); + } + index = (variant->result.variantId * 0x9e3779b97f4a7c15ull) ^ hash; + const auto found = source.memoIndex.find(index); + if (found != source.memoIndex.end() && found->second->variantId == variant->result.variantId && found->second->hash == hash) { + source.memo.splice(source.memo.begin(), source.memo, found->second); + counters.hits.fetch_add(1, std::memory_order_relaxed); + if (memoHit != nullptr) *memoHit = true; + reportResultMemo(); + return found->second->result; + } + } + const auto started = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + auto result = std::make_shared(materializeResult(*variant, request, snapshot)); + result->cacheHit = cacheHit; + if (profile) counters.populateNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); + counters.misses.fetch_add(1, std::memory_order_relaxed); + std::shared_ptr shared = std::move(result); + { + std::lock_guard lock(source.mutex); + const auto found = source.memoIndex.find(index); + if (found != source.memoIndex.end()) { + if (found->second->variantId == variant->result.variantId && found->second->hash == hash) { + source.memo.splice(source.memo.begin(), source.memo, found->second); + shared = found->second->result; + } else { + source.memo.erase(found->second); + source.memoIndex.erase(found); + } + } + if (source.memoIndex.find(index) == source.memoIndex.end()) { + source.memo.push_front({variant->result.variantId, hash, shared}); + source.memoIndex.emplace(index, source.memo.begin()); + while (source.memo.size() > ResultMemoEntries) { + const auto& last = source.memo.back(); + source.memoIndex.erase((last.variantId * 0x9e3779b97f4a7c15ull) ^ last.hash); + source.memo.pop_back(); + counters.evictions.fetch_add(1, std::memory_order_relaxed); + } + } + } + reportResultMemo(); + return shared; +} + // The capture already resolved the source entry (stage input validation included) and materialized // the request over exactly the words the driver captured, so neither is repeated here. -RecompileResult RecompileImpl(const RecompileRequest& request, const ResourceCapture& capture) { +std::shared_ptr RecompileImpl(const RecompileRequest& request, const ResourceCapture& capture, bool* memoHit) { if (!request.useCache || capture.source == nullptr) { auto program = PrepareResourceProgram(request); const auto variant = compileVariant(request, std::move(program), capture.snapshot, capture.specialization); - return materializeResult(variant, request, capture.snapshot); + return std::make_shared(materializeResult(variant, request, capture.snapshot)); } - return materializeVariant(*capture.source, request, capture.snapshot, capture.specialization); + if (!ResultMemo()) return std::make_shared(materializeVariant(*capture.source, request, capture.snapshot, capture.specialization)); + return materializeMemoized(*capture.source, request, capture.snapshot, capture.specialization, memoHit); } template -RecompileResult recompileReporting(const RecompileRequest& request, Impl&& impl) { +auto recompileReporting(const RecompileRequest& request, Impl&& impl) -> decltype(impl()) { try { return impl(); } catch (const std::exception& e) { @@ -402,7 +578,29 @@ std::shared_ptr GetResourcePlan(const RecompileRequest& re return makeResourcePlan(request); } +namespace { + +// The capture's materialization; with pure flat slots in the plan the walk's read addresses are +// traced into the capture (ResourceCapture::readTrace). +void materializeCapture(ResourceCapture& capture, const SrtRuntime& runtime) { + const auto& plan = *capture.plan; + if (std::none_of(plan.pureFlatSlots.begin(), plan.pureFlatSlots.end(), [](std::uint8_t pure) { return pure != 0u; })) { + ResourceMaterializer{}.Materialize(plan, runtime, capture.snapshot, capture.specialization); + return; + } + SrtRuntime traced = runtime; + traced.readTrace = &capture.readTrace; + ResourceMaterializer{}.Materialize(plan, traced, capture.snapshot, capture.specialization); + auto& other = capture.readTrace.otherReads; + std::sort(other.begin(), other.end()); + other.erase(std::unique(other.begin(), other.end()), other.end()); +} + +} + std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto started = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; // Validates the stage inputs once per request, as GetResourcePlan and Recompile(request) do. static_cast(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request))); auto capture = std::make_shared(); @@ -412,7 +610,27 @@ std::shared_ptr CaptureResources(const RecompileRequest& } else { capture->plan = makeResourcePlan(request); } - ResourceMaterializer{}.Materialize(*capture->plan, runtime, capture->snapshot, capture->specialization); + if (profile) capture->sourceNanoseconds = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); + materializeCapture(*capture, runtime); + return capture; +} + +std::shared_ptr ResolveSource(const RecompileRequest& request) { + if (!request.useCache) return nullptr; + static_cast(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request))); + return std::make_shared(SourceHandle{getSource(request)}); +} + +std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime, const SourceHandle& handle) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto started = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + // The whole vertex family (Vertex, Local, TC, TE, Mesh) validates V# fields the memo key does not cover. + if (request.shader.stage != ShaderStage::Compute && request.shader.stage != ShaderStage::Fragment) static_cast(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request))); + auto capture = std::make_shared(); + capture->source = handle.source; + capture->plan = handle.source->plan; + if (profile) capture->sourceNanoseconds = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); + materializeCapture(*capture, runtime); return capture; } @@ -420,8 +638,9 @@ RecompileResult Recompile(const RecompileRequest& request) { return recompileReporting(request, [&] { return RecompileImpl(request); }); } -RecompileResult Recompile(const RecompileRequest& request, const ResourceCapture& capture) { - return recompileReporting(request, [&] { return RecompileImpl(request, capture); }); +std::shared_ptr Recompile(const RecompileRequest& request, const ResourceCapture& capture, bool* memoHit) { + if (memoHit != nullptr) *memoHit = false; + return recompileReporting(request, [&] { return RecompileImpl(request, capture, memoHit); }); } } diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 251fe2fd4a..2cf0221f7a 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -323,6 +323,12 @@ struct RecompileResult { std::vector vertexAttributes; std::int32_t vertexOffsetSgpr = -1; std::int32_t instanceOffsetSgpr = -1; + // The offset SGPR is also read elsewhere in the program (so a value folded into the draw's + // first vertex / instance cannot stand in for it), or two SGPRs were added (the SGPR is -1). + bool vertexOffsetShared = false; + bool instanceOffsetShared = false; + bool vertexOffsetConflict = false; + bool instanceOffsetConflict = false; std::vector parameterExports; std::vector fragmentParameters; bool cacheHit = false; @@ -335,9 +341,12 @@ struct RecompileResult { // The resource plan, snapshot and specialization a driver captured for the request (see // CaptureResources in Optimization/ResourceProgram.hpp): this overload reuses them instead of -// materializing the request's memory regions again, and is otherwise Recompile(request). +// materializing the request's memory regions again, and is otherwise Recompile(request). The +// result is immutable and shared: a capture that reproduces a snapshot the source's variant was +// materialized over before receives the same object (`memoHit`), so the descriptor population runs +// once per distinct snapshot. APS5_NO_RESULT_MEMO=1 materializes every call. struct ResourceCapture; -[[nodiscard]] RecompileResult Recompile(const RecompileRequest& request, const ResourceCapture& capture); +[[nodiscard]] std::shared_ptr Recompile(const RecompileRequest& request, const ResourceCapture& capture, bool* memoHit = nullptr); // Debug aid (see DebugProbe in Translation/TranslationContext.hpp): the APS5_PROBE register probe is // only applied while a driver has it active, so it can be limited to one dispatch; the recompile diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index 00bca4f89a..be56f6c726 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -4,6 +4,7 @@ #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvEmitter.hpp" #include "BdaAbi.hpp" +#include namespace ShaderRecompiler { @@ -19,7 +20,14 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe bool BdaInvocationsMayStop(const IrProgram& program); void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition); std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits); +// Reads the dwords of a 1-4 dword load at address + offset that the program extracts, with one +// table lookup for the whole span; the per-byte lookups of EmitBdaRead remain the fallback (and +// the only path under APS5_BDA_BYTE_READS=1). Dwords the program never extracts are neither read +// nor fault-checked, as when every dword was its own instruction. +std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, std::uint32_t dwords); +bool BdaByteReadsForced(); std::uint32_t AddBdaAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, bool subtract); +std::uint32_t AddBdaImmediate(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::int32_t immediate); } diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index f52867e06c..36c76c3c33 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -49,9 +49,9 @@ std::uint32_t ImageScalarType(SpirvEmitterState& state, IrTextureNumericClass nu std::uint32_t ImageVectorType(SpirvEmitterState& state, IrTextureNumericClass numericClass, std::uint32_t components); std::uint32_t ImageType(SpirvEmitterState& state, const ImageResource& image); std::uint32_t ImageViewSizeType(SpirvEmitterState& state, RdnaImageDimension dimension); -std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource); +std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t slotId = 0); std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t sampler); -std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler); +std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler, std::uint32_t slotId = 0); std::uint32_t StorageImageDescriptorPointer(SpirvEmitterState& state, std::uint32_t resource); void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t mipLod, std::uint32_t coord, std::uint32_t texel); std::uint32_t ExecutionModelForStage(IrShaderStage stage); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index a4d33b643e..0d73dbf0ca 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -205,6 +205,9 @@ void EmitMakeImageAddress(SpirvValueEmitContext& context); void EmitLoadAddressU8(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitLoadAddressU16(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitLoadAddressU32(SpirvValueEmitContext& ctx, const IrValue& inst); +void EmitLoadAddressU32x2(SpirvValueEmitContext& ctx, const IrValue& inst); +void EmitLoadAddressU32x3(SpirvValueEmitContext& ctx, const IrValue& inst); +void EmitLoadAddressU32x4(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitStoreAddressU8(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitStoreAddressU16(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitStoreAddressU32(SpirvValueEmitContext& ctx, const IrValue& inst); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 0693b53e89..373d8ff2a0 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -78,12 +80,23 @@ struct SpirvEmitterState { SpirvRequirements requirements; std::uint32_t laneCount = 1; std::uint32_t laneHalf = 0; + // The target's SPIR-V version and what the device accepts, for capabilities an emitter adds + // only when needed (bindless image tables: see TableImageIndex). + std::uint32_t spirvVersion = 0x00010300u; + std::span supportedCapabilities; + std::span supportedExtensions; + // A bindless table's runtime slot is wave-uniform; it is uniform over the invocation group + // only for a single-wave compute workgroup, elsewhere it needs the NonUniform decoration. + bool tableIndexNonUniform = true; std::uint32_t storageBufferVariable = 0; std::uint32_t storageBufferU64Variable = 0; std::array memoryByteOffsets {}; std::uint32_t bdaPagetableVariable = 0; std::uint32_t faultBufferVariable = 0; std::uint32_t bdaPointerFunction = 0; + // The lookup without fault recording that wide reads try first (see EmitBdaDwordReads); 0 when + // every read takes the byte path. + std::uint32_t bdaProbeFunction = 0; // False for programs with workgroup barriers: faulting BDA accesses then continue (see BdaInvocationsMayStop). bool bdaStopsInvocations = true; // Execution scope of the barriers that keep one guest wave's LDS accesses in program order across diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp index 444a8d8cee..404ac85d12 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp @@ -13,9 +13,10 @@ std::uint32_t ImageScalarType(SpirvEmitterState& state, IrTextureNumericClass nu std::uint32_t ImageVectorType(SpirvEmitterState& state, IrTextureNumericClass numericClass, std::uint32_t components); std::uint32_t ImageType(SpirvEmitterState& state, const ImageResource& image); std::uint32_t ImageViewSizeType(SpirvEmitterState& state, RdnaImageDimension dimension); -std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource); +// `slotId` (a SPIR-V id, 0 for none) selects a bindless table's slot at runtime: element(resource) + slot. +std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t slotId); std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t sampler); -std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler); +std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler, std::uint32_t slotId); std::uint32_t StorageImageDescriptorPointer(SpirvEmitterState& state, std::uint32_t resource); void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t mipLod, std::uint32_t coord, std::uint32_t texel); const RdnaImageDimensionInfo& RdnaImageDimensionInfoFor(RdnaImageDimension dimension); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index 91ec9e6aeb..b67516b279 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -4,8 +4,22 @@ namespace ShaderRecompiler { -void DefineGetBdaPointer(SpirvEmitterState& state) { - if (!state.program.Info().usesDma) return; +namespace { + +void ReturnBdaZeroIf(SpirvEmitterState& state, std::uint32_t condition) { + const auto failed = state.module.AllocateId(); + const auto next = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, next, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, condition, failed, next); + EmitLabel(state, failed); + state.module.AddFunction(spv::OpReturnValue, BdaConstant(state, 0u)); + EmitLabel(state, next); +} + +// (u64 address, u32 bytes, u32 instruction) -> u64 device address of the range holding all the bytes, +// or 0. With recordFaults the failure is published first; without, the caller retries byte-wise, and +// the fault is recorded there if the bytes are unmapped for real. +std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool recordFaults) { const auto u32 = TypeU32(state); const auto u64 = TypeScalarU64(state); const auto boolean = TypeBool(state); @@ -16,9 +30,9 @@ void DefineGetBdaPointer(SpirvEmitterState& state) { state.module.AddFunction(spv::OpLoad, u32, value, pointer); return value; }; - state.bdaPointerFunction = state.module.AllocateId(); - state.module.AddName(state.bdaPointerFunction, "get_bda_pointer"); - state.module.AddFunction(spv::OpFunction, u64, state.bdaPointerFunction, spv::FunctionControlMaskNone, state.module.Type(spv::OpTypeFunction, u64, u64, u32, u32)); + const auto function = state.module.AllocateId(); + state.module.AddName(function, name); + state.module.AddFunction(spv::OpFunction, u64, function, spv::FunctionControlMaskNone, state.module.Type(spv::OpTypeFunction, u64, u64, u32, u32)); const auto address = state.module.AllocateId(); const auto bytes = state.module.AllocateId(); const auto instruction = state.module.AllocateId(); @@ -31,7 +45,13 @@ void DefineGetBdaPointer(SpirvEmitterState& state) { const auto pointer = TypePointer(state, spv::StorageClassFunction, u32); state.module.AddFunction(spv::OpVariable, pointer, low, spv::StorageClassFunction); state.module.AddFunction(spv::OpVariable, pointer, high, spv::StorageClassFunction); - const auto fail = [&](std::uint32_t condition, BdaAbi::FaultReason reason) { ReturnBdaFailureIf(state, condition, address, bytes, instruction, reason); }; + const auto fail = [&](std::uint32_t condition, BdaAbi::FaultReason reason) { + if (recordFaults) { + ReturnBdaFailureIf(state, condition, address, bytes, instruction, reason); + } else { + ReturnBdaZeroIf(state, condition); + } + }; const auto length = state.module.AllocateId(); state.module.AddFunction(spv::OpArrayLength, u32, length, state.bdaPagetableVariable, 0u); fail(binary(spv::OpULessThan, boolean, length, constant(4)), BdaAbi::FaultReason::InvalidTable); @@ -92,6 +112,15 @@ void DefineGetBdaPointer(SpirvEmitterState& state) { fail(binary(spv::OpULessThanEqual, boolean, deviceEnd, result), BdaAbi::FaultReason::Overflow); state.module.AddFunction(spv::OpReturnValue, result); state.module.AddFunction(spv::OpFunctionEnd); + return function; +} + +} + +void DefineGetBdaPointer(SpirvEmitterState& state) { + if (!state.program.Info().usesDma) return; + state.bdaPointerFunction = DefineBdaLookup(state, "get_bda_pointer", true); + if (!BdaByteReadsForced()) state.bdaProbeFunction = DefineBdaLookup(state, "probe_bda_pointer", false); } } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index 8314af119c..5f5c947eed 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -2,11 +2,89 @@ #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" #include +#include #include #include namespace ShaderRecompiler { +namespace { + +void EmitBdaOverflowCheck(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction) { + const auto overflow = Binary(state, spv::OpUGreaterThan, TypeBool(state), address, BdaConstant(state, std::numeric_limits::max() - bytes)); + EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), instruction, BdaAbi::FaultReason::Overflow); }); + StopBdaInvocationIf(state, overflow); +} + +std::uint32_t EmitBdaByte(SpirvEmitterState& state, std::uint32_t guest, std::uint32_t instruction) { + const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u); + const auto bytePointer = TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType); + const auto physical = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaPointerFunction, guest, ConstantU32(state, 1u), instruction); + if (state.bdaStopsInvocations) { + StopBdaInvocationIf(state, Binary(state, spv::OpIEqual, TypeBool(state), physical, BdaConstant(state, 0u))); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, bytePointer, pointer, physical); + const auto loaded = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, byteType, loaded, pointer, spv::MemoryAccessAlignedMask, 1u); + return Unary(state, spv::OpUConvert, TypeU32(state), loaded); + } + // Unmapped bytes read as zero; the fault is recorded and every invocation reaches the + // program's barriers. + const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); + const auto before = state.currentLabel; + const auto loadLabel = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, mapped, loadLabel, merge); + EmitLabel(state, loadLabel); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, bytePointer, pointer, physical); + const auto loaded = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, byteType, loaded, pointer, spv::MemoryAccessAlignedMask, 1u); + const auto widened = Unary(state, spv::OpUConvert, TypeU32(state), loaded); + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, merge); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpPhi, TypeU32(state), value, widened, loadLabel, ConstantU32(state, 0u), before); + return value; +} + +std::uint32_t EmitBdaBytes(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction) { + auto result = ConstantU32(state, 0u); + for (std::uint32_t byte = 0; byte < bytes; ++byte) { + const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte)); + const auto value = EmitBdaByte(state, guest, instruction); + result = Binary(state, spv::OpBitwiseOr, TypeU32(state), result, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), value, ConstantU32(state, byte * 8u))); + } + return result; +} + +// The dwords of a wide load that the program extracts (all of them when the composite is used +// whole). Dead-code elimination has dropped the unused extracts, as it dropped unused dword loads +// when every dword was its own instruction. +std::uint32_t UsedBdaDwords(const IrValue& inst, std::uint32_t dwords) { + const auto all = (1u << dwords) - 1u; + if (dwords == 1u) return all; + std::uint32_t used = 0; + for (const IrValue* user : inst.Uses()) { + const auto opcode = user->Opcode(); + if (opcode != IrOpcode::CompositeExtractU32x2 && opcode != IrOpcode::CompositeExtractU32x3 && opcode != IrOpcode::CompositeExtractU32x4) return all; + const IrValue* index = user->ArgumentCount() > 1u ? user->Argument(1) : nullptr; + if (index == nullptr || !index->HasImmediate() || index->ImmediateU32() >= dwords) return all; + used |= 1u << index->ImmediateU32(); + } + return used; +} + +} + +// APS5_BDA_BYTE_READS=1 restores one lookup and one byte load per byte for every read. +bool BdaByteReadsForced() { + static const bool forced = std::getenv("APS5_BDA_BYTE_READS") != nullptr; + return forced; +} + std::uint32_t AddBdaAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, bool subtract) { auto& state = ctx.state; const auto result = Binary(state, subtract ? spv::OpISub : spv::OpIAdd, TypeScalarU64(state), address, offset); @@ -16,6 +94,14 @@ std::uint32_t AddBdaAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std return result; } +std::uint32_t AddBdaImmediate(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::int32_t immediate) { + if (immediate == 0) { + return address; + } + const auto magnitude = immediate < 0 ? -static_cast(immediate) : static_cast(immediate); + return AddBdaAddress(ctx, inst, address, BdaConstant(ctx.state, static_cast(magnitude)), immediate < 0); +} + void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target) { if (!program.Info().usesDma) return; if (target.bdaAbiVersion != BdaAbi::Version) throw std::runtime_error("unsupported BDA ABI version"); @@ -41,48 +127,89 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: auto& state = ctx.state; if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA read width"); if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); + if (bits == 32u) return EmitBdaDwordReads(ctx, inst, address, 0u, 1u)[0]; const auto instruction = ConstantU32(state, inst.Flags().pc); - const auto overflow = Binary(state, spv::OpUGreaterThan, TypeBool(state), address, BdaConstant(state, std::numeric_limits::max() - bits / 8u)); - EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, bits / 8u), instruction, BdaAbi::FaultReason::Overflow); }); - StopBdaInvocationIf(state, overflow); - const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u); - const auto bytePointer = TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType); - auto result = ConstantU32(state, 0u); - for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { - const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte)); - const auto physical = state.module.AllocateId(); - state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaPointerFunction, guest, ConstantU32(state, 1u), instruction); - std::uint32_t value = 0; - if (state.bdaStopsInvocations) { - StopBdaInvocationIf(state, Binary(state, spv::OpIEqual, TypeBool(state), physical, BdaConstant(state, 0u))); - const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpConvertUToPtr, bytePointer, pointer, physical); - const auto loaded = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, byteType, loaded, pointer, spv::MemoryAccessAlignedMask, 1u); - value = Unary(state, spv::OpUConvert, TypeU32(state), loaded); - } else { - // Unmapped bytes read as zero; the fault is recorded and every invocation reaches the - // program's barriers. - const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); - const auto before = state.currentLabel; - const auto loadLabel = state.module.AllocateId(); - const auto merge = state.module.AllocateId(); - state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); - state.module.AddFunction(spv::OpBranchConditional, mapped, loadLabel, merge); - EmitLabel(state, loadLabel); - const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpConvertUToPtr, bytePointer, pointer, physical); - const auto loaded = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, byteType, loaded, pointer, spv::MemoryAccessAlignedMask, 1u); - const auto widened = Unary(state, spv::OpUConvert, TypeU32(state), loaded); - state.module.AddFunction(spv::OpBranch, merge); - EmitLabel(state, merge); - value = state.module.AllocateId(); - state.module.AddFunction(spv::OpPhi, TypeU32(state), value, widened, loadLabel, ConstantU32(state, 0u), before); + EmitBdaOverflowCheck(state, address, bits / 8u, instruction); + return EmitBdaBytes(state, address, bits / 8u, instruction); +} + +// One probe of the span from the first to the last extracted dword replaces 4 lookups per dword (a +// Bink DC pass spent ~450 dependent table levels per iteration on them). The span is loaded +// dword-wise from the returned device address when it lies in one range and that address is +// 4-aligned; a span crossing ranges (the table is byte-granular, see tests/BdaExecution.cpp), an +// unaligned mirror or a wrapping span falls back to the byte path. That path repeats, per extracted +// dword in order, the address, overflow check and byte lookups of a separate dword load, so every +// fault is recorded exactly as one; the probe records none, and a span it accepts cannot fault. +std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, std::uint32_t dwords) { + auto& state = ctx.state; + if (dwords == 0u || dwords > 4u) ctx.Fail(inst, "unsupported BDA read width"); + if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); + const auto u64 = TypeScalarU64(state); + const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto used = UsedBdaDwords(inst, dwords); + const auto isUsed = [&](std::uint32_t dword) { return (used & (1u << dword)) != 0u; }; + std::array values{}; + std::uint32_t first = dwords; + std::uint32_t last = 0; + for (std::uint32_t dword = 0; dword < dwords; ++dword) { + if (!isUsed(dword)) { + values[dword] = ConstantU32(state, 0u); + continue; } - result = Binary(state, spv::OpBitwiseOr, TypeU32(state), result, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), value, ConstantU32(state, byte * 8u))); + first = std::min(first, dword); + last = dword; } - return result; + if (used == 0u) return values; + std::array addresses{}; + const auto dwordAddress = [&](std::uint32_t dword) { return AddBdaImmediate(ctx, inst, address, static_cast(offset + dword * 4u)); }; + const auto readBytes = [&](std::array& into) { + for (std::uint32_t dword = first; dword <= last; ++dword) { + if (!isUsed(dword)) continue; + if (dword != first) addresses[dword] = dwordAddress(dword); + EmitBdaOverflowCheck(state, addresses[dword], 4u, instruction); + into[dword] = EmitBdaBytes(state, addresses[dword], 4u, instruction); + } + }; + addresses[first] = dwordAddress(first); + if (state.bdaProbeFunction == 0) { + readBytes(values); + return values; + } + const auto physical = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, u64, physical, state.bdaProbeFunction, addresses[first], ConstantU32(state, (last - first + 1u) * 4u), instruction); + const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); + const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, u64, physical, BdaConstant(state, 3u)), BdaConstant(state, 0u)); + const auto wide = Binary(state, spv::OpLogicalAnd, TypeBool(state), mapped, aligned); + const auto wideLabel = state.module.AllocateId(); + const auto byteLabel = state.module.AllocateId(); + const auto byteExit = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, wide, wideLabel, byteLabel); + EmitLabel(state, wideLabel); + std::array wideValues{}; + for (std::uint32_t dword = first; dword <= last; ++dword) { + if (!isUsed(dword)) continue; + const auto element = dword == first ? physical : Binary(state, spv::OpIAdd, u64, physical, BdaConstant(state, (dword - first) * 4u)); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, element); + wideValues[dword] = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), wideValues[dword], pointer, spv::MemoryAccessAlignedMask, 4u); + } + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, byteLabel); + std::array byteValues{}; + readBytes(byteValues); + state.module.AddFunction(spv::OpBranch, byteExit); + EmitLabel(state, byteExit); + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, merge); + for (std::uint32_t dword = first; dword <= last; ++dword) { + if (!isUsed(dword)) continue; + values[dword] = state.module.AllocateId(); + state.module.AddFunction(spv::OpPhi, TypeU32(state), values[dword], wideValues[dword], wideLabel, byteValues[dword], byteExit); + } + return values; } } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index aaf3edb5bd..e9d65296eb 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -243,8 +243,19 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh ValidateBdaTarget(program, target); SpirvEmitterState state(program, inputInfo); state.module.RequireVersion(target.spirvVersion); + state.spirvVersion = target.spirvVersion; + state.supportedCapabilities = target.supportedCapabilities; + state.supportedExtensions = target.supportedExtensions; const auto* workgroup = ShaderWorkgroupInputFor(state); state.laneCount = workgroup != nullptr && program.WaveSize() == 64u && workgroup->hostSubgroupSize == 32u ? 2u : 1u; + if (program.Resources().stage == IrShaderStage::Compute && workgroup != nullptr) { + // The key comes from a subgroup ballot (ReadFirstLane), so the slot is uniform over the + // workgroup only when the workgroup is one wave held by one host subgroup; a wave64 program + // kept at one lane per invocation spans two subgroups (see WaveLdsScope). + const auto threads = std::max(workgroup->threadsNum[0], 1u) * std::max(workgroup->threadsNum[1], 1u) * std::max(workgroup->threadsNum[2], 1u); + const bool oneSubgroup = state.laneCount == 2u || (program.WaveSize() == workgroup->hostSubgroupSize && workgroup->hostSubgroupSize <= 32u); + state.tableIndexNonUniform = threads > program.WaveSize() || !oneSubgroup; + } state.waveLdsScope = WaveLdsScope(program, workgroup, state.laneCount); if (const char* guard = std::getenv("APS5_LOOP_GUARD")) state.loopGuardLimit = static_cast(std::strtoul(guard, nullptr, 0)); // Stopped invocations would leave the wave LDS barriers incomplete. diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 0539474c38..215f616e5c 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -424,6 +424,9 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::LoadAddressU8: return Invoke(EmitLoadAddressU8, ctx, inst); case IrOpcode::LoadAddressU16: return Invoke(EmitLoadAddressU16, ctx, inst); case IrOpcode::LoadAddressU32: return Invoke(EmitLoadAddressU32, ctx, inst); + case IrOpcode::LoadAddressU32x2: return Invoke(EmitLoadAddressU32x2, ctx, inst); + case IrOpcode::LoadAddressU32x3: return Invoke(EmitLoadAddressU32x3, ctx, inst); + case IrOpcode::LoadAddressU32x4: return Invoke(EmitLoadAddressU32x4, ctx, inst); case IrOpcode::StoreAddressU8: return Invoke(EmitStoreAddressU8, ctx, inst); case IrOpcode::StoreAddressU16: return Invoke(EmitStoreAddressU16, ctx, inst); case IrOpcode::StoreAddressU32: return Invoke(EmitStoreAddressU32, ctx, inst); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 3107be84d9..356ee327ea 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -15,11 +15,20 @@ namespace ShaderRecompiler { namespace { +// A bindless table root's runtime slot and whether the key is mapped (SPIR-V ids), both 0 for a +// direct image. +struct TableSelection { + std::uint32_t slot = 0; + std::uint32_t mapped = 0; +}; + struct ImageEmitAccess { const IrValue& inst; const MemoryInfo& mem; const ImageResource& image; const IrValue& address; + TableSelection table; + std::uint32_t slot = table.slot; }; struct SampleSetup { @@ -226,6 +235,13 @@ std::uint32_t SampledComponentZero(SpirvEmitterState& state, IrTextureNumericCla throw std::runtime_error("invalid sampled image numeric class"); } +// An image op's u32x4 result through a bindless table: zeros when the key is not mapped. +std::uint32_t TableResult(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, std::uint32_t result) { + if (access.table.mapped == 0) return result; + auto& state = ctx.state; + return Select(state, TypeU32Vector(state, 4), access.table.mapped, result, ConstantU32CompositeZero(state, 4)); +} + std::uint32_t ResultVector(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, std::uint32_t value, IrTextureNumericClass numericClass, bool dref, bool gather) { auto& state = ctx.state; const auto& mem = access.mem; @@ -289,7 +305,7 @@ std::uint32_t QueryDimensions(SpirvValueEmitContext& ctx, const ImageEmitAccess& auto& state = ctx.state; const auto dimension = access.image.dimension; const auto& info = RdnaImageDimensionInfoFor(dimension); - const auto image = LoadSampledImageDescriptor(state, access.mem.resource); + const auto image = LoadSampledImageDescriptor(state, access.mem.resource, access.slot); const auto size = state.module.AllocateId(); if (info.multisampled != 0u) { state.module.AddFunction(spv::OpImageQuerySize, ImageViewSizeType(state, dimension), size, image); @@ -443,14 +459,14 @@ std::uint32_t EmitOneDimensionalGatherLz(SpirvValueEmitContext& ctx, const Image auto& state = ctx.state; const auto numericClass = access.image.numericClass; state.module.EmitCapability(spv::CapabilityImageQuery); - const auto image = LoadSampledImageDescriptor(state, access.mem.resource); + const auto image = LoadSampledImageDescriptor(state, access.mem.resource, access.slot); const auto width = state.module.AllocateId(); state.module.AddFunction(spv::OpImageQuerySizeLod, TypeU32(state), width, image, ConstantU32(state, 0)); const auto widthF32 = state.module.AllocateId(); state.module.AddFunction(spv::OpConvertUToF, TypeF32(state), widthF32, width); const auto left = state.module.AllocateId(); state.module.AddFunction(spv::OpExtInst, TypeF32(state), left, GlslStd450(state), GLSLstd450Floor, Binary(state, spv::OpFSub, TypeF32(state), Binary(state, spv::OpFMul, TypeF32(state), coord, widthF32), ConstantF32(state, 0x3f000000u))); - const auto sampled = MakeSampledImage(state, access.mem.resource, access.mem.sampler); + const auto sampled = MakeSampledImage(state, access.mem.resource, access.mem.sampler, access.slot); const auto vectorType = ImageVectorType(state, numericClass, 4); const auto scalarType = ImageScalarType(state, numericClass); const auto component = ImageConversionFormat(access.image).format == IrBufferFormat::Invalid ? ImageGatherComponent(EffectiveDmask(access.mem)) : 0u; @@ -543,13 +559,13 @@ const ImageResource& ImageResourceOf(const SpirvEmitterState& state, const Memor void EmitQueryDimensionsOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { ctx.state.module.EmitCapability(spv::CapabilityImageQuery); - ctx.Define(access.inst, QueryDimensions(ctx, access)); + ctx.Define(access.inst, TableResult(ctx, access, QueryDimensions(ctx, access))); } void EmitQueryLodOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { auto& state = ctx.state; state.module.EmitCapability(spv::CapabilityImageQuery); - const auto sampled = MakeSampledImage(state, access.mem.resource, access.mem.sampler); + const auto sampled = MakeSampledImage(state, access.mem.resource, access.mem.sampler, access.slot); const auto coord = CoordF32(ctx, access, 0, RdnaImageDimensionInfoFor(access.image.dimension).spatialComponents); const auto lod = state.module.AllocateId(); state.module.AddFunction(spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled, coord); @@ -561,7 +577,7 @@ void EmitQueryLodOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { } const auto result = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0], values[1], values[2], values[3]); - ctx.Define(access.inst, result); + ctx.Define(access.inst, TableResult(ctx, access, result)); } void EmitReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { @@ -570,7 +586,7 @@ void EmitReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { const auto numericClass = access.image.numericClass; const auto condition = ctx.Arg(access.inst, 2); ctx.Define(access.inst, EmitValueOrDefaultIfCondition(state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4), [&]() { - const auto descriptor = LoadSampledImageDescriptor(state, access.mem.resource); + const auto descriptor = LoadSampledImageDescriptor(state, access.mem.resource, access.slot); const auto color = state.module.AllocateId(); const auto coord = CoordU32(ctx, access); if (dimensionInfo.multisampled != 0u) { @@ -581,12 +597,15 @@ void EmitReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { } else { state.module.AddFunction(spv::OpImageFetch, ImageVectorType(state, numericClass, 4), color, descriptor, coord, spv::ImageOperandsLodMask, LodU32(ctx, access)); } - return ResultVector(ctx, access, UnpackImageTexel(ctx, access, color), numericClass, false, false); + return TableResult(ctx, access, ResultVector(ctx, access, UnpackImageTexel(ctx, access, color), numericClass, false, false)); })); } void EmitWriteOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { auto& state = ctx.state; + if (access.slot != 0) { + ctx.Fail(access.inst, "stores through a bindless image table are unsupported"); + } const bool uintImage = access.image.numericClass == IrTextureNumericClass::Uint; EmitIfCondition(state, ctx.Arg(access.inst, 3), [&]() { const auto mipLod = access.image.mipMode == ImageMipMode::DynamicStorage ? LodU32(ctx, access) : 0u; @@ -598,6 +617,9 @@ void EmitWriteOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { void EmitAtomicOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { auto& state = ctx.state; + if (access.slot != 0) { + ctx.Fail(access.inst, "atomics through a bindless image table are unsupported"); + } const auto atomicOpcode = ImageAtomicOpcode(access.inst.Opcode()); ctx.Define(access.inst, EmitValueOrZeroIfCondition(state, ctx.Arg(access.inst, 3), [&]() { const auto pointer = state.module.AllocateId(); @@ -630,13 +652,13 @@ void EmitGatherOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con ctx.Fail(access.inst, "has an unsupported 1D gather variant"); } const auto sample = EmitOneDimensionalGatherLz(ctx, access, setup.coord); - ctx.Define(access.inst, ResultVector(ctx, access, UnpackImageGather(ctx, access, sample), setup.numericClass, false, true)); + ctx.Define(access.inst, TableResult(ctx, access, ResultVector(ctx, access, UnpackImageGather(ctx, access, sample), setup.numericClass, false, true))); return; } if (dimension == RdnaImageDimension::Dim1DArray) { ctx.Fail(access.inst, "has an unsupported 1D-array gather"); } - const auto sampled = MakeSampledImage(state, mem.resource, mem.sampler); + const auto sampled = MakeSampledImage(state, mem.resource, mem.sampler, access.slot); const auto sample = state.module.AllocateId(); std::vector words; if (setup.dref) { @@ -672,12 +694,15 @@ void EmitGatherOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con gathered = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeConstruct, ImageVectorType(state, resultNumericClass, 4), gathered, component, component, component, component); } - ctx.Define(access.inst, ResultVector(ctx, access, UnpackImageGather(ctx, access, gathered), resultNumericClass, false, true)); + ctx.Define(access.inst, TableResult(ctx, access, ResultVector(ctx, access, UnpackImageGather(ctx, access, gathered), resultNumericClass, false, true))); } -std::uint32_t EmitIndirectImageSelector(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, std::uint32_t key) { +// The bound slot of a bindless table for the runtime key: a binary search over the (key, slot) +// pairs the materializer appended to the flattened SRT. A key the mapping lacks (past the table, +// or a null or unusable entry) selects slot 0 and reports unmapped: its result is zeroed, as a +// null T# samples on hardware. +TableSelection EmitIndirectImageSelector(SpirvValueEmitContext& ctx, const ImageResource& image, std::uint32_t key) { auto& state = ctx.state; - const auto& image = access.image; const auto loadMapping = [&](std::uint32_t index) { const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0), index); @@ -689,6 +714,7 @@ std::uint32_t EmitIndirectImageSelector(SpirvValueEmitContext& ctx, const ImageE auto low = ConstantU32(state, 0u); auto high = loadMapping(mapping); auto selected = ConstantU32(state, 0u); + auto mapped = ConstantBool(state, false); for (std::uint32_t iteration = 0; iteration < image.indirectSearchIterations; iteration++) { const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high); const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state), Binary(state, spv::OpIAdd, TypeU32(state), low, high), ConstantU32(state, 1u)); @@ -699,13 +725,14 @@ std::uint32_t EmitIndirectImageSelector(SpirvValueEmitContext& ctx, const ImageE const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mappedKey, key); const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal); selected = Select(state, TypeU32(state), match, candidate, selected); + mapped = Binary(state, spv::OpLogicalOr, TypeBool(state), mapped, match); const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mappedKey, key); const auto takeUpper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less); const auto takeLower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, Unary(state, spv::OpLogicalNot, TypeBool(state), less)); low = Select(state, TypeU32(state), takeUpper, Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low); high = Select(state, TypeU32(state), takeLower, mid, high); } - return selected; + return {selected, mapped}; } void EmitSampleOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, const SampleSetup& setup) { @@ -738,65 +765,38 @@ void EmitSampleOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con operandMask |= spv::ImageOperandsBiasMask; operands.push_back(AddressF32(ctx, access, setup.layout.bias)); } - const auto emitSample = [&](std::uint32_t resource) { - const auto sampled = MakeSampledImage(state, resource, mem.sampler); - const auto sample = state.module.AllocateId(); - std::vector words = {opcode, resultType, sample, sampled, setup.coord}; - if (setup.dref) { - words.push_back(drefValue); - } - if (operandMask != 0u) { - words.push_back(operandMask); - words.insert(words.end(), operands.begin(), operands.end()); - } - state.module.AddFunction(words); - return sample; - }; - if (image.indirectRoot != mem.resource) { - const auto sample = emitSample(mem.resource); - const auto result = setup.dref ? sample : UnpackImageTexel(ctx, access, sample); - ctx.Define(access.inst, ResultVector(ctx, access, result, setup.numericClass, setup.dref, false)); - return; + const auto sampled = MakeSampledImage(state, mem.resource, mem.sampler, access.slot); + const auto sample = state.module.AllocateId(); + std::vector words = {opcode, resultType, sample, sampled, setup.coord}; + if (setup.dref) { + words.push_back(drefValue); } - const auto* handle = access.inst.Argument(0); + if (operandMask != 0u) { + words.push_back(operandMask); + words.insert(words.end(), operands.begin(), operands.end()); + } + state.module.AddFunction(words); + const auto result = setup.dref ? sample : UnpackImageTexel(ctx, access, sample); + ctx.Define(access.inst, TableResult(ctx, access, ResultVector(ctx, access, result, setup.numericClass, setup.dref, false))); +} + +// A bindless table root's selection for this access (see EmitIndirectImageSelector), ids 0 for +// a direct image. +TableSelection TableSlot(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const ImageResource& image) { + auto& state = ctx.state; + if (image.indirectRoot != mem.resource) { + return {}; + } + const auto* handle = inst.Argument(0); const auto& sources = state.program.Resources().descriptorSources; if (handle == nullptr || image.source >= sources.size() || !sources[image.source].indirectImage.has_value() || sources[image.source].indirectImage->keyArg >= handle->ArgumentCount()) { - ctx.Fail(access.inst, "has invalid indirect image key provenance"); + ctx.Fail(inst, "has invalid bindless image table key provenance"); } - if (state.flattenedSrtVariable == 0 || image.indirectSearchIterations == 0u || image.indirectResources.size() < 2u) { - ctx.Fail(access.inst, "has no indirect image runtime mapping"); + if (state.flattenedSrtVariable == 0 || image.indirectSearchIterations == 0u) { + ctx.Fail(inst, "has no bindless image table runtime mapping"); } const auto key = ctx.Def(handle->Argument(sources[image.source].indirectImage->keyArg)); - const auto selected = EmitIndirectImageSelector(ctx, access, key); - const auto defaultLabel = state.module.AllocateId(); - const auto mergeLabel = state.module.AllocateId(); - std::vector labels(image.indirectResources.size() - 1u); - std::vector switchWords = {spv::OpSwitch, selected, defaultLabel}; - for (std::uint32_t candidate = 1; candidate < image.indirectResources.size(); candidate++) { - labels[candidate - 1u] = state.module.AllocateId(); - switchWords.push_back(candidate); - switchWords.push_back(labels[candidate - 1u]); - } - state.module.AddFunction(spv::OpSelectionMerge, mergeLabel, spv::SelectionControlMaskNone); - state.module.AddFunction(switchWords); - std::vector phiWords = {spv::OpPhi, resultType, state.module.AllocateId()}; - EmitLabel(state, defaultLabel); - phiWords.push_back(emitSample(image.indirectResources[0])); - phiWords.push_back(defaultLabel); - state.module.AddFunction(spv::OpBranch, mergeLabel); - for (std::uint32_t candidate = 1; candidate < image.indirectResources.size(); candidate++) { - EmitLabel(state, labels[candidate - 1u]); - phiWords.push_back(emitSample(image.indirectResources[candidate])); - phiWords.push_back(labels[candidate - 1u]); - state.module.AddFunction(spv::OpBranch, mergeLabel); - } - EmitLabel(state, mergeLabel); - state.module.AddFunction(phiWords); - auto result = phiWords[2]; - if (!setup.dref) { - result = UnpackImageTexel(ctx, access, result); - } - ctx.Define(access.inst, ResultVector(ctx, access, result, setup.numericClass, setup.dref, false)); + return EmitIndirectImageSelector(ctx, image, key); } void EmitSamplingOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { @@ -828,7 +828,7 @@ void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { if (address == nullptr) { ctx.Fail(inst, "has no image address"); } - const ImageEmitAccess access{inst, mem, image, *address}; + const ImageEmitAccess access{inst, mem, image, *address, TableSlot(ctx, inst, mem, image)}; switch (irOpcode) { case IrOpcode::ImageQueryDimensions: EmitQueryDimensionsOp(ctx, access); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp index b99a16e771..b0fee41fa6 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp @@ -2,6 +2,7 @@ #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include "Optimization/ResourceMaterializer.hpp" #include #include #include @@ -59,6 +60,41 @@ std::uint32_t DescriptorElementPointer(SpirvEmitterState& state, std::uint32_t r return pointer; } +// The element of a bindless table's runtime slot, `arrayIndex` (the root's element) + `slotId`. +// The slot is wave-uniform; where that is not uniform over the invocation group (see +// SpirvEmitterState::tableIndexNonUniform) the index carries NonUniform, which needs +// VK_EXT_descriptor_indexing on the device. +std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource, std::uint32_t arrayIndex, std::uint32_t slotId) { + const bool storage = ImageBindingResourceClass(kind) == ImageResourceClass::Storage; + const auto supported = [&](std::uint32_t capability) { + return std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), capability) != state.supportedCapabilities.end(); + }; + const auto dynamic = storage ? spv::CapabilityStorageImageArrayDynamicIndexing : spv::CapabilitySampledImageArrayDynamicIndexing; + if (!supported(dynamic)) { + ExitDescriptorBindingFailure(state, kind, resource, "bindless image table needs image array dynamic indexing, which the device lacks"); + } + state.module.EmitCapability(dynamic); + const auto index = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, arrayIndex), slotId); + if (!state.tableIndexNonUniform) { + return index; + } + const auto nonUniform = storage ? spv::CapabilityStorageImageArrayNonUniformIndexing : spv::CapabilitySampledImageArrayNonUniformIndexing; + const bool extension = state.spirvVersion < 0x00010500u; + if (!supported(spv::CapabilityShaderNonUniform) || !supported(nonUniform) || (extension && std::find(state.supportedExtensions.begin(), state.supportedExtensions.end(), "SPV_EXT_descriptor_indexing") == state.supportedExtensions.end())) { + ResourceMaterializer::CountBindlessRejection(BindlessRejection::NonUniform); + throw std::runtime_error("bindless image table: the slot is not uniform over the workgroup and the device lacks non-uniform image indexing"); + } + state.module.EmitCapability(spv::CapabilityShaderNonUniform); + state.module.EmitCapability(nonUniform); + if (extension) state.module.EmitExtension("SPV_EXT_descriptor_indexing"); + state.module.AddAnnotation(spv::OpDecorate, index, spv::DecorationNonUniform); + return index; +} + +void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t slotId, std::uint32_t id) { + if (slotId != 0 && state.tableIndexNonUniform) state.module.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform); +} + std::uint32_t ImageScalarType(SpirvEmitterState& state, IrTextureNumericClass numericClass) { switch (numericClass) { case IrTextureNumericClass::Float: return TypeF32(state); @@ -108,7 +144,7 @@ std::uint32_t ImageViewSizeType(SpirvEmitterState& state, RdnaImageDimension dim } } -std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource) { +std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t slotId) { const auto& imageResource = state.program.Info().images.at(resource); if (imageResource.resourceClass != ImageResourceClass::Sampled) { FailEmit("sampled image descriptor requested for a non-sampled image"); @@ -118,9 +154,20 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t const auto variable = state.imageVariables.at(ImageBindingIndex(kind)); const auto imageType = ImageType(state, imageResource); const auto pointerType = state.module.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, imageType); - const auto pointer = DescriptorElementPointer(state, pointerType, variable, arrayIndex, kind, resource, "sampled image descriptor array was not emitted"); + if (variable == 0) { + ExitDescriptorBindingFailure(state, kind, resource, "sampled image descriptor array was not emitted"); + } + std::uint32_t pointer = 0; + if (slotId == 0) { + pointer = DescriptorElementPointer(state, pointerType, variable, arrayIndex, kind, resource, "sampled image descriptor array was not emitted"); + } else { + pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, variable, TableImageIndex(state, kind, resource, arrayIndex, slotId)); + DecorateTableNonUniform(state, slotId, pointer); + } const auto image = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, imageType, image, pointer); + DecorateTableNonUniform(state, slotId, image); return image; } @@ -134,13 +181,14 @@ std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t samp return samplerId; } -std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler) { +std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler, std::uint32_t slotId) { const auto& imageResource = state.program.Info().images.at(resource); - const auto image = LoadSampledImageDescriptor(state, resource); + const auto image = LoadSampledImageDescriptor(state, resource, slotId); const auto samplerId = LoadSamplerDescriptor(state, sampler); const auto sampledImage = state.module.AllocateId(); const auto sampledType = state.module.Type(spv::OpTypeSampledImage, ImageType(state, imageResource)); state.module.AddFunction(spv::OpSampledImage, sampledType, sampledImage, image, samplerId); + DecorateTableNonUniform(state, slotId, sampledImage); return sampledImage; } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index edba86edd6..59ac117120 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -172,33 +172,31 @@ std::uint32_t DeviceAddressFromWords(SpirvEmitterState& state, std::uint32_t low return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64); } -std::uint32_t GuestAddress(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { +std::uint32_t GuestAddressBase(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { auto& state = ctx.state; auto low = ctx.Arg(inst, 1); if (mem.kind == ResourceKind::ScalarAddress) { low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u)); } - std::uint32_t address = 0; if (mem.addressIsFull) { - address = DeviceAddressFromWords(state, low, ctx.Arg(inst, 2)); - } else { - const IrValue* argument = inst.Argument(0); - const IrValue* handle = argument != nullptr ? argument->Resolve() : nullptr; - if (handle == nullptr || handle->Opcode() != IrOpcode::GetAddressResource || handle->ArgumentCount() != 2u) { - ctx.Fail(inst, "has no address base pair"); - } - const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1)); - address = AddBdaAddress(ctx, inst, base, Unary(state, spv::OpUConvert, TypeScalarU64(state), low), false); + return DeviceAddressFromWords(state, low, ctx.Arg(inst, 2)); } + const IrValue* argument = inst.Argument(0); + const IrValue* handle = argument != nullptr ? argument->Resolve() : nullptr; + if (handle == nullptr || handle->Opcode() != IrOpcode::GetAddressResource || handle->ArgumentCount() != 2u) { + ctx.Fail(inst, "has no address base pair"); + } + const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1)); + return AddBdaAddress(ctx, inst, base, Unary(state, spv::OpUConvert, TypeScalarU64(state), low), false); +} + +std::uint32_t GuestAddress(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { + const auto address = GuestAddressBase(ctx, inst, mem); auto immediate = static_cast(mem.offset); if (mem.kind == ResourceKind::ScalarAddress) { immediate = static_cast(static_cast(immediate) & ~3u); } - if (immediate == 0) { - return address; - } - const auto magnitude = immediate < 0 ? -static_cast(immediate) : static_cast(immediate); - return AddBdaAddress(ctx, inst, address, ConstantDeviceAddress(state, static_cast(magnitude)), immediate < 0); + return AddBdaImmediate(ctx, inst, address, immediate); } std::uint32_t LoadBda(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t bits) { @@ -595,6 +593,28 @@ void LoadAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t } } +std::uint32_t LoadBdaWide(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { + auto& state = ctx.state; + return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { + return ConstructU32Composite(state, components, EmitBdaDwordReads(ctx, inst, GuestAddressBase(ctx, inst, mem), mem.offset, components)); + }); +} + +void LoadAddressWide(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) { + const auto& mem = ctx.Memory(inst); + if (mem.planningOnly) { + return; + } + switch (mem.kind) { + case ResourceKind::Flat: + case ResourceKind::Global: + ctx.Define(inst, LoadBdaWide(ctx, inst, mem, components)); + return; + default: + ctx.Fail(inst, "must read a physical address resource"); + } +} + void StoreAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t bits) { const auto& mem = ctx.Memory(inst); if (mem.kind != ResourceKind::Scratch) { @@ -947,6 +967,18 @@ void EmitLoadAddressU32(SpirvValueEmitContext& ctx, const IrValue& inst) { LoadAddress(ctx, inst, 32u); } +void EmitLoadAddressU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) { + LoadAddressWide(ctx, inst, 2u); +} + +void EmitLoadAddressU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) { + LoadAddressWide(ctx, inst, 3u); +} + +void EmitLoadAddressU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) { + LoadAddressWide(ctx, inst, 4u); +} + void EmitStoreAddressU8(SpirvValueEmitContext& ctx, const IrValue& inst) { StoreAddress(ctx, inst, 8u); } diff --git a/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp b/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp index e9223b838a..238056cd61 100644 --- a/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp +++ b/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp @@ -18,6 +18,12 @@ struct EmbeddedFetchPlan { std::vector loads; std::int32_t vertexOffsetSgpr = -1; std::int32_t instanceOffsetSgpr = -1; + // Whether the offset SGPR is read (or written) anywhere besides the accumulator add, and + // whether two different SGPRs were added to the accumulator (the SGPR then stays -1). + bool vertexOffsetShared = false; + bool instanceOffsetShared = false; + bool vertexOffsetConflict = false; + bool instanceOffsetConflict = false; }; class EmbeddedVertexFetchAnalyzer { diff --git a/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp b/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp index 2b3d74ac3a..4e5e270afd 100644 --- a/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp +++ b/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp @@ -161,6 +161,43 @@ std::uint32_t embeddedFetchDstSize(const RdnaInstruction& inst) { return inst.op == RdnaOpcode::VMadU64U32 ? 2u : decodedDstSize(inst); } +// How many consecutive SGPRs a scalar operand of the instruction names, over-estimated where the +// decoder does not say (descriptor bases, 64-bit pairs, carry-ins): a register counted as read that +// is not only costs the driver's fold decision, never its correctness. +std::uint32_t scalarOperandWidth(const RdnaInstruction& inst, std::uint32_t operandIndex) { + switch (inst.family) { + case RdnaInstructionFamily::SMEM: + return operandIndex == 0u ? 4u : 1u; + case RdnaInstructionFamily::MUBUF: + case RdnaInstructionFamily::MTBUF: + return operandIndex == 1u ? 4u : 1u; + case RdnaInstructionFamily::MIMG: + return 8u; + case RdnaInstructionFamily::FLAT: + return 2u; + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + return inst.is64Bit || operandIndex == 2u ? 2u : 1u; + default: + return inst.is64Bit ? 2u : 1u; + } +} + +bool touchesSgpr(const RdnaInstruction& inst, std::uint32_t sgpr) { + const std::array sources{&inst.source0, &inst.source1, &inst.source2, &inst.source3}; + for (std::uint32_t i = 0u; i < sources.size(); i++) { + if (!isScalarOperand(*sources[i])) continue; + const auto first = scalarSlot(*sources[i]); + if (sgpr >= first && sgpr < first + scalarOperandWidth(inst, i)) return true; + } + if (isScalarOperand(inst.destination)) { + const auto first = scalarSlot(inst.destination); + if (sgpr >= first && sgpr < first + std::max(inst.is64Bit ? 2u : 1u, decodedDstSize(inst))) return true; + } + return false; +} + } EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& program, std::uint32_t attributeTableRegister, std::uint32_t bufferTableRegister, std::uint32_t userDataBaseRegister, std::uint32_t userDataCount, std::uint32_t waveSize) const { @@ -190,6 +227,8 @@ EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& progra std::int32_t instanceOffsetCandidate = -1; bool vertexOffsetConflict = false; bool instanceOffsetConflict = false; + std::vector vertexAddPcs; + std::vector instanceAddPcs; for (const auto& inst : program.instructions) { const bool vertexIndexAccumulator = isVectorOperand(inst.destination) && (inst.destination.reg == 0u || (userDataBaseRegister == 8u && inst.destination.reg == kVertexIndexVgpr)); @@ -210,6 +249,7 @@ EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& progra } else { candidate = static_cast(reg); } + (vertexIndexAccumulator ? vertexAddPcs : instanceAddPcs).push_back(inst.programCounter); } } switch (inst.op) { @@ -365,6 +405,19 @@ EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& progra } } + plan.vertexOffsetConflict = vertexOffsetConflict; + plan.instanceOffsetConflict = instanceOffsetConflict; + const auto shared = [&](std::int32_t sgpr, const std::vector& addPcs) { + if (sgpr < 0) return false; + for (const auto& inst : program.instructions) { + if (std::find(addPcs.begin(), addPcs.end(), inst.programCounter) != addPcs.end()) continue; + if (touchesSgpr(inst, static_cast(sgpr))) return true; + } + return false; + }; + plan.vertexOffsetShared = shared(plan.vertexOffsetSgpr, vertexAddPcs); + plan.instanceOffsetShared = shared(plan.instanceOffsetSgpr, instanceAddPcs); + return plan; } diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index a16586b296..b000be7825 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -360,6 +360,10 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con if (options.embeddedFetch != nullptr) { program.Info().vertexOffsetSgpr = options.embeddedFetch->vertexOffsetSgpr; program.Info().instanceOffsetSgpr = options.embeddedFetch->instanceOffsetSgpr; + program.Info().vertexOffsetShared = options.embeddedFetch->vertexOffsetShared; + program.Info().instanceOffsetShared = options.embeddedFetch->instanceOffsetShared; + program.Info().vertexOffsetConflict = options.embeddedFetch->vertexOffsetConflict; + program.Info().instanceOffsetConflict = options.embeddedFetch->instanceOffsetConflict; } program.Metadata().cfgFailureKind = cfg.failureKind; program.Metadata().failureReason = cfg.unsupportedReason; diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp index af8c415373..2151b1048f 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp @@ -1,12 +1,31 @@ #include "Translation/MemoryInstructions.hpp" #include "Translation/TranslationContext.hpp" #include +#include #include namespace ShaderRecompiler { namespace { +// A multi-dword global load is one address instruction so the emitter can resolve its guest address +// once (see EmitBdaDwordReads). APS5_BDA_BYTE_READS=1 keeps one LoadAddressU32 per dword, as before. +bool wideAddressLoadsEnabled() { + static const bool byteReads = std::getenv("APS5_BDA_BYTE_READS") != nullptr; + return !byteReads; +} + +IrOpcode wideAddressLoadOpcode(std::uint32_t dwords) { + switch (dwords) { + case 2u: + return IrOpcode::LoadAddressU32x2; + case 3u: + return IrOpcode::LoadAddressU32x3; + default: + return IrOpcode::LoadAddressU32x4; + } +} + ResourceKind flatSegmentResourceKind(std::uint32_t segment) { switch (segment) { case 1u: @@ -54,6 +73,18 @@ bool TranslationContext::flatLoad(const RdnaInstruction& inst) { const AddressOperands address = readAddressOperands(inst, 0u); IrValue& active = ir.GetExec(); const std::uint32_t count = memory.dataBits == 32u ? std::min(memory.dataDwords, 4u) : 1u; + if (count > 1u && memory.kind != ResourceKind::Scratch && wideAddressLoadsEnabled()) { + MemoryInfo group = memory; + group.dataDwords = count; + group.componentCount = count; + group.componentIndex = 0u; + const IrOpcode wide = wideAddressLoadOpcode(count); + IrValue& loaded = ir.Emit(wide, IrOpcodeType(wide), {address.resource, address.low, address.high, &active}, addMemoryInfo(group, inst.programCounter)); + for (std::uint32_t index = 0u; index < count; ++index) { + writeOperand(offsetOperand(inst.destination, index), &ir.CompositeExtract(loaded, index)); + } + return true; + } for (std::uint32_t index = 0u; index < count; ++index) { MemoryInfo component = memory; component.offset += index * 4u; From 6e810feb506442fb00f3c45e846c709de349d4b8 Mon Sep 17 00:00:00 2001 From: owner Date: Sat, 26 Sep 2026 23:30:21 -0600 Subject: [PATCH 16/93] docs: drop SPIR-V gate note from TechnicalDebt.md per maintainer feedback Maintainer noted the opt-in-via-documented-flag SPIR-V validation gap isn't the silent-stub category this doc tracks; the README correction stands on its own. Co-Authored-By: Claude Sonnet 5 --- docs/TechnicalDebt.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/TechnicalDebt.md b/docs/TechnicalDebt.md index 28f74897d4..922c79b780 100644 --- a/docs/TechnicalDebt.md +++ b/docs/TechnicalDebt.md @@ -14,7 +14,6 @@ Throughout the project, every function at every stage either **does exactly what - [libSceSaveDataDialog.native](../core/libs/prx/libSceSaveDataDialog.native/Export.cpp) - [libSceCommonDialog](../core/libs/prx/libSceCommonDialog/Export.cpp) - The shader recompiler [skips baryctric coordinates](../core/shader/recompiler/Recompiler.cpp) (is not even passed to SpirvTargetOptions at row 212). -- [SPIR-V validation](../core/shader/recompiler/Recompiler.cpp) via Spirv-Tools is gated behind the `ANYPS5_ENABLE_SPIRV_TOOLS` CMake option (default OFF) and does not run in a default build. ### Unknown function names From 1379ec001b63e92d34741fc19317f6699bc8b806 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 15:10:21 +0300 Subject: [PATCH 17/93] feat(relinker): add native guest module loading with export priority --- core/relinker/CMakeLists.txt | 5 + .../domain/include/domain/GuestRuntime.hpp | 27 +++ core/relinker/domain/include/domain/Types.hpp | 2 + .../elfpatcher/general/GuestModuleWriter.hpp | 16 ++ .../windows/WindowsEntryStubBuilder.hpp | 2 +- .../windows/WindowsGuestStartup.hpp | 21 ++ .../elfpatcher/windows/WindowsPeFormat.hpp | 2 + .../elfpatcher/windows/WindowsTlsBuilder.hpp | 2 +- .../src/linux/LinuxGuestModuleWriter.cpp | 131 ++++++++++++ .../src/windows/WindowsEntryStubBuilder.cpp | 34 ++- .../src/windows/WindowsGuestModuleWriter.cpp | 160 +++++++++++++++ .../src/windows/WindowsGuestStartup.cpp | 43 ++++ .../src/windows/WindowsPePatcher.cpp | 11 +- .../src/windows/WindowsTlsBuilder.cpp | 3 +- core/relinker/main.cpp | 11 +- .../include/relinker/guest/GuestImage.hpp | 49 +++++ .../relinker/src/guest/GuestImageReader.cpp | 193 ++++++++++++++++++ .../relinker/src/guest/GuestModuleBuilder.cpp | 156 ++++++++++++++ 18 files changed, 856 insertions(+), 12 deletions(-) create mode 100644 core/relinker/domain/include/domain/GuestRuntime.hpp create mode 100644 core/relinker/elfpatcher/include/elfpatcher/general/GuestModuleWriter.hpp create mode 100644 core/relinker/elfpatcher/include/elfpatcher/windows/WindowsGuestStartup.hpp create mode 100644 core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp create mode 100644 core/relinker/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp create mode 100644 core/relinker/elfpatcher/src/windows/WindowsGuestStartup.cpp create mode 100644 core/relinker/relinker/include/relinker/guest/GuestImage.hpp create mode 100644 core/relinker/relinker/src/guest/GuestImageReader.cpp create mode 100644 core/relinker/relinker/src/guest/GuestModuleBuilder.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index a48325819d..66f3d02bd8 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -3,6 +3,11 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/cli/src/CliArgs.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cli/src/Autorun.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsGuestStartup.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/guest/GuestImageReader.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/guest/GuestModuleBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsPePatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsPeFormat.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsLoadImage.cpp diff --git a/core/relinker/domain/include/domain/GuestRuntime.hpp b/core/relinker/domain/include/domain/GuestRuntime.hpp new file mode 100644 index 0000000000..43aae70b09 --- /dev/null +++ b/core/relinker/domain/include/domain/GuestRuntime.hpp @@ -0,0 +1,27 @@ +#ifndef DOMAIN_GUESTRUNTIME_HPP +#define DOMAIN_GUESTRUNTIME_HPP + +#include +#include +#include + +namespace Domain { + +struct GuestImport { + std::string Name; + std::uint32_t TargetRva; + std::uint64_t Addend; + std::uint32_t RelocationType = 1; +}; + +struct GuestRuntime { + std::string Path; + std::vector Imports; + std::uint32_t InitRva = 0; + std::uint32_t FiniRva = 0; + bool UsePlatformTlsResolver = true; +}; + +} + +#endif diff --git a/core/relinker/domain/include/domain/Types.hpp b/core/relinker/domain/include/domain/Types.hpp index f75570c5ce..40bd960ce9 100644 --- a/core/relinker/domain/include/domain/Types.hpp +++ b/core/relinker/domain/include/domain/Types.hpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace Domain { @@ -92,6 +93,7 @@ struct SysVDynamicSection { std::vector DynStrData; std::vector RelaData; std::vector RelaPltData; + std::vector GuestModules; }; struct CallRegistryEntry { diff --git a/core/relinker/elfpatcher/include/elfpatcher/general/GuestModuleWriter.hpp b/core/relinker/elfpatcher/include/elfpatcher/general/GuestModuleWriter.hpp new file mode 100644 index 0000000000..c820c38791 --- /dev/null +++ b/core/relinker/elfpatcher/include/elfpatcher/general/GuestModuleWriter.hpp @@ -0,0 +1,16 @@ +#ifndef ELFPATCHER_GUESTMODULEWRITER_HPP +#define ELFPATCHER_GUESTMODULEWRITER_HPP + +#include + +namespace Elfpatcher { + +class GuestModuleWriter { +public: + std::vector WriteLinux(const Relinker::GuestImage& image, const std::vector& dependencies, const std::string& runPath) const; + std::vector WriteWindows(const Relinker::GuestImage& image, Domain::GuestRuntime& runtime) const; +}; + +} + +#endif diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsEntryStubBuilder.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsEntryStubBuilder.hpp index 1ab27beca0..a7a1225c18 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsEntryStubBuilder.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsEntryStubBuilder.hpp @@ -19,7 +19,7 @@ struct WindowsEntryStub { class WindowsEntryStubBuilder { public: - WindowsEntryStub Build(std::uint32_t dataRva, std::uint32_t entryRva, const WindowsImports& nativeImports, const std::vector& libraries, const std::vector& imports, const std::string& runPath, bool lazyBinding, bool dependencyDiagnostics) const; + WindowsEntryStub Build(std::uint32_t dataRva, std::uint32_t entryRva, const WindowsImports& nativeImports, const std::vector& libraries, const std::vector& imports, const std::string& runPath, bool lazyBinding, bool dependencyDiagnostics, const std::vector& guestModules = {}) const; }; } diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsGuestStartup.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsGuestStartup.hpp new file mode 100644 index 0000000000..209c8d00bd --- /dev/null +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsGuestStartup.hpp @@ -0,0 +1,21 @@ +#ifndef ELFPATCHER_WINDOWSGUESTSTARTUP_HPP +#define ELFPATCHER_WINDOWSGUESTSTARTUP_HPP + +#include +#include + +namespace Elfpatcher::Windows { + +class WindowsGuestStartup { +public: + void WriteImport(WindowsStubEmitter& code, const PeImport& import, std::uint32_t handles) const; + void Initialize(WindowsStubEmitter& code, const std::vector& modules, std::uint32_t handles) const; + void Finalize(WindowsStubEmitter& code, const std::vector& modules, std::uint32_t handles, std::uint32_t finished) const; + std::uint32_t EmitTlsResolver(WindowsStubEmitter& code) const; +private: + void callLifecycle(WindowsStubEmitter& code, std::uint32_t handle, std::uint32_t rva) const; +}; + +} + +#endif diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsPeFormat.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsPeFormat.hpp index 9c2d5036d4..bc1c0984d4 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsPeFormat.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsPeFormat.hpp @@ -32,6 +32,8 @@ struct PeImport { std::string Name; std::uint32_t TargetRva; std::uint64_t Addend; + std::int32_t TargetModule = -1; + std::uint32_t RelocationType = 1; }; struct PeRelocations { diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsBuilder.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsBuilder.hpp index 297ef58854..e7dc1ed7dd 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsBuilder.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsBuilder.hpp @@ -7,7 +7,7 @@ namespace Elfpatcher::Windows { class WindowsTlsBuilder { public: - PeDirectory Build(const std::vector& source, const std::vector& headers, const WindowsLoadImage& image, std::vector& sections, std::vector& relocations, std::uint32_t& nextRva) const; + PeDirectory Build(const std::vector& source, const std::vector& headers, const WindowsLoadImage& image, std::vector& sections, std::vector& relocations, std::uint32_t& nextRva, std::uint32_t* tlsIndexRva = nullptr) const; }; } diff --git a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp new file mode 100644 index 0000000000..31168ebd63 --- /dev/null +++ b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp @@ -0,0 +1,131 @@ +#include +#include +#include +#include + +namespace Elfpatcher { + +std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestImage& image, const std::vector& dependencies, const std::string& runPath) const { + auto bytes = image.Bytes; + std::vector headers; + std::uint64_t end = 0; + for (auto header : image.Headers) { + if (header.Type != 1 && header.Type != 7 && header.Type != 0x6474e550 && header.Type != 0x6474e551) continue; + if (header.Type == 1) { + header.Flags |= 4; + end = std::max(end, header.MappedAddress + header.MemorySize); + } + headers.push_back(header); + } + if (end > std::numeric_limits::max() - 0x4000) throw Domain::RelinkerException("Guest virtual address overflow"); + Io::AlignBuffer(bytes, 0x4000); + const auto extraOffset = bytes.size(); + const auto extraAddress = Io::AlignUp64(end, 0x4000); + const auto address = [&] { return extraAddress + bytes.size() - extraOffset; }; + auto strings = image.Dynamic.DynStrData; + auto symbols = image.Dynamic.DynSymData; + const auto addString = [&](const std::string& value) { + if (strings.size() > std::numeric_limits::max()) throw Domain::RelinkerException("Guest string table too large"); + const auto offset = static_cast(strings.size()); + Io::AppendString(strings, value); + return offset; + }; + bool needsTlsResolver = false; + for (std::size_t index = 1; index < image.Symbols.size(); ++index) { + const auto& symbol = image.Symbols[index]; + if (image.UsePlatformTlsResolver && symbol.Section == 0 && symbol.Name == "vNe1w4diLCs") { + Io::WriteU32(symbols, index * 24, addString("__tls_get_addr")); + needsTlsResolver = true; + } + if (symbol.Section == 0 && (symbol.Info >> 4) == 2) symbols[index * 24 + 4] = static_cast(0x10 | (symbol.Info & 15)); + } + std::vector needed; + for (const auto& dependency : dependencies) needed.push_back(addString(dependency)); + if (needsTlsResolver) needed.push_back(addString("ld-linux-x86-64.so.2")); + const auto soname = addString(image.OutputName); + const auto search = addString(runPath); + const auto strAddress = address(); + bytes.insert(bytes.end(), strings.begin(), strings.end()); + Io::AlignBuffer(bytes, 8); + const auto symAddress = address(); + bytes.insert(bytes.end(), symbols.begin(), symbols.end()); + const auto hashAddress = address(); + const auto count = static_cast(image.Symbols.size()); + Io::AppendU32(bytes, 1); + Io::AppendU32(bytes, count); + Io::AppendU32(bytes, count > 1 ? 1 : 0); + for (std::uint32_t index = 0; index < count; ++index) Io::AppendU32(bytes, index != 0 && index + 1 < count ? index + 1 : 0); + Io::AlignBuffer(bytes, 8); + const auto relaAddress = address(); + bytes.insert(bytes.end(), image.Dynamic.RelaData.begin(), image.Dynamic.RelaData.end()); + const auto pltAddress = address(); + bytes.insert(bytes.end(), image.Dynamic.RelaPltData.begin(), image.Dynamic.RelaPltData.end()); + const auto lifecycle = [&](std::uint64_t target) { + if (target == 0) return std::uint64_t{}; + const auto start = address(); + bytes.insert(bytes.end(), {0x31, 0xff, 0x31, 0xf6, 0x31, 0xd2, 0xe9}); + const auto displacement = static_cast(target) - static_cast(address() + 4); + if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) throw Domain::RelinkerException("Guest initializer exceeds relative branch range"); + Io::AppendU32(bytes, static_cast(displacement)); + return start; + }; + const auto init = lifecycle(image.Init); + const auto fini = lifecycle(image.Fini); + Io::AlignBuffer(bytes, 8); + const auto dynamicOffset = bytes.size(); + const auto dynamicAddress = address(); + const auto tag = [&](std::uint64_t key, std::uint64_t value) { Io::AppendU64(bytes, key); Io::AppendU64(bytes, value); }; + for (const auto offset : needed) tag(1, offset); + tag(14, soname); + tag(29, search); + tag(4, hashAddress); + tag(5, strAddress); + tag(10, strings.size()); + tag(6, symAddress); + tag(11, 24); + if (!image.Dynamic.RelaData.empty()) { + tag(7, relaAddress); + tag(8, image.Dynamic.RelaData.size()); + tag(9, 24); + } + if (!image.Dynamic.RelaPltData.empty()) { + tag(23, pltAddress); + tag(2, image.Dynamic.RelaPltData.size()); + tag(20, 7); + tag(3, image.Got); + } + if (init != 0) tag(12, init); + if (fini != 0) tag(13, fini); + tag(30, 8); + tag(0, 0); + const auto dynamicSize = bytes.size() - dynamicOffset; + const auto phOffset = bytes.size(); + const auto phCount = headers.size() + 2; + if (phCount > std::numeric_limits::max()) throw Domain::RelinkerException("Too many guest program headers"); + const auto extraSize = bytes.size() + phCount * 56 - extraOffset; + headers.push_back({1, 7, extraOffset, extraAddress, extraAddress, extraSize, extraSize, 0x4000}); + headers.push_back({2, 6, dynamicOffset, dynamicAddress, dynamicAddress, dynamicSize, dynamicSize, 8}); + for (const auto& header : headers) { + Io::AppendU32(bytes, header.Type); + Io::AppendU32(bytes, header.Flags); + Io::AppendU64(bytes, header.Offset); + Io::AppendU64(bytes, header.MappedAddress); + Io::AppendU64(bytes, header.PhysicalAddress); + Io::AppendU64(bytes, header.FileSize); + Io::AppendU64(bytes, header.MemorySize); + Io::AppendU64(bytes, header.Alignment); + } + bytes[7] = 0; + bytes[8] = 0; + Io::WriteU16(bytes, 16, 3); + Io::WriteU64(bytes, 24, 0); + Io::WriteU64(bytes, 32, phOffset); + Io::WriteU64(bytes, 40, 0); + Io::WriteU16(bytes, 56, static_cast(phCount)); + Io::WriteU16(bytes, 58, 0); + Io::WriteU16(bytes, 60, 0); + Io::WriteU16(bytes, 62, 0); + return bytes; +} + +} diff --git a/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp index 51f41f827e..c0bde96688 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,7 @@ std::string normalizeRunPath(std::string path) { } -WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, const std::uint32_t entryRva, const WindowsImports& nativeImports, const std::vector& libraries, const std::vector& imports, const std::string& runPath, const bool lazyBinding, const bool dependencyDiagnostics) const { +WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, const std::uint32_t entryRva, const WindowsImports& nativeImports, const std::vector& libraries, const std::vector& imports, const std::string& runPath, const bool lazyBinding, const bool dependencyDiagnostics, const std::vector& guestModules) const { if (!imports.empty() && libraries.empty()) throw Domain::RelinkerException("ELF imports have no DT_NEEDED libraries"); const auto path = normalizeRunPath(runPath); @@ -54,14 +55,17 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con const auto programPath = reserve(PathCapacity); const auto modulePath = reserve(PathCapacity); const auto handles = reserve(libraries.size() * 8); + const auto guestFinished = reserve(4); + const WindowsGuestStartup guestStartup; const auto functionTable = reserve(12 * 32); const auto unwindRva = CheckedRva(dataRva + data.size()); data.insert(data.end(), {1, 10, 6, 0, 10, 0xb2, 6, 0xc0, 4, 0x70, 3, 0x60, 2, 0x50, 1, 0x30}); std::vector libraryPaths; std::vector libraryNames; - for (const auto& library : libraries) { - auto name = path + library; + for (std::size_t index = 0; index < libraries.size(); ++index) { + auto name = index < guestModules.size() ? libraries[index] : path + libraries[index]; + std::replace(name.begin(), name.end(), '/', '\\'); if (name.size() + 1 >= PathCapacity) throw Domain::RelinkerException("Windows library path exceeds the startup buffer: " + name); libraryPaths.push_back(addString(name)); @@ -218,7 +222,7 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con code.Emit({0x49, 0xff, 0xc4}); for (std::size_t index = 0; index < libraries.size(); ++index) { - if (absolutePath) { + if (absolutePath && index >= guestModules.size()) { code.Rip({0x48, 0x8d, 0x0d}, libraryPaths[index]); } else { code.Emit({0x4c, 0x89, 0xe0, 0x48, 0x29, 0xd8, 0x48, 0x05}); @@ -270,7 +274,14 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con std::vector unresolvedBranches; std::vector lazyUnresolvedImports; + std::vector tlsResolverAddresses; for (std::size_t index = 0; index < imports.size(); ++index) { + if (!guestModules.empty() && guestModules.front().UsePlatformTlsResolver && imports[index].Name == "vNe1w4diLCs") { + if (imports[index].Addend != 0) throw Domain::RelinkerException("TLS resolver import has an addend"); + tlsResolverAddresses.push_back(code.Branch({0x48, 0x8d, 0x05})); + guestStartup.WriteImport(code, imports[index], handles); + continue; + } code.Rip({0x48, 0x8d, 0x1d}, handles); code.Rip({0x48, 0x8d, 0x35}, symbolNames[index]); code.Emit({0xbd}); @@ -291,7 +302,7 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con code.U64(imports[index].Addend); code.Emit({0x48, 0x01, 0xd0}); } - code.Rip({0x48, 0x89, 0x05}, imports[index].TargetRva); + guestStartup.WriteImport(code, imports[index], handles); code.PatchBranch(skipGotWrite, code.GetRva()); continue; } @@ -299,20 +310,26 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con writeString(errorRvas.at(3 + index), true); unresolvedBranches.push_back(code.Branch({0xe9})); code.PatchBranch(resolved, code.GetRva()); + if (imports[index].RelocationType == 16) code.Emit({0x48, 0x8b, 0x00}); + if (imports[index].RelocationType == 17) code.Emit({0x48, 0x8b, 0x40, 8}); if (imports[index].Addend != 0) { code.Emit({0x48, 0xba}); code.U64(imports[index].Addend); code.Emit({0x48, 0x01, 0xd0}); } - code.Rip({0x48, 0x89, 0x05}, imports[index].TargetRva); + guestStartup.WriteImport(code, imports[index], handles); } + guestStartup.Initialize(code, guestModules, handles); writeString(enteringElf); code.Emit({0x48, 0xc7, 0x44, 0x24, 0x40, 1, 0, 0, 0}); code.Rip({0x48, 0x8d, 0x05}, programPath); code.Emit({0x48, 0x89, 0x44, 0x24, 0x48, 0x31, 0xc0, 0x48, 0x89, 0x44, 0x24, 0x50, 0x48, 0x89, 0x44, 0x24, 0x58, 0x48, 0x8d, 0x7c, 0x24, 0x40}); const auto exitCallback = code.Branch({0x48, 0x8d, 0x35}); code.Rip({0xe8}, entryRva); + code.Emit({0x89, 0x44, 0x24, 0x58}); + guestStartup.Finalize(code, guestModules, handles, guestFinished); + code.Emit({0x8b, 0x44, 0x24, 0x58}); code.Emit({0x89, 0xc1}); call("ExitProcess"); code.Emit({0x0f, 0x0b}); @@ -332,7 +349,12 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con const auto functionEnd = code.GetRva(); code.PatchBranch(exitCallback, functionEnd); + code.Emit({0x48, 0x83, 0xec, 0x28}); + guestStartup.Finalize(code, guestModules, handles, guestFinished); + code.Emit({0x48, 0x83, 0xc4, 0x28}); code.Emit({0xc3}); + const auto tlsResolver = guestStartup.EmitTlsResolver(code); + for (const auto offset : tlsResolverAddresses) code.PatchBranch(offset, tlsResolver); for (const auto index : lazyUnresolvedImports) { const auto stubRva = code.GetRva(); diff --git a/core/relinker/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp new file mode 100644 index 0000000000..3e7f5be806 --- /dev/null +++ b/core/relinker/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp @@ -0,0 +1,160 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Elfpatcher { + +std::vector GuestModuleWriter::WriteWindows(const Relinker::GuestImage& guest, Domain::GuestRuntime& runtime) const { + using namespace Windows; + WindowsLoadImage image(guest.Bytes, guest.Headers); + std::vector relocations; + std::vector> tlsModules; + std::map targets; + const auto apply = [&](const std::vector& table) { + for (std::size_t offset = 0; offset < table.size(); offset += 24) { + const auto target = Io::ReadU64(table, offset); + const auto info = Io::ReadU64(table, offset + 8); + const auto addend = Io::ReadU64(table, offset + 16); + const auto type = static_cast(info); + const auto symbolIndex = info >> 32; + const auto& symbol = guest.Symbols.at(symbolIndex); + const auto rva = image.GetRva(target, 8); + for (const auto& header : guest.Headers) { + if (header.Type == 7 && target >= header.MappedAddress && target - header.MappedAddress < header.FileSize) throw Domain::RelinkerException("Windows guest TLS template relocations require explicit support", target); + } + const auto next = targets.lower_bound(target); + if ((next != targets.end() && next->first < target + 8) || (next != targets.begin() && std::prev(next)->second > target)) throw Domain::RelinkerException("Overlapping guest relocations", target); + targets.emplace(target, target + 8); + image.RequireWritable(target, 8); + if (type == 8) { + if (symbolIndex != 0) throw Domain::RelinkerException("RELATIVE guest relocation has a symbol", target); + image.WritePointer(target, image.GetRelocatedAddress(addend)); + relocations.push_back(rva); + } else if (type == 16) { + if (addend != 0 || (symbolIndex != 0 && (symbol.Info & 15) != 6)) throw Domain::RelinkerException("Invalid guest TLS module relocation", target); + if (symbolIndex != 0 && symbol.Section == 0) { + image.WritePointer(target, 0); + runtime.Imports.push_back({symbol.Name, rva, 0, 16}); + } else tlsModules.emplace_back(target, rva); + } else if (type == 17) { + if (symbolIndex != 0 && (symbol.Info & 15) != 6) throw Domain::RelinkerException("Invalid guest TLS offset relocation", target); + if (symbolIndex != 0 && symbol.Section == 0) { + image.WritePointer(target, 0); + runtime.Imports.push_back({symbol.Name, rva, addend, 17}); + } else image.WritePointer(target, symbol.Value + addend); + } else if (type == 18) { + const auto tls = std::find_if(guest.Headers.begin(), guest.Headers.end(), [](const auto& header) { return header.Type == 7; }); + if (tls == guest.Headers.end() || (symbolIndex != 0 && (symbol.Section == 0 || (symbol.Info & 15) != 6))) throw Domain::RelinkerException("Unsupported external static TLS relocation", target); + const auto size = Io::AlignUp64(tls->MemorySize, std::max(tls->Alignment, 16)); + image.WritePointer(target, symbol.Value + addend - size); + } else if (type == 1 || type == 6 || type == 7) { + if (symbolIndex == 0 || (type != 1 && addend != 0) || (symbol.Info & 15) == 6) throw Domain::RelinkerException("Invalid guest symbol relocation", target); + if (symbol.Section != 0) { + if (symbol.Section >= 0xff00) throw Domain::RelinkerException("Unsupported special guest symbol section", target); + image.WritePointer(target, image.GetRelocatedAddress(symbol.Value) + addend); + relocations.push_back(rva); + } else { + if (symbol.Name.empty()) throw Domain::RelinkerException("Empty guest import", target); + image.WritePointer(target, 0); + runtime.Imports.push_back({symbol.Name, rva, addend}); + } + } else throw Domain::RelinkerException("Unsupported Windows guest relocation " + std::to_string(type), target); + } + }; + apply(guest.Dynamic.RelaData); + apply(guest.Dynamic.RelaPltData); + auto sections = image.BuildSections(); + auto nextRva = image.GetEndRva(); + std::array directories{}; + std::uint32_t tlsIndex = 0; + directories[9] = WindowsTlsBuilder().Build(guest.Bytes, guest.Headers, image, sections, relocations, nextRva, &tlsIndex); + for (const auto& [target, rva] : tlsModules) { + if (tlsIndex == 0) throw Domain::RelinkerException("Guest TLS relocation has no TLS block", target); + bool written = false; + for (auto& section : sections) { + if (rva < section.Rva || rva - section.Rva > section.Data.size() || section.Data.size() - (rva - section.Rva) < 8) continue; + Io::WriteU64(section.Data, rva - section.Rva, ImageBase + tlsIndex); + written = true; + break; + } + if (!written) throw Domain::RelinkerException("Guest TLS target is unmapped", target); + relocations.push_back(rva); + } + for (const auto& header : guest.Headers) { + if (header.Type != 0x6474e550) continue; + std::vector metadata(4); + Io::WriteU32(metadata, 0, image.GetRva(header.MappedAddress, header.FileSize)); + sections.push_back({".ehmeta", nextRva, SectionRead | 0x40u, std::move(metadata)}); + nextRva = AlignRva(nextRva + sections.back().Data.size()); + } + std::map exports; + PeSection tlsExports{".tlsrefs", nextRva, SectionRead | 0x40u, {}}; + for (const auto& symbol : guest.Symbols) { + if (symbol.Section == 0 || (symbol.Info >> 4) == 0 || symbol.Visibility == 1 || symbol.Visibility == 2) continue; + if (symbol.Section >= 0xff00) throw Domain::RelinkerException("Unsupported guest export section: " + symbol.Name); + std::uint32_t rva = 0; + if ((symbol.Info & 15) == 6) { + if (tlsIndex == 0) throw Domain::RelinkerException("Guest TLS export has no TLS block: " + symbol.Name); + rva = CheckedRva(tlsExports.Rva + tlsExports.Data.size()); + Io::AppendU64(tlsExports.Data, ImageBase + tlsIndex); + Io::AppendU64(tlsExports.Data, symbol.Value); + relocations.push_back(rva); + } else rva = image.GetRva(symbol.Value, std::max(symbol.Size, 1)); + if (!exports.emplace(symbol.Name, rva).second) throw Domain::RelinkerException("Duplicate guest export: " + symbol.Name); + } + if (!tlsExports.Data.empty()) { + nextRva = AlignRva(nextRva + tlsExports.Data.size()); + sections.push_back(std::move(tlsExports)); + } + if (exports.size() > 65535) throw Domain::RelinkerException("Too many guest PE exports"); + PeSection exportSection{".edata", nextRva, SectionRead | 0x40u, std::vector(40)}; + auto& data = exportSection.Data; + const auto count = CheckedRva(exports.size()); + const auto functions = data.size(); + data.resize(data.size() + count * 4); + const auto names = data.size(); + data.resize(data.size() + count * 4); + const auto ordinals = data.size(); + data.resize(data.size() + count * 2); + Io::WriteU32(data, 12, CheckedRva(nextRva + data.size())); + Io::AppendString(data, guest.OutputName); + Io::WriteU32(data, 16, 1); + Io::WriteU32(data, 20, count); + Io::WriteU32(data, 24, count); + Io::WriteU32(data, 28, CheckedRva(nextRva + functions)); + Io::WriteU32(data, 32, CheckedRva(nextRva + names)); + Io::WriteU32(data, 36, CheckedRva(nextRva + ordinals)); + std::size_t index = 0; + for (const auto& [name, rva] : exports) { + Io::WriteU32(data, functions + index * 4, rva); + Io::WriteU32(data, names + index * 4, CheckedRva(nextRva + data.size())); + Io::WriteU16(data, ordinals + index * 2, static_cast(index)); + Io::AppendString(data, name); + ++index; + } + directories[0] = {nextRva, CheckedRva(data.size())}; + nextRva = AlignRva(nextRva + data.size()); + sections.push_back(std::move(exportSection)); + auto relocationData = WindowsRelocationBuilder().BuildBaseRelocations(relocations); + if (!relocationData.empty()) { + directories[5] = {nextRva, CheckedRva(relocationData.size())}; + sections.push_back({".reloc", nextRva, SectionRead | 0x02000040u, std::move(relocationData)}); + nextRva = AlignRva(nextRva + sections.back().Data.size()); + } + const auto entry = nextRva; + sections.push_back({".dllmain", entry, SectionRead | SectionExecute | 0x20u, {0xb8, 1, 0, 0, 0, 0xc3}}); + runtime.InitRva = guest.Init == 0 ? 0 : image.GetRva(guest.Init); + runtime.FiniRva = guest.Fini == 0 ? 0 : image.GetRva(guest.Fini); + auto result = WindowsPeWriter().Write(sections, entry, directories); + const auto characteristics = Io::ReadU16(result, 0x80 + 22); + Io::WriteU16(result, 0x80 + 22, static_cast((characteristics | 0x2000) & ~1u)); + return result; +} + +} diff --git a/core/relinker/elfpatcher/src/windows/WindowsGuestStartup.cpp b/core/relinker/elfpatcher/src/windows/WindowsGuestStartup.cpp new file mode 100644 index 0000000000..b5e212bc9f --- /dev/null +++ b/core/relinker/elfpatcher/src/windows/WindowsGuestStartup.cpp @@ -0,0 +1,43 @@ +#include + +namespace Elfpatcher::Windows { + +void WindowsGuestStartup::WriteImport(WindowsStubEmitter& code, const PeImport& import, const std::uint32_t handles) const { + if (import.TargetModule < 0) { + code.Rip({0x48, 0x89, 0x05}, import.TargetRva); + return; + } + code.Rip({0x48, 0x8b, 0x15}, CheckedRva(handles + static_cast(import.TargetModule) * 8)); + code.Emit({0x41, 0xb8}); + code.U32(import.TargetRva); + code.Emit({0x4a, 0x89, 0x04, 0x02}); +} + +void WindowsGuestStartup::callLifecycle(WindowsStubEmitter& code, const std::uint32_t handle, const std::uint32_t rva) const { + if (rva == 0) return; + code.Rip({0x48, 0x8b, 0x05}, handle); + code.Emit({0xb9}); + code.U32(rva); + code.Emit({0x48, 0x01, 0xc8, 0x31, 0xff, 0x31, 0xf6, 0x31, 0xd2, 0xff, 0xd0}); +} + +void WindowsGuestStartup::Initialize(WindowsStubEmitter& code, const std::vector& modules, const std::uint32_t handles) const { + for (std::size_t index = 0; index < modules.size(); ++index) callLifecycle(code, CheckedRva(handles + index * 8), modules[index].InitRva); +} + +void WindowsGuestStartup::Finalize(WindowsStubEmitter& code, const std::vector& modules, const std::uint32_t handles, const std::uint32_t finished) const { + code.Emit({0xb8, 1, 0, 0, 0}); + code.Rip({0x87, 0x05}, finished); + code.Emit({0x85, 0xc0}); + const auto skip = code.Branch({0x0f, 0x85}); + for (std::size_t index = modules.size(); index > 0; --index) callLifecycle(code, CheckedRva(handles + (index - 1) * 8), modules[index - 1].FiniRva); + code.PatchBranch(skip, code.GetRva()); +} + +std::uint32_t WindowsGuestStartup::EmitTlsResolver(WindowsStubEmitter& code) const { + const auto entry = code.GetRva(); + code.Emit({0x48, 0x8b, 0x07, 0x8b, 0x08, 0x65, 0x48, 0x8b, 0x04, 0x25, 0x58, 0, 0, 0, 0x48, 0x8b, 0x04, 0xc8, 0x48, 0x03, 0x47, 8, 0xc3}); + return entry; +} + +} diff --git a/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp b/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp index 0910e7aad1..a9e5e2a647 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp @@ -78,10 +78,17 @@ std::vector WindowsPePatcher::Patch(const std::vector guestPaths; + for (std::size_t index = 0; index < dynamicSection.GuestModules.size(); ++index) { + const auto& module = dynamicSection.GuestModules[index]; + guestPaths.push_back(module.Path); + for (const auto& import : module.Imports) relocations.Imports.push_back({import.Name, import.TargetRva, import.Addend, static_cast(index), import.RelocationType}); + } + libraries.insert(libraries.begin(), guestPaths.begin(), guestPaths.end()); if (dependencyDiagnostics) writeDiagnosticsImports(relocations.Imports); - auto entry = WindowsEntryStubBuilder().Build(nextRva, image.GetEntryRva(), nativeImports, libraries, relocations.Imports, runPath, lazyBinding, dependencyDiagnostics); + auto entry = WindowsEntryStubBuilder().Build(nextRva, image.GetEntryRva(), nativeImports, libraries, relocations.Imports, runPath, lazyBinding, dependencyDiagnostics, dynamicSection.GuestModules); directories[3] = entry.ExceptionDirectory; const auto entryRva = entry.Code.Rva; sections.push_back(std::move(nativeImports.Section)); diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp index ea58d81389..5721381f86 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp @@ -35,7 +35,7 @@ void patchAccess(std::vector& sections, const TlsAccess& access, cons } -PeDirectory WindowsTlsBuilder::Build(const std::vector& source, const std::vector& headers, const WindowsLoadImage& image, std::vector& sections, std::vector& relocations, std::uint32_t& nextRva) const { +PeDirectory WindowsTlsBuilder::Build(const std::vector& source, const std::vector& headers, const WindowsLoadImage& image, std::vector& sections, std::vector& relocations, std::uint32_t& nextRva, std::uint32_t* tlsIndexRva) const { const Domain::ProgramHeader* tls = nullptr; const Codegen::X64InstructionDecoder decoder; @@ -92,6 +92,7 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co PeSection data{".gtls", nextRva, SectionRead | SectionWrite | 0x40u, std::vector(templateOffset + blockSize + 16)}; const auto indexRva = nextRva + 40; + if (tlsIndexRva != nullptr) *tlsIndexRva = indexRva; const auto callbackTableRva = nextRva + 48; const auto templateRva = nextRva + templateOffset; const auto codeRva = AlignRva(nextRva + data.Data.size()); diff --git a/core/relinker/main.cpp b/core/relinker/main.cpp index 90f087e608..32b8c2b320 100644 --- a/core/relinker/main.cpp +++ b/core/relinker/main.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,8 @@ int main(const int argc, char* argv[]) { for (std::size_t index = 0; index < patch.Bytes.size(); ++index) sourceBytes[patch.Offset + index] = patch.Bytes[index]; } + const auto guestArtifacts = Relinker::GuestModuleBuilder().Build(args.inputPath, absPath, result.DynamicSection, args.toWindows, args.toIntel, args.skipSyscallCheck, args.lazyBinding, args.runPath); + if (args.writeRegistry) { const std::filesystem::path outFsPath(absPath); const std::string registryPath = (outFsPath.parent_path() / (outFsPath.stem().string() + ".registry.json")).string(); @@ -95,7 +98,13 @@ int main(const int argc, char* argv[]) { ); } - fileWriter.Write(absPath, patcher->Patch(sourceBytes, result.OriginalHeaders, result.DynamicSection, result.OriginalPltGotVaddr, args.runPath, args.lazyBinding, args.windowsDiagnostics)); + const auto executableBytes = patcher->Patch(sourceBytes, result.OriginalHeaders, result.DynamicSection, result.OriginalPltGotVaddr, args.runPath, args.lazyBinding, args.windowsDiagnostics); + for (const auto& artifact : guestArtifacts) { + std::filesystem::create_directories(artifact.Path.parent_path()); + fileWriter.Write(artifact.Path.string(), artifact.Bytes); + std::cout << "Guest module: " << artifact.Path.string() << '\n'; + } + fileWriter.Write(absPath, executableBytes); std::cout << "External prx references: " << result.RegistryEntries.size() << "\nOutput file: " << absPath << '\n'; if (args.autorun) return Cli::Autorun(absPath, args.toWindows); diff --git a/core/relinker/relinker/include/relinker/guest/GuestImage.hpp b/core/relinker/relinker/include/relinker/guest/GuestImage.hpp new file mode 100644 index 0000000000..e001c4357b --- /dev/null +++ b/core/relinker/relinker/include/relinker/guest/GuestImage.hpp @@ -0,0 +1,49 @@ +#ifndef RELINKER_GUESTIMAGE_HPP +#define RELINKER_GUESTIMAGE_HPP + +#include +#include + +namespace Relinker { + +struct GuestSymbol { + std::string Name; + std::uint8_t Info; + std::uint8_t Visibility; + std::uint16_t Section; + std::uint64_t Value; + std::uint64_t Size; +}; + +struct GuestImage { + std::filesystem::path SourcePath; + std::string OutputName; + std::vector Bytes; + std::vector Headers; + std::vector Symbols; + std::vector Dependencies; + Domain::SysVDynamicSection Dynamic; + std::uint64_t Init = 0; + std::uint64_t Fini = 0; + std::uint64_t Got = 0; + bool UsePlatformTlsResolver = true; +}; + +class GuestImageReader { +public: + GuestImage Read(const std::filesystem::path& path, std::vector bytes) const; +}; + +struct GuestArtifact { + std::filesystem::path Path; + std::vector Bytes; +}; + +class GuestModuleBuilder { +public: + std::vector Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, bool windows, bool toIntel, bool skipSyscallCheck, bool lazyBinding, const std::string& runPath) const; +}; + +} + +#endif diff --git a/core/relinker/relinker/src/guest/GuestImageReader.cpp b/core/relinker/relinker/src/guest/GuestImageReader.cpp new file mode 100644 index 0000000000..816196a26b --- /dev/null +++ b/core/relinker/relinker/src/guest/GuestImageReader.cpp @@ -0,0 +1,193 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Relinker { + +GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector bytes) const { + const auto fail = [&](const std::string& message) { throw Domain::RelinkerException(path.string() + ": " + message); }; + const auto range = [&](std::uint64_t offset, std::uint64_t size) { + if (offset > bytes.size() || size > bytes.size() - offset) fail("ELF range exceeds file"); + }; + range(0, 64); + if (Io::ReadU32(bytes, 0) != 0x464c457f || bytes[4] != 2 || bytes[5] != 1 || bytes[6] != 1 || Io::ReadU16(bytes, 18) != 62 || Io::ReadU32(bytes, 20) != 1) fail("Expected little-endian ELF64 x86-64"); + const auto type = Io::ReadU16(bytes, 16); + if (type != 3 && type != 0xfe18) fail("ELF is not a shared module"); + if (Io::ReadU16(bytes, 52) != 64 || Io::ReadU16(bytes, 54) != 56 || Io::ReadU16(bytes, 56) == 0) fail("Invalid ELF program header table"); + range(Io::ReadU64(bytes, 32), static_cast(Io::ReadU16(bytes, 56)) * 56); + GuestImage image; + image.SourcePath = path; + image.OutputName = path.filename().string() + ".guest.prx"; + image.Headers = ElfReader(bytes).ReadProgramHeaders(); + const Domain::ProgramHeader* dynamic = nullptr; + const Domain::ProgramHeader* dynlib = nullptr; + const Domain::ProgramHeader* tls = nullptr; + std::map loads; + for (const auto& header : image.Headers) { + range(header.Offset, header.FileSize); + if (header.Type == 1) { + if (header.FileSize > header.MemorySize || header.MemorySize > std::numeric_limits::max() - header.MappedAddress || (header.Flags & ~7u) != 0) fail("Invalid load segment"); + if (header.Alignment > 1 && (!std::has_single_bit(header.Alignment) || header.Offset % header.Alignment != header.MappedAddress % header.Alignment)) fail("Invalid load segment alignment"); + const auto next = loads.lower_bound(header.MappedAddress); + if ((next != loads.end() && next->first < header.MappedAddress + header.MemorySize) || (next != loads.begin() && std::prev(next)->second > header.MappedAddress)) fail("Overlapping load segments"); + loads.emplace(header.MappedAddress, header.MappedAddress + header.MemorySize); + } + if (header.Type == 2) { + if (dynamic != nullptr) fail("Duplicate dynamic segment"); + dynamic = &header; + } + if (header.Type == 0x61000000) { + if (dynlib != nullptr) fail("Duplicate SCE dynamic data"); + dynlib = &header; + } + if (header.Type == 7) { + if (tls != nullptr || header.FileSize > header.MemorySize || !std::has_single_bit(header.Alignment)) fail("Invalid TLS segment"); + tls = &header; + } + } + if (loads.empty() || dynamic == nullptr || dynamic->FileSize % 16 != 0) fail("Missing or invalid ELF load/dynamic segments"); + std::map tags; + std::vector needed; + bool terminated = false; + for (std::uint64_t offset = dynamic->Offset; offset < dynamic->Offset + dynamic->FileSize; offset += 16) { + const auto tag = Io::ReadU64(bytes, offset); + const auto value = Io::ReadU64(bytes, offset + 8); + if (tag == 0) { terminated = true; break; } + if (tag == 1) needed.push_back(value); + else if (tag < 0x60000000 || tag == 0x6100003f || (tag >= 0x61000027 && tag <= 0x6100003b)) { + if (!tags.emplace(tag, value).second) fail("Duplicate dynamic tag " + std::to_string(tag)); + } + } + if (!terminated) fail("Unterminated dynamic segment"); + for (const auto unsupported : {15u, 16u, 17u, 18u, 19u, 22u, 35u, 36u, 37u}) { + if (tags.contains(unsupported)) fail("Unsupported dynamic tag " + std::to_string(unsupported)); + } + if (tags.contains(30) && (tags.at(30) & ~8ull) != 0) fail("Unsupported guest dynamic flags"); + const auto mapped = [&](std::uint64_t address, std::uint64_t size, std::uint32_t flags) { + for (const auto& header : image.Headers) { + if (header.Type == 1 && (header.Flags & flags) == flags && address >= header.MappedAddress && address - header.MappedAddress <= header.MemorySize && size <= header.MemorySize - (address - header.MappedAddress)) return; + } + fail("Unmapped or inaccessible guest address " + std::to_string(address)); + }; + const auto translate = [&](std::uint64_t address, std::uint64_t size) { + for (const auto& header : image.Headers) { + if (header.Type == 1 && address >= header.MappedAddress && address - header.MappedAddress <= header.FileSize && size <= header.FileSize - (address - header.MappedAddress)) return header.Offset + address - header.MappedAddress; + } + fail("Unmapped ELF address " + std::to_string(address)); + return std::uint64_t{}; + }; + const auto get = [&](std::uint64_t standard, std::uint64_t sce) { + if (tags.contains(standard) == tags.contains(sce)) fail("Missing or ambiguous dynamic tag " + std::to_string(standard)); + return tags.at(tags.contains(standard) ? standard : sce); + }; + const auto table = [&](std::uint64_t standard, std::uint64_t sce, std::uint64_t size) { + const auto value = get(standard, sce); + if (tags.contains(standard)) return translate(value, size); + if (dynlib == nullptr || value > dynlib->FileSize || size > dynlib->FileSize - value) fail("SCE table exceeds dynamic data segment"); + return dynlib->Offset + value; + }; + const auto strSize = get(10, 0x61000037); + const auto strOffset = table(5, 0x61000035, strSize); + const auto string = [&](std::uint64_t offset) { + if (offset >= strSize) fail("String offset out of bounds"); + const auto start = bytes.begin() + static_cast(strOffset + offset); + const auto end = bytes.begin() + static_cast(strOffset + strSize); + const auto zero = std::find(start, end, 0); + if (zero == end) fail("Unterminated ELF string"); + return std::string(start, zero); + }; + if (strSize == 0 || bytes[strOffset] != 0 || get(11, 0x6100003b) != 24) fail("Invalid string or symbol table"); + std::uint64_t symSize = 0; + if (tags.contains(0x6100003f)) symSize = tags.at(0x6100003f); + else if (tags.contains(4)) symSize = static_cast(Io::ReadU32(bytes, translate(tags.at(4), 8) + 4)) * 24; + else fail("Missing dynamic symbol count"); + if (symSize == 0 || symSize % 24 != 0 || symSize / 24 > std::numeric_limits::max()) fail("Invalid dynamic symbol count"); + const auto symOffset = table(6, 0x61000039, symSize); + image.Dynamic.DynSymData.assign(bytes.begin() + symOffset, bytes.begin() + symOffset + symSize); + image.Dynamic.DynStrData.push_back(0); + std::set exports; + for (std::uint64_t offset = 0; offset < symSize; offset += 24) { + auto name = string(Io::ReadU32(bytes, symOffset + offset)); + name = name.substr(0, name.find('#')); + const auto info = bytes[symOffset + offset + 4]; + const auto visibility = bytes[symOffset + offset + 5]; + const auto section = Io::ReadU16(bytes, symOffset + offset + 6); + const auto value = Io::ReadU64(bytes, symOffset + offset + 8); + const auto size = Io::ReadU64(bytes, symOffset + offset + 16); + if (offset == 0 && (info != 0 || visibility != 0 || section != 0 || value != 0 || size != 0 || !name.empty())) fail("Invalid null symbol"); + if (offset == 0) { + image.Symbols.push_back({{}, 0, 0, 0, 0, 0}); + continue; + } + if (offset != 0 && ((info >> 4) > 2 || (visibility & ~3u) != 0 || ((info & 15) != 0 && (info & 15) != 1 && (info & 15) != 2 && (info & 15) != 6))) fail("Unsupported symbol attributes: " + name); + if (section != 0 && (info >> 4) != 0 && visibility != 1 && visibility != 2) { + if (name.empty() || !exports.insert(name).second) fail("Duplicate or empty export after stripping #: " + name); + } + if ((info & 15) == 6 && section != 0 && (tls == nullptr || value > tls->MemorySize || size > tls->MemorySize - value)) fail("TLS symbol exceeds TLS segment: " + name); + if (section != 0 && (info & 15) != 6) { + if (section >= 0xff00) fail("Unsupported special symbol section: " + name); + mapped(value, std::max(size, 1), (info & 15) == 2 ? 1 : 0); + } + if (image.Dynamic.DynStrData.size() > std::numeric_limits::max()) fail("String table too large"); + Io::WriteU32(image.Dynamic.DynSymData, offset, static_cast(image.Dynamic.DynStrData.size())); + Io::AppendString(image.Dynamic.DynStrData, name); + image.Symbols.push_back({name, info, visibility, section, value, size}); + } + std::set dependencies; + for (const auto offset : needed) { + const auto name = string(offset); + if (name.empty() || name.find_first_of("/\\:$") != std::string::npos || !dependencies.insert(name).second) fail("Invalid or duplicate dependency: " + name); + image.Dependencies.push_back(name); + } + std::map relocationTargets; + const auto copyRelocations = [&](std::uint64_t addressTag, std::uint64_t sceAddressTag, std::uint64_t sizeTag, std::uint64_t sceSizeTag, std::vector& output, bool plt) { + if (!tags.contains(addressTag) && !tags.contains(sceAddressTag) && !tags.contains(sizeTag) && !tags.contains(sceSizeTag)) return; + const auto size = get(sizeTag, sceSizeTag); + if (size % 24 != 0) fail("Invalid relocation table size"); + const auto offset = table(addressTag, sceAddressTag, size); + output.assign(bytes.begin() + offset, bytes.begin() + offset + size); + for (std::uint64_t index = 0; index < size; index += 24) { + const auto info = Io::ReadU64(output, index + 8); + const auto relocation = static_cast(info); + if ((info >> 32) >= image.Symbols.size() || (plt && relocation != 7) || (relocation != 1 && relocation != 6 && relocation != 7 && relocation != 8 && relocation != 16 && relocation != 17 && relocation != 18)) fail("Unsupported relocation or symbol index"); + const auto target = Io::ReadU64(output, index); + const auto addend = Io::ReadU64(output, index + 16); + mapped(target, 8, 2); + const auto next = relocationTargets.lower_bound(target); + if ((next != relocationTargets.end() && next->first < target + 8) || (next != relocationTargets.begin() && std::prev(next)->second > target)) fail("Overlapping guest relocations"); + relocationTargets.emplace(target, target + 8); + if (relocation == 8) { + if ((info >> 32) != 0) fail("RELATIVE relocation has a symbol"); + mapped(addend, 0, 0); + } else if (relocation == 1 || relocation == 6 || relocation == 7) { + const auto& symbol = image.Symbols[info >> 32]; + if ((info >> 32) == 0 || (symbol.Info & 15) == 6 || (relocation != 1 && addend != 0) || (symbol.Section == 0 && symbol.Name.empty())) fail("Invalid guest symbol relocation"); + } else { + const auto& symbol = image.Symbols[info >> 32]; + if (((info >> 32) == 0 && tls == nullptr) || ((info >> 32) != 0 && (symbol.Info & 15) != 6) || (relocation == 16 && addend != 0)) fail("Invalid guest TLS relocation"); + } + } + }; + if (tags.contains(7) || tags.contains(0x6100002f)) { + if (get(9, 0x61000033) != 24) fail("Invalid RELA entry size"); + } + copyRelocations(7, 0x6100002f, 8, 0x61000031, image.Dynamic.RelaData, false); + copyRelocations(23, 0x61000029, 2, 0x6100002d, image.Dynamic.RelaPltData, true); + if (!image.Dynamic.RelaPltData.empty() && get(20, 0x6100002b) != 7) fail("Unsupported PLT relocation format"); + if (tags.contains(3) || tags.contains(0x61000027)) image.Got = get(3, 0x61000027); + if (tags.contains(12)) image.Init = tags.at(12); + if (tags.contains(13)) image.Fini = tags.at(13); + if (image.Init != 0) mapped(image.Init, 1, 1); + if (image.Fini != 0) mapped(image.Fini, 1, 1); + if ((tags.contains(27) && tags.at(27) != 0) || (tags.contains(28) && tags.at(28) != 0)) fail("Separate guest INIT_ARRAY/FINI_ARRAY is not supported"); + if (tags.contains(33) && tags.at(33) != 0 && image.Init == 0) fail("Guest PREINIT_ARRAY has no module initializer"); + image.Bytes = std::move(bytes); + return image; +} + +} diff --git a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp new file mode 100644 index 0000000000..df86edccc8 --- /dev/null +++ b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp @@ -0,0 +1,156 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Relinker { + +std::vector GuestModuleBuilder::Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, const bool windows, const bool toIntel, const bool skipSyscallCheck, const bool lazyBinding, const std::string& runPath) const { + const auto root = std::filesystem::absolute(inputPath).parent_path(); + const auto singular = root / "sce_module"; + const auto plural = root / "sce_modules"; + const bool hasSingular = std::filesystem::exists(singular); + const bool hasPlural = std::filesystem::exists(plural); + if (hasSingular && hasPlural) throw Domain::RelinkerException("Both sce_module and sce_modules exist beside the input executable"); + if (!hasSingular && !hasPlural) { + std::cout << "WARNING: sce_module not found beside the input executable; it may be unnecessary or missing.\n"; + return {}; + } + const auto directory = hasSingular ? singular : plural; + if (!std::filesystem::is_directory(directory)) throw Domain::RelinkerException("Guest module path is not a directory: " + directory.string()); + std::vector paths; + for (const auto& entry : std::filesystem::directory_iterator(directory)) { + if (!entry.is_regular_file()) continue; + std::ifstream stream(entry.path(), std::ios::binary); + if (!stream) throw Domain::RelinkerException("Cannot read guest candidate: " + entry.path().string()); + char magic[4]{}; + stream.read(magic, 4); + if (stream.bad()) throw Domain::RelinkerException("Cannot read guest candidate magic: " + entry.path().string()); + if (stream.gcount() == 4 && static_cast(magic[0]) == 0x7f && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F') paths.push_back(entry.path()); + } + std::sort(paths.begin(), paths.end()); + if (paths.empty()) return {}; + if (lazyBinding) throw Domain::RelinkerException("Guest modules require eager binding; --lazy-binding is incompatible"); + std::vector images; + std::map exports; + std::set outputNames; + Io::FileReader reader; + for (const auto& path : paths) { + auto image = GuestImageReader().Read(path, reader.Read(path.string())); + if (image.OutputName.find_first_of("$\r\n") != std::string::npos) throw Domain::RelinkerException("Unsupported guest filename: " + image.OutputName); + std::string folded = image.OutputName; + if (windows) { + for (auto& value : folded) { + if (static_cast(value) >= 128 || value == ':' || value == '$') throw Domain::RelinkerException("Unsupported Windows guest filename: " + image.OutputName); + if (value >= 'A' && value <= 'Z') value = static_cast(value + ('a' - 'A')); + } + } + if (!outputNames.insert(folded).second) throw Domain::RelinkerException("Conflicting guest output filename: " + image.OutputName); + for (const auto& symbol : image.Symbols) { + if (symbol.Section == 0 || (symbol.Info >> 4) == 0 || symbol.Visibility == 1 || symbol.Visibility == 2) continue; + const auto [existing, inserted] = exports.emplace(symbol.Name, images.size()); + if (!inserted) throw Domain::RelinkerException("Duplicate guest export after stripping #: " + symbol.Name + " in " + images.at(existing->second).SourcePath.string() + " and " + path.string()); + } + std::vector codeHeaders; + for (const auto& header : image.Headers) if (header.Type == 1 && (header.Flags & 1) != 0) codeHeaders.push_back(header); + if (toIntel) image.Bytes = Codegen::MakeAmd64OnlyConverter()->Convert(std::move(image.Bytes), codeHeaders).Bytes; + if (!skipSyscallCheck) { + const auto scanner = MakeSyscallScanner(); + for (const auto& header : codeHeaders) { + const std::vector code(image.Bytes.begin() + header.Offset, image.Bytes.begin() + header.Offset + header.FileSize); + scanner->ScanCodeSectionForSyscalls(code, header.MappedAddress, header.FileSize); + } + } + images.push_back(std::move(image)); + } + std::vector> dependencies(images.size()); + for (auto& image : images) image.UsePlatformTlsResolver = !exports.contains("vNe1w4diLCs"); + for (std::size_t index = 0; index < images.size(); ++index) { + for (const auto& symbol : images[index].Symbols) { + if (symbol.Section != 0 || symbol.Name.empty()) continue; + const auto found = exports.find(symbol.Name); + if (found != exports.end()) { + const auto& provider = images[found->second]; + const auto exported = std::find_if(provider.Symbols.begin(), provider.Symbols.end(), [&](const auto& candidate) { return candidate.Section != 0 && candidate.Name == symbol.Name && (candidate.Info >> 4) != 0 && candidate.Visibility != 1 && candidate.Visibility != 2; }); + if (exported == provider.Symbols.end() || ((symbol.Info & 15) != 0 && (symbol.Info & 15) != (exported->Info & 15))) throw Domain::RelinkerException("Guest import/export type mismatch: " + symbol.Name); + if (found->second != index) dependencies[index].insert(found->second); + } else if (windows && (symbol.Info & 15) == 6) throw Domain::RelinkerException("Windows guest TLS import requires a guest TLS export: " + symbol.Name); + } + } + std::vector order; + std::vector states(images.size()); + const std::function visit = [&](std::size_t index) { + if (states[index] == 1) throw Domain::RelinkerException("Cyclic guest initialization dependency: " + images[index].SourcePath.string()); + if (states[index] == 2) return; + states[index] = 1; + for (const auto dependency : dependencies[index]) visit(dependency); + states[index] = 2; + order.push_back(index); + }; + for (std::size_t index = 0; index < images.size(); ++index) visit(index); + std::vector hostLibraries; + std::set uniqueHosts; + const auto addHost = [&](const std::string& name) { + if (name.empty() || name.find_first_of("/\\:$") != std::string::npos) throw Domain::RelinkerException("Invalid host dependency: " + name); + if (uniqueHosts.insert(name).second) hostLibraries.push_back(name); + }; + if (dynamic.DynamicSegmentData.size() % 16 != 0) throw Domain::RelinkerException("Invalid executable dependency table"); + for (std::size_t offset = 0; offset < dynamic.DynamicSegmentData.size(); offset += 16) { + if (Io::ReadU64(dynamic.DynamicSegmentData, offset) != 1) throw Domain::RelinkerException("Unexpected executable dependency tag"); + const auto nameOffset = Io::ReadU64(dynamic.DynamicSegmentData, offset + 8); + if (nameOffset >= dynamic.DynStrData.size()) throw Domain::RelinkerException("Invalid dependency string offset"); + const auto start = dynamic.DynStrData.begin() + nameOffset; + const auto end = std::find(start, dynamic.DynStrData.end(), 0); + if (end == dynamic.DynStrData.end()) throw Domain::RelinkerException("Unterminated dependency string"); + addHost(std::string(start, end)); + } + for (const auto& image : images) for (const auto& dependency : image.Dependencies) addHost(dependency); + dynamic.DynamicSegmentData.clear(); + const auto addNeeded = [&](const std::string& name) { + Io::AppendU64(dynamic.DynamicSegmentData, 1); + Io::AppendU64(dynamic.DynamicSegmentData, dynamic.DynStrData.size()); + Io::AppendString(dynamic.DynStrData, name); + }; + const auto relativeDirectory = directory.filename().generic_string(); + for (const auto index : order) if (!windows) addNeeded("$ORIGIN/" + relativeDirectory + "/" + images[index].OutputName); + for (const auto& name : hostLibraries) addNeeded(name); + std::string guestRunPath = runPath; + if (!windows) { + if (guestRunPath == "$ORIGIN") guestRunPath = "$ORIGIN/.."; + else if (guestRunPath.starts_with("$ORIGIN/")) guestRunPath.insert(8, "../"); + else if (!std::filesystem::path(guestRunPath).is_absolute()) throw Domain::RelinkerException("Guest Linux run path must be absolute or begin with $ORIGIN"); + } + std::vector artifacts; + const auto destination = std::filesystem::absolute(outputPath).parent_path() / directory.filename(); + if (std::filesystem::exists(destination) && std::filesystem::equivalent(destination, directory)) throw Domain::RelinkerException("Guest output directory must differ from the source module directory"); + for (const auto index : order) { + const auto& image = images[index]; + const auto target = destination / image.OutputName; + if (target.lexically_normal() == std::filesystem::absolute(outputPath).lexically_normal()) throw Domain::RelinkerException("Guest output collides with the executable output"); + for (const auto& source : paths) if (std::filesystem::exists(target) && std::filesystem::equivalent(source, target)) throw Domain::RelinkerException("Guest output would overwrite an input module: " + target.string()); + if (std::filesystem::exists(target) && std::filesystem::equivalent(inputPath, target)) throw Domain::RelinkerException("Guest output would overwrite the input executable"); + Domain::GuestRuntime runtime; + runtime.UsePlatformTlsResolver = image.UsePlatformTlsResolver; + runtime.Path = relativeDirectory + "/" + image.OutputName; + std::vector output; + if (windows) output = Elfpatcher::GuestModuleWriter().WriteWindows(image, runtime); + else { + std::vector needed; + for (const auto dependency : dependencies[index]) needed.push_back("$ORIGIN/" + images[dependency].OutputName); + needed.insert(needed.end(), hostLibraries.begin(), hostLibraries.end()); + output = Elfpatcher::GuestModuleWriter().WriteLinux(image, needed, guestRunPath); + } + dynamic.GuestModules.push_back(std::move(runtime)); + artifacts.push_back({target, std::move(output)}); + } + return artifacts; +} + +} From 0c6765e6a607c9722b49a58d58ddeac9612841b2 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 15:19:40 +0300 Subject: [PATCH 18/93] fix(relinker): ignore non-loadable SCE segments outside file bounds --- core/relinker/relinker/src/guest/GuestImageReader.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/relinker/relinker/src/guest/GuestImageReader.cpp b/core/relinker/relinker/src/guest/GuestImageReader.cpp index 816196a26b..5b3a8abc88 100644 --- a/core/relinker/relinker/src/guest/GuestImageReader.cpp +++ b/core/relinker/relinker/src/guest/GuestImageReader.cpp @@ -6,13 +6,18 @@ #include #include #include +#include namespace Relinker { GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector bytes) const { const auto fail = [&](const std::string& message) { throw Domain::RelinkerException(path.string() + ": " + message); }; const auto range = [&](std::uint64_t offset, std::uint64_t size) { - if (offset > bytes.size() || size > bytes.size() - offset) fail("ELF range exceeds file"); + if (offset > bytes.size() || size > bytes.size() - offset) { + std::ostringstream message; + message << "ELF range exceeds file: offset=0x" << std::hex << offset << ", size=0x" << size << ", fileSize=0x" << bytes.size(); + fail(message.str()); + } }; range(0, 64); if (Io::ReadU32(bytes, 0) != 0x464c457f || bytes[4] != 2 || bytes[5] != 1 || bytes[6] != 1 || Io::ReadU16(bytes, 18) != 62 || Io::ReadU32(bytes, 20) != 1) fail("Expected little-endian ELF64 x86-64"); @@ -28,7 +33,10 @@ GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector const Domain::ProgramHeader* dynlib = nullptr; const Domain::ProgramHeader* tls = nullptr; std::map loads; + constexpr std::uint32_t sceComment = 0x6fffff00; + constexpr std::uint32_t sceLibVersion = 0x6fffff01; for (const auto& header : image.Headers) { + if (header.Type == sceComment || header.Type == sceLibVersion) continue; range(header.Offset, header.FileSize); if (header.Type == 1) { if (header.FileSize > header.MemorySize || header.MemorySize > std::numeric_limits::max() - header.MappedAddress || (header.Flags & ~7u) != 0) fail("Invalid load segment"); From 6b2db1d8a8342541b0d55901994abc71dd0bdb34 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 15:33:26 +0300 Subject: [PATCH 19/93] fix(relinker): support windows guest tls relocations and accesses --- core/relinker/CMakeLists.txt | 1 + .../windows/WindowsTlsTemplateBuilder.hpp | 15 +++++++++ .../src/windows/WindowsGuestModuleWriter.cpp | 7 +++- .../src/windows/WindowsTlsBuilder.cpp | 23 ++++++++++--- .../src/windows/WindowsTlsTemplateBuilder.cpp | 32 +++++++++++++++++++ 5 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsTemplateBuilder.hpp create mode 100644 core/relinker/elfpatcher/src/windows/WindowsTlsTemplateBuilder.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 66f3d02bd8..95f6399352 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsPeFormat.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsLoadImage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsTlsBuilder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsTlsTemplateBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsRelocationBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsImportBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsTemplateBuilder.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsTemplateBuilder.hpp new file mode 100644 index 0000000000..4441a73c81 --- /dev/null +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsTemplateBuilder.hpp @@ -0,0 +1,15 @@ +#ifndef ELFPATCHER_WINDOWSTLSTEMPLATEBUILDER_HPP +#define ELFPATCHER_WINDOWSTLSTEMPLATEBUILDER_HPP + +#include + +namespace Elfpatcher::Windows { + +class WindowsTlsTemplateBuilder { +public: + std::vector Build(const Domain::ProgramHeader& tls, const WindowsLoadImage& image, const std::vector& sections, std::uint32_t templateRva, std::vector& relocations) const; +}; + +} + +#endif diff --git a/core/relinker/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp index 3e7f5be806..78b7c1d83d 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsGuestModuleWriter.cpp @@ -26,7 +26,12 @@ std::vector GuestModuleWriter::WriteWindows(const Relinker::GuestI const auto& symbol = guest.Symbols.at(symbolIndex); const auto rva = image.GetRva(target, 8); for (const auto& header : guest.Headers) { - if (header.Type == 7 && target >= header.MappedAddress && target - header.MappedAddress < header.FileSize) throw Domain::RelinkerException("Windows guest TLS template relocations require explicit support", target); + if (header.Type != 7 || header.FileSize == 0) continue; + const auto templateRva = image.GetRva(header.MappedAddress, header.FileSize); + const auto templateEnd = static_cast(templateRva) + header.FileSize; + if (rva >= templateEnd || static_cast(rva) + 8 <= templateRva) continue; + if (rva < templateRva || templateEnd - rva < 8) throw Domain::RelinkerException("Guest relocation crosses the TLS template boundary", target); + if (type == 16 || (symbolIndex != 0 && symbol.Section == 0)) throw Domain::RelinkerException("Runtime-bound guest TLS template relocation is not supported", target); } const auto next = targets.lower_bound(target); if ((next != targets.end() && next->first < target + 8) || (next != targets.begin() && std::prev(next)->second > target)) throw Domain::RelinkerException("Overlapping guest relocations", target); diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp index 5721381f86..2e0d8de4ce 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -14,6 +15,8 @@ namespace { struct TlsAccess { std::uint32_t Rva; std::size_t Length; + bool StoreImmediate; + std::uint32_t Immediate; }; void patchAccess(std::vector& sections, const TlsAccess& access, const std::uint32_t target) { @@ -63,9 +66,11 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co if (info.SegmentPrefix != 0) { const auto position = info.OpcodeOffset; - if (info.SegmentPrefix != 0x64 || info.RexPrefix != 0x48 || info.Length - position != 7 || bytes[position] != 0x8b || bytes[position + 1] != 0x04 || bytes[position + 2] != 0x25 || Io::ReadU32(source, header.Offset + offset + position + 3) != 0) + const bool loadPointer = info.Length == 9 && position == 2 && bytes[0] == 0x64 && bytes[1] == 0x48 && bytes[2] == 0x8b && bytes[3] == 0x04 && bytes[4] == 0x25 && Io::ReadU32(source, header.Offset + offset + 5) == 0; + const bool storeImmediate = info.Length == 12 && position == 1 && bytes[0] == 0x64 && bytes[1] == 0xc7 && bytes[2] == 0x04 && bytes[3] == 0x25 && Io::ReadU32(source, header.Offset + offset + 4) == 0x28; + if (!loadPointer && !storeImmediate) throw Domain::RelinkerException("Unsupported Windows guest TLS instruction", header.Offset + offset); - accesses.push_back({rva, info.Length}); + accesses.push_back({rva, info.Length, storeImmediate, storeImmediate ? Io::ReadU32(source, header.Offset + offset + 8) : 0}); } offset += info.Length; } @@ -89,7 +94,8 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co const auto alignment = std::max(tls->Alignment, 16); const auto blockSize = CheckedRva((tls->MemorySize + alignment - 1) & ~(alignment - 1)); const auto templateOffset = CheckedRva((64 + alignment - 1) & ~(alignment - 1)); - PeSection data{".gtls", nextRva, SectionRead | SectionWrite | 0x40u, std::vector(templateOffset + blockSize + 16)}; + constexpr std::uint32_t threadControlBlockSize = 0x30; + PeSection data{".gtls", nextRva, SectionRead | SectionWrite | 0x40u, std::vector(templateOffset + blockSize + threadControlBlockSize)}; const auto indexRva = nextRva + 40; if (tlsIndexRva != nullptr) *tlsIndexRva = indexRva; @@ -121,19 +127,26 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co throw Domain::RelinkerException("Branch enters a guest TLS instruction", *target); patchAccess(sections, access, code.GetRva()); code.Emit({0x48, 0x8d, 0x64, 0x24, 0x80, 0x51}); + if (access.StoreImmediate) code.Emit({0x50}); loadPointer(); + if (access.StoreImmediate) { + code.Emit({0xc7, 0x40, 0x28}); + code.U32(access.Immediate); + code.Emit({0x58}); + } code.Emit({0x59, 0x48, 0x8d, 0xa4, 0x24, 0x80, 0, 0, 0}); code.Rip({0xe9}, CheckedRva(access.Rva + access.Length)); } - std::copy_n(source.begin() + tls->Offset, tls->FileSize, data.Data.begin() + templateOffset); + const auto templateBytes = WindowsTlsTemplateBuilder().Build(*tls, image, sections, templateRva, relocations); + std::copy(templateBytes.begin(), templateBytes.end(), data.Data.begin() + templateOffset); const auto writeAddress = [&](const std::size_t offset, const std::uint32_t rva) { Io::WriteU64(data.Data, offset, ImageBase + rva); relocations.push_back(CheckedRva(data.Rva + offset)); }; writeAddress(0, templateRva); - writeAddress(8, CheckedRva(templateRva + blockSize + 16)); + writeAddress(8, CheckedRva(templateRva + blockSize + threadControlBlockSize)); writeAddress(16, indexRva); writeAddress(24, callbackTableRva); writeAddress(48, codeRva); diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsTemplateBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsTemplateBuilder.cpp new file mode 100644 index 0000000000..4ec5556f07 --- /dev/null +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsTemplateBuilder.cpp @@ -0,0 +1,32 @@ +#include +#include + +namespace Elfpatcher::Windows { + +std::vector WindowsTlsTemplateBuilder::Build(const Domain::ProgramHeader& tls, const WindowsLoadImage& image, const std::vector& sections, const std::uint32_t templateRva, std::vector& relocations) const { + if (tls.FileSize == 0) return {}; + const auto sourceRva = image.GetRva(tls.MappedAddress, tls.FileSize); + const auto sourceEnd = static_cast(sourceRva) + tls.FileSize; + std::vector result; + result.reserve(static_cast(tls.FileSize)); + while (result.size() < tls.FileSize) { + const auto currentRva = static_cast(sourceRva) + result.size(); + const auto section = std::find_if(sections.begin(), sections.end(), [&](const auto& candidate) { + return currentRva >= candidate.Rva && currentRva - candidate.Rva < candidate.Data.size(); + }); + if (section == sections.end()) throw Domain::RelinkerException("TLS template is outside PE section data", currentRva); + const auto offset = static_cast(currentRva - section->Rva); + const auto size = std::min(tls.FileSize - result.size(), section->Data.size() - offset); + result.insert(result.end(), section->Data.begin() + offset, section->Data.begin() + offset + size); + } + const auto originalCount = relocations.size(); + for (std::size_t index = 0; index < originalCount; ++index) { + const auto target = static_cast(relocations[index]); + if (target >= sourceEnd || target + 8 <= sourceRva) continue; + if (target < sourceRva || sourceEnd - target < 8) throw Domain::RelinkerException("PE relocation crosses the TLS template boundary", target); + relocations.push_back(CheckedRva(static_cast(templateRva) + target - sourceRva)); + } + return result; +} + +} From fa10c6d7ebb737dfbf19bdc08d804b3306487629 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 15:41:22 +0300 Subject: [PATCH 20/93] fix(relinker): scan reachable code for windows guest tls accesses --- core/relinker/CMakeLists.txt | 1 + .../src/windows/WindowsTlsBuilder.cpp | 6 +- .../analysis/CodeInstructionCollector.hpp | 16 ++ .../src/analysis/CodeInstructionCollector.cpp | 180 ++++++++++++++++++ 4 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 core/relinker/relinker/include/relinker/analysis/CodeInstructionCollector.hpp create mode 100644 core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 95f6399352..0e95e87f9b 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -37,6 +37,7 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CodeInstructionCollector.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/UnusedNidFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/UnusedNidFilter/StrictNidFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/UnusedNidFilter/StrictReachability.cpp diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp index 2e0d8de4ce..018059e2fd 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co std::vector accesses; std::set branchTargets; + const auto instructions = Relinker::CodeInstructionCollector().Collect(source, headers); for (const auto& header : headers) { if (header.Type == 7) { @@ -53,7 +55,8 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co } if (header.Type != 1 || (header.Flags & 1) == 0) continue; - for (std::uint64_t offset = 0; offset < header.FileSize;) { + for (auto instruction = instructions.lower_bound(header.MappedAddress); instruction != instructions.end() && *instruction - header.MappedAddress < header.FileSize; ++instruction) { + const auto offset = *instruction - header.MappedAddress; const auto* bytes = source.data() + header.Offset + offset; const auto info = decoder.DecodeInstruction(bytes, header.FileSize - offset); const auto rva = image.GetRva(header.MappedAddress + offset, info.Length); @@ -72,7 +75,6 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co throw Domain::RelinkerException("Unsupported Windows guest TLS instruction", header.Offset + offset); accesses.push_back({rva, info.Length, storeImmediate, storeImmediate ? Io::ReadU32(source, header.Offset + offset + 8) : 0}); } - offset += info.Length; } } diff --git a/core/relinker/relinker/include/relinker/analysis/CodeInstructionCollector.hpp b/core/relinker/relinker/include/relinker/analysis/CodeInstructionCollector.hpp new file mode 100644 index 0000000000..2146ab9cd6 --- /dev/null +++ b/core/relinker/relinker/include/relinker/analysis/CodeInstructionCollector.hpp @@ -0,0 +1,16 @@ +#ifndef RELINKER_CODEINSTRUCTIONCOLLECTOR_HPP +#define RELINKER_CODEINSTRUCTIONCOLLECTOR_HPP + +#include +#include + +namespace Relinker { + +class CodeInstructionCollector { +public: + std::set Collect(const std::vector& bytes, const std::vector& headers) const; +}; + +} + +#endif diff --git a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp new file mode 100644 index 0000000000..061c353786 --- /dev/null +++ b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include +#include + +namespace Relinker { + +namespace { + +class CodePointers : public UnusedNidFilter::IRelativeRelocationIndex { +public: + std::map Values; + std::optional TargetOfSlot(Domain::VirtualAddress address) const override { + const auto found = Values.find(address); + if (found == Values.end()) return std::nullopt; + return found->second; + } +}; + +} + +std::set CodeInstructionCollector::Collect(const std::vector& bytes, const std::vector& headers) const { + const auto range = [&](std::uint64_t offset, std::uint64_t size) { + if (offset > bytes.size() || size > bytes.size() - offset) throw Domain::RelinkerException("Code analysis: file range exceeds image", offset); + }; + const auto fileOffset = [&](std::uint64_t address, std::uint64_t size) { + for (const auto& header : headers) { + if (header.Type != 1 || address < header.MappedAddress || address - header.MappedAddress > header.FileSize || size > header.FileSize - (address - header.MappedAddress)) continue; + const auto offset = header.Offset + address - header.MappedAddress; + range(offset, size); + return offset; + } + throw Domain::RelinkerException("Code analysis: unmapped address", address); + }; + const auto isCode = [&](std::uint64_t address) { + for (const auto& header : headers) if (header.Type == 1 && (header.Flags & 1) != 0 && address >= header.MappedAddress && address - header.MappedAddress < header.FileSize) return true; + return false; + }; + std::map tags; + std::optional sceBase; + bool hasDynamic = false; + for (const auto& header : headers) { + if (header.Type == 0x61000000) sceBase = header.Offset; + if (header.Type != 2) continue; + if (hasDynamic || header.FileSize % 16 != 0) throw Domain::RelinkerException("Code analysis: invalid dynamic segment"); + hasDynamic = true; + range(header.Offset, header.FileSize); + bool terminated = false; + for (std::uint64_t offset = header.Offset; offset < header.Offset + header.FileSize; offset += 16) { + const auto tag = Io::ReadU64(bytes, offset); + if (tag == 0) { terminated = true; break; } + const auto value = Io::ReadU64(bytes, offset + 8); + if (tag == 1 || (tag >= 0x61000000 && tag != 0x6100003f && (tag < 0x61000027 || tag > 0x6100003b))) continue; + if (!tags.emplace(tag, value).second) throw Domain::RelinkerException("Code analysis: duplicate dynamic tag", offset); + } + if (!terminated) throw Domain::RelinkerException("Code analysis: unterminated dynamic segment"); + } + const auto value = [&](std::uint64_t standard, std::uint64_t sce) -> std::optional { + if (tags.contains(standard) && tags.contains(sce)) throw Domain::RelinkerException("Code analysis: ambiguous dynamic tag"); + if (tags.contains(standard)) return tags.at(standard); + if (tags.contains(sce)) return tags.at(sce); + return std::nullopt; + }; + const auto table = [&](std::uint64_t standard, std::uint64_t sce, std::uint64_t size) { + const auto address = value(standard, sce); + if (!address) throw Domain::RelinkerException("Code analysis: missing table"); + if (tags.contains(standard)) return fileOffset(*address, size); + if (!sceBase || *address > std::numeric_limits::max() - *sceBase) throw Domain::RelinkerException("Code analysis: invalid SCE table"); + const auto offset = *sceBase + *address; + range(offset, size); + return offset; + }; + std::set roots; + const auto addRoot = [&](std::uint64_t address) { if (isCode(address)) roots.insert(address); }; + range(0, 64); + if (const auto entry = Io::ReadU64(bytes, 24); entry != 0) addRoot(entry); + for (const auto tag : {12u, 13u}) if (const auto address = value(tag, 0x60000000 + tag); address && *address != 0) addRoot(*address); + const auto symbolBytes = value(0x6100003f, 0xffffffffffffffffull); + std::uint64_t symbolCount = 0; + if (symbolBytes) { + if (*symbolBytes % 24 != 0) throw Domain::RelinkerException("Code analysis: invalid symbol table size"); + symbolCount = *symbolBytes / 24; + } else if (tags.contains(4)) symbolCount = Io::ReadU32(bytes, fileOffset(tags.at(4), 8) + 4); + std::uint64_t symbols = 0; + if (symbolCount != 0) { + if (value(11, 0x6100003b) != 24) throw Domain::RelinkerException("Code analysis: invalid symbol entry size"); + symbols = table(6, 0x61000039, symbolCount * 24); + for (std::uint64_t index = 0; index < symbolCount; ++index) { + const auto offset = symbols + index * 24; + if ((bytes[offset + 4] & 15) == 2 && Io::ReadU16(bytes, offset + 6) != 0) addRoot(Io::ReadU64(bytes, offset + 8)); + } + } + CodePointers pointers; + std::set importSlots; + const auto readRelocations = [&](std::uint64_t addressTag, std::uint64_t sceAddressTag, std::uint64_t sizeTag, std::uint64_t sceSizeTag) { + const auto size = value(sizeTag, sceSizeTag); + if (!size) return; + if (*size % 24 != 0) throw Domain::RelinkerException("Code analysis: invalid relocation table size"); + if (*size == 0) return; + const auto start = table(addressTag, sceAddressTag, *size); + for (std::uint64_t offset = start; offset < start + *size; offset += 24) { + const auto slot = Io::ReadU64(bytes, offset); + const auto info = Io::ReadU64(bytes, offset + 8); + const auto type = static_cast(info); + const auto index = info >> 32; + const auto addend = Io::ReadU64(bytes, offset + 16); + if (type == 8) { + if (index != 0 || !pointers.Values.emplace(slot, addend).second) throw Domain::RelinkerException("Code analysis: invalid relative relocation", slot); + addRoot(addend); + } else if (type == 1 || type == 6 || type == 7) { + if (index == 0 || index >= symbolCount) throw Domain::RelinkerException("Code analysis: invalid relocation symbol", slot); + const auto symbol = symbols + index * 24; + if (Io::ReadU16(bytes, symbol + 6) == 0) importSlots.insert(slot); + else { + const auto target = Io::ReadU64(bytes, symbol + 8) + addend; + if (!pointers.Values.emplace(slot, target).second) throw Domain::RelinkerException("Code analysis: duplicate pointer relocation", slot); + addRoot(target); + } + } + } + }; + readRelocations(7, 0x6100002f, 8, 0x61000031); + readRelocations(23, 0x61000029, 2, 0x6100002d); + for (const auto& [addressTag, sizeTag] : {std::pair{25u, 27u}, std::pair{26u, 28u}, std::pair{32u, 33u}}) { + const auto address = value(addressTag, 0x60000000 + addressTag); + const auto size = value(sizeTag, 0x60000000 + sizeTag); + if (address.has_value() != size.has_value()) throw Domain::RelinkerException("Code analysis: incomplete initializer array"); + if (!size || *size == 0) continue; + if (*size % 8 != 0) throw Domain::RelinkerException("Code analysis: invalid initializer array size"); + const auto offset = fileOffset(*address, *size); + for (std::uint64_t index = 0; index < *size; index += 8) { + const auto relocated = pointers.TargetOfSlot(*address + index); + const auto target = relocated ? *relocated : Io::ReadU64(bytes, offset + index); + if (target == 0 || target == std::numeric_limits::max()) continue; + if (!isCode(target)) throw Domain::RelinkerException("Code analysis: initializer lies outside code", target); + roots.insert(target); + } + } + for (const auto& function : UnusedNidFilter::ReadExceptionFunctions(bytes, headers, pointers.Values, importSlots)) { + addRoot(function.Begin); + for (const auto target : function.ExtraTargets) addRoot(target); + } + if (roots.empty()) throw Domain::RelinkerException("Code analysis: no code entry points"); + std::set instructions; + const Codegen::X64InstructionDecoder decoder; + std::size_t previousRoots = 0; + do { + previousRoots = roots.size(); + for (const auto& header : headers) { + if (header.Type != 1 || (header.Flags & 1) == 0 || header.FileSize == 0) continue; + range(header.Offset, header.FileSize); + std::vector entries; + for (const auto root : roots) if (root >= header.MappedAddress && root - header.MappedAddress < header.FileSize) entries.push_back(root); + if (entries.empty()) continue; + const std::vector text(bytes.begin() + header.Offset, bytes.begin() + header.Offset + header.FileSize); + const auto graph = UnusedNidFilter::BuildControlFlowGraph(text, header.MappedAddress, entries.front(), entries, pointers); + for (const auto address : graph->ReachableVaddrs()) { + instructions.insert(address); + const auto offset = address - header.MappedAddress; + const auto info = decoder.DecodeInstruction(text.data() + offset, text.size() - offset); + if (info.HasBranchTarget && !info.HasRipRelativeDisp) addRoot(static_cast(static_cast(address + info.Length) + info.BranchDisp)); + } + } + } while (roots.size() != previousRoots); + std::uint64_t previousEnd = 0; + for (const auto address : instructions) { + if (address < previousEnd) throw Domain::RelinkerException("Code analysis: overlapping instruction boundaries", address); + for (const auto& header : headers) { + if (header.Type != 1 || (header.Flags & 1) == 0 || address < header.MappedAddress || address - header.MappedAddress >= header.FileSize) continue; + const auto offset = address - header.MappedAddress; + previousEnd = address + decoder.DecodeInstruction(bytes.data() + header.Offset + offset, header.FileSize - offset).Length; + break; + } + } + return instructions; +} + +} From 2cc34985c065f6c43b57edf03e34e6c925a2d670 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 15:42:55 +0300 Subject: [PATCH 21/93] fix(codegen): decode int imm8 instruction length correctly --- .../relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp | 2 ++ core/relinker/codegen/src/x86/X64InstructionDecoder.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp index 69bc3598f8..237fb8bb72 100644 --- a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp +++ b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp @@ -5,6 +5,8 @@ namespace Codegen::X64OpcodeConstants { +inline constexpr std::uint8_t OneByteIntImm8 = 0xCD; + inline constexpr std::uint8_t PrefixLock = 0xF0; inline constexpr std::uint8_t PrefixRepne = 0xF2; inline constexpr std::uint8_t PrefixRep = 0xF3; diff --git a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp index 85c42cb9cb..d574fef6c1 100644 --- a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp +++ b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp @@ -200,7 +200,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t immediateSize = ImmSize32; } - if (opcode == OneByteJmpRel8 || opcode == OneByteJrcxz || + if (opcode == OneByteJmpRel8 || opcode == OneByteJrcxz || opcode == OneByteIntImm8 || (opcode >= OneByteLoop && opcode <= OneByteLoopMax) || (opcode >= OneByteInOutImm8Min && opcode <= OneByteInOutImm8Max)) { immediateSize = ImmSize8; From 337e763597c32f17a55e6c225ea4d21f2759b4a6 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 15:46:16 +0300 Subject: [PATCH 22/93] fix(relinker): support operand-size prefixes on fs:[0] access --- .../elfpatcher/src/windows/WindowsTlsBuilder.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp index 018059e2fd..58ee3eb622 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp @@ -69,11 +69,18 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co if (info.SegmentPrefix != 0) { const auto position = info.OpcodeOffset; - const bool loadPointer = info.Length == 9 && position == 2 && bytes[0] == 0x64 && bytes[1] == 0x48 && bytes[2] == 0x8b && bytes[3] == 0x04 && bytes[4] == 0x25 && Io::ReadU32(source, header.Offset + offset + 5) == 0; - const bool storeImmediate = info.Length == 12 && position == 1 && bytes[0] == 0x64 && bytes[1] == 0xc7 && bytes[2] == 0x04 && bytes[3] == 0x25 && Io::ReadU32(source, header.Offset + offset + 4) == 0x28; + bool hasOperandSizePrefix = false; + bool supportedPrefixes = info.SegmentPrefix == 0x64; + for (std::size_t prefix = 0; prefix < position; ++prefix) { + const auto value = bytes[prefix]; + if (value == 0x66) hasOperandSizePrefix = true; + else if (value != 0x64 && !(prefix + 1 == position && value >= 0x40 && value <= 0x4f)) supportedPrefixes = false; + } + const bool loadPointer = supportedPrefixes && info.RexPrefix == 0x48 && info.Length - position == 7 && bytes[position] == 0x8b && bytes[position + 1] == 0x04 && bytes[position + 2] == 0x25 && Io::ReadU32(source, header.Offset + offset + position + 3) == 0; + const bool storeImmediate = supportedPrefixes && !hasOperandSizePrefix && (info.RexPrefix == 0 || info.RexPrefix == 0x40) && info.Length - position == 11 && bytes[position] == 0xc7 && bytes[position + 1] == 0x04 && bytes[position + 2] == 0x25 && Io::ReadU32(source, header.Offset + offset + position + 3) == 0x28; if (!loadPointer && !storeImmediate) throw Domain::RelinkerException("Unsupported Windows guest TLS instruction", header.Offset + offset); - accesses.push_back({rva, info.Length, storeImmediate, storeImmediate ? Io::ReadU32(source, header.Offset + offset + 8) : 0}); + accesses.push_back({rva, info.Length, storeImmediate, storeImmediate ? Io::ReadU32(source, header.Offset + offset + position + 7) : 0}); } } } From 0783e369da957982dc6b33f0eb0beb98971182ca Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 15:57:46 +0300 Subject: [PATCH 23/93] feat(libkernel): add definitions for standard guest libc imports --- core/libs/prx/libkernel/CMakeLists.txt | 1 + .../libkernel/System/src/PosixImportStubs.cpp | 91 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 core/libs/prx/libkernel/System/src/PosixImportStubs.cpp diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 394bad5136..965749d044 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -39,6 +39,7 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Sanitizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Rtld.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Process.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/System/src/PosixImportStubs.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/SystemConfiguration.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Signals.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Time/Time.cpp diff --git a/core/libs/prx/libkernel/System/src/PosixImportStubs.cpp b/core/libs/prx/libkernel/System/src/PosixImportStubs.cpp new file mode 100644 index 0000000000..bfb5cd1456 --- /dev/null +++ b/core/libs/prx/libkernel/System/src/PosixImportStubs.cpp @@ -0,0 +1,91 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" +#include +#include + +struct GuestSignalSet; +struct GuestResourceUsage; + +extern "C" { + +int APS5_VABI _close_nid_postfix(int descriptor) { + (void)descriptor; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +void APS5_VABI _exit_nid_postfix(int status) { + (void)status; + NotImplemented_nid_no_patch(__func__); +} + +int APS5_VABI _nanosleep_nid_postfix(const KernelTimespec* requested, KernelTimespec* remaining) { + (void)requested; + (void)remaining; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _open_nid_postfix(const char* path, int flags, ...) { + (void)path; + (void)flags; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +std::int64_t APS5_VABI _read_nid_postfix(int descriptor, void* buffer, std::size_t count) { + (void)descriptor; + (void)buffer; + (void)count; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _sigprocmask_nid_postfix(int how, const GuestSignalSet* set, GuestSignalSet* previousSet) { + (void)how; + (void)set; + (void)previousSet; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +std::int64_t APS5_VABI _write_nid_postfix(int descriptor, const void* buffer, std::size_t count) { + (void)descriptor; + (void)buffer; + (void)count; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI getrusage_nid_postfix(int who, GuestResourceUsage* usage) { + (void)who; + (void)usage; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI pthread_rwlock_rdlock_nid_postfix(PthreadRwlock* rwlock) { + (void)rwlock; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI pthread_rwlock_unlock_nid_postfix(PthreadRwlock* rwlock) { + (void)rwlock; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI rmdir_nid_postfix(const char* path) { + (void)path; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI unlink_nid_postfix(const char* path) { + (void)path; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} From 40101471eebaf99e5514f93cd313bd9ad02c7457 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 16:32:16 +0300 Subject: [PATCH 24/93] feat(libkernel): add new posix, pthread, semaphore, process funcs definitions --- .../prx/libkernel/DirectMemory/Export.cpp | 11 +++++ core/libs/prx/libkernel/File/src/Stdio.cpp | 25 ++++++++++++ core/libs/prx/libkernel/Module/src/Module.cpp | 11 +++++ .../libs/prx/libkernel/Pthread/Posix/Once.cpp | 14 +++++++ .../libs/prx/libkernel/Pthread/src/Thread.cpp | 9 +++++ .../libs/prx/libkernel/System/src/Process.cpp | 2 + core/libs/prx/libkernel/System/src/Rtld.cpp | 40 +++++++++++++++++++ .../libs/prx/libkernel/System/src/Signals.cpp | 11 +++++ 8 files changed, 123 insertions(+) diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 0ef1a9b988..986f5616bd 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -202,3 +202,14 @@ int APS5_VABI sceKernelBatchMap2(KernelBatchMapEntry* entries, int num_entries, } } + +extern "C" { + +int APS5_VABI sceKernelMlock_nid_postfix(void* address, std::uint64_t length) { + (void)address; + (void)length; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 714948a584..81704d09d1 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -187,3 +187,28 @@ int APS5_VABI sceKernelRmdir(const char* path) { } } + +extern "C" { + +int APS5_VABI sceKernelChmod_nid_postfix(const char* path, std::uint16_t mode) { + (void)path; + (void)mode; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelTruncate_nid_postfix(const char* path, std::int64_t length) { + (void)path; + (void)length; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceKernelUtimes_nid_postfix(const char* path, const KernelTimeval* times) { + (void)path; + (void)times; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libkernel/Module/src/Module.cpp b/core/libs/prx/libkernel/Module/src/Module.cpp index c34c4577a9..24a9e86a1b 100644 --- a/core/libs/prx/libkernel/Module/src/Module.cpp +++ b/core/libs/prx/libkernel/Module/src/Module.cpp @@ -52,3 +52,14 @@ int APS5_VABI sceKernelStopUnloadModule(KernelModule handle, size_t args, const } } + +extern "C" { + +int APS5_VABI __elf_phdr_match_addr_nid_postfix(ModuleInfo* module, std::uint64_t address) { + (void)module; + (void)address; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libkernel/Pthread/Posix/Once.cpp b/core/libs/prx/libkernel/Pthread/Posix/Once.cpp index bd7d8a24f7..8702a17a05 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Once.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Once.cpp @@ -1,3 +1,4 @@ +#include "prx/libc/include/General.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include #include @@ -41,3 +42,16 @@ extern "C" int APS5_VABI pthread_once_nid_postfix( onceChanged.notify_all(); return 0; } + +using OnceRoutine = void (APS5_VABI *)(); + +extern "C" { + +int APS5_VABI scePthreadOnce_nid_postfix(void* onceControl, OnceRoutine initRoutine) { + (void)onceControl; + (void)initRoutine; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index b594ba7a15..7f4f857dd1 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -284,3 +284,12 @@ int APS5_VABI scePthreadSetprio(Pthread thread, int prio) { } } + +extern "C" { + +void APS5_VABI __pthread_cxa_finalize_nid_postfix(void* argument) { + (void)argument; + NotImplemented_nid_no_patch(__func__); +} + +} diff --git a/core/libs/prx/libkernel/System/src/Process.cpp b/core/libs/prx/libkernel/System/src/Process.cpp index 821e2cff34..0e235df6dc 100644 --- a/core/libs/prx/libkernel/System/src/Process.cpp +++ b/core/libs/prx/libkernel/System/src/Process.cpp @@ -6,6 +6,8 @@ extern "C" { +char* __progname_nid_postfix = nullptr; + int APS5_VABI getargc_nid_postfix(void) { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libkernel/System/src/Rtld.cpp b/core/libs/prx/libkernel/System/src/Rtld.cpp index e8b44eae9e..0d131700b1 100644 --- a/core/libs/prx/libkernel/System/src/Rtld.cpp +++ b/core/libs/prx/libkernel/System/src/Rtld.cpp @@ -38,3 +38,43 @@ void APS5_VABI sceKernelSetThreadDtors(thread_dtors_func_t dtors) { } } + +using ThreadAtexitCount = int (APS5_VABI *)(KernelModule module); +using ThreadAtexitReport = void (APS5_VABI *)(KernelModule module); +using ThreadDtors = void (APS5_VABI *)(); + +extern "C" { + +void APS5_VABI _sceKernelRtldSetApplicationHeapAPI_nid_postfix(void* api[]) { + (void)api; + NotImplemented_nid_no_patch(__func__); +} + +int APS5_VABI _sceKernelRtldThreadAtexitDecrement_nid_postfix(std::uint64_t* counter) { + (void)counter; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _sceKernelRtldThreadAtexitIncrement_nid_postfix(std::uint64_t* counter) { + (void)counter; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +void APS5_VABI _sceKernelSetThreadAtexitCount_nid_postfix(ThreadAtexitCount callback) { + (void)callback; + NotImplemented_nid_no_patch(__func__); +} + +void APS5_VABI _sceKernelSetThreadAtexitReport_nid_postfix(ThreadAtexitReport callback) { + (void)callback; + NotImplemented_nid_no_patch(__func__); +} + +void APS5_VABI _sceKernelSetThreadDtors_nid_postfix(ThreadDtors callback) { + (void)callback; + NotImplemented_nid_no_patch(__func__); +} + +} diff --git a/core/libs/prx/libkernel/System/src/Signals.cpp b/core/libs/prx/libkernel/System/src/Signals.cpp index c662d48583..713585c601 100644 --- a/core/libs/prx/libkernel/System/src/Signals.cpp +++ b/core/libs/prx/libkernel/System/src/Signals.cpp @@ -1,3 +1,4 @@ +#include "prx/libc/include/General.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include #include @@ -58,3 +59,13 @@ int APS5_VABI raise_nid_postfix(int guest) { return result ? -1 : 0; } } + +extern "C" { + +int APS5_VABI _is_signal_return_nid_postfix(std::uint64_t programCounter) { + (void)programCounter; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} From 5887c8d6e207a9c0dbcf3e38a1432d7f485ab2b5 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 16:42:54 +0300 Subject: [PATCH 25/93] feat(libkernel): add memory segment, backtrace, heap error report definitions --- core/libs/prx/libc/CMakeLists.txt | 1 + core/libs/prx/libc/src/HeapDiagnostics.cpp | 20 +++++++++++++++++++ core/libs/prx/libkernel/Module/src/Module.cpp | 7 +++++++ .../libs/prx/libkernel/System/src/Process.cpp | 1 + docs/TechnicalDebt.md | 18 ++++++++++------- 5 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 core/libs/prx/libc/src/HeapDiagnostics.cpp diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index c81a2711a7..872c317cd6 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -14,6 +14,7 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestMemoryBacking.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestHeapStorage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestHeap.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/HeapDiagnostics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Mspace.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ApplicationHeap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ApplicationProcessParameters.cpp diff --git a/core/libs/prx/libc/src/HeapDiagnostics.cpp b/core/libs/prx/libc/src/HeapDiagnostics.cpp new file mode 100644 index 0000000000..fc9dd761fe --- /dev/null +++ b/core/libs/prx/libc/src/HeapDiagnostics.cpp @@ -0,0 +1,20 @@ +#include "prx/libc/include/General.hpp" +#include + +extern "C" { + +// unknown signature +void APS5_VABI sceLibcInternalBacktraceForGame_nid_postfix(const char* heapName) { + (void)heapName; + NotImplemented_nid_no_patch(__func__); +} + +// unknown signature +void APS5_VABI sceLibcInternalHeapErrorReportForGame_nid_postfix(void* heap, void* block, std::uint32_t error) { + (void)heap; + (void)block; + (void)error; + NotImplemented_nid_no_patch(__func__); +} + +} diff --git a/core/libs/prx/libkernel/Module/src/Module.cpp b/core/libs/prx/libkernel/Module/src/Module.cpp index 24a9e86a1b..4b0f12136d 100644 --- a/core/libs/prx/libkernel/Module/src/Module.cpp +++ b/core/libs/prx/libkernel/Module/src/Module.cpp @@ -62,4 +62,11 @@ int APS5_VABI __elf_phdr_match_addr_nid_postfix(ModuleInfo* module, std::uint64_ return 0; } +// unknown signature +std::int32_t APS5_VABI sceKernelInternalMemoryGetModuleSegmentInfo_nid_postfix(void* result) { + (void)result; + NotImplemented_nid_no_patch(__func__); + return 0; +} + } diff --git a/core/libs/prx/libkernel/System/src/Process.cpp b/core/libs/prx/libkernel/System/src/Process.cpp index 0e235df6dc..89dca7b711 100644 --- a/core/libs/prx/libkernel/System/src/Process.cpp +++ b/core/libs/prx/libkernel/System/src/Process.cpp @@ -6,6 +6,7 @@ extern "C" { +// unknown signature char* __progname_nid_postfix = nullptr; int APS5_VABI getargc_nid_postfix(void) { diff --git a/docs/TechnicalDebt.md b/docs/TechnicalDebt.md index 922c79b780..35713ad195 100644 --- a/docs/TechnicalDebt.md +++ b/docs/TechnicalDebt.md @@ -15,14 +15,18 @@ Throughout the project, every function at every stage either **does exactly what - [libSceCommonDialog](../core/libs/prx/libSceCommonDialog/Export.cpp) - The shader recompiler [skips baryctric coordinates](../core/shader/recompiler/Recompiler.cpp) (is not even passed to SpirvTargetOptions at row 212). -### Unknown function names +### Unknown function info -- [zARR5aCmkoY](../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - unknown signature -- [qj7QZpgr9Uw](../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) -- [fd5Bp5tGTgo](../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) -- [dolOmWH+huQ](../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) -- [V++UgBtQhn0](../core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp) (libSceAgc) -- [gQkqkLttcpw](../core/libs/prx/libSceAgc/Acb/src/Control.cpp) (libSceAgc) - unknown signature +- [zARR5aCmkoY](../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - unknown name, signature +- [qj7QZpgr9Uw](../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) - unknown name +- [fd5Bp5tGTgo](../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - unknown name +- [dolOmWH+huQ](../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - unknown name +- [V++UgBtQhn0](../core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp) (libSceAgc) - unknown name +- [gQkqkLttcpw](../core/libs/prx/libSceAgc/Acb/src/Control.cpp) (libSceAgc) - unknown name, signature +- [sceKernelInternalMemoryGetModuleSegmentInfo](../core/libs/prx/libkernel/Module/src/Module.cpp) (libkernel) - unknown signature +- [sceLibcInternalBacktraceForGame](../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - unknown signature +- [sceLibcInternalHeapErrorReportForGame](../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - unknown signature +- [__progname](../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - unknown data export ### Functional From 630cb018c077bd268f0c46fe643ad0896a46c48c Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 16:45:41 +0300 Subject: [PATCH 26/93] feat(relinker): add --skip-sce-module flag --- core/relinker/cli/include/Cli.hpp | 1 + core/relinker/cli/src/CliArgs.cpp | 4 +++- core/relinker/main.cpp | 7 ++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/relinker/cli/include/Cli.hpp b/core/relinker/cli/include/Cli.hpp index 369cb7f60d..63c1bd38b5 100644 --- a/core/relinker/cli/include/Cli.hpp +++ b/core/relinker/cli/include/Cli.hpp @@ -8,6 +8,7 @@ namespace Cli { struct Args { bool skipSyscallCheck = false; + bool skipSceModule = false; bool toIntel = false; bool writeRegistry = false; bool toWindows = false; diff --git a/core/relinker/cli/src/CliArgs.cpp b/core/relinker/cli/src/CliArgs.cpp index 12b1631f03..7629f68028 100644 --- a/core/relinker/cli/src/CliArgs.cpp +++ b/core/relinker/cli/src/CliArgs.cpp @@ -13,6 +13,8 @@ Args ParseArgs(int argc, char* argv[]) { const std::string arg = argv[i]; if (arg == "--skip-syscall-check") { args.skipSyscallCheck = true; + } else if (arg == "--skip-sce-module") { + args.skipSceModule = true; } else if (arg == "--to-intel") { args.toIntel = true; } else if (arg.rfind("unused-filter=", 0) == 0) { @@ -51,7 +53,7 @@ Args ParseArgs(int argc, char* argv[]) { if (args.inputPath.empty() || args.outputPath.empty()) throw std::runtime_error( - "Usage: relinker [--windows] [--windows-diagnostics] [--skip-syscall-check] [--to-intel] [unused-filter=0|1|2] [--registry] [--rpath ] [--lazy-binding] [--autorun] \n" + "Usage: relinker [--windows] [--windows-diagnostics] [--skip-syscall-check] [--skip-sce-module] [--to-intel] [unused-filter=0|1|2] [--registry] [--rpath ] [--lazy-binding] [--autorun] \n" "Example: relinker input.elf output.elf" ); diff --git a/core/relinker/main.cpp b/core/relinker/main.cpp index 32b8c2b320..1c530c64e5 100644 --- a/core/relinker/main.cpp +++ b/core/relinker/main.cpp @@ -23,6 +23,7 @@ #include #include #include +#include int main(const int argc, char* argv[]) { Cli::Args args; @@ -66,6 +67,7 @@ int main(const int argc, char* argv[]) { ); std::cout << "System: " << (args.toWindows ? "Windows" : "Linux") << "; unused-filter=" << args.unusedFilterLevel << "\n"; + std::cout << "sce_module/sce_modules processing: " << (args.skipSceModule ? "disabled (--skip-sce-module)" : "enabled") << '\n'; auto result = pipeline->Relink(sourceBytes); for (const auto& patch : result.Patches) { if (patch.Offset > sourceBytes.size() || patch.Bytes.size() > sourceBytes.size() - patch.Offset) @@ -73,7 +75,10 @@ int main(const int argc, char* argv[]) { for (std::size_t index = 0; index < patch.Bytes.size(); ++index) sourceBytes[patch.Offset + index] = patch.Bytes[index]; } - const auto guestArtifacts = Relinker::GuestModuleBuilder().Build(args.inputPath, absPath, result.DynamicSection, args.toWindows, args.toIntel, args.skipSyscallCheck, args.lazyBinding, args.runPath); + std::vector guestArtifacts; + if (!args.skipSceModule) { + guestArtifacts = Relinker::GuestModuleBuilder().Build(args.inputPath, absPath, result.DynamicSection, args.toWindows, args.toIntel, args.skipSyscallCheck, args.lazyBinding, args.runPath); + } if (args.writeRegistry) { const std::filesystem::path outFsPath(absPath); From 2e654e4f895c4b35e46aa40eec7c3628ef46bbac Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 16:54:43 +0300 Subject: [PATCH 27/93] feat(relinker): scan sce modules for syscalls --- core/relinker/main.cpp | 5 +++-- .../relinker/include/relinker/guest/GuestImage.hpp | 3 ++- .../relinker/src/guest/GuestModuleBuilder.cpp | 12 ++++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/core/relinker/main.cpp b/core/relinker/main.cpp index 1c530c64e5..64e3994e13 100644 --- a/core/relinker/main.cpp +++ b/core/relinker/main.cpp @@ -55,10 +55,11 @@ int main(const int argc, char* argv[]) { } auto elfReader = std::make_shared(sourceBytes); + const std::shared_ptr syscallScanner = args.skipSyscallCheck ? Relinker::MakeNullSyscallScanner() : Relinker::MakeSyscallScanner(); const auto pipeline = std::make_shared( elfReader, - args.skipSyscallCheck ? Relinker::MakeNullSyscallScanner() : Relinker::MakeSyscallScanner(), + syscallScanner, Relinker::MakeCallSiteResolver(), std::make_shared(), std::make_shared(), @@ -77,7 +78,7 @@ int main(const int argc, char* argv[]) { std::vector guestArtifacts; if (!args.skipSceModule) { - guestArtifacts = Relinker::GuestModuleBuilder().Build(args.inputPath, absPath, result.DynamicSection, args.toWindows, args.toIntel, args.skipSyscallCheck, args.lazyBinding, args.runPath); + guestArtifacts = Relinker::GuestModuleBuilder().Build(args.inputPath, absPath, result.DynamicSection, args.toWindows, args.toIntel, *syscallScanner, args.lazyBinding, args.runPath); } if (args.writeRegistry) { diff --git a/core/relinker/relinker/include/relinker/guest/GuestImage.hpp b/core/relinker/relinker/include/relinker/guest/GuestImage.hpp index e001c4357b..a71a332893 100644 --- a/core/relinker/relinker/include/relinker/guest/GuestImage.hpp +++ b/core/relinker/relinker/include/relinker/guest/GuestImage.hpp @@ -2,6 +2,7 @@ #define RELINKER_GUESTIMAGE_HPP #include +#include #include namespace Relinker { @@ -41,7 +42,7 @@ struct GuestArtifact { class GuestModuleBuilder { public: - std::vector Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, bool windows, bool toIntel, bool skipSyscallCheck, bool lazyBinding, const std::string& runPath) const; + std::vector Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, bool windows, bool toIntel, ISyscallScanner& syscallScanner, bool lazyBinding, const std::string& runPath) const; }; } diff --git a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp index df86edccc8..c461ffb3c9 100644 --- a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp +++ b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -12,7 +11,7 @@ namespace Relinker { -std::vector GuestModuleBuilder::Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, const bool windows, const bool toIntel, const bool skipSyscallCheck, const bool lazyBinding, const std::string& runPath) const { +std::vector GuestModuleBuilder::Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, const bool windows, const bool toIntel, ISyscallScanner& syscallScanner, const bool lazyBinding, const std::string& runPath) const { const auto root = std::filesystem::absolute(inputPath).parent_path(); const auto singular = root / "sce_module"; const auto plural = root / "sce_modules"; @@ -61,12 +60,9 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path std::vector codeHeaders; for (const auto& header : image.Headers) if (header.Type == 1 && (header.Flags & 1) != 0) codeHeaders.push_back(header); if (toIntel) image.Bytes = Codegen::MakeAmd64OnlyConverter()->Convert(std::move(image.Bytes), codeHeaders).Bytes; - if (!skipSyscallCheck) { - const auto scanner = MakeSyscallScanner(); - for (const auto& header : codeHeaders) { - const std::vector code(image.Bytes.begin() + header.Offset, image.Bytes.begin() + header.Offset + header.FileSize); - scanner->ScanCodeSectionForSyscalls(code, header.MappedAddress, header.FileSize); - } + for (const auto& header : codeHeaders) { + const std::vector code(image.Bytes.begin() + header.Offset, image.Bytes.begin() + header.Offset + header.FileSize); + syscallScanner.ScanCodeSectionForSyscalls(code, header.MappedAddress, header.FileSize); } images.push_back(std::move(image)); } From 55e1c4a23872c0cbb861a841db5efe6ce62fd658 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 17:07:34 +0300 Subject: [PATCH 28/93] feat(libkernel): implement application heap and thread atexit registration --- core/libs/SceTypes.hpp | 6 +- .../prx/libkernel/Pthread/Posix/Thread.cpp | 4 +- .../Pthread/include/ThreadLifecycle.hpp | 15 +++++ .../libs/prx/libkernel/Pthread/src/Thread.cpp | 56 +++++++++++++++---- core/libs/prx/libkernel/System/src/Rtld.cpp | 32 ++++------- 5 files changed, 76 insertions(+), 37 deletions(-) create mode 100644 core/libs/prx/libkernel/Pthread/include/ThreadLifecycle.hpp diff --git a/core/libs/SceTypes.hpp b/core/libs/SceTypes.hpp index 3b149bce37..51b0a9cafc 100644 --- a/core/libs/SceTypes.hpp +++ b/core/libs/SceTypes.hpp @@ -173,9 +173,9 @@ using PthreadCondattr = PthreadCondattrPrivate*; using PthreadKey = int; using pthread_entry_func_t = void* (*)(void*); using pthread_key_destructor_func_t = void (*)(void*); -using thread_dtors_func_t = void (*)(); -using get_thread_atexit_count_func_t = int (*)(KernelModule); -using thread_atexit_report_func_t = void (*)(KernelModule); +using thread_dtors_func_t = void (APS5_VABI *)(); +using get_thread_atexit_count_func_t = int (APS5_VABI *)(KernelModule); +using thread_atexit_report_func_t = void (APS5_VABI *)(KernelModule); struct FileStat { std::uint32_t st_dev; diff --git a/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp b/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp index da9083fc49..3b585b5834 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp @@ -1,6 +1,7 @@ #include #include #include "SceTypes.hpp" +#include "../include/ThreadLifecycle.hpp" #include "prx/libc/include/General.hpp" @@ -32,8 +33,7 @@ int APS5_VABI pthread_detach_nid_postfix(Pthread thread) { } void APS5_VABI pthread_exit_nid_postfix(void* value) { - (void)value; - NotImplemented_nid_no_patch(__func__); + scePthreadExit(value); } int APS5_VABI pthread_getschedparam_nid_postfix(Pthread thread, int* policy, KernelSchedParam* param) { diff --git a/core/libs/prx/libkernel/Pthread/include/ThreadLifecycle.hpp b/core/libs/prx/libkernel/Pthread/include/ThreadLifecycle.hpp new file mode 100644 index 0000000000..f342bb0ca0 --- /dev/null +++ b/core/libs/prx/libkernel/Pthread/include/ThreadLifecycle.hpp @@ -0,0 +1,15 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_PTHREAD_THREADLIFECYCLE_HPP +#define CORE_LIBS_PRX_LIBKERNEL_PTHREAD_THREADLIFECYCLE_HPP + +#include "SceTypes.hpp" + +class ThreadLifecycle { +public: + static void SetThreadDtors(thread_dtors_func_t callback); + static void SetThreadAtexitCount(get_thread_atexit_count_func_t callback); + static void SetThreadAtexitReport(thread_atexit_report_func_t callback); +}; + +extern "C" void APS5_VABI scePthreadExit(void* retval); + +#endif diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index 7f4f857dd1..6be2a0565e 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -1,4 +1,5 @@ #include "../include/Pthread.hpp" +#include "../include/ThreadLifecycle.hpp" #include "prx/libc/include/General.hpp" #include #include @@ -29,7 +30,44 @@ struct ThreadArgs { PthreadPrivate* self; }; -static void FinishThread(PthreadPrivate* self, void* retval) { +static std::atomic threadDtors{nullptr}; +static std::atomic threadAtexitCount{nullptr}; +static std::atomic threadAtexitReport{nullptr}; +static thread_local PthreadPrivate* currentThread = nullptr; +static thread_local bool threadFinishing = false; + +void ThreadLifecycle::SetThreadDtors(thread_dtors_func_t callback) { + if (!callback) + throw std::runtime_error("Thread destructor callback is null"); + thread_dtors_func_t expected = nullptr; + if (!threadDtors.compare_exchange_strong(expected, callback)) + throw std::runtime_error("Thread destructor callback is already registered"); +} + +void ThreadLifecycle::SetThreadAtexitCount(get_thread_atexit_count_func_t callback) { + if (!callback) + throw std::runtime_error("Thread atexit count callback is null"); + get_thread_atexit_count_func_t expected = nullptr; + if (!threadAtexitCount.compare_exchange_strong(expected, callback)) + throw std::runtime_error("Thread atexit count callback is already registered"); +} + +void ThreadLifecycle::SetThreadAtexitReport(thread_atexit_report_func_t callback) { + if (!callback) + throw std::runtime_error("Thread atexit report callback is null"); + thread_atexit_report_func_t expected = nullptr; + if (!threadAtexitReport.compare_exchange_strong(expected, callback)) + throw std::runtime_error("Thread atexit report callback is already registered"); +} + +static void finishThread(PthreadPrivate* self, void* retval) { + if (!self || self != currentThread) + throw std::runtime_error("Finishing an unregistered guest thread"); + if (threadFinishing) + throw std::runtime_error("Guest thread is already finishing"); + threadFinishing = true; + if (const auto callback = threadDtors.load()) + callback(); { std::unique_lock lk(self->_join_mtx); self->_retval = retval; @@ -43,13 +81,13 @@ static void RunThread(std::unique_ptr args) { const auto entry = args->entry; void* arg = args->arg; PthreadPrivate* self = args->self; + currentThread = self; args.reset(); - FinishThread(self, entry(arg)); + finishThread(self, entry(arg)); + currentThread = nullptr; } #ifdef _WIN32 -static thread_local PthreadPrivate* currentThread = nullptr; - static void ReleaseThread(PthreadPrivate* thread) { if (thread->references.fetch_sub(1, std::memory_order_acq_rel) != 1) return; @@ -183,26 +221,22 @@ int APS5_VABI scePthreadDetach(Pthread thread) { } void APS5_VABI scePthreadExit(void* retval) { -#ifdef _WIN32 if (!currentThread) throw std::runtime_error("scePthreadExit: current thread is not registered"); auto* self = currentThread; - FinishThread(self, retval); + finishThread(self, retval); currentThread = nullptr; +#ifdef _WIN32 ReleaseThread(self); _endthreadex(0); #else pthread_exit(retval); #endif - __builtin_unreachable(); + throw std::runtime_error("Native thread exit returned"); } Pthread APS5_VABI scePthreadSelf() { -#ifdef _WIN32 return currentThread; -#else - return nullptr; -#endif } void APS5_VABI scePthreadYield() { diff --git a/core/libs/prx/libkernel/System/src/Rtld.cpp b/core/libs/prx/libkernel/System/src/Rtld.cpp index 0d131700b1..e47bf5b13c 100644 --- a/core/libs/prx/libkernel/System/src/Rtld.cpp +++ b/core/libs/prx/libkernel/System/src/Rtld.cpp @@ -3,6 +3,7 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libkernel/Pthread/include/ThreadLifecycle.hpp" extern "C" { @@ -23,31 +24,23 @@ int APS5_VABI sceKernelRtldThreadAtexitIncrement(uint64_t* c) { } void APS5_VABI sceKernelSetThreadAtexitCount(get_thread_atexit_count_func_t func) { - (void)func; - NotImplemented_nid_no_patch(__func__); + ThreadLifecycle::SetThreadAtexitCount(func); } void APS5_VABI sceKernelSetThreadAtexitReport(thread_atexit_report_func_t func) { - (void)func; - NotImplemented_nid_no_patch(__func__); + ThreadLifecycle::SetThreadAtexitReport(func); } void APS5_VABI sceKernelSetThreadDtors(thread_dtors_func_t dtors) { - (void)dtors; - NotImplemented_nid_no_patch(__func__); + ThreadLifecycle::SetThreadDtors(dtors); } } -using ThreadAtexitCount = int (APS5_VABI *)(KernelModule module); -using ThreadAtexitReport = void (APS5_VABI *)(KernelModule module); -using ThreadDtors = void (APS5_VABI *)(); - extern "C" { void APS5_VABI _sceKernelRtldSetApplicationHeapAPI_nid_postfix(void* api[]) { - (void)api; - NotImplemented_nid_no_patch(__func__); + ApplicationHeapRegister_nid_no_patch(api); } int APS5_VABI _sceKernelRtldThreadAtexitDecrement_nid_postfix(std::uint64_t* counter) { @@ -62,19 +55,16 @@ int APS5_VABI _sceKernelRtldThreadAtexitIncrement_nid_postfix(std::uint64_t* cou return 0; } -void APS5_VABI _sceKernelSetThreadAtexitCount_nid_postfix(ThreadAtexitCount callback) { - (void)callback; - NotImplemented_nid_no_patch(__func__); +void APS5_VABI _sceKernelSetThreadAtexitCount_nid_postfix(get_thread_atexit_count_func_t callback) { + ThreadLifecycle::SetThreadAtexitCount(callback); } -void APS5_VABI _sceKernelSetThreadAtexitReport_nid_postfix(ThreadAtexitReport callback) { - (void)callback; - NotImplemented_nid_no_patch(__func__); +void APS5_VABI _sceKernelSetThreadAtexitReport_nid_postfix(thread_atexit_report_func_t callback) { + ThreadLifecycle::SetThreadAtexitReport(callback); } -void APS5_VABI _sceKernelSetThreadDtors_nid_postfix(ThreadDtors callback) { - (void)callback; - NotImplemented_nid_no_patch(__func__); +void APS5_VABI _sceKernelSetThreadDtors_nid_postfix(thread_dtors_func_t callback) { + ThreadLifecycle::SetThreadDtors(callback); } } From dbb56adcf669093d4c47671c2f5e750ca789e7d2 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 17:24:28 +0300 Subject: [PATCH 29/93] feat(libkernel): implement process, cpu, uuid, gpi, gpo, sync funcs --- core/libs/prx/libc/include/Shutdown.hpp | 1 + core/libs/prx/libc/src/Process.cpp | 6 +- .../libs/prx/libkernel/System/src/Process.cpp | 212 +++++++++++++++--- 3 files changed, 181 insertions(+), 38 deletions(-) diff --git a/core/libs/prx/libc/include/Shutdown.hpp b/core/libs/prx/libc/include/Shutdown.hpp index 048505884f..c1ff166049 100644 --- a/core/libs/prx/libc/include/Shutdown.hpp +++ b/core/libs/prx/libc/include/Shutdown.hpp @@ -3,5 +3,6 @@ extern "C" void LibcRegisterShutdown_nid_postfix(void (*callback)()); extern "C" void LibcRunShutdown_nid_postfix(); +extern "C" [[noreturn]] void LibcExit_nid_no_patch(int code); #endif diff --git a/core/libs/prx/libc/src/Process.cpp b/core/libs/prx/libc/src/Process.cpp index 01b63f7694..dad4e628e6 100644 --- a/core/libs/prx/libc/src/Process.cpp +++ b/core/libs/prx/libc/src/Process.cpp @@ -61,11 +61,15 @@ extern "C" { std::_Exit(code); } -void APS5_VABI exit_nid_postfix(int code) { +[[noreturn]] void LibcExit_nid_no_patch(int code) { LibcRunShutdown_nid_postfix(); std::exit(code); } +void APS5_VABI exit_nid_postfix(int code) { + LibcExit_nid_no_patch(code); +} + [[noreturn]] void abort_nid_postfix( uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5 diff --git a/core/libs/prx/libkernel/System/src/Process.cpp b/core/libs/prx/libkernel/System/src/Process.cpp index 89dca7b711..4fa1e7ea97 100644 --- a/core/libs/prx/libkernel/System/src/Process.cpp +++ b/core/libs/prx/libkernel/System/src/Process.cpp @@ -1,8 +1,120 @@ -#include -#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#include +#endif + #include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" +#include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libc/include/Shutdown.hpp" #include "prx/libkernel/DirectMemory/DirectMemory.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::atomic gpoBits{0}; +constexpr std::array openPsId{'A', 'n', 'y', 'P', 'S', '5', 'O', 'p', 'e', 'n', 'P', 's', 'I', 'd', 0, 1}; + +class ProcessArguments { +public: + ProcessArguments() { +#ifdef _WIN32 + std::array path{}; + const auto size = GetModuleFileNameA(nullptr, path.data(), static_cast(path.size())); + if (size == 0) + throw std::system_error(GetLastError(), std::system_category(), "Reading executable path"); + if (size >= path.size()) + throw std::runtime_error("Executable path exceeds the guest argument buffer"); + arguments.emplace_back(path.data(), size); +#else + std::ifstream stream("/proc/self/cmdline", std::ios::binary); + if (!stream) + throw std::runtime_error("Cannot read process arguments"); + std::string argument; + while (std::getline(stream, argument, '\0')) { + if (stream.eof()) + throw std::runtime_error("Unterminated process argument"); + arguments.push_back(argument); + } + if (stream.bad() || !stream.eof()) + throw std::runtime_error("Reading process arguments failed"); +#endif + if (arguments.empty() || arguments.size() > static_cast(std::numeric_limits::max())) + throw std::runtime_error("Invalid process argument count"); + for (const auto& argument : arguments) + pointers.push_back(argument.c_str()); + pointers.push_back(nullptr); + } + + int GetCount() const { return static_cast(arguments.size()); } + const char** GetValues() { return pointers.data(); } + +private: + std::vector arguments; + std::vector pointers; +}; + +ProcessArguments& getProcessArguments() { + static ProcessArguments arguments; + return arguments; +} + +void validateSchedulingPolicy(int policy) { + if (policy != 1 && policy != 3) + throw std::invalid_argument("Unsupported guest scheduling policy"); +} + +#ifdef _WIN32 +void syncVolumes() { + std::array name{}; + const auto first = FindFirstVolumeW(name.data(), static_cast(name.size())); + if (first == INVALID_HANDLE_VALUE) + throw std::system_error(GetLastError(), std::system_category(), "Enumerating volumes for sync"); + const std::unique_ptr search(first, &FindVolumeClose); + for (;;) { + DWORD flags = 0; + if (!GetVolumeInformationW(name.data(), nullptr, 0, nullptr, nullptr, &flags, nullptr, 0)) + throw std::system_error(GetLastError(), std::system_category(), "Reading volume properties for sync"); + if ((flags & FILE_READ_ONLY_VOLUME) == 0) { + std::wstring path(name.data()); + if (path.empty() || path.back() != L'\\') + throw std::runtime_error("Invalid volume path for sync"); + path.pop_back(); + const auto native = CreateFileW(path.c_str(), GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr); + if (native == INVALID_HANDLE_VALUE) + throw std::system_error(GetLastError(), std::system_category(), "Opening volume for sync"); + const std::unique_ptr volume(native, &CloseHandle); + if (!FlushFileBuffers(volume.get())) + throw std::system_error(GetLastError(), std::system_category(), "Flushing volume"); + } + if (FindNextVolumeW(search.get(), name.data(), static_cast(name.size()))) + continue; + const auto error = GetLastError(); + if (error != ERROR_NO_MORE_FILES) + throw std::system_error(error, std::system_category(), "Enumerating volumes for sync"); + break; + } +} +#endif + +} extern "C" { @@ -10,75 +122,101 @@ extern "C" { char* __progname_nid_postfix = nullptr; int APS5_VABI getargc_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return getProcessArguments().GetCount(); } const char** APS5_VABI getargv_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return nullptr; + return getProcessArguments().GetValues(); } int APS5_VABI getpagesize_nid_postfix(void) { - return PS5_PAGE_SIZE; + return PS5_PAGE_SIZE; } int APS5_VABI getpid_nid_postfix(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +#ifdef _WIN32 + const auto pid = GetCurrentProcessId(); +#else + const auto pid = ::getpid(); +#endif + if (pid == 0 || static_cast(pid) > static_cast(std::numeric_limits::max())) + throw std::runtime_error("Process identifier is outside the guest range"); + return static_cast(pid); } void APS5_VABI exit_nid_postfix(int code) { - (void)code; - NotImplemented_nid_no_patch(__func__); + LibcExit_nid_no_patch(code); } int APS5_VABI sceKernelGetCurrentCpu(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +#ifdef _WIN32 + PROCESSOR_NUMBER processor{}; + GetCurrentProcessorNumberEx(&processor); + unsigned index = processor.Number; + for (WORD group = 0; group < processor.Group; ++group) { + const auto count = GetActiveProcessorCount(group); + if (count == 0) + throw std::system_error(GetLastError(), std::system_category(), "Reading processor group size"); + index += count; + } + return static_cast(index); +#else + const int cpu = ::sched_getcpu(); + if (cpu < 0) + throw std::system_error(errno, std::generic_category(), "Reading current processor"); + return cpu; +#endif } -uint64_t APS5_VABI sceKernelGetGPI(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +std::uint64_t APS5_VABI sceKernelGetGPI(void) { + return gpoBits.load(std::memory_order_relaxed); } -void APS5_VABI sceKernelSetGPO(uint32_t bits) { - (void)bits; - NotImplemented_nid_no_patch(__func__); +void APS5_VABI sceKernelSetGPO(std::uint32_t bits) { + gpoBits.store(bits, std::memory_order_relaxed); } -int APS5_VABI sceKernelGetOpenPsId(void* open_ps_id) { - (void)open_ps_id; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceKernelGetOpenPsId(void* openPsIdOutput) { + if (!openPsIdOutput) + throw std::invalid_argument("sceKernelGetOpenPsId: null output"); + std::memcpy(openPsIdOutput, openPsId.data(), openPsId.size()); + return 0; } void* APS5_VABI sceKernelGetProcParam(void) { - NotImplemented_nid_no_patch(__func__); - return nullptr; + return const_cast(ApplicationProcessParameters_nid_no_patch()); } -int APS5_VABI sceKernelUuidCreate(uint32_t* uuid) { - (void)uuid; - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceKernelUuidCreate(std::uint32_t* uuid) { + if (!uuid) + throw std::invalid_argument("sceKernelUuidCreate: null output"); + static thread_local std::random_device device; + std::uniform_int_distribution distribution; + std::array value; + for (auto& word : value) + word = distribution(device); + value[1] = (value[1] & 0x0fffffffu) | 0x40000000u; + value[2] = (value[2] & 0xffffff3fu) | 0x80u; + std::memcpy(uuid, value.data(), sizeof(value)); + return 0; } void APS5_VABI sceKernelSync(void) { - NotImplemented_nid_no_patch(__func__); +#ifdef _WIN32 + syncVolumes(); +#else + ::sync(); +#endif } int APS5_VABI sched_get_priority_max_nid_postfix(int policy) { - (void)policy; - NotImplemented_nid_no_patch(__func__); - return 0; + validateSchedulingPolicy(policy); + return 256; } int APS5_VABI sched_get_priority_min_nid_postfix(int policy) { - (void)policy; - NotImplemented_nid_no_patch(__func__); - return 0; + validateSchedulingPolicy(policy); + return 767; } } From 9fd68163569d6b8fad8916f7b81118bed9ea682f Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 17:31:13 +0300 Subject: [PATCH 30/93] fix(libkernel): store rtld allocator table separately from application heap --- core/libs/prx/libkernel/System/src/Rtld.cpp | 29 ++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libkernel/System/src/Rtld.cpp b/core/libs/prx/libkernel/System/src/Rtld.cpp index e47bf5b13c..7a2f27d81e 100644 --- a/core/libs/prx/libkernel/System/src/Rtld.cpp +++ b/core/libs/prx/libkernel/System/src/Rtld.cpp @@ -1,14 +1,37 @@ #include #include +#include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" -#include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libkernel/Pthread/include/ThreadLifecycle.hpp" +namespace { + +std::mutex rtldHeapMutex; +std::array rtldHeapApi{}; + +void registerApplicationHeapApi(void* const* api) { + if (api == nullptr) + throw std::invalid_argument("RTLD application heap: null allocator API"); + std::array replacement; + std::memcpy(replacement.data(), api, sizeof(replacement)); + if (replacement[0] == nullptr || replacement[1] == nullptr) + throw std::invalid_argument("RTLD application heap: malloc and free are required"); + std::lock_guard lock(rtldHeapMutex); + if (rtldHeapApi[0] != nullptr && rtldHeapApi != replacement) + throw std::runtime_error("RTLD application heap: cannot replace an active allocator"); + rtldHeapApi = replacement; +} + +} + extern "C" { void APS5_VABI sceKernelRtldSetApplicationHeapAPI(void* api[]) { - ApplicationHeapRegister_nid_no_patch(api); + registerApplicationHeapApi(api); } int APS5_VABI sceKernelRtldThreadAtexitDecrement(uint64_t* c) { @@ -40,7 +63,7 @@ void APS5_VABI sceKernelSetThreadDtors(thread_dtors_func_t dtors) { extern "C" { void APS5_VABI _sceKernelRtldSetApplicationHeapAPI_nid_postfix(void* api[]) { - ApplicationHeapRegister_nid_no_patch(api); + registerApplicationHeapApi(api); } int APS5_VABI _sceKernelRtldThreadAtexitDecrement_nid_postfix(std::uint64_t* counter) { From c9cb3cae39f3f9490e147a4e79a51d5b390abd32 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 17:34:30 +0300 Subject: [PATCH 31/93] feat(libkernel): implement disabled address sanitizer state + replacement tables --- core/libs/prx/libkernel/System/src/Sanitizer.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/core/libs/prx/libkernel/System/src/Sanitizer.cpp b/core/libs/prx/libkernel/System/src/Sanitizer.cpp index 452a937430..15f2d3f5fb 100644 --- a/core/libs/prx/libkernel/System/src/Sanitizer.cpp +++ b/core/libs/prx/libkernel/System/src/Sanitizer.cpp @@ -1,23 +1,19 @@ -#include -#include #include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" extern "C" { int APS5_VABI sceKernelIsAddressSanitizerEnabled(void) { - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } MallocReplace* APS5_VABI sceKernelGetSanitizerMallocReplaceExternal(void) { - NotImplemented_nid_no_patch(__func__); - return nullptr; + static MallocReplace replacement{}; + return &replacement; } NewReplace* APS5_VABI sceKernelGetSanitizerNewReplaceExternal(void) { - NotImplemented_nid_no_patch(__func__); - return nullptr; + static NewReplace replacement{}; + return &replacement; } } From ae2e5cba7ebf85ba915f8f55df52090b0b720ffe Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 17:37:29 +0300 Subject: [PATCH 32/93] feat(libc): implement heap trace info --- core/libs/prx/libSceLibcInternal/Export.cpp | 9 ++----- core/libs/prx/libc/Export.cpp | 4 +-- .../libs/prx/libc/include/HeapDiagnostics.hpp | 8 ++++++ core/libs/prx/libc/src/HeapDiagnostics.cpp | 25 +++++++++++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 core/libs/prx/libc/include/HeapDiagnostics.hpp diff --git a/core/libs/prx/libSceLibcInternal/Export.cpp b/core/libs/prx/libSceLibcInternal/Export.cpp index 7541f8ed18..6597f5f5be 100644 --- a/core/libs/prx/libSceLibcInternal/Export.cpp +++ b/core/libs/prx/libSceLibcInternal/Export.cpp @@ -1,14 +1,9 @@ -#include -#include -#include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" +#include "prx/libc/include/HeapDiagnostics.hpp" extern "C" { - void APS5_VABI sceLibcHeapGetTraceInfo_nid_postfix(Info* info) { - (void)info; - NotImplemented_nid_no_patch(__func__); + LibcHeapTraceInfo_nid_no_patch(info); } } diff --git a/core/libs/prx/libc/Export.cpp b/core/libs/prx/libc/Export.cpp index a7bacbfdec..4e0c3813a2 100644 --- a/core/libs/prx/libc/Export.cpp +++ b/core/libs/prx/libc/Export.cpp @@ -2,6 +2,7 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/HeapDiagnostics.hpp" uint32_t Need_sceLibc = 1; @@ -39,8 +40,7 @@ extern "C" { } void APS5_VABI LibcHeapGetTraceInfo_nid_postfix(LibcHeapInfo* info) { - (void)info; - NotImplemented_nid_no_patch(__func__); + LibcHeapTraceInfo_nid_no_patch(info); } int APS5_VABI LibcInternalExtCxaThreadAtexit_nid_postfix(void (*destructor)(void*), void* object, void* module_id) { diff --git a/core/libs/prx/libc/include/HeapDiagnostics.hpp b/core/libs/prx/libc/include/HeapDiagnostics.hpp new file mode 100644 index 0000000000..7d3d624547 --- /dev/null +++ b/core/libs/prx/libc/include/HeapDiagnostics.hpp @@ -0,0 +1,8 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_HEAPDIAGNOSTICS_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_HEAPDIAGNOSTICS_HPP + +#include "SceTypes.hpp" + +extern "C" void LibcHeapTraceInfo_nid_no_patch(LibcHeapInfo* info); + +#endif diff --git a/core/libs/prx/libc/src/HeapDiagnostics.cpp b/core/libs/prx/libc/src/HeapDiagnostics.cpp index fc9dd761fe..39aaf39511 100644 --- a/core/libs/prx/libc/src/HeapDiagnostics.cpp +++ b/core/libs/prx/libc/src/HeapDiagnostics.cpp @@ -1,8 +1,33 @@ #include "prx/libc/include/General.hpp" +#include "prx/libc/include/HeapDiagnostics.hpp" +#include +#include #include +#include + +namespace { + +std::uint64_t mspaceAtomicIdMask = 0; +std::array mstateTable{}; + +static_assert(sizeof(LibcHeapInfo) == 32); +static_assert(offsetof(LibcHeapInfo, mspace_atomic_id_mask) == 16); +static_assert(offsetof(LibcHeapInfo, mstate_table) == 24); + +} extern "C" { +void LibcHeapTraceInfo_nid_no_patch(LibcHeapInfo* info) { + if (info == nullptr) + throw std::invalid_argument("heap trace info: null output"); + if (info->size != sizeof(LibcHeapInfo)) + throw std::invalid_argument("heap trace info: unsupported structure size"); + info->unknown2 = 0; + info->mspace_atomic_id_mask = &mspaceAtomicIdMask; + info->mstate_table = mstateTable.data(); +} + // unknown signature void APS5_VABI sceLibcInternalBacktraceForGame_nid_postfix(const char* heapName) { (void)heapName; From a709e220f2191427f6fc9866065a61a3cab9e7de Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 17:43:44 +0300 Subject: [PATCH 33/93] feat(libkernel): implement posix mutex locking and timed waits --- .../prx/libkernel/Pthread/Posix/Mutex.cpp | 64 ++--- .../prx/libkernel/Pthread/include/Mutex.hpp | 26 ++ .../prx/libkernel/Pthread/include/Pthread.hpp | 2 +- core/libs/prx/libkernel/Pthread/src/Mutex.cpp | 223 +++++++++++------- 4 files changed, 190 insertions(+), 125 deletions(-) create mode 100644 core/libs/prx/libkernel/Pthread/include/Mutex.hpp diff --git a/core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp b/core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp index 1409b3450a..5482859ba0 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp @@ -1,72 +1,60 @@ +#include "../include/Mutex.hpp" #include -#include -#include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" +#include + +namespace { + +int toPosix(int result) { + if (result == 0) + return 0; + const auto error = static_cast(result); + if ((error & 0xffff0000u) != 0x80020000u) + throw std::runtime_error("Unexpected SCE mutex error"); + return static_cast(error & 0xffffu); +} + +} extern "C" { int APS5_VABI pthread_mutex_destroy_nid_postfix(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexDestroy(mutex)); } int APS5_VABI pthread_mutex_init_nid_postfix(PthreadMutex* mutex, const PthreadMutexattr* attr) { - (void)mutex; - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexInit(mutex, attr, nullptr)); } int APS5_VABI pthread_mutex_lock_nid_postfix(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexLock(mutex)); } int APS5_VABI pthread_mutex_timedlock_nid_postfix(PthreadMutex* mutex, const KernelTimespec* abstime) { - (void)mutex; - (void)abstime; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(MutexOperations::Timedlock(mutex, abstime)); } int APS5_VABI pthread_mutex_trylock_nid_postfix(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexTrylock(mutex)); } int APS5_VABI pthread_mutex_unlock_nid_postfix(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexUnlock(mutex)); } int APS5_VABI pthread_mutexattr_destroy_nid_postfix(PthreadMutexattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexattrDestroy(attr)); } int APS5_VABI pthread_mutexattr_init_nid_postfix(PthreadMutexattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexattrInit(attr)); } int APS5_VABI pthread_mutexattr_setprotocol_nid_postfix(PthreadMutexattr* attr, int protocol) { - (void)attr; - (void)protocol; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexattrSetprotocol(attr, protocol)); } int APS5_VABI pthread_mutexattr_settype_nid_postfix(PthreadMutexattr* attr, int type) { - (void)attr; - (void)type; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadMutexattrSettype(attr, type)); } -} +} \ No newline at end of file diff --git a/core/libs/prx/libkernel/Pthread/include/Mutex.hpp b/core/libs/prx/libkernel/Pthread/include/Mutex.hpp new file mode 100644 index 0000000000..ca734c603d --- /dev/null +++ b/core/libs/prx/libkernel/Pthread/include/Mutex.hpp @@ -0,0 +1,26 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_PTHREAD_MUTEX_HPP +#define CORE_LIBS_PRX_LIBKERNEL_PTHREAD_MUTEX_HPP + +#include "SceTypes.hpp" + +class MutexOperations { +public: + static int Timedlock(PthreadMutex* mutex, const KernelTimespec* abstime); +}; + +extern "C" { + +int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attr); +int APS5_VABI scePthreadMutexattrDestroy(PthreadMutexattr* attr); +int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attr, int type); +int APS5_VABI scePthreadMutexattrSetprotocol(PthreadMutexattr* attr, int protocol); +int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr, const char* name); +int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec); +int APS5_VABI scePthreadMutexTrylock(PthreadMutex* mutex); + +} + +#endif diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index e0adf92b8d..b9b503a1bd 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -28,7 +28,7 @@ struct PthreadMutexPrivate { std::atomic _owner; int _count; - PthreadMutexPrivate() : _type(MutexType::Normal), _count(0) {} + PthreadMutexPrivate() : _type(MutexType::Normal), _owner(std::thread::id{}), _count(0) {} }; struct PthreadCondattrPrivate { diff --git a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp index cc9ac18813..ccf8953822 100644 --- a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp @@ -1,113 +1,164 @@ #include "../include/Pthread.hpp" -#include "prx/libc/include/General.hpp" -#include +#include "../include/Mutex.hpp" #include +#include +#include #include -static constexpr int SCE_OK = 0; -static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; -static constexpr int SCE_KERNEL_ERROR_EDEADLK = 0x80020023; -static constexpr int SCE_KERNEL_ERROR_EPERM = 0x80020001; +namespace { + +constexpr int sceBusy = static_cast(0x80020010u); +constexpr int sceTimedOut = static_cast(0x8002003cu); +std::mutex initializationMutex; + +PthreadMutex destroyedMutex() { + return reinterpret_cast(std::uintptr_t{2}); +} + +PthreadMutex resolveMutex(PthreadMutex* mutex, bool initialize) { + if (!mutex) + throw std::invalid_argument("Mutex pointer is null"); + std::lock_guard lock(initializationMutex); + if (*mutex == destroyedMutex()) + throw std::runtime_error("Mutex has been destroyed"); + if (reinterpret_cast(*mutex) == 1) + throw std::runtime_error("Adaptive mutex initializer is unsupported"); + if (!*mutex) { + if (!initialize) + throw std::runtime_error("Mutex is not initialized"); + *mutex = new PthreadMutexPrivate(); + } + return *mutex; +} + +template +int acquireMutex(PthreadMutex mutex, TAcquire acquire, int unavailable, bool tryOnly) { + const auto thread = std::this_thread::get_id(); + const bool owned = mutex->_owner.load(std::memory_order_acquire) == thread; + if (owned && mutex->_type != MutexType::Recursive) { + if (tryOnly) + return sceBusy; + throw std::runtime_error("Mutex is already owned by the current thread"); + } + if (owned && mutex->_count == std::numeric_limits::max()) + throw std::overflow_error("Recursive mutex lock count overflow"); + if (mutex->_type == MutexType::Recursive) { + if (!acquire(mutex->_rmtx)) + return unavailable; + ++mutex->_count; + } else { + if (!acquire(mutex->_mtx)) + return unavailable; + } + mutex->_owner.store(thread, std::memory_order_release); + return 0; +} + +} + +int MutexOperations::Timedlock(PthreadMutex* mutex, const KernelTimespec* abstime) { + if (!abstime || abstime->tv_sec < 0 || abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) + throw std::invalid_argument("Invalid absolute mutex timeout"); + const auto maximum = std::chrono::nanoseconds::max().count(); + if (abstime->tv_sec > (maximum - abstime->tv_nsec) / 1000000000) + throw std::overflow_error("Absolute mutex timeout exceeds the host clock range"); + const auto duration = std::chrono::nanoseconds(abstime->tv_sec * 1000000000 + abstime->tv_nsec); + if (std::chrono::duration(duration) >= std::chrono::duration(std::chrono::system_clock::duration::max())) + throw std::overflow_error("Absolute mutex timeout exceeds the host clock range"); + const auto deadline = std::chrono::system_clock::time_point(std::chrono::duration_cast(duration)); + return acquireMutex(resolveMutex(mutex, true), [&](auto& native) { return native.try_lock_until(deadline); }, sceTimedOut, false); +} extern "C" { int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attr) { - if (!attr) throw std::runtime_error("scePthreadMutexattrInit: null attr"); - auto* p = new (std::nothrow) PthreadMutexattrPrivate{MutexType::Normal}; - if (!p) return SCE_KERNEL_ERROR_ENOMEM; - *attr = p; - return SCE_OK; + if (!attr) + throw std::invalid_argument("Mutex attribute pointer is null"); + *attr = new PthreadMutexattrPrivate{MutexType::Normal}; + return 0; } int APS5_VABI scePthreadMutexattrDestroy(PthreadMutexattr* attr) { - if (!attr || !*attr) throw std::runtime_error("scePthreadMutexattrDestroy: null attr"); + if (!attr || !*attr) + throw std::invalid_argument("Mutex attributes are not initialized"); delete *attr; *attr = nullptr; - return SCE_OK; + return 0; } int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attr, int type) { - if (!attr || !*attr) throw std::runtime_error("scePthreadMutexattrSettype: null attr"); + if (!attr || !*attr) + throw std::invalid_argument("Mutex attributes are not initialized"); switch (type) { case 1: (*attr)->type = MutexType::ErrorCheck; break; case 2: (*attr)->type = MutexType::Recursive; break; case 3: (*attr)->type = MutexType::Normal; break; - default: throw std::runtime_error("scePthreadMutexattrSettype: invalid type"); + default: throw std::invalid_argument("Invalid mutex type"); } - return SCE_OK; -} - -int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr, const char*) { - if (!mutex) throw std::runtime_error("scePthreadMutexInit: null mutex"); - MutexType t = MutexType::Normal; - if (attr && *attr) t = (*attr)->type; - auto* p = new (std::nothrow) PthreadMutexPrivate(); - if (!p) return SCE_KERNEL_ERROR_ENOMEM; - p->_type = t; - *mutex = p; - return SCE_OK; -} - -int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex) { - if (!mutex || !*mutex) throw std::runtime_error("scePthreadMutexDestroy: null mutex"); - delete *mutex; - *mutex = nullptr; - return SCE_OK; -} - -int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex) { - if (!mutex || !*mutex) throw std::runtime_error("scePthreadMutexLock: null mutex"); - auto* m = *mutex; - const auto tid = std::this_thread::get_id(); - if (m->_type == MutexType::Recursive) { - m->_rmtx.lock(); - m->_owner.store(tid, std::memory_order_relaxed); - ++m->_count; - return SCE_OK; - } - if (m->_type == MutexType::ErrorCheck) { - if (m->_owner.load(std::memory_order_acquire) == tid) return SCE_KERNEL_ERROR_EDEADLK; - } - m->_mtx.lock(); - m->_owner.store(tid, std::memory_order_relaxed); - return SCE_OK; -} - -int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex) { - if (!mutex || !*mutex) throw std::runtime_error("scePthreadMutexUnlock: null mutex"); - auto* m = *mutex; - if (m->_type == MutexType::ErrorCheck || m->_type == MutexType::Normal) { - if (m->_owner.load(std::memory_order_acquire) != std::this_thread::get_id()) - return SCE_KERNEL_ERROR_EPERM; - } - if (m->_type == MutexType::Recursive) { - if (--m->_count == 0) m->_owner.store(std::thread::id{}, std::memory_order_relaxed); - m->_rmtx.unlock(); - return SCE_OK; - } - m->_owner.store(std::thread::id{}, std::memory_order_relaxed); - m->_mtx.unlock(); - return SCE_OK; -} - -int APS5_VABI scePthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec) { - (void)mutex; - (void)usec; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI scePthreadMutexTrylock(PthreadMutex* mutex) { - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + return 0; } int APS5_VABI scePthreadMutexattrSetprotocol(PthreadMutexattr* attr, int protocol) { - (void)attr; - (void)protocol; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) + throw std::invalid_argument("Mutex attributes are not initialized"); + if (protocol != 0) + throw std::invalid_argument("Mutex priority inheritance and protection are unsupported"); + return 0; +} + +int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr, const char*) { + if (!mutex) + throw std::invalid_argument("Mutex pointer is null"); + if (attr && !*attr) + throw std::invalid_argument("Mutex attributes are not initialized"); + auto replacement = std::make_unique(); + if (attr) + replacement->_type = (*attr)->type; + std::lock_guard lock(initializationMutex); + *mutex = replacement.release(); + return 0; +} + +int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex) { + if (!mutex) + throw std::invalid_argument("Mutex pointer is null"); + std::lock_guard lock(initializationMutex); + if (*mutex == destroyedMutex()) + throw std::runtime_error("Mutex has already been destroyed"); + if (reinterpret_cast(*mutex) == 1) + throw std::runtime_error("Adaptive mutex initializer is unsupported"); + if (*mutex && (*mutex)->_owner.load(std::memory_order_acquire) != std::thread::id{}) + throw std::runtime_error("Cannot destroy a locked mutex"); + delete *mutex; + *mutex = destroyedMutex(); + return 0; +} + +int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex) { + return acquireMutex(resolveMutex(mutex, true), [](auto& native) { native.lock(); return true; }, 0, false); +} + +int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex) { + auto* current = resolveMutex(mutex, false); + if (current->_owner.load(std::memory_order_acquire) != std::this_thread::get_id()) + throw std::runtime_error("Cannot unlock a mutex owned by another thread"); + if (current->_type == MutexType::Recursive) { + if (--current->_count == 0) + current->_owner.store(std::thread::id{}, std::memory_order_release); + current->_rmtx.unlock(); + } else { + current->_owner.store(std::thread::id{}, std::memory_order_release); + current->_mtx.unlock(); + } + return 0; +} + +int APS5_VABI scePthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec) { + return acquireMutex(resolveMutex(mutex, true), [=](auto& native) { return native.try_lock_for(std::chrono::microseconds(usec)); }, sceTimedOut, false); +} + +int APS5_VABI scePthreadMutexTrylock(PthreadMutex* mutex) { + return acquireMutex(resolveMutex(mutex, true), [](auto& native) { return native.try_lock(); }, sceBusy, true); } } From 62c16792ff74a787f0cdd4f0cb97a3791c7a9381 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 18:16:47 +0300 Subject: [PATCH 34/93] feat(libkernel): impl pthread condition variables + posix thread attribute queries --- .../libs/prx/libkernel/Pthread/Posix/Cond.cpp | 55 +++--- .../prx/libkernel/Pthread/include/Cond.hpp | 26 +++ core/libs/prx/libkernel/Pthread/src/Attr.cpp | 83 ++++----- core/libs/prx/libkernel/Pthread/src/Cond.cpp | 172 ++++++++++++------ 4 files changed, 208 insertions(+), 128 deletions(-) create mode 100644 core/libs/prx/libkernel/Pthread/include/Cond.hpp diff --git a/core/libs/prx/libkernel/Pthread/Posix/Cond.cpp b/core/libs/prx/libkernel/Pthread/Posix/Cond.cpp index b860a2262b..3c202b17df 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Cond.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Cond.cpp @@ -1,61 +1,52 @@ +#include "prx/libkernel/Pthread/include/Cond.hpp" #include -#include -#include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" +#include + +namespace { + +int toPosix(int result) { + if (result == 0) + return 0; + const auto error = static_cast(result); + if ((error & 0xffff0000u) != 0x80020000u) + throw std::runtime_error("Unexpected SCE condition variable error"); + return static_cast(error & 0xffffu); +} + +} extern "C" { int APS5_VABI pthread_cond_broadcast_nid_postfix(PthreadCond* cond) { - (void)cond; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadCondBroadcast(cond)); } int APS5_VABI pthread_cond_init_nid_postfix(PthreadCond* cond, const PthreadCondattr* attr) { - (void)cond; - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadCondInit(cond, attr, nullptr)); } int APS5_VABI pthread_cond_signal_nid_postfix(PthreadCond* cond) { - (void)cond; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadCondSignal(cond)); } int APS5_VABI pthread_cond_timedwait_nid_postfix(PthreadCond* cond, PthreadMutex* mutex, const KernelTimespec* abstime) { - (void)cond; - (void)mutex; - (void)abstime; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(CondOperations::AbsoluteTimedwait(cond, mutex, abstime)); } int APS5_VABI pthread_cond_wait_nid_postfix(PthreadCond* cond, PthreadMutex* mutex) { - (void)cond; - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadCondWait(cond, mutex)); } int APS5_VABI pthread_condattr_destroy_nid_postfix(PthreadCondattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadCondattrDestroy(attr)); } int APS5_VABI pthread_condattr_init_nid_postfix(PthreadCondattr* attr) { - (void)attr; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadCondattrInit(attr)); } int APS5_VABI pthread_condattr_setclock_nid_postfix(PthreadCondattr* attr, KernelClockid clock_id) { - (void)attr; - (void)clock_id; - NotImplemented_nid_no_patch(__func__); - return 0; + return toPosix(scePthreadCondattrSetclock(attr, clock_id)); } } diff --git a/core/libs/prx/libkernel/Pthread/include/Cond.hpp b/core/libs/prx/libkernel/Pthread/include/Cond.hpp new file mode 100644 index 0000000000..936b957149 --- /dev/null +++ b/core/libs/prx/libkernel/Pthread/include/Cond.hpp @@ -0,0 +1,26 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_PTHREAD_COND_HPP +#define CORE_LIBS_PRX_LIBKERNEL_PTHREAD_COND_HPP + +#include "SceTypes.hpp" + +class CondOperations { +public: + static int AbsoluteTimedwait(PthreadCond* cond, PthreadMutex* mutex, const KernelTimespec* abstime); +}; + +extern "C" { + +int APS5_VABI scePthreadCondattrInit(PthreadCondattr* attr); +int APS5_VABI scePthreadCondattrDestroy(PthreadCondattr* attr); +int APS5_VABI scePthreadCondattrSetclock(PthreadCondattr* attr, KernelClockid clockId); +int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char* name); +int APS5_VABI scePthreadCondDestroy(PthreadCond* cond); +int APS5_VABI scePthreadCondSignal(PthreadCond* cond); +int APS5_VABI scePthreadCondBroadcast(PthreadCond* cond); +int APS5_VABI scePthreadCondSignalto(PthreadCond* cond, Pthread thread); +int APS5_VABI scePthreadCondWait(PthreadCond* cond, PthreadMutex* mutex); +int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, KernelUseconds usec); + +} + +#endif diff --git a/core/libs/prx/libkernel/Pthread/src/Attr.cpp b/core/libs/prx/libkernel/Pthread/src/Attr.cpp index 7f71adb808..d325255645 100644 --- a/core/libs/prx/libkernel/Pthread/src/Attr.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Attr.cpp @@ -1,8 +1,10 @@ -#include "../include/Pthread.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" #include "prx/libc/include/General.hpp" #include +#include static constexpr int SCE_OK = 0; +static constexpr int SCE_KERNEL_ERROR_EINVAL = 0x80020016; static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; static constexpr std::size_t DEFAULT_STACK_SIZE = 1u << 20; @@ -18,7 +20,7 @@ static constexpr int SCHED_FIFO_PS5 = 1; extern "C" { int APS5_VABI scePthreadAttrInit(PthreadAttr* attr) { - if (!attr) throw std::runtime_error("scePthreadAttrInit: null attr"); + if (!attr) throw std::runtime_error(std::string(__func__) + ": null attr"); auto* p = new (std::nothrow) PthreadAttrPrivate{}; if (!p) return SCE_KERNEL_ERROR_ENOMEM; p->_stacksize = DEFAULT_STACK_SIZE; @@ -31,35 +33,35 @@ int APS5_VABI scePthreadAttrInit(PthreadAttr* attr) { } int APS5_VABI scePthreadAttrDestroy(PthreadAttr* attr) { - if (!attr || !*attr) throw std::runtime_error("scePthreadAttrDestroy: null attr"); + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); delete *attr; *attr = nullptr; return SCE_OK; } int APS5_VABI scePthreadAttrSetdetachstate(PthreadAttr* attr, int detachstate) { - if (!attr || !*attr) throw std::runtime_error("scePthreadAttrSetdetachstate: null attr"); + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); if (detachstate != DETACH_JOINABLE && detachstate != DETACH_DETACHED) - throw std::runtime_error("scePthreadAttrSetdetachstate: invalid state"); + throw std::runtime_error(std::string(__func__) + ": invalid state"); (*attr)->_detachstate = detachstate; return SCE_OK; } int APS5_VABI scePthreadAttrSetschedparam(PthreadAttr* attr, const KernelSchedParam* param) { if (!attr || !*attr || !param) - throw std::runtime_error("scePthreadAttrSetschedparam: null arg"); + throw std::runtime_error(std::string(__func__) + ": null arg"); (*attr)->_schedpriority = param->sched_priority; return SCE_OK; } int APS5_VABI scePthreadAttrSetstacksize(PthreadAttr* attr, std::size_t stacksize) { - if (!attr || !*attr) throw std::runtime_error("scePthreadAttrSetstacksize: null attr"); - if (stacksize < 16384) throw std::runtime_error("scePthreadAttrSetstacksize: too small"); + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + if (stacksize < 16384) throw std::runtime_error(std::string(__func__) + ": too small"); #ifdef _WIN32 SYSTEM_INFO system{}; GetSystemInfo(&system); if (stacksize % system.dwPageSize != 0 || stacksize > std::numeric_limits::max()) - throw std::runtime_error("scePthreadAttrSetstacksize: invalid Windows stack size"); + throw std::runtime_error(std::string(__func__) + ": invalid Windows stack size"); #endif (*attr)->_stacksize = stacksize; return SCE_OK; @@ -67,7 +69,7 @@ int APS5_VABI scePthreadAttrSetstacksize(PthreadAttr* attr, std::size_t stacksiz int APS5_VABI scePthreadAttrGetstack(const PthreadAttr* attr, void** stackaddr, std::size_t* stacksize) { if (!attr || !*attr || !stackaddr || !stacksize) - throw std::runtime_error("scePthreadAttrGetstack: null arg"); + throw std::runtime_error(std::string(__func__) + ": null arg"); *stackaddr = (*attr)->stackAddress; *stacksize = (*attr)->_stacksize; return SCE_OK; @@ -75,7 +77,7 @@ int APS5_VABI scePthreadAttrGetstack(const PthreadAttr* attr, void** stackaddr, int APS5_VABI scePthreadAttrGet(Pthread thread, PthreadAttr* attr) { if (!thread || !attr || !*attr) - throw std::runtime_error("scePthreadAttrGet: null arg"); + return SCE_KERNEL_ERROR_EINVAL; (*attr)->_stacksize = thread->stackSize; (*attr)->stackAddress = thread->stackAddress; (*attr)->_detachstate = thread->_detached ? DETACH_DETACHED : DETACH_JOINABLE; @@ -93,10 +95,9 @@ int APS5_VABI scePthreadAttrGetaffinity(const PthreadAttr* attr, KernelCpumask* } int APS5_VABI scePthreadAttrGetdetachstate(const PthreadAttr* attr, int* state) { - (void)attr; - (void)state; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr || !state) throw std::runtime_error(std::string(__func__) + ": null arg"); + *state = (*attr)->_detachstate; + return SCE_OK; } int APS5_VABI scePthreadAttrGetguardsize(const PthreadAttr* attr, size_t* guard_size) { @@ -107,10 +108,9 @@ int APS5_VABI scePthreadAttrGetguardsize(const PthreadAttr* attr, size_t* guard_ } int APS5_VABI scePthreadAttrGetschedparam(const PthreadAttr* attr, KernelSchedParam* param) { - (void)attr; - (void)param; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr || !param) throw std::runtime_error(std::string(__func__) + ": null arg"); + param->sched_priority = (*attr)->_schedpriority; + return SCE_OK; } int APS5_VABI scePthreadAttrGetsolosched(const PthreadAttr* attr, int* solosched) { @@ -121,17 +121,15 @@ int APS5_VABI scePthreadAttrGetsolosched(const PthreadAttr* attr, int* solosched } int APS5_VABI scePthreadAttrGetstackaddr(const PthreadAttr* attr, void** stack_addr) { - (void)attr; - (void)stack_addr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr || !stack_addr) throw std::runtime_error(std::string(__func__) + ": null arg"); + *stack_addr = (*attr)->stackAddress; + return SCE_OK; } int APS5_VABI scePthreadAttrGetstacksize(const PthreadAttr* attr, size_t* stack_size) { - (void)attr; - (void)stack_size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr || !stack_size) throw std::runtime_error(std::string(__func__) + ": null arg"); + *stack_size = (*attr)->_stacksize; + return SCE_OK; } int APS5_VABI scePthreadAttrSetaffinity(PthreadAttr* attr, KernelCpumask mask) { @@ -149,17 +147,15 @@ int APS5_VABI scePthreadAttrSetguardsize(PthreadAttr* attr, size_t guard_size) { } int APS5_VABI scePthreadAttrSetinheritsched(PthreadAttr* attr, int inherit_sched) { - (void)attr; - (void)inherit_sched; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + (*attr)->_inheritsched = inherit_sched; + return SCE_OK; } int APS5_VABI scePthreadAttrSetschedpolicy(PthreadAttr* attr, int policy) { - (void)attr; - (void)policy; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); + (*attr)->_schedpolicy = policy; + return SCE_OK; } int APS5_VABI scePthreadAttrSetsolosched(PthreadAttr* attr, int solosched) { @@ -170,18 +166,17 @@ int APS5_VABI scePthreadAttrSetsolosched(PthreadAttr* attr, int solosched) { } int APS5_VABI scePthreadAttrSetstack(PthreadAttr* attr, void* addr, size_t size) { - (void)attr; - (void)addr; - (void)size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr || !addr) throw std::runtime_error(std::string(__func__) + ": null arg"); + if (size < 16384) throw std::runtime_error(std::string(__func__) + ": too small"); + (*attr)->stackAddress = addr; + (*attr)->_stacksize = size; + return SCE_OK; } int APS5_VABI scePthreadAttrSetstackaddr(PthreadAttr* attr, void* addr) { - (void)attr; - (void)addr; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!attr || !*attr || !addr) throw std::runtime_error(std::string(__func__) + ": null arg"); + (*attr)->stackAddress = addr; + return SCE_OK; } } diff --git a/core/libs/prx/libkernel/Pthread/src/Cond.cpp b/core/libs/prx/libkernel/Pthread/src/Cond.cpp index 754a076f32..93c2902fee 100644 --- a/core/libs/prx/libkernel/Pthread/src/Cond.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Cond.cpp @@ -1,85 +1,153 @@ -#include "../include/Pthread.hpp" -#include "prx/libc/include/General.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" +#include "prx/libkernel/Pthread/include/Mutex.hpp" +#include "prx/libkernel/Pthread/include/Cond.hpp" #include +#include #include -static constexpr int SCE_OK = 0; -static constexpr int SCE_KERNEL_ERROR_ENOMEM = 0x8002000C; -static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = 0x80020062; +namespace { + +constexpr int sceTimedOut = static_cast(0x8002003cu); +std::mutex condInitializationMutex; + +PthreadCond destroyedCond() { + return reinterpret_cast(std::uintptr_t{2}); +} + +PthreadCond resolveCond(PthreadCond* cond) { + if (!cond) + throw std::invalid_argument("Condition variable pointer is null"); + std::lock_guard lock(condInitializationMutex); + if (*cond == destroyedCond()) + throw std::runtime_error("Condition variable has been destroyed"); + if (!*cond) + *cond = new PthreadCondPrivate(); + return *cond; +} + +PthreadMutex lockedMutex(PthreadMutex* mutex) { + if (!mutex || !*mutex) + throw std::invalid_argument("Mutex pointer is null"); + auto* current = *mutex; + if (current->_owner.load(std::memory_order_acquire) != std::this_thread::get_id()) + throw std::runtime_error("Condition wait mutex is not owned by the current thread"); + return current; +} + +int waitUntil(PthreadCond* cond, PthreadMutex* mutex, const std::chrono::system_clock::time_point* deadline) { + auto* c = resolveCond(cond); + auto* m = lockedMutex(mutex); + if (m->_type == MutexType::Recursive) { + std::unique_lock lock(m->_rmtx, std::adopt_lock); + const auto previousCount = m->_count; + m->_count = 0; + m->_owner.store(std::thread::id{}, std::memory_order_release); + const bool timedOut = deadline && c->_cv.wait_until(lock, *deadline) == std::cv_status::timeout; + if (!deadline) + c->_cv.wait(lock); + m->_owner.store(std::this_thread::get_id(), std::memory_order_release); + m->_count = previousCount; + lock.release(); + return timedOut ? sceTimedOut : 0; + } + std::unique_lock lock(m->_mtx, std::adopt_lock); + m->_owner.store(std::thread::id{}, std::memory_order_release); + const bool timedOut = deadline && c->_cv.wait_until(lock, *deadline) == std::cv_status::timeout; + if (!deadline) + c->_cv.wait(lock); + m->_owner.store(std::this_thread::get_id(), std::memory_order_release); + lock.release(); + return timedOut ? sceTimedOut : 0; +} + +std::chrono::system_clock::time_point toDeadline(const KernelTimespec* abstime) { + if (!abstime || abstime->tv_sec < 0 || abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) + throw std::invalid_argument("Invalid absolute condition variable timeout"); + const auto maximum = std::chrono::nanoseconds::max().count(); + if (abstime->tv_sec > (maximum - abstime->tv_nsec) / 1000000000) + throw std::overflow_error("Absolute condition variable timeout exceeds the host clock range"); + const auto duration = std::chrono::nanoseconds(abstime->tv_sec * 1000000000 + abstime->tv_nsec); + if (std::chrono::duration(duration) >= std::chrono::duration(std::chrono::system_clock::duration::max())) + throw std::overflow_error("Absolute condition variable timeout exceeds the host clock range"); + return std::chrono::system_clock::time_point(std::chrono::duration_cast(duration)); +} + +} + +int CondOperations::AbsoluteTimedwait(PthreadCond* cond, PthreadMutex* mutex, const KernelTimespec* abstime) { + const auto deadline = toDeadline(abstime); + return waitUntil(cond, mutex, &deadline); +} extern "C" { int APS5_VABI scePthreadCondattrInit(PthreadCondattr* attr) { - if (!attr) throw std::runtime_error("scePthreadCondattrInit: null attr"); - auto* p = new (std::nothrow) PthreadCondattrPrivate{0}; - if (!p) return SCE_KERNEL_ERROR_ENOMEM; - *attr = p; - return SCE_OK; + if (!attr) + throw std::invalid_argument("Condition attribute pointer is null"); + *attr = new PthreadCondattrPrivate{0}; + return 0; } int APS5_VABI scePthreadCondattrDestroy(PthreadCondattr* attr) { - if (!attr || !*attr) throw std::runtime_error("scePthreadCondattrDestroy: null attr"); + if (!attr || !*attr) + throw std::invalid_argument("Condition attributes are not initialized"); delete *attr; *attr = nullptr; - return SCE_OK; + return 0; } -int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr*, const char*) { - if (!cond) throw std::runtime_error("scePthreadCondInit: null cond"); - auto* p = new (std::nothrow) PthreadCondPrivate{}; - if (!p) return SCE_KERNEL_ERROR_ENOMEM; - *cond = p; - return SCE_OK; +int APS5_VABI scePthreadCondattrSetclock(PthreadCondattr* attr, KernelClockid clockId) { + if (!attr || !*attr) + throw std::invalid_argument("Condition attributes are not initialized"); + (*attr)->_clockid = static_cast(clockId); + return 0; +} + +int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char*) { + if (!cond) + throw std::invalid_argument("Condition variable pointer is null"); + if (attr && !*attr) + throw std::invalid_argument("Condition attributes are not initialized"); + auto replacement = std::make_unique(); + std::lock_guard lock(condInitializationMutex); + *cond = replacement.release(); + return 0; } int APS5_VABI scePthreadCondDestroy(PthreadCond* cond) { - if (!cond || !*cond) throw std::runtime_error("scePthreadCondDestroy: null cond"); + if (!cond) + throw std::invalid_argument("Condition variable pointer is null"); + std::lock_guard lock(condInitializationMutex); + if (*cond == destroyedCond()) + throw std::runtime_error("Condition variable has already been destroyed"); delete *cond; - *cond = nullptr; - return SCE_OK; + *cond = destroyedCond(); + return 0; } int APS5_VABI scePthreadCondSignal(PthreadCond* cond) { - if (!cond || !*cond) throw std::runtime_error("scePthreadCondSignal: null cond"); - (*cond)->_cv.notify_one(); - return SCE_OK; + resolveCond(cond)->_cv.notify_one(); + return 0; } int APS5_VABI scePthreadCondBroadcast(PthreadCond* cond) { - if (!cond || !*cond) throw std::runtime_error("scePthreadCondBroadcast: null cond"); - (*cond)->_cv.notify_all(); - return SCE_OK; -} - -int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, unsigned int usec) { - if (!cond || !*cond || !mutex || !*mutex) - throw std::runtime_error("scePthreadCondTimedwait: null arg"); - auto* m = *mutex; - auto* c = *cond; - if (m->_type == MutexType::Recursive) { - std::unique_lock lk(m->_rmtx, std::adopt_lock); - auto res = c->_cv.wait_for(lk, std::chrono::microseconds(usec)); - lk.release(); - return res == std::cv_status::timeout ? SCE_KERNEL_ERROR_ETIMEDOUT : SCE_OK; - } - std::unique_lock lk(m->_mtx, std::adopt_lock); - auto res = c->_cv.wait_for(lk, std::chrono::microseconds(usec)); - lk.release(); - return res == std::cv_status::timeout ? SCE_KERNEL_ERROR_ETIMEDOUT : SCE_OK; + resolveCond(cond)->_cv.notify_all(); + return 0; } int APS5_VABI scePthreadCondSignalto(PthreadCond* cond, Pthread thread) { - (void)cond; - (void)thread; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)thread; + resolveCond(cond)->_cv.notify_all(); + return 0; } int APS5_VABI scePthreadCondWait(PthreadCond* cond, PthreadMutex* mutex) { - (void)cond; - (void)mutex; - NotImplemented_nid_no_patch(__func__); - return 0; + return waitUntil(cond, mutex, nullptr); +} + +int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, KernelUseconds usec) { + const auto deadline = std::chrono::system_clock::now() + std::chrono::microseconds(usec); + return waitUntil(cond, mutex, &deadline); } } From 13c648bebbdcdc501b4b67ed01670e84647cf91d Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 18:35:20 +0300 Subject: [PATCH 35/93] feat(libkernel): impl file, time, process, signal for posix --- core/libs/prx/libkernel/CMakeLists.txt | 1 - core/libs/prx/libkernel/File/src/Stdio.cpp | 61 +++++++++++- .../prx/libkernel/Pthread/Posix/Rwlock.cpp | 22 +++++ .../prx/libkernel/Pthread/include/Rwlock.hpp | 21 ++++ .../libs/prx/libkernel/Pthread/src/Rwlock.cpp | 1 + .../libkernel/System/src/PosixImportStubs.cpp | 91 ------------------ .../libs/prx/libkernel/System/src/Process.cpp | 95 +++++++++++++++++++ .../libs/prx/libkernel/System/src/Signals.cpp | 26 +++++ core/libs/prx/libkernel/Time/Time.cpp | 4 + 9 files changed, 227 insertions(+), 95 deletions(-) create mode 100644 core/libs/prx/libkernel/Pthread/include/Rwlock.hpp delete mode 100644 core/libs/prx/libkernel/System/src/PosixImportStubs.cpp diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 965749d044..394bad5136 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -39,7 +39,6 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Sanitizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Rtld.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Process.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/System/src/PosixImportStubs.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/SystemConfiguration.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Signals.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Time/Time.cpp diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 81704d09d1..c0cefcd8b9 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -2,11 +2,23 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/File/include/File.hpp" +#include "prx/libkernel/File/include/FileFlags.hpp" #include "prx/libkernel/Socket/include/SocketRuntime.hpp" +#include +#include +#include #ifdef _WIN32 #include +#include +static int NativeRmdir(const std::filesystem::path& path) { + return ::_wrmdir(path.wstring().c_str()); +} #else #include +static int NativeRmdir(const std::filesystem::path& path) { + return ::rmdir(path.c_str()); +} #endif extern "C" { @@ -27,6 +39,10 @@ int APS5_VABI close_nid_postfix(int d) { #endif } +int APS5_VABI _close_nid_postfix(int descriptor) { + return close_nid_postfix(descriptor); +} + int APS5_VABI flock_nid_postfix(int d, int operation) { (void)d; (void)operation; @@ -71,6 +87,24 @@ int APS5_VABI open_nid_postfix(const char* path, int flags, int mode) { return 0; } +int APS5_VABI _open_nid_postfix(const char* path, int flags, ...) { + std::uint16_t mode = 0; + if (flags & SCE_KERNEL_O_CREAT) { +#ifdef _WIN32 + __builtin_sysv_va_list arguments; + __builtin_sysv_va_start(arguments, flags); + mode = static_cast(__builtin_va_arg(arguments, int)); + __builtin_sysv_va_end(arguments); +#else + std::va_list arguments; + va_start(arguments, flags); + mode = static_cast(va_arg(arguments, int)); + va_end(arguments); +#endif + } + return sceKernelOpen(path, flags, mode); +} + int64_t APS5_VABI pread_nid_postfix(int d, void* buf, size_t nbytes, int64_t offset) { (void)d; (void)buf; @@ -97,6 +131,10 @@ int64_t APS5_VABI read_nid_postfix(int d, void* buf, uint64_t nbytes) { return 0; } +std::int64_t APS5_VABI _read_nid_postfix(int descriptor, void* buffer, std::size_t count) { + return sceKernelRead(descriptor, buffer, count); +} + int64_t APS5_VABI write_nid_postfix(int d, const char* str, int64_t size) { (void)d; (void)str; @@ -105,6 +143,10 @@ int64_t APS5_VABI write_nid_postfix(int d, const char* str, int64_t size) { return 0; } +std::int64_t APS5_VABI _write_nid_postfix(int descriptor, const void* buffer, std::size_t count) { + return sceKernelWrite(descriptor, buffer, count); +} + int APS5_VABI stat_nid_postfix(const char* path, FileStat* sb) { (void)path; (void)sb; @@ -112,6 +154,10 @@ int APS5_VABI stat_nid_postfix(const char* path, FileStat* sb) { return 0; } +int APS5_VABI unlink_nid_postfix(const char* path) { + return sceKernelUnlink(path); +} + int APS5_VABI sceKernelCheckReachability(const char* path) { (void)path; NotImplemented_nid_no_patch(__func__); @@ -181,9 +227,18 @@ int APS5_VABI sceKernelRename(const char* from, const char* to) { } int APS5_VABI sceKernelRmdir(const char* path) { - (void)path; - NotImplemented_nid_no_patch(__func__); - return 0; + if (path == nullptr) { + throw std::invalid_argument(std::string(__func__) + ": path is null"); + } + auto native = ResolvePath_nid_no_patch(path); + if (NativeRmdir(native) != 0) { + throw std::runtime_error(std::string(__func__) + ": rmdir failed for " + native.string() + ", errno=" + std::to_string(errno)); + } + return 0; +} + +int APS5_VABI rmdir_nid_postfix(const char* path) { + return sceKernelRmdir(path); } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Rwlock.cpp b/core/libs/prx/libkernel/Pthread/Posix/Rwlock.cpp index 2f1ee6a983..52f67399e5 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Rwlock.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Rwlock.cpp @@ -2,6 +2,20 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "../include/Rwlock.hpp" + +namespace { + +int toPosix(int result) { + if (result == 0) + return 0; + const auto error = static_cast(result); + if ((error & 0xffff0000u) != 0x80020000u) + throw std::runtime_error("Unexpected SCE rwlock error"); + return static_cast(error & 0xffffu); +} + +} extern "C" { @@ -18,6 +32,14 @@ int APS5_VABI pthread_rwlock_init_nid_postfix(PthreadRwlock* rwlock, const Pthre return 0; } +int APS5_VABI pthread_rwlock_rdlock_nid_postfix(PthreadRwlock* rwlock) { + return toPosix(scePthreadRwlockRdlock(rwlock)); +} + +int APS5_VABI pthread_rwlock_unlock_nid_postfix(PthreadRwlock* rwlock) { + return toPosix(scePthreadRwlockUnlock(rwlock)); +} + int APS5_VABI pthread_rwlock_wrlock_nid_postfix(PthreadRwlock* rwlock) { (void)rwlock; NotImplemented_nid_no_patch(__func__); diff --git a/core/libs/prx/libkernel/Pthread/include/Rwlock.hpp b/core/libs/prx/libkernel/Pthread/include/Rwlock.hpp new file mode 100644 index 0000000000..0c9668003d --- /dev/null +++ b/core/libs/prx/libkernel/Pthread/include/Rwlock.hpp @@ -0,0 +1,21 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_PTHREAD_RWLOCK_HPP +#define CORE_LIBS_PRX_LIBKERNEL_PTHREAD_RWLOCK_HPP + +#include "SceTypes.hpp" + +extern "C" { + +int APS5_VABI scePthreadRwlockattrInit(PthreadRwlockattr* attr); +int APS5_VABI scePthreadRwlockattrDestroy(PthreadRwlockattr* attr); +int APS5_VABI scePthreadRwlockattrSettype(PthreadRwlockattr* attr, int type); +int APS5_VABI scePthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr, const char* name); +int APS5_VABI scePthreadRwlockDestroy(PthreadRwlock* rwlock); +int APS5_VABI scePthreadRwlockRdlock(PthreadRwlock* rwlock); +int APS5_VABI scePthreadRwlockTryrdlock(PthreadRwlock* rwlock); +int APS5_VABI scePthreadRwlockWrlock(PthreadRwlock* rwlock); +int APS5_VABI scePthreadRwlockTrywrlock(PthreadRwlock* rwlock); +int APS5_VABI scePthreadRwlockUnlock(PthreadRwlock* rwlock); + +} + +#endif diff --git a/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp b/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp index 12a6040a28..4804e2dac6 100644 --- a/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp @@ -1,4 +1,5 @@ #include "../include/Pthread.hpp" +#include "../include/Rwlock.hpp" #include "prx/libc/include/General.hpp" extern "C" { diff --git a/core/libs/prx/libkernel/System/src/PosixImportStubs.cpp b/core/libs/prx/libkernel/System/src/PosixImportStubs.cpp deleted file mode 100644 index bfb5cd1456..0000000000 --- a/core/libs/prx/libkernel/System/src/PosixImportStubs.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#include "SceTypes.hpp" -#include "prx/libc/include/General.hpp" -#include -#include - -struct GuestSignalSet; -struct GuestResourceUsage; - -extern "C" { - -int APS5_VABI _close_nid_postfix(int descriptor) { - (void)descriptor; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -void APS5_VABI _exit_nid_postfix(int status) { - (void)status; - NotImplemented_nid_no_patch(__func__); -} - -int APS5_VABI _nanosleep_nid_postfix(const KernelTimespec* requested, KernelTimespec* remaining) { - (void)requested; - (void)remaining; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _open_nid_postfix(const char* path, int flags, ...) { - (void)path; - (void)flags; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -std::int64_t APS5_VABI _read_nid_postfix(int descriptor, void* buffer, std::size_t count) { - (void)descriptor; - (void)buffer; - (void)count; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI _sigprocmask_nid_postfix(int how, const GuestSignalSet* set, GuestSignalSet* previousSet) { - (void)how; - (void)set; - (void)previousSet; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -std::int64_t APS5_VABI _write_nid_postfix(int descriptor, const void* buffer, std::size_t count) { - (void)descriptor; - (void)buffer; - (void)count; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI getrusage_nid_postfix(int who, GuestResourceUsage* usage) { - (void)who; - (void)usage; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI pthread_rwlock_rdlock_nid_postfix(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI pthread_rwlock_unlock_nid_postfix(PthreadRwlock* rwlock) { - (void)rwlock; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI rmdir_nid_postfix(const char* path) { - (void)path; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI unlink_nid_postfix(const char* path) { - (void)path; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -} diff --git a/core/libs/prx/libkernel/System/src/Process.cpp b/core/libs/prx/libkernel/System/src/Process.cpp index 4fa1e7ea97..7fe1ad6bb4 100644 --- a/core/libs/prx/libkernel/System/src/Process.cpp +++ b/core/libs/prx/libkernel/System/src/Process.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,9 @@ #include #include #include +#ifndef _WIN32 +#include +#endif namespace { @@ -116,6 +120,25 @@ void syncVolumes() { } +struct GuestResourceUsage { + KernelTimeval ru_utime; + KernelTimeval ru_stime; + std::int64_t ru_maxrss; + std::int64_t ru_ixrss; + std::int64_t ru_idrss; + std::int64_t ru_isrss; + std::int64_t ru_minflt; + std::int64_t ru_majflt; + std::int64_t ru_nswap; + std::int64_t ru_inblock; + std::int64_t ru_oublock; + std::int64_t ru_msgsnd; + std::int64_t ru_msgrcv; + std::int64_t ru_nsignals; + std::int64_t ru_nvcsw; + std::int64_t ru_nivcsw; +}; + extern "C" { // unknown signature @@ -148,6 +171,10 @@ void APS5_VABI exit_nid_postfix(int code) { LibcExit_nid_no_patch(code); } +[[noreturn]] void APS5_VABI _exit_nid_postfix(int status) { + std::_Exit(status); +} + int APS5_VABI sceKernelGetCurrentCpu(void) { #ifdef _WIN32 PROCESSOR_NUMBER processor{}; @@ -219,4 +246,72 @@ int APS5_VABI sched_get_priority_min_nid_postfix(int policy) { return 767; } +int APS5_VABI getrusage_nid_postfix(int who, GuestResourceUsage* usage) { + if (usage == nullptr) + throw std::invalid_argument("getrusage: usage is null"); + if (who != 0 && who != 1) + throw std::invalid_argument("getrusage: unsupported who"); +#ifdef _WIN32 + FILETIME creation{}; + FILETIME exitTime{}; + FILETIME kernel{}; + FILETIME user{}; + if (who == 0) { + if (!GetProcessTimes(GetCurrentProcess(), &creation, &exitTime, &kernel, &user)) + throw std::system_error(GetLastError(), std::system_category(), "getrusage: GetProcessTimes failed"); + } else { + if (!GetThreadTimes(GetCurrentThread(), &creation, &exitTime, &kernel, &user)) + throw std::system_error(GetLastError(), std::system_category(), "getrusage: GetThreadTimes failed"); + } + const auto toMicros = [](const FILETIME& time) { + return ((static_cast(time.dwHighDateTime) << 32) | time.dwLowDateTime) / 10ULL; + }; + const auto userMicros = toMicros(user); + const auto kernelMicros = toMicros(kernel); + usage->ru_utime.tv_sec = static_cast(userMicros / 1000000ULL); + usage->ru_utime.tv_usec = static_cast(userMicros % 1000000ULL); + usage->ru_stime.tv_sec = static_cast(kernelMicros / 1000000ULL); + usage->ru_stime.tv_usec = static_cast(kernelMicros % 1000000ULL); + usage->ru_maxrss = 0; + usage->ru_ixrss = 0; + usage->ru_idrss = 0; + usage->ru_isrss = 0; + usage->ru_minflt = 0; + usage->ru_majflt = 0; + usage->ru_nswap = 0; + usage->ru_inblock = 0; + usage->ru_oublock = 0; + usage->ru_msgsnd = 0; + usage->ru_msgrcv = 0; + usage->ru_nsignals = 0; + usage->ru_nvcsw = 0; + usage->ru_nivcsw = 0; +#else + if (who == 1) + throw std::invalid_argument("getrusage: RUSAGE_THREAD is not supported on this platform"); + struct rusage native{}; + if (::getrusage(RUSAGE_SELF, &native) != 0) + throw std::system_error(errno, std::generic_category(), "getrusage: getrusage failed"); + usage->ru_utime.tv_sec = static_cast(native.ru_utime.tv_sec); + usage->ru_utime.tv_usec = static_cast(native.ru_utime.tv_usec); + usage->ru_stime.tv_sec = static_cast(native.ru_stime.tv_sec); + usage->ru_stime.tv_usec = static_cast(native.ru_stime.tv_usec); + usage->ru_maxrss = static_cast(native.ru_maxrss); + usage->ru_ixrss = static_cast(native.ru_ixrss); + usage->ru_idrss = static_cast(native.ru_idrss); + usage->ru_isrss = static_cast(native.ru_isrss); + usage->ru_minflt = static_cast(native.ru_minflt); + usage->ru_majflt = static_cast(native.ru_majflt); + usage->ru_nswap = static_cast(native.ru_nswap); + usage->ru_inblock = static_cast(native.ru_inblock); + usage->ru_oublock = static_cast(native.ru_oublock); + usage->ru_msgsnd = static_cast(native.ru_msgsnd); + usage->ru_msgrcv = static_cast(native.ru_msgrcv); + usage->ru_nsignals = static_cast(native.ru_nsignals); + usage->ru_nvcsw = static_cast(native.ru_nvcsw); + usage->ru_nivcsw = static_cast(native.ru_nivcsw); +#endif + return 0; +} + } diff --git a/core/libs/prx/libkernel/System/src/Signals.cpp b/core/libs/prx/libkernel/System/src/Signals.cpp index 713585c601..94a4e3be5f 100644 --- a/core/libs/prx/libkernel/System/src/Signals.cpp +++ b/core/libs/prx/libkernel/System/src/Signals.cpp @@ -10,6 +10,8 @@ namespace { using GuestHandler = void (APS5_VABI *)(int); std::atomic handlers[32]{}; static_assert(std::atomic::is_always_lock_free); +std::atomic blockedMask{0}; +static_assert(std::atomic::is_always_lock_free); std::mutex registration; int NativeSignal(int guest) { switch (guest) { @@ -31,10 +33,16 @@ void Dispatch(int native) { // Preserve the guest's persistent registration across CRT delivery. std::signal(native, Dispatch); #endif + if ((blockedMask.load() & (1u << guest)) != 0) return; const auto callback = handlers[guest].load(); if (reinterpret_cast(callback) > 1) callback(guest); } } + +struct GuestSignalSet { + std::uint32_t bits[4]; +}; + extern "C" { GuestHandler APS5_VABI signal_nid_postfix(int guest, GuestHandler handler) { const auto invalid = reinterpret_cast(static_cast(-1)); @@ -58,6 +66,24 @@ int APS5_VABI raise_nid_postfix(int guest) { if (result) *__error_nid_postfix() = 22; return result ? -1 : 0; } +int APS5_VABI _sigprocmask_nid_postfix(int how, const GuestSignalSet* set, GuestSignalSet* previousSet) { + std::lock_guard lock(registration); + if (previousSet != nullptr) { + previousSet->bits[0] = blockedMask.load(); + previousSet->bits[1] = 0; + previousSet->bits[2] = 0; + previousSet->bits[3] = 0; + } + if (set != nullptr) { + switch (how) { + case 1: blockedMask.fetch_or(set->bits[0]); break; + case 2: blockedMask.fetch_and(~set->bits[0]); break; + case 3: blockedMask.store(set->bits[0]); break; + default: throw std::invalid_argument("_sigprocmask: invalid how"); + } + } + return 0; +} } extern "C" { diff --git a/core/libs/prx/libkernel/Time/Time.cpp b/core/libs/prx/libkernel/Time/Time.cpp index aeb9cafb9f..7b641b016c 100644 --- a/core/libs/prx/libkernel/Time/Time.cpp +++ b/core/libs/prx/libkernel/Time/Time.cpp @@ -92,6 +92,10 @@ int APS5_VABI nanosleep_nid_postfix(const KernelTimespec* rqtp, KernelTimespec* return sceKernelNanosleep(rqtp, rmtp); } +int APS5_VABI _nanosleep_nid_postfix(const KernelTimespec* rqtp, KernelTimespec* rmtp) { + return sceKernelNanosleep(rqtp, rmtp); +} + int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp) { if (tp == nullptr) { APS5_INVALID_ARG_EX; From 7a925c68e5d535d8d6a14d9b99542d80642ca8a9 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 18:48:40 +0300 Subject: [PATCH 36/93] fix(libkernel): handle clock_second 13 in clock_gettime and clock_getres --- core/libs/prx/libkernel/Time/Time.cpp | 32 +++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libkernel/Time/Time.cpp b/core/libs/prx/libkernel/Time/Time.cpp index 7b641b016c..28cd5647e9 100644 --- a/core/libs/prx/libkernel/Time/Time.cpp +++ b/core/libs/prx/libkernel/Time/Time.cpp @@ -101,7 +101,7 @@ int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp) { APS5_INVALID_ARG_EX; } #ifdef _WIN32 - if (clockId == 0 || clockId == 9 || clockId == 10) { + if (clockId == 0 || clockId == 9) { FILETIME ft{}; GetSystemTimePreciseAsFileTime(&ft); std::uint64_t t = (static_cast(ft.dwHighDateTime) << 32) | ft.dwLowDateTime; @@ -111,6 +111,16 @@ int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp) { tp->tv_nsec = static_cast(t % 1000000000ULL); return 0; } + if (clockId == 10 || clockId == 13) { + FILETIME ft{}; + GetSystemTimeAsFileTime(&ft); + std::uint64_t t = (static_cast(ft.dwHighDateTime) << 32) | ft.dwLowDateTime; + t -= 116444736000000000ULL; + t *= 100ULL; + tp->tv_sec = static_cast(t / 1000000000ULL); + tp->tv_nsec = static_cast(t % 1000000000ULL); + return 0; + } if (clockId == 4 || clockId == 1 || clockId == 5 || clockId == 7 || clockId == 8 || clockId == 11 || clockId == 12) { std::uint64_t nanos = GetMonotonicNanos(); tp->tv_sec = static_cast(nanos / 1000000000ULL); @@ -123,9 +133,16 @@ int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp) { switch (clockId) { case 0: case 9: - case 10: nativeId = CLOCK_REALTIME; break; + case 10: + case 13: +#ifdef CLOCK_REALTIME_COARSE + nativeId = CLOCK_REALTIME_COARSE; +#else + nativeId = CLOCK_REALTIME; +#endif + break; case 4: case 7: case 11: @@ -191,7 +208,7 @@ int APS5_VABI clock_getres_nid_postfix(int clockId, KernelTimespec* res) { res->tv_nsec = 100LL; return 0; } - if (clockId == 4 || clockId == 1 || clockId == 5 || clockId == 7 || clockId == 8 || clockId == 10 || clockId == 11 || clockId == 12) { + if (clockId == 4 || clockId == 1 || clockId == 5 || clockId == 7 || clockId == 8 || clockId == 10 || clockId == 11 || clockId == 12 || clockId == 13) { static const std::uint64_t freq = [] { LARGE_INTEGER f{}; QueryPerformanceFrequency(&f); @@ -208,9 +225,16 @@ int APS5_VABI clock_getres_nid_postfix(int clockId, KernelTimespec* res) { switch (clockId) { case 0: case 9: - case 10: nativeId = CLOCK_REALTIME; break; + case 10: + case 13: +#ifdef CLOCK_REALTIME_COARSE + nativeId = CLOCK_REALTIME_COARSE; +#else + nativeId = CLOCK_REALTIME; +#endif + break; case 4: case 7: case 11: From f5ea673c1085f6c1627273534827fe9ff618a020 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 18:58:32 +0300 Subject: [PATCH 37/93] feat(libkernel): impl posix stdio wrappers, reusing sceKernel file ops --- .../prx/libkernel/File/include/NativeStat.hpp | 1 + .../prx/libkernel/File/src/NativeStat.cpp | 32 ++- core/libs/prx/libkernel/File/src/Stdio.cpp | 207 +++++++++++++----- 3 files changed, 177 insertions(+), 63 deletions(-) diff --git a/core/libs/prx/libkernel/File/include/NativeStat.hpp b/core/libs/prx/libkernel/File/include/NativeStat.hpp index 8ea664c419..07ee81c91d 100644 --- a/core/libs/prx/libkernel/File/include/NativeStat.hpp +++ b/core/libs/prx/libkernel/File/include/NativeStat.hpp @@ -7,6 +7,7 @@ namespace File { void FillFileStat(const std::filesystem::path& nativePath, FileStat* sb); +void FillFileStat(int nativeDescriptor, FileStat* sb); } diff --git a/core/libs/prx/libkernel/File/src/NativeStat.cpp b/core/libs/prx/libkernel/File/src/NativeStat.cpp index 5f92973b89..f25a72e980 100644 --- a/core/libs/prx/libkernel/File/src/NativeStat.cpp +++ b/core/libs/prx/libkernel/File/src/NativeStat.cpp @@ -10,21 +10,21 @@ using NativeStat = struct __stat64; static int DoStat(const std::filesystem::path& p, NativeStat* st) { return _wstat64(p.wstring().c_str(), st); } +static int DoFstat(int fd, NativeStat* st) { + return _fstat64(fd, st); +} #else #include using NativeStat = struct stat; static int DoStat(const std::filesystem::path& p, NativeStat* st) { return ::stat(p.c_str(), st); } +static int DoFstat(int fd, NativeStat* st) { + return ::fstat(fd, st); +} #endif -namespace File { - -void FillFileStat(const std::filesystem::path& nativePath, FileStat* sb) { - NativeStat st{}; - if (DoStat(nativePath, &st) != 0) { - throw std::runtime_error(std::string("FillFileStat: stat failed for ") + nativePath.string()); - } +static void CopyNativeStat(const NativeStat& st, FileStat* sb) { *sb = FileStat{}; sb->st_mode = static_cast(st.st_mode); sb->st_size = static_cast(st.st_size); @@ -72,4 +72,22 @@ void FillFileStat(const std::filesystem::path& nativePath, FileStat* sb) { #endif } +namespace File { + +void FillFileStat(const std::filesystem::path& nativePath, FileStat* sb) { + NativeStat st{}; + if (DoStat(nativePath, &st) != 0) { + throw std::runtime_error(std::string("FillFileStat: stat failed for ") + nativePath.string()); + } + CopyNativeStat(st, sb); +} + +void FillFileStat(int nativeDescriptor, FileStat* sb) { + NativeStat st{}; + if (DoFstat(nativeDescriptor, &st) != 0) { + throw std::runtime_error(std::string("FillFileStat: fstat failed for fd ") + std::to_string(nativeDescriptor)); + } + CopyNativeStat(st, sb); +} + } diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index c0cefcd8b9..ef652c6172 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -1,33 +1,115 @@ #include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include "prx/libkernel/File/include/File.hpp" #include "prx/libkernel/File/include/FileFlags.hpp" +#include "prx/libkernel/File/include/NativeStat.hpp" #include "prx/libkernel/Socket/include/SocketRuntime.hpp" #include #include #include #ifdef _WIN32 +#include #include #include +#include static int NativeRmdir(const std::filesystem::path& path) { return ::_wrmdir(path.wstring().c_str()); } +static int NativeMkdir(const std::filesystem::path& path, std::uint16_t mode) { + (void)mode; + return ::_wmkdir(path.wstring().c_str()); +} +static int NativeChmod(const std::filesystem::path& path, int mode) { + return ::_wchmod(path.wstring().c_str(), mode); +} +static int NativeFtruncate(int descriptor, std::int64_t length) { + return static_cast(::_chsize_s(descriptor, length)); +} +static int NativeFlock(int descriptor, int operation) { + HANDLE handle = reinterpret_cast(::_get_osfhandle(descriptor)); + if (handle == INVALID_HANDLE_VALUE) { + return -1; + } + OVERLAPPED overlapped{}; + if (operation & 8) { + return ::UnlockFileEx(handle, 0, MAXDWORD, MAXDWORD, &overlapped) ? 0 : -1; + } + DWORD flags = 0; + if (operation & 2) flags |= LOCKFILE_EXCLUSIVE_LOCK; + if (operation & 4) flags |= LOCKFILE_FAIL_IMMEDIATELY; + return ::LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped) ? 0 : -1; +} +static std::int64_t NativePread(int descriptor, void* buf, std::size_t nbytes, std::int64_t offset) { + if (nbytes > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("NativePread: nbytes exceeds platform limit"); + } + std::int64_t saved = ::_lseeki64(descriptor, 0, SEEK_CUR); + if (saved < 0) { + return -1; + } + if (::_lseeki64(descriptor, offset, SEEK_SET) < 0) { + return -1; + } + int n = ::_read(descriptor, buf, static_cast(nbytes)); + ::_lseeki64(descriptor, saved, SEEK_SET); + return n; +} +static std::int64_t NativePwrite(int descriptor, const void* buf, std::size_t nbytes, std::int64_t offset) { + if (nbytes > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("NativePwrite: nbytes exceeds platform limit"); + } + std::int64_t saved = ::_lseeki64(descriptor, 0, SEEK_CUR); + if (saved < 0) { + return -1; + } + if (::_lseeki64(descriptor, offset, SEEK_SET) < 0) { + return -1; + } + int n = ::_write(descriptor, buf, static_cast(nbytes)); + ::_lseeki64(descriptor, saved, SEEK_SET); + return n; +} #else #include +#include +#include static int NativeRmdir(const std::filesystem::path& path) { return ::rmdir(path.c_str()); } +static int NativeMkdir(const std::filesystem::path& path, std::uint16_t mode) { + return ::mkdir(path.c_str(), static_cast(mode)); +} +static int NativeChmod(const std::filesystem::path& path, int mode) { + return ::chmod(path.c_str(), static_cast(mode)); +} +static int NativeFtruncate(int descriptor, std::int64_t length) { + return ::ftruncate(descriptor, static_cast(length)); +} +static int NativeFlock(int descriptor, int operation) { + return ::flock(descriptor, operation); +} +static std::int64_t NativePread(int descriptor, void* buf, std::size_t nbytes, std::int64_t offset) { + return static_cast(::pread(descriptor, buf, nbytes, static_cast(offset))); +} +static std::int64_t NativePwrite(int descriptor, const void* buf, std::size_t nbytes, std::int64_t offset) { + return static_cast(::pwrite(descriptor, buf, nbytes, static_cast(offset))); +} #endif extern "C" { int APS5_VABI chmod_nid_postfix(const char* path, int mode) { - (void)path; - (void)mode; - NotImplemented_nid_no_patch(__func__); - return 0; + if (path == nullptr) { + APS5_INVALID_ARG_EX; + } + auto native = ResolvePath_nid_no_patch(path); + if (NativeChmod(native, mode) != 0) { + throw std::runtime_error(std::string(__func__) + ": chmod failed for " + native.string() + ", errno=" + std::to_string(errno)); + } + return 0; } int APS5_VABI close_nid_postfix(int d) { @@ -44,47 +126,58 @@ int APS5_VABI _close_nid_postfix(int descriptor) { } int APS5_VABI flock_nid_postfix(int d, int operation) { - (void)d; - (void)operation; - NotImplemented_nid_no_patch(__func__); - return 0; + if (NativeFlock(d, operation) != 0) { +#ifdef _WIN32 + throw std::runtime_error(std::string(__func__) + ": flock failed, fd=" + std::to_string(d) + ", error=" + std::to_string(::GetLastError())); +#else + throw std::runtime_error(std::string(__func__) + ": flock failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); +#endif + } + return 0; } int64_t APS5_VABI fstat_nid_disambig1_nid_postfix(int d, FileStat* sb) { - (void)d; - (void)sb; - NotImplemented_nid_no_patch(__func__); - return 0; + if (sb == nullptr) { + APS5_INVALID_ARG_EX; + } + File::FillFileStat(d, sb); + return 0; } int APS5_VABI ftruncate_nid_postfix(int d, int64_t length) { - (void)d; - (void)length; - NotImplemented_nid_no_patch(__func__); - return 0; + if (length < 0) { + APS5_INVALID_ARG_EX; + } +#ifdef _WIN32 + int error = NativeFtruncate(d, length); + if (error != 0) { + throw std::runtime_error(std::string(__func__) + ": ftruncate failed, fd=" + std::to_string(d) + ", error=" + std::to_string(error)); + } +#else + if (NativeFtruncate(d, length) != 0) { + throw std::runtime_error(std::string(__func__) + ": ftruncate failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); + } +#endif + return 0; } int64_t APS5_VABI lseek_nid_postfix(int d, int64_t offset, int whence) { - (void)d; - (void)offset; - (void)whence; - NotImplemented_nid_no_patch(__func__); - return 0; + return static_cast(sceKernelLseek(d, offset, whence)); } int APS5_VABI mkdir_nid_postfix(const char* path, uint16_t mode) { - (void)path; - (void)mode; - NotImplemented_nid_no_patch(__func__); - return 0; + if (path == nullptr) { + APS5_INVALID_ARG_EX; + } + auto native = ResolvePath_nid_no_patch(path); + if (NativeMkdir(native, mode) != 0) { + throw std::runtime_error(std::string(__func__) + ": mkdir failed for " + native.string() + ", errno=" + std::to_string(errno)); + } + return 0; } int APS5_VABI open_nid_postfix(const char* path, int flags, int mode) { - (void)path; - (void)flags; - (void)mode; - NotImplemented_nid_no_patch(__func__); - return 0; + return sceKernelOpen(path, flags, static_cast(mode)); } int APS5_VABI _open_nid_postfix(const char* path, int flags, ...) { @@ -106,29 +199,35 @@ int APS5_VABI _open_nid_postfix(const char* path, int flags, ...) { } int64_t APS5_VABI pread_nid_postfix(int d, void* buf, size_t nbytes, int64_t offset) { - (void)d; - (void)buf; - (void)nbytes; - (void)offset; - NotImplemented_nid_no_patch(__func__); - return 0; + if (buf == nullptr) { + APS5_INVALID_ARG_EX; + } + if (offset < 0) { + APS5_INVALID_ARG_EX; + } + auto n = NativePread(d, buf, nbytes, offset); + if (n < 0) { + throw std::runtime_error(std::string(__func__) + ": pread failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); + } + return n; } int64_t APS5_VABI pwrite_nid_disambig1_nid_postfix(int d, const void* buf, size_t nbytes, int64_t offset) { - (void)d; - (void)buf; - (void)nbytes; - (void)offset; - NotImplemented_nid_no_patch(__func__); - return 0; + if (buf == nullptr) { + APS5_INVALID_ARG_EX; + } + if (offset < 0) { + APS5_INVALID_ARG_EX; + } + auto n = NativePwrite(d, buf, nbytes, offset); + if (n < 0) { + throw std::runtime_error(std::string(__func__) + ": pwrite failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); + } + return n; } int64_t APS5_VABI read_nid_postfix(int d, void* buf, uint64_t nbytes) { - (void)d; - (void)buf; - (void)nbytes; - NotImplemented_nid_no_patch(__func__); - return 0; + return sceKernelRead(d, buf, static_cast(nbytes)); } std::int64_t APS5_VABI _read_nid_postfix(int descriptor, void* buffer, std::size_t count) { @@ -136,11 +235,10 @@ std::int64_t APS5_VABI _read_nid_postfix(int descriptor, void* buffer, std::size } int64_t APS5_VABI write_nid_postfix(int d, const char* str, int64_t size) { - (void)d; - (void)str; - (void)size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (size < 0) { + APS5_INVALID_ARG_EX; + } + return sceKernelWrite(d, str, static_cast(size)); } std::int64_t APS5_VABI _write_nid_postfix(int descriptor, const void* buffer, std::size_t count) { @@ -148,10 +246,7 @@ std::int64_t APS5_VABI _write_nid_postfix(int descriptor, const void* buffer, st } int APS5_VABI stat_nid_postfix(const char* path, FileStat* sb) { - (void)path; - (void)sb; - NotImplemented_nid_no_patch(__func__); - return 0; + return sceKernelStat(path, sb); } int APS5_VABI unlink_nid_postfix(const char* path) { From 9cc36a02fecfa57a875c430684efc51a0a862c8f Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sat, 26 Sep 2026 19:06:03 +0300 Subject: [PATCH 38/93] feat(libkernel): add __progname str value --- core/libs/prx/libkernel/System/src/Process.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libkernel/System/src/Process.cpp b/core/libs/prx/libkernel/System/src/Process.cpp index 7fe1ad6bb4..db034a2109 100644 --- a/core/libs/prx/libkernel/System/src/Process.cpp +++ b/core/libs/prx/libkernel/System/src/Process.cpp @@ -141,8 +141,8 @@ struct GuestResourceUsage { extern "C" { -// unknown signature -char* __progname_nid_postfix = nullptr; +// unknown data +const char* __progname_nid_postfix = "eboot.bin"; int APS5_VABI getargc_nid_postfix(void) { return getProcessArguments().GetCount(); From bfa1ac0751d6cec02b4bc13d26ca64cd0320be1b Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sun, 27 Sep 2026 09:00:11 +0300 Subject: [PATCH 39/93] fix(relinker): scan complete function ranges for windows tls accesses --- core/relinker/CMakeLists.txt | 4 +- .../src/analysis/CodeInstructionCollector.cpp | 49 +++++++- .../tests/test_tls_function_coverage.py | 116 ++++++++++++++++++ 3 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 core/relinker/relinker/tests/test_tls_function_coverage.py diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 0e95e87f9b..6062598776 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -86,8 +86,10 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_optional_plt.py $) add_test(NAME empty_tls COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_empty_tls.py $) + add_test(NAME tls_function_coverage COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_tls_function_coverage.py $) else() - message(STATUS "Python3 interpreter not found; skipping optional_plt and empty_tls tests") + message(STATUS "Python3 interpreter not found; skipping Python relinker tests") endif() add_executable(strict_nid_filter_tests ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/StrictNidFilterTests.cpp diff --git a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp index 061c353786..0b009c39f9 100644 --- a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp +++ b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp @@ -2,8 +2,11 @@ #include #include #include +#include #include #include +#include +#include namespace Relinker { @@ -73,7 +76,27 @@ std::set CodeInstructionCollector::Collect(const std::ve return offset; }; std::set roots; + std::map functions; const auto addRoot = [&](std::uint64_t address) { if (isCode(address)) roots.insert(address); }; + const auto addFunction = [&](std::uint64_t begin, std::uint64_t size) { + if (size == 0) return; + if (!isCode(begin) || size > std::numeric_limits::max() - begin) + throw Domain::RelinkerException("Code analysis: invalid function range", begin); + bool mapped = false; + for (const auto& header : headers) { + if (header.Type != 1 || (header.Flags & 1) == 0 || begin < header.MappedAddress || begin - header.MappedAddress >= header.FileSize) continue; + if (size > header.FileSize - (begin - header.MappedAddress)) + throw Domain::RelinkerException("Code analysis: function exceeds executable segment", begin); + range(header.Offset + begin - header.MappedAddress, size); + mapped = true; + break; + } + if (!mapped) throw Domain::RelinkerException("Code analysis: function is not file-backed", begin); + const auto [position, inserted] = functions.emplace(begin, begin + size); + if (!inserted && position->second != begin + size) + throw Domain::RelinkerException("Code analysis: conflicting function ranges", begin); + roots.insert(begin); + }; range(0, 64); if (const auto entry = Io::ReadU64(bytes, 24); entry != 0) addRoot(entry); for (const auto tag : {12u, 13u}) if (const auto address = value(tag, 0x60000000 + tag); address && *address != 0) addRoot(*address); @@ -89,7 +112,11 @@ std::set CodeInstructionCollector::Collect(const std::ve symbols = table(6, 0x61000039, symbolCount * 24); for (std::uint64_t index = 0; index < symbolCount; ++index) { const auto offset = symbols + index * 24; - if ((bytes[offset + 4] & 15) == 2 && Io::ReadU16(bytes, offset + 6) != 0) addRoot(Io::ReadU64(bytes, offset + 8)); + if ((bytes[offset + 4] & 15) == 2 && Io::ReadU16(bytes, offset + 6) != 0) { + const auto address = Io::ReadU64(bytes, offset + 8); + addRoot(address); + addFunction(address, Io::ReadU64(bytes, offset + 16)); + } } } CodePointers pointers; @@ -139,12 +166,30 @@ std::set CodeInstructionCollector::Collect(const std::ve } } for (const auto& function : UnusedNidFilter::ReadExceptionFunctions(bytes, headers, pointers.Values, importSlots)) { - addRoot(function.Begin); + if (function.End <= function.Begin) throw Domain::RelinkerException("Code analysis: invalid unwind function range", function.Begin); + addFunction(function.Begin, function.End - function.Begin); for (const auto target : function.ExtraTargets) addRoot(target); } if (roots.empty()) throw Domain::RelinkerException("Code analysis: no code entry points"); std::set instructions; const Codegen::X64InstructionDecoder decoder; + for (const auto& [begin, end] : functions) { + const auto offset = fileOffset(begin, end - begin); + for (auto address = begin; address < end;) { + Codegen::DecodedInstructionInfo info; + try { + info = decoder.DecodeInstruction(bytes.data() + offset + address - begin, end - address); + } catch (const Codegen::CodegenException& error) { + throw Domain::RelinkerException(std::string("Code analysis: ") + error.what(), address); + } + if (info.Length == 0 || info.Length > end - address) + throw Domain::RelinkerException("Code analysis: instruction crosses function boundary", address); + instructions.insert(address); + if (info.HasBranchTarget && !info.HasRipRelativeDisp) + addRoot(address + info.Length + static_cast(info.BranchDisp)); + address += info.Length; + } + } std::size_t previousRoots = 0; do { previousRoots = roots.size(); diff --git a/core/relinker/relinker/tests/test_tls_function_coverage.py b/core/relinker/relinker/tests/test_tls_function_coverage.py new file mode 100644 index 0000000000..c0bc8145b4 --- /dev/null +++ b/core/relinker/relinker/tests/test_tls_function_coverage.py @@ -0,0 +1,116 @@ +import os +from pathlib import Path +import struct +import subprocess +import sys +import tempfile + +from test_optional_plt import fixture + + +TLS_LOAD = bytes.fromhex("66 66 66 64 48 8b 04 25 00 00 00 00") + + +def make_image(transfer, metadata, extent=None): + image = fixture() + image.extend(b"\x90" * 0x1000) + struct.pack_into(" Date: Sun, 27 Sep 2026 09:37:01 +0300 Subject: [PATCH 40/93] feat(libc): replace guest locale implementation with std::locale --- core/libs/CMakeLists.txt | 6 - core/libs/prx/libc/include/GuestLocale.hpp | 66 ------- core/libs/prx/libc/src/LocaleSupport.cpp | 216 ++++++++------------- core/libs/tests/GuestCharacter.cpp | 4 +- core/libs/tests/GuestLocale.cpp | 158 --------------- 5 files changed, 79 insertions(+), 371 deletions(-) delete mode 100644 core/libs/prx/libc/include/GuestLocale.hpp delete mode 100644 core/libs/tests/GuestLocale.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index ac01dac0e5..933cd9093f 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -361,12 +361,6 @@ add_executable(application_heap_tests EXCLUDE_FROM_ALL tests/ApplicationHeap.cpp target_include_directories(application_heap_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(application_heap_tests PROPERTIES CXX_EXTENSIONS OFF) -add_executable(guest_locale_tests EXCLUDE_FROM_ALL tests/GuestLocale.cpp) -target_include_directories(guest_locale_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -set_target_properties(guest_locale_tests PROPERTIES CXX_EXTENSIONS OFF) -target_link_libraries(guest_locale_tests PRIVATE libc) -configure_windows_unwind(guest_locale_tests) - if(MINGW) foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static) string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") diff --git a/core/libs/prx/libc/include/GuestLocale.hpp b/core/libs/prx/libc/include/GuestLocale.hpp deleted file mode 100644 index 1e46c92102..0000000000 --- a/core/libs/prx/libc/include/GuestLocale.hpp +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTLOCALE_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTLOCALE_HPP - -#include -#include -#include "general/VabiMacros.hpp" - -namespace GuestLocale { - -struct LocinfoStorage { - alignas(std::uint64_t) unsigned char bytes[64]; -}; - -struct Facet; - -struct FacetVtable { - void (APS5_VABI *destroy)(Facet* self); - void (APS5_VABI *deleteObject)(Facet* self); - void (APS5_VABI *retain)(Facet* self); - Facet* (APS5_VABI *release)(Facet* self); -}; - -struct Facet { - const FacetVtable* vtable; - std::uint32_t references; - std::uint32_t reserved; -}; - -struct Implementation { - Facet base; - Facet** facets; - std::uint64_t facetCount; - std::uint32_t category; - bool transparent; - const char* name; -}; - -struct IosBase { - const void* vtable; - std::int64_t standardStream; - std::int32_t state; - std::int32_t exceptions; - std::int32_t flags; - std::int32_t precision; - std::int32_t width; - std::uint32_t reserved; - void* storage; - void* callbacks; - Implementation** locale; -}; - -static_assert(sizeof(Facet) == 0x10); -static_assert(offsetof(Implementation, facets) == 0x10); -static_assert(offsetof(Implementation, facetCount) == 0x18); -static_assert(offsetof(Implementation, transparent) == 0x24); -static_assert(offsetof(Implementation, name) == 0x28); -static_assert(offsetof(IosBase, state) == 0x10); -static_assert(offsetof(IosBase, storage) == 0x28); -static_assert(offsetof(IosBase, callbacks) == 0x30); -static_assert(offsetof(IosBase, locale) == 0x38); -static_assert(sizeof(IosBase) == 0x40); -static_assert(alignof(LocinfoStorage) == 8); - -} - -#endif diff --git a/core/libs/prx/libc/src/LocaleSupport.cpp b/core/libs/prx/libc/src/LocaleSupport.cpp index 0d178f8993..8d3d0e7b4f 100644 --- a/core/libs/prx/libc/src/LocaleSupport.cpp +++ b/core/libs/prx/libc/src/LocaleSupport.cpp @@ -1,203 +1,141 @@ +#include +#include #include -#include +#include #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include #include "prx/libc/include/General.hpp" #include "prx/libc/include/ApplicationHeap.hpp" -#include "prx/libc/include/GuestLocale.hpp" namespace { -std::mutex g_localeInitMutex; -bool g_localeInitialized = false; -std::vector g_registeredFacets; - -void APS5_VABI DestroyClassicLocale(GuestLocale::Facet*) { - throw std::runtime_error("Cannot destroy the classic locale"); +void ValidateCharacter(int value) { + if (value != EOF && (value < 0 || value > UCHAR_MAX)) throw std::invalid_argument("Invalid character value"); } -void APS5_VABI RetainLocale(GuestLocale::Facet* self) { - if (self == nullptr) throw std::invalid_argument("locale retain: null facet"); - std::atomic_ref references(self->references); - auto count = references.load(); - do { - if (count == 0 || count == UINT32_MAX) throw std::runtime_error("locale retain: invalid reference count"); - } while (!references.compare_exchange_weak(count, count + 1)); +int ClassifyCharacter(int value, std::ctype_base::mask mask) { + ValidateCharacter(value); + if (value == EOF) return 0; + return std::use_facet>(std::locale::classic()).is(mask, static_cast(value)); } -GuestLocale::Facet* APS5_VABI ReleaseLocale(GuestLocale::Facet* self) { - if (self == nullptr) throw std::invalid_argument("locale release: null facet"); - std::atomic_ref references(self->references); - auto count = references.load(); - do { - if (count <= 1) throw std::runtime_error("classic locale: unbalanced release"); - } while (!references.compare_exchange_weak(count, count - 1)); - return nullptr; +int ConvertCharacter(int value, bool upper) { + ValidateCharacter(value); + if (value == EOF) return EOF; + const auto& facet = std::use_facet>(std::locale::classic()); + const auto character = static_cast(value); + return static_cast(upper ? facet.toupper(character) : facet.tolower(character)); } -const GuestLocale::FacetVtable g_localeVtable{DestroyClassicLocale, DestroyClassicLocale, RetainLocale, ReleaseLocale}; -GuestLocale::Facet* g_classicFacets[1]{}; -GuestLocale::Implementation g_classicLocale{{&g_localeVtable, 1, 0}, g_classicFacets, 1, 0, false, "C"}; - -constexpr std::array MakeClassificationTable() { - std::array table{}; - for (unsigned int value = 0; value < 256; ++value) { - short mask = 0; - if (value < 32 || value == 127) mask |= 0x80; - if (value == ' ') mask |= 0x04; - if (value >= '\t' && value <= '\r') mask |= 0x40; - if (value >= 'A' && value <= 'Z') mask |= 0x02; - if (value >= 'a' && value <= 'z') mask |= 0x10; - if (value >= '0' && value <= '9') mask |= 0x20; - if ((value >= '0' && value <= '9') || (value >= 'A' && value <= 'F') || (value >= 'a' && value <= 'f')) mask |= 0x01; - if (value >= 33 && value <= 126 && (mask & 0x232) == 0) mask |= 0x08; - table[value + 1] = mask; - } +std::array MakeCaseTable(bool upper) { + std::array table{}; + table[0] = EOF; + for (int value = 0; value <= UCHAR_MAX; ++value) table[value + 1] = ConvertCharacter(value, upper); return table; } -constexpr std::array MakeCaseTable(bool upper) { - std::array table{}; - table[0] = -1; - for (unsigned int value = 0; value < 256; ++value) { - auto converted = value; - if (upper && value >= 'a' && value <= 'z') converted -= 'a' - 'A'; - if (!upper && value >= 'A' && value <= 'Z') converted += 'a' - 'A'; - table[value + 1] = static_cast(converted); - } - return table; -} - -constexpr auto g_classificationTable = MakeClassificationTable(); -constexpr auto g_lowerTable = MakeCaseTable(false); -constexpr auto g_upperTable = MakeCaseTable(true); - } extern "C" { -// The runtime currently exposes the classic C locale. Keep byte classification -// independent of any locale selected by host-side libraries. -int APS5_VABI isupper_nid_postfix(int c) { return c >= 'A' && c <= 'Z'; } -int APS5_VABI islower_nid_postfix(int c) { return c >= 'a' && c <= 'z'; } -int APS5_VABI isalpha_nid_postfix(int c) { return isupper_nid_postfix(c) || islower_nid_postfix(c); } -int APS5_VABI isdigit_nid_postfix(int c) { return c >= '0' && c <= '9'; } -int APS5_VABI isalnum_nid_postfix(int c) { return isalpha_nid_postfix(c) || isdigit_nid_postfix(c); } -int APS5_VABI isspace_nid_postfix(int c) { return c == ' ' || (c >= '\t' && c <= '\r'); } -int APS5_VABI isblank_nid_postfix(int c) { return c == ' ' || c == '\t'; } -int APS5_VABI iscntrl_nid_postfix(int c) { return (c >= 0 && c < 32) || c == 127; } -int APS5_VABI isprint_nid_postfix(int c) { return c >= 32 && c <= 126; } -int APS5_VABI isgraph_nid_postfix(int c) { return c >= 33 && c <= 126; } -int APS5_VABI ispunct_nid_postfix(int c) { return isgraph_nid_postfix(c) && !isalnum_nid_postfix(c); } -int APS5_VABI isxdigit_nid_postfix(int c) { - return isdigit_nid_postfix(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); -} -int APS5_VABI toupper_nid_postfix(int c) { return islower_nid_postfix(c) ? c - ('a' - 'A') : c; } -int APS5_VABI tolower_nid_postfix(int c) { return isupper_nid_postfix(c) ? c + ('a' - 'A') : c; } - -std::uint64_t _ZNSt5ctypeIcE2idE_nid_postfix = 0; -std::uint64_t _ZNSt5ctypeIwE2idE_nid_postfix = 0; -std::uint64_t _ZNSt7collateIwE2idE_nid_postfix = 0; -std::uint64_t _ZNSt7num_putIcSt19ostreambuf_iteratorIcSt11char_traitsIcEEE2idE_nid_postfix = 0; -std::uintptr_t _ZTVSt7num_putIcSt19ostreambuf_iteratorIcSt11char_traitsIcEEE_nid_postfix[12] {}; +int APS5_VABI isupper_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::upper); } +int APS5_VABI islower_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::lower); } +int APS5_VABI isalpha_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::alpha); } +int APS5_VABI isdigit_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::digit); } +int APS5_VABI isalnum_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::alnum); } +int APS5_VABI isspace_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::space); } +int APS5_VABI isblank_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::blank); } +int APS5_VABI iscntrl_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::cntrl); } +int APS5_VABI isprint_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::print); } +int APS5_VABI isgraph_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::graph); } +int APS5_VABI ispunct_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::punct); } +int APS5_VABI isxdigit_nid_postfix(int c) { return ClassifyCharacter(c, std::ctype_base::xdigit); } +int APS5_VABI toupper_nid_postfix(int c) { return ConvertCharacter(c, true); } +int APS5_VABI tolower_nid_postfix(int c) { return ConvertCharacter(c, false); } std::streamoff _ZSt7_BADOFF_nid_postfix = -1; -std::fpos_t _ZSt4_Fpz_nid_postfix {}; -std::int32_t _ZNSt6locale2id7_Id_cntE_nid_postfix = 0; +std::fpos_t _ZSt4_Fpz_nid_postfix{}; -GuestLocale::Implementation* _ZSt21_sceLibcClassicLocale_nid_postfix = &g_classicLocale; +const std::locale* _ZSt21_sceLibcClassicLocale_nid_postfix = &std::locale::classic(); -void APS5_VABI _ZNSt8ios_baseD2Ev_nid_postfix(GuestLocale::IosBase* self) { +void APS5_VABI _ZNSt8ios_baseD2Ev_nid_postfix(std::ios_base* self) { if (self == nullptr) throw std::invalid_argument("ios_base destructor: null object"); - if (self->standardStream != 0 || self->storage != nullptr || self->callbacks != nullptr) throw std::runtime_error("ios_base destructor: unsupported stream storage or callbacks"); - if (self->locale != &_ZSt21_sceLibcClassicLocale_nid_postfix) throw std::runtime_error("ios_base destructor: unsupported locale ownership"); - self->locale = nullptr; + self->std::ios_base::~ios_base(); } -GuestLocale::Implementation* APS5_VABI _ZNSt6locale5_InitEv_nid_postfix() { - std::lock_guard lock(g_localeInitMutex); - if (!g_localeInitialized) { - g_localeInitialized = true; - } - return &g_classicLocale; +const std::locale* APS5_VABI _ZNSt6locale5_InitEv_nid_postfix() { + return &std::locale::classic(); } -void APS5_VABI _ZNSt6locale5facet9_RegisterEv_nid_postfix(GuestLocale::Facet* self) { - if (self == nullptr || self->vtable == nullptr) throw std::invalid_argument("locale register: invalid facet"); - std::lock_guard lock(g_localeInitMutex); - for (const auto* facet : g_registeredFacets) { - if (facet == self) throw std::runtime_error("locale register: duplicate facet"); - } - g_registeredFacets.push_back(self); +void APS5_VABI _ZNSt6locale5facet9_RegisterEv_nid_postfix(std::locale::facet*) { + NotImplemented_nid_no_patch("locale::facet::_Register"); } -GuestLocale::Implementation* APS5_VABI _ZNSt6locale16_GetgloballocaleEv_nid_postfix() { - return &g_classicLocale; +const std::locale* APS5_VABI _ZNSt6locale16_GetgloballocaleEv_nid_postfix() { + thread_local std::locale locale; + locale = std::locale(); + return &locale; } -void APS5_VABI _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet**, const GuestLocale::Implementation*) { - NotImplemented_nid_no_patch(__func__); +void APS5_VABI _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(const std::locale::facet**, const std::locale*) { + NotImplemented_nid_no_patch("collate::_Getcat"); } -void APS5_VABI _ZNSt8_LocinfoC1EPKc_nid_postfix(GuestLocale::LocinfoStorage* self, const char* localeName) { - if (self == nullptr || localeName == nullptr || std::strcmp(localeName, "C") != 0) throw std::invalid_argument("_Locinfo: only the C locale is supported"); - new (self) GuestLocale::LocinfoStorage{}; +void APS5_VABI _ZNSt8_LocinfoC1EPKc_nid_postfix(std::locale* self, const char* localeName) { + if (self == nullptr || localeName == nullptr) throw std::invalid_argument("Locale constructor: null argument"); + std::construct_at(self, localeName); } -void APS5_VABI _ZNSt8_LocinfoD1Ev_nid_postfix(GuestLocale::LocinfoStorage* self) { - if (self == nullptr) throw std::invalid_argument("_Locinfo destructor: null object"); +void APS5_VABI _ZNSt8_LocinfoD1Ev_nid_postfix(std::locale* self) { + if (self == nullptr) throw std::invalid_argument("Locale destructor: null object"); + std::destroy_at(self); } -int APS5_VABI _Mbtowcx_nid_postfix(std::uint16_t* dst, const char* src, std::size_t count, mbstate_t* st) { - if (dst == nullptr || src == nullptr || st == nullptr || count == 0) throw std::invalid_argument("_Mbtowcx: invalid conversion arguments"); - wchar_t converted{}; - const auto result = std::mbrtowc(&converted, src, count, st); +std::size_t APS5_VABI _Mbtowcx_nid_postfix(wchar_t* dst, const char* src, std::size_t count, std::mbstate_t* state) { + if (dst == nullptr || src == nullptr || state == nullptr || count == 0) throw std::invalid_argument("_Mbtowcx: invalid conversion arguments"); + const auto result = std::mbrtowc(dst, src, count, state); if (result == static_cast(-1)) throw std::runtime_error("_Mbtowcx: invalid multibyte character"); if (result == static_cast(-2)) throw std::runtime_error("_Mbtowcx: incomplete multibyte character"); - if (result > static_cast(std::numeric_limits::max()) || static_cast(converted) > 0xffff) throw std::runtime_error("_Mbtowcx: conversion exceeds guest character limits"); - *dst = static_cast(converted); - return static_cast(result); + return result; } -int APS5_VABI _Wctombx_nid_postfix(char* dst, std::uint16_t src, mbstate_t* st) { - if (dst == nullptr || st == nullptr) throw std::invalid_argument("_Wctombx: invalid conversion arguments"); - const auto result = std::wcrtomb(dst, static_cast(src), st); +std::size_t APS5_VABI _Wctombx_nid_postfix(char* dst, wchar_t src, std::mbstate_t* state) { + if (dst == nullptr || state == nullptr) throw std::invalid_argument("_Wctombx: invalid conversion arguments"); + const auto result = std::wcrtomb(dst, src, state); if (result == static_cast(-1)) throw std::runtime_error("_Wctombx: invalid wide character"); - if (result > static_cast(std::numeric_limits::max())) throw std::runtime_error("_Wctombx: conversion size exceeds guest limits"); - return static_cast(result); + return result; } -const short* APS5_VABI _Getpctype_nid_postfix() { - return g_classificationTable.data() + 1; +const std::ctype::mask* APS5_VABI _Getpctype_nid_postfix() { + return std::ctype::classic_table(); } -const short* APS5_VABI _Getptolower_nid_postfix() { - return g_lowerTable.data() + 1; +const int* APS5_VABI _Getptolower_nid_postfix() { + static const auto table = MakeCaseTable(false); + return table.data() + 1; } -const short* APS5_VABI _Getptoupper_nid_postfix() { - return g_upperTable.data() + 1; +const int* APS5_VABI _Getptoupper_nid_postfix() { + static const auto table = MakeCaseTable(true); + return table.data() + 1; } -mbstate_t* APS5_VABI _Getpmbstate_nid_postfix() { - thread_local mbstate_t state {}; +std::mbstate_t* APS5_VABI _Getpmbstate_nid_postfix() { + thread_local std::mbstate_t state{}; return &state; } -mbstate_t* APS5_VABI _Getpwcstate_nid_postfix() { - thread_local mbstate_t state {}; +std::mbstate_t* APS5_VABI _Getpwcstate_nid_postfix() { + thread_local std::mbstate_t state{}; return &state; } -wint_t APS5_VABI _Towctrans_nid_postfix(wint_t c, wctrans_t desc) { +std::wint_t APS5_VABI _Towctrans_nid_postfix(std::wint_t c, std::wctrans_t desc) { + if (desc == std::wctrans_t{}) throw std::invalid_argument("_Towctrans: invalid transformation"); return std::towctrans(c, desc); } diff --git a/core/libs/tests/GuestCharacter.cpp b/core/libs/tests/GuestCharacter.cpp index 60331fc87e..ab299a4506 100644 --- a/core/libs/tests/GuestCharacter.cpp +++ b/core/libs/tests/GuestCharacter.cpp @@ -18,8 +18,8 @@ int APS5_VABI ispunct_nid_postfix(int); int APS5_VABI isxdigit_nid_postfix(int); int APS5_VABI toupper_nid_postfix(int); int APS5_VABI tolower_nid_postfix(int); -const short* APS5_VABI _Getptolower_nid_postfix(); -const short* APS5_VABI _Getptoupper_nid_postfix(); +const int* APS5_VABI _Getptolower_nid_postfix(); +const int* APS5_VABI _Getptoupper_nid_postfix(); } static void Require(bool value) { if (!value) std::abort(); } int main() { diff --git a/core/libs/tests/GuestLocale.cpp b/core/libs/tests/GuestLocale.cpp deleted file mode 100644 index dc1635dd96..0000000000 --- a/core/libs/tests/GuestLocale.cpp +++ /dev/null @@ -1,158 +0,0 @@ -#include "prx/libc/include/GuestLocale.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" { -extern GuestLocale::Implementation* _ZSt21_sceLibcClassicLocale_nid_postfix; -GuestLocale::Implementation* APS5_VABI _ZNSt6locale16_GetgloballocaleEv_nid_postfix(); -GuestLocale::Implementation* APS5_VABI _ZNSt6locale5_InitEv_nid_postfix(); -void APS5_VABI _ZNSt6locale5facet9_RegisterEv_nid_postfix(GuestLocale::Facet* self); -void APS5_VABI _ZNSt8_LocinfoC1EPKc_nid_postfix(GuestLocale::LocinfoStorage* self, const char* localeName); -void APS5_VABI _ZNSt8_LocinfoD1Ev_nid_postfix(GuestLocale::LocinfoStorage* self); -void APS5_VABI _ZNSt8ios_baseD2Ev_nid_postfix(GuestLocale::IosBase* self); -const short* APS5_VABI _Getpctype_nid_postfix(); -const short* APS5_VABI _Getptolower_nid_postfix(); -const short* APS5_VABI _Getptoupper_nid_postfix(); -int APS5_VABI _Mbtowcx_nid_postfix(std::uint16_t* dst, const char* src, std::size_t count, std::mbstate_t* st); -int APS5_VABI _Wctombx_nid_postfix(char* dst, std::uint16_t src, std::mbstate_t* st); -void APS5_VABI _Locksyslock_nid_postfix(); -void APS5_VABI _Unlocksyslock_nid_postfix(); - -} - -static void Require(bool condition) { - if (!condition) throw std::runtime_error("Guest locale check failed"); -} - -template -static void RequireException(TAction action) { - bool caught = false; - try { - action(); - } catch (const std::exception&) { - caught = true; - } - Require(caught); -} - -static void CheckGuestCalls() { - auto* locale = _ZSt21_sceLibcClassicLocale_nid_postfix; - Require(locale != nullptr && locale == _ZNSt6locale16_GetgloballocaleEv_nid_postfix()); - Require(_ZNSt6locale5_InitEv_nid_postfix() == locale); - Require(_ZNSt6locale5_InitEv_nid_postfix() == locale); - const std::byte* table = nullptr; - std::memcpy(&table, locale, sizeof(table)); - void (APS5_VABI *retain)(GuestLocale::Facet*) = nullptr; - GuestLocale::Facet* (APS5_VABI *release)(GuestLocale::Facet*) = nullptr; - std::memcpy(&retain, table + 0x10, sizeof(retain)); - std::memcpy(&release, table + 0x18, sizeof(release)); - const auto initial = locale->base.references; - retain(&locale->base); - Require(locale->base.references == initial + 1); - Require(release(&locale->base) == nullptr); - Require(locale->base.references == initial); - RequireException([&] { release(&locale->base); }); - RequireException([&] { retain(nullptr); }); - RequireException([&] { locale->base.vtable->deleteObject(&locale->base); }); - Require(locale->base.references == initial); - std::vector threads; - for (int thread = 0; thread < 4; ++thread) { - threads.emplace_back([&] { - for (int iteration = 0; iteration < 10000; ++iteration) { - retain(&locale->base); - release(&locale->base); - } - }); - } - for (auto& thread : threads) thread.join(); - Require(locale->base.references == initial); - Require(locale->facetCount == 1 && locale->facets[0] == nullptr); - Require(std::strcmp(locale->name, "C") == 0 && !locale->transparent); -} - -static void CheckLocinfoAlignment() { - alignas(16) std::array bytes; - bytes.fill(std::byte{0x5a}); - auto* storage = reinterpret_cast(bytes.data() + 8); - _ZNSt8_LocinfoC1EPKc_nid_postfix(storage, "C"); - for (std::size_t index = 0; index < bytes.size(); ++index) Require(bytes[index] == (index >= 8 && index < 72 ? std::byte{} : std::byte{0x5a})); - _ZNSt8_LocinfoD1Ev_nid_postfix(storage); - RequireException([&] { _ZNSt8_LocinfoC1EPKc_nid_postfix(storage, "unknown-locale"); }); - RequireException([] { _ZNSt8_LocinfoC1EPKc_nid_postfix(nullptr, "C"); }); -} - -static void CheckCharacterTables() { - const auto* classification = _Getpctype_nid_postfix(); - const auto* lower = _Getptolower_nid_postfix(); - const auto* upper = _Getptoupper_nid_postfix(); - Require(classification[-1] == 0 && lower[-1] == -1 && upper[-1] == -1); - Require(classification['A'] == 0x03 && classification['a'] == 0x11); - Require(classification['G'] == 0x02 && classification['0'] == 0x21); - Require(classification[' '] == 0x04 && classification['\t'] == 0xc0); - Require(classification['\n'] == 0xc0 && classification['!'] == 0x08); - for (int value = 0; value < 256; ++value) { - Require(lower[value] == (value >= 'A' && value <= 'Z' ? value + 32 : value)); - Require(upper[value] == (value >= 'a' && value <= 'z' ? value - 32 : value)); - if (value >= 128) Require(classification[value] == 0); - } -} - -static void CheckCharacterConversions() { - const auto* classification = _Getpctype_nid_postfix(); - for (std::uint16_t value = 0; value < 128; ++value) { - std::array bytes{}; - bytes.fill('!'); - std::mbstate_t encodeState{}; - Require(_Wctombx_nid_postfix(bytes.data(), value, &encodeState) == 1); - Require(bytes[0] == static_cast(value) && bytes[1] == '!'); - std::array wide{0xffff, 0x1234}; - std::mbstate_t decodeState{}; - Require(_Mbtowcx_nid_postfix(wide.data(), bytes.data(), 2, &decodeState) == (value == 0 ? 0 : 1)); - Require(wide[0] == value && wide[1] == 0x1234); - const bool alnum = (value >= '0' && value <= '9') || (value >= 'A' && value <= 'Z') || (value >= 'a' && value <= 'z'); - Require(((classification[static_cast(bytes[0])] & 0x232) != 0) == alnum); - } - std::mbstate_t state{}; - char byte{}; - std::uint16_t wide{}; - RequireException([&] { _Mbtowcx_nid_postfix(&wide, "]", 0, &state); }); - RequireException([&] { _Mbtowcx_nid_postfix(nullptr, "]", 1, &state); }); - RequireException([&] { _Mbtowcx_nid_postfix(&wide, nullptr, 1, &state); }); - RequireException([&] { _Mbtowcx_nid_postfix(&wide, "]", 1, nullptr); }); - RequireException([&] { _Wctombx_nid_postfix(nullptr, ']', &state); }); - RequireException([&] { _Wctombx_nid_postfix(&byte, ']', nullptr); }); -} - -static void CheckStreamDestruction() { - struct GuardedStream { - std::uint64_t before = 0x12345678; - GuestLocale::IosBase stream{}; - std::uint64_t after = 0x87654321; - } object; - object.stream.locale = &_ZSt21_sceLibcClassicLocale_nid_postfix; - _ZNSt8ios_baseD2Ev_nid_postfix(&object.stream); - Require(object.stream.locale == nullptr && object.before == 0x12345678 && object.after == 0x87654321); - object.stream.locale = &_ZSt21_sceLibcClassicLocale_nid_postfix; - object.stream.callbacks = &object; - RequireException([&] { _ZNSt8ios_baseD2Ev_nid_postfix(&object.stream); }); - Require(object.stream.locale == &_ZSt21_sceLibcClassicLocale_nid_postfix); -} - -int main() { - CheckGuestCalls(); - CheckLocinfoAlignment(); - CheckCharacterTables(); - CheckCharacterConversions(); - CheckStreamDestruction(); - _Locksyslock_nid_postfix(); - _Locksyslock_nid_postfix(); - _Unlocksyslock_nid_postfix(); - _Unlocksyslock_nid_postfix(); - std::cout << "Guest locale checks passed\n"; -} From 52b683d20845481ee44022608af98616b4f7bfe6 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sun, 27 Sep 2026 10:23:28 +0300 Subject: [PATCH 41/93] fix(relinker): skip generated guest prx files during sce module scan --- core/relinker/relinker/include/relinker/guest/GuestImage.hpp | 2 ++ core/relinker/relinker/src/guest/GuestImageReader.cpp | 2 +- core/relinker/relinker/src/guest/GuestModuleBuilder.cpp | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/relinker/relinker/include/relinker/guest/GuestImage.hpp b/core/relinker/relinker/include/relinker/guest/GuestImage.hpp index a71a332893..71cca90fa9 100644 --- a/core/relinker/relinker/include/relinker/guest/GuestImage.hpp +++ b/core/relinker/relinker/include/relinker/guest/GuestImage.hpp @@ -7,6 +7,8 @@ namespace Relinker { +inline constexpr char GuestModuleSuffix[] = ".guest.prx"; + struct GuestSymbol { std::string Name; std::uint8_t Info; diff --git a/core/relinker/relinker/src/guest/GuestImageReader.cpp b/core/relinker/relinker/src/guest/GuestImageReader.cpp index 5b3a8abc88..d0e2afb410 100644 --- a/core/relinker/relinker/src/guest/GuestImageReader.cpp +++ b/core/relinker/relinker/src/guest/GuestImageReader.cpp @@ -27,7 +27,7 @@ GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector range(Io::ReadU64(bytes, 32), static_cast(Io::ReadU16(bytes, 56)) * 56); GuestImage image; image.SourcePath = path; - image.OutputName = path.filename().string() + ".guest.prx"; + image.OutputName = path.filename().string() + GuestModuleSuffix; image.Headers = ElfReader(bytes).ReadProgramHeaders(); const Domain::ProgramHeader* dynamic = nullptr; const Domain::ProgramHeader* dynlib = nullptr; diff --git a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp index c461ffb3c9..f8fd3e8031 100644 --- a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp +++ b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp @@ -26,6 +26,7 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path if (!std::filesystem::is_directory(directory)) throw Domain::RelinkerException("Guest module path is not a directory: " + directory.string()); std::vector paths; for (const auto& entry : std::filesystem::directory_iterator(directory)) { + if (entry.path().filename().string().ends_with(GuestModuleSuffix)) continue; if (!entry.is_regular_file()) continue; std::ifstream stream(entry.path(), std::ios::binary); if (!stream) throw Domain::RelinkerException("Cannot read guest candidate: " + entry.path().string()); From e3dc9a6acc6ee85520524e98e5be468c70c77d43 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sun, 27 Sep 2026 12:41:52 +0300 Subject: [PATCH 42/93] fix(relinker): accept zero alignment for empty tls segments --- core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp | 2 +- core/relinker/relinker/src/guest/GuestImageReader.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp index 58ee3eb622..f0dd778e70 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp @@ -91,7 +91,7 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co return {}; } - if (tls->FileSize > tls->MemorySize || tls->Offset > source.size() || tls->FileSize > source.size() - tls->Offset || !std::has_single_bit(tls->Alignment) || tls->Alignment > 8192 || tls->MemorySize > 0x7fff0000u) + if (tls->FileSize > tls->MemorySize || tls->Offset > source.size() || tls->FileSize > source.size() - tls->Offset || (tls->Alignment > 1 && !std::has_single_bit(tls->Alignment)) || tls->Alignment > 8192 || tls->MemorySize > 0x7fff0000u) throw Domain::RelinkerException("Invalid or unsupported ELF TLS layout", tls->Offset); // Native homebrew linkers can emit an empty PT_TLS placeholder. // It needs no Windows TLS directory unless guest code accesses TLS. diff --git a/core/relinker/relinker/src/guest/GuestImageReader.cpp b/core/relinker/relinker/src/guest/GuestImageReader.cpp index d0e2afb410..8dec3cf8ac 100644 --- a/core/relinker/relinker/src/guest/GuestImageReader.cpp +++ b/core/relinker/relinker/src/guest/GuestImageReader.cpp @@ -54,7 +54,7 @@ GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector dynlib = &header; } if (header.Type == 7) { - if (tls != nullptr || header.FileSize > header.MemorySize || !std::has_single_bit(header.Alignment)) fail("Invalid TLS segment"); + if (tls != nullptr || header.FileSize > header.MemorySize || (header.Alignment > 1 && !std::has_single_bit(header.Alignment))) fail("Invalid TLS segment"); tls = &header; } } From 43d2e8b4dc41d3bfcf7b5934e11c1ef21704b0b0 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sun, 27 Sep 2026 12:49:46 +0300 Subject: [PATCH 43/93] fix(codegen): decode xadd byte and popcnt instruction lengths fix --- .../codegen/include/codegen/x86/X64OpcodeConstants.hpp | 2 ++ core/relinker/codegen/src/x86/X64InstructionDecoder.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp index 237fb8bb72..7448093dfa 100644 --- a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp +++ b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp @@ -153,6 +153,8 @@ inline constexpr std::uint8_t TwoByteShldCl = 0xA5; inline constexpr std::uint8_t TwoByteShrdImm8 = 0xAC; inline constexpr std::uint8_t TwoByteShrdCl = 0xAD; inline constexpr std::uint8_t TwoByteGrp15 = 0xAE; +inline constexpr std::uint8_t TwoBytePopcnt = 0xB8; +inline constexpr std::uint8_t TwoByteXaddRm8 = 0xC0; inline constexpr std::uint8_t TwoByteXadd = 0xC1; inline constexpr std::uint8_t TwoBytePextrw = 0xC5; inline constexpr std::uint8_t TwoByteShufpsImm8 = 0xC2; diff --git a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp index d574fef6c1..665cf6fd24 100644 --- a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp +++ b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp @@ -246,7 +246,8 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t opcode == TwoByteImulRmModRm || opcode == TwoByteGrp7 || opcode == TwoByteGrp15 || - opcode == TwoByteXadd || + opcode == TwoBytePopcnt || + opcode == TwoByteXaddRm8 || opcode == TwoByteXadd || opcode == TwoByteGrp9 || opcode == TwoByteNopModRm || opcode == TwoByteEndbr) { hasModRm = true; @@ -497,7 +498,8 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( (op >= TwoBytePrefetchGrpMin && op <= TwoBytePrefetchGrpMax) || (op >= TwoByteSetccMin && op <= TwoByteSetccMax) || op == TwoByteImulRmModRm || op == TwoByteGrp7 || - op == TwoByteGrp15 || op == TwoByteXadd || + op == TwoBytePopcnt || + op == TwoByteGrp15 || op == TwoByteXaddRm8 || op == TwoByteXadd || op == TwoByteGrp9 || op == TwoByteNopModRm || op == TwoByteEndbr || op == TwoByteMovImm8ModRm || (op >= TwoByteShiftImm8Min && op <= TwoByteShiftImm8Max) || From 0722f3e5fc92ba684832842aa5d8dd88e1ee2cea Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 11:57:42 +0200 Subject: [PATCH 44/93] prx: add libRenoirCore.PS5 replacement with 23 Renoir stubs for libcohtml Minecraft Legends' libcohtml.Prospero.prx imports 23 NIDs from lib libRenoirCore.PS5_v1 (docs/legends-unresolved-nids.json). Clear names are unknown, so each export is exposed under its raw NID via APS5_EXPORT alias (_nid_no_patch_cut), which nid_patcher strips to the literal NID instead of hashing. Stubs follow KytyPS5 UnresolvedImportStub policy (log call + return 0) via APS5_LOG_OUT rather than throwing NotImplemented. --- core/libs/prx/libRenoirCore.PS5/Export.cpp | 158 +++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 core/libs/prx/libRenoirCore.PS5/Export.cpp diff --git a/core/libs/prx/libRenoirCore.PS5/Export.cpp b/core/libs/prx/libRenoirCore.PS5/Export.cpp new file mode 100644 index 0000000000..a502a85354 --- /dev/null +++ b/core/libs/prx/libRenoirCore.PS5/Export.cpp @@ -0,0 +1,158 @@ +#include +#include + +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +// libRenoirCore.PS5 replacement (game-custom renderer lib, e.g. Minecraft Legends). +// +// The 23 NIDs below are imported by libcohtml.Prospero.prx (lib libRenoirCore.PS5_v1) +// and their clear names are unknown (obfuscated NIDs), so each export is exposed +// with its raw NID string via APS5_EXPORT (alias "_nid_no_patch_cut", which the +// nid_patcher strips to the literal NID instead of hashing it). +// +// Stub policy mirrors KytyPS5's UnresolvedImportStub (log the call, return 0) rather +// than the local NotImplemented_nid_no_patch helper (which throws): Renoir calls may +// happen per-frame and must not unwind through the guest. + +extern "C" { + +APS5_EXPORT("0DsDTr534gc", renoirStub00); +uint64_t APS5_VABI renoirStub00(void) { + APS5_LOG_OUT("stub NID=%s called", "0DsDTr534gc"); + return 0; +} + +APS5_EXPORT("2T56HgbPW08", renoirStub01); +uint64_t APS5_VABI renoirStub01(void) { + APS5_LOG_OUT("stub NID=%s called", "2T56HgbPW08"); + return 0; +} + +APS5_EXPORT("3HnCozh8Az8", renoirStub02); +uint64_t APS5_VABI renoirStub02(void) { + APS5_LOG_OUT("stub NID=%s called", "3HnCozh8Az8"); + return 0; +} + +APS5_EXPORT("5Ysce+s-CWU", renoirStub03); +uint64_t APS5_VABI renoirStub03(void) { + APS5_LOG_OUT("stub NID=%s called", "5Ysce+s-CWU"); + return 0; +} + +APS5_EXPORT("AdlSqA4lGS8", renoirStub04); +uint64_t APS5_VABI renoirStub04(void) { + APS5_LOG_OUT("stub NID=%s called", "AdlSqA4lGS8"); + return 0; +} + +APS5_EXPORT("bC37hk7rHqs", renoirStub05); +uint64_t APS5_VABI renoirStub05(void) { + APS5_LOG_OUT("stub NID=%s called", "bC37hk7rHqs"); + return 0; +} + +APS5_EXPORT("dfPK9R6M0qM", renoirStub06); +uint64_t APS5_VABI renoirStub06(void) { + APS5_LOG_OUT("stub NID=%s called", "dfPK9R6M0qM"); + return 0; +} + +APS5_EXPORT("fZiBbVa2ZDU", renoirStub07); +uint64_t APS5_VABI renoirStub07(void) { + APS5_LOG_OUT("stub NID=%s called", "fZiBbVa2ZDU"); + return 0; +} + +APS5_EXPORT("hkWBH8-WfW4", renoirStub08); +uint64_t APS5_VABI renoirStub08(void) { + APS5_LOG_OUT("stub NID=%s called", "hkWBH8-WfW4"); + return 0; +} + +APS5_EXPORT("hUxvGdi64Io", renoirStub09); +uint64_t APS5_VABI renoirStub09(void) { + APS5_LOG_OUT("stub NID=%s called", "hUxvGdi64Io"); + return 0; +} + +APS5_EXPORT("NxqsZxGisrA", renoirStub10); +uint64_t APS5_VABI renoirStub10(void) { + APS5_LOG_OUT("stub NID=%s called", "NxqsZxGisrA"); + return 0; +} + +APS5_EXPORT("OgZVIkaNmb0", renoirStub11); +uint64_t APS5_VABI renoirStub11(void) { + APS5_LOG_OUT("stub NID=%s called", "OgZVIkaNmb0"); + return 0; +} + +APS5_EXPORT("ojLQdRUnVgA", renoirStub12); +uint64_t APS5_VABI renoirStub12(void) { + APS5_LOG_OUT("stub NID=%s called", "ojLQdRUnVgA"); + return 0; +} + +APS5_EXPORT("s0qDfgsAgVg", renoirStub13); +uint64_t APS5_VABI renoirStub13(void) { + APS5_LOG_OUT("stub NID=%s called", "s0qDfgsAgVg"); + return 0; +} + +APS5_EXPORT("sgrd-r1Nw60", renoirStub14); +uint64_t APS5_VABI renoirStub14(void) { + APS5_LOG_OUT("stub NID=%s called", "sgrd-r1Nw60"); + return 0; +} + +APS5_EXPORT("sjTRb8YD7dQ", renoirStub15); +uint64_t APS5_VABI renoirStub15(void) { + APS5_LOG_OUT("stub NID=%s called", "sjTRb8YD7dQ"); + return 0; +} + +APS5_EXPORT("tG86XGc2VtE", renoirStub16); +uint64_t APS5_VABI renoirStub16(void) { + APS5_LOG_OUT("stub NID=%s called", "tG86XGc2VtE"); + return 0; +} + +APS5_EXPORT("umlGxOcNzgc", renoirStub17); +uint64_t APS5_VABI renoirStub17(void) { + APS5_LOG_OUT("stub NID=%s called", "umlGxOcNzgc"); + return 0; +} + +APS5_EXPORT("vPxA9OFW0os", renoirStub18); +uint64_t APS5_VABI renoirStub18(void) { + APS5_LOG_OUT("stub NID=%s called", "vPxA9OFW0os"); + return 0; +} + +APS5_EXPORT("X2qEBlOLkNo", renoirStub19); +uint64_t APS5_VABI renoirStub19(void) { + APS5_LOG_OUT("stub NID=%s called", "X2qEBlOLkNo"); + return 0; +} + +APS5_EXPORT("ynh4E+BIaqY", renoirStub20); +uint64_t APS5_VABI renoirStub20(void) { + APS5_LOG_OUT("stub NID=%s called", "ynh4E+BIaqY"); + return 0; +} + +APS5_EXPORT("YXe6ABbHZPs", renoirStub21); +uint64_t APS5_VABI renoirStub21(void) { + APS5_LOG_OUT("stub NID=%s called", "YXe6ABbHZPs"); + return 0; +} + +APS5_EXPORT("z0jsXZVz75I", renoirStub22); +uint64_t APS5_VABI renoirStub22(void) { + APS5_LOG_OUT("stub NID=%s called", "z0jsXZVz75I"); + return 0; +} + +} From a4248cb1142531a18ac1b6a71c65e176ecefb1d7 Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 12:00:15 +0200 Subject: [PATCH 45/93] feat(libcohtml): add stub prx exporting 50 Minecraft Legends NIDs --- core/libs/prx/libcohtml/CMakeLists.txt | 6 + core/libs/prx/libcohtml/Export.cpp | 308 +++++++++++++++++++++++++ 2 files changed, 314 insertions(+) create mode 100644 core/libs/prx/libcohtml/CMakeLists.txt create mode 100644 core/libs/prx/libcohtml/Export.cpp diff --git a/core/libs/prx/libcohtml/CMakeLists.txt b/core/libs/prx/libcohtml/CMakeLists.txt new file mode 100644 index 0000000000..842799d34f --- /dev/null +++ b/core/libs/prx/libcohtml/CMakeLists.txt @@ -0,0 +1,6 @@ +add_library(libcohtml SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp +) +target_include_directories(libcohtml PRIVATE ${LIBS_INCLUDE_DIR}) +set_target_properties(libcohtml PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(libcohtml) diff --git a/core/libs/prx/libcohtml/Export.cpp b/core/libs/prx/libcohtml/Export.cpp new file mode 100644 index 0000000000..fae3870a5f --- /dev/null +++ b/core/libs/prx/libcohtml/Export.cpp @@ -0,0 +1,308 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +APS5_EXPORT("1fOA53oGwU8", cohtml_stub_00); +std::uint64_t APS5_VABI cohtml_stub_00(void) { + std::printf("[libcohtml] 1fOA53oGwU8 called\n"); + return 0; +} + +APS5_EXPORT("2vxXHkZlhbw", cohtml_stub_01); +std::uint64_t APS5_VABI cohtml_stub_01(void) { + std::printf("[libcohtml] 2vxXHkZlhbw called\n"); + return 0; +} + +APS5_EXPORT("3rew70M0SmY", cohtml_stub_02); +std::uint64_t APS5_VABI cohtml_stub_02(void) { + std::printf("[libcohtml] 3rew70M0SmY called\n"); + return 0; +} + +APS5_EXPORT("4Le0SBPvnF8", cohtml_stub_03); +std::uint64_t APS5_VABI cohtml_stub_03(void) { + std::printf("[libcohtml] 4Le0SBPvnF8 called\n"); + return 0; +} + +APS5_EXPORT("53ZTBeo2a54", cohtml_stub_04); +std::uint64_t APS5_VABI cohtml_stub_04(void) { + std::printf("[libcohtml] 53ZTBeo2a54 called\n"); + return 0; +} + +APS5_EXPORT("6PaNuQipllM", cohtml_stub_05); +std::uint64_t APS5_VABI cohtml_stub_05(void) { + std::printf("[libcohtml] 6PaNuQipllM called\n"); + return 0; +} + +APS5_EXPORT("7hn5DAs2bLk", cohtml_stub_06); +std::uint64_t APS5_VABI cohtml_stub_06(void) { + std::printf("[libcohtml] 7hn5DAs2bLk called\n"); + return 0; +} + +APS5_EXPORT("8EYWdAM2-R0", cohtml_stub_07); +std::uint64_t APS5_VABI cohtml_stub_07(void) { + std::printf("[libcohtml] 8EYWdAM2-R0 called\n"); + return 0; +} + +APS5_EXPORT("9bGvfp1zbKo", cohtml_stub_08); +std::uint64_t APS5_VABI cohtml_stub_08(void) { + std::printf("[libcohtml] 9bGvfp1zbKo called\n"); + return 0; +} + +APS5_EXPORT("a4E5dJh0onE", cohtml_stub_09); +std::uint64_t APS5_VABI cohtml_stub_09(void) { + std::printf("[libcohtml] a4E5dJh0onE called\n"); + return 0; +} + +APS5_EXPORT("BhL-ZtSHmB8", cohtml_stub_10); +std::uint64_t APS5_VABI cohtml_stub_10(void) { + std::printf("[libcohtml] BhL-ZtSHmB8 called\n"); + return 0; +} + +APS5_EXPORT("blvwzOvsEAA", cohtml_stub_11); +std::uint64_t APS5_VABI cohtml_stub_11(void) { + std::printf("[libcohtml] blvwzOvsEAA called\n"); + return 0; +} + +APS5_EXPORT("dko-cMqv5yw", cohtml_stub_12); +std::uint64_t APS5_VABI cohtml_stub_12(void) { + std::printf("[libcohtml] dko-cMqv5yw called\n"); + return 0; +} + +APS5_EXPORT("EG4kqkxLU3Q", cohtml_stub_13); +std::uint64_t APS5_VABI cohtml_stub_13(void) { + std::printf("[libcohtml] EG4kqkxLU3Q called\n"); + return 0; +} + +APS5_EXPORT("-El4B0xJF2U", cohtml_stub_14); +std::uint64_t APS5_VABI cohtml_stub_14(void) { + std::printf("[libcohtml] -El4B0xJF2U called\n"); + return 0; +} + +APS5_EXPORT("EMhQNylkSwk", cohtml_stub_15); +std::uint64_t APS5_VABI cohtml_stub_15(void) { + std::printf("[libcohtml] EMhQNylkSwk called\n"); + return 0; +} + +APS5_EXPORT("-eNSKAypRz0", cohtml_stub_16); +std::uint64_t APS5_VABI cohtml_stub_16(void) { + std::printf("[libcohtml] -eNSKAypRz0 called\n"); + return 0; +} + +APS5_EXPORT("FG9VRZfson0", cohtml_stub_17); +std::uint64_t APS5_VABI cohtml_stub_17(void) { + std::printf("[libcohtml] FG9VRZfson0 called\n"); + return 0; +} + +APS5_EXPORT("fNvtExaHN4U", cohtml_stub_18); +std::uint64_t APS5_VABI cohtml_stub_18(void) { + std::printf("[libcohtml] fNvtExaHN4U called\n"); + return 0; +} + +APS5_EXPORT("g10DkPARoa8", cohtml_stub_19); +std::uint64_t APS5_VABI cohtml_stub_19(void) { + std::printf("[libcohtml] g10DkPARoa8 called\n"); + return 0; +} + +APS5_EXPORT("gwQczxoDmG0", cohtml_stub_20); +std::uint64_t APS5_VABI cohtml_stub_20(void) { + std::printf("[libcohtml] gwQczxoDmG0 called\n"); + return 0; +} + +APS5_EXPORT("HQF4O9D+Di4", cohtml_stub_21); +std::uint64_t APS5_VABI cohtml_stub_21(void) { + std::printf("[libcohtml] HQF4O9D+Di4 called\n"); + return 0; +} + +APS5_EXPORT("igt3a6Q0VMc", cohtml_stub_22); +std::uint64_t APS5_VABI cohtml_stub_22(void) { + std::printf("[libcohtml] igt3a6Q0VMc called\n"); + return 0; +} + +APS5_EXPORT("kyApuhEHHxQ", cohtml_stub_23); +std::uint64_t APS5_VABI cohtml_stub_23(void) { + std::printf("[libcohtml] kyApuhEHHxQ called\n"); + return 0; +} + +APS5_EXPORT("Ljp75QXUi7s", cohtml_stub_24); +std::uint64_t APS5_VABI cohtml_stub_24(void) { + std::printf("[libcohtml] Ljp75QXUi7s called\n"); + return 0; +} + +APS5_EXPORT("loJ5Ef3MZrs", cohtml_stub_25); +std::uint64_t APS5_VABI cohtml_stub_25(void) { + std::printf("[libcohtml] loJ5Ef3MZrs called\n"); + return 0; +} + +APS5_EXPORT("LWe+u8SHKgU", cohtml_stub_26); +std::uint64_t APS5_VABI cohtml_stub_26(void) { + std::printf("[libcohtml] LWe+u8SHKgU called\n"); + return 0; +} + +APS5_EXPORT("nhn1ds+I2t4", cohtml_stub_27); +std::uint64_t APS5_VABI cohtml_stub_27(void) { + std::printf("[libcohtml] nhn1ds+I2t4 called\n"); + return 0; +} + +APS5_EXPORT("NmVsRWgsKt4", cohtml_stub_28); +std::uint64_t APS5_VABI cohtml_stub_28(void) { + std::printf("[libcohtml] NmVsRWgsKt4 called\n"); + return 0; +} + +APS5_EXPORT("ONEnhMinebs", cohtml_stub_29); +std::uint64_t APS5_VABI cohtml_stub_29(void) { + std::printf("[libcohtml] ONEnhMinebs called\n"); + return 0; +} + +APS5_EXPORT("PdOmR8dy1fM", cohtml_stub_30); +std::uint64_t APS5_VABI cohtml_stub_30(void) { + std::printf("[libcohtml] PdOmR8dy1fM called\n"); + return 0; +} + +APS5_EXPORT("PSwGBN5Icv8", cohtml_stub_31); +std::uint64_t APS5_VABI cohtml_stub_31(void) { + std::printf("[libcohtml] PSwGBN5Icv8 called\n"); + return 0; +} + +APS5_EXPORT("RalyfwiTt8o", cohtml_stub_32); +std::uint64_t APS5_VABI cohtml_stub_32(void) { + std::printf("[libcohtml] RalyfwiTt8o called\n"); + return 0; +} + +APS5_EXPORT("t+OHUjVWIwU", cohtml_stub_33); +std::uint64_t APS5_VABI cohtml_stub_33(void) { + std::printf("[libcohtml] t+OHUjVWIwU called\n"); + return 0; +} + +APS5_EXPORT("Ter8YCVZ9tQ", cohtml_stub_34); +std::uint64_t APS5_VABI cohtml_stub_34(void) { + std::printf("[libcohtml] Ter8YCVZ9tQ called\n"); + return 0; +} + +APS5_EXPORT("tG3LJfAOH3k", cohtml_stub_35); +std::uint64_t APS5_VABI cohtml_stub_35(void) { + std::printf("[libcohtml] tG3LJfAOH3k called\n"); + return 0; +} + +APS5_EXPORT("ukSFhKPmCTg", cohtml_stub_36); +std::uint64_t APS5_VABI cohtml_stub_36(void) { + std::printf("[libcohtml] ukSFhKPmCTg called\n"); + return 0; +} + +APS5_EXPORT("V7KGze2XjJc", cohtml_stub_37); +std::uint64_t APS5_VABI cohtml_stub_37(void) { + std::printf("[libcohtml] V7KGze2XjJc called\n"); + return 0; +} + +APS5_EXPORT("vlmAvJqR7mQ", cohtml_stub_38); +std::uint64_t APS5_VABI cohtml_stub_38(void) { + std::printf("[libcohtml] vlmAvJqR7mQ called\n"); + return 0; +} + +APS5_EXPORT("VzhxjXDzYSI", cohtml_stub_39); +std::uint64_t APS5_VABI cohtml_stub_39(void) { + std::printf("[libcohtml] VzhxjXDzYSI called\n"); + return 0; +} + +APS5_EXPORT("X3FzDPzpD2Y", cohtml_stub_40); +std::uint64_t APS5_VABI cohtml_stub_40(void) { + std::printf("[libcohtml] X3FzDPzpD2Y called\n"); + return 0; +} + +APS5_EXPORT("x68HPh+vxsE", cohtml_stub_41); +std::uint64_t APS5_VABI cohtml_stub_41(void) { + std::printf("[libcohtml] x68HPh+vxsE called\n"); + return 0; +} + +APS5_EXPORT("XUe8OAKB-vQ", cohtml_stub_42); +std::uint64_t APS5_VABI cohtml_stub_42(void) { + std::printf("[libcohtml] XUe8OAKB-vQ called\n"); + return 0; +} + +APS5_EXPORT("y6UNQtnr0wY", cohtml_stub_43); +std::uint64_t APS5_VABI cohtml_stub_43(void) { + std::printf("[libcohtml] y6UNQtnr0wY called\n"); + return 0; +} + +APS5_EXPORT("YoHAXkBV06c", cohtml_stub_44); +std::uint64_t APS5_VABI cohtml_stub_44(void) { + std::printf("[libcohtml] YoHAXkBV06c called\n"); + return 0; +} + +APS5_EXPORT("Z9XFJCfmId4", cohtml_stub_45); +std::uint64_t APS5_VABI cohtml_stub_45(void) { + std::printf("[libcohtml] Z9XFJCfmId4 called\n"); + return 0; +} + +APS5_EXPORT("zEniuuzG2W0", cohtml_stub_46); +std::uint64_t APS5_VABI cohtml_stub_46(void) { + std::printf("[libcohtml] zEniuuzG2W0 called\n"); + return 0; +} + +APS5_EXPORT("zH9Z2nK1fmA", cohtml_stub_47); +std::uint64_t APS5_VABI cohtml_stub_47(void) { + std::printf("[libcohtml] zH9Z2nK1fmA called\n"); + return 0; +} + +APS5_EXPORT("ZHGgdWeYTo8", cohtml_stub_48); +std::uint64_t APS5_VABI cohtml_stub_48(void) { + std::printf("[libcohtml] ZHGgdWeYTo8 called\n"); + return 0; +} + +APS5_EXPORT("ZWftPxZ4E6k", cohtml_stub_49); +std::uint64_t APS5_VABI cohtml_stub_49(void) { + std::printf("[libcohtml] ZWftPxZ4E6k called\n"); + return 0; +} + +} From 179841caff2945c6d6c4f2f77d054e47269fb27a Mon Sep 17 00:00:00 2001 From: boykopovar Date: Sun, 27 Sep 2026 13:10:06 +0300 Subject: [PATCH 46/93] feat(relinker): report full module path when windows loading fails --- .../elfpatcher/src/windows/WindowsEntryStubBuilder.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp index c0bde96688..94d25b832b 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp @@ -82,6 +82,7 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con const auto loading = addString("Loading PRX: "); const auto loaded = addString(" -> OK\n"); const auto loadFailed = addString(" -> FAILED\n"); + const auto failedModule = addString("Failed to load module: "); const auto errorPrefix = addString("GetLastError: "); const auto messageSeparator = addString(" - "); const auto newline = addString("\n"); @@ -257,6 +258,9 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con const auto loadSucceeded = code.Branch({0x0f, 0x85}); captureLastError(); writeString(loadFailed, true); + writeString(failedModule, true); + writeString(resolvedPaths[index], true); + writeString(newline, true); writeLastError(); if (dependencyDiagnostics) { code.Rip({0x48, 0x8d, 0x0d}, resolvedPaths[index]); From c0515bdcce3ab1d089c0319f5551b8f936280776 Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 12:32:13 +0200 Subject: [PATCH 47/93] feat(prx): add libfmod and libfmodstudio replacements covering Legends NIDs --- core/libs/prx/libfmod/CMakeLists.txt | 10 + core/libs/prx/libfmod/Export.cpp | 1435 ++++++++++++++++++++ core/libs/prx/libfmodstudio/CMakeLists.txt | 10 + core/libs/prx/libfmodstudio/Export.cpp | 870 ++++++++++++ 4 files changed, 2325 insertions(+) create mode 100644 core/libs/prx/libfmod/CMakeLists.txt create mode 100644 core/libs/prx/libfmod/Export.cpp create mode 100644 core/libs/prx/libfmodstudio/CMakeLists.txt create mode 100644 core/libs/prx/libfmodstudio/Export.cpp diff --git a/core/libs/prx/libfmod/CMakeLists.txt b/core/libs/prx/libfmod/CMakeLists.txt new file mode 100644 index 0000000000..a48f55b15d --- /dev/null +++ b/core/libs/prx/libfmod/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(libfmod SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp +) + +target_include_directories(libfmod PRIVATE + ${LIBS_INCLUDE_DIR} +) +target_link_libraries(libfmod PRIVATE libkernel libc) +target_compile_options(libfmod PRIVATE -fkeep-inline-functions) +configure_windows_unwind(libfmod) diff --git a/core/libs/prx/libfmod/Export.cpp b/core/libs/prx/libfmod/Export.cpp new file mode 100644 index 0000000000..334b9ffb0a --- /dev/null +++ b/core/libs/prx/libfmod/Export.cpp @@ -0,0 +1,1435 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +enum FMOD_OUTPUTTYPE { + FMOD_OUTPUTTYPE_AUTODETECT = 0, + FMOD_OUTPUTTYPE_UNKNOWN = 1, + FMOD_OUTPUTTYPE_NOSOUND = 2, + FMOD_OUTPUTTYPE_WAVWRITER = 3, + FMOD_OUTPUTTYPE_NOSOUND_NRT = 4, + FMOD_OUTPUTTYPE_WAVWRITER_NRT = 5, + FMOD_OUTPUTTYPE_WASAPI = 6, + FMOD_OUTPUTTYPE_ASIO = 7, + FMOD_OUTPUTTYPE_PULSEAUDIO = 8, + FMOD_OUTPUTTYPE_ALSA = 9, + FMOD_OUTPUTTYPE_COREAUDIO = 10, + FMOD_OUTPUTTYPE_AUDIOTRACK = 11, + FMOD_OUTPUTTYPE_OPENSL = 12, + FMOD_OUTPUTTYPE_AUDIOOUT = 13, + FMOD_OUTPUTTYPE_AUDIO3D = 14, + FMOD_OUTPUTTYPE_MAX = 15 +}; + +enum FMOD_SPEAKERMODE { + FMOD_SPEAKERMODE_DEFAULT = 0, + FMOD_SPEAKERMODE_RAW = 1, + FMOD_SPEAKERMODE_MONO = 2, + FMOD_SPEAKERMODE_STEREO = 3, + FMOD_SPEAKERMODE_QUAD = 4, + FMOD_SPEAKERMODE_SURROUND = 5, + FMOD_SPEAKERMODE_5POINT1 = 6, + FMOD_SPEAKERMODE_7POINT1 = 7, + FMOD_SPEAKERMODE_7POINT1POINT4 = 8, + FMOD_SPEAKERMODE_MAX = 9 +}; + +enum FMOD_SOUND_TYPE { + FMOD_SOUND_TYPE_UNKNOWN = 0, + FMOD_SOUND_TYPE_AIFF = 1, + FMOD_SOUND_TYPE_ASF = 2, + FMOD_SOUND_TYPE_DLS = 3, + FMOD_SOUND_TYPE_FLAC = 4, + FMOD_SOUND_TYPE_FSB = 5, + FMOD_SOUND_TYPE_IT = 6, + FMOD_SOUND_TYPE_MIDI = 7, + FMOD_SOUND_TYPE_MOD = 8, + FMOD_SOUND_TYPE_MPEG = 9, + FMOD_SOUND_TYPE_OGGVORBIS = 10, + FMOD_SOUND_TYPE_PLAYLIST = 11, + FMOD_SOUND_TYPE_RAW = 12, + FMOD_SOUND_TYPE_S3M = 13, + FMOD_SOUND_TYPE_USER = 14, + FMOD_SOUND_TYPE_XM = 15, + FMOD_SOUND_TYPE_XMA = 16, + FMOD_SOUND_TYPE_AUDIOQUEUE = 17, + FMOD_SOUND_TYPE_AT9 = 18, + FMOD_SOUND_TYPE_VB = 19, + FMOD_SOUND_TYPE_MAX = 20 +}; + +enum FMOD_SOUND_FORMAT { + FMOD_SOUND_FORMAT_NONE = 0, + FMOD_SOUND_FORMAT_PCM8 = 1, + FMOD_SOUND_FORMAT_PCM16 = 2, + FMOD_SOUND_FORMAT_PCM24 = 3, + FMOD_SOUND_FORMAT_PCM32 = 4, + FMOD_SOUND_FORMAT_PCMFLOAT = 5, + FMOD_SOUND_FORMAT_BITSTREAM = 6, + FMOD_SOUND_FORMAT_MAX = 7 +}; + +enum FMOD_OPENSTATE { + FMOD_OPENSTATE_READY = 0, + FMOD_OPENSTATE_LOADING = 1, + FMOD_OPENSTATE_ERROR = 2, + FMOD_OPENSTATE_CONNECTING = 3, + FMOD_OPENSTATE_BUFFERING = 4, + FMOD_OPENSTATE_SEEKING = 5, + FMOD_OPENSTATE_PLAYING = 6, + FMOD_OPENSTATE_SETPOSITION = 7, + FMOD_OPENSTATE_MAX = 8 +}; + +enum FMOD_SOUNDGROUP_BEHAVIOR { + FMOD_SOUNDGROUP_BEHAVIOR_FAIL = 0, + FMOD_SOUNDGROUP_BEHAVIOR_MUTE = 1, + FMOD_SOUNDGROUP_BEHAVIOR_STEALLOWEST = 2, + FMOD_SOUNDGROUP_BEHAVIOR_MAX = 3 +}; + +enum FMOD_DSP_TYPE { + FMOD_DSP_TYPE_UNKNOWN = 0, + FMOD_DSP_TYPE_MIXER = 1, + FMOD_DSP_TYPE_OSCILLATOR = 2, + FMOD_DSP_TYPE_LOWPASS = 3, + FMOD_DSP_TYPE_ITLOWPASS = 4, + FMOD_DSP_TYPE_HIGHPASS = 5, + FMOD_DSP_TYPE_ECHO = 6, + FMOD_DSP_TYPE_FADER = 7, + FMOD_DSP_TYPE_FLANGE = 8, + FMOD_DSP_TYPE_DISTORTION = 9, + FMOD_DSP_TYPE_NORMALIZE = 10, + FMOD_DSP_TYPE_LIMITER = 11, + FMOD_DSP_TYPE_PARAMEQ = 12, + FMOD_DSP_TYPE_PITCHSHIFT = 13, + FMOD_DSP_TYPE_CHORUS = 14, + FMOD_DSP_TYPE_VSTPLUGIN = 15, + FMOD_DSP_TYPE_WINAMPPLUGIN = 16, + FMOD_DSP_TYPE_ITECHO = 17, + FMOD_DSP_TYPE_COMPRESSOR = 18, + FMOD_DSP_TYPE_SFXREVERB = 19, + FMOD_DSP_TYPE_LOWPASS_SIMPLE = 20, + FMOD_DSP_TYPE_DELAY = 21, + FMOD_DSP_TYPE_TREMOLO = 22, + FMOD_DSP_TYPE_SEND = 24, + FMOD_DSP_TYPE_RETURN = 25, + FMOD_DSP_TYPE_HIGHPASS_SIMPLE = 26, + FMOD_DSP_TYPE_PAN = 27, + FMOD_DSP_TYPE_THREE_EQ = 28, + FMOD_DSP_TYPE_FFT = 29, + FMOD_DSP_TYPE_LOUDNESS_METER = 30, + FMOD_DSP_TYPE_ENVELOPEFOLLOWER = 31, + FMOD_DSP_TYPE_CONVOLUTIONREVERB = 32, + FMOD_DSP_TYPE_TRANSCEIVER = 33, + FMOD_DSP_TYPE_MAX = 34 +}; + +enum FMOD_DSPCONNECTION_TYPE { + FMOD_DSPCONNECTION_TYPE_STANDARD = 0, + FMOD_DSPCONNECTION_TYPE_SIDECHAIN = 1, + FMOD_DSPCONNECTION_TYPE_SEND = 2, + FMOD_DSPCONNECTION_TYPE_SEND_SIDECHAIN = 3, + FMOD_DSPCONNECTION_TYPE_MAX = 4 +}; + +enum FMOD_DSP_FFT_WINDOW { + FMOD_DSP_FFT_WINDOW_RECT = 0, + FMOD_DSP_FFT_WINDOW_TRIANGLE = 1, + FMOD_DSP_FFT_WINDOW_HAMMING = 2, + FMOD_DSP_FFT_WINDOW_HANNING = 3, + FMOD_DSP_FFT_WINDOW_BLACKMAN = 4, + FMOD_DSP_FFT_WINDOW_BLACKMANHARRIS = 5, + FMOD_DSP_FFT_WINDOW_MAX = 6 +}; + +enum FMOD_PLUGINTYPE { + FMOD_PLUGINTYPE_OUTPUT = 0, + FMOD_PLUGINTYPE_CODEC = 1, + FMOD_PLUGINTYPE_DSP = 2, + FMOD_PLUGINTYPE_MAX = 3 +}; + +enum FMOD_DRIVER_STATE { + FMOD_DRIVER_STATE_CONNECTED = 1, + FMOD_DRIVER_STATE_DEFAULT = 2 +}; + +enum FMOD_PORT_TYPE { + FMOD_PORT_TYPE_MUSIC = 0, + FMOD_PORT_TYPE_COPY = 1, + FMOD_PORT_TYPE_SPLIT = 2, + FMOD_PORT_TYPE_FEEDBACK = 3, + FMOD_PORT_TYPE_MAX = 4 +}; + +enum FMOD_SYSTEM_CALLBACK_TYPE { + FMOD_SYSTEM_CALLBACK_DEVICELISTCHANGED = 1, + FMOD_SYSTEM_CALLBACK_DEVICELOST = 2, + FMOD_SYSTEM_CALLBACK_MEMORYALLOCATIONFAILED = 4, + FMOD_SYSTEM_CALLBACK_THREADCREATED = 8, + FMOD_SYSTEM_CALLBACK_BADDSPCONNECTION = 16, + FMOD_SYSTEM_CALLBACK_PREMIX = 32, + FMOD_SYSTEM_CALLBACK_POSTMIX = 64, + FMOD_SYSTEM_CALLBACK_ERROR = 128, + FMOD_SYSTEM_CALLBACK_MIDMIX = 256, + FMOD_SYSTEM_CALLBACK_THREADDESTROYED = 512, + FMOD_SYSTEM_CALLBACK_PREUPDATE = 1024, + FMOD_SYSTEM_CALLBACK_POSTUPDATE = 2048, + FMOD_SYSTEM_CALLBACK_ALL = 0xFFFFFFFF +}; + +enum FMOD_CHANNELCONTROL_TYPE { + FMOD_CHANNELCONTROL_CHANNEL = 0, + FMOD_CHANNELCONTROL_CHANNELGROUP = 1, + FMOD_CHANNELCONTROL_MAX = 2 +}; + +enum FMOD_CHANNELCONTROL_CALLBACK_TYPE { + FMOD_CHANNELCONTROL_CALLBACK_END = 0, + FMOD_CHANNELCONTROL_CALLBACK_VIRTUALVOICE = 1, + FMOD_CHANNELCONTROL_CALLBACK_SYNCPOINT = 2, + FMOD_CHANNELCONTROL_CALLBACK_OCCLUSION = 3, + FMOD_CHANNELCONTROL_CALLBACK_MAX = 4 +}; + +enum FMOD_TAGTYPE { + FMOD_TAGTYPE_UNKNOWN = 0, + FMOD_TAGTYPE_ID3V1 = 1, + FMOD_TAGTYPE_ID3V2 = 2, + FMOD_TAGTYPE_VORBISCOMMENT = 3, + FMOD_TAGTYPE_SHOUTCAST = 4, + FMOD_TAGTYPE_ICECAST = 5, + FMOD_TAGTYPE_ASF = 6, + FMOD_TAGTYPE_MIDI = 7, + FMOD_TAGTYPE_PLAYLIST = 8, + FMOD_TAGTYPE_FMOD = 9, + FMOD_TAGTYPE_USER = 10, + FMOD_TAGTYPE_MAX = 11 +}; + +enum FMOD_TAGDATATYPE { + FMOD_TAGDATATYPE_BINARY = 0, + FMOD_TAGDATATYPE_INT = 1, + FMOD_TAGDATATYPE_FLOAT = 2, + FMOD_TAGDATATYPE_STRING = 3, + FMOD_TAGDATATYPE_STRING_UTF16 = 4, + FMOD_TAGDATATYPE_STRING_UTF16BE = 5, + FMOD_TAGDATATYPE_STRING_UTF8 = 6, + FMOD_TAGDATATYPE_CDTOC = 7, + FMOD_TAGDATATYPE_MAX = 8 +}; + +struct FMOD_GUID { + unsigned Data1; + unsigned short Data2; + unsigned short Data3; + unsigned char Data4[8]; +}; + +struct FMOD_VECTOR { + float x; + float y; + float z; +}; + +struct FMOD_3D_ATTRIBUTES { + FMOD_VECTOR position; + FMOD_VECTOR velocity; + FMOD_VECTOR forward; + FMOD_VECTOR up; +}; + +struct FMOD_CREATESOUNDEXINFO { + int cbsize; + unsigned length; + unsigned fileoffset; + int numchannels; + int defaultfrequency; + FMOD_SOUND_FORMAT format; + unsigned decodebuffersize; + int initialsubsound; + int numsubsounds; + void* inclusionlist; + unsigned inclusionlistnum; + unsigned char dlsrawdata[8]; + char* encryptionkey; +}; + +struct FMOD_REVERB_PROPERTIES { + int Instance; + float Environment; + float EnvSize; + float EnvDiffusion; + int Room; + int RoomHF; + int RoomLF; + float DecayTime; + float DecayHFRatio; + float DecayLFRatio; + int Reflections; + float ReflectionsDelay; + int Reverb; + float ReverbDelay; + float EchoTime; + float EchoDepth; + float ModulationTime; + float ModulationDepth; + float AirAbsorptionHF; + float HFReference; + float LFReference; + float RoomRolloffFactor; + float Diffusion; + float Density; + unsigned Flags; +}; + +struct FMOD_ADVANCEDSETTINGS { + int cbSize; + unsigned maxMPEGCodecs; + unsigned maxADPCMCodecs; + unsigned maxXMACodecs; + unsigned maxVorbisCodecs; + unsigned maxAT9Codecs; + unsigned maxFADPCMCodecs; + unsigned maxPCMCodecs; + unsigned ASIONumChannels; + char* ASIOChannelList; + char* ASIOSpeakerList; + float vol0virtualvol; + unsigned defaultDecodeBufferSize; + unsigned short profilePort; + unsigned geometryMaxFadeTime; + float distanceFilterCenterFreq; + int reverb3Dinstance; + unsigned DSPBufferPoolSize; + unsigned stackSizeStream; + unsigned stackSizeNonBlocking; + unsigned stackSizeMixer; + int resamplerMethod; + unsigned commandQueueSize; + unsigned handleInitialSize; +}; + +struct FMOD_CPU_USAGE { + float dsp; + float stream; + float geometry; + float update; + float convolution1; + float convolution2; +}; + +struct FMOD_TAG { + FMOD_TAGTYPE type; + FMOD_TAGDATATYPE datatype; + char* name; + void* data; + unsigned datalen; + bool updated; +}; + +struct FMOD_SYNC_POINT { + void* opaque; +}; + +struct FMOD_DSP_DESCRIPTION { + unsigned pluginsdkversion; + char name[32]; + unsigned version; + int numinputbuffers; + int numoutputbuffers; + void* create; + void* release; + void* reset; + void* read; + void* process; + void* setposition; + int numparameters; + void* paramdesc; + void* setparameterfloat; + void* setparameterint; + void* setparameterbool; + void* setparameterdata; + void* getparameterfloat; + void* getparameterint; + void* getparameterbool; + void* getparameterdata; + void* shouldiprocess; + void* userdata; + void* sys_register; + void* sys_deregister; + void* sys_mix; +}; + +struct FMOD_DSP_PARAMETER_DESC { + int type; + char name[16]; + char label[16]; + char* description; +}; + +struct FMOD_DSP_METERING_INFO { + int numsamples; + float peaklevel[32]; + float rmslevel[32]; + short numchannels; +}; + +struct FMOD_DSP_BUFFER_ARRAY { + int numbuffers; + int* buffernumchannels; + int* bufferchannelmask; + float** buffers; + FMOD_SPEAKERMODE speakermode; +}; + +struct FMOD_DSP_STATE { + void* instance; + void* plugindata; + int channelmask; + FMOD_SPEAKERMODE source_speakermode; + float* sidechaindata; + int sidechainchannels; + void* callbacks; + int systemobject; +}; + +struct FMOD_SYSTEM { + int unused; +}; + +struct FMOD_CHANNELCONTROL { + int unused; +}; + +struct FMOD_CHANNELGROUP { + int unused; +}; + +typedef int (*FMOD_SYSTEM_CALLBACK)(FMOD_SYSTEM* system, FMOD_SYSTEM_CALLBACK_TYPE type, void* commanddata1, void* commanddata2, void* userdata); +typedef int (*FMOD_CHANNELCONTROL_CALLBACK)(FMOD_CHANNELCONTROL* channelcontrol, FMOD_CHANNELCONTROL_TYPE controltype, FMOD_CHANNELCONTROL_CALLBACK_TYPE callbacktype, void* commanddata1, void* commanddata2); + +namespace { + +std::mutex gFmodMutex; + +constexpr int FMOD_OK = 0; +constexpr unsigned FMOD_VERSION_CURRENT = 0x00020206; + +} + +namespace FMOD { + +class System; +class ChannelGroup; +class Channel; +class Sound; +class SoundGroup; +class DSP; +class DSPConnection; +class Geometry; +class Reverb3D; + +class Sound { +public: + std::string name; + unsigned mode = 0; + unsigned length = 0; + int loopCount = 0; + float minDistance = 0.0f; + float maxDistance = 10000.0f; + SoundGroup* soundGroup = nullptr; + void* userData = nullptr; + int APS5_VABI release() { + std::lock_guard lock(gFmodMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI getSystemObject(System** system); + int APS5_VABI setMode(unsigned mode) { + std::lock_guard lock(gFmodMutex); + this->mode = mode; + return FMOD_OK; + } + int APS5_VABI getMode(unsigned* mode) { + std::lock_guard lock(gFmodMutex); + if (mode) *mode = this->mode; + return FMOD_OK; + } + int APS5_VABI set3DMinMaxDistance(float mindistance, float maxdistance) { + std::lock_guard lock(gFmodMutex); + minDistance = mindistance; + maxDistance = maxdistance; + return FMOD_OK; + } + int APS5_VABI getNumSubSounds(int* numsubsounds) { + std::lock_guard lock(gFmodMutex); + if (numsubsounds) *numsubsounds = 0; + return FMOD_OK; + } + int APS5_VABI getLength(unsigned* length, unsigned lengthtype) { + (void)lengthtype; + std::lock_guard lock(gFmodMutex); + if (length) *length = this->length; + return FMOD_OK; + } + int APS5_VABI getFormat(FMOD_SOUND_TYPE* type, FMOD_SOUND_FORMAT* format, int* channels, int* bits) { + std::lock_guard lock(gFmodMutex); + if (type) *type = FMOD_SOUND_TYPE_UNKNOWN; + if (format) *format = FMOD_SOUND_FORMAT_PCM16; + if (channels) *channels = 2; + if (bits) *bits = 16; + return FMOD_OK; + } + int APS5_VABI getName(char* name, int namelen) { + std::lock_guard lock(gFmodMutex); + if (name && namelen > 0) { + std::strncpy(name, this->name.c_str(), static_cast(namelen - 1)); + name[namelen - 1] = '\0'; + } + return FMOD_OK; + } + int APS5_VABI setLoopCount(int loopcount) { + std::lock_guard lock(gFmodMutex); + loopCount = loopcount; + return FMOD_OK; + } + int APS5_VABI getLoopCount(int* loopcount) { + std::lock_guard lock(gFmodMutex); + if (loopcount) *loopcount = loopCount; + return FMOD_OK; + } + int APS5_VABI setSoundGroup(SoundGroup* soundgroup) { + std::lock_guard lock(gFmodMutex); + soundGroup = soundgroup; + return FMOD_OK; + } + int APS5_VABI getSoundGroup(SoundGroup** soundgroup) { + std::lock_guard lock(gFmodMutex); + if (soundgroup) *soundgroup = soundGroup; + return FMOD_OK; + } + int APS5_VABI getSubSound(int index, Sound** subsound) { + (void)index; + std::lock_guard lock(gFmodMutex); + if (subsound) *subsound = nullptr; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gFmodMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gFmodMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } + int APS5_VABI getOpenState(FMOD_OPENSTATE* openstate, unsigned* percentbuffered, bool* starving, bool* diskbusy) { + std::lock_guard lock(gFmodMutex); + if (openstate) *openstate = FMOD_OPENSTATE_READY; + if (percentbuffered) *percentbuffered = 100; + if (starving) *starving = false; + if (diskbusy) *diskbusy = false; + return FMOD_OK; + } + int APS5_VABI readData(void* buffer, unsigned length, unsigned* read) { + std::lock_guard lock(gFmodMutex); + if (buffer && length) std::memset(buffer, 0, length); + if (read) *read = length; + return FMOD_OK; + } + int APS5_VABI seekData(unsigned pcm) { + (void)pcm; + return FMOD_OK; + } +}; + +class SoundGroup { +public: + int maxAudible = 0; + float volume = 1.0f; + void* userData = nullptr; + int APS5_VABI release() { + std::lock_guard lock(gFmodMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI setMaxAudible(int maxaudible) { + std::lock_guard lock(gFmodMutex); + maxAudible = maxaudible; + return FMOD_OK; + } + int APS5_VABI getMaxAudible(int* maxaudible) { + std::lock_guard lock(gFmodMutex); + if (maxaudible) *maxaudible = maxAudible; + return FMOD_OK; + } + int APS5_VABI setVolume(float volume) { + std::lock_guard lock(gFmodMutex); + this->volume = volume; + return FMOD_OK; + } + int APS5_VABI getVolume(float* volume) { + std::lock_guard lock(gFmodMutex); + if (volume) *volume = this->volume; + return FMOD_OK; + } + int APS5_VABI stop() { return FMOD_OK; } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gFmodMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gFmodMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } + int APS5_VABI getNumSounds(int* numsounds) { + std::lock_guard lock(gFmodMutex); + if (numsounds) *numsounds = 0; + return FMOD_OK; + } +}; + +class DSPConnection { +public: + DSP* input = nullptr; + DSP* output = nullptr; + void* userData = nullptr; + int APS5_VABI getInput(DSP** input) { + std::lock_guard lock(gFmodMutex); + if (input) *input = this->input; + return FMOD_OK; + } + int APS5_VABI getOutput(DSP** output) { + std::lock_guard lock(gFmodMutex); + if (output) *output = this->output; + return FMOD_OK; + } + int APS5_VABI getType(FMOD_DSPCONNECTION_TYPE* type) { + std::lock_guard lock(gFmodMutex); + if (type) *type = FMOD_DSPCONNECTION_TYPE_STANDARD; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gFmodMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gFmodMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class DSP { +public: + FMOD_DSP_TYPE type = FMOD_DSP_TYPE_UNKNOWN; + bool active = true; + bool bypass = false; + void* userData = nullptr; + int APS5_VABI release() { + std::lock_guard lock(gFmodMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI setActive(bool active) { + std::lock_guard lock(gFmodMutex); + this->active = active; + return FMOD_OK; + } + int APS5_VABI getActive(bool* active) { + std::lock_guard lock(gFmodMutex); + if (active) *active = this->active; + return FMOD_OK; + } + int APS5_VABI setBypass(bool bypass) { + std::lock_guard lock(gFmodMutex); + this->bypass = bypass; + return FMOD_OK; + } + int APS5_VABI getBypass(bool* bypass) { + std::lock_guard lock(gFmodMutex); + if (bypass) *bypass = this->bypass; + return FMOD_OK; + } + int APS5_VABI reset() { return FMOD_OK; } + int APS5_VABI setParameterFloat(int index, float value) { + (void)index; (void)value; + return FMOD_OK; + } + int APS5_VABI getParameterFloat(int index, float* value) { + (void)index; + std::lock_guard lock(gFmodMutex); + if (value) *value = 0.0f; + return FMOD_OK; + } + int APS5_VABI getType(FMOD_DSP_TYPE* type) { + std::lock_guard lock(gFmodMutex); + if (type) *type = this->type; + return FMOD_OK; + } + int APS5_VABI getInfo(char* name, unsigned* version, int* channels, int* configwidth, int* configheight) { + std::lock_guard lock(gFmodMutex); + if (name) name[0] = '\0'; + if (version) *version = 0; + if (channels) *channels = 0; + if (configwidth) *configwidth = 0; + if (configheight) *configheight = 0; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gFmodMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gFmodMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class ChannelControl { +public: + float volume = 1.0f; + float pitch = 1.0f; + bool paused = false; + bool mute = false; + bool playing = false; + unsigned mode = 0; + bool volumeRamp = false; + void* userData = nullptr; + int APS5_VABI setVolume(float volume) { + std::lock_guard lock(gFmodMutex); + this->volume = volume; + return FMOD_OK; + } + int APS5_VABI getVolume(float* volume) { + std::lock_guard lock(gFmodMutex); + if (volume) *volume = this->volume; + return FMOD_OK; + } + int APS5_VABI setPitch(float pitch) { + std::lock_guard lock(gFmodMutex); + this->pitch = pitch; + return FMOD_OK; + } + int APS5_VABI getPitch(float* pitch) { + std::lock_guard lock(gFmodMutex); + if (pitch) *pitch = this->pitch; + return FMOD_OK; + } + int APS5_VABI setPaused(bool paused) { + std::lock_guard lock(gFmodMutex); + this->paused = paused; + return FMOD_OK; + } + int APS5_VABI getPaused(bool* paused) { + std::lock_guard lock(gFmodMutex); + if (paused) *paused = this->paused; + return FMOD_OK; + } + int APS5_VABI setMute(bool mute) { + std::lock_guard lock(gFmodMutex); + this->mute = mute; + return FMOD_OK; + } + int APS5_VABI getMute(bool* mute) { + std::lock_guard lock(gFmodMutex); + if (mute) *mute = this->mute; + return FMOD_OK; + } + int APS5_VABI stop() { + std::lock_guard lock(gFmodMutex); + playing = false; + return FMOD_OK; + } + int APS5_VABI isPlaying(bool* isplaying) { + std::lock_guard lock(gFmodMutex); + if (isplaying) *isplaying = playing && !paused; + return FMOD_OK; + } + int APS5_VABI setMode(unsigned mode) { + std::lock_guard lock(gFmodMutex); + this->mode = mode; + return FMOD_OK; + } + int APS5_VABI getMode(unsigned* mode) { + std::lock_guard lock(gFmodMutex); + if (mode) *mode = this->mode; + return FMOD_OK; + } + int APS5_VABI addFadePoint(unsigned long long dspclock, float volume) { + (void)dspclock; (void)volume; + return FMOD_OK; + } + int APS5_VABI setDelay(unsigned long long dspclock_start, unsigned long long dspclock_end, bool stopchannels) { + (void)dspclock_start; (void)dspclock_end; (void)stopchannels; + return FMOD_OK; + } + int APS5_VABI getDelay(unsigned long long* dspclock_start, unsigned long long* dspclock_end, bool* stopchannels) { + std::lock_guard lock(gFmodMutex); + if (dspclock_start) *dspclock_start = 0; + if (dspclock_end) *dspclock_end = 0; + if (stopchannels) *stopchannels = false; + return FMOD_OK; + } + int APS5_VABI setVolumeRamp(bool ramp) { + std::lock_guard lock(gFmodMutex); + volumeRamp = ramp; + return FMOD_OK; + } + int APS5_VABI getVolumeRamp(bool* ramp) { + std::lock_guard lock(gFmodMutex); + if (ramp) *ramp = volumeRamp; + return FMOD_OK; + } + int APS5_VABI setCallback(FMOD_CHANNELCONTROL_CALLBACK callback) { + (void)callback; + return FMOD_OK; + } + int APS5_VABI isVirtual(bool* isvirtual) { + std::lock_guard lock(gFmodMutex); + if (isvirtual) *isvirtual = false; + return FMOD_OK; + } + int APS5_VABI getIndex(int* index) { + std::lock_guard lock(gFmodMutex); + if (index) *index = 0; + return FMOD_OK; + } + int APS5_VABI getSystemObject(System** system); + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gFmodMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gFmodMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } + int APS5_VABI set3DAttributes(const FMOD_VECTOR* pos, const FMOD_VECTOR* vel) { + (void)pos; (void)vel; + return FMOD_OK; + } + int APS5_VABI setLowPassGain(float gain) { + (void)gain; + return FMOD_OK; + } + int APS5_VABI getLowPassGain(float* gain) { + std::lock_guard lock(gFmodMutex); + if (gain) *gain = 1.0f; + return FMOD_OK; + } + int APS5_VABI setPan(float pan) { + (void)pan; + return FMOD_OK; + } + int APS5_VABI getDSPClock(unsigned long long* dspclock, unsigned long long* parentclock) { + std::lock_guard lock(gFmodMutex); + if (dspclock) *dspclock = 0; + if (parentclock) *parentclock = 0; + return FMOD_OK; + } + int APS5_VABI getDSP(int index, DSP** dsp) { + (void)index; + std::lock_guard lock(gFmodMutex); + if (dsp) *dsp = nullptr; + return FMOD_OK; + } +}; + +class Channel : public ChannelControl { +public: + Sound* sound = nullptr; + ChannelGroup* group = nullptr; + float frequency = 48000.0f; + int priority = 128; + unsigned position = 0; + int loopCount = 0; + int APS5_VABI setPriority(int priority) { + std::lock_guard lock(gFmodMutex); + this->priority = priority; + return FMOD_OK; + } + int APS5_VABI getPriority(int* priority) { + std::lock_guard lock(gFmodMutex); + if (priority) *priority = this->priority; + return FMOD_OK; + } + int APS5_VABI setFrequency(float frequency) { + std::lock_guard lock(gFmodMutex); + this->frequency = frequency; + return FMOD_OK; + } + int APS5_VABI getFrequency(float* frequency) { + std::lock_guard lock(gFmodMutex); + if (frequency) *frequency = this->frequency; + return FMOD_OK; + } + int APS5_VABI setPosition(unsigned position, unsigned postype) { + (void)postype; + std::lock_guard lock(gFmodMutex); + this->position = position; + return FMOD_OK; + } + int APS5_VABI getPosition(unsigned* position, unsigned postype) { + (void)postype; + std::lock_guard lock(gFmodMutex); + if (position) *position = this->position; + return FMOD_OK; + } + int APS5_VABI setChannelGroup(ChannelGroup* channelgroup) { + std::lock_guard lock(gFmodMutex); + group = channelgroup; + return FMOD_OK; + } + int APS5_VABI getChannelGroup(ChannelGroup** channelgroup) { + std::lock_guard lock(gFmodMutex); + if (channelgroup) *channelgroup = group; + return FMOD_OK; + } + int APS5_VABI setLoopCount(int loopcount) { + std::lock_guard lock(gFmodMutex); + loopCount = loopcount; + return FMOD_OK; + } + int APS5_VABI getCurrentSound(Sound** sound) { + std::lock_guard lock(gFmodMutex); + if (sound) *sound = this->sound; + return FMOD_OK; + } +}; + +class ChannelGroup : public ChannelControl { +public: + std::string name; + int APS5_VABI release() { + std::lock_guard lock(gFmodMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI addGroup(ChannelGroup* group, bool propagatedspclock, DSPConnection** connection) { + (void)group; (void)propagatedspclock; + std::lock_guard lock(gFmodMutex); + if (connection) *connection = nullptr; + return FMOD_OK; + } + int APS5_VABI getNumGroups(int* numgroups) { + std::lock_guard lock(gFmodMutex); + if (numgroups) *numgroups = 0; + return FMOD_OK; + } + int APS5_VABI getGroup(int index, ChannelGroup** group) { + (void)index; + std::lock_guard lock(gFmodMutex); + if (group) *group = nullptr; + return FMOD_OK; + } + int APS5_VABI getParentGroup(ChannelGroup** group) { + std::lock_guard lock(gFmodMutex); + if (group) *group = nullptr; + return FMOD_OK; + } + int APS5_VABI getName(char* name, int namelen) { + std::lock_guard lock(gFmodMutex); + if (name && namelen > 0) { + std::strncpy(name, this->name.c_str(), static_cast(namelen - 1)); + name[namelen - 1] = '\0'; + } + return FMOD_OK; + } + int APS5_VABI getNumChannels(int* numchannels) { + std::lock_guard lock(gFmodMutex); + if (numchannels) *numchannels = 0; + return FMOD_OK; + } + int APS5_VABI getChannel(int index, Channel** channel) { + (void)index; + std::lock_guard lock(gFmodMutex); + if (channel) *channel = nullptr; + return FMOD_OK; + } +}; + +class Geometry { +public: + void* userData = nullptr; + int APS5_VABI release() { + std::lock_guard lock(gFmodMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI addPolygon(float directocclusion, float reverbocclusion, bool doublesided, int numvertices, const FMOD_VECTOR* vertices, int* polygonindex) { + (void)directocclusion; (void)reverbocclusion; (void)doublesided; (void)numvertices; (void)vertices; + std::lock_guard lock(gFmodMutex); + if (polygonindex) *polygonindex = 0; + return FMOD_OK; + } + int APS5_VABI setActive(bool active) { + (void)active; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gFmodMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gFmodMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class Reverb3D { +public: + void* userData = nullptr; + int APS5_VABI release() { + std::lock_guard lock(gFmodMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI set3DAttributes(const FMOD_VECTOR* pos, float mindistance, float maxdistance) { + (void)pos; (void)mindistance; (void)maxdistance; + return FMOD_OK; + } + int APS5_VABI setActive(bool active) { + (void)active; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gFmodMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gFmodMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class System { +public: + bool initialized = false; + int maxChannels = 0; + unsigned initFlags = 0; + int driver = 0; + FMOD_OUTPUTTYPE output = FMOD_OUTPUTTYPE_AUTODETECT; + int softwareChannels = 0; + int numListeners = 1; + float dopplerScale = 1.0f; + float distanceFactor = 1.0f; + float rolloffScale = 1.0f; + void* userData = nullptr; + ChannelGroup* masterChannelGroup = nullptr; + SoundGroup* masterSoundGroup = nullptr; + static int APS5_VABI create(System** system) { + std::lock_guard lock(gFmodMutex); + if (!system) return FMOD_OK; + *system = new System(); + return FMOD_OK; + } + static int APS5_VABI create(System** system, unsigned headerversion) { + (void)headerversion; + return create(system); + } + int APS5_VABI release() { + std::lock_guard lock(gFmodMutex); + delete masterChannelGroup; + delete masterSoundGroup; + masterChannelGroup = nullptr; + masterSoundGroup = nullptr; + delete this; + return FMOD_OK; + } + int APS5_VABI init(int maxchannels, unsigned flags, void* extradriverdata) { + (void)extradriverdata; + std::lock_guard lock(gFmodMutex); + maxChannels = maxchannels; + initFlags = flags; + initialized = true; + if (!masterChannelGroup) masterChannelGroup = new ChannelGroup(); + if (!masterSoundGroup) masterSoundGroup = new SoundGroup(); + return FMOD_OK; + } + int APS5_VABI close() { + std::lock_guard lock(gFmodMutex); + initialized = false; + return FMOD_OK; + } + int APS5_VABI update() { return FMOD_OK; } + int APS5_VABI setOutput(FMOD_OUTPUTTYPE output) { + std::lock_guard lock(gFmodMutex); + this->output = output; + return FMOD_OK; + } + int APS5_VABI getOutput(FMOD_OUTPUTTYPE* output) { + std::lock_guard lock(gFmodMutex); + if (output) *output = this->output; + return FMOD_OK; + } + int APS5_VABI getNumDrivers(int* numdrivers) { + std::lock_guard lock(gFmodMutex); + if (numdrivers) *numdrivers = 1; + return FMOD_OK; + } + int APS5_VABI getDriverInfo(int id, char* name, int namelen, FMOD_GUID* guid, int* systemrate, FMOD_SPEAKERMODE* speakermode, int* speakermodechannels) { + (void)id; + std::lock_guard lock(gFmodMutex); + if (name && namelen > 0) { + const char* driverName = "AnyPS5 Null Output"; + std::strncpy(name, driverName, static_cast(namelen - 1)); + name[namelen - 1] = '\0'; + } + if (guid) std::memset(guid, 0, sizeof(*guid)); + if (systemrate) *systemrate = 48000; + if (speakermode) *speakermode = FMOD_SPEAKERMODE_STEREO; + if (speakermodechannels) *speakermodechannels = 2; + return FMOD_OK; + } + int APS5_VABI setDriver(int driver) { + std::lock_guard lock(gFmodMutex); + this->driver = driver; + return FMOD_OK; + } + int APS5_VABI getDriver(int* driver) { + std::lock_guard lock(gFmodMutex); + if (driver) *driver = this->driver; + return FMOD_OK; + } + int APS5_VABI setSoftwareChannels(int numsoftwarechannels) { + std::lock_guard lock(gFmodMutex); + softwareChannels = numsoftwarechannels; + return FMOD_OK; + } + int APS5_VABI getSoftwareChannels(int* numsoftwarechannels) { + std::lock_guard lock(gFmodMutex); + if (numsoftwarechannels) *numsoftwarechannels = softwareChannels; + return FMOD_OK; + } + int APS5_VABI setSoftwareFormat(int samplerate, FMOD_SPEAKERMODE speakermode, int numrawspeakers) { + (void)samplerate; (void)speakermode; (void)numrawspeakers; + return FMOD_OK; + } + int APS5_VABI getSoftwareFormat(int* samplerate, FMOD_SPEAKERMODE* speakermode, int* numrawspeakers) { + std::lock_guard lock(gFmodMutex); + if (samplerate) *samplerate = 48000; + if (speakermode) *speakermode = FMOD_SPEAKERMODE_STEREO; + if (numrawspeakers) *numrawspeakers = 0; + return FMOD_OK; + } + int APS5_VABI setDSPBufferSize(unsigned bufferlength, int numbuffers) { + (void)bufferlength; (void)numbuffers; + return FMOD_OK; + } + int APS5_VABI getDSPBufferSize(unsigned* bufferlength, int* numbuffers) { + std::lock_guard lock(gFmodMutex); + if (bufferlength) *bufferlength = 1024; + if (numbuffers) *numbuffers = 4; + return FMOD_OK; + } + int APS5_VABI setAdvancedSettings(FMOD_ADVANCEDSETTINGS* settings) { + (void)settings; + return FMOD_OK; + } + int APS5_VABI getAdvancedSettings(FMOD_ADVANCEDSETTINGS* settings) { + std::lock_guard lock(gFmodMutex); + if (settings) std::memset(settings, 0, sizeof(*settings)); + return FMOD_OK; + } + int APS5_VABI setCallback(FMOD_SYSTEM_CALLBACK callback, FMOD_SYSTEM_CALLBACK_TYPE callbackmask, void* userdata) { + (void)callback; (void)callbackmask; (void)userdata; + return FMOD_OK; + } + int APS5_VABI setPluginPath(const char* pluginpath) { + (void)pluginpath; + return FMOD_OK; + } + int APS5_VABI loadPlugin(const char* filename, unsigned* handle, FMOD_PLUGINTYPE plugintype) { + (void)filename; (void)plugintype; + std::lock_guard lock(gFmodMutex); + if (handle) *handle = 0; + return FMOD_OK; + } + int APS5_VABI getVersion(unsigned* version) { + std::lock_guard lock(gFmodMutex); + if (version) *version = FMOD_VERSION_CURRENT; + return FMOD_OK; + } + int APS5_VABI getOutputHandle(void** handle) { + std::lock_guard lock(gFmodMutex); + if (handle) *handle = nullptr; + return FMOD_OK; + } + int APS5_VABI getChannelsPlaying(int* channels, int* realchannels) { + std::lock_guard lock(gFmodMutex); + if (channels) *channels = 0; + if (realchannels) *realchannels = 0; + return FMOD_OK; + } + int APS5_VABI getCPUUsage(FMOD_CPU_USAGE* usage) { + std::lock_guard lock(gFmodMutex); + if (usage) std::memset(usage, 0, sizeof(*usage)); + return FMOD_OK; + } + int APS5_VABI getFileUsage(long long* samplebytesread, long long* streambytesread, long long* otherbytesread) { + std::lock_guard lock(gFmodMutex); + if (samplebytesread) *samplebytesread = 0; + if (streambytesread) *streambytesread = 0; + if (otherbytesread) *otherbytesread = 0; + return FMOD_OK; + } + int APS5_VABI getSpectrum(float* spectrumarray, int numvalues, int channeloffset, FMOD_DSP_FFT_WINDOW window) { + (void)channeloffset; (void)window; + std::lock_guard lock(gFmodMutex); + if (spectrumarray && numvalues > 0) std::memset(spectrumarray, 0, sizeof(float) * static_cast(numvalues)); + return FMOD_OK; + } + int APS5_VABI getWaveData(float* wavearray, int numvalues, int channeloffset) { + (void)channeloffset; + std::lock_guard lock(gFmodMutex); + if (wavearray && numvalues > 0) std::memset(wavearray, 0, sizeof(float) * static_cast(numvalues)); + return FMOD_OK; + } + int APS5_VABI createSound(const char* name_or_data, unsigned mode, FMOD_CREATESOUNDEXINFO* exinfo, Sound** sound) { + std::lock_guard lock(gFmodMutex); + if (!sound) return FMOD_OK; + Sound* created = new Sound(); + if (name_or_data) created->name = name_or_data; + created->mode = mode; + if (exinfo) created->length = exinfo->length; + *sound = created; + return FMOD_OK; + } + int APS5_VABI createStream(const char* name_or_data, unsigned mode, FMOD_CREATESOUNDEXINFO* exinfo, Sound** sound) { + return createSound(name_or_data, mode, exinfo, sound); + } + int APS5_VABI createDSP(FMOD_DSP_DESCRIPTION* description, DSP** dsp) { + (void)description; + std::lock_guard lock(gFmodMutex); + if (dsp) *dsp = new DSP(); + return FMOD_OK; + } + int APS5_VABI createDSPByType(FMOD_DSP_TYPE type, DSP** dsp) { + std::lock_guard lock(gFmodMutex); + if (dsp) { + *dsp = new DSP(); + (*dsp)->type = type; + } + return FMOD_OK; + } + int APS5_VABI playSound(Sound* sound, ChannelGroup* channelgroup, bool paused, Channel** channel) { + std::lock_guard lock(gFmodMutex); + if (!channel) return FMOD_OK; + Channel* created = new Channel(); + created->sound = sound; + created->group = channelgroup ? channelgroup : masterChannelGroup; + created->paused = paused; + created->playing = true; + *channel = created; + return FMOD_OK; + } + int APS5_VABI playDSP(DSP* dsp, ChannelGroup* channelgroup, bool paused, Channel** channel) { + (void)dsp; + std::lock_guard lock(gFmodMutex); + if (!channel) return FMOD_OK; + Channel* created = new Channel(); + created->group = channelgroup ? channelgroup : masterChannelGroup; + created->paused = paused; + created->playing = true; + *channel = created; + return FMOD_OK; + } + int APS5_VABI getChannel(int channelid, Channel** channel) { + (void)channelid; + std::lock_guard lock(gFmodMutex); + if (channel) *channel = nullptr; + return FMOD_OK; + } + int APS5_VABI getMasterSoundGroup(SoundGroup** soundgroup) { + std::lock_guard lock(gFmodMutex); + if (soundgroup) *soundgroup = masterSoundGroup; + return FMOD_OK; + } + int APS5_VABI getMasterChannelGroup(ChannelGroup** channelgroup) { + std::lock_guard lock(gFmodMutex); + if (channelgroup) *channelgroup = masterChannelGroup; + return FMOD_OK; + } + int APS5_VABI setReverbProperties(int instance, FMOD_REVERB_PROPERTIES* prop) { + (void)instance; (void)prop; + return FMOD_OK; + } + int APS5_VABI getReverbProperties(int instance, FMOD_REVERB_PROPERTIES* prop) { + (void)instance; + std::lock_guard lock(gFmodMutex); + if (prop) std::memset(prop, 0, sizeof(*prop)); + return FMOD_OK; + } + int APS5_VABI createReverb3D(Reverb3D** reverb) { + std::lock_guard lock(gFmodMutex); + if (reverb) *reverb = new Reverb3D(); + return FMOD_OK; + } + int APS5_VABI createGeometry(int maxpolygons, int maxvertices, Geometry** geometry) { + (void)maxpolygons; (void)maxvertices; + std::lock_guard lock(gFmodMutex); + if (geometry) *geometry = new Geometry(); + return FMOD_OK; + } + int APS5_VABI loadGeometry(const void* data, int datasize, Geometry** geometry) { + (void)data; (void)datasize; + std::lock_guard lock(gFmodMutex); + if (geometry) *geometry = new Geometry(); + return FMOD_OK; + } + int APS5_VABI setGeometrySettings(float maxworldsize) { + (void)maxworldsize; + return FMOD_OK; + } + int APS5_VABI getGeometrySettings(float* maxworldsize) { + std::lock_guard lock(gFmodMutex); + if (maxworldsize) *maxworldsize = 0.0f; + return FMOD_OK; + } + int APS5_VABI set3DNumListeners(int numlisteners) { + std::lock_guard lock(gFmodMutex); + numListeners = numlisteners; + return FMOD_OK; + } + int APS5_VABI get3DNumListeners(int* numlisteners) { + std::lock_guard lock(gFmodMutex); + if (numlisteners) *numlisteners = numListeners; + return FMOD_OK; + } + int APS5_VABI set3DListenerAttributes(int listener, const FMOD_VECTOR* pos, const FMOD_VECTOR* vel, const FMOD_VECTOR* forward, const FMOD_VECTOR* up) { + (void)listener; (void)pos; (void)vel; (void)forward; (void)up; + return FMOD_OK; + } + int APS5_VABI mixerSuspend() { return FMOD_OK; } + int APS5_VABI mixerResume() { return FMOD_OK; } + int APS5_VABI set3DSettings(float dopplerscale, float distancefactor, float rolloffscale) { + std::lock_guard lock(gFmodMutex); + dopplerScale = dopplerscale; + distanceFactor = distancefactor; + rolloffScale = rolloffscale; + return FMOD_OK; + } + int APS5_VABI get3DSettings(float* dopplerscale, float* distancefactor, float* rolloffscale) { + std::lock_guard lock(gFmodMutex); + if (dopplerscale) *dopplerscale = dopplerScale; + if (distancefactor) *distancefactor = distanceFactor; + if (rolloffscale) *rolloffscale = rolloffScale; + return FMOD_OK; + } + int APS5_VABI createChannelGroup(const char* name, ChannelGroup** channelgroup) { + std::lock_guard lock(gFmodMutex); + if (!channelgroup) return FMOD_OK; + *channelgroup = new ChannelGroup(); + if (name) (*channelgroup)->name = name; + return FMOD_OK; + } + int APS5_VABI createSoundGroup(SoundGroup** soundgroup) { + std::lock_guard lock(gFmodMutex); + if (soundgroup) *soundgroup = new SoundGroup(); + return FMOD_OK; + } + int APS5_VABI lockDSP() { return FMOD_OK; } + int APS5_VABI unlockDSP() { return FMOD_OK; } + int APS5_VABI getSoundRAM(int* currentalloced, int* maxalloced, int* total) { + std::lock_guard lock(gFmodMutex); + if (currentalloced) *currentalloced = 0; + if (maxalloced) *maxalloced = 0; + if (total) *total = 0; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gFmodMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gFmodMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } + int APS5_VABI setStreamBufferSize(unsigned filebuffersize, unsigned filebuffersizetype) { + (void)filebuffersize; (void)filebuffersizetype; + return FMOD_OK; + } + int APS5_VABI attachChannelGroupToPort(FMOD_PORT_TYPE porttype, unsigned long long portindex, ChannelGroup* channelgroup) { + (void)porttype; (void)portindex; (void)channelgroup; + return FMOD_OK; + } + int APS5_VABI detachChannelGroupFromPort(ChannelGroup* channelgroup) { + (void)channelgroup; + return FMOD_OK; + } + int APS5_VABI setNetworkProxy(const char* proxy) { + (void)proxy; + return FMOD_OK; + } +}; + +int APS5_VABI Sound::getSystemObject(System** system) { + std::lock_guard lock(gFmodMutex); + if (system) *system = nullptr; + return FMOD_OK; +} + +int APS5_VABI ChannelControl::getSystemObject(System** system) { + std::lock_guard lock(gFmodMutex); + if (system) *system = nullptr; + return FMOD_OK; +} + +} + +extern "C" { + +int APS5_VABI FMOD_System_Create(FMOD_SYSTEM** system) { + std::lock_guard lock(gFmodMutex); + if (!system) return 0; + *system = reinterpret_cast(new FMOD::System()); + return 0; +} + +int APS5_VABI FMOD_Memory_Initialize(void* poolmem, int poollen, void* useralloc, void* userrealloc, void* userfree, unsigned memtypeflags) { + (void)poolmem; (void)poollen; (void)useralloc; (void)userrealloc; (void)userfree; (void)memtypeflags; + return 0; +} + +int APS5_VABI FMOD_Memory_GetStats(int* currentalloced, int* maxalloced, int blocking) { + (void)blocking; + std::lock_guard lock(gFmodMutex); + if (currentalloced) *currentalloced = 0; + if (maxalloced) *maxalloced = 0; + return 0; +} + +namespace { + +__attribute__((used)) int fmod_nid_stub_00() { APS5_LOG_ERR("%s", "VS1Vg5yOLH0"); return 0; } +__attribute__((used)) int fmod_nid_stub_01() { APS5_LOG_ERR("%s", "WAU72lnwYic"); return 0; } + +} + +APS5_EXPORT("VS1Vg5yOLH0", fmod_nid_stub_00); +APS5_EXPORT("WAU72lnwYic", fmod_nid_stub_01); + +} diff --git a/core/libs/prx/libfmodstudio/CMakeLists.txt b/core/libs/prx/libfmodstudio/CMakeLists.txt new file mode 100644 index 0000000000..38c8ce4843 --- /dev/null +++ b/core/libs/prx/libfmodstudio/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(libfmodstudio SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp +) + +target_include_directories(libfmodstudio PRIVATE + ${LIBS_INCLUDE_DIR} +) +target_link_libraries(libfmodstudio PRIVATE libkernel libc) +target_compile_options(libfmodstudio PRIVATE -fkeep-inline-functions) +configure_windows_unwind(libfmodstudio) diff --git a/core/libs/prx/libfmodstudio/Export.cpp b/core/libs/prx/libfmodstudio/Export.cpp new file mode 100644 index 0000000000..c676d3746a --- /dev/null +++ b/core/libs/prx/libfmodstudio/Export.cpp @@ -0,0 +1,870 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +enum FMOD_STUDIO_STOP_MODE { + FMOD_STUDIO_STOP_ALLOWFADEOUT = 0, + FMOD_STUDIO_STOP_IMMEDIATE = 1, + FMOD_STUDIO_STOP_MAX = 2 +}; + +enum FMOD_STUDIO_PLAYBACK_STATE { + FMOD_STUDIO_PLAYBACK_PLAYING = 0, + FMOD_STUDIO_PLAYBACK_SUSTAINING = 1, + FMOD_STUDIO_PLAYBACK_STOPPED = 2, + FMOD_STUDIO_PLAYBACK_STARTING = 3, + FMOD_STUDIO_PLAYBACK_STOPPING = 4, + FMOD_STUDIO_PLAYBACK_MAX = 5 +}; + +enum FMOD_STUDIO_LOADING_STATE { + FMOD_STUDIO_LOADING_STATE_UNLOADING = 0, + FMOD_STUDIO_LOADING_STATE_UNLOADED = 1, + FMOD_STUDIO_LOADING_STATE_LOADING = 2, + FMOD_STUDIO_LOADING_STATE_LOADED = 3, + FMOD_STUDIO_LOADING_STATE_ERROR = 4, + FMOD_STUDIO_LOADING_STATE_MAX = 5 +}; + +enum FMOD_STUDIO_EVENT_PROPERTY { + FMOD_STUDIO_EVENT_PROPERTY_CHANNELPRIORITY = 0, + FMOD_STUDIO_EVENT_PROPERTY_SCHEDULE_DELAY = 1, + FMOD_STUDIO_EVENT_PROPERTY_SCHEDULE_LOOKAHEAD = 2, + FMOD_STUDIO_EVENT_PROPERTY_MINIMUM_DISTANCE = 3, + FMOD_STUDIO_EVENT_PROPERTY_MAXIMUM_DISTANCE = 4, + FMOD_STUDIO_EVENT_PROPERTY_MAX = 5 +}; + +enum FMOD_STUDIO_SYSTEM_CALLBACK_TYPE { + FMOD_STUDIO_SYSTEM_CALLBACK_PREUPDATE = 1, + FMOD_STUDIO_SYSTEM_CALLBACK_POSTUPDATE = 2, + FMOD_STUDIO_SYSTEM_CALLBACK_BANK_UNLOAD = 4, + FMOD_STUDIO_SYSTEM_CALLBACK_ALL = 0xFFFFFFFF +}; + +enum FMOD_STUDIO_EVENT_CALLBACK_TYPE { + FMOD_STUDIO_EVENT_CALLBACK_CREATED = 1, + FMOD_STUDIO_EVENT_CALLBACK_DESTROYED = 2, + FMOD_STUDIO_EVENT_CALLBACK_STARTING = 4, + FMOD_STUDIO_EVENT_CALLBACK_STARTED = 8, + FMOD_STUDIO_EVENT_CALLBACK_RESTARTED = 16, + FMOD_STUDIO_EVENT_CALLBACK_STOPPED = 32, + FMOD_STUDIO_EVENT_CALLBACK_START_FAILED = 64, + FMOD_STUDIO_EVENT_CALLBACK_ALL = 0xFFFFFFFF +}; + +struct FMOD_GUID { + unsigned Data1; + unsigned short Data2; + unsigned short Data3; + unsigned char Data4[8]; +}; + +struct FMOD_VECTOR { + float x; + float y; + float z; +}; + +struct FMOD_3D_ATTRIBUTES { + FMOD_VECTOR position; + FMOD_VECTOR velocity; + FMOD_VECTOR forward; + FMOD_VECTOR up; +}; + +struct FMOD_CHANNELGROUP { + int unused; +}; + +struct FMOD_DSP_DESCRIPTION { + unsigned pluginsdkversion; + char name[32]; + unsigned version; +}; + +struct FMOD_STUDIO_PARAMETER_ID { + unsigned data1; + unsigned data2; +}; + +struct FMOD_STUDIO_PARAMETER_DESCRIPTION { + char name[256]; + float minimum; + float maximum; + float defaultvalue; +}; + +struct FMOD_STUDIO_BANK_INFO { + int size; + void* userdata; + int userdatalength; +}; + +struct FMOD_STUDIO_ADVANCEDSETTINGS { + int cbsize; + unsigned commandqueuesize; + unsigned handleinitialsize; +}; + +struct FMOD_STUDIO_CPU_USAGE { + float update; + float nonstreaming; + float streaming; +}; + +struct FMOD_STUDIO_BUFFER_USAGE { + int studiocommandqueue; + int studiohandle; + int values[6]; +}; + +struct FMOD_STUDIO_MEMORY_USAGE { + int exclusive; + int inclusive; + int sampledata; +}; + +struct FMOD_STUDIO_SOUND_INFO { + const char* name_or_data; + unsigned mode; + int subsoundindex; +}; + +struct FMOD_STUDIO_SYSTEM { + int unused; +}; + +struct FMOD_STUDIO_EVENTINSTANCE { + int unused; +}; + +typedef int (*FMOD_STUDIO_SYSTEM_CALLBACK)(FMOD_STUDIO_SYSTEM* system, FMOD_STUDIO_SYSTEM_CALLBACK_TYPE type, void* commanddata, void* userdata); +typedef int (*FMOD_STUDIO_EVENT_CALLBACK)(FMOD_STUDIO_EVENT_CALLBACK_TYPE type, FMOD_STUDIO_EVENTINSTANCE* event, void* parameters); + +namespace { + +std::mutex gStudioMutex; + +constexpr int FMOD_OK = 0; + +} + +namespace FMOD { +namespace Studio { + +class System; +class Bank; +class EventDescription; +class EventInstance; +class Bus; +class VCA; +class CommandReplay; + +class ID : public FMOD_GUID { +}; + +class Bank { +public: + std::string path; + bool sampleDataLoaded = false; + void* userData = nullptr; + bool APS5_VABI isValid() { + return true; + } + int APS5_VABI getID(ID* id) { + std::lock_guard lock(gStudioMutex); + if (id) std::memset(id, 0, sizeof(*id)); + return FMOD_OK; + } + int APS5_VABI getPath(char* path, int size, int* retrieved) { + std::lock_guard lock(gStudioMutex); + if (path && size > 0) { + std::strncpy(path, this->path.c_str(), static_cast(size - 1)); + path[size - 1] = '\0'; + } + if (retrieved) *retrieved = static_cast(this->path.size() + 1); + return FMOD_OK; + } + int APS5_VABI unload() { + std::lock_guard lock(gStudioMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI loadSampleData() { + std::lock_guard lock(gStudioMutex); + sampleDataLoaded = true; + return FMOD_OK; + } + int APS5_VABI unloadSampleData() { + std::lock_guard lock(gStudioMutex); + sampleDataLoaded = false; + return FMOD_OK; + } + int APS5_VABI getLoadingState(FMOD_STUDIO_LOADING_STATE* state) { + std::lock_guard lock(gStudioMutex); + if (state) *state = FMOD_STUDIO_LOADING_STATE_LOADED; + return FMOD_OK; + } + int APS5_VABI getSampleLoadingState(FMOD_STUDIO_LOADING_STATE* state) { + std::lock_guard lock(gStudioMutex); + if (state) *state = sampleDataLoaded ? FMOD_STUDIO_LOADING_STATE_LOADED : FMOD_STUDIO_LOADING_STATE_UNLOADED; + return FMOD_OK; + } + int APS5_VABI getStringCount(int* count) { + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI getEventCount(int* count) { + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI getEventList(EventDescription** array, int capacity, int* count) { + (void)array; (void)capacity; + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI getBusCount(int* count) { + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI getBusList(Bus** array, int capacity, int* count) { + (void)array; (void)capacity; + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI getVCACount(int* count) { + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI getVCAList(VCA** array, int capacity, int* count) { + (void)array; (void)capacity; + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gStudioMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gStudioMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class EventDescription { +public: + std::string path; + bool sampleDataLoaded = false; + void* userData = nullptr; + int APS5_VABI isValid() { + return true; + } + int APS5_VABI getID(ID* id) { + std::lock_guard lock(gStudioMutex); + if (id) std::memset(id, 0, sizeof(*id)); + return FMOD_OK; + } + int APS5_VABI getPath(char* path, int size, int* retrieved) { + std::lock_guard lock(gStudioMutex); + if (path && size > 0) { + std::strncpy(path, this->path.c_str(), static_cast(size - 1)); + path[size - 1] = '\0'; + } + if (retrieved) *retrieved = static_cast(this->path.size() + 1); + return FMOD_OK; + } + int APS5_VABI getInstanceCount(int* count) { + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI getInstanceList(EventInstance** array, int capacity, int* count) { + (void)array; (void)capacity; + std::lock_guard lock(gStudioMutex); + if (count) *count = 0; + return FMOD_OK; + } + int APS5_VABI createInstance(EventInstance** instance); + int APS5_VABI loadSampleData() { + std::lock_guard lock(gStudioMutex); + sampleDataLoaded = true; + return FMOD_OK; + } + int APS5_VABI unloadSampleData() { + std::lock_guard lock(gStudioMutex); + sampleDataLoaded = false; + return FMOD_OK; + } + int APS5_VABI getSampleLoadingState(FMOD_STUDIO_LOADING_STATE* state) { + std::lock_guard lock(gStudioMutex); + if (state) *state = sampleDataLoaded ? FMOD_STUDIO_LOADING_STATE_LOADED : FMOD_STUDIO_LOADING_STATE_UNLOADED; + return FMOD_OK; + } + int APS5_VABI getLength(int* length) { + std::lock_guard lock(gStudioMutex); + if (length) *length = 0; + return FMOD_OK; + } + int APS5_VABI isSnapshot(bool* snapshot) { + std::lock_guard lock(gStudioMutex); + if (snapshot) *snapshot = false; + return FMOD_OK; + } + int APS5_VABI isOneshot(bool* oneshot) { + std::lock_guard lock(gStudioMutex); + if (oneshot) *oneshot = false; + return FMOD_OK; + } + int APS5_VABI isStream(bool* isstream) { + std::lock_guard lock(gStudioMutex); + if (isstream) *isstream = false; + return FMOD_OK; + } + int APS5_VABI is3D(bool* is3d) { + std::lock_guard lock(gStudioMutex); + if (is3d) *is3d = false; + return FMOD_OK; + } + int APS5_VABI hasSustainPoint(bool* sustainpoint) { + std::lock_guard lock(gStudioMutex); + if (sustainpoint) *sustainpoint = false; + return FMOD_OK; + } + int APS5_VABI getMinimumDistance(float* distance) { + std::lock_guard lock(gStudioMutex); + if (distance) *distance = 0.0f; + return FMOD_OK; + } + int APS5_VABI getMaximumDistance(float* distance) { + std::lock_guard lock(gStudioMutex); + if (distance) *distance = 10000.0f; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gStudioMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gStudioMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class EventInstance { +public: + EventDescription* description = nullptr; + bool playing = false; + bool paused = false; + float volume = 1.0f; + float pitch = 1.0f; + void* userData = nullptr; + std::unordered_map paramsByName; + int APS5_VABI isValid() { + return true; + } + int APS5_VABI getDescription(EventDescription** description) { + std::lock_guard lock(gStudioMutex); + if (description) *description = this->description; + return FMOD_OK; + } + int APS5_VABI getVolume(float* volume, float* finalvolume) { + std::lock_guard lock(gStudioMutex); + if (volume) *volume = this->volume; + if (finalvolume) *finalvolume = this->volume; + return FMOD_OK; + } + int APS5_VABI setVolume(float volume) { + std::lock_guard lock(gStudioMutex); + this->volume = volume; + return FMOD_OK; + } + int APS5_VABI getPitch(float* pitch, float* finalpitch) { + std::lock_guard lock(gStudioMutex); + if (pitch) *pitch = this->pitch; + if (finalpitch) *finalpitch = this->pitch; + return FMOD_OK; + } + int APS5_VABI setPitch(float pitch) { + std::lock_guard lock(gStudioMutex); + this->pitch = pitch; + return FMOD_OK; + } + int APS5_VABI getPaused(bool* paused) { + std::lock_guard lock(gStudioMutex); + if (paused) *paused = this->paused; + return FMOD_OK; + } + int APS5_VABI setPaused(bool paused) { + std::lock_guard lock(gStudioMutex); + this->paused = paused; + return FMOD_OK; + } + int APS5_VABI start() { + std::lock_guard lock(gStudioMutex); + playing = true; + return FMOD_OK; + } + int APS5_VABI stop(FMOD_STUDIO_STOP_MODE mode) { + (void)mode; + std::lock_guard lock(gStudioMutex); + playing = false; + return FMOD_OK; + } + int APS5_VABI release() { + std::lock_guard lock(gStudioMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI getPlaybackState(FMOD_STUDIO_PLAYBACK_STATE* state) { + std::lock_guard lock(gStudioMutex); + if (state) *state = playing ? FMOD_STUDIO_PLAYBACK_PLAYING : FMOD_STUDIO_PLAYBACK_STOPPED; + return FMOD_OK; + } + int APS5_VABI setTimelinePosition(int position) { + (void)position; + return FMOD_OK; + } + int APS5_VABI getTimelinePosition(int* position) { + std::lock_guard lock(gStudioMutex); + if (position) *position = 0; + return FMOD_OK; + } + int APS5_VABI setParameterByName(const char* name, float value, bool ignoreseekspeed) { + (void)ignoreseekspeed; + std::lock_guard lock(gStudioMutex); + if (name) paramsByName[name] = value; + return FMOD_OK; + } + int APS5_VABI getParameterByName(const char* name, float* value, float* finalvalue) { + std::lock_guard lock(gStudioMutex); + float stored = 0.0f; + if (name) { + auto it = paramsByName.find(name); + if (it != paramsByName.end()) stored = it->second; + } + if (value) *value = stored; + if (finalvalue) *finalvalue = stored; + return FMOD_OK; + } + int APS5_VABI setParameterByID(FMOD_STUDIO_PARAMETER_ID id, float value, bool ignoreseekspeed) { + (void)id; (void)ignoreseekspeed; + return FMOD_OK; + } + int APS5_VABI keyOff() { return FMOD_OK; } + int APS5_VABI setReverbLevel(int index, float level) { + (void)index; (void)level; + return FMOD_OK; + } + int APS5_VABI set3DAttributes(const FMOD_3D_ATTRIBUTES* attributes) { + (void)attributes; + return FMOD_OK; + } + int APS5_VABI setListenerMask(unsigned mask) { + (void)mask; + return FMOD_OK; + } + int APS5_VABI getChannelGroup(FMOD_CHANNELGROUP** group) { + std::lock_guard lock(gStudioMutex); + if (group) *group = nullptr; + return FMOD_OK; + } + int APS5_VABI setCallback(FMOD_STUDIO_EVENT_CALLBACK callback) { + (void)callback; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gStudioMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gStudioMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } + int APS5_VABI hasSustainPoint(bool* sustainpoint) { + std::lock_guard lock(gStudioMutex); + if (sustainpoint) *sustainpoint = false; + return FMOD_OK; + } + int APS5_VABI triggerCue() { return FMOD_OK; } +}; + +int APS5_VABI EventDescription::createInstance(EventInstance** instance) { + std::lock_guard lock(gStudioMutex); + if (!instance) return FMOD_OK; + *instance = new EventInstance(); + (*instance)->description = this; + return FMOD_OK; +} + +class Bus { +public: + std::string path; + float volume = 1.0f; + bool paused = false; + bool mute = false; + void* userData = nullptr; + int APS5_VABI isValid() { + return true; + } + int APS5_VABI getVolume(float* volume, float* finalvolume) { + std::lock_guard lock(gStudioMutex); + if (volume) *volume = this->volume; + if (finalvolume) *finalvolume = this->volume; + return FMOD_OK; + } + int APS5_VABI setVolume(float volume) { + std::lock_guard lock(gStudioMutex); + this->volume = volume; + return FMOD_OK; + } + int APS5_VABI getPaused(bool* paused) { + std::lock_guard lock(gStudioMutex); + if (paused) *paused = this->paused; + return FMOD_OK; + } + int APS5_VABI setPaused(bool paused) { + std::lock_guard lock(gStudioMutex); + this->paused = paused; + return FMOD_OK; + } + int APS5_VABI getMute(bool* mute) { + std::lock_guard lock(gStudioMutex); + if (mute) *mute = this->mute; + return FMOD_OK; + } + int APS5_VABI setMute(bool mute) { + std::lock_guard lock(gStudioMutex); + this->mute = mute; + return FMOD_OK; + } + int APS5_VABI stopAllEvents(FMOD_STUDIO_STOP_MODE mode) { + (void)mode; + return FMOD_OK; + } + int APS5_VABI getChannelGroup(FMOD_CHANNELGROUP** group) { + std::lock_guard lock(gStudioMutex); + if (group) *group = nullptr; + return FMOD_OK; + } + int APS5_VABI lockChannelGroup() { return FMOD_OK; } + int APS5_VABI unlockChannelGroup() { return FMOD_OK; } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gStudioMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gStudioMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class VCA { +public: + std::string path; + float volume = 1.0f; + void* userData = nullptr; + int APS5_VABI isValid() { + return true; + } + int APS5_VABI getVolume(float* volume, float* finalvolume) { + std::lock_guard lock(gStudioMutex); + if (volume) *volume = this->volume; + if (finalvolume) *finalvolume = this->volume; + return FMOD_OK; + } + int APS5_VABI setVolume(float volume) { + std::lock_guard lock(gStudioMutex); + this->volume = volume; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gStudioMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gStudioMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class CommandReplay { +public: + void* userData = nullptr; + int APS5_VABI isValid() { + return true; + } + int APS5_VABI getLength(float* length) { + std::lock_guard lock(gStudioMutex); + if (length) *length = 0.0f; + return FMOD_OK; + } + int APS5_VABI start() { return FMOD_OK; } + int APS5_VABI stop() { return FMOD_OK; } + int APS5_VABI seekToTime(float time) { + (void)time; + return FMOD_OK; + } + int APS5_VABI release() { + std::lock_guard lock(gStudioMutex); + delete this; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gStudioMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gStudioMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } +}; + +class System { +public: + bool initialized = false; + int maxChannels = 0; + int numListeners = 1; + void* userData = nullptr; + std::unordered_map paramsByName; + std::vector banks; + static int APS5_VABI create(System** system, unsigned headerversion) { + (void)headerversion; + std::lock_guard lock(gStudioMutex); + if (!system) return FMOD_OK; + *system = new System(); + return FMOD_OK; + } + int APS5_VABI isValid() { + return true; + } + int APS5_VABI release() { + std::lock_guard lock(gStudioMutex); + for (Bank* bank : banks) delete bank; + banks.clear(); + delete this; + return FMOD_OK; + } + int APS5_VABI initialize(int maxchannels, unsigned studioflags, unsigned flags, void* extradriverdata) { + (void)studioflags; (void)flags; (void)extradriverdata; + std::lock_guard lock(gStudioMutex); + maxChannels = maxchannels; + initialized = true; + return FMOD_OK; + } + int APS5_VABI update() { return FMOD_OK; } + int APS5_VABI getEvent(const char* pathOrID, EventDescription** event) { + std::lock_guard lock(gStudioMutex); + if (!event) return FMOD_OK; + *event = new EventDescription(); + if (pathOrID) (*event)->path = pathOrID; + return FMOD_OK; + } + int APS5_VABI getBankList(Bank** array, int capacity, int* count) { + std::lock_guard lock(gStudioMutex); + if (count) *count = static_cast(banks.size()); + if (array && capacity > 0) { + int filled = capacity < static_cast(banks.size()) ? capacity : static_cast(banks.size()); + for (int i = 0; i < filled; i++) array[i] = banks[static_cast(i)]; + } + return FMOD_OK; + } + int APS5_VABI getBankCount(int* count) { + std::lock_guard lock(gStudioMutex); + if (count) *count = static_cast(banks.size()); + return FMOD_OK; + } + int APS5_VABI loadBankFile(const char* filename, unsigned flags, Bank** bank) { + (void)flags; + std::lock_guard lock(gStudioMutex); + if (!bank) return FMOD_OK; + *bank = new Bank(); + if (filename) (*bank)->path = filename; + banks.push_back(*bank); + return FMOD_OK; + } + int APS5_VABI loadBankMemory(const char* buffer, int length, unsigned mode, unsigned flags, Bank** bank) { + (void)buffer; (void)length; (void)mode; (void)flags; + std::lock_guard lock(gStudioMutex); + if (!bank) return FMOD_OK; + *bank = new Bank(); + banks.push_back(*bank); + return FMOD_OK; + } + int APS5_VABI unloadAll() { + std::lock_guard lock(gStudioMutex); + for (Bank* bank : banks) delete bank; + banks.clear(); + return FMOD_OK; + } + int APS5_VABI flushCommands() { return FMOD_OK; } + int APS5_VABI flushSampleLoading() { return FMOD_OK; } + int APS5_VABI setNumListeners(int numlisteners) { + std::lock_guard lock(gStudioMutex); + numListeners = numlisteners; + return FMOD_OK; + } + int APS5_VABI getNumListeners(int* numlisteners) { + std::lock_guard lock(gStudioMutex); + if (numlisteners) *numlisteners = numListeners; + return FMOD_OK; + } + int APS5_VABI setListenerAttributes(int index, const FMOD_3D_ATTRIBUTES* attributes, const FMOD_VECTOR* attenuationposition) { + (void)index; (void)attributes; (void)attenuationposition; + return FMOD_OK; + } + int APS5_VABI getBus(const char* pathOrID, Bus** bus) { + std::lock_guard lock(gStudioMutex); + if (!bus) return FMOD_OK; + *bus = new Bus(); + if (pathOrID) (*bus)->path = pathOrID; + return FMOD_OK; + } + int APS5_VABI getVCA(const char* pathOrID, VCA** vca) { + std::lock_guard lock(gStudioMutex); + if (!vca) return FMOD_OK; + *vca = new VCA(); + if (pathOrID) (*vca)->path = pathOrID; + return FMOD_OK; + } + int APS5_VABI getParameterByName(const char* name, float* value, float* finalvalue) { + std::lock_guard lock(gStudioMutex); + float stored = 0.0f; + if (name) { + auto it = paramsByName.find(name); + if (it != paramsByName.end()) stored = it->second; + } + if (value) *value = stored; + if (finalvalue) *finalvalue = stored; + return FMOD_OK; + } + int APS5_VABI setParameterByName(const char* name, float value, bool ignoreseekspeed) { + (void)ignoreseekspeed; + std::lock_guard lock(gStudioMutex); + if (name) paramsByName[name] = value; + return FMOD_OK; + } + int APS5_VABI setCallback(FMOD_STUDIO_SYSTEM_CALLBACK callback, FMOD_STUDIO_SYSTEM_CALLBACK_TYPE callbackmask) { + (void)callback; (void)callbackmask; + return FMOD_OK; + } + int APS5_VABI registerPlugin(const FMOD_DSP_DESCRIPTION* description) { + (void)description; + return FMOD_OK; + } + int APS5_VABI setUserData(void* userdata) { + std::lock_guard lock(gStudioMutex); + userData = userdata; + return FMOD_OK; + } + int APS5_VABI getUserData(void** userdata) { + std::lock_guard lock(gStudioMutex); + if (userdata) *userdata = userData; + return FMOD_OK; + } + int APS5_VABI getCPUUsage(FMOD_STUDIO_CPU_USAGE* usage) { + std::lock_guard lock(gStudioMutex); + if (usage) std::memset(usage, 0, sizeof(*usage)); + return FMOD_OK; + } + int APS5_VABI getBufferUsage(FMOD_STUDIO_BUFFER_USAGE* usage) { + std::lock_guard lock(gStudioMutex); + if (usage) std::memset(usage, 0, sizeof(*usage)); + return FMOD_OK; + } + int APS5_VABI resetBufferUsage() { return FMOD_OK; } + int APS5_VABI getMemoryUsage(FMOD_STUDIO_MEMORY_USAGE* memoryusage) { + std::lock_guard lock(gStudioMutex); + if (memoryusage) std::memset(memoryusage, 0, sizeof(*memoryusage)); + return FMOD_OK; + } +}; + +} +} + +extern "C" { + +namespace { + +__attribute__((used)) int fmodstudio_nid_stub_00() { APS5_LOG_ERR("%s", "+V2QcIcvWpw"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_01() { APS5_LOG_ERR("%s", "2QOSvz5hvAg"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_02() { APS5_LOG_ERR("%s", "3loRRCoqXfU"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_03() { APS5_LOG_ERR("%s", "6E6MUTGjxhI"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_04() { APS5_LOG_ERR("%s", "6nWYF1yMOl4"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_05() { APS5_LOG_ERR("%s", "7hd4bRJuLMg"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_06() { APS5_LOG_ERR("%s", "-7VafNNns2A"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_07() { APS5_LOG_ERR("%s", "AdR01fo-WaM"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_08() { APS5_LOG_ERR("%s", "alleMuKEWr8"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_09() { APS5_LOG_ERR("%s", "Bu8uFP7ME2k"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_10() { APS5_LOG_ERR("%s", "Cm2cmtCv8cA"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_11() { APS5_LOG_ERR("%s", "DsFsPD0MWNc"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_12() { APS5_LOG_ERR("%s", "EfM9xXvBmxk"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_13() { APS5_LOG_ERR("%s", "--f9RJwAh1A"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_14() { APS5_LOG_ERR("%s", "fbAVXdyBuso"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_15() { APS5_LOG_ERR("%s", "Js90KQXVS4s"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_16() { APS5_LOG_ERR("%s", "LG53EJZJDnA"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_17() { APS5_LOG_ERR("%s", "nhBPjhZ+VWs"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_18() { APS5_LOG_ERR("%s", "oCYWES02VPc"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_19() { APS5_LOG_ERR("%s", "omDBr+dcDVc"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_20() { APS5_LOG_ERR("%s", "uWXqw4LeM5Q"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_21() { APS5_LOG_ERR("%s", "VdQjbIsdhXQ"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_22() { APS5_LOG_ERR("%s", "vVwA2cZA5e4"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_23() { APS5_LOG_ERR("%s", "WP51b8simn8"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_24() { APS5_LOG_ERR("%s", "Zi1n4MvZ3-0"); return 0; } + +} + +APS5_EXPORT("+V2QcIcvWpw", fmodstudio_nid_stub_00); +APS5_EXPORT("2QOSvz5hvAg", fmodstudio_nid_stub_01); +APS5_EXPORT("3loRRCoqXfU", fmodstudio_nid_stub_02); +APS5_EXPORT("6E6MUTGjxhI", fmodstudio_nid_stub_03); +APS5_EXPORT("6nWYF1yMOl4", fmodstudio_nid_stub_04); +APS5_EXPORT("7hd4bRJuLMg", fmodstudio_nid_stub_05); +APS5_EXPORT("-7VafNNns2A", fmodstudio_nid_stub_06); +APS5_EXPORT("AdR01fo-WaM", fmodstudio_nid_stub_07); +APS5_EXPORT("alleMuKEWr8", fmodstudio_nid_stub_08); +APS5_EXPORT("Bu8uFP7ME2k", fmodstudio_nid_stub_09); +APS5_EXPORT("Cm2cmtCv8cA", fmodstudio_nid_stub_10); +APS5_EXPORT("DsFsPD0MWNc", fmodstudio_nid_stub_11); +APS5_EXPORT("EfM9xXvBmxk", fmodstudio_nid_stub_12); +APS5_EXPORT("--f9RJwAh1A", fmodstudio_nid_stub_13); +APS5_EXPORT("fbAVXdyBuso", fmodstudio_nid_stub_14); +APS5_EXPORT("Js90KQXVS4s", fmodstudio_nid_stub_15); +APS5_EXPORT("LG53EJZJDnA", fmodstudio_nid_stub_16); +APS5_EXPORT("nhBPjhZ+VWs", fmodstudio_nid_stub_17); +APS5_EXPORT("oCYWES02VPc", fmodstudio_nid_stub_18); +APS5_EXPORT("omDBr+dcDVc", fmodstudio_nid_stub_19); +APS5_EXPORT("uWXqw4LeM5Q", fmodstudio_nid_stub_20); +APS5_EXPORT("VdQjbIsdhXQ", fmodstudio_nid_stub_21); +APS5_EXPORT("vVwA2cZA5e4", fmodstudio_nid_stub_22); +APS5_EXPORT("WP51b8simn8", fmodstudio_nid_stub_23); +APS5_EXPORT("Zi1n4MvZ3-0", fmodstudio_nid_stub_24); + +} From 279587cb16ff75851223357d5aea6f7cf2e59040 Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 12:57:23 +0200 Subject: [PATCH 48/93] fix(libcohtml): provide libcohtml.Prospero.prx for DT_NEEDED Copy the patched libcohtml.prx to libcohtml.Prospero.prx (exact DT_NEEDED name) via a post-build copy in the module CMakeLists, wired into the libs umbrella target. OUTPUT_NAME with a dot was rejected: the parent pipeline derives unpatched/patched paths from the directory target name, so renaming output would break it; dotted CMake target names are supported (e.g. libSceSaveData.native) but the parent-owned NID-patch rule is kept untouched. Adds DEPLOY.md deploy note. All 50 raw NIDs re-verified in the renamed file export table. --- core/libs/prx/libcohtml/CMakeLists.txt | 13 +++++++++++++ core/libs/prx/libcohtml/DEPLOY.md | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 core/libs/prx/libcohtml/DEPLOY.md diff --git a/core/libs/prx/libcohtml/CMakeLists.txt b/core/libs/prx/libcohtml/CMakeLists.txt index 842799d34f..f32848eb25 100644 --- a/core/libs/prx/libcohtml/CMakeLists.txt +++ b/core/libs/prx/libcohtml/CMakeLists.txt @@ -4,3 +4,16 @@ add_library(libcohtml SHARED EXCLUDE_FROM_ALL target_include_directories(libcohtml PRIVATE ${LIBS_INCLUDE_DIR}) set_target_properties(libcohtml PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(libcohtml) + +set(LIBCOHTML_PATCHED ${PATCHED_DIR}/libcohtml.prx) +set(LIBCOHTML_PROSPERO_PATCHED ${PATCHED_DIR}/libcohtml.Prospero.prx) +add_custom_command( + OUTPUT ${LIBCOHTML_PROSPERO_PATCHED} + COMMAND ${CMAKE_COMMAND} -E copy ${LIBCOHTML_PATCHED} ${LIBCOHTML_PROSPERO_PATCHED} + DEPENDS ${LIBCOHTML_PATCHED} + COMMENT "Providing libcohtml.Prospero.prx for DT_NEEDED" + VERBATIM +) +add_custom_target(libcohtml_prospero DEPENDS ${LIBCOHTML_PROSPERO_PATCHED}) +list(APPEND PATCHED_OUTPUTS ${LIBCOHTML_PROSPERO_PATCHED}) +set(PATCHED_OUTPUTS ${PATCHED_OUTPUTS} PARENT_SCOPE) diff --git a/core/libs/prx/libcohtml/DEPLOY.md b/core/libs/prx/libcohtml/DEPLOY.md new file mode 100644 index 0000000000..de4c3ffea8 --- /dev/null +++ b/core/libs/prx/libcohtml/DEPLOY.md @@ -0,0 +1,16 @@ +# libcohtml — deploy note + +Game `DT_NEEDED` expects exactly: `libcohtml.Prospero.prx`. + +Copy next to the relinked eboot (same directory): + +- `build/core/libs/libs/libcohtml.Prospero.prx` (patched build output, 50 NIDs) — required. +- `libgcc_s_seh-1.dll`, `libstdc++-6.dll`, `libwinpthread-1.dll` from `winlibs-gcc15/mingw64/bin` — required on Windows (see `docs/TechnicalDebt.md`). + +Do NOT deploy: + +- `libcohtml.prx` (build intermediate kept for the NID-patch pipeline, wrong `DT_NEEDED` name). +- Anything under `build/core/libs/libs/unpatched/` (NIDs not patched). +- `libc.prx` / `libkernel.prx` are not runtime imports of this module (no such DLL dependency); the game may need its other modules separately (out of scope). + +Scope: raw-NID stubs only; real Cohtml rendering stays out of scope. From 030e62dfa33ae7cdf34f5110beac5a691ff11fe1 Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 13:24:34 +0200 Subject: [PATCH 49/93] feat(prx): fmod real audio output via SDL + identify 22/27 NIDs - libfmod: route createSound (OPENMEMORY WAV/RAW, file placeholder sine) + playSound + update() mixer pump into SDL queue (48k stereo F32, mirrors libSceAudioOut). Channel::setVolume/isPlaying/position now track real PCM cursor with loop support. Silent fallback via ANYPS5_FMOD_SILENT=1, NOSOUND output, or SDL open failure. Link SDL2-static like libSceAudioOut. - NIDs: recompute from public FMOD headers (fmod.h/hpp + fmod_studio 2.0) with exact Itanium manglings (g++ nm) + SHA1+suffix (NidCompute.cpp). Identify 2/2 libfmod (setFileSystem, setCallback) + 20/25 studio (const createInstance, getEvent, getCoreSystem, getSoundInfo, etc.). Replace stubs with ABI-compatible real impls; 5 studio remain unknown (Cm2cmtCv8cA, EfM9xXvBmxk, Js90KQXVS4s, oCYWES02VPc, vVwA2cZA5e4). - Tests: coverage OK (51+42 Legends NIDs, missing 0), audio smoke (WAV 4800f + RAW 2048f, volume, channelsPlaying, stop) OK real and silent. --- core/libs/prx/libfmod/CMakeLists.txt | 3 +- core/libs/prx/libfmod/Export.cpp | 509 ++++++++++++++++++++++++- core/libs/prx/libfmodstudio/Export.cpp | 219 +++++++++-- 3 files changed, 669 insertions(+), 62 deletions(-) diff --git a/core/libs/prx/libfmod/CMakeLists.txt b/core/libs/prx/libfmod/CMakeLists.txt index a48f55b15d..5d1cb47f71 100644 --- a/core/libs/prx/libfmod/CMakeLists.txt +++ b/core/libs/prx/libfmod/CMakeLists.txt @@ -4,7 +4,8 @@ add_library(libfmod SHARED EXCLUDE_FROM_ALL target_include_directories(libfmod PRIVATE ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/3rdparty/SDL2/include ) -target_link_libraries(libfmod PRIVATE libkernel libc) +target_link_libraries(libfmod PRIVATE libkernel libc SDL2-static) target_compile_options(libfmod PRIVATE -fkeep-inline-functions) configure_windows_unwind(libfmod) diff --git a/core/libs/prx/libfmod/Export.cpp b/core/libs/prx/libfmod/Export.cpp index 334b9ffb0a..6c5a7476ff 100644 --- a/core/libs/prx/libfmod/Export.cpp +++ b/core/libs/prx/libfmod/Export.cpp @@ -1,13 +1,28 @@ #include #include #include +#include #include +#include #include #include #include #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "SDL.h" + +// FMOD_MODE bits (subset, from fmod_common.h) — used to detect memory buffers. +constexpr unsigned FMOD_LOOP_OFF = 0x00000001u; +constexpr unsigned FMOD_LOOP_NORMAL = 0x00000002u; +constexpr unsigned FMOD_LOOP_BIDI = 0x00000004u; +constexpr unsigned FMOD_2D_FLAG = 0x00000008u; +constexpr unsigned FMOD_3D_FLAG = 0x00000010u; +constexpr unsigned FMOD_CREATESTREAM_FLAG = 0x00000080u; +constexpr unsigned FMOD_OPENMEMORY_FLAG = 0x00000800u; +constexpr unsigned FMOD_OPENMEMORY_POINT_FLAG = 0x10000000u; +constexpr unsigned FMOD_OPENRAW_FLAG = 0x00001000u; +constexpr unsigned FMOD_NONBLOCKING_FLAG = 0x00010000u; enum FMOD_OUTPUTTYPE { FMOD_OUTPUTTYPE_AUTODETECT = 0, @@ -424,6 +439,149 @@ std::mutex gFmodMutex; constexpr int FMOD_OK = 0; constexpr unsigned FMOD_VERSION_CURRENT = 0x00020206; +// ---- Real-audio backend (SDL queue, mirrors libSceAudioOut) ---- +// Silent fallback is kept via ANYPS5_FMOD_SILENT=1 or NOSOUND output type +// or when SDL cannot be opened. When silent, update() is a no-op and +// isPlaying() keeps the previous sticky behaviour so the old smoke test +// (playSound(nullptr) -> playing==true) still passes. +constexpr int kFmodOutRate = 48000; +constexpr int kFmodOutCh = 2; +constexpr int kFmodMixFrames = 1024; + +static SDL_AudioDeviceID gFmodDevice = 0; +static SDL_AudioSpec gFmodSpec = {}; +static bool gFmodSdlTried = false; +static bool gFmodSilent = false; + +static bool FmodSilentRequested() { + const char* s = std::getenv("ANYPS5_FMOD_SILENT"); + if (s && (s[0] == '1' || s[0] == 'y' || s[0] == 'Y')) return true; + s = std::getenv("ANYPS5_FMOD_NOSOUND"); + if (s && (s[0] == '1' || s[0] == 'y' || s[0] == 'Y')) return true; + return false; +} + +static bool FmodEnsureDevice() { + if (gFmodDevice != 0) return true; + if (gFmodSdlTried) return gFmodDevice != 0; + gFmodSdlTried = true; + if (FmodSilentRequested()) { + gFmodSilent = true; + return false; + } + if (SDL_InitSubSystem(SDL_INIT_AUDIO) < 0) { + gFmodSilent = true; + return false; + } + SDL_AudioSpec desired{}; + desired.freq = kFmodOutRate; + desired.format = AUDIO_F32SYS; + desired.channels = static_cast(kFmodOutCh); + desired.samples = 1024; + desired.callback = nullptr; + SDL_AudioSpec obtained{}; + gFmodDevice = SDL_OpenAudioDevice(nullptr, 0, &desired, &obtained, SDL_AUDIO_ALLOW_ANY_CHANGE); + if (gFmodDevice == 0) { + gFmodSilent = true; + return false; + } + gFmodSpec = obtained; + SDL_PauseAudioDevice(gFmodDevice, 0); + return true; +} + +static void FmodMakeSine(std::vector& out, int rate, int ch, float freqHz, float seconds) { + const size_t frames = static_cast(rate * seconds); + out.resize(frames * static_cast(ch)); + const float phaseInc = 2.0f * 3.14159265358979323846f * freqHz / static_cast(rate); + float phase = 0.0f; + for (size_t f = 0; f < frames; f++) { + const float s = std::sin(phase) * 0.25f; + phase += phaseInc; + for (int c = 0; c < ch; c++) out[f * static_cast(ch) + static_cast(c)] = s; + } +} + +// Minimal WAV (RIFF/WAVE) parser -> float stereo 48k. Returns false if not WAV. +static bool FmodParseWav(const uint8_t* data, size_t len, std::vector& outPcm, int& outRate, int& outCh) { + if (len < 44 || std::memcmp(data, "RIFF", 4) != 0 || std::memcmp(data + 8, "WAVE", 4) != 0) return false; + uint16_t audioFmt = 0, numCh = 0, bitsPerSample = 0; + uint32_t sampleRate = 0; + const uint8_t* snd = nullptr; + size_t sndLen = 0; + size_t off = 12; + auto rd16 = [](const uint8_t* p) -> uint16_t { return static_cast(p[0] | (p[1] << 8)); }; + auto rd32 = [](const uint8_t* p) -> uint32_t { return static_cast(p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24)); }; + while (off + 8 <= len) { + const uint8_t* chunk = data + off; + uint32_t chunkLen = rd32(chunk + 4); + if (off + 8 + chunkLen > len) break; + if (std::memcmp(chunk, "fmt ", 4) == 0 && chunkLen >= 16) { + audioFmt = rd16(chunk + 8); + numCh = rd16(chunk + 10); + sampleRate = rd32(chunk + 12); + bitsPerSample = rd16(chunk + 22); + } else if (std::memcmp(chunk, "data", 4) == 0) { + snd = chunk + 8; + sndLen = chunkLen; + } + off += 8 + ((chunkLen + 1) & ~static_cast(1)); + } + if (!snd || sndLen == 0 || numCh == 0 || sampleRate == 0) return false; + if (audioFmt != 1 && audioFmt != 3) return false; // PCM or FLOAT + int bytesPerSamp = bitsPerSample / 8; + if (bytesPerSamp <= 0) return false; + size_t frames = sndLen / (static_cast(numCh) * static_cast(bytesPerSamp)); + if (frames == 0) return false; + if (frames > 48000u * 30u) frames = 48000u * 30u; // cap 30s + std::vector tmp; + tmp.reserve(frames * 2); + for (size_t f = 0; f < frames; f++) { + float l = 0.0f, r = 0.0f; + for (int c = 0; c < numCh && c < 8; c++) { + const uint8_t* p = snd + (f * numCh + c) * bytesPerSamp; + float s = 0.0f; + if (audioFmt == 1) { + if (bitsPerSample == 8) s = (static_cast(p[0]) - 128) / 128.0f; + else if (bitsPerSample == 16) { int16_t v; std::memcpy(&v, p, 2); s = v / 32768.0f; } + else if (bitsPerSample == 24) { int32_t v = (p[0] | (p[1] << 8) | (p[2] << 16)); if (v & 0x800000) v |= ~0xFFFFFF; s = v / 8388608.0f; } + else if (bitsPerSample == 32) { int32_t v; std::memcpy(&v, p, 4); s = v / 2147483648.0f; } + else return false; + } else { + if (bitsPerSample == 32) { float v; std::memcpy(&v, p, 4); s = v; } + else return false; + } + if (c == 0) l = s; + else if (c == 1) r = s; + else { l += s; r += s; } + } + if (numCh == 1) r = l; + tmp.push_back(l); + tmp.push_back(r); + } + // Resample to 48k if needed (linear). + if (sampleRate != static_cast(kFmodOutRate)) { + const size_t inFrames = tmp.size() / 2; + const size_t outFrames = static_cast((static_cast(inFrames) * kFmodOutRate) / sampleRate); + std::vector rs; + rs.reserve(outFrames * 2); + for (size_t i = 0; i < outFrames; i++) { + double pos = (static_cast(i) * sampleRate) / kFmodOutRate; + size_t i0 = static_cast(pos); + size_t i1 = i0 + 1 < inFrames ? i0 + 1 : i0; + float t = static_cast(pos - i0); + rs.push_back(tmp[i0 * 2] * (1 - t) + tmp[i1 * 2] * t); + rs.push_back(tmp[i0 * 2 + 1] * (1 - t) + tmp[i1 * 2 + 1] * t); + } + outPcm.swap(rs); + } else { + outPcm.swap(tmp); + } + outRate = kFmodOutRate; + outCh = 2; + return true; +} + } namespace FMOD { @@ -448,11 +606,11 @@ public: float maxDistance = 10000.0f; SoundGroup* soundGroup = nullptr; void* userData = nullptr; - int APS5_VABI release() { - std::lock_guard lock(gFmodMutex); - delete this; - return FMOD_OK; - } + // Real-audio decoded PCM (float stereo 48k, interleaved). Empty = silent/placeholder. + std::vector pcm; + int pcmRate = kFmodOutRate; + int pcmCh = 2; + int APS5_VABI release(); int APS5_VABI getSystemObject(System** system); int APS5_VABI setMode(unsigned mode) { std::lock_guard lock(gFmodMutex); @@ -476,13 +634,29 @@ public: return FMOD_OK; } int APS5_VABI getLength(unsigned* length, unsigned lengthtype) { - (void)lengthtype; std::lock_guard lock(gFmodMutex); - if (length) *length = this->length; + if (!length) return FMOD_OK; + if (!pcm.empty()) { + const unsigned frames = static_cast(pcm.size() / 2); + // FMOD_TIMEUNIT: MS=1, PCM=2, PCMBYTES=4, RAWBYTES=8 + if (lengthtype == 1) *length = (frames * 1000u) / static_cast(kFmodOutRate); + else if (lengthtype == 2) *length = frames; + else if (lengthtype == 4 || lengthtype == 8) *length = frames * 2u * sizeof(float); + else *length = frames; + return FMOD_OK; + } + *length = this->length; return FMOD_OK; } int APS5_VABI getFormat(FMOD_SOUND_TYPE* type, FMOD_SOUND_FORMAT* format, int* channels, int* bits) { std::lock_guard lock(gFmodMutex); + if (!pcm.empty()) { + if (type) *type = FMOD_SOUND_TYPE_UNKNOWN; + if (format) *format = FMOD_SOUND_FORMAT_PCMFLOAT; + if (channels) *channels = 2; + if (bits) *bits = 32; + return FMOD_OK; + } if (type) *type = FMOD_SOUND_TYPE_UNKNOWN; if (format) *format = FMOD_SOUND_FORMAT_PCM16; if (channels) *channels = 2; @@ -711,6 +885,12 @@ public: unsigned mode = 0; bool volumeRamp = false; void* userData = nullptr; + // Mixer state for real-audio backend (used when this control is a Channel + // with decoded PCM; groups keep mixHasData==false and use sticky flag). + size_t mixCursor = 0; + size_t mixTotal = 0; + bool mixHasData = false; + bool mixLoop = false; int APS5_VABI setVolume(float volume) { std::lock_guard lock(gFmodMutex); this->volume = volume; @@ -758,7 +938,12 @@ public: } int APS5_VABI isPlaying(bool* isplaying) { std::lock_guard lock(gFmodMutex); - if (isplaying) *isplaying = playing && !paused; + if (!isplaying) return FMOD_OK; + if (mixHasData) { + *isplaying = playing && !paused && (mixLoop || mixCursor < mixTotal); + } else { + *isplaying = playing && !paused; + } return FMOD_OK; } int APS5_VABI setMode(unsigned mode) { @@ -860,6 +1045,7 @@ public: int priority = 128; unsigned position = 0; int loopCount = 0; + System* owner = nullptr; int APS5_VABI setPriority(int priority) { std::lock_guard lock(gFmodMutex); this->priority = priority; @@ -884,12 +1070,15 @@ public: (void)postype; std::lock_guard lock(gFmodMutex); this->position = position; + // Keep mixer cursor in sync (PCM frames). postype==2 (PCM) assumed. + mixCursor = static_cast(position); + if (mixHasData && mixCursor > mixTotal) mixCursor = mixTotal; return FMOD_OK; } int APS5_VABI getPosition(unsigned* position, unsigned postype) { (void)postype; std::lock_guard lock(gFmodMutex); - if (position) *position = this->position; + if (position) *position = static_cast(mixHasData ? mixCursor : this->position); return FMOD_OK; } int APS5_VABI setChannelGroup(ChannelGroup* channelgroup) { @@ -1023,6 +1212,23 @@ public: } }; +static std::vector gFmodChannels; + +static int APS5_VABI SoundReleaseImpl(Sound* self) { + std::lock_guard lock(gFmodMutex); + for (Channel* ch : gFmodChannels) { + if (ch && ch->sound == self) { + ch->playing = false; + ch->sound = nullptr; + ch->mixHasData = false; + } + } + delete self; + return FMOD_OK; +} + +int APS5_VABI Sound::release() { return SoundReleaseImpl(this); } + class System { public: bool initialized = false; @@ -1050,6 +1256,16 @@ public: } int APS5_VABI release() { std::lock_guard lock(gFmodMutex); + // Drop channels owned by this system (game holds raw pointers; mark + // them stopped and free to avoid unbounded growth). + for (auto it = gFmodChannels.begin(); it != gFmodChannels.end();) { + if (*it && (*it)->owner == this) { + delete *it; + it = gFmodChannels.erase(it); + } else { + ++it; + } + } delete masterChannelGroup; delete masterSoundGroup; masterChannelGroup = nullptr; @@ -1065,6 +1281,21 @@ public: initialized = true; if (!masterChannelGroup) masterChannelGroup = new ChannelGroup(); if (!masterSoundGroup) masterSoundGroup = new SoundGroup(); + // Real-audio: try SDL unless NOSOUND output or env requests silent. + // Keep silent backend as fallback (no failure returned). + bool wantSilent = FmodSilentRequested() || + output == FMOD_OUTPUTTYPE_NOSOUND || + output == FMOD_OUTPUTTYPE_NOSOUND_NRT; + if (!wantSilent) { + // Unlock during SDL open (SDL may call back)? FmodEnsureDevice uses + // no FMOD lock internally except SDL, so we can call with lock held + // for simplicity; SDL_OpenAudioDevice does not call into FMOD. + if (!FmodEnsureDevice()) { + APS5_LOG_ERR("%s", "FMOD: SDL audio unavailable, silent fallback"); + } else { + APS5_LOG_OUT("%s", "FMOD: SDL audio output active (48k stereo)"); + } + } return FMOD_OK; } int APS5_VABI close() { @@ -1072,10 +1303,119 @@ public: initialized = false; return FMOD_OK; } - int APS5_VABI update() { return FMOD_OK; } + int APS5_VABI update() { + std::lock_guard lock(gFmodMutex); + if (!initialized) return FMOD_OK; + if (gFmodSilent || gFmodDevice == 0) return FMOD_OK; // silent fallback + // Throttle like libSceAudioOut: avoid unbounded queue growth. + Uint32 queued = SDL_GetQueuedAudioSize(gFmodDevice); + const Uint32 oneBuf = static_cast(kFmodMixFrames * kFmodOutCh * sizeof(float)); + if (queued > oneBuf * 8u) return FMOD_OK; + float mix[kFmodMixFrames * 2] = {}; + bool anyActive = false; + for (Channel* ch : gFmodChannels) { + if (!ch || ch->owner != this) continue; + if (!ch->playing || ch->paused || ch->mute) continue; + if (!ch->mixHasData || !ch->sound || ch->sound->pcm.empty()) continue; + Sound* s = ch->sound; + const size_t total = ch->mixTotal; + if (total == 0) continue; + size_t cur = ch->mixCursor; + const float vol = ch->volume; + if (vol <= 0.0f) { + // Still advance cursor so silent channels finish. + size_t adv = kFmodMixFrames; + if (!ch->mixLoop) { + if (cur + adv >= total) { cur = total; ch->playing = false; } + else cur += adv; + } else { + cur = (cur + adv) % (total ? total : 1); + } + ch->mixCursor = cur; + ch->position = static_cast(cur); + continue; + } + anyActive = true; + const float* pcm = s->pcm.data(); + size_t pos = cur; + for (int i = 0; i < kFmodMixFrames; i++) { + if (pos >= total) { + if (ch->mixLoop || s->loopCount != 0 || + (s->mode & FMOD_LOOP_NORMAL) != 0) { + pos = 0; + } else { + break; + } + } + mix[i * 2] += pcm[pos * 2] * vol; + mix[i * 2 + 1] += pcm[pos * 2 + 1] * vol; + pos++; + } + // Advance cursor by frames consumed (approx, ignoring loop wrap details above). + size_t adv = static_cast(kFmodMixFrames); + if (!ch->mixLoop && s->loopCount == 0 && (s->mode & FMOD_LOOP_NORMAL) == 0) { + if (cur + adv >= total) { + cur = total; + ch->playing = false; + } else { + cur += adv; + } + } else { + cur = total ? ((cur + adv) % total) : 0; + } + ch->mixCursor = cur; + ch->position = static_cast(cur); + } + if (!anyActive) { + // Still queue silence rarely to keep device alive? No — skip to + // avoid filling queue with silence when nothing plays. + return FMOD_OK; + } + for (int i = 0; i < kFmodMixFrames * 2; i++) { + if (mix[i] > 1.0f) mix[i] = 1.0f; + else if (mix[i] < -1.0f) mix[i] = -1.0f; + } + // Convert to device format if needed. + if (gFmodSpec.format == AUDIO_F32SYS || gFmodSpec.format == 0) { + if (SDL_QueueAudio(gFmodDevice, mix, sizeof(mix)) < 0) { + APS5_LOG_ERR("%s", "FMOD: SDL_QueueAudio failed"); + } + } else { + int16_t tmp[kFmodMixFrames * 2]; + for (int i = 0; i < kFmodMixFrames * 2; i++) { + float v = mix[i] * 32767.0f; + if (v > 32767.0f) v = 32767.0f; + if (v < -32768.0f) v = -32768.0f; + tmp[i] = static_cast(v); + } + // Resample/format convert via SDL if spec differs (channels/rate). + // For minimal path, queue directly when spec is S16 stereo 48k; + // otherwise use SDL_ConvertAudio. + if (gFmodSpec.channels == 2 && gFmodSpec.freq == kFmodOutRate && + (gFmodSpec.format == AUDIO_S16SYS)) { + SDL_QueueAudio(gFmodDevice, tmp, sizeof(tmp)); + } else { + SDL_AudioCVT cvt{}; + if (SDL_BuildAudioCVT(&cvt, AUDIO_F32SYS, 2, kFmodOutRate, + gFmodSpec.format, gFmodSpec.channels, gFmodSpec.freq) >= 0) { + std::vector buf(sizeof(mix) * 4u); + std::memcpy(buf.data(), mix, sizeof(mix)); + cvt.buf = buf.data(); + cvt.len = static_cast(sizeof(mix)); + if (SDL_ConvertAudio(&cvt) == 0) { + SDL_QueueAudio(gFmodDevice, cvt.buf, static_cast(cvt.len_cvt)); + } + } + } + } + return FMOD_OK; + } int APS5_VABI setOutput(FMOD_OUTPUTTYPE output) { std::lock_guard lock(gFmodMutex); this->output = output; + if (output == FMOD_OUTPUTTYPE_NOSOUND || output == FMOD_OUTPUTTYPE_NOSOUND_NRT) { + gFmodSilent = true; + } return FMOD_OK; } int APS5_VABI getOutput(FMOD_OUTPUTTYPE* output) { @@ -1092,7 +1432,7 @@ public: (void)id; std::lock_guard lock(gFmodMutex); if (name && namelen > 0) { - const char* driverName = "AnyPS5 Null Output"; + const char* driverName = (gFmodDevice != 0 && !gFmodSilent) ? "AnyPS5 FMOD Output" : "AnyPS5 Null Output"; std::strncpy(name, driverName, static_cast(namelen - 1)); name[namelen - 1] = '\0'; } @@ -1152,8 +1492,13 @@ public: if (settings) std::memset(settings, 0, sizeof(*settings)); return FMOD_OK; } - int APS5_VABI setCallback(FMOD_SYSTEM_CALLBACK callback, FMOD_SYSTEM_CALLBACK_TYPE callbackmask, void* userdata) { - (void)callback; (void)callbackmask; (void)userdata; + int APS5_VABI setCallback(FMOD_SYSTEM_CALLBACK callback, FMOD_SYSTEM_CALLBACK_TYPE callbackmask) { + (void)callback; (void)callbackmask; + return FMOD_OK; + } + int APS5_VABI setFileSystem(void* useropen, void* userclose, void* userread, void* userseek, void* userasyncread, void* userasynccancel, int blockalign) { + (void)useropen; (void)userclose; (void)userread; (void)userseek; + (void)userasyncread; (void)userasynccancel; (void)blockalign; return FMOD_OK; } int APS5_VABI setPluginPath(const char* pluginpath) { @@ -1178,8 +1523,18 @@ public: } int APS5_VABI getChannelsPlaying(int* channels, int* realchannels) { std::lock_guard lock(gFmodMutex); - if (channels) *channels = 0; - if (realchannels) *realchannels = 0; + int active = 0; + for (Channel* ch : gFmodChannels) { + if (!ch || ch->owner != this) continue; + if (!ch->playing || ch->paused) continue; + if (ch->mixHasData) { + if (ch->mixLoop || ch->mixCursor < ch->mixTotal) active++; + } + // Null/empty sounds (smoke test) are not counted to preserve old + // behaviour (previously always returned 0). + } + if (channels) *channels = active; + if (realchannels) *realchannels = active; return FMOD_OK; } int APS5_VABI getCPUUsage(FMOD_CPU_USAGE* usage) { @@ -1210,9 +1565,93 @@ public: std::lock_guard lock(gFmodMutex); if (!sound) return FMOD_OK; Sound* created = new Sound(); - if (name_or_data) created->name = name_or_data; created->mode = mode; - if (exinfo) created->length = exinfo->length; + unsigned exLen = exinfo ? exinfo->length : 0; + if (exLen) created->length = exLen; + const bool isMem = (mode & (FMOD_OPENMEMORY_FLAG | FMOD_OPENMEMORY_POINT_FLAG)) != 0; + const bool isRaw = (mode & FMOD_OPENRAW_FLAG) != 0; + if (isMem && name_or_data && exLen > 0 && exLen < (1u << 30)) { + const uint8_t* buf = reinterpret_cast(name_or_data); + // Keep a copy of the name only for file-based sounds; for memory + // buffers the pointer is binary data (not a C string). + created->name.clear(); + std::vector pcm; + int rate = kFmodOutRate, ch = 2; + bool decoded = false; + if (!isRaw) { + decoded = FmodParseWav(buf, exLen, pcm, rate, ch); + } + if (!decoded && isRaw) { + // OPENRAW: interpret buffer using exinfo (default stereo 48k PCM16). + int rawCh = exinfo ? exinfo->numchannels : 0; + int rawRate = exinfo ? exinfo->defaultfrequency : 0; + int rawFmt = exinfo ? static_cast(exinfo->format) : 2; + if (rawCh <= 0 || rawCh > 8) rawCh = 2; + if (rawRate <= 0) rawRate = kFmodOutRate; + size_t bytesPerSamp = 2; + if (rawFmt == 1) bytesPerSamp = 1; // PCM8 + else if (rawFmt == 2) bytesPerSamp = 2; // PCM16 + else if (rawFmt == 5) bytesPerSamp = 4; // FLOAT + else bytesPerSamp = 2; + size_t frames = exLen / (static_cast(rawCh) * bytesPerSamp); + if (frames > 0 && frames < 48000u * 60u) { + pcm.reserve(frames * 2); + for (size_t f = 0; f < frames; f++) { + float l = 0, r = 0; + for (int c = 0; c < rawCh; c++) { + const uint8_t* p = buf + (f * rawCh + c) * bytesPerSamp; + float s = 0; + if (rawFmt == 1) s = (static_cast(p[0]) - 128) / 128.0f; + else if (rawFmt == 2) { int16_t v; std::memcpy(&v, p, 2); s = v / 32768.0f; } + else if (rawFmt == 5) { float v; std::memcpy(&v, p, 4); s = v; } + if (c == 0) l = s; else if (c == 1) r = s; else { l += s; r += s; } + } + if (rawCh == 1) r = l; + pcm.push_back(l); pcm.push_back(r); + } + // Resample if needed. + if (rawRate != kFmodOutRate && !pcm.empty()) { + size_t inF = pcm.size() / 2; + size_t outF = static_cast((static_cast(inF) * kFmodOutRate) / rawRate); + std::vector rs; rs.reserve(outF * 2); + for (size_t i = 0; i < outF; i++) { + double pos = (static_cast(i) * rawRate) / kFmodOutRate; + size_t i0 = static_cast(pos); + size_t i1 = i0 + 1 < inF ? i0 + 1 : i0; + float t = static_cast(pos - i0); + rs.push_back(pcm[i0 * 2] * (1 - t) + pcm[i1 * 2] * t); + rs.push_back(pcm[i0 * 2 + 1] * (1 - t) + pcm[i1 * 2 + 1] * t); + } + pcm.swap(rs); + } + decoded = !pcm.empty(); + rate = kFmodOutRate; ch = 2; + } + } + if (decoded) { + created->pcm.swap(pcm); + created->pcmRate = rate; + created->pcmCh = ch; + } else { + // Compressed (Vorbis/MP3/FSB/AT9...) or unknown: audible + // placeholder so the mixer path can be heard. Real decoders + // (Vorbis/MP3/AT9) remain future work — see docs/TechnicalDebt. + FmodMakeSine(created->pcm, kFmodOutRate, 2, 440.0f, 1.0f); + created->pcmRate = kFmodOutRate; + created->pcmCh = 2; + } + } else { + if (name_or_data) { + // File-based: store name only (first bytes may still be checked + // for embedded RIFF in case game passes a path that is data). + size_t nlen = std::strlen(name_or_data); + if (nlen < 256) created->name = name_or_data; + } + // Placeholder 0.5s sine so file-based playSound is audible. + FmodMakeSine(created->pcm, kFmodOutRate, 2, 330.0f, 0.5f); + created->pcmRate = kFmodOutRate; + created->pcmCh = 2; + } *sound = created; return FMOD_OK; } @@ -1241,6 +1680,19 @@ public: created->group = channelgroup ? channelgroup : masterChannelGroup; created->paused = paused; created->playing = true; + created->owner = this; + created->mixCursor = 0; + if (sound && !sound->pcm.empty()) { + created->mixHasData = true; + created->mixTotal = sound->pcm.size() / 2; + const bool loopMode = (sound->mode & FMOD_LOOP_NORMAL) != 0; + created->mixLoop = loopMode || sound->loopCount != 0; + } else { + created->mixHasData = false; + created->mixTotal = 0; + created->mixLoop = false; + } + gFmodChannels.push_back(created); *channel = created; return FMOD_OK; } @@ -1252,6 +1704,9 @@ public: created->group = channelgroup ? channelgroup : masterChannelGroup; created->paused = paused; created->playing = true; + created->owner = this; + created->mixHasData = false; + gFmodChannels.push_back(created); *channel = created; return FMOD_OK; } @@ -1424,12 +1879,24 @@ int APS5_VABI FMOD_Memory_GetStats(int* currentalloced, int* maxalloced, int blo namespace { -__attribute__((used)) int fmod_nid_stub_00() { APS5_LOG_ERR("%s", "VS1Vg5yOLH0"); return 0; } -__attribute__((used)) int fmod_nid_stub_01() { APS5_LOG_ERR("%s", "WAU72lnwYic"); return 0; } +// Round2 NID identification via real FMOD headers (const-correct manglings). +// VS1Vg5yOLH0 = FMOD::System::setFileSystem(open,close,read,seek,asyncRead,asyncCancel,blockalign) +// WAU72lnwYic = FMOD::System::setCallback(callback,mask) +// Both are ABI-compatible with void*/int (func ptrs are 8B, mask 4B); we keep +// the silent backend (return OK) — file callbacks are not needed for +// OPENMEMORY sounds used by Legends, and callbacks are stored nowhere. +__attribute__((used)) int APS5_VABI fmod_system_setfilesystem(void* self, void* a1, void* a2, void* a3, void* a4, void* a5, void* a6, int a7) { + (void)self; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; (void)a6; (void)a7; + return 0; +} +__attribute__((used)) int APS5_VABI fmod_system_setcallback(void* self, void* cb, unsigned mask) { + (void)self; (void)cb; (void)mask; + return 0; +} } -APS5_EXPORT("VS1Vg5yOLH0", fmod_nid_stub_00); -APS5_EXPORT("WAU72lnwYic", fmod_nid_stub_01); +APS5_EXPORT("VS1Vg5yOLH0", fmod_system_setfilesystem); +APS5_EXPORT("WAU72lnwYic", fmod_system_setcallback); } diff --git a/core/libs/prx/libfmodstudio/Export.cpp b/core/libs/prx/libfmodstudio/Export.cpp index c676d3746a..61fb3eeb40 100644 --- a/core/libs/prx/libfmodstudio/Export.cpp +++ b/core/libs/prx/libfmodstudio/Export.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -813,58 +814,196 @@ extern "C" { namespace { -__attribute__((used)) int fmodstudio_nid_stub_00() { APS5_LOG_ERR("%s", "+V2QcIcvWpw"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_01() { APS5_LOG_ERR("%s", "2QOSvz5hvAg"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_02() { APS5_LOG_ERR("%s", "3loRRCoqXfU"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_03() { APS5_LOG_ERR("%s", "6E6MUTGjxhI"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_04() { APS5_LOG_ERR("%s", "6nWYF1yMOl4"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_05() { APS5_LOG_ERR("%s", "7hd4bRJuLMg"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_06() { APS5_LOG_ERR("%s", "-7VafNNns2A"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_07() { APS5_LOG_ERR("%s", "AdR01fo-WaM"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_08() { APS5_LOG_ERR("%s", "alleMuKEWr8"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_09() { APS5_LOG_ERR("%s", "Bu8uFP7ME2k"); return 0; } +// Round2: 20/25 studio stubs identified via real FMOD headers (const-correct +// Itanium manglings, verified with g++ + c++filt + NidCompute SHA1+suffix). +// Wrappers below are ABI-compatible (this+params, SysV) with the real const +// methods; most delegate to the existing non-const logic (const is +// compile-time only, same calling convention). 5 remain unknown (kept as +// logging stubs): Cm2cmtCv8cA, EfM9xXvBmxk, Js90KQXVS4s, oCYWES02VPc, vVwA2cZA5e4. + +// +V2QcIcvWpw = EventDescription::createInstance(EventInstance**) const +__attribute__((used)) int APS5_VABI fmodstudio_real_createInstance(const void* self, void** out) { + std::lock_guard lock(gStudioMutex); + if (!out) return 0; + auto* desc = const_cast(static_cast(self)); + auto* inst = new FMOD::Studio::EventInstance(); + inst->description = desc; + *out = inst; + return 0; +} +// 2QOSvz5hvAg = EventInstance::getPlaybackState(PLAYBACK_STATE*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_getPlaybackState(const void* self, int* state) { + std::lock_guard lock(gStudioMutex); + const auto* inst = static_cast(self); + if (state) *state = (inst && inst->playing) ? 0 : 2; // PLAYING=0, STOPPED=2 + return 0; +} +// 3loRRCoqXfU = EventInstance::isValid() const +__attribute__((used)) int APS5_VABI fmodstudio_real_ei_isValid(const void* self) { + (void)self; + return 1; +} +// 6E6MUTGjxhI = System::getEvent(const char*, EventDescription**) const +__attribute__((used)) int APS5_VABI fmodstudio_real_getEvent(const void* self, const char* path, void** out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (!out) return 0; + auto* ed = new FMOD::Studio::EventDescription(); + if (path) ed->path = path; + *out = ed; + return 0; +} +// 6nWYF1yMOl4 = Bank::getLoadingState(LOADING_STATE*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_bankLoading(const void* self, int* state) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (state) *state = 3; // LOADED + return 0; +} +// 7hd4bRJuLMg = System::getCoreSystem(CoreSystem**) const (placeholder 256B zeroed) +__attribute__((used)) int APS5_VABI fmodstudio_real_getCoreSystem(const void* self, void** out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (!out) return 0; + void* placeholder = std::calloc(1, 256); + *out = placeholder; + return 0; +} +// -7VafNNns2A = EventInstance::getUserData(void**) const +__attribute__((used)) int APS5_VABI fmodstudio_real_ei_getUserData(const void* self, void** out) { + std::lock_guard lock(gStudioMutex); + const auto* inst = static_cast(self); + if (out) *out = inst ? inst->userData : nullptr; + return 0; +} +// AdR01fo-WaM = System::getParameterByName(const char*, float*, float*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_sysGetParam(const void* self, const char* name, float* v, float* fv) { + std::lock_guard lock(gStudioMutex); + const auto* sys = static_cast(self); + float stored = 0.0f; + if (sys && name) { + auto it = sys->paramsByName.find(name); + if (it != sys->paramsByName.end()) stored = it->second; + } + if (v) *v = stored; + if (fv) *fv = stored; + return 0; +} +// alleMuKEWr8 = EventDescription::isOneshot(bool*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_isOneshot(const void* self, bool* out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (out) *out = false; + return 0; +} +// Bu8uFP7ME2k = System::getVCA(const char*, VCA**) const +__attribute__((used)) int APS5_VABI fmodstudio_real_getVCA(const void* self, const char* path, void** out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (!out) return 0; + auto* vca = new FMOD::Studio::VCA(); + if (path) vca->path = path; + *out = vca; + return 0; +} +// DsFsPD0MWNc = System::getSoundInfo(const char*, SOUND_INFO*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_getSoundInfo(const void* self, const char* key, void* info) { + (void)self; (void)key; + std::lock_guard lock(gStudioMutex); + if (info) std::memset(info, 0, sizeof(FMOD_STUDIO_SOUND_INFO)); + return 0; +} +// --f9RJwAh1A = EventDescription::getSampleLoadingState(LOADING_STATE*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_edSampleState(const void* self, int* state) { + std::lock_guard lock(gStudioMutex); + const auto* ed = static_cast(self); + if (state) *state = (ed && ed->sampleDataLoaded) ? 3 : 1; // LOADED=3, UNLOADED=1 + return 0; +} +// fbAVXdyBuso = Bank::isValid() const +__attribute__((used)) int APS5_VABI fmodstudio_real_bankIsValid(const void* self) { + (void)self; + return 1; +} +// LG53EJZJDnA = EventInstance::setCallback(callback, mask) (non-const, 2 params) +__attribute__((used)) int APS5_VABI fmodstudio_real_eiSetCallback(void* self, void* cb, unsigned mask) { + (void)self; (void)cb; (void)mask; + return 0; +} +// nhBPjhZ+VWs = EventDescription::is3D(bool*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_is3D(const void* self, bool* out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (out) *out = false; + return 0; +} +// omDBr+dcDVc = EventDescription::getInstanceCount(int*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_getInstanceCount(const void* self, int* out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (out) *out = 0; + return 0; +} +// uWXqw4LeM5Q = EventDescription::isValid() const +__attribute__((used)) int APS5_VABI fmodstudio_real_edIsValid(const void* self) { + (void)self; + return 1; +} +// VdQjbIsdhXQ = EventDescription::getLength(int*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_edGetLength(const void* self, int* out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (out) *out = 0; + return 0; +} +// WP51b8simn8 = EventInstance::getTimelinePosition(int*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_getTimeline(const void* self, int* out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (out) *out = 0; + return 0; +} +// Zi1n4MvZ3-0 = System::getBufferUsage(BUFFER_USAGE*) const +__attribute__((used)) int APS5_VABI fmodstudio_real_getBufferUsage(const void* self, void* out) { + (void)self; + std::lock_guard lock(gStudioMutex); + if (out) std::memset(out, 0, sizeof(FMOD_STUDIO_BUFFER_USAGE)); + return 0; +} + +// Remaining 5 unknown (no public FMOD match found via exact manglings): __attribute__((used)) int fmodstudio_nid_stub_10() { APS5_LOG_ERR("%s", "Cm2cmtCv8cA"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_11() { APS5_LOG_ERR("%s", "DsFsPD0MWNc"); return 0; } __attribute__((used)) int fmodstudio_nid_stub_12() { APS5_LOG_ERR("%s", "EfM9xXvBmxk"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_13() { APS5_LOG_ERR("%s", "--f9RJwAh1A"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_14() { APS5_LOG_ERR("%s", "fbAVXdyBuso"); return 0; } __attribute__((used)) int fmodstudio_nid_stub_15() { APS5_LOG_ERR("%s", "Js90KQXVS4s"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_16() { APS5_LOG_ERR("%s", "LG53EJZJDnA"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_17() { APS5_LOG_ERR("%s", "nhBPjhZ+VWs"); return 0; } __attribute__((used)) int fmodstudio_nid_stub_18() { APS5_LOG_ERR("%s", "oCYWES02VPc"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_19() { APS5_LOG_ERR("%s", "omDBr+dcDVc"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_20() { APS5_LOG_ERR("%s", "uWXqw4LeM5Q"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_21() { APS5_LOG_ERR("%s", "VdQjbIsdhXQ"); return 0; } __attribute__((used)) int fmodstudio_nid_stub_22() { APS5_LOG_ERR("%s", "vVwA2cZA5e4"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_23() { APS5_LOG_ERR("%s", "WP51b8simn8"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_24() { APS5_LOG_ERR("%s", "Zi1n4MvZ3-0"); return 0; } } -APS5_EXPORT("+V2QcIcvWpw", fmodstudio_nid_stub_00); -APS5_EXPORT("2QOSvz5hvAg", fmodstudio_nid_stub_01); -APS5_EXPORT("3loRRCoqXfU", fmodstudio_nid_stub_02); -APS5_EXPORT("6E6MUTGjxhI", fmodstudio_nid_stub_03); -APS5_EXPORT("6nWYF1yMOl4", fmodstudio_nid_stub_04); -APS5_EXPORT("7hd4bRJuLMg", fmodstudio_nid_stub_05); -APS5_EXPORT("-7VafNNns2A", fmodstudio_nid_stub_06); -APS5_EXPORT("AdR01fo-WaM", fmodstudio_nid_stub_07); -APS5_EXPORT("alleMuKEWr8", fmodstudio_nid_stub_08); -APS5_EXPORT("Bu8uFP7ME2k", fmodstudio_nid_stub_09); +APS5_EXPORT("+V2QcIcvWpw", fmodstudio_real_createInstance); +APS5_EXPORT("2QOSvz5hvAg", fmodstudio_real_getPlaybackState); +APS5_EXPORT("3loRRCoqXfU", fmodstudio_real_ei_isValid); +APS5_EXPORT("6E6MUTGjxhI", fmodstudio_real_getEvent); +APS5_EXPORT("6nWYF1yMOl4", fmodstudio_real_bankLoading); +APS5_EXPORT("7hd4bRJuLMg", fmodstudio_real_getCoreSystem); +APS5_EXPORT("-7VafNNns2A", fmodstudio_real_ei_getUserData); +APS5_EXPORT("AdR01fo-WaM", fmodstudio_real_sysGetParam); +APS5_EXPORT("alleMuKEWr8", fmodstudio_real_isOneshot); +APS5_EXPORT("Bu8uFP7ME2k", fmodstudio_real_getVCA); APS5_EXPORT("Cm2cmtCv8cA", fmodstudio_nid_stub_10); -APS5_EXPORT("DsFsPD0MWNc", fmodstudio_nid_stub_11); +APS5_EXPORT("DsFsPD0MWNc", fmodstudio_real_getSoundInfo); APS5_EXPORT("EfM9xXvBmxk", fmodstudio_nid_stub_12); -APS5_EXPORT("--f9RJwAh1A", fmodstudio_nid_stub_13); -APS5_EXPORT("fbAVXdyBuso", fmodstudio_nid_stub_14); +APS5_EXPORT("--f9RJwAh1A", fmodstudio_real_edSampleState); +APS5_EXPORT("fbAVXdyBuso", fmodstudio_real_bankIsValid); APS5_EXPORT("Js90KQXVS4s", fmodstudio_nid_stub_15); -APS5_EXPORT("LG53EJZJDnA", fmodstudio_nid_stub_16); -APS5_EXPORT("nhBPjhZ+VWs", fmodstudio_nid_stub_17); +APS5_EXPORT("LG53EJZJDnA", fmodstudio_real_eiSetCallback); +APS5_EXPORT("nhBPjhZ+VWs", fmodstudio_real_is3D); APS5_EXPORT("oCYWES02VPc", fmodstudio_nid_stub_18); -APS5_EXPORT("omDBr+dcDVc", fmodstudio_nid_stub_19); -APS5_EXPORT("uWXqw4LeM5Q", fmodstudio_nid_stub_20); -APS5_EXPORT("VdQjbIsdhXQ", fmodstudio_nid_stub_21); +APS5_EXPORT("omDBr+dcDVc", fmodstudio_real_getInstanceCount); +APS5_EXPORT("uWXqw4LeM5Q", fmodstudio_real_edIsValid); +APS5_EXPORT("VdQjbIsdhXQ", fmodstudio_real_edGetLength); APS5_EXPORT("vVwA2cZA5e4", fmodstudio_nid_stub_22); -APS5_EXPORT("WP51b8simn8", fmodstudio_nid_stub_23); -APS5_EXPORT("Zi1n4MvZ3-0", fmodstudio_nid_stub_24); +APS5_EXPORT("WP51b8simn8", fmodstudio_real_getTimeline); +APS5_EXPORT("Zi1n4MvZ3-0", fmodstudio_real_getBufferUsage); } From a719359cd18dd2903e86396b882236296b0b2392 Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 15:49:18 +0200 Subject: [PATCH 50/93] prx: make libRenoirCore.PS5 stubs throw NotImplemented --- core/libs/prx/libRenoirCore.PS5/Export.cpp | 105 +++++++++------------ 1 file changed, 46 insertions(+), 59 deletions(-) diff --git a/core/libs/prx/libRenoirCore.PS5/Export.cpp b/core/libs/prx/libRenoirCore.PS5/Export.cpp index a502a85354..9130b50f37 100644 --- a/core/libs/prx/libRenoirCore.PS5/Export.cpp +++ b/core/libs/prx/libRenoirCore.PS5/Export.cpp @@ -1,158 +1,145 @@ #include -#include - #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" -// libRenoirCore.PS5 replacement (game-custom renderer lib, e.g. Minecraft Legends). -// -// The 23 NIDs below are imported by libcohtml.Prospero.prx (lib libRenoirCore.PS5_v1) -// and their clear names are unknown (obfuscated NIDs), so each export is exposed -// with its raw NID string via APS5_EXPORT (alias "_nid_no_patch_cut", which the -// nid_patcher strips to the literal NID instead of hashing it). -// -// Stub policy mirrors KytyPS5's UnresolvedImportStub (log the call, return 0) rather -// than the local NotImplemented_nid_no_patch helper (which throws): Renoir calls may -// happen per-frame and must not unwind through the guest. - extern "C" { APS5_EXPORT("0DsDTr534gc", renoirStub00); uint64_t APS5_VABI renoirStub00(void) { - APS5_LOG_OUT("stub NID=%s called", "0DsDTr534gc"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("2T56HgbPW08", renoirStub01); uint64_t APS5_VABI renoirStub01(void) { - APS5_LOG_OUT("stub NID=%s called", "2T56HgbPW08"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("3HnCozh8Az8", renoirStub02); uint64_t APS5_VABI renoirStub02(void) { - APS5_LOG_OUT("stub NID=%s called", "3HnCozh8Az8"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("5Ysce+s-CWU", renoirStub03); uint64_t APS5_VABI renoirStub03(void) { - APS5_LOG_OUT("stub NID=%s called", "5Ysce+s-CWU"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("AdlSqA4lGS8", renoirStub04); uint64_t APS5_VABI renoirStub04(void) { - APS5_LOG_OUT("stub NID=%s called", "AdlSqA4lGS8"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("bC37hk7rHqs", renoirStub05); uint64_t APS5_VABI renoirStub05(void) { - APS5_LOG_OUT("stub NID=%s called", "bC37hk7rHqs"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("dfPK9R6M0qM", renoirStub06); uint64_t APS5_VABI renoirStub06(void) { - APS5_LOG_OUT("stub NID=%s called", "dfPK9R6M0qM"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("fZiBbVa2ZDU", renoirStub07); uint64_t APS5_VABI renoirStub07(void) { - APS5_LOG_OUT("stub NID=%s called", "fZiBbVa2ZDU"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("hkWBH8-WfW4", renoirStub08); uint64_t APS5_VABI renoirStub08(void) { - APS5_LOG_OUT("stub NID=%s called", "hkWBH8-WfW4"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("hUxvGdi64Io", renoirStub09); uint64_t APS5_VABI renoirStub09(void) { - APS5_LOG_OUT("stub NID=%s called", "hUxvGdi64Io"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("NxqsZxGisrA", renoirStub10); uint64_t APS5_VABI renoirStub10(void) { - APS5_LOG_OUT("stub NID=%s called", "NxqsZxGisrA"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("OgZVIkaNmb0", renoirStub11); uint64_t APS5_VABI renoirStub11(void) { - APS5_LOG_OUT("stub NID=%s called", "OgZVIkaNmb0"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("ojLQdRUnVgA", renoirStub12); uint64_t APS5_VABI renoirStub12(void) { - APS5_LOG_OUT("stub NID=%s called", "ojLQdRUnVgA"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("s0qDfgsAgVg", renoirStub13); uint64_t APS5_VABI renoirStub13(void) { - APS5_LOG_OUT("stub NID=%s called", "s0qDfgsAgVg"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("sgrd-r1Nw60", renoirStub14); uint64_t APS5_VABI renoirStub14(void) { - APS5_LOG_OUT("stub NID=%s called", "sgrd-r1Nw60"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("sjTRb8YD7dQ", renoirStub15); uint64_t APS5_VABI renoirStub15(void) { - APS5_LOG_OUT("stub NID=%s called", "sjTRb8YD7dQ"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("tG86XGc2VtE", renoirStub16); uint64_t APS5_VABI renoirStub16(void) { - APS5_LOG_OUT("stub NID=%s called", "tG86XGc2VtE"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("umlGxOcNzgc", renoirStub17); uint64_t APS5_VABI renoirStub17(void) { - APS5_LOG_OUT("stub NID=%s called", "umlGxOcNzgc"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("vPxA9OFW0os", renoirStub18); uint64_t APS5_VABI renoirStub18(void) { - APS5_LOG_OUT("stub NID=%s called", "vPxA9OFW0os"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("X2qEBlOLkNo", renoirStub19); uint64_t APS5_VABI renoirStub19(void) { - APS5_LOG_OUT("stub NID=%s called", "X2qEBlOLkNo"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("ynh4E+BIaqY", renoirStub20); uint64_t APS5_VABI renoirStub20(void) { - APS5_LOG_OUT("stub NID=%s called", "ynh4E+BIaqY"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("YXe6ABbHZPs", renoirStub21); uint64_t APS5_VABI renoirStub21(void) { - APS5_LOG_OUT("stub NID=%s called", "YXe6ABbHZPs"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } APS5_EXPORT("z0jsXZVz75I", renoirStub22); uint64_t APS5_VABI renoirStub22(void) { - APS5_LOG_OUT("stub NID=%s called", "z0jsXZVz75I"); - return 0; + NotImplemented_nid_no_patch(__func__); + return 0; } } From b6dea0b51b268d084adb06986cbe41e9214be725 Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 15:53:27 +0200 Subject: [PATCH 51/93] Renoir stubs throw with NID name for progress monitoring --- core/libs/prx/libRenoirCore.PS5/Export.cpp | 46 +++++++++++----------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/core/libs/prx/libRenoirCore.PS5/Export.cpp b/core/libs/prx/libRenoirCore.PS5/Export.cpp index 9130b50f37..c0cf955a38 100644 --- a/core/libs/prx/libRenoirCore.PS5/Export.cpp +++ b/core/libs/prx/libRenoirCore.PS5/Export.cpp @@ -6,139 +6,139 @@ extern "C" { APS5_EXPORT("0DsDTr534gc", renoirStub00); uint64_t APS5_VABI renoirStub00(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("0DsDTr534gc"); return 0; } APS5_EXPORT("2T56HgbPW08", renoirStub01); uint64_t APS5_VABI renoirStub01(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("2T56HgbPW08"); return 0; } APS5_EXPORT("3HnCozh8Az8", renoirStub02); uint64_t APS5_VABI renoirStub02(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("3HnCozh8Az8"); return 0; } APS5_EXPORT("5Ysce+s-CWU", renoirStub03); uint64_t APS5_VABI renoirStub03(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("5Ysce+s-CWU"); return 0; } APS5_EXPORT("AdlSqA4lGS8", renoirStub04); uint64_t APS5_VABI renoirStub04(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("AdlSqA4lGS8"); return 0; } APS5_EXPORT("bC37hk7rHqs", renoirStub05); uint64_t APS5_VABI renoirStub05(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("bC37hk7rHqs"); return 0; } APS5_EXPORT("dfPK9R6M0qM", renoirStub06); uint64_t APS5_VABI renoirStub06(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("dfPK9R6M0qM"); return 0; } APS5_EXPORT("fZiBbVa2ZDU", renoirStub07); uint64_t APS5_VABI renoirStub07(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("fZiBbVa2ZDU"); return 0; } APS5_EXPORT("hkWBH8-WfW4", renoirStub08); uint64_t APS5_VABI renoirStub08(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("hkWBH8-WfW4"); return 0; } APS5_EXPORT("hUxvGdi64Io", renoirStub09); uint64_t APS5_VABI renoirStub09(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("hUxvGdi64Io"); return 0; } APS5_EXPORT("NxqsZxGisrA", renoirStub10); uint64_t APS5_VABI renoirStub10(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("NxqsZxGisrA"); return 0; } APS5_EXPORT("OgZVIkaNmb0", renoirStub11); uint64_t APS5_VABI renoirStub11(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("OgZVIkaNmb0"); return 0; } APS5_EXPORT("ojLQdRUnVgA", renoirStub12); uint64_t APS5_VABI renoirStub12(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("ojLQdRUnVgA"); return 0; } APS5_EXPORT("s0qDfgsAgVg", renoirStub13); uint64_t APS5_VABI renoirStub13(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("s0qDfgsAgVg"); return 0; } APS5_EXPORT("sgrd-r1Nw60", renoirStub14); uint64_t APS5_VABI renoirStub14(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("sgrd-r1Nw60"); return 0; } APS5_EXPORT("sjTRb8YD7dQ", renoirStub15); uint64_t APS5_VABI renoirStub15(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("sjTRb8YD7dQ"); return 0; } APS5_EXPORT("tG86XGc2VtE", renoirStub16); uint64_t APS5_VABI renoirStub16(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("tG86XGc2VtE"); return 0; } APS5_EXPORT("umlGxOcNzgc", renoirStub17); uint64_t APS5_VABI renoirStub17(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("umlGxOcNzgc"); return 0; } APS5_EXPORT("vPxA9OFW0os", renoirStub18); uint64_t APS5_VABI renoirStub18(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("vPxA9OFW0os"); return 0; } APS5_EXPORT("X2qEBlOLkNo", renoirStub19); uint64_t APS5_VABI renoirStub19(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("X2qEBlOLkNo"); return 0; } APS5_EXPORT("ynh4E+BIaqY", renoirStub20); uint64_t APS5_VABI renoirStub20(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("ynh4E+BIaqY"); return 0; } APS5_EXPORT("YXe6ABbHZPs", renoirStub21); uint64_t APS5_VABI renoirStub21(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("YXe6ABbHZPs"); return 0; } APS5_EXPORT("z0jsXZVz75I", renoirStub22); uint64_t APS5_VABI renoirStub22(void) { - NotImplemented_nid_no_patch(__func__); + NotImplemented_nid_no_patch("z0jsXZVz75I"); return 0; } From 7cdb23eacdfe3b5d47b135fc7cfa48aa2c5238a9 Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 16:06:18 +0200 Subject: [PATCH 52/93] fix(libcohtml): throw via NotImplemented_nid_no_patch for 50 stubs --- core/libs/prx/libcohtml/Export.cpp | 101 ++++++++++++++--------------- 1 file changed, 50 insertions(+), 51 deletions(-) diff --git a/core/libs/prx/libcohtml/Export.cpp b/core/libs/prx/libcohtml/Export.cpp index fae3870a5f..4ae1996b45 100644 --- a/core/libs/prx/libcohtml/Export.cpp +++ b/core/libs/prx/libcohtml/Export.cpp @@ -1,5 +1,4 @@ #include -#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" @@ -7,301 +6,301 @@ extern "C" { APS5_EXPORT("1fOA53oGwU8", cohtml_stub_00); std::uint64_t APS5_VABI cohtml_stub_00(void) { - std::printf("[libcohtml] 1fOA53oGwU8 called\n"); + NotImplemented_nid_no_patch("1fOA53oGwU8"); return 0; } APS5_EXPORT("2vxXHkZlhbw", cohtml_stub_01); std::uint64_t APS5_VABI cohtml_stub_01(void) { - std::printf("[libcohtml] 2vxXHkZlhbw called\n"); + NotImplemented_nid_no_patch("2vxXHkZlhbw"); return 0; } APS5_EXPORT("3rew70M0SmY", cohtml_stub_02); std::uint64_t APS5_VABI cohtml_stub_02(void) { - std::printf("[libcohtml] 3rew70M0SmY called\n"); + NotImplemented_nid_no_patch("3rew70M0SmY"); return 0; } APS5_EXPORT("4Le0SBPvnF8", cohtml_stub_03); std::uint64_t APS5_VABI cohtml_stub_03(void) { - std::printf("[libcohtml] 4Le0SBPvnF8 called\n"); + NotImplemented_nid_no_patch("4Le0SBPvnF8"); return 0; } APS5_EXPORT("53ZTBeo2a54", cohtml_stub_04); std::uint64_t APS5_VABI cohtml_stub_04(void) { - std::printf("[libcohtml] 53ZTBeo2a54 called\n"); + NotImplemented_nid_no_patch("53ZTBeo2a54"); return 0; } APS5_EXPORT("6PaNuQipllM", cohtml_stub_05); std::uint64_t APS5_VABI cohtml_stub_05(void) { - std::printf("[libcohtml] 6PaNuQipllM called\n"); + NotImplemented_nid_no_patch("6PaNuQipllM"); return 0; } APS5_EXPORT("7hn5DAs2bLk", cohtml_stub_06); std::uint64_t APS5_VABI cohtml_stub_06(void) { - std::printf("[libcohtml] 7hn5DAs2bLk called\n"); + NotImplemented_nid_no_patch("7hn5DAs2bLk"); return 0; } APS5_EXPORT("8EYWdAM2-R0", cohtml_stub_07); std::uint64_t APS5_VABI cohtml_stub_07(void) { - std::printf("[libcohtml] 8EYWdAM2-R0 called\n"); + NotImplemented_nid_no_patch("8EYWdAM2-R0"); return 0; } APS5_EXPORT("9bGvfp1zbKo", cohtml_stub_08); std::uint64_t APS5_VABI cohtml_stub_08(void) { - std::printf("[libcohtml] 9bGvfp1zbKo called\n"); + NotImplemented_nid_no_patch("9bGvfp1zbKo"); return 0; } APS5_EXPORT("a4E5dJh0onE", cohtml_stub_09); std::uint64_t APS5_VABI cohtml_stub_09(void) { - std::printf("[libcohtml] a4E5dJh0onE called\n"); + NotImplemented_nid_no_patch("a4E5dJh0onE"); return 0; } APS5_EXPORT("BhL-ZtSHmB8", cohtml_stub_10); std::uint64_t APS5_VABI cohtml_stub_10(void) { - std::printf("[libcohtml] BhL-ZtSHmB8 called\n"); + NotImplemented_nid_no_patch("BhL-ZtSHmB8"); return 0; } APS5_EXPORT("blvwzOvsEAA", cohtml_stub_11); std::uint64_t APS5_VABI cohtml_stub_11(void) { - std::printf("[libcohtml] blvwzOvsEAA called\n"); + NotImplemented_nid_no_patch("blvwzOvsEAA"); return 0; } APS5_EXPORT("dko-cMqv5yw", cohtml_stub_12); std::uint64_t APS5_VABI cohtml_stub_12(void) { - std::printf("[libcohtml] dko-cMqv5yw called\n"); + NotImplemented_nid_no_patch("dko-cMqv5yw"); return 0; } APS5_EXPORT("EG4kqkxLU3Q", cohtml_stub_13); std::uint64_t APS5_VABI cohtml_stub_13(void) { - std::printf("[libcohtml] EG4kqkxLU3Q called\n"); + NotImplemented_nid_no_patch("EG4kqkxLU3Q"); return 0; } APS5_EXPORT("-El4B0xJF2U", cohtml_stub_14); std::uint64_t APS5_VABI cohtml_stub_14(void) { - std::printf("[libcohtml] -El4B0xJF2U called\n"); + NotImplemented_nid_no_patch("-El4B0xJF2U"); return 0; } APS5_EXPORT("EMhQNylkSwk", cohtml_stub_15); std::uint64_t APS5_VABI cohtml_stub_15(void) { - std::printf("[libcohtml] EMhQNylkSwk called\n"); + NotImplemented_nid_no_patch("EMhQNylkSwk"); return 0; } APS5_EXPORT("-eNSKAypRz0", cohtml_stub_16); std::uint64_t APS5_VABI cohtml_stub_16(void) { - std::printf("[libcohtml] -eNSKAypRz0 called\n"); + NotImplemented_nid_no_patch("-eNSKAypRz0"); return 0; } APS5_EXPORT("FG9VRZfson0", cohtml_stub_17); std::uint64_t APS5_VABI cohtml_stub_17(void) { - std::printf("[libcohtml] FG9VRZfson0 called\n"); + NotImplemented_nid_no_patch("FG9VRZfson0"); return 0; } APS5_EXPORT("fNvtExaHN4U", cohtml_stub_18); std::uint64_t APS5_VABI cohtml_stub_18(void) { - std::printf("[libcohtml] fNvtExaHN4U called\n"); + NotImplemented_nid_no_patch("fNvtExaHN4U"); return 0; } APS5_EXPORT("g10DkPARoa8", cohtml_stub_19); std::uint64_t APS5_VABI cohtml_stub_19(void) { - std::printf("[libcohtml] g10DkPARoa8 called\n"); + NotImplemented_nid_no_patch("g10DkPARoa8"); return 0; } APS5_EXPORT("gwQczxoDmG0", cohtml_stub_20); std::uint64_t APS5_VABI cohtml_stub_20(void) { - std::printf("[libcohtml] gwQczxoDmG0 called\n"); + NotImplemented_nid_no_patch("gwQczxoDmG0"); return 0; } APS5_EXPORT("HQF4O9D+Di4", cohtml_stub_21); std::uint64_t APS5_VABI cohtml_stub_21(void) { - std::printf("[libcohtml] HQF4O9D+Di4 called\n"); + NotImplemented_nid_no_patch("HQF4O9D+Di4"); return 0; } APS5_EXPORT("igt3a6Q0VMc", cohtml_stub_22); std::uint64_t APS5_VABI cohtml_stub_22(void) { - std::printf("[libcohtml] igt3a6Q0VMc called\n"); + NotImplemented_nid_no_patch("igt3a6Q0VMc"); return 0; } APS5_EXPORT("kyApuhEHHxQ", cohtml_stub_23); std::uint64_t APS5_VABI cohtml_stub_23(void) { - std::printf("[libcohtml] kyApuhEHHxQ called\n"); + NotImplemented_nid_no_patch("kyApuhEHHxQ"); return 0; } APS5_EXPORT("Ljp75QXUi7s", cohtml_stub_24); std::uint64_t APS5_VABI cohtml_stub_24(void) { - std::printf("[libcohtml] Ljp75QXUi7s called\n"); + NotImplemented_nid_no_patch("Ljp75QXUi7s"); return 0; } APS5_EXPORT("loJ5Ef3MZrs", cohtml_stub_25); std::uint64_t APS5_VABI cohtml_stub_25(void) { - std::printf("[libcohtml] loJ5Ef3MZrs called\n"); + NotImplemented_nid_no_patch("loJ5Ef3MZrs"); return 0; } APS5_EXPORT("LWe+u8SHKgU", cohtml_stub_26); std::uint64_t APS5_VABI cohtml_stub_26(void) { - std::printf("[libcohtml] LWe+u8SHKgU called\n"); + NotImplemented_nid_no_patch("LWe+u8SHKgU"); return 0; } APS5_EXPORT("nhn1ds+I2t4", cohtml_stub_27); std::uint64_t APS5_VABI cohtml_stub_27(void) { - std::printf("[libcohtml] nhn1ds+I2t4 called\n"); + NotImplemented_nid_no_patch("nhn1ds+I2t4"); return 0; } APS5_EXPORT("NmVsRWgsKt4", cohtml_stub_28); std::uint64_t APS5_VABI cohtml_stub_28(void) { - std::printf("[libcohtml] NmVsRWgsKt4 called\n"); + NotImplemented_nid_no_patch("NmVsRWgsKt4"); return 0; } APS5_EXPORT("ONEnhMinebs", cohtml_stub_29); std::uint64_t APS5_VABI cohtml_stub_29(void) { - std::printf("[libcohtml] ONEnhMinebs called\n"); + NotImplemented_nid_no_patch("ONEnhMinebs"); return 0; } APS5_EXPORT("PdOmR8dy1fM", cohtml_stub_30); std::uint64_t APS5_VABI cohtml_stub_30(void) { - std::printf("[libcohtml] PdOmR8dy1fM called\n"); + NotImplemented_nid_no_patch("PdOmR8dy1fM"); return 0; } APS5_EXPORT("PSwGBN5Icv8", cohtml_stub_31); std::uint64_t APS5_VABI cohtml_stub_31(void) { - std::printf("[libcohtml] PSwGBN5Icv8 called\n"); + NotImplemented_nid_no_patch("PSwGBN5Icv8"); return 0; } APS5_EXPORT("RalyfwiTt8o", cohtml_stub_32); std::uint64_t APS5_VABI cohtml_stub_32(void) { - std::printf("[libcohtml] RalyfwiTt8o called\n"); + NotImplemented_nid_no_patch("RalyfwiTt8o"); return 0; } APS5_EXPORT("t+OHUjVWIwU", cohtml_stub_33); std::uint64_t APS5_VABI cohtml_stub_33(void) { - std::printf("[libcohtml] t+OHUjVWIwU called\n"); + NotImplemented_nid_no_patch("t+OHUjVWIwU"); return 0; } APS5_EXPORT("Ter8YCVZ9tQ", cohtml_stub_34); std::uint64_t APS5_VABI cohtml_stub_34(void) { - std::printf("[libcohtml] Ter8YCVZ9tQ called\n"); + NotImplemented_nid_no_patch("Ter8YCVZ9tQ"); return 0; } APS5_EXPORT("tG3LJfAOH3k", cohtml_stub_35); std::uint64_t APS5_VABI cohtml_stub_35(void) { - std::printf("[libcohtml] tG3LJfAOH3k called\n"); + NotImplemented_nid_no_patch("tG3LJfAOH3k"); return 0; } APS5_EXPORT("ukSFhKPmCTg", cohtml_stub_36); std::uint64_t APS5_VABI cohtml_stub_36(void) { - std::printf("[libcohtml] ukSFhKPmCTg called\n"); + NotImplemented_nid_no_patch("ukSFhKPmCTg"); return 0; } APS5_EXPORT("V7KGze2XjJc", cohtml_stub_37); std::uint64_t APS5_VABI cohtml_stub_37(void) { - std::printf("[libcohtml] V7KGze2XjJc called\n"); + NotImplemented_nid_no_patch("V7KGze2XjJc"); return 0; } APS5_EXPORT("vlmAvJqR7mQ", cohtml_stub_38); std::uint64_t APS5_VABI cohtml_stub_38(void) { - std::printf("[libcohtml] vlmAvJqR7mQ called\n"); + NotImplemented_nid_no_patch("vlmAvJqR7mQ"); return 0; } APS5_EXPORT("VzhxjXDzYSI", cohtml_stub_39); std::uint64_t APS5_VABI cohtml_stub_39(void) { - std::printf("[libcohtml] VzhxjXDzYSI called\n"); + NotImplemented_nid_no_patch("VzhxjXDzYSI"); return 0; } APS5_EXPORT("X3FzDPzpD2Y", cohtml_stub_40); std::uint64_t APS5_VABI cohtml_stub_40(void) { - std::printf("[libcohtml] X3FzDPzpD2Y called\n"); + NotImplemented_nid_no_patch("X3FzDPzpD2Y"); return 0; } APS5_EXPORT("x68HPh+vxsE", cohtml_stub_41); std::uint64_t APS5_VABI cohtml_stub_41(void) { - std::printf("[libcohtml] x68HPh+vxsE called\n"); + NotImplemented_nid_no_patch("x68HPh+vxsE"); return 0; } APS5_EXPORT("XUe8OAKB-vQ", cohtml_stub_42); std::uint64_t APS5_VABI cohtml_stub_42(void) { - std::printf("[libcohtml] XUe8OAKB-vQ called\n"); + NotImplemented_nid_no_patch("XUe8OAKB-vQ"); return 0; } APS5_EXPORT("y6UNQtnr0wY", cohtml_stub_43); std::uint64_t APS5_VABI cohtml_stub_43(void) { - std::printf("[libcohtml] y6UNQtnr0wY called\n"); + NotImplemented_nid_no_patch("y6UNQtnr0wY"); return 0; } APS5_EXPORT("YoHAXkBV06c", cohtml_stub_44); std::uint64_t APS5_VABI cohtml_stub_44(void) { - std::printf("[libcohtml] YoHAXkBV06c called\n"); + NotImplemented_nid_no_patch("YoHAXkBV06c"); return 0; } APS5_EXPORT("Z9XFJCfmId4", cohtml_stub_45); std::uint64_t APS5_VABI cohtml_stub_45(void) { - std::printf("[libcohtml] Z9XFJCfmId4 called\n"); + NotImplemented_nid_no_patch("Z9XFJCfmId4"); return 0; } APS5_EXPORT("zEniuuzG2W0", cohtml_stub_46); std::uint64_t APS5_VABI cohtml_stub_46(void) { - std::printf("[libcohtml] zEniuuzG2W0 called\n"); + NotImplemented_nid_no_patch("zEniuuzG2W0"); return 0; } APS5_EXPORT("zH9Z2nK1fmA", cohtml_stub_47); std::uint64_t APS5_VABI cohtml_stub_47(void) { - std::printf("[libcohtml] zH9Z2nK1fmA called\n"); + NotImplemented_nid_no_patch("zH9Z2nK1fmA"); return 0; } APS5_EXPORT("ZHGgdWeYTo8", cohtml_stub_48); std::uint64_t APS5_VABI cohtml_stub_48(void) { - std::printf("[libcohtml] ZHGgdWeYTo8 called\n"); + NotImplemented_nid_no_patch("ZHGgdWeYTo8"); return 0; } APS5_EXPORT("ZWftPxZ4E6k", cohtml_stub_49); std::uint64_t APS5_VABI cohtml_stub_49(void) { - std::printf("[libcohtml] ZWftPxZ4E6k called\n"); + NotImplemented_nid_no_patch("ZWftPxZ4E6k"); return 0; } From 5099184d4d08d7eda6b0703d36de0a8f2097bc60 Mon Sep 17 00:00:00 2001 From: Pouare514 Date: Sun, 27 Sep 2026 16:10:41 +0200 Subject: [PATCH 53/93] feat(prx): fmodstudio unknown NIDs throw via NotImplemented Convert 5 remaining pure stubs (log + return 0) to throwing with literal NID, keep trailing return 0. Real SDL/state-tracking impls (20 studio + 2 libfmod + C++ classes) untouched. Remove stale logging-stub comments only around touched stubs. Throwing: Cm2cmtCv8cA, EfM9xXvBmxk, Js90KQXVS4s, oCYWES02VPc, vVwA2cZA5e4 --- core/libs/prx/libfmodstudio/Export.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libfmodstudio/Export.cpp b/core/libs/prx/libfmodstudio/Export.cpp index 61fb3eeb40..8f7baefc38 100644 --- a/core/libs/prx/libfmodstudio/Export.cpp +++ b/core/libs/prx/libfmodstudio/Export.cpp @@ -818,8 +818,7 @@ namespace { // Itanium manglings, verified with g++ + c++filt + NidCompute SHA1+suffix). // Wrappers below are ABI-compatible (this+params, SysV) with the real const // methods; most delegate to the existing non-const logic (const is -// compile-time only, same calling convention). 5 remain unknown (kept as -// logging stubs): Cm2cmtCv8cA, EfM9xXvBmxk, Js90KQXVS4s, oCYWES02VPc, vVwA2cZA5e4. +// compile-time only, same calling convention). // +V2QcIcvWpw = EventDescription::createInstance(EventInstance**) const __attribute__((used)) int APS5_VABI fmodstudio_real_createInstance(const void* self, void** out) { @@ -971,12 +970,11 @@ __attribute__((used)) int APS5_VABI fmodstudio_real_getBufferUsage(const void* s return 0; } -// Remaining 5 unknown (no public FMOD match found via exact manglings): -__attribute__((used)) int fmodstudio_nid_stub_10() { APS5_LOG_ERR("%s", "Cm2cmtCv8cA"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_12() { APS5_LOG_ERR("%s", "EfM9xXvBmxk"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_15() { APS5_LOG_ERR("%s", "Js90KQXVS4s"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_18() { APS5_LOG_ERR("%s", "oCYWES02VPc"); return 0; } -__attribute__((used)) int fmodstudio_nid_stub_22() { APS5_LOG_ERR("%s", "vVwA2cZA5e4"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_10() { NotImplemented_nid_no_patch("Cm2cmtCv8cA"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_12() { NotImplemented_nid_no_patch("EfM9xXvBmxk"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_15() { NotImplemented_nid_no_patch("Js90KQXVS4s"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_18() { NotImplemented_nid_no_patch("oCYWES02VPc"); return 0; } +__attribute__((used)) int fmodstudio_nid_stub_22() { NotImplemented_nid_no_patch("vVwA2cZA5e4"); return 0; } } From 5fdcd571cd37a3996b6479e43a10eaecaa9b3295 Mon Sep 17 00:00:00 2001 From: Pacuka <69674426+Pacuka@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:59:46 +0200 Subject: [PATCH 54/93] fix(recompiler): widen constants of 64-bit operands instead of repeating them readU32Pair read a literal or inline constant into both halves of a 64-bit source. A literal is now zero-extended, an integer inline constant sign-extended and a float inline constant becomes the matching double. Co-Authored-By: Claude Opus 5.5 --- .../src/TranslationContext/SubwordPacking.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp index e1ba9818f5..d8b32b7e0b 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp @@ -1,5 +1,7 @@ #include "Translation/TranslationContext.hpp" #include +#include +#include namespace ShaderRecompiler { @@ -8,7 +10,15 @@ IrU32 TranslationContext::readU32(const RdnaOperand& operand) { } std::array TranslationContext::readU32Pair(const RdnaOperand& operand) { - return {readU32(operand), readU32(offsetOperand(operand, 1u))}; + switch (operand.kind) { + case RdnaOperandKind::LiteralConstant: return {readU32(operand), IrU32(ir.Constant(0u))}; + case RdnaOperandKind::IntegerInlineConstant: return {readU32(operand), IrU32(ir.Constant(operand.signedVal < 0 ? 0xffffffffu : 0u))}; + case RdnaOperandKind::FloatInlineConstant: { + const auto bits = std::bit_cast(static_cast(std::bit_cast(operand.value))); + return {IrU32(ir.Constant(static_cast(bits))), IrU32(ir.Constant(static_cast(bits >> 32u)))}; + } + default: return {readU32(operand), readU32(offsetOperand(operand, 1u))}; + } } IrU64 TranslationContext::readU64(const RdnaOperand& operand) { From 83f43d1004115d85bcca0940116fe208a1bb7169 Mon Sep 17 00:00:00 2001 From: Pacuka <69674426+Pacuka@users.noreply.github.com> Date: Sun, 27 Sep 2026 19:43:16 +0200 Subject: [PATCH 55/93] feat(agc): emit indexed indirect draws sceAgcDcbDrawIndexIndirect and sceAgcDcbDrawIndexIndirectMulti emit DRAW_INDEX_INDIRECT (0x25) and DRAW_INDEX_INDIRECT_MULTI (0x38) like their non-indexed counterparts, with 20-byte records. Co-Authored-By: Claude Opus 5.5 --- .../prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp | 29 ++++++++++--------- core/libs/prx/libSceAgc/tests/Commands.cpp | 21 ++++++++++++++ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/core/libs/prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp b/core/libs/prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp index 8c81f1e8f3..d8ccb70df6 100644 --- a/core/libs/prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp +++ b/core/libs/prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp @@ -40,11 +40,9 @@ uint32_t APS5_VABI sceAgcDcbDrawIndexOffsetGetSize(void) { } uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirect(CommandBuffer* buf, uint32_t data_offset_in_bytes, uint64_t modifier) { - (void)buf; - (void)data_offset_in_bytes; - (void)modifier; - NotImplemented_nid_no_patch(__func__); - return nullptr; + Agc::Command::Require((data_offset_in_bytes & 3u) == 0, __func__, "misaligned indirect argument offset"); + const auto offsets = Agc::Command::DrawPatchOffsets(modifier, __func__); + return Agc::Command::Emit(buf, 0x25u, {data_offset_in_bytes, static_cast(offsets), static_cast(offsets >> 32u), Agc::Command::DrawInitiator(modifier, true, __func__)}, __func__); } std::uint32_t APS5_VABI sceAgcDcbDrawIndexIndirectGetSize() { @@ -52,15 +50,18 @@ std::uint32_t APS5_VABI sceAgcDcbDrawIndexIndirectGetSize() { } uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirectMulti(CommandBuffer* buf, uint32_t data_offset_in_bytes, uint32_t count_indirect, uint32_t max_count_or_count, const volatile void* count_addr, uint32_t stride_in_bytes, uint64_t modifier) { - (void)buf; - (void)data_offset_in_bytes; - (void)count_indirect; - (void)max_count_or_count; - (void)count_addr; - (void)stride_in_bytes; - (void)modifier; - NotImplemented_nid_no_patch(__func__); - return nullptr; + Agc::Command::CheckBits(count_indirect, 1, __func__); + Agc::Command::Require((data_offset_in_bytes & 3u) == 0 && (stride_in_bytes & 3u) == 0 && stride_in_bytes >= 20, __func__, "invalid indirect draw offset or stride"); + const auto address = reinterpret_cast(count_addr); + if (count_indirect != 0) { + Agc::Command::CheckAddress(address, 4, __func__); + } else { + Agc::Command::Require(address == 0, __func__, "count address supplied for a direct draw count"); + } + const auto offsets = Agc::Command::DrawPatchOffsets(modifier, __func__); + const auto low = static_cast(modifier); + const auto control = Agc::Command::DrawIndexLocation(modifier) | ((low & 0x10u) << 23u) | (count_indirect << 30u) | ((low & 8u) << 28u); + return Agc::Command::Emit(buf, 0x38u, {data_offset_in_bytes, static_cast(offsets), static_cast(offsets >> 32u), control, max_count_or_count, static_cast(address), static_cast(address >> 32u), stride_in_bytes, Agc::Command::DrawInitiator(modifier, true, __func__)}, __func__); } std::uint32_t APS5_VABI sceAgcDcbDrawIndexIndirectMultiGetSize() { diff --git a/core/libs/prx/libSceAgc/tests/Commands.cpp b/core/libs/prx/libSceAgc/tests/Commands.cpp index 421ed8a0f4..d936c998b7 100644 --- a/core/libs/prx/libSceAgc/tests/Commands.cpp +++ b/core/libs/prx/libSceAgc/tests/Commands.cpp @@ -14,6 +14,8 @@ extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetFlip(CommandBuffer* buf, std::ui extern "C" int APS5_VABI sceAgcSuspendPoint(); extern "C" int APS5_VABI sceAgcInit(std::uint32_t* state, std::uint32_t version); extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexAuto(CommandBuffer* buf, std::uint32_t indexCount, std::uint64_t modifier); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirect(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint64_t modifier); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirectMulti(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint32_t countIndirect, std::uint32_t maxCountOrCount, const volatile void* countAddress, std::uint32_t strideInBytes, std::uint64_t modifier); extern "C" int APS5_VABI sceAgcWaitRegMemPatchReference(std::uint32_t* cmd, std::uint64_t reference); extern "C" int APS5_VABI sceAgcGetDataPacketPayloadAddressUnk(std::uint32_t** addr, std::uint32_t* cmd, int type); extern "C" std::uint32_t* APS5_VABI sceAgcCbSetShRegisterRangeDirect(CommandBuffer* buf, std::uint32_t offset, const std::uint32_t* values, std::uint32_t numValues); @@ -96,6 +98,24 @@ bool APS5_VABI growContext(CommandBuffer* buffer, std::uint32_t count, void* use return true; } +void testIndexedIndirectDraws() { + Storage storage; + const auto* single = sceAgcDcbDrawIndexIndirect(&storage.buffer, 0x40, 0); + const std::array expectedSingle{0xc0032500u, 0x40, 0x280, 0x280, 0}; + check(single == storage.words.data() && std::equal(expectedSingle.begin(), expectedSingle.end(), single), "indexed indirect draw packet mismatch"); + alignas(4) std::uint32_t count = 0; + const auto* multi = sceAgcDcbDrawIndexIndirectMulti(&storage.buffer, 0x80, 1, 8, &count, 20, 0); + const auto address = reinterpret_cast(&count); + const std::array expectedMulti{0xc0083800u, 0x80, 0x280, 0x280, 0x40000280u, 8, static_cast(address), static_cast(address >> 32u), 20, 0}; + check(multi == storage.words.data() + expectedSingle.size() && std::equal(expectedMulti.begin(), expectedMulti.end(), multi), "indexed indirect multi draw packet mismatch"); + check(storage.buffer.cursor_up == storage.words.data() + expectedSingle.size() + expectedMulti.size(), "incorrect indexed indirect cursor advance"); + const auto before = storage.words; + expectFailure([&] { sceAgcDcbDrawIndexIndirect(&storage.buffer, 2, 0); }); + expectFailure([&] { sceAgcDcbDrawIndexIndirectMulti(&storage.buffer, 0, 1, 8, &count, 16, 0); }); + expectFailure([&] { sceAgcDcbDrawIndexIndirectMulti(&storage.buffer, 0, 0, 8, &count, 20, 0); }); + check(storage.words == before, "invalid indexed indirect draw modified packet memory"); +} + void testContextState() { const std::array, 4> sizes{{{5}, {5, 8, 9, 3, 2}, {3, 5, 8, 9, 2}, {5, 8, 9, 3, 2, 5}}}; const std::array, 4> reservations{{{5}, {22, 3, 2}, {3, 22, 2}, {22, 3, 2, 5}}}; @@ -272,6 +292,7 @@ void testDefaults() { int main() { try { testPackets(); + testIndexedIndirectDraws(); testContextState(); testFlip(); testRegisters(); From 1e4430e917773eb090aa9ab37501c2ae61fe9264 Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:53:11 -0700 Subject: [PATCH 56/93] fix(agc): drop cached textures on Vulkan device replacement When the presentation device was replaced, textures created on the old device stayed in the global sampled and storage texture caches and kept a reference to the destroyed TextureDetiler, whose VkDevice was then read corrupted ("missing Vulkan function: vkResetDescriptorPool" at startup). The replacement now flushes the old device's cached storage textures and publishes every unit shadow before waiting for the device, and the device teardown erases the cache entries that belong to it. Patch author: boykopovar (fix-vulkan-texture-lifetime.patch, PR #5 review). Co-Authored-By: boykopovar Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- .../Execution/include/VulkanDevice.hpp | 1 + .../libSceAgcDriver/Execution/src/Driver.cpp | 2 +- .../Execution/src/VulkanDevice.cpp | 9 +++++ .../Graphics/include/ShaderResources.hpp | 3 ++ .../Graphics/src/ShaderResources.cpp | 39 +++++++++++++++++-- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index c94e666bd0..c94e40eb53 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -42,6 +42,7 @@ public: // resize, device replacement, CPU fill fallback and APS5_DRAIN_ALL; the packet-loop drains use the // three-step form below so the GPU wait happens without the mutex. void WaitIdle(); + void PrepareForReplacement(); // Sends recorded work to the GPU without waiting for it. With `reapFirst` it first retires batches // that already finished, so the in-flight list stays short (APS5_NO_OPPORTUNISTIC_REAP=1 skips // that). A reap runs completion actions, and a write-back can wait for a later batch under the diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index 84699adb22..14a2f4614e 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -651,7 +651,7 @@ public: std::lock_guard lock(GuestMemory::GpuMutex()); timing.Mark("gpu_mutex_wait"); if (device == nullptr || device->Window() == nullptr) { - if (device) device->WaitIdle(); + if (device) device->PrepareForReplacement(); device = std::make_shared(&window); } require(device->Window() == window.context, "presentation window does not match device surface"); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 500b035946..ddd49fc0a3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -426,6 +426,7 @@ struct VulkanDevice::State { // Every ShaderResources (kept by the recorder or the resource cache) is gone now, so the // sets and samplers they borrowed can go. resourceCache.Clear(); + Graphics::ClearCachedTextures(device); descriptorCache.reset(); samplerCache.reset(); textureCache.reset(); @@ -873,6 +874,14 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u VulkanDevice::~VulkanDevice() = default; +void VulkanDevice::PrepareForReplacement() { + GuestMemory::AssertGpuLockHeld("VulkanDevice::PrepareForReplacement"); + require(state->recorder != nullptr && Graphics::Recorder::Active() == state->recorder.get(), "device replacement requires its active recorder"); + Graphics::FlushCachedTextures(state->device); + Graphics::PublishAllShadows(state->context, Graphics::PublishReason::Teardown); + WaitIdle(); +} + void VulkanDevice::WaitIdle() { APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle begin"); RetirePresents(0); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 2ac3fc2fd8..67b0511e06 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -21,6 +21,9 @@ namespace AgcDriver::Graphics { class Recorder; +void FlushCachedTextures(VkDevice device); +void ClearCachedTextures(VkDevice device); + // The cached storage image of a surface (render targets use it as their resident image); brought up // to date with guest memory before it is returned. std::shared_ptr CachedStorageSurface(const Context& context, const GuestTextureResource& resource); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index ec222e29af..f55e3ebf60 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -515,10 +515,41 @@ std::shared_ptr cachedStorageTexture(const Context& context, std } -// Whether `image` is still the storage cache's image of its surface, i.e. what a lookup would return -// (an evicted image next to a newer one of the same memory must not be reused: two images of one -// surface would hold results). A use through the fast Revalidate or a draw recipe's target proof -// counts as a use for the cache's eviction order, as a lookup would. +void FlushCachedTextures(VkDevice device) { + Require(device != VK_NULL_HANDLE, "cannot flush textures without a Vulkan device"); + GuestMemory::AssertGpuLockHeld("FlushCachedTextures"); + auto& cache = StorageTextures(); + std::lock_guard lock(cache.mutex); + for (const auto& entry : cache.entries) { + if (entry.key.device == device) entry.texture->Flush(); + } +} + +void ClearCachedTextures(VkDevice device) { + Require(device != VK_NULL_HANDLE, "cannot clear textures without a Vulkan device"); + auto& sampled = Textures(); + { + std::lock_guard lock(sampled.mutex); + for (auto it = sampled.entries.begin(); it != sampled.entries.end();) { + if (it->key.device == device) eraseTexture(sampled, it++); + else ++it; + } + } + auto& storage = StorageTextures(); + std::lock_guard lock(storage.mutex); + for (auto it = storage.entries.begin(); it != storage.entries.end();) { + if (it->key.device != device) { + ++it; + continue; + } + it->texture->SetCached(false); + storage.bytes -= it->texture->GuestBytes(); + storage.index.erase(it->key); + storage.byImage.erase(it->texture.get()); + it = storage.entries.erase(it); + } +} + bool StorageImageCached(const Context& context, const StorageTexture* image) { auto& cache = StorageTextures(); std::lock_guard lock(cache.mutex); From e764385d51304f9a04542d930b34a464f9e10fa8 Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:26:58 -0700 Subject: [PATCH 57/93] fix(libkernel): drop the scePthreadOnce stub that shadowed the implementation The merge brought a NotImplemented scePthreadOnce_nid_postfix next to the existing scePthreadOnce. The NID patcher gives the NID to the postfixed symbol, so the stub replaced the working export and the title aborted at its first once-initialisation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/libs/prx/libkernel/Pthread/Posix/Once.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/core/libs/prx/libkernel/Pthread/Posix/Once.cpp b/core/libs/prx/libkernel/Pthread/Posix/Once.cpp index 8702a17a05..32ed2ee791 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Once.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Once.cpp @@ -42,16 +42,3 @@ extern "C" int APS5_VABI pthread_once_nid_postfix( onceChanged.notify_all(); return 0; } - -using OnceRoutine = void (APS5_VABI *)(); - -extern "C" { - -int APS5_VABI scePthreadOnce_nid_postfix(void* onceControl, OnceRoutine initRoutine) { - (void)onceControl; - (void)initRoutine; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -} From 72336201b2d118174d37353973e24fb03a9b1e84 Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:26:59 -0700 Subject: [PATCH 58/93] perf(libkernel): resolve initialised mutexes and conditions without the global lock Every lock, unlock, signal and wait took the process-wide initialisation mutex to read the handle, serialising all guest synchronisation. The handle is now read atomically and the lock is only taken to initialise a static initializer or to report a destroyed object. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/libs/prx/libkernel/Pthread/src/Cond.cpp | 19 ++++++++---- core/libs/prx/libkernel/Pthread/src/Mutex.cpp | 30 ++++++++++++------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/core/libs/prx/libkernel/Pthread/src/Cond.cpp b/core/libs/prx/libkernel/Pthread/src/Cond.cpp index cfeec60be2..9eb2c9449a 100644 --- a/core/libs/prx/libkernel/Pthread/src/Cond.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Cond.cpp @@ -4,6 +4,7 @@ #include "prx/libkernel/Pthread/Posix/Common.hpp" #include "prx/libkernel/Time/include/Time.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" +#include #include #include #include @@ -23,12 +24,18 @@ PthreadCond destroyedCond() { PthreadCond resolveCond(PthreadCond* cond) { if (!cond) throw std::invalid_argument("Condition variable pointer is null"); + std::atomic_ref slot(*cond); + if (const auto current = slot.load(std::memory_order_acquire); current && current != destroyedCond()) + return current; std::lock_guard lock(condInitializationMutex); - if (*cond == destroyedCond()) + const auto current = slot.load(std::memory_order_acquire); + if (current == destroyedCond()) throw std::runtime_error("Condition variable has been destroyed"); - if (!*cond) - *cond = new PthreadCondPrivate(); - return *cond; + if (current) + return current; + auto* created = new PthreadCondPrivate(); + slot.store(created, std::memory_order_release); + return created; } PthreadMutex lockedMutex(PthreadMutex* mutex) { @@ -112,7 +119,7 @@ int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, if (attr) replacement->_clockid = (*attr)->_clockid; std::lock_guard lock(condInitializationMutex); - *cond = replacement.release(); + std::atomic_ref(*cond).store(replacement.release(), std::memory_order_release); return 0; } @@ -123,7 +130,7 @@ int APS5_VABI scePthreadCondDestroy(PthreadCond* cond) { if (*cond == destroyedCond()) throw std::runtime_error("Condition variable has already been destroyed"); delete *cond; - *cond = destroyedCond(); + std::atomic_ref(*cond).store(destroyedCond(), std::memory_order_release); return 0; } diff --git a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp index d37a9161b0..270c476556 100644 --- a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp @@ -1,6 +1,7 @@ #include "../include/Pthread.hpp" #include "../include/Mutex.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" +#include #include #include #include @@ -19,20 +20,29 @@ PthreadMutex destroyedMutex() { return reinterpret_cast(std::uintptr_t{2}); } +bool isInitializedMutex(PthreadMutex mutex) { + return mutex && mutex != destroyedMutex() && reinterpret_cast(mutex) != 1; +} + PthreadMutex resolveMutex(PthreadMutex* mutex, bool initialize) { if (!mutex) throw std::invalid_argument("Mutex pointer is null"); + std::atomic_ref slot(*mutex); + if (const auto current = slot.load(std::memory_order_acquire); isInitializedMutex(current)) + return current; std::lock_guard lock(initializationMutex); - if (*mutex == destroyedMutex()) + const auto current = slot.load(std::memory_order_acquire); + if (current == destroyedMutex()) throw std::runtime_error("Mutex has been destroyed"); - if (reinterpret_cast(*mutex) == 1) + if (reinterpret_cast(current) == 1) throw std::runtime_error("Adaptive mutex initializer is unsupported"); - if (!*mutex) { - if (!initialize) - throw std::runtime_error("Mutex is not initialized"); - *mutex = new PthreadMutexPrivate(); - } - return *mutex; + if (current) + return current; + if (!initialize) + throw std::runtime_error("Mutex is not initialized"); + auto* created = new PthreadMutexPrivate(); + slot.store(created, std::memory_order_release); + return created; } template @@ -122,7 +132,7 @@ int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* a if (attr) replacement->_type = (*attr)->type; std::lock_guard lock(initializationMutex); - *mutex = replacement.release(); + std::atomic_ref(*mutex).store(replacement.release(), std::memory_order_release); return 0; } @@ -137,7 +147,7 @@ int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex) { if (*mutex && (*mutex)->_owner.load(std::memory_order_acquire) != std::thread::id{}) throw std::runtime_error("Cannot destroy a locked mutex"); delete *mutex; - *mutex = destroyedMutex(); + std::atomic_ref(*mutex).store(destroyedMutex(), std::memory_order_release); return 0; } From 9325ac08de9e0e50ad3fc75226fe60f67c5d1c5f Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:54:31 -0700 Subject: [PATCH 59/93] docs(md): relink titles that ship their own libc with --skip-sce-module Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 13abea598b..c22ffe6ca0 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Includes a [relinker](core/relinker) that converts executable to the target syst `relinker --to-intel` rewrites the Zen 2-only instructions a title may contain (EXTRQ, INSERTQ, MOVNTSS, MOVNTSD) at relink time, in place when an equal-length Intel encoding exists and otherwise through a jump to a generated stub section, so Intel hosts need no runtime emulation. +By default the relinker also converts the title's own `sce_module`/`sce_modules` prx files into guest modules that take priority at link time (they require eager binding). A title whose `sce_module` ships its own `libc.prx`, such as Demon's Souls (PPSA01342), is relinked with `--skip-sce-module`: its libc would replace the project's [libc.prx](core/libs/prx/libc) (guest memory arena, heap, crash reporting) and imports kernel functions that are not implemented, so the executable fails to bind at startup. + Releases will be published after the first full successful launch of at least one game. ## Status From 9e5f8397f29adee0796bd5d7d473ff79e924d8c2 Mon Sep 17 00:00:00 2001 From: devilismyfriend <87043616+devilismyfriend@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:14:48 -0700 Subject: [PATCH 60/93] refactor(agc): drop outside references from the copy kernel comments Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177gthrWEBBfBu5xJDXgBfo --- core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index 14a2f4614e..a2f9e338ca 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -1852,7 +1852,7 @@ private: // user[8..11], stride-4 32_UINT V#s and a stride-16 one). Demon's Souls runs it ~27 times per // frame at the movie stage, mostly on small records whose destinations later captures read, so // every such capture waited for the kernel's batch. Matched by the code hash and the exact V# - // control words (Kyty's guard set) and done by VulkanDevice::CopyBuffer: a CPU copy at packet + // control words and done by VulkanDevice::CopyBuffer: a CPU copy at packet // time when the copy is small and nothing recorded may still read the destination in place // (the recorder's read tracking) or write either range (the bytes are settled, with no pending // write for readers to wait on), else a transfer between the host imports. @@ -2021,7 +2021,7 @@ private: std::memcpy(&period, reinterpret_cast(record + 4), 4); const auto bytes = static_cast(count) * 4u; // A plain copy only: n >= count (no repetition), every record inside both V#s, exactly the - // groups for the count (Kyty's guard), at most 16 MiB, dword-aligned ranges, and the count + // groups for the count, at most 16 MiB, dword-aligned ranges, and the count // record outside the destination (every wave of the kernel re-reads it after other waves // stored, while the HLE reads it once). if (count == 0 || period < count || groups[0] != (count + 63u) / 64u || count > userData[2] || count > userData[6] || bytes > (16u << 20u) || source % 4 != 0 || destination % 4 != 0 || (record < destination + bytes && destination < record + 8)) { From 42594eecd4bc181f196408679397f247df84f66e Mon Sep 17 00:00:00 2001 From: Darko Gjorgjijoski Date: Mon, 28 Sep 2026 00:25:42 +0200 Subject: [PATCH 61/93] fix(relinker): report buffered output write failures --- core/relinker/CMakeLists.txt | 13 +++++++ core/relinker/io/src/FileWriter.cpp | 2 + core/relinker/io/tests/FileWriterTests.cpp | 43 ++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 core/relinker/io/tests/FileWriterTests.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 42ebb17ba2..ee8aa80fc7 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -106,6 +106,19 @@ endif() if(BUILD_TESTING) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_executable(file_writer_tests + ${CMAKE_CURRENT_SOURCE_DIR}/io/tests/FileWriterTests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/io/src/FileWriter.cpp + ) + target_include_directories(file_writer_tests PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/domain/include + ${CMAKE_CURRENT_SOURCE_DIR}/io/include + ) + add_test(NAME file_writer_binary COMMAND file_writer_tests binary) + add_test(NAME file_writer_text COMMAND file_writer_tests text) + endif() + add_executable(amd64_only_converter_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/Amd64OnlyConverterTests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/Amd64OnlyConverter.cpp diff --git a/core/relinker/io/src/FileWriter.cpp b/core/relinker/io/src/FileWriter.cpp index f30fdae52a..a8c65fd637 100644 --- a/core/relinker/io/src/FileWriter.cpp +++ b/core/relinker/io/src/FileWriter.cpp @@ -9,6 +9,7 @@ void FileWriter::Write(const std::string& path, const std::vector& if (!f) throw Domain::RelinkerException("Cannot open output file: " + path); f.write(reinterpret_cast(data.data()), static_cast(data.size())); + f.close(); if (!f) throw Domain::RelinkerException("Failed to write file: " + path); } @@ -18,6 +19,7 @@ void FileWriter::Write(const std::string& path, const std::string& content) { if (!f) throw Domain::RelinkerException("Cannot open output file: " + path); f << content; + f.close(); if (!f) throw Domain::RelinkerException("Failed to write file: " + path); } diff --git a/core/relinker/io/tests/FileWriterTests.cpp b/core/relinker/io/tests/FileWriterTests.cpp new file mode 100644 index 0000000000..8edc710d55 --- /dev/null +++ b/core/relinker/io/tests/FileWriterTests.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include + +namespace { + +template +void requireWriteFailure(const TValue& data) { + Io::FileWriter writer; + writer.Write("/dev/null", data); + try { + writer.Write("/dev/full", data); + } catch (const Domain::RelinkerException& e) { + if (std::string(e.what()) != "Failed to write file: /dev/full") + throw std::runtime_error("Unexpected write failure: " + std::string(e.what())); + return; + } + throw std::runtime_error("Write to /dev/full reported success"); +} + +} + +int main(int argc, char* argv[]) { + try { + if (argc != 2) throw std::runtime_error("Expected binary or text"); + const std::string mode = argv[1]; + for (const std::size_t size : {std::size_t{3}, std::size_t{65536}}) { + if (mode == "binary") + requireWriteFailure(std::vector(size, 0x41)); + else if (mode == "text") + requireWriteFailure(std::string(size, 'A')); + else + throw std::runtime_error("Unknown test mode: " + mode); + } + } catch (const std::exception& e) { + std::cerr << e.what() << '\n'; + return 1; + } + return 0; +} From 67fce999c521984efb7301ebd93340f70f170d9d Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:13:37 +1000 Subject: [PATCH 62/93] fix(libc): support SDK startup without allocator replacements --- core/libs/CMakeLists.txt | 14 +++++++-- core/libs/prx/libc/src/ApplicationHeap.cpp | 32 +++++++++++++++++-- core/libs/tests/ApplicationHeap.cpp | 36 +++++++++++++++++++++- 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index b0572b5bfe..2785b42536 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -357,11 +357,21 @@ if(BUILD_TESTING) endif() endif() -add_executable(application_heap_tests EXCLUDE_FROM_ALL tests/ApplicationHeap.cpp prx/libc/src/ApplicationHeap.cpp) +add_executable(application_heap_tests EXCLUDE_FROM_ALL tests/ApplicationHeap.cpp) target_include_directories(application_heap_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -set_target_properties(application_heap_tests PROPERTIES CXX_EXTENSIONS OFF) target_link_libraries(application_heap_tests PRIVATE libc) configure_windows_unwind(application_heap_tests) +set_target_properties(application_heap_tests PROPERTIES CXX_EXTENSIONS OFF) +if(BUILD_TESTING) + set_target_properties(application_heap_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME application_heap_replacement COMMAND application_heap_tests) + add_test(NAME application_heap_invalid_metadata COMMAND application_heap_tests invalid) + add_test(NAME application_heap_default COMMAND application_heap_tests default) + if(WIN32) + set_tests_properties(application_heap_replacement application_heap_invalid_metadata application_heap_default + PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endif() add_executable(guest_locale_tests EXCLUDE_FROM_ALL tests/GuestLocale.cpp) target_include_directories(guest_locale_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/core/libs/prx/libc/src/ApplicationHeap.cpp b/core/libs/prx/libc/src/ApplicationHeap.cpp index 81e17250dd..6976923ae9 100644 --- a/core/libs/prx/libc/src/ApplicationHeap.cpp +++ b/core/libs/prx/libc/src/ApplicationHeap.cpp @@ -1,4 +1,5 @@ #include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libc/include/GuestHeap.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include #include @@ -9,6 +10,7 @@ #include #include #include +#include namespace { @@ -28,6 +30,28 @@ Initialize heapFinalize = nullptr; bool heapFinalized = false; thread_local bool heapCallbackActive = false; +void* APS5_VABI defaultAllocate(std::size_t bytes) { return GuestHeap::GuestHeapAllocate_nid_postfix(bytes); } +void APS5_VABI defaultFree(void* pointer) { GuestHeap::GuestHeapFree_nid_postfix(pointer); } +void* APS5_VABI defaultReallocate(void* pointer, std::size_t bytes) { return GuestHeap::GuestHeapReallocate_nid_postfix(pointer, bytes); } +void* APS5_VABI defaultCalloc(std::size_t count, std::size_t bytes) { + if (bytes != 0 && count > std::numeric_limits::max() / bytes) throw std::bad_alloc(); + auto* pointer = defaultAllocate(count * bytes); + std::memset(pointer, 0, count * bytes); + return pointer; +} +void* APS5_VABI defaultAlign(std::size_t alignment, std::size_t bytes) { return GuestHeap::GuestHeapAlign_nid_postfix(alignment, bytes); } +int APS5_VABI defaultPosixAlign(void** pointer, std::size_t alignment, std::size_t bytes) { + if (pointer == nullptr || alignment < sizeof(void*) || (alignment & (alignment - 1)) != 0) return 22; + try { *pointer = defaultAlign(alignment, bytes); return 0; } + catch (const std::bad_alloc&) { return 12; } +} + +std::array defaultApi() { + return {reinterpret_cast(defaultAllocate), reinterpret_cast(defaultFree), + reinterpret_cast(defaultCalloc), reinterpret_cast(defaultReallocate), + reinterpret_cast(defaultAlign), nullptr, reinterpret_cast(defaultPosixAlign)}; +} + class CallbackScope { public: CallbackScope() { @@ -87,8 +111,12 @@ void ApplicationHeapRegister_nid_no_patch(void* const* api) { if (api == nullptr) throw std::invalid_argument("application heap: null allocator API"); std::array replacement; std::memcpy(replacement.data(), api, sizeof(replacement)); - for (std::size_t index = 0; index < 7; ++index) { - if (replacement[index] == nullptr) throw std::invalid_argument("application heap: incomplete allocator API"); + if (std::all_of(replacement.begin(), replacement.end(), [](const void* entry) { return entry == nullptr; })) { + replacement = defaultApi(); + } else { + for (std::size_t index = 0; index < 7; ++index) { + if (replacement[index] == nullptr) throw std::invalid_argument("application heap: incomplete allocator API"); + } } std::lock_guard lock(heapMutex); if (heapFailure) std::rethrow_exception(heapFailure); diff --git a/core/libs/tests/ApplicationHeap.cpp b/core/libs/tests/ApplicationHeap.cpp index 48009f88ef..ccb0507823 100644 --- a/core/libs/tests/ApplicationHeap.cpp +++ b/core/libs/tests/ApplicationHeap.cpp @@ -70,7 +70,7 @@ void write(std::array& data, std::size_t offset, TValue value) } -int main(int argc, char**) { +int main(int argc, char** argv) { reject([] { ApplicationHeapAllocate_nid_no_patch(64); }); reject([] { ApplicationHeapRegister_nid_no_patch(nullptr); }); std::array process{}; @@ -92,6 +92,40 @@ int main(int argc, char**) { write(replacement, 0x40, &align); write(replacement, 0x48, &reallocate); write(replacement, 0x50, &posixAlign); + if (argc > 1 && std::strcmp(argv[1], "default") == 0) { + std::array partial{}; + partial[0] = reinterpret_cast(&allocate); + reject([&] { ApplicationHeapRegister_nid_no_patch(partial.data()); }); + std::memset(replacement.data() + 0x20, 0, sizeof(partial)); + ApplicationHeapInitialize_nid_no_patch(process.data()); + require(initializes == 1); + auto* pointer = static_cast(ApplicationHeapCalloc_nid_no_patch(7, 9)); + for (unsigned i = 0; i < 63; ++i) require(pointer[i] == 0); + std::memset(pointer, 0x5a, 63); + pointer = static_cast(ApplicationHeapReallocate_nid_no_patch(pointer, 150)); + for (unsigned i = 0; i < 63; ++i) require(pointer[i] == 0x5a); + pointer = static_cast(ApplicationHeapReallocate_nid_no_patch(pointer, 11)); + for (unsigned i = 0; i < 11; ++i) require(pointer[i] == 0x5a); + require(ApplicationHeapReallocate_nid_no_patch(pointer, 0) == nullptr); + pointer = static_cast(ApplicationHeapReallocate_nid_no_patch(nullptr, 32)); + require(pointer != nullptr); + ApplicationHeapFree_nid_no_patch(pointer); + for (std::size_t alignment : {16, 64, 4096}) { + auto* aligned = ApplicationHeapAlign_nid_no_patch(alignment, 37); + require(reinterpret_cast(aligned) % alignment == 0); + ApplicationHeapFree_nid_no_patch(aligned); + } + void* aligned = nullptr; + require(ApplicationHeapPosixAlign_nid_no_patch(&aligned, 256, 99) == 0); + require(reinterpret_cast(aligned) % 256 == 0); + ApplicationHeapFree_nid_no_patch(aligned); + ApplicationHeapFree_nid_no_patch(nullptr); + reject([] { ApplicationHeapCalloc_nid_no_patch(SIZE_MAX, 2); }); + reject([] { ApplicationHeapAlign_nid_no_patch(3, 16); }); + ApplicationHeapInitialize_nid_no_patch(process.data()); + require(initializes == 1); + return 0; + } if (argc > 1) { write(replacement, 8, std::uint64_t{99}); reject([&] { ApplicationHeapInitialize_nid_no_patch(process.data()); }); From 07930c6726921b82a3164b0bddaa314e170db216 Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:31:47 +1000 Subject: [PATCH 63/93] feat(videoout): present linear framebuffer layouts --- core/libs/CMakeLists.txt | 9 +++++ .../Execution/include/DisplayBuffer.hpp | 2 ++ .../Execution/src/DisplayBuffer.cpp | 13 +++++-- .../Execution/src/VulkanDevice.cpp | 5 +++ core/libs/prx/libSceVideoOut/src/Buffer.cpp | 4 +-- core/libs/tests/LinearDisplayBuffer.cpp | 35 +++++++++++++++++++ 6 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 core/libs/tests/LinearDisplayBuffer.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index 2785b42536..de0090ebd0 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -109,6 +109,15 @@ endforeach() add_custom_target(libs DEPENDS ${PATCHED_OUTPUTS}) if(BUILD_TESTING) + add_executable(linear_display_buffer_tests tests/LinearDisplayBuffer.cpp) + target_include_directories(linear_display_buffer_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(linear_display_buffer_tests PRIVATE libSceAgcDriver libc) + configure_windows_unwind(linear_display_buffer_tests) + add_test(NAME linear_display_buffer COMMAND linear_display_buffer_tests) + if(WIN32) + set_tests_properties(linear_display_buffer PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_executable(guest_rendering_wait_tests tests/GuestRenderingWait.cpp) target_include_directories(guest_rendering_wait_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(guest_rendering_wait_tests PRIVATE libSceAgcDriver libc) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp index e5c499a667..b7f70c3d80 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp @@ -13,6 +13,8 @@ struct DisplayBuffer { std::uint64_t pixelFormat; std::uint32_t width; std::uint32_t height; + std::uint32_t tilingMode = 0; + std::uint32_t pitchInPixel = 0; }; std::size_t DisplayBufferSize(const DisplayBuffer& buffer); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp index e9294fea79..837b13f02c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp @@ -38,14 +38,19 @@ std::size_t DisplayBufferSize(const DisplayBuffer& buffer) { throw std::runtime_error(message); } require(buffer.address != 0 && (buffer.address & 65535u) == 0, "VideoOut: display buffer requires 64 KiB alignment"); - const auto size = static_cast((buffer.width + 127u) / 128u) * ((buffer.height + 127u) / 128u) * 65536u; + require(buffer.tilingMode <= 1, "VideoOut: unsupported display tiling mode"); + require(buffer.tilingMode != 0 || buffer.pitchInPixel == 0, "VideoOut: tiled display pitch is unsupported"); + const auto pitch = buffer.pitchInPixel == 0 ? buffer.width : buffer.pitchInPixel; + require(pitch >= buffer.width && pitch <= 16384, "VideoOut: invalid linear display pitch"); + const auto size = buffer.tilingMode == 1 ? static_cast(pitch) * buffer.height * 4u + : static_cast((buffer.width + 127u) / 128u) * ((buffer.height + 127u) / 128u) * 65536u; require(size <= std::numeric_limits::max() && size <= std::numeric_limits::max() - buffer.address, "VideoOut: display buffer range overflow"); return static_cast(size); } std::vector DecodeDisplayBuffer(const DisplayBuffer& buffer, std::span source) { PerformanceTimer timing("DisplayBuffer.Decode"); - require(source.size() == DisplayBufferSize(buffer), "VideoOut: invalid tiled display buffer size"); + require(source.size() == DisplayBufferSize(buffer), "VideoOut: invalid display buffer size"); std::vector pixels(static_cast(buffer.width) * buffer.height * 4); const auto blocksPerRow = (buffer.width + 127u) / 128u; const bool rgba = baseFormat(buffer.pixelFormat) == PixelFormatR8G8B8A8; @@ -55,7 +60,9 @@ std::vector DecodeDisplayBuffer(const DisplayBuffer& buffer, std::spa timing.Mark("validate_allocate"); for (std::uint32_t y = 0; y < buffer.height; ++y) { for (std::uint32_t x = 0; x < buffer.width; ++x) { - const auto tiled = (static_cast(y / 128u) * blocksPerRow + x / 128u) * 65536u + tileOffset(x, y); + const auto pitch = buffer.pitchInPixel == 0 ? buffer.width : buffer.pitchInPixel; + const auto tiled = buffer.tilingMode == 1 ? (static_cast(y) * pitch + x) * 4u + : (static_cast(y / 128u) * blocksPerRow + x / 128u) * 65536u + tileOffset(x, y); const auto linear = (static_cast(y) * buffer.width + x) * 4; if (tenBit) { std::uint32_t word = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index ddd49fc0a3..a3a590e638 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1629,6 +1629,10 @@ FrameDumps& Dumps() { } bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { + if (buffer.tilingMode == 1) { + const auto pixels = ReadDisplayBuffer(buffer); + return present(buffer.width, buffer.height, true, pixels); + } static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; // The display buffer is usually a resident render target whose results are still on the GPU: it // is blitted from that image, which skips the write-back retile, the 33 MB read of guest memory, @@ -1934,6 +1938,7 @@ VulkanDevice::PresentStatistics VulkanDevice::PresentCounts() { bool VulkanDevice::PresentWaitsForSlots(const DisplayBuffer* buffer) const { if (buffer == nullptr || !state->SharedSlotInFlight()) return false; + if (buffer->tilingMode == 1) return true; const auto& dumps = Dumps(); if ((dumps.dumped < dumps.limit && !dumps.cpu) || NoResidentPresent()) return true; VkFilter filter = VK_FILTER_LINEAR; diff --git a/core/libs/prx/libSceVideoOut/src/Buffer.cpp b/core/libs/prx/libSceVideoOut/src/Buffer.cpp index 5c260c1fc3..28430fe297 100644 --- a/core/libs/prx/libSceVideoOut/src/Buffer.cpp +++ b/core/libs/prx/libSceVideoOut/src/Buffer.cpp @@ -10,9 +10,9 @@ AgcDriver::DisplayBuffer DescribeVideoOutBuffer(const VideoOutBuffer& buffer, const BufferAttributeGroup& group) { const auto& attribute = group.attribute; if (attribute.reserved0 != 0 || attribute.pad0 != 0 || attribute.reserved1[0] != 0 || attribute.reserved1[1] != 0 || attribute.reserved1[2] != 0) throw std::runtime_error("VideoOut: reserved buffer attribute bits are set"); - if (attribute.tiling_mode != 0 || attribute.pitch_in_pixel != 0 || attribute.aspect_ratio != 0 || attribute.option != 0) throw std::runtime_error("VideoOut: unsupported tiling, pitch, aspect ratio or buffer option"); + if (attribute.tiling_mode > 1 || (attribute.tiling_mode == 0 && attribute.pitch_in_pixel != 0) || attribute.aspect_ratio != 0 || attribute.option != 0) throw std::runtime_error("VideoOut: unsupported tiling, pitch, aspect ratio or buffer option"); if (group.category != VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_UNCOMPRESSED || buffer.metadataAddress != 0 || attribute.dcc_control != 0 || attribute.dcc_cb_register_clear_color != 0) throw std::runtime_error("VideoOut: DCC presentation is not implemented"); - const AgcDriver::DisplayBuffer result{buffer.dataAddress, attribute.pixel_format, attribute.width, attribute.height}; + const AgcDriver::DisplayBuffer result{buffer.dataAddress, attribute.pixel_format, attribute.width, attribute.height, attribute.tiling_mode, attribute.pitch_in_pixel}; static_cast(AgcDriverDisplayBufferSize_nid_postfix(result)); return result; } diff --git a/core/libs/tests/LinearDisplayBuffer.cpp b/core/libs/tests/LinearDisplayBuffer.cpp new file mode 100644 index 0000000000..ad10b3f421 --- /dev/null +++ b/core/libs/tests/LinearDisplayBuffer.cpp @@ -0,0 +1,35 @@ +#include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" +#include +#include +#include + +static void Require(bool condition) { if (!condition) std::abort(); } +template static void Reject(TAction action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + Require(rejected); +} + +int main() { + AgcDriver::DisplayBuffer buffer{65536, 0x8000000000000000ull, 2, 2, 1, 3}; + const std::array input{1,2,3,4, 5,6,7,8, 99,99,99,99, + 9,10,11,12, 13,14,15,16, 99,99,99,99}; + const auto source = std::as_bytes(std::span(input)); + Require(AgcDriver::DisplayBufferSize(buffer) == input.size()); + auto pixels = AgcDriver::DecodeDisplayBuffer(buffer, source); + Require(pixels.size() == 16); + for (unsigned i = 0; i < pixels.size(); ++i) Require(std::to_integer(pixels[i]) == i + 1); + buffer.pixelFormat = 0x8000000022000000ull; + pixels = AgcDriver::DecodeDisplayBuffer(buffer, source); + const std::array swapped{3,2,1,4, 7,6,5,8, 11,10,9,12, 15,14,13,16}; + for (unsigned i = 0; i < pixels.size(); ++i) Require(std::to_integer(pixels[i]) == swapped[i]); + Reject([&] { AgcDriver::DecodeDisplayBuffer(buffer, source.first(23)); }); + buffer.pitchInPixel = 1; + Reject([&] { AgcDriver::DisplayBufferSize(buffer); }); + buffer.pitchInPixel = 0; + Require(AgcDriver::DisplayBufferSize(buffer) == 16); + buffer.tilingMode = 2; + Reject([&] { AgcDriver::DisplayBufferSize(buffer); }); + buffer.tilingMode = 0; + Require(AgcDriver::DisplayBufferSize(buffer) == 65536); +} From b25025c6820eb3863f97564a1a827d3cdf3a54bf Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:49:13 +1000 Subject: [PATCH 64/93] fix(agc): support explicit indexed draw commands --- core/libs/CMakeLists.txt | 9 +++ .../prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp | 11 +-- .../prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp | 6 +- .../libSceAgcDriver/Execution/include/Pm4.hpp | 2 +- .../libSceAgcDriver/Execution/src/Driver.cpp | 2 +- .../prx/libSceAgcDriver/Execution/src/Pm4.cpp | 34 ++++++--- core/libs/tests/GuestIndexedDraw.cpp | 74 +++++++++++++++++++ 7 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 core/libs/tests/GuestIndexedDraw.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index de0090ebd0..d972362601 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -136,6 +136,15 @@ if(BUILD_TESTING) set_tests_properties(guest_buffer_reuse PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_executable(guest_indexed_draw_tests tests/GuestIndexedDraw.cpp) + target_include_directories(guest_indexed_draw_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(guest_indexed_draw_tests PRIVATE libSceAgc libSceAgcDriver libc) + configure_windows_unwind(guest_indexed_draw_tests) + add_test(NAME guest_indexed_draw COMMAND guest_indexed_draw_tests) + if(WIN32) + set_tests_properties(guest_indexed_draw PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_executable(guest_shader_link_tests tests/GuestShaderLink.cpp) target_include_directories(guest_shader_link_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(guest_shader_link_tests PRIVATE libSceAgc libc) diff --git a/core/libs/prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp b/core/libs/prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp index 8c81f1e8f3..d557ba0dc8 100644 --- a/core/libs/prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp +++ b/core/libs/prx/libSceAgc/DcbDraw/src/DrawIndexed.cpp @@ -9,13 +9,10 @@ extern "C" { -uint32_t* APS5_VABI sceAgcDcbDrawIndex(CommandBuffer* buf, uint32_t index_count, const volatile void* index_addr, uint64_t modifier) { - (void)buf; - (void)index_count; - (void)index_addr; - (void)modifier; - NotImplemented_nid_no_patch(__func__); - return nullptr; +std::uint32_t* APS5_VABI sceAgcDcbDrawIndex(CommandBuffer* buf, std::uint32_t indexCount, const volatile void* indexAddress, std::uint64_t modifier) { + const auto address = reinterpret_cast(indexAddress); + Agc::Command::CheckAddress(address, 1, __func__); + return Agc::Command::Emit(buf, 0x27u, {indexCount, static_cast(address), static_cast(address >> 32u), indexCount, Agc::Command::DrawInitiator(modifier, true, __func__)}, __func__); } std::uint32_t APS5_VABI sceAgcDcbDrawIndexGetSize() { diff --git a/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp b/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp index 9ff4fb9fdb..e7c71b3b68 100644 --- a/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp +++ b/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp @@ -18,12 +18,12 @@ std::uint32_t APS5_VABI sceAgcDcbSetIndexBufferGetSize() { return 0; } -uint32_t* APS5_VABI sceAgcDcbSetIndexCount(CommandBuffer* buf, uint32_t index_count) { - return Agc::Command::Emit(buf, 0x13u, {index_count}, __func__); +std::uint32_t* APS5_VABI sceAgcDcbSetIndexCount(CommandBuffer* buf, std::uint32_t indexCount) { + return Agc::Command::Emit(buf, 0x13u, {indexCount}, __func__); } std::uint32_t APS5_VABI sceAgcDcbSetIndexCountGetSize() { - return 2; + return 8; } std::uint32_t* APS5_VABI sceAgcDcbSetIndexSize(CommandBuffer* buf, std::uint8_t indexSize, std::uint8_t cachePolicy) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp index 9543e5a628..6035dc5605 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp @@ -73,7 +73,7 @@ struct DrawArguments { DrawArguments ReadDrawArguments(const DrawParameters::IndirectDraw& indirect, std::uint32_t record); std::uint32_t ReadDrawCount(const DrawParameters::IndirectDraw& indirect); inline bool IndirectDrawOpcode(std::uint32_t opcode) { return opcode == 0x24 || opcode == 0x25 || opcode == 0x2c || opcode == 0x38; } -inline bool DrawOpcode(std::uint32_t opcode) { return opcode == 0x2d || opcode == 0x35 || IndirectDrawOpcode(opcode); } +inline bool DrawOpcode(std::uint32_t opcode) { return opcode == 0x27 || opcode == 0x2d || opcode == 0x35 || IndirectDrawOpcode(opcode); } std::string Name(std::uint32_t header); // A PM4 type-2 packet is a one-dword filler (command-buffer padding); type 3 and type 0 carry a diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index a2f9e338ca..b647e97356 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -441,7 +441,7 @@ bool NeedsRecordedLabels(std::uint32_t header) { bool PacketLocksItself(std::uint32_t header) { if (header == FlipPacketHeader) return true; switch ((header >> 8u) & 0xffu) { - case 0x15: case 0x16: case 0x2d: case 0x35: case 0x24: case 0x25: case 0x2c: case 0x38: return true; + case 0x15: case 0x16: case 0x27: case 0x2d: case 0x35: case 0x24: case 0x25: case 0x2c: case 0x38: return true; default: return false; } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index 21dd47ae27..a89bc0ecab 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -137,14 +137,14 @@ std::string_view UnsupportedReason(std::uint32_t header) { } } switch (opcode) { - case 0x11: case 0x12: case 0x13: case 0x15: case 0x16: case 0x26: + case 0x11: case 0x12: case 0x13: case 0x15: case 0x16: case 0x26: case 0x27: case 0x2a: case 0x2d: case 0x2f: case 0x35: case 0x37: case 0x40: case 0x42: case 0x46: case 0x50: case 0x58: case 0x63: case 0x64: case 0x69: case 0x76: case 0x79: case 0x7a: case 0x81: case 0x83: case 0x9f: return {}; case 0x24: case 0x25: case 0x2c: case 0x38: if (IndirectDrawsDisabled()) return "graphics draw, shader stages and guest render-target materialization are not implemented"; return {}; - case 0x27: case 0x3a: case 0x8d: + case 0x3a: case 0x8d: return "graphics draw, shader stages and guest render-target materialization are not implemented"; case 0x20: return "GPU query predication is not implemented"; case 0x22: return "conditional command execution and conditional flip reservation are not implemented"; @@ -219,6 +219,13 @@ void Validate(std::span packet, std::uint32_t queue) { case 0x12: graphics(); size(2); require((packet[1] & ~0xfu) == 0, "unsupported CLEAR_STATE payload bits"); break; case 0x13: case 0x2f: graphics(); size(2); break; case 0x26: graphics(); size(3); break; + case 0x27: + graphics(); + size(6); + require(packet[3] <= 0xffffu, "unsupported index address bits"); + require(packet[4] <= packet[1], "index count exceeds maximum index size"); + require((packet[5] & ~0x20u) == 0, "unsupported indexed draw flags"); + break; case 0x2a: graphics(); size(2); require(packet[1] <= 3, "unsupported index-type modifiers"); break; case 0x2d: graphics(); @@ -470,7 +477,7 @@ bool UsesGpuCacheBarrier(std::span packet) { bool AccessesMemory(std::uint32_t header) { switch ((header >> 8u) & 0xffu) { - case 0x3c: case 0x93: case 0x49: case 0x16: case 0x2d: case 0x35: case 0x24: case 0x25: case 0x2c: case 0x38: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true; + case 0x27: case 0x3c: case 0x93: case 0x49: case 0x16: case 0x2d: case 0x35: case 0x24: case 0x25: case 0x2c: case 0x38: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true; default: return false; } } @@ -578,16 +585,21 @@ DrawParameters ResolveDraw(std::span packet, const QueueSta require(packet[1] == 0 || firstVertex <= std::numeric_limits::max() - (packet[1] - 1u), "auto draw vertex range overflow"); return {0, packet[1], 0, queue.instanceCount, packet[2] & 0x20u, false, firstVertex, 0}; } - require(((packet[0] >> 8u) & 0xffu) == 0x35, "expected DRAW_INDEX_OFFSET_2 packet"); - const auto indexSize = indexSizeOf(queue); - const auto offset = static_cast(packet[2]) * indexSize; - require(offset <= std::numeric_limits::max() - queue.indexBase, "index address overflow"); - const auto address = queue.indexBase + offset; - const auto bytes = static_cast(packet[3]) * indexSize; + const bool explicitAddress = ((packet[0] >> 8u) & 0xffu) == 0x27; + require(explicitAddress || ((packet[0] >> 8u) & 0xffu) == 0x35, "expected indexed draw packet"); + require(queue.indexType <= 2, "unsupported index type"); + const std::uint32_t indexSize = queue.indexType == 0 ? 2 : queue.indexType == 1 ? 4 : 1; + const auto base = explicitAddress ? address(packet[2], packet[3]) : queue.indexBase; + const auto count = explicitAddress ? packet[4] : packet[3]; + require(base != 0 && base % indexSize == 0, "null or misaligned index base"); + const auto offset = explicitAddress ? 0 : static_cast(packet[2]) * indexSize; + require(offset <= std::numeric_limits::max() - base, "index address overflow"); + const auto address = base + offset; + const auto bytes = static_cast(count) * indexSize; require(bytes <= std::numeric_limits::max(), "index range size overflow"); GuestMemory::CheckRange(reinterpret_cast(address), static_cast(bytes), indexSize); - APS5_LOG_OUT_DEBUG("ResolveDraw context targetMask=0x%x shaderMask=0x%x indexCount=%u indexType=%u instances=%u", queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u, packet[3], queue.indexType, queue.instanceCount); - return {address, packet[3], indexSize, queue.instanceCount, packet[4]}; + APS5_LOG_OUT_DEBUG("ResolveDraw context targetMask=0x%x shaderMask=0x%x indexCount=%u indexType=%u instances=%u", queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u, count, queue.indexType, queue.instanceCount); + return {address, count, indexSize, queue.instanceCount, packet.back()}; } void Execute(std::span packet, QueueState& queue) { diff --git a/core/libs/tests/GuestIndexedDraw.cpp b/core/libs/tests/GuestIndexedDraw.cpp new file mode 100644 index 0000000000..3c6ed23b0b --- /dev/null +++ b/core/libs/tests/GuestIndexedDraw.cpp @@ -0,0 +1,74 @@ +#include "SceTypes.hpp" +#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include +#include +#include + +extern "C" { +std::uint32_t* APS5_VABI sceAgcDcbSetIndexCount(CommandBuffer*, std::uint32_t); +std::uint32_t APS5_VABI sceAgcDcbSetIndexCountGetSize(); +std::uint32_t* APS5_VABI sceAgcDcbDrawIndex(CommandBuffer*, std::uint32_t, const volatile void*, std::uint64_t); +std::uint32_t APS5_VABI sceAgcDcbDrawIndexGetSize(); +} +static void Require(bool condition) { if (!condition) std::abort(); } +template static void Reject(TAction action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + Require(rejected); +} +int main() { + std::array words{}; + CommandBuffer buffer{words.data(), words.data()+words.size(), words.data(), words.data()+words.size(), nullptr, nullptr, 0}; + auto* sizePacket = sceAgcDcbSetIndexCount(&buffer, 3); + Require(sizePacket == words.data() && words[0] == 0xc0001300u && words[1] == 3); + Require(sceAgcDcbSetIndexCountGetSize() == 8); + AgcDriver::QueueState queue; + AgcDriver::Pm4::Validate({sizePacket,2},0); + AgcDriver::Pm4::Execute({sizePacket,2},queue); + Require(queue.indexBufferSize == 3); + alignas(4) std::array indices{2,0,1}; + auto* draw = sceAgcDcbDrawIndex(&buffer,3,indices.data(),0); + Require(draw == words.data()+2 && draw[0] == 0xc0042700u && draw[1] == 3 && draw[4] == 3 && draw[5] == 0); + Require(sceAgcDcbDrawIndexGetSize() == 24); + Require(AgcDriver::Pm4::AccessesMemory(draw[0])); + Require(AgcDriver::Pm4::DrawOpcode((draw[0] >> 8u) & 0xffu)); + queue.indexBase = 1; + for (unsigned type=0;type<3;++type) { + queue.indexType=type; + const auto result=AgcDriver::Pm4::ResolveDraw({draw,6},queue); + Require(result.indexAddress == reinterpret_cast(indices.data())); + Require(result.indexCount == 3 && result.indexSize == (type==0?2:type==1?4:1) && result.indexed); + } + Require(queue.indexBase == 1); + queue.indexType = 1; + queue.indexBase = reinterpret_cast(indices.data()); + const std::array offsetDraw{0xc0033500u,2,1,2,0}; + const auto offsetResult = AgcDriver::Pm4::ResolveDraw(offsetDraw,queue); + Require(offsetResult.indexAddress == queue.indexBase+4 && offsetResult.indexCount == 2); + draw[5] = 0x20; + Require(AgcDriver::Pm4::ResolveDraw({draw,6},queue).flags == 0x20); + draw[5] = 0; + draw[1] = 0; draw[4] = 0; + Require(AgcDriver::Pm4::ResolveDraw({draw,6},queue).indexCount == 0); + draw[1] = 3; draw[4] = 3; + draw[4]=4; + Reject([&]{AgcDriver::Pm4::Validate({draw,6},0);}); + draw[4]=3; + draw[5]=2; + Reject([&]{AgcDriver::Pm4::Validate({draw,6},0);}); + draw[5]=0; + Reject([&]{AgcDriver::Pm4::Validate({draw,5},0);}); + Reject([&]{AgcDriver::Pm4::Validate({draw,6},1);}); + draw[3]=0x10000; + Reject([&]{AgcDriver::Pm4::Validate({draw,6},0);}); + draw[2]=1; draw[3]=0; queue.indexType=1; + Reject([&]{AgcDriver::Pm4::ResolveDraw({draw,6},queue);}); + auto* saved=buffer.cursor_up; + Reject([&]{sceAgcDcbDrawIndex(&buffer,3,nullptr,0);}); + Reject([&]{sceAgcDcbDrawIndex(&buffer,3,indices.data(),1ull<<40);}); + Require(buffer.cursor_up==saved); + buffer.cursor_down=buffer.cursor_up+5; + Reject([&]{sceAgcDcbDrawIndex(&buffer,3,indices.data(),0);}); + Require(buffer.cursor_up==saved); + return 0; +} From fc26cff30ff132d862e635722da16617b3c89183 Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:00:03 +1000 Subject: [PATCH 65/93] test(shader): cover decoded buffer address operands --- core/libs/CMakeLists.txt | 8 +++++ core/libs/tests/ShaderBufferAddress.cpp | 42 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 core/libs/tests/ShaderBufferAddress.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index d972362601..3374519bab 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -137,6 +137,14 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() add_executable(guest_indexed_draw_tests tests/GuestIndexedDraw.cpp) + add_executable(shader_buffer_address_tests tests/ShaderBufferAddress.cpp) + target_link_libraries(shader_buffer_address_tests PRIVATE shader_recompiler_agc_driver libc) + configure_windows_unwind(shader_buffer_address_tests) + add_test(NAME shader_buffer_address COMMAND shader_buffer_address_tests) + if(WIN32) + set_tests_properties(shader_buffer_address PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() target_include_directories(guest_indexed_draw_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(guest_indexed_draw_tests PRIVATE libSceAgc libSceAgcDriver libc) configure_windows_unwind(guest_indexed_draw_tests) diff --git a/core/libs/tests/ShaderBufferAddress.cpp b/core/libs/tests/ShaderBufferAddress.cpp new file mode 100644 index 0000000000..8062fb67dd --- /dev/null +++ b/core/libs/tests/ShaderBufferAddress.cpp @@ -0,0 +1,42 @@ +#include "Translation/TranslationContext.hpp" +#include "RdnaDecoder/RdnaMemoryOpDecoder.hpp" +#include +#include + +using namespace ShaderRecompiler; +static void Require(bool value) { if (!value) throw std::runtime_error("buffer address operand regression"); } +static void CheckRegister(IrValue* value, IrOpcode opcode, unsigned index) { + Require(value->Opcode() == opcode); + Require(value->Argument(0)->Register().index == index); +} +static void Check(bool typed, unsigned opcode, IrOpcode expected, bool idxen, bool offen, bool immediate) { + const std::array code{ + (typed ? (0x3au<<26)|(opcode<<16)|(4u<<19) : (0x38u<<26)|(opcode<<18)) + | (unsigned(idxen)<<13)|(unsigned(offen)<<12)|12u, + ((immediate?128u:20u)<<24)|(2u<<16)|(4u<<8)|10u + }; + const auto instruction = typed ? DecodeRdnaMtbuf(0,code,0) : DecodeRdnaMubuf(0,code,0); + IrProgram program; + auto& block=program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program,block,256); + context.TranslateInstruction(instruction); + IrValue* memory=nullptr; + for(auto* value:block.Instructions()) if(value->Opcode()==expected) memory=value; + Require(memory!=nullptr); + if(idxen) CheckRegister(memory->Argument(1),IrOpcode::GetVectorRegister,10); + else Require(memory->Argument(1)->HasImmediate() && memory->Argument(1)->ImmediateU32()==0); + if(offen) CheckRegister(memory->Argument(2),IrOpcode::GetVectorRegister,idxen?11:10); + else Require(memory->Argument(2)->HasImmediate() && memory->Argument(2)->ImmediateU32()==0); + if(immediate) Require(memory->Argument(3)->HasImmediate() && memory->Argument(3)->ImmediateU32()==0); + else CheckRegister(memory->Argument(3),IrOpcode::GetScalarRegister,20); +} +int main() { + for(bool idxen:{false,true}) for(bool offen:{false,true}) for(bool immediate:{false,true}) { + Check(false,0x0c,IrOpcode::LoadBufferU32,idxen,offen,immediate); + Check(false,0x1c,IrOpcode::StoreBufferU32,idxen,offen,immediate); + Check(false,0x32,IrOpcode::BufferAtomicIAdd32,idxen,offen,immediate); + Check(true,0,IrOpcode::LoadBufferU32,idxen,offen,immediate); + Check(true,4,IrOpcode::StoreBufferU32,idxen,offen,immediate); + } +} From e1e3a122270ed575a4588af412c5b24a9b8267d4 Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:07:34 +1000 Subject: [PATCH 66/93] fix(build): restrict POSIX exception runtime test to Unix --- core/libs/CMakeLists.txt | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index 3374519bab..63bd99a002 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -372,14 +372,11 @@ if(BUILD_TESTING) set_tests_properties(guest_strings PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - add_executable(exception_runtime_tests tests/ExceptionRuntime.cpp) - target_include_directories(exception_runtime_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) - target_link_libraries(exception_runtime_tests PRIVATE libc) - configure_windows_unwind(exception_runtime_tests) - add_test(NAME exception_runtime COMMAND exception_runtime_tests) - if(WIN32) - set_tests_properties(exception_runtime PROPERTIES ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:$") + if(UNIX) + add_executable(exception_runtime_tests tests/ExceptionRuntime.cpp) + target_include_directories(exception_runtime_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(exception_runtime_tests PRIVATE libc) + add_test(NAME exception_runtime COMMAND exception_runtime_tests) endif() endif() From 9814c93f757a6cc1194ecd9d23ec115dc5078681 Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:55:00 +1000 Subject: [PATCH 67/93] fix(agc): avoid exception-driven render target format probing --- core/libs/CMakeLists.txt | 9 +++++++++ .../Graphics/include/TextureFormat.hpp | 2 ++ .../prx/libSceAgcDriver/Graphics/src/Draw.cpp | 14 +------------- .../Graphics/src/TextureFormat.cpp | 7 +++++++ core/libs/tests/TextureFormatLookup.cpp | 15 +++++++++++++++ 5 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 core/libs/tests/TextureFormatLookup.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index 63bd99a002..cb33a343ff 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -109,6 +109,15 @@ endforeach() add_custom_target(libs DEPENDS ${PATCHED_OUTPUTS}) if(BUILD_TESTING) + add_executable(texture_format_lookup_tests tests/TextureFormatLookup.cpp) + target_include_directories(texture_format_lookup_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) + target_link_libraries(texture_format_lookup_tests PRIVATE libSceAgcDriver libc) + configure_windows_unwind(texture_format_lookup_tests) + add_test(NAME texture_format_lookup COMMAND texture_format_lookup_tests) + if(WIN32) + set_tests_properties(texture_format_lookup PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_executable(linear_display_buffer_tests tests/LinearDisplayBuffer.cpp) target_include_directories(linear_display_buffer_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(linear_display_buffer_tests PRIVATE libSceAgcDriver libc) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp index a631ca03e3..3f2d1f0d88 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp @@ -6,10 +6,12 @@ #endif #include #include +#include namespace AgcDriver::Graphics { VkFormat ResolveTextureFormat(std::uint32_t guestFormat); +std::optional FindGuestTextureFormat(VkFormat format, std::uint32_t elementBytes); std::uint32_t BytesPerElement(std::uint32_t guestFormat); bool IsBlockCompressed(std::uint32_t guestFormat); std::uint32_t BlockWidth(std::uint32_t guestFormat); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index ed00f454a2..9f008e3aa5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -34,19 +34,7 @@ namespace { // A guest texture format code with the same Vulkan format and texel size as a color buffer. std::uint32_t GuestFormatFor(VkFormat format, std::uint32_t elementBytes) { - static std::mutex mutex; - static std::map, std::uint32_t> known; - std::lock_guard lock(mutex); - if (const auto found = known.find({format, elementBytes}); found != known.end()) return found->second; - for (std::uint32_t code = 0; code < 256; ++code) { - try { - if (!IsBlockCompressed(code) && ResolveTextureFormat(code) == format && BytesPerElement(code) == elementBytes) { - known.emplace(std::make_pair(format, elementBytes), code); - return code; - } - } catch (const std::exception&) { - } - } + if (const auto guest = FindGuestTextureFormat(format, elementBytes)) return *guest; throw std::runtime_error("AGC graphics: no guest texture format matches the color buffer format " + std::to_string(static_cast(format))); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp index 06c67ae59c..83b6bfbec0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp @@ -108,6 +108,13 @@ VkFormat ResolveTextureFormat(std::uint32_t guestFormat) { return findFormatEntry(guestFormat).vkFormat; } +std::optional FindGuestTextureFormat(VkFormat format, std::uint32_t elementBytes) { + for (const auto& entry : kFormatLookup) { + if (!entry.blockCompressed && entry.vkFormat == format && entry.bytesPerElement == elementBytes) return entry.guestFormat; + } + return std::nullopt; +} + std::uint32_t BytesPerElement(std::uint32_t guestFormat) { return findFormatEntry(guestFormat).bytesPerElement; } diff --git a/core/libs/tests/TextureFormatLookup.cpp b/core/libs/tests/TextureFormatLookup.cpp new file mode 100644 index 0000000000..fd8901eb54 --- /dev/null +++ b/core/libs/tests/TextureFormatLookup.cpp @@ -0,0 +1,15 @@ +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" +#include + +int main() { + using namespace AgcDriver::Graphics; + for (auto format : {VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R16G16B16A16_SFLOAT}) { + const unsigned bytes = format == VK_FORMAT_R8_UNORM ? 1 : format == VK_FORMAT_R8G8B8A8_UNORM ? 4 : 8; + const auto guest = FindGuestTextureFormat(format, bytes); + if (!guest || ResolveTextureFormat(*guest) != format || BytesPerElement(*guest) != bytes) std::abort(); + if (FindGuestTextureFormat(format, bytes + 1)) std::abort(); + } + if (FindGuestTextureFormat(VK_FORMAT_B8G8R8A8_UNORM, 4)) std::abort(); + if (FindGuestTextureFormat(VK_FORMAT_UNDEFINED, 4)) std::abort(); + if (FindGuestTextureFormat(VK_FORMAT_BC1_RGBA_UNORM_BLOCK, 8)) std::abort(); +} From e424b6b7bfd625eeeb9e3146c631b0a536806f1a Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:55:01 +1000 Subject: [PATCH 68/93] fix(agc): preserve recorder buffers through Windows thread teardown --- core/libs/CMakeLists.txt | 10 +++++ .../libSceAgcDriver/Graphics/src/Recorder.cpp | 44 ++++++++++--------- .../libs/prx/libc/include/HostThreadLocal.hpp | 28 ++++++++++++ core/libs/tests/HostThreadLocal.cpp | 24 ++++++++++ core/libs/tests/HostThreadLocalLibrary.cpp | 31 +++++++++++++ 5 files changed, 117 insertions(+), 20 deletions(-) create mode 100644 core/libs/prx/libc/include/HostThreadLocal.hpp create mode 100644 core/libs/tests/HostThreadLocal.cpp create mode 100644 core/libs/tests/HostThreadLocalLibrary.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index cb33a343ff..762c9b1d7e 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -109,6 +109,16 @@ endforeach() add_custom_target(libs DEPENDS ${PATCHED_OUTPUTS}) if(BUILD_TESTING) + add_library(host_thread_local_test_library SHARED tests/HostThreadLocalLibrary.cpp) + target_include_directories(host_thread_local_test_library PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + add_executable(host_thread_local_tests tests/HostThreadLocal.cpp) + target_link_libraries(host_thread_local_tests PRIVATE host_thread_local_test_library) + if(MINGW) + target_compile_options(host_thread_local_test_library PRIVATE -fasynchronous-unwind-tables) + target_compile_options(host_thread_local_tests PRIVATE -fasynchronous-unwind-tables) + endif() + add_test(NAME host_thread_local COMMAND host_thread_local_tests) + set_tests_properties(host_thread_local PROPERTIES TIMEOUT 15) add_executable(texture_format_lookup_tests tests/TextureFormatLookup.cpp) target_include_directories(texture_format_lookup_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) target_link_libraries(texture_format_lookup_tests PRIVATE libSceAgcDriver libc) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 8c11f3ccd7..d9184240f0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -1,3 +1,4 @@ +#include "prx/libc/include/HostThreadLocal.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" @@ -276,7 +277,8 @@ struct DeferredBatch { std::vector> kept; std::vector> completions; }; -thread_local std::vector deferredBatches; +struct DeferredBatchesTag {}; +auto& DeferredBatches() { return HostThreadLocal, DeferredBatchesTag>(); } std::atomic deferredPending{0}; // APS5_PROFILE_DRAW, for the [recorder] line: batches and objects released after an unlock and the // time that took, on the release thread and inline (on the unlocking thread: the kill switch, a @@ -392,11 +394,11 @@ void JoinReleaseThread() { // The unlock hook: hands what this thread deferred to the release thread, or destroys it here // (APS5_RELEASE_ON_UNLOCK=1, the queue is full or stopping, or the hand-off cannot be made). void ReleaseDeferredKeeps() { - if (deferredBatches.empty()) return; + if (DeferredBatches().empty()) return; // Taken off the thread's list first: a destructor that took and released the mutex (none is // known to) would re-enter here and must find nothing. - auto releasing = std::move(deferredBatches); - deferredBatches.clear(); + auto releasing = std::move(DeferredBatches()); + DeferredBatches().clear(); if (ReleaseOnUnlock()) { DestroyDeferred(std::move(releasing), false); return; @@ -475,10 +477,12 @@ Recorder* labelTableOwner = nullptr; // The ranges of the calling worker's queued labels (NoteQueuedLabel, cleared by // ForgetQueuedLabels): the flush hook, on the same thread, has them recorded before an access that // overlaps one, through the function the driver installed. -thread_local std::vector> queuedLabelRanges; +struct QueuedLabelRangesTag {}; +auto& QueuedLabelRanges() { return HostThreadLocal>, QueuedLabelRangesTag>(); } std::atomic queuedLabelRecorder{nullptr}; // The dwords (with their stamps) this thread noted since its last CloseLabelGroup (see Recorder::CloseLabelGroup). -thread_local std::vector> labelGroupDwords; +struct LabelGroupDwordsTag {}; +auto& LabelGroupDwords() { return HostThreadLocal>, LabelGroupDwordsTag>(); } // Late-rule counters (Recorder::LateCounts), per thread: lookups run on the waiting worker. thread_local Recorder::LateStatistics lateCounts{}; @@ -548,7 +552,7 @@ bool SeparateQueuedLabels() { bool QueuedLabelOverlaps(std::uint64_t address, std::size_t bytes) { const auto end = address + bytes; - for (const auto& [begin, finish] : queuedLabelRanges) { + for (const auto& [begin, finish] : QueuedLabelRanges()) { if (address < finish && begin < end) return true; } return false; @@ -943,7 +947,7 @@ void FlushForAccess(std::uint64_t address, std::size_t bytes) { // would reap nested in the reap in progress, landing later batches' write-backs before the // rest of this one's, and this queue's queued label follows every batch of its own in flight // anyway, so the write-back landing first is the order the hardware gives. - if (!queuedLabelRanges.empty() && QueuedLabelOverlaps(address, bytes)) { + if (!QueuedLabelRanges().empty() && QueuedLabelOverlaps(address, bytes)) { if (completionDepth != 0 && HookCompletionGuard()) { queuedLabelHookInCompletion.fetch_add(1, std::memory_order_relaxed); } else if (auto* record = queuedLabelRecorder.load(std::memory_order_acquire); record != nullptr) { @@ -1074,9 +1078,9 @@ Recorder::~Recorder() { // other threads (an unlock hook that found the queue stopping or full) are waited for before // the caller destroys the device. The count is global: a newer recorder's batches (device // replacement) are waited for too, which only prolongs the spin. - if (!deferredBatches.empty()) { - auto own = std::move(deferredBatches); - deferredBatches.clear(); + if (!DeferredBatches().empty()) { + auto own = std::move(DeferredBatches()); + DeferredBatches().clear(); DestroyDeferred(std::move(own), false); } JoinReleaseThread(); @@ -1160,9 +1164,9 @@ Recorder::LateStatistics Recorder::LateCounts() { } void Recorder::CloseLabelGroup(std::uint64_t trackerGeneration) { - if (labelGroupDwords.empty()) return; - auto group = std::move(labelGroupDwords); - labelGroupDwords.clear(); + if (LabelGroupDwords().empty()) return; + auto group = std::move(LabelGroupDwords()); + LabelGroupDwords().clear(); if (trackerGeneration == 0) return; std::lock_guard tableLock(labelTableMutex); if (labelTableOwner == nullptr) return; @@ -1183,7 +1187,7 @@ void Recorder::NoteQueuedLabel(std::uint64_t address, std::span if (bytes.size() < 4 || address % 4 != 0) return; // The range before the table: the hook covers the group whether or not a table exists to // enter it in (no owner: the callback records it outright, by a CPU store when need be). - queuedLabelRanges.emplace_back(address, address + bytes.size()); + QueuedLabelRanges().emplace_back(address, address + bytes.size()); queuedLabelsNoted.fetch_add(1, std::memory_order_relaxed); std::lock_guard tableLock(labelTableMutex); if (labelTableOwner == nullptr) return; @@ -1199,9 +1203,9 @@ void Recorder::NoteQueuedLabel(std::uint64_t address, std::span } void Recorder::ForgetQueuedLabels() { - if (queuedLabelRanges.empty()) return; - auto ranges = std::move(queuedLabelRanges); - queuedLabelRanges.clear(); + if (QueuedLabelRanges().empty()) return; + auto ranges = std::move(QueuedLabelRanges()); + QueuedLabelRanges().clear(); std::lock_guard tableLock(labelTableMutex); if (labelTableOwner == nullptr) return; auto& recorded = labelTableOwner->labels; @@ -2124,7 +2128,7 @@ void Recorder::noteLabelOn(Batch& batch, std::uint64_t address, std::span batch, bool wait, int source) { // destroys the batch's objects here, under the mutex, counts nothing the destructor would // wait for and must not escape (the rest of finish() releases the fence and command buffer). try { - deferredBatches.push_back({std::move(batch->kept), std::move(batch->completions)}); + DeferredBatches().push_back({std::move(batch->kept), std::move(batch->completions)}); deferredPending.fetch_add(1, std::memory_order_acq_rel); } catch (...) { } diff --git a/core/libs/prx/libc/include/HostThreadLocal.hpp b/core/libs/prx/libc/include/HostThreadLocal.hpp new file mode 100644 index 0000000000..076459cc9a --- /dev/null +++ b/core/libs/prx/libc/include/HostThreadLocal.hpp @@ -0,0 +1,28 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_HOSTTHREADLOCAL_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_HOSTTHREADLOCAL_HPP + +#include +#include +#ifdef _WIN32 +#include +#endif + +template +TValue& HostThreadLocal() { +#ifdef _WIN32 + static const DWORD slot = FlsAlloc(+[](void* value) { delete static_cast(value); }); + if (slot == FLS_OUT_OF_INDEXES) throw std::runtime_error("host thread-local slot allocation failed"); + auto* value = static_cast(FlsGetValue(slot)); + if (value == nullptr) { + auto created = std::make_unique(); + if (!FlsSetValue(slot, created.get())) throw std::runtime_error("host thread-local value registration failed"); + value = created.release(); + } + return *value; +#else + static thread_local TValue value; + return value; +#endif +} + +#endif diff --git a/core/libs/tests/HostThreadLocal.cpp b/core/libs/tests/HostThreadLocal.cpp new file mode 100644 index 0000000000..4fedf466a2 --- /dev/null +++ b/core/libs/tests/HostThreadLocal.cpp @@ -0,0 +1,24 @@ +#include +#include +#ifdef _WIN32 +#include +#endif + +extern "C" void TouchHostThreadLocal(); +extern "C" unsigned DestroyedHostThreadLocals(); + +int main() { + for (unsigned i = 0; i < 64; ++i) { + std::thread worker([] { TouchHostThreadLocal(); TouchHostThreadLocal(); }); + worker.join(); + if (DestroyedHostThreadLocals() != (i + 1) * 2) std::abort(); + } +#ifdef _WIN32 + for (unsigned i = 0; i < 64; ++i) { + const auto worker = CreateThread(nullptr, 0, +[](void*) -> DWORD { TouchHostThreadLocal(); return 0; }, nullptr, 0, nullptr); + if (!worker || WaitForSingleObject(worker, 5000) != WAIT_OBJECT_0) std::abort(); + CloseHandle(worker); + if (DestroyedHostThreadLocals() != 128 + (i + 1) * 2) std::abort(); + } +#endif +} diff --git a/core/libs/tests/HostThreadLocalLibrary.cpp b/core/libs/tests/HostThreadLocalLibrary.cpp new file mode 100644 index 0000000000..9f11e94d27 --- /dev/null +++ b/core/libs/tests/HostThreadLocalLibrary.cpp @@ -0,0 +1,31 @@ +#include "prx/libc/include/HostThreadLocal.hpp" +#include +#include +#include + +namespace { +std::atomic destroyed{0}; +struct Value { + std::vector data = std::vector(256, 42); + ~Value() { + for (auto item : data) if (item != 42) std::abort(); + ++destroyed; + } +}; +struct First {}; +struct Second {}; +} + +#ifdef _WIN32 +#define TEST_EXPORT __declspec(dllexport) +#else +#define TEST_EXPORT +#endif + +extern "C" TEST_EXPORT void TouchHostThreadLocal() { + auto& first = HostThreadLocal(); + auto& second = HostThreadLocal(); + if (&first == &second || &first != &HostThreadLocal()) std::abort(); +} + +extern "C" TEST_EXPORT unsigned DestroyedHostThreadLocals() { return destroyed.load(); } From 93a214836ffa446564c6fc1216aa8a0f42440c60 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 13:32:36 +1300 Subject: [PATCH 69/93] fix(libkernel): build sceKernelVirtualQuery on linux The host-region fallback added with the Demon's Souls work calls VirtualQuery and includes windows.h unconditionally, so libkernel no longer compiles on Linux. On Linux, read the containing mapping and its permissions from /proc/self/maps. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- .../prx/libkernel/DirectMemory/Export.cpp | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 14d1333d8e..b39ce5addf 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -5,9 +5,14 @@ #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "DirectMemory.hpp" +#ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #define NOMINMAX #include +#else +#include +#include +#endif #include #include #include @@ -179,12 +184,33 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf } // Memory the registry does not know (the title's own heap blocks, stacks): the host's committed // region around the address is the honest extent; unmapped memory is an error, as on the PS5. + #ifdef _WIN32 MEMORY_BASIC_INFORMATION host{}; if (VirtualQuery(addr, &host, sizeof(host)) == 0 || host.State != MEM_COMMIT) return SCE_KERNEL_ERROR_EACCES; info->start = reinterpret_cast(host.BaseAddress); info->end = info->start + host.RegionSize; const bool writable = (host.Protect & (PAGE_READWRITE | PAGE_EXECUTE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_WRITECOPY)) != 0; const bool executable = (host.Protect & (PAGE_EXECUTE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY)) != 0; + #else + std::ifstream maps("/proc/self/maps"); + std::string line; + bool found = false, writable = false, executable = false; + while (std::getline(maps, line)) { + std::istringstream fields(line); + uintptr_t begin = 0, end = 0; + char dash = 0; + std::string perms; + if (!(fields >> std::hex >> begin >> dash >> end >> perms) || address < begin || address >= end) continue; + if (perms.size() < 3 || perms[0] != 'r') return SCE_KERNEL_ERROR_EACCES; + info->start = begin; + info->end = end; + writable = perms[1] == 'w'; + executable = perms[2] == 'x'; + found = true; + break; + } + if (!found) return SCE_KERNEL_ERROR_EACCES; + #endif info->protection = 1 | (writable ? 2 : 0) | (executable ? 4 : 0); info->is_flexible = 1; info->is_committed = 1; From 346b7280e260f1293592b8c4b5ee590ddd6c19ce Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 13:32:36 +1300 Subject: [PATCH 70/93] fix(relinker): skip the flagless dynlib data segment in linux guest modules GuestModuleWriter::WriteLinux adds read permission to every PT_LOAD, including the SCE dynlib data segment, which has no flags and is never mapped on the PS5. When that segment starts inside the last page of the preceding RW segment, the read-only mapping covers the tail of its bss and the module's initializer faults on its first write there. The symbol and relocation tables it holds are already copied into the appended segment, so it no longer needs to be mapped. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp index 31168ebd63..11b2348eff 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp @@ -12,8 +12,9 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma for (auto header : image.Headers) { if (header.Type != 1 && header.Type != 7 && header.Type != 0x6474e550 && header.Type != 0x6474e551) continue; if (header.Type == 1) { - header.Flags |= 4; end = std::max(end, header.MappedAddress + header.MemorySize); + if (header.Flags == 0) continue; + header.Flags |= 4; } headers.push_back(header); } From c13f0eee673bb80ccb687aefa688b6df349d83d5 Mon Sep 17 00:00:00 2001 From: MartinCajiao Date: Sun, 27 Sep 2026 19:54:19 -0500 Subject: [PATCH 71/93] feat(input): allow configurable keyboard and mouse bindings --- README.md | 4 + .../prx/libScePad/include/InputMapping.hpp | 5 +- core/libs/prx/libSceVideoOut/CMakeLists.txt | 1 + .../prx/libSceVideoOut/include/PadInput.hpp | 7 +- .../prx/libSceVideoOut/src/InputMapping.cpp | 172 ++++++++++++++++++ core/libs/prx/libSceVideoOut/src/PadInput.cpp | 33 ++-- .../prx/libSceVideoOut/src/VideoOutDriver.cpp | 2 +- docs/INPUT_MAPPING.md | 28 +++ docs/TechnicalDebt.md | 1 - 9 files changed, 233 insertions(+), 20 deletions(-) create mode 100644 core/libs/prx/libSceVideoOut/src/InputMapping.cpp create mode 100644 docs/INPUT_MAPPING.md diff --git a/README.md b/README.md index c22ffe6ca0..d54a4320bd 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ The relinker uses only the C++20 standard library and should build with any conf The project targets maximum compiler portability. Support for additional compilers will be addressed after the first successful game launch. +## Input mapping + +Keyboard and mouse controls can be configured with an `anyps5-input.ini` file. See [input mapping](docs/INPUT_MAPPING.md) for the format and behavior. + ## Disclaimer This project is intended for interoperability, research, preservation, and compatibility purposes. It does not include, distribute, or require copyrighted software, firmware, cryptographic keys, or proprietary libraries. Users are responsible for ensuring that any binaries used with this project are obtained and used in accordance with applicable laws and their respective license terms. diff --git a/core/libs/prx/libScePad/include/InputMapping.hpp b/core/libs/prx/libScePad/include/InputMapping.hpp index 67c310a19f..ea3513278e 100644 --- a/core/libs/prx/libScePad/include/InputMapping.hpp +++ b/core/libs/prx/libScePad/include/InputMapping.hpp @@ -3,6 +3,7 @@ #include #include +#include #include "SDL_scancode.h" #include "SDL_mouse.h" @@ -50,6 +51,8 @@ inline constexpr std::array InputMapping{ InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::None, InputControl::Button, PadButton::Down, -1} }; +std::vector LoadInputMapping(); + } -#endif \ No newline at end of file +#endif diff --git a/core/libs/prx/libSceVideoOut/CMakeLists.txt b/core/libs/prx/libSceVideoOut/CMakeLists.txt index 452f68c24e..e41cf34bdd 100644 --- a/core/libs/prx/libSceVideoOut/CMakeLists.txt +++ b/core/libs/prx/libSceVideoOut/CMakeLists.txt @@ -6,6 +6,7 @@ add_library(libSceVideoOut SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/src/VideoOutDriver.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/DisplayWindow.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/PadInput.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/InputMapping.cpp ) target_include_directories(libSceVideoOut PRIVATE diff --git a/core/libs/prx/libSceVideoOut/include/PadInput.hpp b/core/libs/prx/libSceVideoOut/include/PadInput.hpp index e86efd382f..99928b941e 100644 --- a/core/libs/prx/libSceVideoOut/include/PadInput.hpp +++ b/core/libs/prx/libSceVideoOut/include/PadInput.hpp @@ -5,19 +5,22 @@ #include "prx/libScePad/include/InputMapping.hpp" #include #include +#include class DisplayWindow; class PadInput { public: + PadInput(); void HandleEvent(const SDL_Event& event, DisplayWindow& window); void Update(); private: void publish(); void setMouseMode(bool enabled); - std::array pressed{}; - std::array wheelReleaseTimes{}; + std::vector bindings; + std::vector pressed; + std::vector wheelReleaseTimes; std::array mouseStick{128, 128}; std::chrono::steady_clock::time_point nextMousePoll{}; bool mouseEnabled = false; diff --git a/core/libs/prx/libSceVideoOut/src/InputMapping.cpp b/core/libs/prx/libSceVideoOut/src/InputMapping.cpp new file mode 100644 index 0000000000..3b925fc913 --- /dev/null +++ b/core/libs/prx/libSceVideoOut/src/InputMapping.cpp @@ -0,0 +1,172 @@ +#include "prx/libScePad/include/InputMapping.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SDL_keyboard.h" + +namespace { + +struct InputAction { + std::string_view name; + Pad::InputControl control; + Pad::PadButton button; +}; + +constexpr auto actions = std::array{ + InputAction{"Cross", Pad::InputControl::Button, Pad::PadButton::Cross}, + InputAction{"Circle", Pad::InputControl::Button, Pad::PadButton::Circle}, + InputAction{"Triangle", Pad::InputControl::Button, Pad::PadButton::Triangle}, + InputAction{"Square", Pad::InputControl::Button, Pad::PadButton::Square}, + InputAction{"L1", Pad::InputControl::Button, Pad::PadButton::L1}, + InputAction{"R1", Pad::InputControl::Button, Pad::PadButton::R1}, + InputAction{"L2", Pad::InputControl::Button, Pad::PadButton::L2}, + InputAction{"R2", Pad::InputControl::Button, Pad::PadButton::R2}, + InputAction{"L3", Pad::InputControl::Button, Pad::PadButton::L3}, + InputAction{"R3", Pad::InputControl::Button, Pad::PadButton::R3}, + InputAction{"Options", Pad::InputControl::Button, Pad::PadButton::Options}, + InputAction{"Up", Pad::InputControl::Button, Pad::PadButton::Up}, + InputAction{"Right", Pad::InputControl::Button, Pad::PadButton::Right}, + InputAction{"Down", Pad::InputControl::Button, Pad::PadButton::Down}, + InputAction{"Left", Pad::InputControl::Button, Pad::PadButton::Left}, + InputAction{"LeftStickLeft", Pad::InputControl::LeftStickLeft, Pad::PadButton::None}, + InputAction{"LeftStickRight", Pad::InputControl::LeftStickRight, Pad::PadButton::None}, + InputAction{"LeftStickUp", Pad::InputControl::LeftStickUp, Pad::PadButton::None}, + InputAction{"LeftStickDown", Pad::InputControl::LeftStickDown, Pad::PadButton::None}, + InputAction{"RightStickLeft", Pad::InputControl::RightStickLeft, Pad::PadButton::None}, + InputAction{"RightStickRight", Pad::InputControl::RightStickRight, Pad::PadButton::None}, + InputAction{"RightStickUp", Pad::InputControl::RightStickUp, Pad::PadButton::None}, + InputAction{"RightStickDown", Pad::InputControl::RightStickDown, Pad::PadButton::None}, + InputAction{"TouchLeft", Pad::InputControl::TouchLeft, Pad::PadButton::None}, + InputAction{"TouchRight", Pad::InputControl::TouchRight, Pad::PadButton::None}, + InputAction{"ToggleMouse", Pad::InputControl::ToggleMouse, Pad::PadButton::None}, + InputAction{"ToggleFullscreen", Pad::InputControl::ToggleFullscreen, Pad::PadButton::None} +}; + +std::string upper(std::string_view value) { + std::string result(value); + std::transform(result.begin(), result.end(), result.begin(), [](unsigned char character) { + return static_cast(std::toupper(character)); + }); + return result; +} + +std::string_view trim(std::string_view value) { + while (!value.empty() && std::isspace(static_cast(value.front())) != 0) value.remove_prefix(1); + while (!value.empty() && std::isspace(static_cast(value.back())) != 0) value.remove_suffix(1); + return value; +} + +const InputAction* findAction(std::string_view name) { + const auto normalized = upper(name); + for (const auto& action : actions) { + if (upper(action.name) == normalized) return &action; + } + return nullptr; +} + +bool matchesAction(const Pad::InputBinding& binding, const InputAction& action) { + if (action.control == Pad::InputControl::Button) { + return binding.control == Pad::InputControl::Button && binding.button == action.button; + } + return binding.control == action.control; +} + +Pad::MouseButton parseMouseButton(std::string_view name) { + const auto normalized = upper(name); + if (normalized == "LEFT") return Pad::MouseButton::Left; + if (normalized == "MIDDLE") return Pad::MouseButton::Middle; + if (normalized == "RIGHT") return Pad::MouseButton::Right; + if (normalized == "X1") return Pad::MouseButton::X1; + if (normalized == "X2") return Pad::MouseButton::X2; + throw std::runtime_error("unknown mouse button '" + std::string(name) + "'"); +} + +Pad::InputBinding parseBinding(const InputAction& action, std::string_view source) { + const auto separator = source.find(':'); + if (separator == std::string_view::npos) throw std::runtime_error("source must use TYPE:VALUE format"); + const auto type = upper(trim(source.substr(0, separator))); + const auto value = trim(source.substr(separator + 1)); + if (value.empty()) throw std::runtime_error("source value is empty"); + + if (type == "KEY") { + const std::string keyName(value); + const auto key = SDL_GetScancodeFromName(keyName.c_str()); + if (key == SDL_SCANCODE_UNKNOWN) throw std::runtime_error("unknown SDL key name '" + keyName + "'"); + return {key, Pad::MouseButton::None, action.control, action.button, 0}; + } + if (type == "MOUSE") { + if (action.control == Pad::InputControl::ToggleFullscreen) { + throw std::runtime_error("ToggleFullscreen can only use a keyboard key"); + } + return {SDL_SCANCODE_UNKNOWN, parseMouseButton(value), action.control, action.button, 0}; + } + if (type == "WHEEL") { + if (action.control != Pad::InputControl::Button) throw std::runtime_error("mouse wheel can only map to a pad button"); + const auto direction = upper(value); + if (direction == "UP") return {SDL_SCANCODE_UNKNOWN, Pad::MouseButton::None, action.control, action.button, 1}; + if (direction == "DOWN") return {SDL_SCANCODE_UNKNOWN, Pad::MouseButton::None, action.control, action.button, -1}; + throw std::runtime_error("mouse wheel direction must be UP or DOWN"); + } + throw std::runtime_error("source type must be KEY, MOUSE or WHEEL"); +} + +[[noreturn]] void invalidLine(const std::filesystem::path& path, std::size_t line, const std::string& reason) { + throw std::runtime_error("Pad: invalid input mapping " + path.string() + ":" + std::to_string(line) + ": " + reason); +} + +} + +std::vector Pad::LoadInputMapping() { + std::vector bindings(InputMapping.begin(), InputMapping.end()); + const char* configuredPath = std::getenv("ANYPS5_INPUT_CONFIG"); + const bool explicitPath = configuredPath != nullptr && configuredPath[0] != '\0'; + const std::filesystem::path path = explicitPath ? configuredPath : "anyps5-input.ini"; + std::ifstream file(path); + if (!file) { + if (explicitPath || std::filesystem::exists(path)) { + throw std::runtime_error("Pad: cannot read input mapping '" + path.string() + "'"); + } + return bindings; + } + + std::unordered_set overriddenActions; + std::string line; + std::size_t lineNumber = 0; + while (std::getline(file, line)) { + ++lineNumber; + const auto comment = line.find_first_of("#;"); + const std::string_view content = trim(std::string_view(line).substr(0, comment)); + if (content.empty()) continue; + const auto separator = content.find('='); + if (separator == std::string_view::npos) invalidLine(path, lineNumber, "expected Action = TYPE:VALUE"); + + const auto actionName = trim(content.substr(0, separator)); + const auto source = trim(content.substr(separator + 1)); + const auto* action = findAction(actionName); + if (action == nullptr) invalidLine(path, lineNumber, "unknown action '" + std::string(actionName) + "'"); + if (source.empty()) invalidLine(path, lineNumber, "source is empty"); + + try { + auto binding = parseBinding(*action, source); + const auto normalizedAction = upper(action->name); + if (overriddenActions.insert(normalizedAction).second) { + std::erase_if(bindings, [action](const InputBinding& existing) { return matchesAction(existing, *action); }); + } + bindings.push_back(binding); + } catch (const std::runtime_error& error) { + invalidLine(path, lineNumber, error.what()); + } + } + if (file.bad()) throw std::runtime_error("Pad: cannot read input mapping '" + path.string() + "'"); + + return bindings; +} diff --git a/core/libs/prx/libSceVideoOut/src/PadInput.cpp b/core/libs/prx/libSceVideoOut/src/PadInput.cpp index 6631fb5a93..ff08df7518 100644 --- a/core/libs/prx/libSceVideoOut/src/PadInput.cpp +++ b/core/libs/prx/libSceVideoOut/src/PadInput.cpp @@ -10,6 +10,9 @@ #include "prx/libScePad/include/PadState.hpp" #include "prx/libScePad/include/PadInputTypes.hpp" +PadInput::PadInput() + : bindings(Pad::LoadInputMapping()), pressed(bindings.size()), wheelReleaseTimes(bindings.size()) {} + void PadInput::setMouseMode(bool enabled) { if (SDL_SetRelativeMouseMode(enabled ? SDL_TRUE : SDL_FALSE) != 0) throw std::runtime_error(std::string("Pad: relative mouse mode failed: ") + SDL_GetError()); int deltaX = 0; @@ -22,8 +25,8 @@ void PadInput::setMouseMode(bool enabled) { void PadInput::HandleEvent(const SDL_Event& event, DisplayWindow& window) { if (event.type == SDL_WINDOWEVENT && (event.window.event == SDL_WINDOWEVENT_FOCUS_LOST || event.window.event == SDL_WINDOWEVENT_CLOSE)) { - pressed.fill(false); - wheelReleaseTimes.fill({}); + std::fill(pressed.begin(), pressed.end(), false); + std::fill(wheelReleaseTimes.begin(), wheelReleaseTimes.end(), std::chrono::steady_clock::time_point{}); if (mouseEnabled) setMouseMode(false); publish(); return; @@ -37,8 +40,8 @@ void PadInput::HandleEvent(const SDL_Event& event, DisplayWindow& window) { if (event.wheel.direction == SDL_MOUSEWHEEL_FLIPPED) direction = -direction; if (direction == 0) return; const auto releaseTime = std::chrono::steady_clock::now() + std::chrono::milliseconds(Pad::WheelPressDurationMs); - for (std::size_t index = 0; index < Pad::InputMapping.size(); ++index) { - const auto& binding = Pad::InputMapping[index]; + for (std::size_t index = 0; index < bindings.size(); ++index) { + const auto& binding = bindings[index]; if (binding.wheelDirection == 0) continue; pressed[index] = binding.wheelDirection == direction; wheelReleaseTimes[index] = pressed[index] ? releaseTime : std::chrono::steady_clock::time_point{}; @@ -52,11 +55,11 @@ void PadInput::HandleEvent(const SDL_Event& event, DisplayWindow& window) { if (ignoreInput) return; if (keyboard && event.key.repeat != 0) return; const bool down = event.type == SDL_KEYDOWN || event.type == SDL_MOUSEBUTTONDOWN; - for (std::size_t index = 0; index < Pad::InputMapping.size(); ++index) { - const auto& binding = Pad::InputMapping[index]; - const bool keyMatches = binding.key != SDL_SCANCODE_UNKNOWN && binding.key == event.key.keysym.scancode; - const bool mouseMatches = binding.mouseButton != Pad::MouseButton::None && binding.mouseButton == static_cast(event.button.button); - const bool matches = keyboard ? keyMatches : mouseMatches; + for (std::size_t index = 0; index < bindings.size(); ++index) { + const auto& binding = bindings[index]; + const bool matches = keyboard + ? binding.key != SDL_SCANCODE_UNKNOWN && binding.key == event.key.keysym.scancode + : binding.mouseButton != Pad::MouseButton::None && binding.mouseButton == static_cast(event.button.button); if (!matches) continue; if (binding.control == Pad::InputControl::ToggleFullscreen) { @@ -71,8 +74,8 @@ void PadInput::HandleEvent(const SDL_Event& event, DisplayWindow& window) { void PadInput::Update() { const auto now = std::chrono::steady_clock::now(); bool released = false; - for (std::size_t index = 0; index < Pad::InputMapping.size(); ++index) { - if (Pad::InputMapping[index].wheelDirection == 0 || !pressed[index] || now < wheelReleaseTimes[index]) continue; + for (std::size_t index = 0; index < bindings.size(); ++index) { + if (bindings[index].wheelDirection == 0 || !pressed[index] || now < wheelReleaseTimes[index]) continue; pressed[index] = false; wheelReleaseTimes[index] = {}; released = true; @@ -80,8 +83,8 @@ void PadInput::Update() { if (released) publish(); if (!mouseEnabled) return; if (SDL_GetKeyboardFocus() == nullptr) { - pressed.fill(false); - wheelReleaseTimes.fill({}); + std::fill(pressed.begin(), pressed.end(), false); + std::fill(wheelReleaseTimes.begin(), wheelReleaseTimes.end(), std::chrono::steady_clock::time_point{}); setMouseMode(false); publish(); return; @@ -105,9 +108,9 @@ void PadInput::publish() { PadInputState state; std::array negative{}; std::array positive{}; - for (std::size_t index = 0; index < Pad::InputMapping.size(); ++index) { + for (std::size_t index = 0; index < bindings.size(); ++index) { if (!pressed[index]) continue; - const auto& binding = Pad::InputMapping[index]; + const auto& binding = bindings[index]; switch (binding.control) { case Pad::InputControl::Button: state.buttons |= static_cast(binding.button); break; case Pad::InputControl::LeftStickLeft: negative[0] = true; break; diff --git a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp index a929e098b0..f1ad469269 100644 --- a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp +++ b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp @@ -463,8 +463,8 @@ void VideoOutDriver::processFlip(FlipRequest& req) { void VideoOutDriver::presentLoop(std::stop_token token) { std::shared_ptr current; - PadInput padInput; try { + PadInput padInput; while (!token.stop_requested()) { { std::unique_lock lock(flipQueue->mutex); diff --git a/docs/INPUT_MAPPING.md b/docs/INPUT_MAPPING.md new file mode 100644 index 0000000000..3904986f9d --- /dev/null +++ b/docs/INPUT_MAPPING.md @@ -0,0 +1,28 @@ +# Keyboard and mouse mapping + +AnyPS5 uses its built-in keyboard and mouse bindings when no configuration file is present. To change selected bindings, create `anyps5-input.ini` in the game's working directory. Set `ANYPS5_INPUT_CONFIG` to use a file at another path. + +Each non-empty line has the form `Action = Type:Value`. Action names are case-insensitive. A `#` or `;` starts a comment. The first line for an action replaces its built-in bindings; later lines for the same action add alternate inputs. Actions omitted from the file keep their built-in bindings. + +Supported input sources are: + +- `KEY:Return`, `KEY:Space`, or another key name accepted by SDL. +- `MOUSE:Left`, `MOUSE:Middle`, `MOUSE:Right`, `MOUSE:X1`, or `MOUSE:X2`. +- `WHEEL:Up` or `WHEEL:Down` for pad buttons. + +Supported actions are `Cross`, `Circle`, `Triangle`, `Square`, `L1`, `R1`, `L2`, `R2`, `L3`, `R3`, `Options`, `Up`, `Right`, `Down`, `Left`, `LeftStickLeft`, `LeftStickRight`, `LeftStickUp`, `LeftStickDown`, `RightStickLeft`, `RightStickRight`, `RightStickUp`, `RightStickDown`, `TouchLeft`, `TouchRight`, `ToggleMouse`, and `ToggleFullscreen`. + +For example, this changes Cross to F, adds IJKL alternatives for the left stick, uses the mouse buttons for Square and R2, and keeps all other built-in bindings: + +```ini +Cross = KEY:F +LeftStickLeft = KEY:J +LeftStickRight = KEY:L +LeftStickUp = KEY:I +LeftStickDown = KEY:K +Square = MOUSE:Left +R2 = MOUSE:Right +ToggleMouse = MOUSE:Middle +``` + +An invalid line reports the file and line number and stops input initialization. If the configured file does not exist or cannot be read, AnyPS5 reports an error. With no `anyps5-input.ini` and no `ANYPS5_INPUT_CONFIG`, the built-in mapping is used. diff --git a/docs/TechnicalDebt.md b/docs/TechnicalDebt.md index 6156464b1a..87b34664bc 100644 --- a/docs/TechnicalDebt.md +++ b/docs/TechnicalDebt.md @@ -30,7 +30,6 @@ Throughout the project, every function at every stage either **does exactly what ### Functional -- There's no way to specify keyboard and mouse input mapping when using a gamepad. The [default mapping](../core/libs/prx/libScePad/include/InputMapping.hpp) is always used. - [Shader recompilation](../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../core/shader/recompiler) was written completely independently from [libs/prx](../core/libs/prx). - The executable file that [relinker](../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) generates opens the console when launched, which is inconvenient for playability. - [Relinker](../core/relinker/elfpatcher/src) doesn't add an icon to the generated executable. This should be done without adding dependencies (only standard). From 6dfa449e52fd3622f2de438dbb68807bc9fe533c Mon Sep 17 00:00:00 2001 From: MartinCajiao Date: Sun, 27 Sep 2026 19:58:38 -0500 Subject: [PATCH 72/93] fix(input): handle UTF-8 BOM and duplicate bindings --- core/libs/prx/libSceVideoOut/src/InputMapping.cpp | 8 +++++++- docs/INPUT_MAPPING.md | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceVideoOut/src/InputMapping.cpp b/core/libs/prx/libSceVideoOut/src/InputMapping.cpp index 3b925fc913..33df257b3a 100644 --- a/core/libs/prx/libSceVideoOut/src/InputMapping.cpp +++ b/core/libs/prx/libSceVideoOut/src/InputMapping.cpp @@ -143,6 +143,7 @@ std::vector Pad::LoadInputMapping() { std::size_t lineNumber = 0; while (std::getline(file, line)) { ++lineNumber; + if (lineNumber == 1 && line.starts_with("\xEF\xBB\xBF")) line.erase(0, 3); const auto comment = line.find_first_of("#;"); const std::string_view content = trim(std::string_view(line).substr(0, comment)); if (content.empty()) continue; @@ -161,7 +162,12 @@ std::vector Pad::LoadInputMapping() { if (overriddenActions.insert(normalizedAction).second) { std::erase_if(bindings, [action](const InputBinding& existing) { return matchesAction(existing, *action); }); } - bindings.push_back(binding); + const bool duplicate = std::any_of(bindings.begin(), bindings.end(), [&binding](const InputBinding& existing) { + return existing.key == binding.key && existing.mouseButton == binding.mouseButton && + existing.control == binding.control && existing.button == binding.button && + existing.wheelDirection == binding.wheelDirection; + }); + if (!duplicate) bindings.push_back(binding); } catch (const std::runtime_error& error) { invalidLine(path, lineNumber, error.what()); } diff --git a/docs/INPUT_MAPPING.md b/docs/INPUT_MAPPING.md index 3904986f9d..e79ffd25a5 100644 --- a/docs/INPUT_MAPPING.md +++ b/docs/INPUT_MAPPING.md @@ -2,7 +2,7 @@ AnyPS5 uses its built-in keyboard and mouse bindings when no configuration file is present. To change selected bindings, create `anyps5-input.ini` in the game's working directory. Set `ANYPS5_INPUT_CONFIG` to use a file at another path. -Each non-empty line has the form `Action = Type:Value`. Action names are case-insensitive. A `#` or `;` starts a comment. The first line for an action replaces its built-in bindings; later lines for the same action add alternate inputs. Actions omitted from the file keep their built-in bindings. +Each non-empty line has the form `Action = Type:Value`. Action names are case-insensitive. A `#` or `;` starts a comment. The first line for an action replaces its built-in bindings; later lines for the same action add alternate inputs. Duplicate entries for the same action are ignored. Actions omitted from the file keep their built-in bindings. UTF-8 files with or without a byte-order mark are supported. Supported input sources are: @@ -12,10 +12,11 @@ Supported input sources are: Supported actions are `Cross`, `Circle`, `Triangle`, `Square`, `L1`, `R1`, `L2`, `R2`, `L3`, `R3`, `Options`, `Up`, `Right`, `Down`, `Left`, `LeftStickLeft`, `LeftStickRight`, `LeftStickUp`, `LeftStickDown`, `RightStickLeft`, `RightStickRight`, `RightStickUp`, `RightStickDown`, `TouchLeft`, `TouchRight`, `ToggleMouse`, and `ToggleFullscreen`. -For example, this changes Cross to F, adds IJKL alternatives for the left stick, uses the mouse buttons for Square and R2, and keeps all other built-in bindings: +For example, this changes Cross to F or Space, moves the left stick to IJKL, uses the mouse buttons for Square and R2, and keeps all other built-in bindings: ```ini Cross = KEY:F +Cross = KEY:Space LeftStickLeft = KEY:J LeftStickRight = KEY:L LeftStickUp = KEY:I From 1007f6d64ec22467bbd6e381adafbfbee3c47aa1 Mon Sep 17 00:00:00 2001 From: MartinCajiao Date: Sun, 27 Sep 2026 20:20:17 -0500 Subject: [PATCH 73/93] fix(input): load per-game config beside executable --- core/libs/prx/libSceVideoOut/src/InputMapping.cpp | 10 +++++++++- docs/INPUT_MAPPING.md | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceVideoOut/src/InputMapping.cpp b/core/libs/prx/libSceVideoOut/src/InputMapping.cpp index 33df257b3a..d3d3d72752 100644 --- a/core/libs/prx/libSceVideoOut/src/InputMapping.cpp +++ b/core/libs/prx/libSceVideoOut/src/InputMapping.cpp @@ -6,12 +6,14 @@ #include #include #include +#include #include #include #include #include #include "SDL_keyboard.h" +#include "SDL_filesystem.h" namespace { @@ -123,13 +125,19 @@ Pad::InputBinding parseBinding(const InputAction& action, std::string_view sourc throw std::runtime_error("Pad: invalid input mapping " + path.string() + ":" + std::to_string(line) + ": " + reason); } +std::filesystem::path defaultConfigPath() { + std::unique_ptr basePath(SDL_GetBasePath(), SDL_free); + if (basePath) return std::filesystem::path(basePath.get()) / "anyps5-input.ini"; + return "anyps5-input.ini"; +} + } std::vector Pad::LoadInputMapping() { std::vector bindings(InputMapping.begin(), InputMapping.end()); const char* configuredPath = std::getenv("ANYPS5_INPUT_CONFIG"); const bool explicitPath = configuredPath != nullptr && configuredPath[0] != '\0'; - const std::filesystem::path path = explicitPath ? configuredPath : "anyps5-input.ini"; + const std::filesystem::path path = explicitPath ? configuredPath : defaultConfigPath(); std::ifstream file(path); if (!file) { if (explicitPath || std::filesystem::exists(path)) { diff --git a/docs/INPUT_MAPPING.md b/docs/INPUT_MAPPING.md index e79ffd25a5..da678ffbd8 100644 --- a/docs/INPUT_MAPPING.md +++ b/docs/INPUT_MAPPING.md @@ -1,6 +1,6 @@ # Keyboard and mouse mapping -AnyPS5 uses its built-in keyboard and mouse bindings when no configuration file is present. To change selected bindings, create `anyps5-input.ini` in the game's working directory. Set `ANYPS5_INPUT_CONFIG` to use a file at another path. +AnyPS5 uses its built-in keyboard and mouse bindings when no configuration file is present. To change selected bindings, create `anyps5-input.ini` beside the generated game executable. Set `ANYPS5_INPUT_CONFIG` to use a file at another path. Each non-empty line has the form `Action = Type:Value`. Action names are case-insensitive. A `#` or `;` starts a comment. The first line for an action replaces its built-in bindings; later lines for the same action add alternate inputs. Duplicate entries for the same action are ignored. Actions omitted from the file keep their built-in bindings. UTF-8 files with or without a byte-order mark are supported. From 06b3413961246fd41db99a0edfd5ce9ad085921a Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 14:22:48 +1300 Subject: [PATCH 74/93] fix(relinker): align the linux header load segment to the largest segment alignment Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/relinker/CMakeLists.txt | 2 + .../general/ProgramHeaderLayoutBuilder.cpp | 6 +- .../tests/test_linux_load_alignment.py | 56 +++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 core/relinker/relinker/tests/test_linux_load_alignment.py diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 42ebb17ba2..f8a4a4a096 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -140,6 +140,8 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_empty_tls.py $) add_test(NAME tls_function_coverage COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_tls_function_coverage.py $) + add_test(NAME linux_load_alignment COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_load_alignment.py $) else() message(STATUS "Python3 interpreter not found; skipping Python relinker tests") endif() diff --git a/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp b/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp index 36245adeef..b12f1ad90f 100644 --- a/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp +++ b/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp @@ -144,7 +144,11 @@ std::uint16_t ProgramHeaderLayoutBuilder::WriteLayout( ", available " + std::to_string(request.PhNum)); const std::uint64_t headerBlockSize = request.PhOff + static_cast(neededPh) * request.PhEntSize; - const std::uint64_t headerBlockAlign = kDefaultLoadAlignment; + std::uint64_t headerBlockAlign = kDefaultLoadAlignment; + for (const auto& ph : request.OriginalHeaders) { + if (ph.Type == PT_LOAD && !_segmentFilter->ShouldSkip(ph) && ph.Alignment > headerBlockAlign && (ph.Alignment & (ph.Alignment - 1)) == 0) + headerBlockAlign = ph.Alignment; + } const std::uint64_t headerBlockVaddr = (request.ExtraBlockVaddr + request.ExtraBlockSize + headerBlockAlign - 1) & ~(headerBlockAlign - 1); diff --git a/core/relinker/relinker/tests/test_linux_load_alignment.py b/core/relinker/relinker/tests/test_linux_load_alignment.py new file mode 100644 index 0000000000..2e2d8afaf9 --- /dev/null +++ b/core/relinker/relinker/tests/test_linux_load_alignment.py @@ -0,0 +1,56 @@ +"""Check that a Linux relink keeps the image base aligned to the guest segment alignment.""" + +from pathlib import Path +import struct +import subprocess +import sys +import tempfile + +PT_LOAD = 1 +PT_SCE_VERSION = 0x6FFFFF01 + + +def fixture(): + image = bytearray(0x8000) + image[:16] = b"\x7fELF\x02\x01\x01" + bytes(9) + struct.pack_into(" Date: Mon, 28 Sep 2026 14:22:48 +1300 Subject: [PATCH 75/93] feat(libkernel): hinted and no-overwrite mappings, virtual range names and usleep Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- .../libkernel/DirectMemory/DirectMemory.cpp | 65 ++++++++++++++++-- .../prx/libkernel/DirectMemory/Export.cpp | 68 ++++++++++++++++--- core/libs/prx/libkernel/Time/Time.cpp | 5 ++ core/libs/tests/GuestMemory.cpp | 41 +++++++++++ 4 files changed, 165 insertions(+), 14 deletions(-) diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index 2d0569a6e0..c003b35649 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -15,6 +15,10 @@ #if defined(__linux__) #include +#include +#include +#include +#include #else #include @@ -128,6 +132,37 @@ static int mprotect(void* addr, size_t len, int prot) { namespace { +constexpr int GuestMapFixedFlag = 0x10; + +#if defined(__linux__) +void* MapAtOrAbove(std::uintptr_t start, size_t len, int prot, size_t alignment) { + constexpr std::uintptr_t UserLimit = 0x7fff00000000ull; + for (int attempt = 0; attempt < 8; ++attempt) { + std::vector> used; + std::ifstream maps("/proc/self/maps"); + std::string line; + while (std::getline(maps, line)) { + std::istringstream fields(line); + std::uintptr_t begin = 0, end = 0; + char dash = 0; + if (fields >> std::hex >> begin >> dash >> end) used.emplace_back(begin, end); + } + std::sort(used.begin(), used.end()); + auto candidate = (start + alignment - 1) & ~(static_cast(alignment) - 1); + for (const auto& [begin, end] : used) { + if (end <= candidate) continue; + if (begin >= candidate + len) break; + candidate = (end + alignment - 1) & ~(static_cast(alignment) - 1); + } + if (candidate + len > UserLimit || candidate + len < candidate) throw std::runtime_error("No free range above the mapping address hint"); + void* result = mmap(reinterpret_cast(candidate), len, prot, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (result != MAP_FAILED) return result; + if (errno != EEXIST) throw std::system_error(errno, std::generic_category(), "Hinted mmap failed"); + } + throw std::runtime_error("Hinted mmap kept racing with other mappings"); +} +#endif + void ValidateLength(size_t len) { if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) { // return SCE_KERNEL_ERROR_EINVAL; @@ -182,7 +217,8 @@ void Trace(const char* format, ...) { bool RemapFixedIntoRegistered(GuestAllocations::Mutation& mutation, void* addr, size_t len, int prot, int flags, int64_t physStart = -1) { constexpr int GuestMapFixed = 0x10; - if (addr == nullptr || (flags & GuestMapFixed) == 0 || !mutation.Covers(addr, len)) return false; + constexpr int GuestMapNoOverwrite = 0x80; + if (addr == nullptr || (flags & GuestMapFixed) == 0 || (flags & GuestMapNoOverwrite) != 0 || !mutation.Covers(addr, len)) return false; ValidateRange(addr, len, PS5_PAGE_SIZE); Trace("remap fixed %p+0x%zx prot=0x%x phys=0x%llx", addr, len, prot, static_cast(physStart)); const auto nativeProtection = LinuxProtFromSce(prot); @@ -208,13 +244,26 @@ void* MapAligned(void* addr, size_t len, int prot, int flags, size_t alignment) ValidateLength(len); alignment = ValidateAlignment(alignment); constexpr int GuestMapFixed = 0x10; + constexpr int GuestMapNoOverwrite = 0x80; constexpr int GuestMapNoCoalesce = 0x400000; - if ((flags & ~(GuestMapFixed | GuestMapNoCoalesce)) != 0) { - throw std::invalid_argument("Unsupported memory mapping flags"); +#if defined(__linux__) + constexpr int SupportedFlags = GuestMapFixed | GuestMapNoOverwrite | GuestMapNoCoalesce; +#else + constexpr int SupportedFlags = GuestMapFixed | GuestMapNoCoalesce; +#endif + if ((flags & ~SupportedFlags) != 0) { + char message[64]; + std::snprintf(message, sizeof(message), "Unsupported memory mapping flags 0x%x", flags); + throw std::invalid_argument(message); } if ((flags & GuestMapFixed) != 0) { ValidateRange(addr, len, alignment); - void* result = mmap(addr, len, prot, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); +#if defined(__linux__) + const int placement = (flags & GuestMapNoOverwrite) != 0 ? MAP_FIXED_NOREPLACE : MAP_FIXED; +#else + const int placement = MAP_FIXED; +#endif + void* result = mmap(addr, len, prot, MAP_PRIVATE | MAP_ANONYMOUS | placement, -1, 0); if (result == MAP_FAILED) { // return SCE_KERNEL_ERROR_ENOMEM; throw std::system_error(errno, std::generic_category(), "Fixed mmap failed"); @@ -222,7 +271,11 @@ void* MapAligned(void* addr, size_t len, int prot, int flags, size_t alignment) return result; } if (addr) { +#if defined(__linux__) + return MapAtOrAbove(reinterpret_cast(addr), len, prot, alignment); +#else throw std::invalid_argument("Non-fixed mapping address hints are not implemented"); +#endif } #ifdef _WIN32 return mmap_aligned(len, prot, alignment); @@ -270,7 +323,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, } GuestAllocations::Mutation mutation; if (RemapFixedIntoRegistered(mutation, *addr, len, prot, flags)) return 0; - if (*addr != nullptr) mutation.RequireAvailable(*addr, len); + if (*addr != nullptr && (flags & GuestMapFixedFlag) != 0) mutation.RequireAvailable(*addr, len); void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, alignment); try { mutation.Add(mapped, len, (prot & 3) != 0, (prot & 2) != 0); @@ -288,7 +341,7 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags) { if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL; GuestAllocations::Mutation mutation; if (RemapFixedIntoRegistered(mutation, *addr, len, prot, flags)) return 0; - if (*addr != nullptr) mutation.RequireAvailable(*addr, len); + if (*addr != nullptr && (flags & GuestMapFixedFlag) != 0) mutation.RequireAvailable(*addr, len); void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, PS5_PAGE_SIZE); try { mutation.Add(mapped, len, (prot & 3) != 0, (prot & 2) != 0); diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index b39ce5addf..9a9d4eb6a8 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,42 @@ struct PrtAperture { size_t length; }; +struct NamedRange { + uintptr_t end; + std::string name; +}; + +std::mutex g_rangeNameLock; +std::map g_rangeNames; + +void EraseRangeNames(uintptr_t start, uintptr_t end) { + auto it = g_rangeNames.lower_bound(start); + if (it != g_rangeNames.begin() && std::prev(it)->second.end > start) --it; + while (it != g_rangeNames.end() && it->first < end) { + const auto rangeStart = it->first; + const auto range = it->second; + it = g_rangeNames.erase(it); + if (rangeStart < start) g_rangeNames.emplace(rangeStart, NamedRange{start, range.name}); + if (range.end > end) it = g_rangeNames.emplace(end, NamedRange{range.end, range.name}).first; + } +} + +void ApplyRangeName(uintptr_t address, VirtualQueryInfo* info) { + std::lock_guard lock(g_rangeNameLock); + auto next = g_rangeNames.upper_bound(address); + if (next != g_rangeNames.begin()) { + const auto containing = std::prev(next); + if (address < containing->second.end) { + info->start = std::max(info->start, containing->first); + info->end = std::min(info->end, containing->second.end); + std::strncpy(info->name, containing->second.name.c_str(), sizeof(info->name) - 1); + return; + } + info->start = std::max(info->start, containing->second.end); + } + if (next != g_rangeNames.end()) info->end = std::min(info->end, next->first); +} + std::mutex g_prtLock; PrtAperture g_prtApertures[PRT_APERTURE_COUNT] = {}; @@ -127,14 +164,18 @@ int APS5_VABI sceKernelMapFlexibleMemory(void** addr_in_out, size_t len, int pro return _mapFlexible(addr_in_out, len, prot, flags); } +int APS5_VABI sceKernelSetVirtualRangeName(const void* addr, uint64_t len, const char* name); + int APS5_VABI sceKernelMapNamedDirectMemory(void** addr, size_t len, int prot, int flags, int64_t direct_memory_start, size_t alignment, const char* name) { - (void)name; - return DoMapDirect(addr, len, prot, flags, direct_memory_start, alignment); + const int result = DoMapDirect(addr, len, prot, flags, direct_memory_start, alignment); + if (result == 0 && name) sceKernelSetVirtualRangeName(*addr, len, name); + return result; } int32_t APS5_VABI sceKernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags, const char* name) { - (void)name; - return _mapFlexible(addr_in_out, len, prot, flags); + const int result = _mapFlexible(addr_in_out, len, prot, flags); + if (result == 0 && name) sceKernelSetVirtualRangeName(*addr_in_out, len, name); + return result; } int APS5_VABI sceKernelMprotect(const void* addr, size_t len, int prot) { @@ -180,6 +221,7 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf info->protection = (best->readable ? 1 : 0) | (best->writable ? 2 : 0) | (!best->releasable ? 4 : 0); info->is_direct = best->releasable ? 1u : 0u; info->is_committed = 1; + ApplyRangeName(std::max(address, info->start), info); return 0; } // Memory the registry does not know (the title's own heap blocks, stacks): the host's committed @@ -214,6 +256,7 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf info->protection = 1 | (writable ? 2 : 0) | (executable ? 4 : 0); info->is_flexible = 1; info->is_committed = 1; + ApplyRangeName(std::max(address, info->start), info); return 0; } @@ -268,10 +311,19 @@ int APS5_VABI sceKernelConfiguredFlexibleMemorySize(size_t* size) { } int APS5_VABI sceKernelSetVirtualRangeName(const void* addr, uint64_t len, const char* name) { - (void)addr; - (void)len; - (void)name; - NotImplemented_nid_no_patch(__func__); + const auto start = reinterpret_cast(addr); + if (!addr || len == 0 || !name || len > UINTPTR_MAX - start) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(g_rangeNameLock); + EraseRangeNames(start, start + len); + g_rangeNames.emplace(start, NamedRange{start + len, std::string(name, strnlen(name, 31))}); + return 0; +} + +int APS5_VABI sceKernelClearVirtualRangeName(const void* addr, uint64_t len) { + const auto start = reinterpret_cast(addr); + if (!addr || len == 0 || len > UINTPTR_MAX - start) return SCE_KERNEL_ERROR_EINVAL; + std::lock_guard lock(g_rangeNameLock); + EraseRangeNames(start, start + len); return 0; } diff --git a/core/libs/prx/libkernel/Time/Time.cpp b/core/libs/prx/libkernel/Time/Time.cpp index f379bcf0a4..2d2ca3cc57 100644 --- a/core/libs/prx/libkernel/Time/Time.cpp +++ b/core/libs/prx/libkernel/Time/Time.cpp @@ -245,6 +245,11 @@ int APS5_VABI _nanosleep_nid_postfix(const KernelTimespec* rqtp, KernelTimespec* return sceKernelNanosleep(rqtp, rmtp); } +int APS5_VABI usleep_nid_postfix(KernelUseconds microseconds) { + TimedWait::SleepNanos(static_cast(microseconds) * 1000ULL); + return 0; +} + int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp) { if (tp == nullptr) { APS5_INVALID_ARG_EX; diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index e0e906e3d7..a1953e7b0d 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -1,5 +1,8 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libc/include/GuestAllocations.hpp" +#include "SceTypes.hpp" +#include +#include #include #include #include @@ -9,6 +12,12 @@ extern "C" { void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept; int APS5_VABI munmap_nid_postfix(void*, std::size_t) noexcept; int* APS5_VABI __error_nid_postfix(); +int APS5_VABI sceKernelMapNamedFlexibleMemory(void**, std::size_t, int, int, const char*); +int APS5_VABI sceKernelMapFlexibleMemory(void**, std::size_t, int, int); +int APS5_VABI sceKernelMunmap(void*, std::size_t); +int APS5_VABI sceKernelVirtualQuery(const void*, int, VirtualQueryInfo*, std::uint64_t); +int APS5_VABI sceKernelSetVirtualRangeName(const void*, std::uint64_t, const char*); +int APS5_VABI sceKernelClearVirtualRangeName(const void*, std::uint64_t); } static void Require(bool condition) { @@ -18,7 +27,39 @@ static void Require(bool condition) { } } +static const char* NameAt(const void* address) { + static VirtualQueryInfo info; + Require(sceKernelVirtualQuery(address, 0, &info, sizeof(info)) == 0); + return info.name; +} + +static void CheckNamedAndHintedMappings() { + constexpr std::size_t length = 0x10000; + void* first = nullptr; + Require(sceKernelMapNamedFlexibleMemory(&first, length, 3, 0, "first mapping") == 0); + Require(std::strcmp(NameAt(first), "first mapping") == 0); + auto* middle = static_cast(first) + 0x4000; + Require(sceKernelSetVirtualRangeName(middle, 0x4000, "middle") == 0); + Require(std::strcmp(NameAt(first), "first mapping") == 0); + Require(std::strcmp(NameAt(middle), "middle") == 0); + Require(sceKernelClearVirtualRangeName(first, length) == 0); + Require(NameAt(middle)[0] == '\0'); + Require(sceKernelSetVirtualRangeName(nullptr, length, "x") != 0); +#if defined(__linux__) + void* hinted = first; + Require(sceKernelMapFlexibleMemory(&hinted, length, 3, 0) == 0); + Require(hinted > first && (reinterpret_cast(hinted) & 0x3fff) == 0); + bool rejected = false; + void* overwrite = first; + try { sceKernelMapFlexibleMemory(&overwrite, 0x4000, 3, 0x90); } catch (const std::exception&) { rejected = true; } + Require(rejected && overwrite == first); + Require(sceKernelMunmap(hinted, length) == 0); +#endif + Require(sceKernelMunmap(first, length) == 0); +} + int main() { + CheckNamedAndHintedMappings(); constexpr std::size_t page = 0x4000; const auto failed = reinterpret_cast(static_cast(-1)); const auto reject = [&](std::size_t length, int protection, int flags, int fd, From 3058980d96f9f0b043407fdf72c3beb5ab43960d Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 14:22:48 +1300 Subject: [PATCH 76/93] feat(libc): mspace memalign, malloc stats, thread-unsafe flag and nested mspaces Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libc/src/Mspace.cpp | 61 +++++++++++++++++++++++++++++-- core/libs/tests/Mspace.cpp | 33 +++++++++++++++++ 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libc/src/Mspace.cpp b/core/libs/prx/libc/src/Mspace.cpp index 5e1119fa5d..c718fed932 100644 --- a/core/libs/prx/libc/src/Mspace.cpp +++ b/core/libs/prx/libc/src/Mspace.cpp @@ -18,7 +18,10 @@ struct alignas(16) Arena { Arena* next; Block* first; std::uintptr_t end; + std::size_t inUse; + std::size_t peakInUse; }; +constexpr unsigned MspaceThreadUnsafe = 1; std::mutex arenaMutex; Arena* arenas = nullptr; @@ -59,12 +62,47 @@ void* Allocate(Arena* arena, std::size_t size, std::size_t alignment) { } block->pointer = reinterpret_cast(aligned); block->used = size; + arena->inUse += sizeof(Block) + block->capacity; + arena->peakInUse = std::max(arena->peakInUse, arena->inUse); return block->pointer; } Error(12); return nullptr; } +struct MallocManagedSize { + std::uint16_t size; + std::uint16_t version; + std::uint32_t reserved; + std::size_t maxSystemSize; + std::size_t currentSystemSize; + std::size_t maxInuseSize; + std::size_t currentInuseSize; +}; +static_assert(sizeof(MallocManagedSize) == 0x28); + +int FillStats(void* handle, MallocManagedSize* stats) { + if (!stats || stats->size < sizeof(MallocManagedSize)) return 22; + std::lock_guard lock(arenaMutex); + auto* arena = Find(handle); + if (!arena) return 22; + const auto system = arena->end - reinterpret_cast(arena); + stats->maxSystemSize = system; + stats->currentSystemSize = system; + stats->maxInuseSize = arena->peakInUse; + stats->currentInuseSize = arena->inUse; + return 0; +} + +bool InsideAllocation(Arena* arena, std::uintptr_t start, std::size_t size) { + for (auto* block = arena->first; block; block = block->next) { + const auto pointer = reinterpret_cast(block->pointer); + if (pointer && start >= pointer && size <= block->used && start - pointer <= block->used - size) return true; + } + return false; +} + void Release(Arena* arena, Block* released) { + arena->inUse -= sizeof(Block) + released->capacity; released->pointer = nullptr; released->used = 0; for (auto* block = arena->first; block && block->next;) { @@ -82,13 +120,13 @@ void* APS5_VABI sceLibcMspaceCreate_nid_postfix(const char* name, void* base, (void)name; const auto start = reinterpret_cast(base); if (!base || (start & 15) || size < sizeof(Arena) + sizeof(Block) + 16 || - size > std::numeric_limits::max() - start || flags != 0) { + size > std::numeric_limits::max() - start || (flags & ~MspaceThreadUnsafe) != 0) { Error(22); return nullptr; } std::lock_guard lock(arenaMutex); for (auto* arena = arenas; arena; arena = arena->next) { - if (start < arena->end && reinterpret_cast(arena) < start + size) { + if (start < arena->end && reinterpret_cast(arena) < start + size && !InsideAllocation(arena, start, size)) { Error(22); return nullptr; } @@ -96,7 +134,7 @@ void* APS5_VABI sceLibcMspaceCreate_nid_postfix(const char* name, void* base, auto* arena = static_cast(base); auto* block = reinterpret_cast(arena + 1); *block = {size - sizeof(Arena) - sizeof(Block), nullptr, nullptr, 0}; - *arena = {arenas, block, start + size}; + *arena = {arenas, block, start + size, 0, 0}; arenas = arena; return arena; } @@ -165,6 +203,23 @@ int APS5_VABI sceLibcMspacePosixMemalign_nid_postfix(void* handle, void** result return 0; } +void* APS5_VABI sceLibcMspaceMemalign_nid_postfix(void* handle, std::size_t alignment, std::size_t size) { + if (alignment == 0 || (alignment & (alignment - 1))) { + Error(22); + return nullptr; + } + std::lock_guard lock(arenaMutex); + return Allocate(Find(handle), size, std::max(alignment, 16)); +} + +int APS5_VABI sceLibcMspaceMallocStats_nid_postfix(void* handle, MallocManagedSize* stats) { + return FillStats(handle, stats); +} + +int APS5_VABI sceLibcMspaceMallocStatsFast_nid_postfix(void* handle, MallocManagedSize* stats) { + return FillStats(handle, stats); +} + std::size_t APS5_VABI sceLibcMspaceMallocUsableSize_nid_postfix(const void* pointer) { if (!pointer) return 0; std::lock_guard lock(arenaMutex); diff --git a/core/libs/tests/Mspace.cpp b/core/libs/tests/Mspace.cpp index f286a7dc58..2cb6143ce6 100644 --- a/core/libs/tests/Mspace.cpp +++ b/core/libs/tests/Mspace.cpp @@ -16,7 +16,19 @@ void* APS5_VABI sceLibcMspaceRealloc_nid_postfix(void*, void*, std::size_t); void APS5_VABI sceLibcMspaceFree_nid_postfix(void*, void*); int APS5_VABI sceLibcMspacePosixMemalign_nid_postfix(void*, void**, std::size_t, std::size_t); std::size_t APS5_VABI sceLibcMspaceMallocUsableSize_nid_postfix(const void*); +void* APS5_VABI sceLibcMspaceMemalign_nid_postfix(void*, std::size_t, std::size_t); +int APS5_VABI sceLibcMspaceMallocStats_nid_postfix(void*, void*); +int APS5_VABI sceLibcMspaceMallocStatsFast_nid_postfix(void*, void*); } +struct MallocManagedSize { + std::uint16_t size; + std::uint16_t version; + std::uint32_t reserved; + std::size_t maxSystemSize; + std::size_t currentSystemSize; + std::size_t maxInuseSize; + std::size_t currentInuseSize; +}; static void Require(bool value) { if (!value) { std::fputs("Mspace check failed\n", stderr); std::abort(); } } @@ -58,6 +70,27 @@ int main() { } }); for (auto& worker : workers) worker.join(); + MallocManagedSize stats{sizeof(MallocManagedSize), 1, 0, 0, 0, 0, 0}; + Require(sceLibcMspaceMallocStats_nid_postfix(arena, &stats) == 0); + Require(stats.currentSystemSize == storage.size() && stats.maxSystemSize == storage.size()); + Require(stats.currentInuseSize == 0 && stats.maxInuseSize >= storage.size() - 256); + MallocManagedSize shortStats{8, 1, 0, 0, 0, 0, 0}; + Require(sceLibcMspaceMallocStatsFast_nid_postfix(arena, &shortStats) == 22); + void* memaligned = sceLibcMspaceMemalign_nid_postfix(arena, 256, 100); + Require(memaligned && (reinterpret_cast(memaligned) & 255) == 0); + Require(sceLibcMspaceMemalign_nid_postfix(arena, 24, 100) == nullptr); + Require(sceLibcMspaceMallocStatsFast_nid_postfix(arena, &stats) == 0 && stats.currentInuseSize >= 100); + sceLibcMspaceFree_nid_postfix(arena, memaligned); + void* region = sceLibcMspaceMalloc_nid_postfix(arena, 8192); + Require(region != nullptr); + Require(sceLibcMspaceCreate_nid_postfix("outside", static_cast(region) + 8192, 4096, 0) == nullptr); + void* nested = sceLibcMspaceCreate_nid_postfix("nested", region, 8192, 1); + Require(nested == region); + void* inner = sceLibcMspaceMalloc_nid_postfix(nested, 64); + Require(inner > region && inner < static_cast(region) + 8192); + Require(sceLibcMspaceCreate_nid_postfix("flags", storage.data(), storage.size(), 2) == nullptr); + Require(sceLibcMspaceDestroy_nid_postfix(nested) == 0); + sceLibcMspaceFree_nid_postfix(arena, region); Require(sceLibcMspaceDestroy_nid_postfix(arena) == 0); Require(sceLibcMspaceMalloc_nid_postfix(arena, 8) == nullptr); Require(sceLibcMspaceCreate_nid_postfix("reuse", storage.data(), storage.size(), 0) == arena); From a599eca7276dee4ef679edbb0f388fa5c999ef06 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 14:22:48 +1300 Subject: [PATCH 77/93] feat(libc): operator new/delete, bounds-checked functions, char locale facets and runtime helpers Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libSceLibcInternal/Export.cpp | 7 ++ core/libs/prx/libc/include/General.hpp | 1 + .../libc/include/specifics/gcc/AtomicOps.hpp | 8 ++ core/libs/prx/libc/src/FileAndHeap.cpp | 28 ++++++ core/libs/prx/libc/src/Formatting.cpp | 50 ++++++++++ core/libs/prx/libc/src/LocaleSupport.cpp | 7 ++ core/libs/prx/libc/src/RuntimeSupport.cpp | 91 ++++++++++++++++--- core/libs/prx/libc/src/StdioExtras.cpp | 43 +++++++++ core/libs/prx/libc/src/Strings.cpp | 73 +++++++++++++++ core/libs/prx/libc/src/Time.cpp | 4 + core/libs/tests/ApplicationHeap.cpp | 15 +++ core/libs/tests/GuestStrings.cpp | 32 +++++++ 12 files changed, 348 insertions(+), 11 deletions(-) diff --git a/core/libs/prx/libSceLibcInternal/Export.cpp b/core/libs/prx/libSceLibcInternal/Export.cpp index 6597f5f5be..49699b6ca1 100644 --- a/core/libs/prx/libSceLibcInternal/Export.cpp +++ b/core/libs/prx/libSceLibcInternal/Export.cpp @@ -1,7 +1,14 @@ +#include "prx/libc/include/General.hpp" #include "prx/libc/include/HeapDiagnostics.hpp" extern "C" { +int Need_sceLibcInternal_nid_postfix = 1; + +void APS5_VABI __cxa_finalize_nid_postfix(void* dsoHandle) { + CxaFinalize_nid_no_patch(dsoHandle); +} + void APS5_VABI sceLibcHeapGetTraceInfo_nid_postfix(Info* info) { LibcHeapTraceInfo_nid_no_patch(info); } diff --git a/core/libs/prx/libc/include/General.hpp b/core/libs/prx/libc/include/General.hpp index f1ed6ca204..35c9126a58 100644 --- a/core/libs/prx/libc/include/General.hpp +++ b/core/libs/prx/libc/include/General.hpp @@ -9,6 +9,7 @@ #include "general/ExportMacros.hpp" extern "C" void NotImplemented_nid_no_patch(const char* funcName); +extern "C" void CxaFinalize_nid_no_patch(void* dsoHandle); extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path); // Guest path aliases: a guest prefix such as "/_sm/0" (a save-data mount point) resolves to a host diff --git a/core/libs/prx/libc/include/specifics/gcc/AtomicOps.hpp b/core/libs/prx/libc/include/specifics/gcc/AtomicOps.hpp index 37a0e544cc..bfeef359d4 100644 --- a/core/libs/prx/libc/include/specifics/gcc/AtomicOps.hpp +++ b/core/libs/prx/libc/include/specifics/gcc/AtomicOps.hpp @@ -9,4 +9,12 @@ inline unsigned int GccAtomicFetchSub(volatile unsigned int* target, unsigned in return __atomic_fetch_sub(target, value, __ATOMIC_SEQ_CST); } +inline bool GccAtomicCompareExchangeWeak(volatile unsigned int* target, unsigned int* expected, unsigned int desired) { + return __atomic_compare_exchange_n(target, expected, desired, true, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} + +inline unsigned int GccAtomicLoad(volatile unsigned int* target) { + return __atomic_load_n(target, __ATOMIC_SEQ_CST); +} + #endif diff --git a/core/libs/prx/libc/src/FileAndHeap.cpp b/core/libs/prx/libc/src/FileAndHeap.cpp index b3e3ed183e..be4ac1c83d 100644 --- a/core/libs/prx/libc/src/FileAndHeap.cpp +++ b/core/libs/prx/libc/src/FileAndHeap.cpp @@ -63,6 +63,14 @@ FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode) return stream.release(); } +int APS5_VABI fopen_s_nid_postfix(FileStream** result, const char* filename, const char* mode) { + constexpr int GuestEinval = 22; + constexpr int GuestEnoent = 2; + if (!result || !filename || !mode) return GuestEinval; + *result = fopen_nid_postfix(filename, mode); + return *result ? 0 : GuestEnoent; +} + int APS5_VABI fclose_nid_postfix(FileStream* stream) { GetNativeStream(stream); std::unique_ptr owner(stream->IsDynamic() ? stream : nullptr); @@ -121,6 +129,17 @@ int APS5_VABI fseek_nid_postfix(FileStream* stream, std::int64_t offset, int ori std::int64_t APS5_VABI ftell_nid_postfix(FileStream* stream) { return ftello_nid_postfix(stream); } +int APS5_VABI fgetpos_nid_postfix(FileStream* stream, std::int64_t* position) { + if (!position) throw std::invalid_argument("fgetpos: null position"); + *position = ftello_nid_postfix(stream); + return 0; +} + +int APS5_VABI fsetpos_nid_postfix(FileStream* stream, const std::int64_t* position) { + if (!position) throw std::invalid_argument("fsetpos: null position"); + return fseeko_nid_postfix(stream, *position, SEEK_SET); +} + int APS5_VABI fputs_nid_postfix(const char* str, FileStream* stream) { if (!str) throw std::runtime_error("fputs: null string"); const int result = std::fputs(str, GetNativeStream(stream)); @@ -149,6 +168,15 @@ void* APS5_VABI memalign_nid_postfix(size_t alignment, size_t size) { return ApplicationHeapAlign_nid_no_patch(alignment, size); } +void* APS5_VABI aligned_alloc_nid_postfix(size_t alignment, size_t size) { + return ApplicationHeapAlign_nid_no_patch(alignment, size); +} + +void* APS5_VABI reallocalign_nid_postfix(void* ptr, size_t size, size_t alignment) { + if (ptr != nullptr) NotImplemented_nid_no_patch("reallocalign of an existing block"); + return ApplicationHeapAlign_nid_no_patch(alignment, size); +} + void* APS5_VABI calloc_nid_postfix(size_t count, size_t size) { return ApplicationHeapCalloc_nid_no_patch(count, size); } diff --git a/core/libs/prx/libc/src/Formatting.cpp b/core/libs/prx/libc/src/Formatting.cpp index 207069071c..bac3baaf8a 100644 --- a/core/libs/prx/libc/src/Formatting.cpp +++ b/core/libs/prx/libc/src/Formatting.cpp @@ -169,6 +169,56 @@ int APS5_VABI vsnprintf_nid_postfix(char* str, size_t size, const char* format, #endif } +#ifdef _WIN32 + +int APS5_VABI snprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) { + __builtin_sysv_va_list args; + __builtin_sysv_va_start(args, format); + const int result = LibcDetail::FormatWindows(buffer, size, format, args); + __builtin_sysv_va_end(args); + return result; +} + +int APS5_VABI printf_s_nid_postfix(const char* format, ...) { + __builtin_sysv_va_list args; + __builtin_sysv_va_start(args, format); + const int result = LibcDetail::PrintWindows(format, args); + __builtin_sysv_va_end(args); + return result; +} + +#else + +int APS5_VABI vsprintf_s_nid_postfix(char* buffer, size_t size, const char* format, VaList* args) { + return std::vsnprintf(buffer, size, format, *reinterpret_cast(args)); +} + +int APS5_VABI sprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) { + std::va_list args; + va_start(args, format); + const int result = std::vsnprintf(buffer, size, format, args); + va_end(args); + return result; +} + +int APS5_VABI snprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) { + std::va_list args; + va_start(args, format); + const int result = std::vsnprintf(buffer, size, format, args); + va_end(args); + return result; +} + +int APS5_VABI printf_s_nid_postfix(const char* format, ...) { + std::va_list args; + va_start(args, format); + const int result = std::vprintf(format, args); + va_end(args); + return result; +} + +#endif + int APS5_VABI puts_nid_postfix(const char* s) { return std::puts(s); } diff --git a/core/libs/prx/libc/src/LocaleSupport.cpp b/core/libs/prx/libc/src/LocaleSupport.cpp index 4c52e8ad2b..4bc5eb94e7 100644 --- a/core/libs/prx/libc/src/LocaleSupport.cpp +++ b/core/libs/prx/libc/src/LocaleSupport.cpp @@ -143,6 +143,9 @@ int APS5_VABI tolower_nid_postfix(int c) { return ConvertCharacter(c, false); } std::uint64_t _ZNSt5ctypeIcE2idE_nid_postfix = 0; std::uint64_t _ZNSt5ctypeIwE2idE_nid_postfix = 0; std::uint64_t _ZNSt7collateIwE2idE_nid_postfix = 0; +std::uint64_t _ZNSt7collateIcE2idE_nid_postfix = 0; +std::uint64_t _ZNSt7codecvtIcc9_MbstatetE2idE_nid_postfix = 0; +std::uintptr_t _ZTVSt7codecvtIcc9_MbstatetE_nid_postfix[16] {}; std::uint64_t _ZNSt7num_putIcSt19ostreambuf_iteratorIcSt11char_traitsIcEEE2idE_nid_postfix = 0; std::uintptr_t _ZTVSt7num_putIcSt19ostreambuf_iteratorIcSt11char_traitsIcEEE_nid_postfix[12] {}; @@ -185,6 +188,10 @@ void APS5_VABI _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(Gu NotImplemented_nid_no_patch(__func__); } +void APS5_VABI _ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet**, const GuestLocale::Implementation*) { + NotImplemented_nid_no_patch(__func__); +} + void APS5_VABI _ZNSt8_LocinfoC1EPKc_nid_postfix(GuestLocale::LocinfoStorage* self, const char* localeName) { if (self == nullptr || localeName == nullptr || std::strcmp(localeName, "C") != 0) throw std::invalid_argument("_Locinfo: only the C locale is supported"); new (self) GuestLocale::LocinfoStorage{}; diff --git a/core/libs/prx/libc/src/RuntimeSupport.cpp b/core/libs/prx/libc/src/RuntimeSupport.cpp index c2772ac47c..bf1ada3ebc 100644 --- a/core/libs/prx/libc/src/RuntimeSupport.cpp +++ b/core/libs/prx/libc/src/RuntimeSupport.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -6,6 +8,9 @@ #include #include #include +#include +#include +#include #include "prx/libc/include/General.hpp" #include "prx/libc/include/specifics/gcc/AtomicOps.hpp" @@ -15,6 +20,16 @@ namespace { std::recursive_mutex g_sysLock; +struct ExitDestructor { + void (*func)(void*); + void* arg; + void* dsoHandle; +}; + +std::mutex g_exitMutex; +std::vector g_exitDestructors; +bool g_exitRunnerRegistered = false; + } extern "C" { @@ -24,21 +39,31 @@ FileStream _Stdout_nid_postfix{stdout}; FileStream _Stdin_nid_postfix{stdin}; int APS5_VABI __cxa_atexit_nid_postfix(void (*func)(void*), void* arg, void* dsoHandle) { - (void)dsoHandle; - static std::vector> destructors; - static bool runnerRegistered = false; - destructors.emplace_back(func, arg); - if (!runnerRegistered) { - runnerRegistered = true; - std::atexit([] { - for (auto it = destructors.rbegin(); it != destructors.rend(); ++it) { - it->first(it->second); - } - }); + std::lock_guard lock(g_exitMutex); + g_exitDestructors.push_back({func, arg, dsoHandle}); + if (!g_exitRunnerRegistered) { + g_exitRunnerRegistered = true; + std::atexit([] { CxaFinalize_nid_no_patch(nullptr); }); } return 0; } +void CxaFinalize_nid_no_patch(void* dsoHandle) { + for (;;) { + ExitDestructor destructor{}; + { + std::lock_guard lock(g_exitMutex); + const auto found = std::find_if(g_exitDestructors.rbegin(), g_exitDestructors.rend(), [&](const ExitDestructor& entry) { + return dsoHandle == nullptr || entry.dsoHandle == dsoHandle; + }); + if (found == g_exitDestructors.rend()) return; + destructor = *found; + g_exitDestructors.erase(std::next(found).base()); + } + destructor.func(destructor.arg); + } +} + unsigned int APS5_VABI _Atomic_fetch_add_4_nid_postfix(volatile unsigned int* target, unsigned int value, int memoryOrder) { (void)memoryOrder; return GccAtomicFetchAdd(target, value); @@ -49,6 +74,50 @@ unsigned int APS5_VABI _Atomic_fetch_sub_4_nid_postfix(volatile unsigned int* ta return GccAtomicFetchSub(target, value); } +int APS5_VABI _Atomic_compare_exchange_weak_4_nid_postfix(volatile unsigned int* target, unsigned int* expected, unsigned int desired, int successOrder, int failureOrder) { + (void)successOrder; + (void)failureOrder; + return GccAtomicCompareExchangeWeak(target, expected, desired) ? 1 : 0; +} + +unsigned int APS5_VABI _Atomic_load_4_nid_postfix(volatile unsigned int* target, int memoryOrder) { + (void)memoryOrder; + return GccAtomicLoad(target); +} + +std::uintmax_t APS5_VABI strtoumax_nid_postfix(const char* str, char** endptr, int base) { + return std::strtoumax(str, endptr, base); +} + +unsigned int APS5_VABI _ZSt14_Random_devicev_nid_postfix() { + static std::random_device device; + static std::mutex mutex; + std::lock_guard lock(mutex); + return device(); +} + +[[noreturn]] void APS5_VABI _ZSt14_Throw_C_errori_nid_postfix(int code) { + throw std::runtime_error("_Throw_C_error: C11 thread error " + std::to_string(code)); +} + +void APS5_VABI _Lockfilelock_nid_postfix(FileStream* stream) { + if (stream == nullptr) throw std::invalid_argument("_Lockfilelock: null stream"); +#ifdef _WIN32 + _lock_file(GetNativeStream(stream)); +#else + flockfile(GetNativeStream(stream)); +#endif +} + +void APS5_VABI _Unlockfilelock_nid_postfix(FileStream* stream) { + if (stream == nullptr) throw std::invalid_argument("_Unlockfilelock: null stream"); +#ifdef _WIN32 + _unlock_file(GetNativeStream(stream)); +#else + funlockfile(GetNativeStream(stream)); +#endif +} + unsigned long APS5_VABI _Stoul_nid_postfix(const char* str, char** endptr, int base) { return std::strtoul(str, endptr, base); } diff --git a/core/libs/prx/libc/src/StdioExtras.cpp b/core/libs/prx/libc/src/StdioExtras.cpp index 90a21b057e..d2aabc81d7 100644 --- a/core/libs/prx/libc/src/StdioExtras.cpp +++ b/core/libs/prx/libc/src/StdioExtras.cpp @@ -6,6 +6,7 @@ #include "prx/libc/include/FileStream.hpp" #include "prx/libc/include/GuestHeap.hpp" +#include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libc/include/General.hpp" #include "SceTypes.hpp" @@ -19,6 +20,10 @@ using GuestNewHandler = void (APS5_VABI*)(); GuestNewHandler g_newHandler = nullptr; +void* Allocate(std::size_t size) { + return ApplicationHeapAllocate_nid_no_patch(size == 0 ? 1 : size); +} + } extern "C" { @@ -58,6 +63,44 @@ GuestNewHandler APS5_VABI _ZSt15set_new_handlerPFvvE_nid_postfix(GuestNewHandler return previous; } +GuestNewHandler APS5_VABI _ZSt15get_new_handlerv_nid_postfix() { + return g_newHandler; +} + +unsigned char _ZSt7nothrow_nid_postfix = 0; + +void* APS5_VABI _Znwm_nid_postfix(std::size_t size) { + return Allocate(size); +} + +void* APS5_VABI _Znam_nid_postfix(std::size_t size) { + return Allocate(size); +} + +void* APS5_VABI _ZnwmRKSt9nothrow_t_nid_postfix(std::size_t size, const void*) { + return Allocate(size); +} + +void* APS5_VABI _ZnamRKSt9nothrow_t_nid_postfix(std::size_t size, const void*) { + return Allocate(size); +} + +void APS5_VABI _ZdlPv_nid_postfix(void* pointer) { + if (pointer != nullptr) ApplicationHeapFree_nid_no_patch(pointer); +} + +void APS5_VABI _ZdaPv_nid_postfix(void* pointer) { + if (pointer != nullptr) ApplicationHeapFree_nid_no_patch(pointer); +} + +void APS5_VABI _ZdlPvm_nid_postfix(void* pointer, std::size_t) { + if (pointer != nullptr) ApplicationHeapFree_nid_no_patch(pointer); +} + +void APS5_VABI _ZdaPvm_nid_postfix(void* pointer, std::size_t) { + if (pointer != nullptr) ApplicationHeapFree_nid_no_patch(pointer); +} + void APS5_VABI _ZdlPvSt11align_val_t_nid_postfix(void* pointer, std::size_t alignment) { (void)alignment; GuestHeap::GuestHeapFree_nid_postfix(pointer); diff --git a/core/libs/prx/libc/src/Strings.cpp b/core/libs/prx/libc/src/Strings.cpp index c3fdec2d63..1f70ddc442 100644 --- a/core/libs/prx/libc/src/Strings.cpp +++ b/core/libs/prx/libc/src/Strings.cpp @@ -188,6 +188,79 @@ int APS5_VABI strncpy_s_nid_postfix(char* dest, size_t destsz, const char* src, return 0; } +int APS5_VABI strcpy_s_nid_postfix(char* dest, size_t destsz, const char* src) { + return strncpy_s_nid_postfix(dest, destsz, src, static_cast(-1)); +} + +int APS5_VABI strncat_s_nid_postfix(char* dest, size_t destsz, const char* src, size_t count) { + constexpr int GuestEinval = 22; + constexpr int GuestErange = 34; + if (!dest || destsz == 0) return GuestEinval; + size_t used = 0; + while (used < destsz && dest[used] != '\0') ++used; + if (used == destsz || !src) { + dest[0] = '\0'; + return used == destsz ? GuestErange : GuestEinval; + } + size_t length = 0; + while (length < count && src[length] != '\0') ++length; + if (length >= destsz - used) { + dest[0] = '\0'; + return GuestErange; + } + std::memcpy(dest + used, src, length); + dest[used + length] = '\0'; + return 0; +} + +int APS5_VABI strcat_s_nid_postfix(char* dest, size_t destsz, const char* src) { + return strncat_s_nid_postfix(dest, destsz, src, static_cast(-1)); +} + +int APS5_VABI memcpy_s_nid_postfix(void* dest, size_t destsz, const void* src, size_t count) { + constexpr int GuestEinval = 22; + constexpr int GuestErange = 34; + if (!dest) return GuestEinval; + if (!src || count > destsz) { + std::memset(dest, 0, destsz); + return src ? GuestErange : GuestEinval; + } + std::memcpy(dest, src, count); + return 0; +} + +int APS5_VABI memmove_s_nid_postfix(void* dest, size_t destsz, const void* src, size_t count) { + constexpr int GuestEinval = 22; + constexpr int GuestErange = 34; + if (!dest) return GuestEinval; + if (!src || count > destsz) { + std::memset(dest, 0, destsz); + return src ? GuestErange : GuestEinval; + } + std::memmove(dest, src, count); + return 0; +} + +int APS5_VABI memset_s_nid_postfix(void* dest, size_t destsz, int value, size_t count) { + constexpr int GuestEinval = 22; + constexpr int GuestErange = 34; + if (!dest) return GuestEinval; + const auto length = count > destsz ? destsz : count; + auto* bytes = static_cast(dest); + for (size_t index = 0; index < length; ++index) bytes[index] = static_cast(value); + return count > destsz ? GuestErange : 0; +} + +char* APS5_VABI strnstr_nid_postfix(const char* haystack, const char* needle, size_t length) { + const size_t needleLength = std::strlen(needle); + if (needleLength == 0) return const_cast(haystack); + for (size_t index = 0; index < length && haystack[index] != '\0'; ++index) { + if (needleLength > length - index) break; + if (std::strncmp(haystack + index, needle, needleLength) == 0) return const_cast(haystack + index); + } + return nullptr; +} + unsigned long long APS5_VABI _Stoull_nid_postfix(const char* str, char** endptr, int base) { return std::strtoull(str, endptr, base); } diff --git a/core/libs/prx/libc/src/Time.cpp b/core/libs/prx/libc/src/Time.cpp index 6d03d5fb1a..64bc20ddb5 100644 --- a/core/libs/prx/libc/src/Time.cpp +++ b/core/libs/prx/libc/src/Time.cpp @@ -89,6 +89,10 @@ size_t APS5_VABI libc_strftime_nid_postfix(char* str, size_t count, const char* return std::strftime(str, count, format, timeptr); } +char* APS5_VABI asctime_nid_postfix(const std::tm* timeptr) { + return std::asctime(timeptr); +} + size_t APS5_VABI strftime_nid_postfix(char* str, size_t count, const char* format, const std::tm* timeptr) { return std::strftime(str, count, format, timeptr); } diff --git a/core/libs/tests/ApplicationHeap.cpp b/core/libs/tests/ApplicationHeap.cpp index 48009f88ef..7818274602 100644 --- a/core/libs/tests/ApplicationHeap.cpp +++ b/core/libs/tests/ApplicationHeap.cpp @@ -7,6 +7,13 @@ #include #include +extern "C" { +void* APS5_VABI _Znwm_nid_postfix(std::size_t); +void* APS5_VABI _ZnamRKSt9nothrow_t_nid_postfix(std::size_t, const void*); +void APS5_VABI _ZdlPv_nid_postfix(void*); +void APS5_VABI _ZdaPv_nid_postfix(void*); +} + namespace { alignas(64) std::array storage{}; @@ -117,7 +124,15 @@ int main(int argc, char**) { require(ApplicationHeapPosixAlign_nid_no_patch(&pointer, 64, 128) == 0 && pointer == storage.data() && lastAlignment == 64); reject([] { ApplicationHeapAlign_nid_no_patch(3, 64); }); reject([] { ApplicationHeapCalloc_nid_no_patch(2, std::numeric_limits::max()); }); + require(_Znwm_nid_postfix(0) == storage.data() && lastSize == 1); + _ZdlPv_nid_postfix(storage.data()); + _ZdlPv_nid_postfix(nullptr); + require(frees == 3); + require(_ZnamRKSt9nothrow_t_nid_postfix(24, nullptr) == storage.data() && lastSize == 24); + _ZdaPv_nid_postfix(storage.data()); + require(frees == 4); fail = true; + reject([] { _Znwm_nid_postfix(8); }); reject([] { ApplicationHeapAlign_nid_no_patch(4, 64); }); reject([] { ApplicationHeapAllocate_nid_no_patch(64); }); void* unchanged = storage.data(); diff --git a/core/libs/tests/GuestStrings.cpp b/core/libs/tests/GuestStrings.cpp index cc8eefab69..c10a9f3bb7 100644 --- a/core/libs/tests/GuestStrings.cpp +++ b/core/libs/tests/GuestStrings.cpp @@ -15,6 +15,14 @@ std::size_t APS5_VABI strlcat_nid_postfix(char*, const char*, std::size_t); char* APS5_VABI strtok_r_nid_postfix(char*, const char*, char**); char* APS5_VABI strtok_nid_postfix(char*, const char*); char* APS5_VABI strcasestr_nid_postfix(const char*, const char*); +int APS5_VABI strcpy_s_nid_postfix(char*, std::size_t, const char*); +int APS5_VABI strcat_s_nid_postfix(char*, std::size_t, const char*); +int APS5_VABI strncat_s_nid_postfix(char*, std::size_t, const char*, std::size_t); +int APS5_VABI memcpy_s_nid_postfix(void*, std::size_t, const void*, std::size_t); +int APS5_VABI memmove_s_nid_postfix(void*, std::size_t, const void*, std::size_t); +int APS5_VABI memset_s_nid_postfix(void*, std::size_t, int, std::size_t); +char* APS5_VABI strnstr_nid_postfix(const char*, const char*, std::size_t); +int APS5_VABI snprintf_s_nid_postfix(char*, std::size_t, const char*, ...); } static void Require(bool condition) { @@ -24,7 +32,31 @@ static void Require(bool condition) { } } +static void CheckBoundsCheckedFunctions() { + char small[4] = "zz"; + Require(strcpy_s_nid_postfix(small, sizeof(small), "abc") == 0 && std::strcmp(small, "abc") == 0); + Require(strcpy_s_nid_postfix(small, sizeof(small), "abcd") == 34 && small[0] == '\0'); + Require(strcpy_s_nid_postfix(nullptr, 4, "a") == 22); + char joined[8] = "ab"; + Require(strcat_s_nid_postfix(joined, sizeof(joined), "cd") == 0 && std::strcmp(joined, "abcd") == 0); + Require(strncat_s_nid_postfix(joined, sizeof(joined), "efgh", 2) == 0 && std::strcmp(joined, "abcdef") == 0); + Require(strcat_s_nid_postfix(joined, sizeof(joined), "gh") == 34 && joined[0] == '\0'); + char bytes[4] = {1, 2, 3, 4}; + const char source[4] = {5, 6, 7, 8}; + Require(memcpy_s_nid_postfix(bytes, sizeof(bytes), source, 2) == 0 && bytes[0] == 5 && bytes[2] == 3); + Require(memcpy_s_nid_postfix(bytes, 2, source, 4) == 34 && bytes[0] == 0 && bytes[1] == 0 && bytes[2] == 3); + char overlap[6] = "abcde"; + Require(memmove_s_nid_postfix(overlap + 1, 5, overlap, 3) == 0 && std::strcmp(overlap, "aabce") == 0); + Require(memset_s_nid_postfix(bytes, sizeof(bytes), 9, 8) == 34 && bytes[3] == 9); + Require(strnstr_nid_postfix("haystack", "st", 4) == nullptr); + const char haystack[] = "haystack"; + Require(strnstr_nid_postfix(haystack, "st", 6) == haystack + 3); + char formatted[8]; + Require(snprintf_s_nid_postfix(formatted, sizeof(formatted), "%d-%s", 42, "x") == 4 && std::strcmp(formatted, "42-x") == 0); +} + int main() { + CheckBoundsCheckedFunctions(); Require(std::strcmp(basename_nid_postfix(nullptr), ".") == 0); Require(std::strcmp(basename_nid_postfix(""), ".") == 0); Require(std::strcmp(basename_nid_postfix("////"), "/") == 0); From eb79f0d41240adb98027b898556c50577e61dca2 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 14:22:48 +1300 Subject: [PATCH 78/93] fix(libc): don't walk guest frame pointers when reporting an unhandled exception Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libc/src/exception/Runtime.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/libs/prx/libc/src/exception/Runtime.cpp b/core/libs/prx/libc/src/exception/Runtime.cpp index 411edbf916..244bc1f6fb 100644 --- a/core/libs/prx/libc/src/exception/Runtime.cpp +++ b/core/libs/prx/libc/src/exception/Runtime.cpp @@ -240,8 +240,7 @@ void APS5_VABI __cxa_free_exception_nid_postfix(void* object) { header->unwind.exception_cleanup = Cleanup; ++globals.uncaught; _Unwind_RaiseException_nid_postfix(&header->unwind); - RawLog("[libc] unhandled exception '%s' thrown from %p (caller %p)\n", type ? type->name() : "?", __builtin_return_address(0), - static_cast(static_cast(__builtin_frame_address(0))[0])[1]); + RawLog("[libc] unhandled exception '%s' thrown from %p\n", type ? type->name() : "?", __builtin_return_address(0)); __cxa_begin_catch_nid_postfix(&header->unwind); InvokeTerminate(header->terminate); } From 99bdf81364ffc87f66617842318302273acf2402 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 14:22:48 +1300 Subject: [PATCH 79/93] feat(userservice): user name, user number and accessibility settings of the initial user Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libSceUserService/Export.cpp | 68 +++++++++++-------- .../prx/libSceUserService/UserService.hpp | 4 ++ 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/core/libs/prx/libSceUserService/Export.cpp b/core/libs/prx/libSceUserService/Export.cpp index b4d913b5d7..c5e1467317 100644 --- a/core/libs/prx/libSceUserService/Export.cpp +++ b/core/libs/prx/libSceUserService/Export.cpp @@ -3,43 +3,49 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include +#include #include "prx/libSceUserService/UserService.hpp" extern "C" { int APS5_VABI sceUserServiceGetAccessibilityChatTranscription(int user_id, int32_t* chat_transcription) { - (void)user_id; - (void)chat_transcription; - NotImplemented_nid_no_patch(__func__); - return 0; + if (chat_transcription == nullptr || user_id != USER_SERVICE_INITIAL_USER_ID) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + *chat_transcription = 0; + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceGetAccessibilityPressAndHoldDelay(int user_id, int32_t* press_and_hold_delay) { - (void)user_id; - (void)press_and_hold_delay; - NotImplemented_nid_no_patch(__func__); - return 0; + if (press_and_hold_delay == nullptr || user_id != USER_SERVICE_INITIAL_USER_ID) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + *press_and_hold_delay = 0; + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceGetAccessibilityTriggerEffect(int user_id, int32_t* trigger_effect) { - (void)user_id; - (void)trigger_effect; - NotImplemented_nid_no_patch(__func__); - return 0; + if (trigger_effect == nullptr || user_id != USER_SERVICE_INITIAL_USER_ID) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + *trigger_effect = 0; + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceGetAccessibilityVibration(int user_id, int32_t* vibration) { - (void)user_id; - (void)vibration; - NotImplemented_nid_no_patch(__func__); - return 0; + if (vibration == nullptr || user_id != USER_SERVICE_INITIAL_USER_ID) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + *vibration = 0; + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceGetAccessibilityZoomEnabled(int user_id, int32_t* zoom_enabled) { - (void)user_id; - (void)zoom_enabled; - NotImplemented_nid_no_patch(__func__); - return 0; + if (zoom_enabled == nullptr || user_id != USER_SERVICE_INITIAL_USER_ID) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + *zoom_enabled = 0; + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceGetAgeLevel(int user_id, uint32_t* age_level) { @@ -102,18 +108,22 @@ int APS5_VABI sceUserServiceGetLoginUserIdList(UserServiceLoginUserIdList* user_ } int APS5_VABI sceUserServiceGetUserName(int user_id, char* name, size_t size) { - (void)user_id; - (void)name; - (void)size; - NotImplemented_nid_no_patch(__func__); - return 0; + if (name == nullptr || user_id != USER_SERVICE_INITIAL_USER_ID) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (size < USER_SERVICE_MAX_USER_NAME_LENGTH + 1) { + return USER_SERVICE_ERROR_BUFFER_TOO_SHORT; + } + std::strncpy(name, USER_SERVICE_INITIAL_USER_NAME, size); + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceGetUserNumber(int user_id, int32_t* number) { - (void)user_id; - (void)number; - NotImplemented_nid_no_patch(__func__); - return 0; + if (number == nullptr || user_id != USER_SERVICE_INITIAL_USER_ID) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + *number = 1; + return USER_SERVICE_OK; } int APS5_VABI sceUserServiceInitialize(const void* params) { diff --git a/core/libs/prx/libSceUserService/UserService.hpp b/core/libs/prx/libSceUserService/UserService.hpp index 48237cc37e..7a37d82c25 100644 --- a/core/libs/prx/libSceUserService/UserService.hpp +++ b/core/libs/prx/libSceUserService/UserService.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBSCEUSERSERVICE_USERSERVICE_HPP #define CORE_LIBS_PRX_LIBSCEUSERSERVICE_USERSERVICE_HPP +#include #include constexpr int USER_SERVICE_OK = 0; @@ -8,5 +9,8 @@ constexpr int USER_SERVICE_ERROR_INVALID_ARGUMENT = -2137653246; constexpr int USER_SERVICE_ERROR_NO_EVENT = -2137653241; constexpr int USER_SERVICE_USER_ID_INVALID = -1; constexpr int USER_SERVICE_INITIAL_USER_ID = 0x10000000; +constexpr int USER_SERVICE_ERROR_BUFFER_TOO_SHORT = -2137653238; +constexpr std::size_t USER_SERVICE_MAX_USER_NAME_LENGTH = 16; +constexpr const char* USER_SERVICE_INITIAL_USER_NAME = "Player"; #endif From 0644a8c38f07fa0d1265dee8bff62c8f671a3b0f Mon Sep 17 00:00:00 2001 From: MartinCajiao Date: Sun, 27 Sep 2026 20:27:22 -0500 Subject: [PATCH 80/93] feat(input): support SDL game controllers --- CMakeLists.txt | 2 +- README.md | 2 +- core/libs/prx/libScePad/include/PadState.hpp | 2 + core/libs/prx/libScePad/src/PadState.cpp | 8 +- .../prx/libSceVideoOut/include/PadInput.hpp | 6 + core/libs/prx/libSceVideoOut/src/PadInput.cpp | 108 +++++++++++++++++- .../prx/libSceVideoOut/src/VideoOutDriver.cpp | 8 +- docs/INPUT_MAPPING.md | 8 +- 8 files changed, 130 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8308a76139..30724ac322 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ set(SDL_WAYLAND OFF CACHE BOOL "" FORCE) set(SDL_OPENGL OFF CACHE BOOL "" FORCE) set(SDL_OPENGLES OFF CACHE BOOL "" FORCE) set(SDL_VULKAN ON CACHE BOOL "" FORCE) -set(SDL_JOYSTICK OFF CACHE BOOL "" FORCE) +set(SDL_JOYSTICK ON CACHE BOOL "" FORCE) set(SDL_HAPTIC OFF CACHE BOOL "" FORCE) set(SDL_HIDAPI OFF CACHE BOOL "" FORCE) set(SDL_SENSOR OFF CACHE BOOL "" FORCE) diff --git a/README.md b/README.md index d54a4320bd..a98b5a96fe 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The project targets maximum compiler portability. Support for additional compile ## Input mapping -Keyboard and mouse controls can be configured with an `anyps5-input.ini` file. See [input mapping](docs/INPUT_MAPPING.md) for the format and behavior. +SDL-mapped game controllers are supported, including analog sticks and triggers. Keyboard and mouse controls can be configured with an `anyps5-input.ini` file. See [input mapping](docs/INPUT_MAPPING.md) for the supported devices and configuration format. ## Disclaimer diff --git a/core/libs/prx/libScePad/include/PadState.hpp b/core/libs/prx/libScePad/include/PadState.hpp index 80f20807fe..4d8646d295 100644 --- a/core/libs/prx/libScePad/include/PadState.hpp +++ b/core/libs/prx/libScePad/include/PadState.hpp @@ -9,6 +9,8 @@ struct PadInputState { std::uint32_t buttons = 0; std::array sticks{128, 128, 128, 128}; + std::uint8_t analogButtonsL2 = 0; + std::uint8_t analogButtonsR2 = 0; bool touchLeft = false; bool touchRight = false; }; diff --git a/core/libs/prx/libScePad/src/PadState.cpp b/core/libs/prx/libScePad/src/PadState.cpp index d0c03e615a..b9158a4f3b 100644 --- a/core/libs/prx/libScePad/src/PadState.cpp +++ b/core/libs/prx/libScePad/src/PadState.cpp @@ -28,8 +28,8 @@ PadData Pad::ReadState() { data.left_stick_y = state.sticks[1]; data.right_stick_x = state.sticks[2]; data.right_stick_y = state.sticks[3]; - data.analog_buttons_l2 = (state.buttons & 0x100) != 0 ? 255 : 0; - data.analog_buttons_r2 = (state.buttons & 0x200) != 0 ? 255 : 0; + data.analog_buttons_l2 = state.analogButtonsL2; + data.analog_buttons_r2 = state.analogButtonsR2; data.acceleration_y = 1.0f; data.orientation_w = 1.0f; data.connected = true; @@ -47,7 +47,9 @@ PadData Pad::ReadState() { extern "C" void PadPublishInput_nid_postfix(const PadInputState& input) { std::lock_guard lock(stateMutex); if (failure) std::rethrow_exception(failure); - if (state.buttons == input.buttons && state.sticks == input.sticks && state.touchLeft == input.touchLeft && state.touchRight == input.touchRight) return; + if (state.buttons == input.buttons && state.sticks == input.sticks && + state.analogButtonsL2 == input.analogButtonsL2 && state.analogButtonsR2 == input.analogButtonsR2 && + state.touchLeft == input.touchLeft && state.touchRight == input.touchRight) return; state = input; timestamp = sceKernelGetProcessTime(); } diff --git a/core/libs/prx/libSceVideoOut/include/PadInput.hpp b/core/libs/prx/libSceVideoOut/include/PadInput.hpp index 99928b941e..f06f904e91 100644 --- a/core/libs/prx/libSceVideoOut/include/PadInput.hpp +++ b/core/libs/prx/libSceVideoOut/include/PadInput.hpp @@ -2,6 +2,7 @@ #define CORE_LIBS_PRX_LIBSCEVIDEOOUT_PADINPUT_HPP #include "SDL_events.h" +#include "SDL_gamecontroller.h" #include "prx/libScePad/include/InputMapping.hpp" #include #include @@ -12,17 +13,22 @@ class DisplayWindow; class PadInput { public: PadInput(); + ~PadInput(); void HandleEvent(const SDL_Event& event, DisplayWindow& window); void Update(); private: void publish(); void setMouseMode(bool enabled); + void openFirstAvailableController(); + void openController(int deviceIndex); + void closeController(); std::vector bindings; std::vector pressed; std::vector wheelReleaseTimes; std::array mouseStick{128, 128}; std::chrono::steady_clock::time_point nextMousePoll{}; + SDL_GameController* controller = nullptr; bool mouseEnabled = false; }; diff --git a/core/libs/prx/libSceVideoOut/src/PadInput.cpp b/core/libs/prx/libSceVideoOut/src/PadInput.cpp index ff08df7518..e1aeb6018a 100644 --- a/core/libs/prx/libSceVideoOut/src/PadInput.cpp +++ b/core/libs/prx/libSceVideoOut/src/PadInput.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -9,9 +10,42 @@ #include "prx/libSceVideoOut/include/DisplayWindow.hpp" #include "prx/libScePad/include/PadState.hpp" #include "prx/libScePad/include/PadInputTypes.hpp" +#include "general/LogMacros.hpp" PadInput::PadInput() - : bindings(Pad::LoadInputMapping()), pressed(bindings.size()), wheelReleaseTimes(bindings.size()) {} + : bindings(Pad::LoadInputMapping()), pressed(bindings.size()), wheelReleaseTimes(bindings.size()) { + openFirstAvailableController(); +} + +PadInput::~PadInput() { + closeController(); +} + +void PadInput::openFirstAvailableController() { + if (controller != nullptr) return; + for (int deviceIndex = 0; deviceIndex < SDL_NumJoysticks(); ++deviceIndex) { + if (!SDL_IsGameController(deviceIndex)) continue; + openController(deviceIndex); + if (controller != nullptr) return; + } +} + +void PadInput::openController(int deviceIndex) { + if (controller != nullptr || !SDL_IsGameController(deviceIndex)) return; + controller = SDL_GameControllerOpen(deviceIndex); + if (controller == nullptr) { + APS5_LOG_ERR("Pad: could not open game controller %d: %s", deviceIndex, SDL_GetError()); + return; + } + const char* name = SDL_GameControllerName(controller); + APS5_LOG_OUT("Pad: connected game controller: %s", name != nullptr ? name : "unknown"); +} + +void PadInput::closeController() { + if (controller == nullptr) return; + SDL_GameControllerClose(controller); + controller = nullptr; +} void PadInput::setMouseMode(bool enabled) { if (SDL_SetRelativeMouseMode(enabled ? SDL_TRUE : SDL_FALSE) != 0) throw std::runtime_error(std::string("Pad: relative mouse mode failed: ") + SDL_GetError()); @@ -24,6 +58,24 @@ void PadInput::setMouseMode(bool enabled) { } void PadInput::HandleEvent(const SDL_Event& event, DisplayWindow& window) { + if (event.type == SDL_CONTROLLERDEVICEADDED) { + openController(event.cdevice.which); + return; + } + if (event.type == SDL_CONTROLLERDEVICEREMOVED && controller != nullptr) { + const auto instanceId = SDL_JoystickInstanceID(SDL_GameControllerGetJoystick(controller)); + if (instanceId == event.cdevice.which) { + closeController(); + openFirstAvailableController(); + publish(); + } + return; + } + if (event.type == SDL_CONTROLLERBUTTONDOWN || event.type == SDL_CONTROLLERBUTTONUP || + event.type == SDL_CONTROLLERAXISMOTION || event.type == SDL_CONTROLLERDEVICEREMAPPED) { + publish(); + return; + } if (event.type == SDL_WINDOWEVENT && (event.window.event == SDL_WINDOWEVENT_FOCUS_LOST || event.window.event == SDL_WINDOWEVENT_CLOSE)) { std::fill(pressed.begin(), pressed.end(), false); std::fill(wheelReleaseTimes.begin(), wheelReleaseTimes.end(), std::chrono::steady_clock::time_point{}); @@ -73,6 +125,10 @@ void PadInput::HandleEvent(const SDL_Event& event, DisplayWindow& window) { void PadInput::Update() { const auto now = std::chrono::steady_clock::now(); + if (controller != nullptr) { + SDL_GameControllerUpdate(); + publish(); + } bool released = false; for (std::size_t index = 0; index < bindings.size(); ++index) { if (bindings[index].wheelDirection == 0 || !pressed[index] || now < wheelReleaseTimes[index]) continue; @@ -106,13 +162,59 @@ void PadInput::Update() { void PadInput::publish() { PadInputState state; + if (controller != nullptr && SDL_GameControllerGetAttached(controller) == SDL_TRUE) { + const auto readButton = [this](SDL_GameControllerButton button) { + return SDL_GameControllerGetButton(controller, button) != 0; + }; + const auto addButton = [&state, &readButton](SDL_GameControllerButton source, Pad::PadButton button) { + if (readButton(source)) state.buttons |= static_cast(button); + }; + addButton(SDL_CONTROLLER_BUTTON_A, Pad::PadButton::Cross); + addButton(SDL_CONTROLLER_BUTTON_B, Pad::PadButton::Circle); + addButton(SDL_CONTROLLER_BUTTON_X, Pad::PadButton::Square); + addButton(SDL_CONTROLLER_BUTTON_Y, Pad::PadButton::Triangle); + addButton(SDL_CONTROLLER_BUTTON_LEFTSHOULDER, Pad::PadButton::L1); + addButton(SDL_CONTROLLER_BUTTON_RIGHTSHOULDER, Pad::PadButton::R1); + addButton(SDL_CONTROLLER_BUTTON_START, Pad::PadButton::Options); + addButton(SDL_CONTROLLER_BUTTON_LEFTSTICK, Pad::PadButton::L3); + addButton(SDL_CONTROLLER_BUTTON_RIGHTSTICK, Pad::PadButton::R3); + addButton(SDL_CONTROLLER_BUTTON_DPAD_UP, Pad::PadButton::Up); + addButton(SDL_CONTROLLER_BUTTON_DPAD_RIGHT, Pad::PadButton::Right); + addButton(SDL_CONTROLLER_BUTTON_DPAD_DOWN, Pad::PadButton::Down); + addButton(SDL_CONTROLLER_BUTTON_DPAD_LEFT, Pad::PadButton::Left); + addButton(SDL_CONTROLLER_BUTTON_TOUCHPAD, Pad::PadButton::TouchPad); + + const auto triggerValue = [this](SDL_GameControllerAxis axis) { + const auto value = std::clamp(SDL_GameControllerGetAxis(controller, axis), 0, 32767); + return static_cast((value * 255 + 16383) / 32767); + }; + state.analogButtonsL2 = triggerValue(SDL_CONTROLLER_AXIS_TRIGGERLEFT); + state.analogButtonsR2 = triggerValue(SDL_CONTROLLER_AXIS_TRIGGERRIGHT); + if (state.analogButtonsL2 != 0) state.buttons |= static_cast(Pad::PadButton::L2); + if (state.analogButtonsR2 != 0) state.buttons |= static_cast(Pad::PadButton::R2); + + const auto stickValue = [this](SDL_GameControllerAxis axis) { + const auto value = static_cast(SDL_GameControllerGetAxis(controller, axis)) + 32768; + return static_cast((value * 255 + 32767) / 65535); + }; + state.sticks = { + stickValue(SDL_CONTROLLER_AXIS_LEFTX), + stickValue(SDL_CONTROLLER_AXIS_LEFTY), + stickValue(SDL_CONTROLLER_AXIS_RIGHTX), + stickValue(SDL_CONTROLLER_AXIS_RIGHTY) + }; + } std::array negative{}; std::array positive{}; for (std::size_t index = 0; index < bindings.size(); ++index) { if (!pressed[index]) continue; const auto& binding = bindings[index]; switch (binding.control) { - case Pad::InputControl::Button: state.buttons |= static_cast(binding.button); break; + case Pad::InputControl::Button: + state.buttons |= static_cast(binding.button); + if (binding.button == Pad::PadButton::L2) state.analogButtonsL2 = 255; + if (binding.button == Pad::PadButton::R2) state.analogButtonsR2 = 255; + break; case Pad::InputControl::LeftStickLeft: negative[0] = true; break; case Pad::InputControl::LeftStickRight: positive[0] = true; break; case Pad::InputControl::LeftStickUp: negative[1] = true; break; @@ -128,7 +230,7 @@ void PadInput::publish() { } } for (std::size_t axis = 0; axis < state.sticks.size(); ++axis) { - state.sticks[axis] = negative[axis] == positive[axis] ? 128 : negative[axis] ? 0 : 255; + if (negative[axis] || positive[axis]) state.sticks[axis] = negative[axis] == positive[axis] ? 128 : negative[axis] ? 0 : 255; } if (mouseEnabled) { state.sticks[2] = mouseStick[0]; diff --git a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp index f1ad469269..d413eab44f 100644 --- a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp +++ b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp @@ -198,8 +198,8 @@ VideoOutDriver& VideoOutDriver::Get() { } VideoOutDriver::VideoOutDriver() { - if (SDL_InitSubSystem(SDL_INIT_VIDEO) < 0) { - throw std::runtime_error(std::string("SDL_InitSubSystem(VIDEO) failed: ") + SDL_GetError()); + if (SDL_InitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER) < 0) { + throw std::runtime_error(std::string("SDL_InitSubSystem(VIDEO | GAMECONTROLLER) failed: ") + SDL_GetError()); } try { AgcDriverWaitIdle_nid_postfix(); @@ -217,7 +217,7 @@ VideoOutDriver::VideoOutDriver() { flipQueue->changed.notify_all(); presentThread.join(); } - SDL_QuitSubSystem(SDL_INIT_VIDEO); + SDL_QuitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER); throw; } } @@ -249,7 +249,7 @@ void VideoOutDriver::Shutdown() { AgcDriverReleaseWindow_nid_postfix(window.Handle()); window.Destroy(); } - SDL_QuitSubSystem(SDL_INIT_VIDEO); + SDL_QuitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER); stopped = true; std::lock_guard lock(flipQueue->mutex); if (flipQueue->failure) std::rethrow_exception(flipQueue->failure); diff --git a/docs/INPUT_MAPPING.md b/docs/INPUT_MAPPING.md index da678ffbd8..56509963d9 100644 --- a/docs/INPUT_MAPPING.md +++ b/docs/INPUT_MAPPING.md @@ -1,6 +1,8 @@ -# Keyboard and mouse mapping +# Controller, keyboard and mouse input -AnyPS5 uses its built-in keyboard and mouse bindings when no configuration file is present. To change selected bindings, create `anyps5-input.ini` beside the generated game executable. Set `ANYPS5_INPUT_CONFIG` to use a file at another path. +AnyPS5 accepts SDL-mapped game controllers as well as its built-in keyboard and mouse bindings. Xbox-style controllers and PlayStation controllers recognized by SDL use the standard PS button layout; sticks and analog triggers are passed through, and controllers can be connected or disconnected while the game is running. The first recognized controller is used. AnyPS5 uses its built-in keyboard and mouse bindings when no configuration file is present. To change selected keyboard and mouse bindings, create `anyps5-input.ini` beside the generated game executable. Set `ANYPS5_INPUT_CONFIG` to use a file at another path. + +For an Xbox-layout controller, A/B/X/Y map to Cross/Circle/Square/Triangle, LB/RB map to L1/R1, and LT/RT map to analog L2/R2. Start maps to Options. D-pad and stick-click buttons are supported. On a PlayStation controller, the touchpad click button is forwarded; finger touch coordinates are not implemented. Each non-empty line has the form `Action = Type:Value`. Action names are case-insensitive. A `#` or `;` starts a comment. The first line for an action replaces its built-in bindings; later lines for the same action add alternate inputs. Duplicate entries for the same action are ignored. Actions omitted from the file keep their built-in bindings. UTF-8 files with or without a byte-order mark are supported. @@ -10,6 +12,8 @@ Supported input sources are: - `MOUSE:Left`, `MOUSE:Middle`, `MOUSE:Right`, `MOUSE:X1`, or `MOUSE:X2`. - `WHEEL:Up` or `WHEEL:Down` for pad buttons. +Game-controller bindings are enabled automatically and are not affected by keyboard or mouse overrides in this file. + Supported actions are `Cross`, `Circle`, `Triangle`, `Square`, `L1`, `R1`, `L2`, `R2`, `L3`, `R3`, `Options`, `Up`, `Right`, `Down`, `Left`, `LeftStickLeft`, `LeftStickRight`, `LeftStickUp`, `LeftStickDown`, `RightStickLeft`, `RightStickRight`, `RightStickUp`, `RightStickDown`, `TouchLeft`, `TouchRight`, `ToggleMouse`, and `ToggleFullscreen`. For example, this changes Cross to F or Space, moves the left stick to IJKL, uses the mouse buttons for Square and R2, and keeps all other built-in bindings: From ff1fa9eb69e81c81747ad44a6045451794b02624 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 16:37:15 +1300 Subject: [PATCH 81/93] feat(relinker): lower register-form EXTRQ and relocate following instructions for short sites Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- .../x86/IAmd64OnlyInstructionMatcher.hpp | 9 +- .../include/codegen/x86/Sse4aLowering.hpp | 4 +- .../codegen/src/Amd64OnlyConverter.cpp | 54 ++++++-- .../src/x86/Amd64OnlyInstructionMatcher.cpp | 56 ++++++-- .../codegen/src/x86/Sse4aLowering.cpp | 122 ++++++++++++------ .../codegen/tests/Amd64OnlyConverterTests.cpp | 75 ++++++++++- docs/TechnicalDebt.md | 2 +- 7 files changed, 256 insertions(+), 66 deletions(-) diff --git a/core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp b/core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp index d001f34ae7..65d6f04c1e 100644 --- a/core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp +++ b/core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace Codegen { @@ -14,7 +15,13 @@ public: [[nodiscard]] virtual std::optional Match( const std::uint8_t* data, - std::size_t length + std::size_t length, + std::span trailing = {} + ) const = 0; + + [[nodiscard]] virtual std::optional MatchSequence( + std::span> instructions, + std::span trailing ) const = 0; }; diff --git a/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp b/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp index 6064b9cca1..cff554934c 100644 --- a/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace Codegen { @@ -17,7 +18,8 @@ struct LoweredBody { class Sse4aLowering { public: [[nodiscard]] std::optional> LowerInPlace(const Sse4aOperands& operands, std::size_t originalLength) const; - [[nodiscard]] LoweredBody LowerOutOfLine(const Sse4aOperands& operands) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const Sse4aOperands& operands, std::span trailing = {}) const; + [[nodiscard]] LoweredBody LowerOutOfLine(std::span sequence, std::span trailing) const; }; } diff --git a/core/relinker/codegen/src/Amd64OnlyConverter.cpp b/core/relinker/codegen/src/Amd64OnlyConverter.cpp index 41cd3c2116..887d4154f4 100644 --- a/core/relinker/codegen/src/Amd64OnlyConverter.cpp +++ b/core/relinker/codegen/src/Amd64OnlyConverter.cpp @@ -8,7 +8,9 @@ #include #include #include +#include #include +#include namespace Codegen { @@ -32,6 +34,7 @@ public: private: struct Pending { + std::size_t Index; InstructionMatch Instruction; Amd64OnlyMatch Substitution; }; @@ -89,10 +92,11 @@ void Amd64OnlyConverter::_convertSegment( const auto matches = _atFileOffset(ph.Offset, [&] { return _scanner->ScanCodeSection(seg, 0, seg.size()); }); std::vector pending; - for (const auto& match : matches) { + for (std::size_t index = 0; index < matches.size(); ++index) { + const auto& match = matches[index]; auto substitution = _atFileOffset(ph.Offset + match.Offset, [&] { return _matcher->Match(seg.data() + match.Offset, match.Length); }); if (substitution.has_value()) - pending.push_back({match, std::move(*substitution)}); + pending.push_back({index, match, std::move(*substitution)}); } if (pending.empty()) return; @@ -102,7 +106,10 @@ void Amd64OnlyConverter::_convertSegment( }); const auto branchTargets = needsBranchTargets ? _collectBranchTargets(seg, matches, ph) : std::set{}; + std::set consumed; for (const auto& item : pending) { + if (consumed.contains(item.Index)) + continue; const auto& match = item.Instruction; const auto& substitution = item.Substitution; const auto fileOffset = static_cast(ph.Offset + match.Offset); @@ -119,21 +126,48 @@ void Amd64OnlyConverter::_convertSegment( break; } case Amd64OnlyLowering::Trampoline: { - if (match.Length < Amd64OnlySubstitutionTable::kJmpRel32.Size) - throw CodegenException("AMD-only instruction too short for a jump", fileOffset); + std::size_t siteLength = match.Length; + auto stub = substitution; + if (siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size) { + const X64InstructionDecoder decoder; + std::vector> sequence{std::span(seg.data() + match.Offset, match.Length)}; + std::size_t trailingBytes = 0; + for (auto next = item.Index + 1; siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size; ++next) { + if (next >= matches.size() || matches[next].Offset != match.Offset + siteLength) + throw CodegenException("AMD-only instruction too short for a jump and not followed by an instruction", fileOffset); + const auto& following = matches[next]; + const std::span bytes(seg.data() + following.Offset, following.Length); + const auto info = decoder.DecodeInstruction(bytes.data(), bytes.size()); + const bool amdOnly = _matcher->Match(bytes.data(), bytes.size()).has_value(); + if (amdOnly && trailingBytes == 0) { + sequence.push_back(bytes); + consumed.insert(next); + } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget) { + throw CodegenException("AMD-only instruction too short for a jump is followed by an instruction that cannot move", ph.Offset + following.Offset); + } else { + trailingBytes += following.Length; + } + siteLength += following.Length; + } + const std::span trailing(seg.data() + match.Offset + siteLength - trailingBytes, trailingBytes); + auto relocated = _atFileOffset(fileOffset, [&] { return _matcher->MatchSequence(sequence, trailing); }); + if (!relocated.has_value() || relocated->Lowering != Amd64OnlyLowering::Trampoline) + throw CodegenException("AMD-only instruction sequence has no out-of-line lowering", fileOffset); + stub = std::move(*relocated); + } const auto hit = branchTargets.upper_bound(address); - if (hit != branchTargets.end() && *hit < address + match.Length) + if (hit != branchTargets.end() && *hit < address + siteLength) throw CodegenException("Branch enters an AMD-only instruction", ph.Offset + (*hit - ph.MappedAddress)); const auto begin = seg.begin() + static_cast(match.Offset); result.Trampolines.push_back({ fileOffset, address, - match.Length, - std::vector(begin, begin + static_cast(match.Length)), - substitution.StubBody, - substitution.ReturnBranchOffset + siteLength, + std::vector(begin, begin + static_cast(siteLength)), + stub.StubBody, + stub.ReturnBranchOffset }); - replacementLength = substitution.StubBody.size(); + replacementLength = stub.StubBody.size(); break; } case Amd64OnlyLowering::Unsupported: diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index f371299d3a..78548a1e45 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include namespace Codegen { @@ -21,14 +23,20 @@ class Amd64OnlyInstructionMatcher : public IAmd64OnlyInstructionMatcher { public: [[nodiscard]] std::optional Match( const std::uint8_t* data, - std::size_t length + std::size_t length, + std::span trailing = {} + ) const override; + + [[nodiscard]] std::optional MatchSequence( + std::span> instructions, + std::span trailing ) const override; private: Sse4aLowering _lowering; [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const; - [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry) const; + [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const; }; Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstruction& instr, const Entry& entry) const { @@ -44,19 +52,45 @@ Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstructio return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0}; } -Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry) const { +Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const { const auto operands = DecodeSse4a(instr.Data, instr.Length); - if (operands.RegisterForm) + if (operands.RegisterForm && operands.Insertq) return _unsupported(registerFormEntry, instr.Length); - if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length)) - return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0}; - auto body = _lowering.LowerOutOfLine(operands); - return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; + if (!operands.RegisterForm && trailing.empty()) { + if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length)) + return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0}; + } + auto body = _lowering.LowerOutOfLine(operands, trailing); + const auto& name = operands.RegisterForm ? registerFormEntry.Name : entry.Name; + return Amd64OnlyMatch{name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; +} + +std::optional Amd64OnlyInstructionMatcher::MatchSequence( + std::span> instructions, + std::span trailing +) const { + if (instructions.empty()) + return std::nullopt; + std::vector sequence; + for (const auto& instruction : instructions) { + const DecodedInstruction instr{instruction.data(), instruction.size()}; + if (!instr.IsExtrq() && !instr.IsInsertq()) + return std::nullopt; + const auto operands = DecodeSse4a(instr.Data, instr.Length); + if (operands.RegisterForm && operands.Insertq) + return std::nullopt; + sequence.push_back(operands); + } + const auto& first = sequence.front(); + const auto& name = first.RegisterForm ? (first.Insertq ? kInsertqRegisterForm.Name : kExtrqRegisterForm.Name) : (first.Insertq ? kInsertq.Name : kExtrq.Name); + auto body = _lowering.LowerOutOfLine(std::span(sequence), trailing); + return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; } std::optional Amd64OnlyInstructionMatcher::Match( const std::uint8_t* data, - std::size_t length + std::size_t length, + std::span trailing ) const { const DecodedInstruction instr{data, length}; @@ -67,10 +101,10 @@ std::optional Amd64OnlyInstructionMatcher::Match( return _matchMovnts(instr, kMovntsd); if (instr.IsExtrq()) - return _matchSse4a(instr, kExtrq, kExtrqRegisterForm); + return _matchSse4a(instr, kExtrq, kExtrqRegisterForm, trailing); if (instr.IsInsertq()) - return _matchSse4a(instr, kInsertq, kInsertqRegisterForm); + return _matchSse4a(instr, kInsertq, kInsertqRegisterForm, trailing); if (instr.IsMonitorx()) return _unsupported(kMonitorx, length); diff --git a/core/relinker/codegen/src/x86/Sse4aLowering.cpp b/core/relinker/codegen/src/x86/Sse4aLowering.cpp index 758028a365..d183ec9bbb 100644 --- a/core/relinker/codegen/src/x86/Sse4aLowering.cpp +++ b/core/relinker/codegen/src/x86/Sse4aLowering.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace Codegen { @@ -89,6 +90,10 @@ public: _bytes.insert(_bytes.end(), kLeaRspRestore.Bytes, kLeaRspRestore.Bytes + kLeaRspRestore.Size); } + void Raw(std::span bytes) { + _bytes.insert(_bytes.end(), bytes.begin(), bytes.end()); + } + LoweredBody Finish() { const auto returnBranchOffset = _bytes.size(); _bytes.insert(_bytes.end(), kJmpRel32.Bytes, kJmpRel32.Bytes + kJmpRel32.Size); @@ -120,52 +125,41 @@ private: std::vector _constants; }; -} - -std::optional> Sse4aLowering::LowerInPlace(const Sse4aOperands& operands, const std::size_t originalLength) const { - if (operands.RegisterForm) - return std::nullopt; - const auto length = operands.Length; - const auto index = operands.Index; - const auto dst = operands.Destination; - const auto src = operands.Source; - std::vector sequence; - if (operands.Insertq) { - if (dst == src && index == 0) { - } else if (length == kFieldBits && index == 0) { - _sse(sequence, kPrefixScalar, {0x0F, 0x7E}, dst, src); - } else if (index == 0 && length % 16 == 0) { - _sse(sequence, kPrefixPacked, {0x0F, 0x3A, 0x0E}, dst, src); - sequence.push_back(static_cast((1u << (length / 16)) - 1)); - } else { - return std::nullopt; - } - } else { - if (index == 0 && length == kFieldBits) { - } else if (index + length == kFieldBits) { - _shiftImm(sequence, kShiftRight, dst, index); - } else if (index == 0 && (length == 8 || length == 16 || length == 32)) { - const std::uint8_t opcode = length == 8 ? 0x32 : (length == 16 ? 0x34 : 0x35); - _sse(sequence, kPrefixPacked, {0x0F, 0x38, opcode}, dst, dst); - } else { - return std::nullopt; - } +void _emitOutOfLine(BodyBuilder& body, const Sse4aOperands& operands) { + if (operands.RegisterForm) { + if (operands.Insertq) + throw CodegenException("INSERTQ register form has no Intel lowering"); + const auto dst = operands.Destination; + const auto src = operands.Source; + std::array scratch{}; + for (std::uint8_t reg = 0, found = 0; found < scratch.size(); ++reg) + if (reg != dst && reg != src) scratch[found++] = reg; + Constant fieldMask{}; + fieldMask[0] = kFieldBits - 1; + Constant one{}; + one[0] = 1; + body.Spill(scratch[0]); + body.Spill(scratch[1]); + body.Sse(kPrefixPacked, {0x0F, 0x6F}, scratch[0], src); + body.ShiftImm(kShiftRight, scratch[0], 8); + body.RipOperand({0x0F, 0xDB}, scratch[0], fieldMask); + body.Sse(kPrefixPacked, {0x0F, 0x6F}, scratch[1], src); + body.RipOperand({0x0F, 0xDB}, scratch[1], fieldMask); + body.RipOperand({0x0F, 0xEF}, scratch[1], fieldMask); + body.RipOperand({0x0F, 0xD4}, scratch[1], one); + body.RipOperand({0x0F, 0xDB}, scratch[1], fieldMask); + body.Sse(kPrefixPacked, {0x0F, 0xD3}, dst, scratch[0]); + body.Sse(kPrefixPacked, {0x0F, 0xF3}, dst, scratch[1]); + body.Sse(kPrefixPacked, {0x0F, 0xD3}, dst, scratch[1]); + body.Restore(scratch[1]); + body.Restore(scratch[0]); + return; } - if (sequence.size() > originalLength) - return std::nullopt; - _nopFill(sequence, originalLength - sequence.size()); - return sequence; -} - -LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands) const { - if (operands.RegisterForm) - throw CodegenException("EXTRQ/INSERTQ register form has no Intel lowering"); const auto length = operands.Length; const auto index = operands.Index; const auto dst = operands.Destination; const auto src = operands.Source; const bool byteAligned = length % 8 == 0 && index % 8 == 0; - BodyBuilder body; if (!operands.Insertq) { if (byteAligned) { Constant mask; @@ -208,6 +202,54 @@ LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands) const { body.Sse(kPrefixPacked, {0x0F, 0xEF}, dst, scratch); body.Restore(scratch); } +} + +} + +std::optional> Sse4aLowering::LowerInPlace(const Sse4aOperands& operands, const std::size_t originalLength) const { + if (operands.RegisterForm) + return std::nullopt; + const auto length = operands.Length; + const auto index = operands.Index; + const auto dst = operands.Destination; + const auto src = operands.Source; + std::vector sequence; + if (operands.Insertq) { + if (dst == src && index == 0) { + } else if (length == kFieldBits && index == 0) { + _sse(sequence, kPrefixScalar, {0x0F, 0x7E}, dst, src); + } else if (index == 0 && length % 16 == 0) { + _sse(sequence, kPrefixPacked, {0x0F, 0x3A, 0x0E}, dst, src); + sequence.push_back(static_cast((1u << (length / 16)) - 1)); + } else { + return std::nullopt; + } + } else { + if (index == 0 && length == kFieldBits) { + } else if (index + length == kFieldBits) { + _shiftImm(sequence, kShiftRight, dst, index); + } else if (index == 0 && (length == 8 || length == 16 || length == 32)) { + const std::uint8_t opcode = length == 8 ? 0x32 : (length == 16 ? 0x34 : 0x35); + _sse(sequence, kPrefixPacked, {0x0F, 0x38, opcode}, dst, dst); + } else { + return std::nullopt; + } + } + if (sequence.size() > originalLength) + return std::nullopt; + _nopFill(sequence, originalLength - sequence.size()); + return sequence; +} + +LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands, std::span trailing) const { + return LowerOutOfLine(std::span(&operands, 1), trailing); +} + +LoweredBody Sse4aLowering::LowerOutOfLine(std::span sequence, std::span trailing) const { + BodyBuilder body; + for (const auto& operands : sequence) + _emitOutOfLine(body, operands); + body.Raw(trailing); return body.Finish(); } diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index 88a57717d3..280e829809 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -11,6 +11,10 @@ #include #include #include +#include +#ifdef __linux__ +#include +#endif #include #include #include @@ -138,7 +142,9 @@ void matcherSubstitutions() { const auto monitorx = match({0x0F, 0x01, 0xFA}); require(monitorx && monitorx->Lowering == Codegen::Amd64OnlyLowering::Unsupported && monitorx->InstructionName == "MONITORX", "MONITORX was not reported as unsupported"); const auto registerForm = match({0x66, 0x0F, 0x79, 0xCA}); - require(registerForm && registerForm->Lowering == Codegen::Amd64OnlyLowering::Unsupported, "EXTRQ register form was not reported as unsupported"); + require(registerForm && registerForm->Lowering == Codegen::Amd64OnlyLowering::Trampoline && registerForm->InstructionName == "EXTRQ register form", "EXTRQ register form was not lowered through a stub"); + const auto insertqRegisterForm = match({0xF2, 0x0F, 0x79, 0xCA}); + require(insertqRegisterForm && insertqRegisterForm->Lowering == Codegen::Amd64OnlyLowering::Unsupported, "INSERTQ register form was not reported as unsupported"); require(!match({0x66, 0x0F, 0x2B, 0x07}) && !match({0x0F, 0x2B, 0x07}) && !match({0x48, 0x8B, 0x05, 0, 0, 0, 0}), "Ordinary instruction was matched"); const auto stub = match(kInsertqHighSite); require(stub && stub->Lowering == Codegen::Amd64OnlyLowering::Trampoline && stub->StubBody == kInsertqHighBody && stub->ReturnBranchOffset == 15 && stub->InstructionName == "INSERTQ", "INSERTQ was not lowered through a stub"); @@ -221,7 +227,14 @@ void converterSegment() { auto registerForm = file; const Bytes extrqRegister = {0x66, 0x0F, 0x79, 0xCA}; std::copy(extrqRegister.begin(), extrqRegister.end(), registerForm.begin() + 0x20F); - requireFailure([&] { (void)converter->Convert(registerForm, {segmentHeader(20)}); }, "EXTRQ register form was silently kept"); + requireFailure([&] { (void)converter->Convert(registerForm, {segmentHeader(20)}); }, "Short EXTRQ followed by a return was relocated"); + registerForm[0x213] = 0x90; + const auto relocated = converter->Convert(registerForm, {segmentHeader(20)}); + require(relocated.Trampolines.size() == 2, "Short EXTRQ register form was not lowered through a stub"); + const auto& shortSite = relocated.Trampolines[1]; + const Bytes shortOriginal = {0x66, 0x0F, 0x79, 0xCA, 0x90}; + require(shortSite.Offset == 0x20F && shortSite.Length == 5 && shortSite.OriginalBytes == shortOriginal, "Short EXTRQ site did not absorb the following instruction"); + require(shortSite.Body[shortSite.ReturnBranchOffset - 1] == 0x90 && shortSite.Body[shortSite.ReturnBranchOffset] == 0xE9, "Absorbed instruction does not run before the return jump"); requireFailure([&] { (void)converter->Convert(file, {segmentHeader(0x200)}); }, "Segment exceeding the file was accepted"); } @@ -322,12 +335,70 @@ void linuxPlacement() { } +#if defined(__linux__) && defined(__x86_64__) +std::uint64_t extrqReference(std::uint64_t value, std::uint64_t control) { + const auto length = static_cast(control & 0x3f); + const auto index = static_cast((control >> 8) & 0x3f); + const auto shifted = value >> index; + return length == 0 ? shifted : shifted & ((std::uint64_t{1} << length) - 1); +} + +std::uint64_t runExtrqStub(const Bytes& site, std::uint64_t destination, std::uint64_t control) { + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); + const auto match = matcher->Match(site.data(), site.size()); + require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "EXTRQ register form stub was not produced"); + auto body = match->StubBody; + const auto ret = body.size(); + body.push_back(0xC3); + const auto displacement = static_cast(ret - (match->ReturnBranchOffset + 5)); + std::memcpy(body.data() + match->ReturnBranchOffset + 1, &displacement, sizeof(displacement)); + void* code = mmap(nullptr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + require(code != MAP_FAILED, "cannot map executable memory for the stub"); + std::memcpy(code, body.data(), body.size()); + alignas(16) std::uint64_t destinationIn[2] = {destination, 0x1122334455667788ull}; + alignas(16) std::uint64_t controlIn[2] = {control, 0}; + alignas(16) std::uint64_t out[2] = {}; + alignas(16) std::uint64_t scratchIn[2] = {0x0123456789abcdefull, 0xfedcba9876543210ull}; + alignas(16) std::uint64_t scratchOut[2] = {}; + const bool same = site[3] == 0xD2; + asm volatile( + "movdqu (%[scratch]), %%xmm0\n\t" + "movdqu (%[dst]), %%xmm2\n\t" + "movdqu (%[ctl]), %%xmm5\n\t" + "sub $128, %%rsp\n\t" + "call *%[code]\n\t" + "add $128, %%rsp\n\t" + "movdqu %%xmm2, (%[out])\n\t" + "movdqu %%xmm0, (%[scratchOut])\n\t" + : + : [scratch] "r"(scratchIn), [dst] "r"(same ? controlIn : destinationIn), [ctl] "r"(controlIn), [code] "r"(code), [out] "r"(out), [scratchOut] "r"(scratchOut) + : "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "memory", "cc"); + munmap(code, 4096); + require(scratchOut[0] == scratchIn[0] && scratchOut[1] == scratchIn[1], "EXTRQ stub clobbered a scratch register"); + return out[0]; +} + +void registerFormExecution() { + const Bytes distinct = {0x66, 0x0F, 0x79, 0xD5}; + const Bytes same = {0x66, 0x0F, 0x79, 0xD2}; + const std::uint64_t value = 0x9e3779b97f4a7c15ull; + for (const auto [length, index] : {std::pair{8u, 4u}, {0u, 0u}, {40u, 20u}, {63u, 1u}, {1u, 63u}, {16u, 48u}}) { + const auto control = static_cast(length) | (static_cast(index) << 8) | 0xffffc000ull; + require(runExtrqStub(distinct, value, control) == extrqReference(value, control), "EXTRQ register form stub computed the wrong field"); + require(runExtrqStub(same, control, control) == extrqReference(control, control), "EXTRQ register form stub with equal operands computed the wrong field"); + } +} +#else +void registerFormExecution() {} +#endif + int main() { try { decoderLengths(); sse4aOperands(); matcherSubstitutions(); goldenBodies(); + registerFormExecution(); converterSegment(); converterFailureOffsets(); linuxPlacement(); diff --git a/docs/TechnicalDebt.md b/docs/TechnicalDebt.md index 6156464b1a..b044d536ac 100644 --- a/docs/TechnicalDebt.md +++ b/docs/TechnicalDebt.md @@ -34,7 +34,7 @@ Throughout the project, every function at every stage either **does exactly what - [Shader recompilation](../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../core/shader/recompiler) was written completely independently from [libs/prx](../core/libs/prx). - The executable file that [relinker](../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) generates opens the console when launched, which is inconvenient for playability. - [Relinker](../core/relinker/elfpatcher/src) doesn't add an icon to the generated executable. This should be done without adding dependencies (only standard). -- `--to-intel` does not lower the register forms of EXTRQ/INSERTQ (`0F 79`, 4 or 5 bytes, too short for a jump) nor MONITORX/MWAITX/CLZERO/RDPRU/MCOMMIT; the [lowering](../core/relinker/codegen/src/x86/Sse4aLowering.cpp) fails the relink instead. SHA-NI is not substituted. +- `--to-intel` does not lower the register form of INSERTQ (`F2 0F 79`) nor MONITORX/MWAITX/CLZERO/RDPRU/MCOMMIT; the [lowering](../core/relinker/codegen/src/x86/Sse4aLowering.cpp) fails the relink instead. SHA-NI is not substituted. - The length-changing path of the [instruction rewriter](../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust VEX/0F38/0F3A RIP-relative operands, data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. - The libc [SSE4a trap emulation](../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it. From f7e1bdf0b95e7da121912f48f46aae703edc5e5d Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 16:40:00 +1300 Subject: [PATCH 82/93] feat(libkernel): pread, pwrite, getdents and getdirentries on linux Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/CMakeLists.txt | 6 + core/libs/prx/libkernel/File/src/Stdio.cpp | 77 +++++++++++++ core/libs/tests/GuestDirectoryEntries.cpp | 122 +++++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 core/libs/tests/GuestDirectoryEntries.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index b0572b5bfe..f5425dfef7 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -328,6 +328,12 @@ if(BUILD_TESTING) set_tests_properties(guest_memory PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + if(NOT WIN32) + add_executable(guest_directory_entry_tests tests/GuestDirectoryEntries.cpp) + target_include_directories(guest_directory_entry_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(guest_directory_entry_tests PRIVATE libkernel libc) + add_test(NAME guest_directory_entries COMMAND guest_directory_entry_tests) + endif() add_executable(guest_search_tests tests/GuestSearch.cpp) target_include_directories(guest_search_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(guest_search_tests PRIVATE libc) diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 30ee4ccf79..ece3fae6d9 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -8,6 +8,7 @@ #include "prx/libkernel/File/include/NativeStat.hpp" #include "prx/libkernel/Socket/include/SocketRuntime.hpp" #include +#include #include #include #include @@ -78,6 +79,9 @@ static std::int64_t NativePwrite(int descriptor, const void* buf, std::size_t nb #include #include #include +#include +#include +#include static int NativeRmdir(const std::filesystem::path& path) { return ::rmdir(path.c_str()); } @@ -103,7 +107,10 @@ static std::int64_t NativePwrite(int descriptor, const void* buf, std::size_t nb static constexpr int GUEST_ENOENT = 2; static constexpr int GUEST_EIO = 5; +static constexpr int GUEST_EFAULT = 14; static constexpr int GUEST_EEXIST = 17; +static constexpr int GUEST_EINVAL = 22; +static constexpr int GUEST_ENAMETOOLONG = 63; static constexpr int GUEST_ENOTDIR = 20; static constexpr int GUEST_ENOTEMPTY = 66; @@ -283,6 +290,8 @@ int APS5_VABI sceKernelFsync(int fd) { return 0; } +#ifdef _WIN32 + int APS5_VABI sceKernelGetdents(int fd, char* buf, int nbytes) { (void)fd; (void)buf; @@ -300,6 +309,54 @@ int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* bas return 0; } +#else + +int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* basep) { + constexpr std::size_t GuestHeaderBytes = 8; + constexpr std::size_t GuestMaxName = 255; + if (buf == nullptr) return SceErrorFromErrno(GUEST_EFAULT); + if (nbytes <= 0) return SceErrorFromErrno(GUEST_EINVAL); + const off_t base = ::lseek(fd, 0, SEEK_CUR); + if (base < 0) return SceErrorFromErrno(errno); + if (basep != nullptr) *basep = static_cast(base); + std::vector host(static_cast(nbytes) < 4096 ? 4096 : static_cast(nbytes)); + const auto read = ::syscall(SYS_getdents64, fd, host.data(), host.size()); + if (read < 0) return SceErrorFromErrno(errno); + std::size_t written = 0; + off_t resume = base; + for (long offset = 0; offset < read;) { + const auto* entry = reinterpret_cast(host.data() + offset); + const auto nameLength = std::strlen(entry->d_name); + const auto record = (GuestHeaderBytes + nameLength + 1 + 3) & ~std::size_t{3}; + if (nameLength > GuestMaxName || written + record > static_cast(nbytes)) { + if (::lseek(fd, resume, SEEK_SET) < 0) return SceErrorFromErrno(errno); + if (written != 0) return static_cast(written); + return nameLength > GuestMaxName ? static_cast(0x80020000u | GUEST_ENAMETOOLONG) : SceErrorFromErrno(GUEST_EINVAL); + } + char* out = buf + written; + std::memset(out, 0, record); + const auto fileNumber = static_cast(entry->d_ino); + const auto recordLength = static_cast(record); + const auto nameBytes = static_cast(nameLength); + std::memcpy(out, &fileNumber, sizeof(fileNumber)); + std::memcpy(out + 4, &recordLength, sizeof(recordLength)); + out[6] = static_cast(entry->d_type); + out[7] = static_cast(nameBytes); + std::memcpy(out + GuestHeaderBytes, entry->d_name, nameLength); + written += record; + resume = entry->d_off; + offset += entry->d_reclen; + } + if (::lseek(fd, resume, SEEK_SET) < 0) return SceErrorFromErrno(errno); + return static_cast(written); +} + +int APS5_VABI sceKernelGetdents(int fd, char* buf, int nbytes) { + return sceKernelGetdirentries(fd, buf, nbytes, nullptr); +} + +#endif + int APS5_VABI sceKernelMkdir(const char* path, uint16_t mode) { (void)mode; if (path == nullptr) throw std::invalid_argument("sceKernelMkdir: path is null"); @@ -311,6 +368,8 @@ int APS5_VABI sceKernelMkdir(const char* path, uint16_t mode) { return 0; } +#ifdef _WIN32 + int64_t APS5_VABI sceKernelPread(int d, void* buf, size_t nbytes, int64_t offset) { (void)d; (void)buf; @@ -329,6 +388,24 @@ int64_t APS5_VABI sceKernelPwrite(int d, const void* buf, size_t nbytes, int64_t return 0; } +#else + +int64_t APS5_VABI sceKernelPread(int d, void* buf, size_t nbytes, int64_t offset) { + if (buf == nullptr && nbytes != 0) return SceErrorFromErrno(GUEST_EFAULT); + if (offset < 0) return SceErrorFromErrno(GUEST_EINVAL); + const auto result = NativePread(d, buf, nbytes, offset); + return result < 0 ? SceErrorFromErrno(errno) : result; +} + +int64_t APS5_VABI sceKernelPwrite(int d, const void* buf, size_t nbytes, int64_t offset) { + if (buf == nullptr && nbytes != 0) return SceErrorFromErrno(GUEST_EFAULT); + if (offset < 0) return SceErrorFromErrno(GUEST_EINVAL); + const auto result = NativePwrite(d, buf, nbytes, offset); + return result < 0 ? SceErrorFromErrno(errno) : result; +} + +#endif + int APS5_VABI sceKernelRename(const char* from, const char* to) { if (from == nullptr || to == nullptr) throw std::invalid_argument("sceKernelRename: path is null"); const auto source = ResolvePath_nid_no_patch(from); diff --git a/core/libs/tests/GuestDirectoryEntries.cpp b/core/libs/tests/GuestDirectoryEntries.cpp new file mode 100644 index 0000000000..912247d447 --- /dev/null +++ b/core/libs/tests/GuestDirectoryEntries.cpp @@ -0,0 +1,122 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libkernel/File/include/FileFlags.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceKernelOpen(const char*, int, std::uint16_t); +int APS5_VABI sceKernelClose(int); +std::int64_t APS5_VABI sceKernelRead(int, void*, std::size_t); +int APS5_VABI sceKernelLseek(int, std::int64_t, int); +int APS5_VABI sceKernelGetdents(int, char*, int); +int APS5_VABI sceKernelGetdirentries(int, char*, int, std::int64_t*); +std::int64_t APS5_VABI sceKernelPread(int, void*, std::size_t, std::int64_t); +std::int64_t APS5_VABI sceKernelPwrite(int, const void*, std::size_t, std::int64_t); +} + +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "Directory entry check failed at line %d\n", line); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) + +static constexpr int ErrorEinval = static_cast(0x80020016u); +static constexpr int ErrorEfault = static_cast(0x8002000Eu); +static constexpr int ErrorEbadf = static_cast(0x80020009u); + +struct Entry { + std::uint8_t type; + std::uint16_t recordLength; +}; + +static int Collect(const char* buffer, int bytes, std::map& entries) { + int records = 0; + for (int offset = 0; offset < bytes; ++records) { + std::uint16_t recordLength = 0; + std::memcpy(&recordLength, buffer + offset + 4, sizeof(recordLength)); + const auto nameLength = static_cast(buffer[offset + 7]); + const std::string name(buffer + offset + 8); + Require(name.size() == nameLength); + Require(recordLength == ((8 + nameLength + 1 + 3) & ~3)); + Require(offset + recordLength <= bytes); + Require(entries.emplace(name, Entry{static_cast(buffer[offset + 6]), recordLength}).second); + offset += recordLength; + } + return records; +} + +int main() { + const auto root = std::filesystem::path("anyps5-directory-entries-test-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + Require(std::filesystem::create_directory(root)); + const std::string longName = "a-much-longer-entry-name-0123456789"; + { std::ofstream stream(root / "data.bin", std::ios::binary); stream << "0123456789"; } + { std::ofstream stream(root / "b"); } + { std::ofstream stream(root / longName); } + Require(std::filesystem::create_directory(root / "sub")); + + const int directory = sceKernelOpen(root.string().c_str(), SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0); + Require(directory >= 0); + std::array small{}; + std::map entries; + int calls = 0; + std::int64_t previousBase = -1; + for (;;) { + std::int64_t base = -1; + const int read = sceKernelGetdirentries(directory, small.data(), static_cast(small.size()), &base); + Require(read >= 0 && read <= static_cast(small.size())); + Require(base >= 0 && base != previousBase); + if (calls == 0) Require(base == 0); + previousBase = base; + if (read == 0) break; + Require(Collect(small.data(), read, entries) > 0); + ++calls; + } + Require(calls >= 3); + Require(entries.size() == 6); + for (const char* name : {".", "..", "data.bin", "b", "sub"}) Require(entries.contains(name)); + Require(entries.contains(longName) && entries[longName].recordLength == 44); + Require(entries["b"].recordLength == 12); + Require(entries["sub"].type == 4 || entries["sub"].type == 0); + Require(entries["data.bin"].type == 8 || entries["data.bin"].type == 0); + + Require(sceKernelLseek(directory, 0, 0) == 0); + std::array tooSmall{}; + Require(sceKernelGetdents(directory, tooSmall.data(), static_cast(tooSmall.size())) == ErrorEinval); + Require(sceKernelGetdents(directory, nullptr, 64) == ErrorEfault); + Require(sceKernelGetdents(directory, small.data(), 0) == ErrorEinval); + std::array large{}; + std::map all; + const int readAll = sceKernelGetdents(directory, large.data(), static_cast(large.size())); + Require(readAll > 0 && Collect(large.data(), readAll, all) == 6); + Require(sceKernelGetdents(directory, large.data(), static_cast(large.size())) == 0); + Require(sceKernelClose(directory) == 0); + + const int file = sceKernelOpen((root / "data.bin").string().c_str(), SCE_KERNEL_O_RDWR, 0); + Require(file >= 0); + Require(sceKernelLseek(file, 2, 0) == 2); + char bytes[16] = {}; + Require(sceKernelPread(file, bytes, 4, 5) == 4 && std::memcmp(bytes, "5678", 4) == 0); + Require(sceKernelLseek(file, 0, 1) == 2); + Require(sceKernelPwrite(file, "XY", 2, 8) == 2); + Require(sceKernelLseek(file, 0, 1) == 2); + Require(sceKernelRead(file, bytes, 3) == 3 && std::memcmp(bytes, "234", 3) == 0); + Require(sceKernelPread(file, bytes, sizeof(bytes), 0) == 10 && std::memcmp(bytes, "01234567XY", 10) == 0); + Require(sceKernelPread(file, bytes, sizeof(bytes), 10) == 0); + Require(sceKernelPread(file, bytes, 1, -1) == ErrorEinval); + Require(sceKernelPwrite(file, nullptr, 1, 0) == ErrorEfault); + Require(sceKernelClose(file) == 0); + Require(sceKernelPread(file, bytes, 1, 0) == ErrorEbadf); + + std::filesystem::remove_all(root); +} From 390e4cd8006b66bcb03ddebac62c95d0fe9d3296 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 16:42:29 +1300 Subject: [PATCH 83/93] feat(json2): implement sce::Json values, strings, arrays, objects, parser and null-access callback Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/CMakeLists.txt | 8 + core/libs/prx/libSceJson2/Export.cpp | 711 ++++++++++++++++++++++++++- core/libs/tests/GuestJson.cpp | 250 ++++++++++ 3 files changed, 959 insertions(+), 10 deletions(-) create mode 100644 core/libs/tests/GuestJson.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index b0572b5bfe..d7553bc0f8 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -310,6 +310,14 @@ if(BUILD_TESTING) set_tests_properties(guest_rtc PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_executable(guest_json_tests tests/GuestJson.cpp) + target_include_directories(guest_json_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(guest_json_tests PRIVATE libSceJson2) + add_test(NAME guest_json COMMAND guest_json_tests) + if(WIN32) + set_tests_properties(guest_json PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_executable(mspace_tests tests/Mspace.cpp) target_include_directories(mspace_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(mspace_tests PRIVATE libc) diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 848de74a45..cd5ac7f294 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -1,19 +1,462 @@ +#include +#include #include #include +#include +#include +#include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +enum ValueType : std::int32_t { + TypeNull = 0, + TypeBoolean = 1, + TypeInteger = 2, + TypeUInteger = 3, + TypeReal = 4, + TypeString = 5, + TypeArray = 6, + TypeObject = 7, +}; + +constexpr int JsonErrorParse = static_cast(0x80920101); + +struct Node; +struct Value { Node* node; }; +struct String { std::string* text; }; +struct Array { std::list* items; }; +struct Pair; +struct Object { std::list* items; }; +struct ArrayIterator { std::list::iterator it; }; +struct ObjectIterator { std::list::iterator it; }; +struct Pair { + String key; + std::uint64_t reserved; + Value value; +}; +static_assert(sizeof(Value) == 8 && sizeof(String) == 8 && sizeof(Array) == 8 && sizeof(Object) == 8); +static_assert(sizeof(ArrayIterator) == 8 && sizeof(ObjectIterator) == 8); +static_assert(offsetof(Pair, value) == 0x10); + +struct Node { + ValueType type = TypeNull; + bool boolean = false; + std::int64_t integer = 0; + std::uint64_t uinteger = 0; + double real = 0; + String string{}; + Array array{}; + Object object{}; +}; + +using NullAccessCallback = const Value& (APS5_VABI*)(ValueType, const Value*, void*); + +struct Globals { + std::mutex mutex; + NullAccessCallback callback = nullptr; + void* context = nullptr; +}; + +Globals& State() { + static Globals globals; + return globals; +} + +void Construct(String& s, const std::string& text) { s.text = new std::string(text); } +void Construct(Array& a) { a.items = new std::list(); } +void Construct(Object& o) { o.items = new std::list(); } +void Destroy(Value& v); +void CopyInto(Value& destination, const Value& source); + +void Destroy(String& s) { delete s.text; s.text = nullptr; } +void Destroy(Array& a) { + if (!a.items) return; + for (auto& item : *a.items) Destroy(item); + delete a.items; + a.items = nullptr; +} +void Destroy(Object& o) { + if (!o.items) return; + for (auto& pair : *o.items) { + Destroy(pair.key); + Destroy(pair.value); + } + delete o.items; + o.items = nullptr; +} + +void Clear(Node& n) { + if (n.type == TypeString) Destroy(n.string); + if (n.type == TypeArray) Destroy(n.array); + if (n.type == TypeObject) Destroy(n.object); + n = Node{}; +} + +void Destroy(Value& v) { + if (!v.node) return; + Clear(*v.node); + delete v.node; + v.node = nullptr; +} + +Node& NodeOf(const Value& v) { + if (!v.node) throw std::runtime_error("sce::Json::Value used before construction"); + return *v.node; +} + +void Construct(Value& v) { v.node = new Node(); } + +void CopyArray(Array& destination, const Array& source) { + Construct(destination); + for (const auto& item : *source.items) { + Value copy{}; + CopyInto(copy, item); + destination.items->push_back(copy); + } +} + +void CopyObject(Object& destination, const Object& source) { + Construct(destination); + for (const auto& pair : *source.items) { + Pair copy{}; + Construct(copy.key, *pair.key.text); + CopyInto(copy.value, pair.value); + destination.items->push_back(copy); + } +} + +void AssignNode(Node& destination, const Node& source) { + Node copy{}; + copy.type = source.type; + copy.boolean = source.boolean; + copy.integer = source.integer; + copy.uinteger = source.uinteger; + copy.real = source.real; + if (source.type == TypeString) Construct(copy.string, *source.string.text); + if (source.type == TypeArray) CopyArray(copy.array, source.array); + if (source.type == TypeObject) CopyObject(copy.object, source.object); + Clear(destination); + destination = copy; +} + +void CopyInto(Value& destination, const Value& source) { + Construct(destination); + AssignNode(*destination.node, NodeOf(source)); +} + +void SetType(Node& n, ValueType type) { + Clear(n); + n.type = type; + if (type == TypeString) Construct(n.string, std::string()); + if (type == TypeArray) Construct(n.array); + if (type == TypeObject) Construct(n.object); + if (type < TypeNull || type > TypeObject) throw std::invalid_argument("sce::Json::Value::set: invalid value type"); +} + +const Value& Defaults(ValueType type) { + static Value values[8]{}; + static std::once_flag once; + std::call_once(once, [] { + for (int i = 0; i < 8; ++i) { + Construct(values[i]); + SetType(*values[i].node, static_cast(i)); + } + }); + return values[type]; +} + +const Value& NullAccess(ValueType requested, const Value* parent) { + auto& state = State(); + NullAccessCallback callback; + void* context; + { + std::lock_guard lock(state.mutex); + callback = state.callback; + context = state.context; + } + if (callback) { + const Value& result = callback(requested, parent, context); + if (result.node && (requested == TypeNull || result.node->type == requested)) return result; + } + return Defaults(requested); +} + +const Node& Typed(const Value* self, ValueType requested) { + const Node& n = NodeOf(*self); + if (n.type == requested) return n; + return NodeOf(NullAccess(requested, self)); +} + +void AppendEscaped(std::string& out, const std::string& text) { + out += '"'; + for (const unsigned char c : text) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) { + char buffer[8]; + std::snprintf(buffer, sizeof(buffer), "\\u%04x", c); + out += buffer; + } else out += static_cast(c); + } + } + out += '"'; +} + +std::string RealText(double value) { + if (!std::isfinite(value)) throw std::runtime_error("sce::Json: cannot serialize a non-finite number"); + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "%.17g", value); + return buffer; +} + +void Serialize(std::string& out, const Node& n) { + switch (n.type) { + case TypeNull: out += "null"; return; + case TypeBoolean: out += n.boolean ? "true" : "false"; return; + case TypeInteger: out += std::to_string(n.integer); return; + case TypeUInteger: out += std::to_string(n.uinteger); return; + case TypeReal: out += RealText(n.real); return; + case TypeString: AppendEscaped(out, *n.string.text); return; + case TypeArray: { + out += '['; + bool first = true; + for (const auto& item : *n.array.items) { + if (!first) out += ','; + first = false; + Serialize(out, NodeOf(item)); + } + out += ']'; + return; + } + case TypeObject: { + out += '{'; + bool first = true; + for (const auto& pair : *n.object.items) { + if (!first) out += ','; + first = false; + AppendEscaped(out, *pair.key.text); + out += ':'; + Serialize(out, NodeOf(pair.value)); + } + out += '}'; + return; + } + } + throw std::runtime_error("sce::Json: invalid value type"); +} + +class Parser { +public: + Parser(const char* text, std::size_t size) : _text(text), _end(text + size) {} + + bool Parse(Node& out) { + skip(); + if (!value(out)) return false; + skip(); + return _text == _end || *_text == '\0'; + } + +private: + const char* _text; + const char* _end; + + void skip() { while (_text < _end && (*_text == ' ' || *_text == '\t' || *_text == '\n' || *_text == '\r')) ++_text; } + bool literal(const char* word) { + const auto length = std::strlen(word); + if (static_cast(_end - _text) < length || std::memcmp(_text, word, length) != 0) return false; + _text += length; + return true; + } + static void appendUtf8(std::string& out, std::uint32_t code) { + if (code < 0x80) out += static_cast(code); + else if (code < 0x800) { out += static_cast(0xc0 | (code >> 6)); out += static_cast(0x80 | (code & 0x3f)); } + else if (code < 0x10000) { out += static_cast(0xe0 | (code >> 12)); out += static_cast(0x80 | ((code >> 6) & 0x3f)); out += static_cast(0x80 | (code & 0x3f)); } + else { out += static_cast(0xf0 | (code >> 18)); out += static_cast(0x80 | ((code >> 12) & 0x3f)); out += static_cast(0x80 | ((code >> 6) & 0x3f)); out += static_cast(0x80 | (code & 0x3f)); } + } + bool hex4(std::uint32_t& code) { + if (_end - _text < 4) return false; + code = 0; + for (int i = 0; i < 4; ++i) { + const char c = *_text++; + code <<= 4; + if (c >= '0' && c <= '9') code |= static_cast(c - '0'); + else if (c >= 'a' && c <= 'f') code |= static_cast(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') code |= static_cast(c - 'A' + 10); + else return false; + } + return true; + } + bool string(std::string& out) { + if (_text >= _end || *_text != '"') return false; + ++_text; + while (_text < _end && *_text != '"') { + const char c = *_text++; + if (static_cast(c) < 0x20) return false; + if (c != '\\') { out += c; continue; } + if (_text >= _end) return false; + switch (*_text++) { + case '"': out += '"'; break; + case '\\': out += '\\'; break; + case '/': out += '/'; break; + case 'b': out += '\b'; break; + case 'f': out += '\f'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'u': { + std::uint32_t code = 0; + if (!hex4(code)) return false; + if (code >= 0xd800 && code < 0xdc00) { + std::uint32_t low = 0; + if (!literal("\\u") || !hex4(low) || low < 0xdc00 || low >= 0xe000) return false; + code = 0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00); + } else if (code >= 0xdc00 && code < 0xe000) return false; + appendUtf8(out, code); + break; + } + default: return false; + } + } + if (_text >= _end) return false; + ++_text; + return true; + } + bool number(Node& out) { + const char* start = _text; + if (_text < _end && *_text == '-') ++_text; + if (_text >= _end || *_text < '0' || *_text > '9') return false; + if (*_text == '0') ++_text; + else while (_text < _end && *_text >= '0' && *_text <= '9') ++_text; + bool integral = true; + if (_text < _end && *_text == '.') { + integral = false; + ++_text; + if (_text >= _end || *_text < '0' || *_text > '9') return false; + while (_text < _end && *_text >= '0' && *_text <= '9') ++_text; + } + if (_text < _end && (*_text == 'e' || *_text == 'E')) { + integral = false; + ++_text; + if (_text < _end && (*_text == '+' || *_text == '-')) ++_text; + if (_text >= _end || *_text < '0' || *_text > '9') return false; + while (_text < _end && *_text >= '0' && *_text <= '9') ++_text; + } + const std::string text(start, _text); + if (integral) { + errno = 0; + if (text[0] == '-') { + const long long value = std::strtoll(text.c_str(), nullptr, 10); + if (errno == 0) { out.type = TypeInteger; out.integer = value; return true; } + } else { + const unsigned long long value = std::strtoull(text.c_str(), nullptr, 10); + if (errno == 0) { + if (value <= static_cast(INT64_MAX)) { out.type = TypeInteger; out.integer = static_cast(value); } + else { out.type = TypeUInteger; out.uinteger = value; } + return true; + } + } + } + out.type = TypeReal; + out.real = std::strtod(text.c_str(), nullptr); + return true; + } + bool value(Node& out) { + if (_text >= _end) return false; + switch (*_text) { + case 'n': if (!literal("null")) return false; SetType(out, TypeNull); return true; + case 't': if (!literal("true")) return false; SetType(out, TypeBoolean); out.boolean = true; return true; + case 'f': if (!literal("false")) return false; SetType(out, TypeBoolean); return true; + case '"': { + std::string text; + if (!string(text)) return false; + SetType(out, TypeString); + *out.string.text = std::move(text); + return true; + } + case '[': { + ++_text; + SetType(out, TypeArray); + skip(); + if (_text < _end && *_text == ']') { ++_text; return true; } + for (;;) { + Value item{}; + Construct(item); + out.array.items->push_back(item); + skip(); + if (!value(*item.node)) return false; + skip(); + if (_text < _end && *_text == ',') { ++_text; continue; } + if (_text < _end && *_text == ']') { ++_text; return true; } + return false; + } + } + case '{': { + ++_text; + SetType(out, TypeObject); + skip(); + if (_text < _end && *_text == '}') { ++_text; return true; } + for (;;) { + skip(); + std::string key; + if (!string(key)) return false; + skip(); + if (_text >= _end || *_text != ':') return false; + ++_text; + skip(); + Pair pair{}; + Construct(pair.key, key); + Construct(pair.value); + out.object.items->push_back(pair); + if (!value(*pair.value.node)) return false; + skip(); + if (_text < _end && *_text == ',') { ++_text; continue; } + if (_text < _end && *_text == '}') { ++_text; return true; } + return false; + } + } + default: + return number(out); + } + } +}; + +[[noreturn]] void APS5_VABI JsonMemAllocatorPureVirtual() { + throw std::runtime_error("sce::Json::MemAllocator: pure virtual function called"); +} + +void* const JsonMemAllocatorVtable[] = { + nullptr, nullptr, + reinterpret_cast(&JsonMemAllocatorPureVirtual), reinterpret_cast(&JsonMemAllocatorPureVirtual), + reinterpret_cast(&JsonMemAllocatorPureVirtual), reinterpret_cast(&JsonMemAllocatorPureVirtual), + reinterpret_cast(&JsonMemAllocatorPureVirtual), reinterpret_cast(&JsonMemAllocatorPureVirtual), +}; + +Value& ObjectEntry(Object& object, const std::string& key) { + for (auto& pair : *object.items) + if (*pair.key.text == key) return pair.value; + Pair pair{}; + Construct(pair.key, key); + Construct(pair.value); + object.items->push_back(pair); + return object.items->back().value; +} + +} extern "C" { -// Initialization succeeds as on an offline console: the title tears its whole web API layer down on -// an init failure and later dereferences it anyway. Requests made through it fail as offline. -int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* self, const void* parameter) { - (void)self; - (void)parameter; - return 0; -} - int APS5_VABI _ZN3sce4Json11InitializerC1Ev(void* self) { (void)self; return 0; @@ -24,13 +467,261 @@ int APS5_VABI _ZN3sce4Json11InitializerD1Ev(void* self) { return 0; } -int APS5_VABI _ZN3sce4Json12MemAllocatorC2Ev(void* self) { +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void* self, const void* parameter) { (void)self; + if (parameter == nullptr) throw std::invalid_argument("sce::Json::Initializer::initialize: null parameter"); return 0; } -int APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void* self) { +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void* self) { (void)self; + auto& state = State(); + std::lock_guard lock(state.mutex); + state.callback = nullptr; + state.context = nullptr; + return 0; +} + +int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void* self, NullAccessCallback callback, void* context) { + (void)self; + auto& state = State(); + std::lock_guard lock(state.mutex); + state.callback = callback; + state.context = context; + return 0; +} + +void APS5_VABI _ZN3sce4Json12MemAllocatorC2Ev(void* self) { + *static_cast(self) = &JsonMemAllocatorVtable[2]; +} + +void APS5_VABI _ZN3sce4Json12MemAllocatorD2Ev(void* self) { + *static_cast(self) = &JsonMemAllocatorVtable[2]; +} + +void APS5_VABI _ZN3sce4Json6StringC1Ev(String* self) { Construct(*self, std::string()); } +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String* self, const char* text) { + if (text == nullptr) throw std::invalid_argument("sce::Json::String: null text"); + Construct(*self, text); +} +void APS5_VABI _ZN3sce4Json6StringC1ERKS1_(String* self, const String* other) { Construct(*self, *other->text); } +void APS5_VABI _ZN3sce4Json6StringD1Ev(String* self) { Destroy(*self); } +String* APS5_VABI _ZN3sce4Json6StringaSERKS1_(String* self, const String* other) { + if (self != other) *self->text = *other->text; + return self; +} +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String* self) { return self->text->c_str(); } +bool APS5_VABI _ZNK3sce4Json6String5emptyEv(const String* self) { return self->text->empty(); } +std::size_t APS5_VABI _ZNK3sce4Json6String6lengthEv(const String* self) { return self->text->size(); } +bool APS5_VABI _ZNK3sce4Json6StringeqEPKc(const String* self, const char* text) { return text != nullptr && *self->text == text; } + +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array* self) { Construct(*self); } +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array* self) { Destroy(*self); } +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array* self, const Value* value) { + Value copy{}; + CopyInto(copy, *value); + self->items->push_back(copy); +} +ArrayIterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(ArrayIterator* result, const Array* self) { result->it = self->items->begin(); return result; } +ArrayIterator* APS5_VABI _ZNK3sce4Json5Array3endEv(ArrayIterator* result, const Array* self) { result->it = self->items->end(); return result; } +bool APS5_VABI _ZNK3sce4Json5Array5emptyEv(const Array* self) { return self->items->empty(); } +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array* self) { return self->items->size(); } +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array* self) { + if (self->items->empty()) throw std::out_of_range("sce::Json::Array::back: empty array"); + return &self->items->back(); +} +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(ArrayIterator* self) { (void)self; } +ArrayIterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(ArrayIterator* self) { ++self->it; return self; } +Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const ArrayIterator* self) { return &*self->it; } +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const ArrayIterator* self, const ArrayIterator* other) { return self->it != other->it; } + +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object* self) { Construct(*self); } +void APS5_VABI _ZN3sce4Json6ObjectC1ERKS1_(Object* self, const Object* other) { CopyObject(*self, *other); } +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object* self) { Destroy(*self); } +Object* APS5_VABI _ZN3sce4Json6ObjectaSERKS1_(Object* self, const Object* other) { + if (self == other) return self; + Object copy{}; + CopyObject(copy, *other); + Destroy(*self); + *self = copy; + return self; +} +void APS5_VABI _ZN3sce4Json6Object5clearEv(Object* self) { + for (auto& pair : *self->items) { + Destroy(pair.key); + Destroy(pair.value); + } + self->items->clear(); +} +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object* self, const String* key) { return &ObjectEntry(*self, *key->text); } +ObjectIterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(ObjectIterator* result, const Object* self) { result->it = self->items->begin(); return result; } +ObjectIterator* APS5_VABI _ZNK3sce4Json6Object3endEv(ObjectIterator* result, const Object* self) { result->it = self->items->end(); return result; } +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(ObjectIterator* self) { (void)self; } +ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEv(ObjectIterator* self) { ++self->it; return self; } +Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const ObjectIterator* self) { return &*self->it; } +bool APS5_VABI _ZNK3sce4Json6Object8iteratorneERKS2_(const ObjectIterator* self, const ObjectIterator* other) { return self->it != other->it; } + +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value* self) { Construct(*self); } +void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value* self, bool value) { Construct(*self); self->node->type = TypeBoolean; self->node->boolean = value; } +void APS5_VABI _ZN3sce4Json5ValueC1El(Value* self, std::int64_t value) { Construct(*self); self->node->type = TypeInteger; self->node->integer = value; } +void APS5_VABI _ZN3sce4Json5ValueC1Em(Value* self, std::uint64_t value) { Construct(*self); self->node->type = TypeUInteger; self->node->uinteger = value; } +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value* self, double value) { Construct(*self); self->node->type = TypeReal; self->node->real = value; } +void APS5_VABI _ZN3sce4Json5ValueC1EPKc(Value* self, const char* text) { + if (text == nullptr) throw std::invalid_argument("sce::Json::Value: null text"); + Construct(*self); + SetType(*self->node, TypeString); + *self->node->string.text = text; +} +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6StringE(Value* self, const String* text) { + Construct(*self); + SetType(*self->node, TypeString); + *self->node->string.text = *text->text; +} +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_5ArrayE(Value* self, const Array* array) { + Construct(*self); + self->node->type = TypeArray; + CopyArray(self->node->array, *array); +} +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value* self, const Object* object) { + Construct(*self); + self->node->type = TypeObject; + CopyObject(self->node->object, *object); +} +void APS5_VABI _ZN3sce4Json5ValueC1ERKS1_(Value* self, const Value* other) { CopyInto(*self, *other); } +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value* self) { Destroy(*self); } +Value* APS5_VABI _ZN3sce4Json5ValueaSERKS1_(Value* self, const Value* other) { + if (self != other) AssignNode(NodeOf(*self), NodeOf(*other)); + return self; +} + +int APS5_VABI _ZN3sce4Json5Value3setENS0_9ValueTypeE(Value* self, ValueType type) { SetType(NodeOf(*self), type); return 0; } +int APS5_VABI _ZN3sce4Json5Value3setEb(Value* self, bool value) { SetType(NodeOf(*self), TypeBoolean); self->node->boolean = value; return 0; } +int APS5_VABI _ZN3sce4Json5Value3setEl(Value* self, std::int64_t value) { SetType(NodeOf(*self), TypeInteger); self->node->integer = value; return 0; } +int APS5_VABI _ZN3sce4Json5Value3setEm(Value* self, std::uint64_t value) { SetType(NodeOf(*self), TypeUInteger); self->node->uinteger = value; return 0; } +int APS5_VABI _ZN3sce4Json5Value3setEd(Value* self, double value) { SetType(NodeOf(*self), TypeReal); self->node->real = value; return 0; } +int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value* self, const char* text) { + if (text == nullptr) throw std::invalid_argument("sce::Json::Value::set: null text"); + const std::string copy(text); + SetType(NodeOf(*self), TypeString); + *self->node->string.text = copy; + return 0; +} +int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6StringE(Value* self, const String* text) { + const std::string copy(*text->text); + SetType(NodeOf(*self), TypeString); + *self->node->string.text = copy; + return 0; +} +int APS5_VABI _ZN3sce4Json5Value3setERKNS0_5ArrayE(Value* self, const Array* array) { + Array copy{}; + CopyArray(copy, *array); + Clear(NodeOf(*self)); + self->node->type = TypeArray; + self->node->array = copy; + return 0; +} +int APS5_VABI _ZN3sce4Json5Value3setERKNS0_6ObjectE(Value* self, const Object* object) { + Object copy{}; + CopyObject(copy, *object); + Clear(NodeOf(*self)); + self->node->type = TypeObject; + self->node->object = copy; + return 0; +} +int APS5_VABI _ZN3sce4Json5Value3setERKS1_(Value* self, const Value* other) { + if (self != other) AssignNode(NodeOf(*self), NodeOf(*other)); + return 0; +} + +ValueType APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value* self) { return NodeOf(*self).type; } +bool APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value* self) { return Typed(self, TypeBoolean).boolean; } +std::int64_t APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value* self) { + const Node& n = NodeOf(*self); + if (n.type == TypeUInteger && n.uinteger <= static_cast(INT64_MAX)) return static_cast(n.uinteger); + return Typed(self, TypeInteger).integer; +} +std::uint64_t APS5_VABI _ZNK3sce4Json5Value11getUIntegerEv(const Value* self) { + const Node& n = NodeOf(*self); + if (n.type == TypeInteger && n.integer >= 0) return static_cast(n.integer); + return Typed(self, TypeUInteger).uinteger; +} +double APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value* self) { + const Node& n = NodeOf(*self); + if (n.type == TypeInteger) return static_cast(n.integer); + if (n.type == TypeUInteger) return static_cast(n.uinteger); + return Typed(self, TypeReal).real; +} +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value* self) { return &Typed(self, TypeString).string; } +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value* self) { return &Typed(self, TypeArray).array; } +const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value* self) { return &Typed(self, TypeObject).object; } +std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value* self) { + const Node& n = NodeOf(*self); + if (n.type == TypeArray) return n.array.items->size(); + if (n.type == TypeObject) return n.object.items->size(); + return 0; +} +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value* self, const char* key) { + const Node& n = NodeOf(*self); + if (key != nullptr && n.type == TypeObject) + for (const auto& pair : *n.object.items) + if (*pair.key.text == key) return &pair.value; + return &NullAccess(TypeNull, self); +} +const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value* self, std::size_t index) { + const Node& n = NodeOf(*self); + if (n.type == TypeArray && index < n.array.items->size()) { + auto it = n.array.items->begin(); + std::advance(it, static_cast(index)); + return &*it; + } + return &NullAccess(TypeNull, self); +} +Value* APS5_VABI _ZN3sce4Json5Value10referValueERKNS0_6StringE(Value* self, const String* key) { + Node& n = NodeOf(*self); + if (n.type == TypeNull) SetType(n, TypeObject); + if (n.type != TypeObject) throw std::runtime_error("sce::Json::Value::referValue: value is not an object"); + return &ObjectEntry(n.object, *key->text); +} +Array* APS5_VABI _ZN3sce4Json5Value10referArrayEv(Value* self) { + Node& n = NodeOf(*self); + if (n.type == TypeNull) SetType(n, TypeArray); + if (n.type != TypeArray) throw std::runtime_error("sce::Json::Value::referArray: value is not an array"); + return &n.array; +} +Object* APS5_VABI _ZN3sce4Json5Value11referObjectEv(Value* self) { + Node& n = NodeOf(*self); + if (n.type == TypeNull) SetType(n, TypeObject); + if (n.type != TypeObject) throw std::runtime_error("sce::Json::Value::referObject: value is not an object"); + return &n.object; +} +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value* self, String* out) { + std::string text; + Serialize(text, NodeOf(*self)); + *out->text = std::move(text); + return 0; +} +int APS5_VABI _ZNK3sce4Json5Value8toStringERNS0_6StringE(const Value* self, String* out) { + const Node& n = NodeOf(*self); + if (n.type == TypeString) { + *out->text = *n.string.text; + return 0; + } + std::string text; + Serialize(text, n); + *out->text = std::move(text); + return 0; +} + +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value* out, const char* text, std::size_t size) { + if (text == nullptr) throw std::invalid_argument("sce::Json::Parser::parse: null text"); + Node parsed{}; + Parser parser(text, size); + if (!parser.Parse(parsed)) { + Clear(parsed); + return JsonErrorParse; + } + Clear(NodeOf(*out)); + *out->node = parsed; return 0; } diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp new file mode 100644 index 0000000000..2dfbdb93c5 --- /dev/null +++ b/core/libs/tests/GuestJson.cpp @@ -0,0 +1,250 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +struct Value { void* node; }; +struct String { void* text; }; +struct Array { void* items; }; +struct Object { void* items; }; +struct Iterator { void* position; }; +struct Pair { + String key; + std::uint64_t reserved; + Value value; +}; +using NullAccessCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); + +extern "C" { +int APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); +int APS5_VABI _ZN3sce4Json11InitializerD1Ev(void*); +int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const void*); +int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); +int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void*, NullAccessCallback, void*); +void APS5_VABI _ZN3sce4Json6StringC1Ev(String*); +void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); +void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); +const char* APS5_VABI _ZNK3sce4Json6String5c_strEv(const String*); +std::size_t APS5_VABI _ZNK3sce4Json6String6lengthEv(const String*); +void APS5_VABI _ZN3sce4Json5ArrayC1Ev(Array*); +void APS5_VABI _ZN3sce4Json5ArrayD1Ev(Array*); +void APS5_VABI _ZN3sce4Json5Array9push_backERKNS0_5ValueE(Array*, const Value*); +Iterator* APS5_VABI _ZNK3sce4Json5Array5beginEv(Iterator*, const Array*); +Iterator* APS5_VABI _ZNK3sce4Json5Array3endEv(Iterator*, const Array*); +std::size_t APS5_VABI _ZNK3sce4Json5Array4sizeEv(const Array*); +const Value* APS5_VABI _ZNK3sce4Json5Array4backEv(const Array*); +void APS5_VABI _ZN3sce4Json5Array8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json5Array8iteratorppEv(Iterator*); +Value* APS5_VABI _ZNK3sce4Json5Array8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json5Array8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json6ObjectC1Ev(Object*); +void APS5_VABI _ZN3sce4Json6ObjectD1Ev(Object*); +Value* APS5_VABI _ZN3sce4Json6ObjectixERKNS0_6StringE(Object*, const String*); +Iterator* APS5_VABI _ZNK3sce4Json6Object5beginEv(Iterator*, const Object*); +Iterator* APS5_VABI _ZNK3sce4Json6Object3endEv(Iterator*, const Object*); +void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(Iterator*); +Iterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEv(Iterator*); +Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const Iterator*); +bool APS5_VABI _ZNK3sce4Json6Object8iteratorneERKS2_(const Iterator*, const Iterator*); +void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value*); +void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value*, bool); +void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); +void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value*, const Object*); +void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); +int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value*, const char*); +std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); +bool APS5_VABI _ZNK3sce4Json5Value10getBooleanEv(const Value*); +std::int64_t APS5_VABI _ZNK3sce4Json5Value10getIntegerEv(const Value*); +std::uint64_t APS5_VABI _ZNK3sce4Json5Value11getUIntegerEv(const Value*); +double APS5_VABI _ZNK3sce4Json5Value7getRealEv(const Value*); +const String* APS5_VABI _ZNK3sce4Json5Value9getStringEv(const Value*); +const Array* APS5_VABI _ZNK3sce4Json5Value8getArrayEv(const Value*); +std::size_t APS5_VABI _ZNK3sce4Json5Value5countEv(const Value*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEPKc(const Value*, const char*); +const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value*, std::size_t); +int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); +int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +} + +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "Json check failed at line %d\n", line); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) + +enum : std::int32_t { TypeNull, TypeBoolean, TypeInteger, TypeUInteger, TypeReal, TypeString, TypeArray, TypeObject }; + +static Value fallback{}; +static int callbackCalls = 0; +static std::int32_t lastRequested = -1; +static const Value* lastParent = nullptr; +static void* lastContext = nullptr; + +static const Value& APS5_VABI OnNullAccess(std::int32_t requested, const Value* parent, void* context) { + ++callbackCalls; + lastRequested = requested; + lastParent = parent; + lastContext = context; + return fallback; +} + +static std::string Serialize(Value& value) { + String out{}; + _ZN3sce4Json6StringC1Ev(&out); + Require(_ZN3sce4Json5Value9serializeERNS0_6StringE(&value, &out) == 0); + std::string text(_ZNK3sce4Json6String5c_strEv(&out), _ZNK3sce4Json6String6lengthEv(&out)); + _ZN3sce4Json6StringD1Ev(&out); + return text; +} + +static const Value& Member(const Value& value, const char* key) { + return *_ZNK3sce4Json5ValueixEPKc(&value, key); +} + +static void ParseAndRoundTrip() { + const std::string text = + "{ \"name\" : \"x\\u00e9\\n\", \"count\": 3, \"big\": 18446744073709551615, \"neg\": -5," + " \"pi\": 1.5, \"ok\": true, \"none\": null, \"list\": [1, \"two\", [3]] }"; + Value root{}; + _ZN3sce4Json5ValueC1Ev(&root); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, text.c_str(), text.size()) == 0); + Require(_ZNK3sce4Json5Value7getTypeEv(&root) == TypeObject); + Require(_ZNK3sce4Json5Value5countEv(&root) == 8); + Require(std::strcmp(_ZNK3sce4Json6String5c_strEv(_ZNK3sce4Json5Value9getStringEv(&Member(root, "name"))), "x\xc3\xa9\n") == 0); + Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "count")) == TypeInteger); + Require(_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "count")) == 3); + Require(_ZNK3sce4Json5Value11getUIntegerEv(&Member(root, "count")) == 3); + Require(_ZNK3sce4Json5Value7getRealEv(&Member(root, "count")) == 3.0); + Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "big")) == TypeUInteger); + Require(_ZNK3sce4Json5Value11getUIntegerEv(&Member(root, "big")) == std::numeric_limits::max()); + Require(_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "neg")) == -5); + Require(_ZNK3sce4Json5Value7getRealEv(&Member(root, "pi")) == 1.5); + Require(_ZNK3sce4Json5Value10getBooleanEv(&Member(root, "ok"))); + Require(_ZNK3sce4Json5Value7getTypeEv(&Member(root, "none")) == TypeNull); + + const Value& list = Member(root, "list"); + const Array* array = _ZNK3sce4Json5Value8getArrayEv(&list); + Require(_ZNK3sce4Json5Array4sizeEv(array) == 3); + std::vector types; + Iterator it{}, end{}; + _ZNK3sce4Json5Array5beginEv(&it, array); + _ZNK3sce4Json5Array3endEv(&end, array); + for (; _ZNK3sce4Json5Array8iteratorneERKS2_(&it, &end); _ZN3sce4Json5Array8iteratorppEv(&it)) + types.push_back(_ZNK3sce4Json5Value7getTypeEv(_ZNK3sce4Json5Array8iteratordeEv(&it))); + _ZN3sce4Json5Array8iteratorD1Ev(&it); + _ZN3sce4Json5Array8iteratorD1Ev(&end); + Require((types == std::vector{TypeInteger, TypeString, TypeArray})); + Require(_ZNK3sce4Json5Value10getIntegerEv(_ZNK3sce4Json5ValueixEm(_ZNK3sce4Json5ValueixEm(&list, 2), 0)) == 3); + + const std::string serialized = Serialize(root); + Require(serialized == "{\"name\":\"x\xc3\xa9\\n\",\"count\":3,\"big\":18446744073709551615,\"neg\":-5," + "\"pi\":1.5,\"ok\":true,\"none\":null,\"list\":[1,\"two\",[3]]}"); + Value reparsed{}; + _ZN3sce4Json5ValueC1Ev(&reparsed); + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&reparsed, serialized.c_str(), serialized.size()) == 0); + Require(Serialize(reparsed) == serialized); + _ZN3sce4Json5ValueD1Ev(&reparsed); + + for (const char* invalid : {"{\"a\":}", "[1,]", "\"open", "tru", "{} x", "01"}) { + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, invalid, std::strlen(invalid)) < 0); + Require(_ZNK3sce4Json5Value7getTypeEv(&root) == TypeObject && _ZNK3sce4Json5Value5countEv(&root) == 8); + } + _ZN3sce4Json5ValueD1Ev(&root); +} + +static void ObjectsAndArrays() { + Object object{}; + _ZN3sce4Json6ObjectC1Ev(&object); + String alpha{}, beta{}; + _ZN3sce4Json6StringC1EPKc(&alpha, "alpha"); + _ZN3sce4Json6StringC1EPKc(&beta, "beta"); + Value* first = _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha); + Require(_ZNK3sce4Json5Value7getTypeEv(first) == TypeNull); + Require(_ZN3sce4Json5Value3setEl(first, 1) == 0); + Require(_ZN3sce4Json5Value3setEPKc(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &beta), "b") == 0); + Require(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha) == first); + + std::vector keys; + Iterator it{}, end{}; + _ZNK3sce4Json6Object5beginEv(&it, &object); + _ZNK3sce4Json6Object3endEv(&end, &object); + for (; _ZNK3sce4Json6Object8iteratorneERKS2_(&it, &end); _ZN3sce4Json6Object8iteratorppEv(&it)) { + const Pair* pair = _ZNK3sce4Json6Object8iteratordeEv(&it); + keys.emplace_back(_ZNK3sce4Json6String5c_strEv(&pair->key)); + if (keys.size() == 1) Require(&pair->value == first); + } + _ZN3sce4Json6Object8iteratorD1Ev(&it); + _ZN3sce4Json6Object8iteratorD1Ev(&end); + Require((keys == std::vector{"alpha", "beta"})); + + Value wrapped{}; + _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(&wrapped, &object); + Require(Serialize(wrapped) == "{\"alpha\":1,\"beta\":\"b\"}"); + _ZN3sce4Json5ValueD1Ev(&wrapped); + _ZN3sce4Json6StringD1Ev(&alpha); + _ZN3sce4Json6StringD1Ev(&beta); + _ZN3sce4Json6ObjectD1Ev(&object); + + Array array{}; + _ZN3sce4Json5ArrayC1Ev(&array); + Value flag{}, real{}; + _ZN3sce4Json5ValueC1Eb(&flag, true); + _ZN3sce4Json5ValueC1Ed(&real, 2.5); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &flag); + _ZN3sce4Json5Array9push_backERKNS0_5ValueE(&array, &real); + _ZN3sce4Json5ValueD1Ev(&flag); + _ZN3sce4Json5ValueD1Ev(&real); + Require(_ZNK3sce4Json5Array4sizeEv(&array) == 2); + Require(_ZNK3sce4Json5Value7getRealEv(_ZNK3sce4Json5Array4backEv(&array)) == 2.5); + int visited = 0; + _ZNK3sce4Json5Array5beginEv(&it, &array); + _ZNK3sce4Json5Array3endEv(&end, &array); + for (; _ZNK3sce4Json5Array8iteratorneERKS2_(&it, &end); _ZN3sce4Json5Array8iteratorppEv(&it)) ++visited; + Require(visited == 2); + _ZN3sce4Json5ArrayD1Ev(&array); +} + +static void NullAccess() { + alignas(16) std::uint8_t initializer[16]{}; + alignas(16) std::uint8_t parameter[64]{}; + Require(_ZN3sce4Json11InitializerC1Ev(initializer) == 0); + Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(initializer, parameter) == 0); + int context = 0; + Require(_ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(initializer, OnNullAccess, &context) == 0); + _ZN3sce4Json5ValueC1Ev(&fallback); + Require(_ZN3sce4Json5Value3setEl(&fallback, 99) == 0); + + Value root{}; + _ZN3sce4Json5ValueC1Ev(&root); + const char text[] = "{\"present\":\"text\"}"; + Require(_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(&root, text, sizeof(text) - 1) == 0); + Require(&Member(root, "missing") == &fallback); + Require(callbackCalls == 1 && lastRequested == TypeNull && lastParent == &root && lastContext == &context); + Require(_ZNK3sce4Json5Value10getIntegerEv(&Member(root, "present")) == 99); + Require(callbackCalls == 2 && lastRequested == TypeInteger && lastParent == &Member(root, "present")); + Require(!_ZNK3sce4Json5Value10getBooleanEv(&Member(root, "present"))); + Require(callbackCalls == 3 && lastRequested == TypeBoolean); + Require(_ZNK3sce4Json5Value7getTypeEv(_ZNK3sce4Json5ValueixEm(&root, 0)) == TypeInteger); + + Require(_ZN3sce4Json11Initializer9terminateEv(initializer) == 0); + const Value& unhandled = Member(root, "missing"); + Require(&unhandled != &fallback && _ZNK3sce4Json5Value7getTypeEv(&unhandled) == TypeNull); + Require(callbackCalls == 4); + _ZN3sce4Json5ValueD1Ev(&root); + _ZN3sce4Json5ValueD1Ev(&fallback); + Require(_ZN3sce4Json11InitializerD1Ev(initializer) == 0); +} + +int main() { + ParseAndRoundTrip(); + ObjectsAndArrays(); + NullAccess(); +} From f6a23c9d154f7ccebfc260a1d252e7d1bc4b2093 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 16:47:02 +1300 Subject: [PATCH 84/93] fix(agc): keep a replaced Vulkan device alive Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index a2f9e338ca..6957417d5b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -651,7 +651,11 @@ public: std::lock_guard lock(GuestMemory::GpuMutex()); timing.Mark("gpu_mutex_wait"); if (device == nullptr || device->Window() == nullptr) { - if (device) device->PrepareForReplacement(); + if (device) { + device->PrepareForReplacement(); + static std::vector> replacedDevices; + replacedDevices.push_back(device); + } device = std::make_shared(&window); } require(device->Window() == window.context, "presentation window does not match device surface"); From 0479a5f5f08ec478c534c5f60c37d374261dd797 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 16:48:05 +1300 Subject: [PATCH 85/93] fix(agc): reject primitive restart only for indexed draws Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp | 9 ++++++++- core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp | 2 +- core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp | 6 +++--- core/libs/prx/libSceAgcDriver/tests/Graphics.cpp | 8 ++++++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index a2f9e338ca..8d876f376c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -3836,8 +3836,15 @@ private: const bool colorWrites = targetMask != queue.context.end() && shaderMask != queue.context.end() && (targetMask->second & shaderMask->second) != 0; if (!colorWrites && !queue.shader.contains(0x8)) return DrawVerdict::Nothing; } + if (drawParameters.indexed) { + const auto restart = queue.userConfig.find(0x24b); + if (restart != queue.userConfig.end() && restart->second != 0) { + rejected = "AGC graphics: primitive restart (GE_MULTI_PRIM_IB_RESET_EN) is unsupported for indexed draws"; + return DrawVerdict::Rejected; + } + } if (DrawPrecheck()) { - rejected = Graphics::DrawRejection(queue); + rejected = Graphics::DrawRejection(queue, drawParameters.indexed); if (!rejected.empty()) return DrawVerdict::Rejected; } phase(DrawRowPrecheck); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp index fb84496c98..fd90957c32 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp @@ -67,7 +67,7 @@ State DecodeState(const QueueState& queue); // The message DecodeState (or the pixel stage decode after it) would throw for the register rules // this precheck covers, evaluated without exceptions before the draw is decoded; empty when they // pass (DecodeState still checks everything). A register a rule needs that is absent is no verdict. -std::string DrawRejection(const QueueState& queue); +std::string DrawRejection(const QueueState& queue, bool indexed); // The recording facade of the draw decoders (design_cpu_final M8, step 8a): every register read // of DecodeShaderStages, DecodeState, DrawRejection, DecodePixelStageInfo and Driver::draw's diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index 8d214688f1..dcc5a608e1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -293,7 +293,7 @@ State DecodeState(const QueueState& queue) { default: throw std::runtime_error("AGC graphics: unsupported primitive type " + std::to_string(primitive)); } APS5_LOG_OUT_DEBUG("Topology=%u", static_cast(result.topology)); - zero(queue.userConfig, 0x24b, ~0u, "primitive restart (GE_MULTI_PRIM_IB_RESET_EN)", RegisterBank::UserConfig); + (void)read(queue.userConfig, 0x24b, RegisterBank::UserConfig); if ((read(cx, 0x207) & LayerExports) != 0) { static bool reported = false; if (!reported) { @@ -498,7 +498,7 @@ State DecodeState(const QueueState& queue) { return result; } -std::string DrawRejection(const QueueState& queue) { +std::string DrawRejection(const QueueState& queue, bool indexed) { const auto& cx = queue.context; // A register a rule needs that is absent gives no verdict here: DecodeState reports it. const auto value = [&](const Registers& registers, std::uint32_t offset, std::uint32_t& out) { @@ -514,7 +514,7 @@ std::string DrawRejection(const QueueState& queue) { const auto require = [&](bool condition, const char* reason) { return condition ? std::string() : "AGC graphics: " + std::string(reason); }; - if (auto reason = nonzero(queue.userConfig, 0x24b, ~0u, "primitive restart (GE_MULTI_PRIM_IB_RESET_EN)"); !reason.empty()) return reason; + if (auto reason = indexed ? nonzero(queue.userConfig, 0x24b, ~0u, "primitive restart (GE_MULTI_PRIM_IB_RESET_EN)") : std::string(); !reason.empty()) return reason; if (auto reason = nonzero(cx, 0x207, ~LayerExports, "clip distances, layer, viewport or auxiliary vertex exports"); !reason.empty()) return reason; if (value(cx, 0x200, word)) { if (!depthPassThrough(word) && !IgnoreDepthTest() && (word & DepthControlMask) != 0) return zeroMessage(0x200, word, "depth, stencil or conditional color writes"); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index a96f0b446f..dacf921c2b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -85,7 +85,11 @@ void stateTests() { Require(state.viewport.y == 4 && state.viewport.height == -4, "negative viewport height was lost"); Require(state.color.format == VK_FORMAT_R8G8B8A8_UNORM, "RGBA format changed"); queue.userConfig[0x24b] = 1; - expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "GE_MULTI_PRIM_IB_RESET_EN"); + queue.context[0x1b3] = 2; + queue.context[0x1b4] = 2; + (void)AgcDriver::Graphics::DecodeState(queue); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "primitive restart rejected a non-indexed draw"); + Require(AgcDriver::Graphics::DrawRejection(queue, true).find("GE_MULTI_PRIM_IB_RESET_EN") != std::string::npos, "primitive restart was accepted for an indexed draw"); queue = makeState(); queue.userConfig.erase(0x24b); queue.context[0x2a5] = 0; @@ -123,7 +127,7 @@ void stateTests() { queue.context[0x1b3] = 2; queue.context[0x1b4] = 2; state = AgcDriver::Graphics::DecodeState(queue); - Require(AgcDriver::Graphics::DrawRejection(queue).empty(), "precheck rejected the reference state"); + Require(AgcDriver::Graphics::DrawRejection(queue, true).empty(), "precheck rejected the reference state"); static_cast(AgcDriver::Graphics::DecodePixelStageInfo(queue.context, state.hasColorTarget, state.color.componentMapping)); AgcDriver::Graphics::RegisterReadLog() = nullptr; Require(!log.empty(), "the register facade recorded nothing"); From e1a8371bff2d741aa3f811ccb0fbf38bcd33f69a Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 16:51:01 +1300 Subject: [PATCH 86/93] feat(agc): DMA_DATA cache policy hints and GDS source/destination Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- .../prx/libSceAgcDriver/Execution/src/Pm4.cpp | 56 ++++++++++++++++--- core/libs/prx/libSceAgcDriver/tests/Pm4.cpp | 20 ++++++- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index 21dd47ae27..cc90565935 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -94,6 +94,31 @@ std::vector copySource(std::uint64_t source, std::size_t bytes, bool return data; } +constexpr std::uint32_t DmaSelectGds = 1; +constexpr std::size_t GdsBytes = 0x10000; + +struct GdsStorage { + std::mutex mutex; + std::array bytes{}; +}; + +GdsStorage& Gds() { + static GdsStorage storage; + return storage; +} + +bool gdsRange(std::uint64_t offset, std::size_t bytes) { + return offset <= GdsBytes && bytes <= GdsBytes - offset; +} + +std::vector dmaSourceBytes(std::span packet) { + const std::size_t bytes = packet[6] & 0x3ffffffu; + if (dmaSource(packet) != DmaSelectGds) return copySource(address(packet[2], packet[3]), bytes, dmaSource(packet) == 2); + auto& gds = Gds(); + std::lock_guard lock(gds.mutex); + return {gds.bytes.begin() + packet[2], gds.bytes.begin() + packet[2] + bytes}; +} + void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t bytes, bool immediate) { if (bytes == 0) return; GuestMemory::CheckRange(reinterpret_cast(destination), bytes, 1, true); @@ -164,6 +189,9 @@ std::string_view UnsupportedReason(std::uint32_t header) { } } +constexpr std::uint32_t DmaSourceCachePolicy = 3u << 13u; +constexpr std::uint32_t DmaDestinationCachePolicy = 3u << 25u; + void Validate(std::span packet, std::uint32_t queue) { require(!packet.empty(), "truncated PM4 header"); const auto header = packet[0]; @@ -348,9 +376,11 @@ void Validate(std::span packet, std::uint32_t queue) { } case 0x50: size(7); - require((packet[1] & ~0xe0300001u) == 0, "DMA_DATA cache or reserved fields are not implemented"); - require(memorySelector(dmaDestination(packet)), "DMA_DATA register, GDS or prefetch destination is not implemented"); - require(memorySelector(dmaSource(packet)) || dmaSource(packet) == 2, "DMA_DATA register or GDS source is not implemented"); + require((packet[1] & ~(0xe0300001u | DmaSourceCachePolicy | DmaDestinationCachePolicy)) == 0, "DMA_DATA reserved fields are not implemented"); + require(memorySelector(dmaDestination(packet)) || dmaDestination(packet) == DmaSelectGds, "DMA_DATA register or prefetch destination is not implemented"); + require(memorySelector(dmaSource(packet)) || dmaSource(packet) == 2 || dmaSource(packet) == DmaSelectGds, "DMA_DATA register source is not implemented"); + require(dmaSource(packet) != DmaSelectGds || (packet[3] == 0 && gdsRange(packet[2], packet[6] & 0x3ffffffu)), "DMA_DATA GDS source range exceeds the GDS"); + require(dmaDestination(packet) != DmaSelectGds || (packet[5] == 0 && gdsRange(packet[4], packet[6] & 0x3ffffffu)), "DMA_DATA GDS destination range exceeds the GDS"); require(dmaSource(packet) != 2 || packet[3] == 0, "DMA_DATA immediate exceeds 32 bits"); break; default: throw std::runtime_error("known packet has no validator"); @@ -444,11 +474,11 @@ std::optional ResolveStore(std::span packet, co return StoreWrite{destination, {}, copySource(address(packet[2], packet[3]), bytes, source >= 10)}; } case 0x50: { - if (packet.size() < 7) return std::nullopt; + if (packet.size() < 7 || dmaDestination(packet) == DmaSelectGds) return std::nullopt; const std::size_t bytes = packet[6] & 0x3ffffffu; const auto destination = address(packet[4], packet[5]); if (!fits(destination, bytes)) return std::nullopt; - return StoreWrite{destination, {}, copySource(address(packet[2], packet[3]), bytes, dmaSource(packet) == 2)}; + return StoreWrite{destination, {}, dmaSourceBytes(packet)}; } case 0x83: { if (packet.size() < 5 || packet[1] / 4 > queue.constantRam.size() || packet[2] > queue.constantRam.size() - packet[1] / 4) return std::nullopt; @@ -697,9 +727,21 @@ void Execute(std::span packet, QueueState& queue) { copyMemory(address(packet[2], packet[3]), address(packet[4], packet[5]), (packet[1] & 0x10000u) != 0 ? 8 : 4, source >= 10); return; } - case 0x50: - copyMemory(address(packet[2], packet[3]), address(packet[4], packet[5]), packet[6] & 0x3ffffffu, dmaSource(packet) == 2); + case 0x50: { + const std::size_t bytes = packet[6] & 0x3ffffffu; + if (bytes == 0) return; + auto data = dmaSourceBytes(packet); + if (dmaDestination(packet) == DmaSelectGds) { + auto& gds = Gds(); + std::lock_guard lock(gds.mutex); + std::copy(data.begin(), data.end(), gds.bytes.begin() + packet[4]); + return; + } + const auto destination = address(packet[4], packet[5]); + GuestMemory::CheckRange(reinterpret_cast(destination), bytes, 1, true); + GuestMemory::Write(destination, data); return; + } default: throw std::runtime_error("packet requires driver execution"); } } diff --git a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp index 057638e04a..e18b7baf13 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp @@ -5,6 +5,7 @@ #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/Shutdown.hpp" #include +#include #include #include #include @@ -270,7 +271,24 @@ void testCopies() { check(source == destination, "DMA_DATA copy failed"); execute(state, makePacket(0x50, {0x40000000, 0x44332211, 0, low(destination.data()), high(destination.data()), 6})); check(destination[0] == 0x44332211 && destination[1] == 0x00002211, "DMA_DATA byte fill failed"); - expectFailure([&] { execute(state, makePacket(0x50, {0x60100000, low(source.data()), high(source.data()), 0, 0, 4})); }, "GDS"); + constexpr std::uint32_t cachePolicies = (1u << 13u) | (2u << 25u); + const auto toGds = makePacket(0x50, {0x60100000 | cachePolicies, low(source.data()), high(source.data()), 0x100, 0, 16}); + check(!AgcDriver::Pm4::ResolveStore(toGds, state, 64).has_value(), "DMA_DATA to GDS resolved as a memory store"); + execute(state, toGds); + execute(state, makePacket(0x50, {0x20100000, 0x104, 0, 0xfff8, 0, 8})); + destination = {}; + execute(state, makePacket(0x50, {0x20000000 | cachePolicies, 0xfff8, 0, low(destination.data()), high(destination.data()), 8})); + check(destination[0] == 12 && destination[1] == 13 && destination[2] == 0, "DMA_DATA GDS to GDS round trip failed"); + const auto fromGds = makePacket(0x50, {0x20000000, 0x100, 0, low(destination.data()), high(destination.data()), 16}); + const auto store = AgcDriver::Pm4::ResolveStore(fromGds, state, 64); + check(store.has_value() && store->Bytes().size() == 16 && std::memcmp(store->Bytes().data(), source.data(), 16) == 0, "DMA_DATA from GDS did not resolve its source bytes"); + destination = {}; + execute(state, fromGds); + check(source == destination, "DMA_DATA GDS round trip failed"); + expectFailure([&] { execute(state, makePacket(0x50, {0x60100000, low(source.data()), high(source.data()), 0xfffc, 0, 8})); }, "exceeds the GDS"); + expectFailure([&] { execute(state, makePacket(0x50, {0x20000000, 0, 1, low(destination.data()), high(destination.data()), 4})); }, "exceeds the GDS"); + expectFailure([&] { execute(state, makePacket(0x50, {0x60200000, low(source.data()), high(source.data()), 0, 0, 4})); }, "destination is not implemented"); + expectFailure([&] { execute(state, makePacket(0x50, {0x60000000 | (1u << 15u), low(source.data()), high(source.data()), low(destination.data()), high(destination.data()), 4})); }, "reserved fields"); expectFailure([&] { execute(state, makePacket(0x37, {0x100, 0x1000, 0, 1})); }, "guest"); #ifdef _WIN32 auto* memory = VirtualAlloc(nullptr, 4096, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); From 20cc23438f274095bf2abc8db6ad4fbc3c46ef81 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 16:53:29 +1300 Subject: [PATCH 87/93] feat(libc): sscanf_s on linux Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libc/Export.cpp | 5 -- core/libs/prx/libc/src/Formatting.cpp | 115 ++++++++++++++++++++++++++ core/libs/tests/GuestStrings.cpp | 18 ++++ 3 files changed, 133 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libc/Export.cpp b/core/libs/prx/libc/Export.cpp index ec1701d745..625b27ff93 100644 --- a/core/libs/prx/libc/Export.cpp +++ b/core/libs/prx/libc/Export.cpp @@ -81,11 +81,6 @@ int APS5_VABI fscanf_nid_postfix() { return 0; } -int APS5_VABI sscanf_s_nid_postfix() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI vswprintf_nid_postfix() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libc/src/Formatting.cpp b/core/libs/prx/libc/src/Formatting.cpp index 207069071c..10d37e32ea 100644 --- a/core/libs/prx/libc/src/Formatting.cpp +++ b/core/libs/prx/libc/src/Formatting.cpp @@ -1,7 +1,11 @@ #include #include #include +#include #include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/VarArgsAbi.hpp" #include "prx/libc/include/FileStream.hpp" @@ -143,6 +147,117 @@ int APS5_VABI sscanf_nid_postfix(VA_ARGS) { ); } +#ifdef _WIN32 + +int APS5_VABI sscanf_s_nid_postfix(const char* buffer, const char* format, ...) { + (void)buffer; + (void)format; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +#else + +int APS5_VABI sscanf_s_nid_postfix(const char* buffer, const char* format, ...) { + if (buffer == nullptr || format == nullptr) return EOF; + std::va_list args; + va_start(args, format); + const char* input = buffer; + int assigned = 0; + bool converted = false; + const char* cursor = format; + const auto finish = [&](int result) { + va_end(args); + return result; + }; + while (*cursor != '\0') { + if (std::isspace(static_cast(*cursor))) { + while (std::isspace(static_cast(*cursor))) ++cursor; + while (std::isspace(static_cast(*input))) ++input; + continue; + } + if (*cursor != '%' || cursor[1] == '%') { + const char expected = *cursor == '%' ? '%' : *cursor; + cursor += *cursor == '%' ? 2 : 1; + if (expected == '%') while (std::isspace(static_cast(*input))) ++input; + if (*input != expected) return finish(!converted && *input == '\0' ? EOF : assigned); + ++input; + continue; + } + ++cursor; + const bool suppress = *cursor == '*'; + if (suppress) ++cursor; + unsigned long width = 0; + while (std::isdigit(static_cast(*cursor))) width = width * 10 + static_cast(*cursor++ - '0'); + const char* lengthStart = cursor; + while (std::strchr("hljztL", *cursor) != nullptr && *cursor != '\0') ++cursor; + const std::string length(lengthStart, cursor); + const char conversion = *cursor; + if (conversion == '\0') return finish(assigned); + std::string specifier(1, conversion); + if (conversion == '[') { + const char* setStart = cursor++; + if (*cursor == '^') ++cursor; + if (*cursor == ']') ++cursor; + while (*cursor != '\0' && *cursor != ']') ++cursor; + if (*cursor != ']') return finish(assigned); + specifier.assign(setStart, cursor + 1); + } + ++cursor; + if (conversion == 'n') { + if (!suppress) *va_arg(args, int*) = static_cast(input - buffer); + continue; + } + if (conversion != 'c' && conversion != '[') { + const char* probe = input; + while (std::isspace(static_cast(*probe))) ++probe; + if (*probe == '\0') return finish(converted ? assigned : EOF); + } + const bool sized = conversion == 's' || conversion == 'c' || conversion == '['; + std::string directive = "%"; + void* target = nullptr; + if (suppress) directive += '*'; + else target = va_arg(args, void*); + if (sized && !suppress) { + const auto capacity = static_cast(va_arg(args, unsigned int)); + const unsigned long limit = conversion == 'c' ? capacity : (capacity == 0 ? 0 : capacity - 1); + const unsigned long wanted = conversion == 'c' ? (width == 0 ? 1 : width) : width; + if (limit == 0 || (conversion == 'c' && wanted > limit)) { + if (capacity != 0) static_cast(target)[0] = '\0'; + return finish(assigned); + } + width = wanted == 0 ? limit : std::min(wanted, limit); + } + if (width != 0) directive += std::to_string(width); + directive += length; + directive += specifier; + directive += "%n"; + int consumed = -1; + const int matched = suppress ? std::sscanf(input, directive.c_str(), &consumed) : std::sscanf(input, directive.c_str(), target, &consumed); + if (consumed < 0 || (!suppress && matched != 1)) return finish(!converted && *input == '\0' ? EOF : assigned); + if (sized && !suppress && conversion != 'c') { + const char next = input[consumed]; + bool overflow = false; + if (conversion == 's') overflow = next != '\0' && !std::isspace(static_cast(next)); + else if (next != '\0') { + char probe[2]; + const std::string test = "%1" + specifier; + overflow = std::sscanf(input + consumed, test.c_str(), probe) == 1; + } + if (overflow) { + static_cast(target)[0] = '\0'; + return finish(assigned); + } + } + input += consumed; + converted = true; + if (!suppress) ++assigned; + } + return finish(assigned); +} + +#endif + int APS5_VABI vprintf_nid_postfix(const char* str, VaList* c) { #ifdef _WIN32 return LibcDetail::PrintWindows(str, c); diff --git a/core/libs/tests/GuestStrings.cpp b/core/libs/tests/GuestStrings.cpp index cc8eefab69..fe71ab71d9 100644 --- a/core/libs/tests/GuestStrings.cpp +++ b/core/libs/tests/GuestStrings.cpp @@ -15,6 +15,7 @@ std::size_t APS5_VABI strlcat_nid_postfix(char*, const char*, std::size_t); char* APS5_VABI strtok_r_nid_postfix(char*, const char*, char**); char* APS5_VABI strtok_nid_postfix(char*, const char*); char* APS5_VABI strcasestr_nid_postfix(const char*, const char*); +int APS5_VABI sscanf_s_nid_postfix(const char*, const char*, ...); } static void Require(bool condition) { @@ -24,7 +25,24 @@ static void Require(bool condition) { } } +static void CheckSscanfS() { +#ifndef _WIN32 + int number = 0; + char word[4] = "zz"; + char letter = 0; + char value[8] = {}; + Require(sscanf_s_nid_postfix(" 12 abc x", "%d %s %c", &number, word, 4u, &letter, 1u) == 3 && number == 12 && std::strcmp(word, "abc") == 0 && letter == 'x'); + Require(sscanf_s_nid_postfix("12 abcd", "%d %s", &number, word, 4u) == 1 && word[0] == '\0'); + Require(sscanf_s_nid_postfix("key=val", "%3[a-z]=%3s", word, 4u, value, 8u) == 2 && std::strcmp(word, "key") == 0 && std::strcmp(value, "val") == 0); + int position = 0; + Require(sscanf_s_nid_postfix("7 %", "%d %%%n", &number, &position) == 1 && position == 3); + Require(sscanf_s_nid_postfix(" ", "%d", &number) == EOF); + Require(sscanf_s_nid_postfix("x", "%d", &number) == 0); +#endif +} + int main() { + CheckSscanfS(); Require(std::strcmp(basename_nid_postfix(nullptr), ".") == 0); Require(std::strcmp(basename_nid_postfix(""), ".") == 0); Require(std::strcmp(basename_nid_postfix("////"), "/") == 0); From a8151f17d944e571b8a42fdc372b3365a3a39f00 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Mon, 28 Sep 2026 08:13:48 +0300 Subject: [PATCH 88/93] fix(libc): add missing header for not impl helper --- core/libs/prx/libc/src/Formatting.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/libs/prx/libc/src/Formatting.cpp b/core/libs/prx/libc/src/Formatting.cpp index 10d37e32ea..ccedf1cb6c 100644 --- a/core/libs/prx/libc/src/Formatting.cpp +++ b/core/libs/prx/libc/src/Formatting.cpp @@ -6,6 +6,8 @@ #include #include #include + +#include "prx/libc/include/General.hpp" #include "SceTypes.hpp" #include "prx/libc/include/VarArgsAbi.hpp" #include "prx/libc/include/FileStream.hpp" From 9dd86e99bf6e0baa4ec8e26c22881d14ccd83d36 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Mon, 28 Sep 2026 08:41:20 +0300 Subject: [PATCH 89/93] fix(pad): add missing header for log macros to pad input cpp --- core/libs/prx/libSceVideoOut/src/PadInput.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceVideoOut/src/PadInput.cpp b/core/libs/prx/libSceVideoOut/src/PadInput.cpp index e1aeb6018a..aa6e387d6d 100644 --- a/core/libs/prx/libSceVideoOut/src/PadInput.cpp +++ b/core/libs/prx/libSceVideoOut/src/PadInput.cpp @@ -10,7 +10,7 @@ #include "prx/libSceVideoOut/include/DisplayWindow.hpp" #include "prx/libScePad/include/PadState.hpp" #include "prx/libScePad/include/PadInputTypes.hpp" -#include "general/LogMacros.hpp" +#include "prx/libc/include/General.hpp" PadInput::PadInput() : bindings(Pad::LoadInputMapping()), pressed(bindings.size()), wheelReleaseTimes(bindings.size()) { From fa787cfbe9aaf49c32737660f552e599bf02b0f8 Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:17:35 +1000 Subject: [PATCH 90/93] fix(agc): keep BGRA color targets resident between draws --- .../libSceAgcDriver/Execution/src/VulkanDevice.cpp | 11 +++++------ .../Graphics/include/TextureFormat.hpp | 1 + core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp | 3 +-- .../prx/libSceAgcDriver/Graphics/src/Texture.cpp | 13 ++++++++----- .../libSceAgcDriver/Graphics/src/TextureFormat.cpp | 9 +++++++++ core/libs/tests/TextureFormatLookup.cpp | 8 ++++++++ 6 files changed, 32 insertions(+), 13 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index a3a590e638..1e34962efb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1581,12 +1581,11 @@ bool ResidentPresentable(const Graphics::Context& context, const Graphics::Stora if (descriptor.width != buffer.width || descriptor.height != buffer.height || descriptor.mipCount != 1 || image.ImageLayers() != 1 || image.ImageDepth() != 1) return false; if (image.GuestBytes() != DisplayBufferSize(buffer)) return false; const auto format = Graphics::StorageFormatForGuest(context, descriptor.format); - switch (format) { - case VK_FORMAT_R8G8B8A8_UNORM: case VK_FORMAT_B8G8R8A8_UNORM: case VK_FORMAT_A8B8G8R8_UNORM_PACK32: - case VK_FORMAT_A2B10G10R10_UNORM_PACK32: case VK_FORMAT_A2R10G10B10_UNORM_PACK32: - break; - default: return false; - } + constexpr std::uint64_t tenBitFormat = 0x0100000000000000ull; + const bool tenBit = (buffer.pixelFormat & tenBitFormat) != 0; + const bool rgba = (buffer.pixelFormat & ~tenBitFormat) == 0x8000000022000000ull; + const auto displayFormat = tenBit ? VK_FORMAT_A2B10G10R10_UNORM_PACK32 : rgba ? VK_FORMAT_R8G8B8A8_UNORM : VK_FORMAT_B8G8R8A8_UNORM; + if (format != displayFormat) return false; VkFormatProperties properties{}; context.formatProperties(context.physical, format, &properties); if ((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT) == 0) return false; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp index 3f2d1f0d88..b23f2f1daa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp @@ -12,6 +12,7 @@ namespace AgcDriver::Graphics { VkFormat ResolveTextureFormat(std::uint32_t guestFormat); std::optional FindGuestTextureFormat(VkFormat format, std::uint32_t elementBytes); +std::optional FindGuestColorTargetFormat(VkFormat format, std::uint32_t elementBytes); std::uint32_t BytesPerElement(std::uint32_t guestFormat); bool IsBlockCompressed(std::uint32_t guestFormat); std::uint32_t BlockWidth(std::uint32_t guestFormat); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 9f008e3aa5..b90d4969ac 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -32,9 +32,8 @@ namespace AgcDriver::Graphics { namespace { -// A guest texture format code with the same Vulkan format and texel size as a color buffer. std::uint32_t GuestFormatFor(VkFormat format, std::uint32_t elementBytes) { - if (const auto guest = FindGuestTextureFormat(format, elementBytes)) return *guest; + if (const auto guest = FindGuestColorTargetFormat(format, elementBytes)) return *guest; throw std::runtime_error("AGC graphics: no guest texture format matches the color buffer format " + std::to_string(static_cast(format))); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index cacb96d29b..da0d220eeb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -869,7 +869,10 @@ VkImageView StorageTexture::AttachmentView(VkFormat format) { Require(attachable, "storage image cannot be a color attachment"); const auto found = attachmentViews.find(format); if (found != attachmentViews.end()) return found->second; + VkImageViewUsageCreateInfo usage{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO}; + usage.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.pNext = &usage; viewInfo.image = image; viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; viewInfo.format = format; @@ -1299,7 +1302,7 @@ void StorageTexture::upload(const std::vector* layers) { commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import->base; - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { if (layers != nullptr && !(*layers)[layer]) continue; for (const auto& mip : mips) { @@ -1545,7 +1548,7 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import.base; - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); std::vector regions; for (std::size_t i = 0; i < windows.size(); ++i) { const auto& window = windows[i]; @@ -1716,7 +1719,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp if (recorder != nullptr && (covered & transferAccess) == transferAccess && Recorder::MergeBarriers()) { Recorder::CountMerged(Recorder::CommandClass::StorageWriteBack); } else { - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, transferAccess); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, transferAccess); Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack); } for (const auto& [begin, end, slab] : seeds) { @@ -1736,7 +1739,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); for (std::size_t i = 0; i < windows.size(); ++i) { const auto& window = windows[i]; detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, window.window); @@ -2996,7 +2999,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { if (storedLayers != nullptr && !(*storedLayers)[layer]) continue; for (const auto& mip : mips) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp index 83b6bfbec0..31affce9cf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp @@ -119,6 +119,15 @@ std::uint32_t BytesPerElement(std::uint32_t guestFormat) { return findFormatEntry(guestFormat).bytesPerElement; } +std::optional FindGuestColorTargetFormat(VkFormat format, std::uint32_t elementBytes) { + switch (format) { + case VK_FORMAT_B8G8R8A8_UNORM: format = VK_FORMAT_R8G8B8A8_UNORM; break; + case VK_FORMAT_B8G8R8A8_SRGB: format = VK_FORMAT_R8G8B8A8_SRGB; break; + default: break; + } + return FindGuestTextureFormat(format, elementBytes); +} + bool IsBlockCompressed(std::uint32_t guestFormat) { return findFormatEntry(guestFormat).blockCompressed; } diff --git a/core/libs/tests/TextureFormatLookup.cpp b/core/libs/tests/TextureFormatLookup.cpp index fd8901eb54..8838b9a950 100644 --- a/core/libs/tests/TextureFormatLookup.cpp +++ b/core/libs/tests/TextureFormatLookup.cpp @@ -12,4 +12,12 @@ int main() { if (FindGuestTextureFormat(VK_FORMAT_B8G8R8A8_UNORM, 4)) std::abort(); if (FindGuestTextureFormat(VK_FORMAT_UNDEFINED, 4)) std::abort(); if (FindGuestTextureFormat(VK_FORMAT_BC1_RGBA_UNORM_BLOCK, 8)) std::abort(); + if (FindGuestColorTargetFormat(VK_FORMAT_B8G8R8A8_UNORM, 4) != 56) std::abort(); + if (FindGuestColorTargetFormat(VK_FORMAT_B8G8R8A8_SRGB, 4) != 130) std::abort(); + if (FindGuestColorTargetFormat(VK_FORMAT_R8G8B8A8_UNORM, 4) != 56) std::abort(); + if (FindGuestColorTargetFormat(VK_FORMAT_R16G16B16A16_SFLOAT, 8) != 71) std::abort(); + if (FindGuestColorTargetFormat(VK_FORMAT_B8G8R8A8_UNORM, 8)) std::abort(); + if (FindGuestColorTargetFormat(VK_FORMAT_B8G8R8A8_SRGB, 1)) std::abort(); + if (FindGuestColorTargetFormat(VK_FORMAT_BC1_RGBA_UNORM_BLOCK, 8)) std::abort(); + if (FindGuestColorTargetFormat(VK_FORMAT_UNDEFINED, 4)) std::abort(); } From ba9201874f6618d5daa31d6f039aa2f514f14df4 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 14:22:48 +1300 Subject: [PATCH 91/93] feat(fiber): implement fibers on linux Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/CMakeLists.txt | 8 +++ core/libs/prx/libSceFiber/Export.cpp | 88 ++++++++++++++++++++-------- core/libs/tests/Fiber.cpp | 81 +++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 core/libs/tests/Fiber.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index ee9c993f54..b86ddf5c51 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -363,6 +363,14 @@ if(BUILD_TESTING) set_tests_properties(guest_json PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_executable(fiber_tests tests/Fiber.cpp) + target_include_directories(fiber_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(fiber_tests PRIVATE libSceFiber) + add_test(NAME fiber COMMAND fiber_tests) + if(WIN32) + set_tests_properties(fiber PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_executable(mspace_tests tests/Mspace.cpp) target_include_directories(mspace_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(mspace_tests PRIVATE libc) diff --git a/core/libs/prx/libSceFiber/Export.cpp b/core/libs/prx/libSceFiber/Export.cpp index b24a8ea920..adbe258a53 100644 --- a/core/libs/prx/libSceFiber/Export.cpp +++ b/core/libs/prx/libSceFiber/Export.cpp @@ -27,8 +27,6 @@ static constexpr std::size_t FIBER_OBJECT_SIZE = 0x100; static constexpr std::size_t FIBER_OPT_PARAM_SIZE = 0x80; static constexpr std::size_t FIBER_MIN_CONTEXT_SIZE = 512; -#ifdef _WIN32 - using GuestFiberEntry = void (APS5_VABI*)(std::uint64_t argOnInitialize, std::uint64_t argOnRun); // A fiber that is switching away stays Suspending until its context is saved; whoever runs next @@ -92,6 +90,8 @@ static void CompletePendingSuspend() { } } +#ifdef _WIN32 + extern "C" void Aps5FiberSwitchStack(void** save, void* load); extern "C" void Aps5FiberTrampoline(); @@ -180,6 +180,68 @@ static void SetBounds(const StackBounds& bounds) { *reinterpret_cast(teb + 0x1478) = bounds.deallocation; } +#else + +extern "C" __attribute__((visibility("hidden"))) void Aps5FiberSwitchStack(void** save, void* load); +extern "C" __attribute__((visibility("hidden"))) void Aps5FiberTrampoline(); + +asm(R"( + .text + .globl Aps5FiberSwitchStack + .hidden Aps5FiberSwitchStack + .type Aps5FiberSwitchStack, @function +Aps5FiberSwitchStack: + push %rbp + push %rbx + push %r12 + push %r13 + push %r14 + push %r15 + sub $8, %rsp + stmxcsr 0(%rsp) + fnstcw 4(%rsp) + mov %rsp, (%rdi) + mov %rsi, %rsp + ldmxcsr 0(%rsp) + fldcw 4(%rsp) + add $8, %rsp + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbx + pop %rbp + ret + .size Aps5FiberSwitchStack, .-Aps5FiberSwitchStack + + .globl Aps5FiberTrampoline + .hidden Aps5FiberTrampoline + .type Aps5FiberTrampoline, @function +Aps5FiberTrampoline: + mov %r12, %rdi + and $-16, %rsp + call Aps5FiberMain + ud2 + .size Aps5FiberTrampoline, .-Aps5FiberTrampoline +)"); + +struct InitialFrame { + std::uint32_t mxcsr; + std::uint16_t fpuControl; + std::uint16_t padding; + std::uint64_t r15, r14, r13, r12, rbx, rbp; + std::uint64_t returnAddress; +}; +static_assert(sizeof(InitialFrame) == 8 + 6 * 8 + 8, "initial fiber frame must match Aps5FiberSwitchStack"); + +static StackBounds CurrentBounds() { + return {}; +} + +static void SetBounds(const StackBounds&) {} + +#endif + static StackBounds FiberBounds(const Fiber* fiber) { return {fiber->context + fiber->contextSize, fiber->context, fiber->context}; } @@ -189,7 +251,7 @@ static Fiber* AsFiber(FiberObject* object) { return fiber && fiber->magic == FIBER_MAGIC ? fiber : nullptr; } -extern "C" [[noreturn]] void Aps5FiberMain(Fiber* fiber) { +extern "C" [[noreturn]] __attribute__((visibility("hidden"))) void Aps5FiberMain(Fiber* fiber) { CompletePendingSuspend(); fiber->entry(fiber->argOnInitialize, ThreadState().transfer); throw std::runtime_error(std::string("sceFiber: entry function of fiber '") + fiber->name + "' returned"); @@ -377,23 +439,3 @@ int32_t APS5_VABI sceFiberStopContextSizeCheck(void) { } -#else - -extern "C" { - -int32_t APS5_VABI _sceFiberInitializeImpl_nid_postfix() { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberFinalize(FiberObject*) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberRun_nid_postfix(FiberObject*, uint64_t, uint64_t*) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberSwitch(FiberObject*, uint64_t, uint64_t*) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberReturnToThread(uint64_t, uint64_t*) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberGetSelf(FiberObject**) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberGetInfo(FiberObject*, FiberInfo*) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberRename(FiberObject*, const char*) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberOptParamInitialize(FiberOptParam*) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberGetThreadFramePointerAddress(uint64_t*) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberStartContextSizeCheck(uint32_t) { NotImplemented_nid_no_patch(__func__); return 0; } -int32_t APS5_VABI sceFiberStopContextSizeCheck(void) { NotImplemented_nid_no_patch(__func__); return 0; } - -} - -#endif diff --git a/core/libs/tests/Fiber.cpp b/core/libs/tests/Fiber.cpp new file mode 100644 index 0000000000..2f01090843 --- /dev/null +++ b/core/libs/tests/Fiber.cpp @@ -0,0 +1,81 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include + +using Entry = void (APS5_VABI*)(std::uint64_t, std::uint64_t); + +extern "C" { +std::int32_t APS5_VABI _sceFiberInitializeImpl_nid_postfix(FiberObject*, const char*, Entry, std::uint64_t, void*, std::uint64_t, const void*, std::uint32_t); +std::int32_t APS5_VABI sceFiberFinalize(FiberObject*); +std::int32_t APS5_VABI sceFiberRun_nid_postfix(FiberObject*, std::uint64_t, std::uint64_t*); +std::int32_t APS5_VABI sceFiberSwitch(FiberObject*, std::uint64_t, std::uint64_t*); +std::int32_t APS5_VABI sceFiberReturnToThread(std::uint64_t, std::uint64_t*); +std::int32_t APS5_VABI sceFiberGetSelf(FiberObject**); +} + +static void Require(bool value, const char* what) { + if (!value) { std::fprintf(stderr, "Fiber check failed: %s\n", what); std::abort(); } +} + +namespace { + +constexpr std::int32_t FiberErrorPermission = static_cast(0x80590005); +constexpr std::int32_t FiberErrorState = static_cast(0x80590006); + +alignas(16) std::array g_firstContext; +alignas(16) std::array g_secondContext; +struct alignas(8) FiberStorage { + unsigned char bytes[0x100]; +}; + +FiberStorage g_firstStorage; +FiberStorage g_secondStorage; +auto* const g_first = reinterpret_cast(&g_firstStorage); +auto* const g_second = reinterpret_cast(&g_secondStorage); + +void APS5_VABI FirstEntry(std::uint64_t argOnInitialize, std::uint64_t argOnRun) { + Require(argOnInitialize == 11 && argOnRun == 100, "first fiber arguments"); + FiberObject* self = nullptr; + Require(sceFiberGetSelf(&self) == 0 && self == g_first, "first fiber self"); + volatile double carried = 1.5; + std::uint64_t received = 0; + Require(sceFiberSwitch(g_second, 200, &received) == 0, "switch to second fiber"); + Require(received == 300 && carried == 1.5, "switch back to first fiber"); + for (;;) { + Require(sceFiberReturnToThread(received + 1, &received) == 0, "first fiber return"); + } +} + +void APS5_VABI SecondEntry(std::uint64_t argOnInitialize, std::uint64_t argOnRun) { + Require(argOnInitialize == 22 && argOnRun == 200, "second fiber arguments"); + Require(sceFiberSwitch(g_second, 0, nullptr) == FiberErrorState, "switch to self"); + _mm_setcsr(_mm_getcsr() | 0x8000u); + std::uint64_t received = 0; + Require(sceFiberReturnToThread(250, &received) == 0 && received == 260, "second fiber resumed on another thread"); + Require((_mm_getcsr() & 0x8000u) != 0, "second fiber keeps its MXCSR"); + Require(sceFiberSwitch(g_first, 300, nullptr) == 0, "switch to first fiber"); + Require(false, "second fiber resumed after its last switch"); +} + +} + +int main() { + FiberObject* self = nullptr; + Require(sceFiberGetSelf(&self) == FiberErrorPermission, "no fiber on the thread"); + Require(_sceFiberInitializeImpl_nid_postfix(g_first, "first", FirstEntry, 11, g_firstContext.data(), g_firstContext.size(), nullptr, 0) == 0, "initialize first"); + Require(_sceFiberInitializeImpl_nid_postfix(g_second, "second", SecondEntry, 22, g_secondContext.data(), g_secondContext.size(), nullptr, 0) == 0, "initialize second"); + const auto threadCsr = _mm_getcsr(); + std::uint64_t returned = 0; + Require(sceFiberRun_nid_postfix(g_first, 100, &returned) == 0 && returned == 250, "run first until second returns"); + Require(_mm_getcsr() == threadCsr, "thread MXCSR restored after the fibers"); + std::thread([&] { + Require(sceFiberRun_nid_postfix(g_second, 260, &returned) == 0 && returned == 301, "resume second on another thread"); + }).join(); + Require(sceFiberFinalize(g_first) == 0, "finalize first"); + Require(sceFiberFinalize(g_second) == 0, "finalize second"); +} From 41fb65761fd2d5ae987264e1c6e7b747bc51159f Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 19:46:38 +1300 Subject: [PATCH 92/93] refactor(fiber): keep the assembly entry points out of NID patching with _nid_no_patch Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libSceFiber/Export.cpp | 54 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/core/libs/prx/libSceFiber/Export.cpp b/core/libs/prx/libSceFiber/Export.cpp index adbe258a53..81562fa18a 100644 --- a/core/libs/prx/libSceFiber/Export.cpp +++ b/core/libs/prx/libSceFiber/Export.cpp @@ -92,14 +92,14 @@ static void CompletePendingSuspend() { #ifdef _WIN32 -extern "C" void Aps5FiberSwitchStack(void** save, void* load); -extern "C" void Aps5FiberTrampoline(); +extern "C" void Aps5FiberSwitchStack_nid_no_patch(void** save, void* load); +extern "C" void Aps5FiberTrampoline_nid_no_patch(); asm(R"( .text - .globl Aps5FiberSwitchStack - .def Aps5FiberSwitchStack; .scl 2; .type 32; .endef -Aps5FiberSwitchStack: + .globl Aps5FiberSwitchStack_nid_no_patch + .def Aps5FiberSwitchStack_nid_no_patch; .scl 2; .type 32; .endef +Aps5FiberSwitchStack_nid_no_patch: push %rbp push %rbx push %rdi @@ -146,13 +146,13 @@ Aps5FiberSwitchStack: pop %rbp ret - .globl Aps5FiberTrampoline - .def Aps5FiberTrampoline; .scl 2; .type 32; .endef -Aps5FiberTrampoline: + .globl Aps5FiberTrampoline_nid_no_patch + .def Aps5FiberTrampoline_nid_no_patch; .scl 2; .type 32; .endef +Aps5FiberTrampoline_nid_no_patch: mov %r12, %rcx and $-16, %rsp sub $32, %rsp - call Aps5FiberMain + call Aps5FiberMain_nid_no_patch ud2 )"); @@ -164,7 +164,7 @@ struct InitialFrame { std::uint64_t r15, r14, r13, r12, rsi, rdi, rbx, rbp; std::uint64_t returnAddress; }; -static_assert(sizeof(InitialFrame) == 0xa8 + 8 * 8 + 8, "initial fiber frame must match Aps5FiberSwitchStack"); +static_assert(sizeof(InitialFrame) == 0xa8 + 8 * 8 + 8, "initial fiber frame must match Aps5FiberSwitchStack_nid_no_patch"); static StackBounds CurrentBounds() { auto* tib = reinterpret_cast(NtCurrentTeb()); @@ -182,15 +182,14 @@ static void SetBounds(const StackBounds& bounds) { #else -extern "C" __attribute__((visibility("hidden"))) void Aps5FiberSwitchStack(void** save, void* load); -extern "C" __attribute__((visibility("hidden"))) void Aps5FiberTrampoline(); +extern "C" void Aps5FiberSwitchStack_nid_no_patch(void** save, void* load); +extern "C" void Aps5FiberTrampoline_nid_no_patch(); asm(R"( .text - .globl Aps5FiberSwitchStack - .hidden Aps5FiberSwitchStack - .type Aps5FiberSwitchStack, @function -Aps5FiberSwitchStack: + .globl Aps5FiberSwitchStack_nid_no_patch + .type Aps5FiberSwitchStack_nid_no_patch, @function +Aps5FiberSwitchStack_nid_no_patch: push %rbp push %rbx push %r12 @@ -212,17 +211,16 @@ Aps5FiberSwitchStack: pop %rbx pop %rbp ret - .size Aps5FiberSwitchStack, .-Aps5FiberSwitchStack + .size Aps5FiberSwitchStack_nid_no_patch, .-Aps5FiberSwitchStack_nid_no_patch - .globl Aps5FiberTrampoline - .hidden Aps5FiberTrampoline - .type Aps5FiberTrampoline, @function -Aps5FiberTrampoline: + .globl Aps5FiberTrampoline_nid_no_patch + .type Aps5FiberTrampoline_nid_no_patch, @function +Aps5FiberTrampoline_nid_no_patch: mov %r12, %rdi and $-16, %rsp - call Aps5FiberMain + call Aps5FiberMain_nid_no_patch ud2 - .size Aps5FiberTrampoline, .-Aps5FiberTrampoline + .size Aps5FiberTrampoline_nid_no_patch, .-Aps5FiberTrampoline_nid_no_patch )"); struct InitialFrame { @@ -232,7 +230,7 @@ struct InitialFrame { std::uint64_t r15, r14, r13, r12, rbx, rbp; std::uint64_t returnAddress; }; -static_assert(sizeof(InitialFrame) == 8 + 6 * 8 + 8, "initial fiber frame must match Aps5FiberSwitchStack"); +static_assert(sizeof(InitialFrame) == 8 + 6 * 8 + 8, "initial fiber frame must match Aps5FiberSwitchStack_nid_no_patch"); static StackBounds CurrentBounds() { return {}; @@ -251,7 +249,7 @@ static Fiber* AsFiber(FiberObject* object) { return fiber && fiber->magic == FIBER_MAGIC ? fiber : nullptr; } -extern "C" [[noreturn]] __attribute__((visibility("hidden"))) void Aps5FiberMain(Fiber* fiber) { +extern "C" [[noreturn]] void Aps5FiberMain_nid_no_patch(Fiber* fiber) { CompletePendingSuspend(); fiber->entry(fiber->argOnInitialize, ThreadState().transfer); throw std::runtime_error(std::string("sceFiber: entry function of fiber '") + fiber->name + "' returned"); @@ -266,7 +264,7 @@ static void PrepareInitialStack(Fiber* fiber) { asm volatile("fnstcw %0" : "=m"(control)); frame->fpuControl = control; frame->r12 = reinterpret_cast(fiber); - frame->returnAddress = reinterpret_cast(&Aps5FiberTrampoline); + frame->returnAddress = reinterpret_cast(&Aps5FiberTrampoline_nid_no_patch); fiber->savedStack = frame; } @@ -298,7 +296,7 @@ static void Resume(Fiber* target, void** save, std::uint64_t argOnRun) { ThreadState().current = target; ThreadState().transfer = argOnRun; SetBounds(FiberBounds(target)); - Aps5FiberSwitchStack(save, target->savedStack); + Aps5FiberSwitchStack_nid_no_patch(save, target->savedStack); } extern "C" { @@ -380,7 +378,7 @@ int32_t APS5_VABI sceFiberReturnToThread(uint64_t arg_on_return, uint64_t* arg_o ThreadState().current = nullptr; ThreadState().transfer = arg_on_return; SetBounds(ThreadState().threadBounds); - Aps5FiberSwitchStack(&self->savedStack, ThreadState().threadStack); + Aps5FiberSwitchStack_nid_no_patch(&self->savedStack, ThreadState().threadStack); CompletePendingSuspend(); if (arg_on_run) *arg_on_run = ThreadState().transfer; return SCE_OK; From 0c06382d895b2c8ad6be09acc22bb06cb73a7f69 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Mon, 28 Sep 2026 16:52:18 +1300 Subject: [PATCH 93/93] feat(agc): ACB markers, index buffer unbinding and address in errors Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ --- core/libs/prx/libSceAgc/Acb/src/Control.cpp | 21 +++----- .../libs/prx/libSceAgc/Command/src/Packet.cpp | 6 ++- .../prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp | 2 +- .../prx/libSceAgc/DcbState/include/Marker.hpp | 10 ++++ .../prx/libSceAgc/DcbState/src/Marker.cpp | 16 +++--- core/libs/prx/libSceAgc/tests/Commands.cpp | 50 +++++++++++++++++++ 6 files changed, 84 insertions(+), 21 deletions(-) diff --git a/core/libs/prx/libSceAgc/Acb/src/Control.cpp b/core/libs/prx/libSceAgc/Acb/src/Control.cpp index 9bcea70414..012d3e4dfb 100644 --- a/core/libs/prx/libSceAgc/Acb/src/Control.cpp +++ b/core/libs/prx/libSceAgc/Acb/src/Control.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgc/Command/include/Memory.hpp" #include "prx/libSceAgc/Command/include/Packet.hpp" +#include "prx/libSceAgc/DcbState/include/Marker.hpp" #include #include #include "SceTypes.hpp" @@ -73,25 +74,19 @@ std::uint32_t* APS5_VABI sceAgcAcbSetFlip(CommandBuffer* buf, std::uint32_t vide } uint32_t* APS5_VABI sceAgcAcbPushMarker(CommandBuffer* buf, const char* str, uint32_t color) { - (void)buf; - (void)str; - (void)color; - NotImplemented_nid_no_patch(__func__); - return nullptr; + (void)color; + return Agc::Marker::Push(buf, str, __func__); } uint32_t* APS5_VABI sceAgcAcbPopMarker(CommandBuffer* buf) { - (void)buf; - NotImplemented_nid_no_patch(__func__); - return nullptr; + return Agc::Marker::Pop(buf, __func__); } uint32_t* APS5_VABI sceAgcAcbSetMarker(CommandBuffer* buf, const char* str, uint32_t color) { - (void)buf; - (void)str; - (void)color; - NotImplemented_nid_no_patch(__func__); - return nullptr; + (void)color; + auto* packet = Agc::Marker::Push(buf, str, __func__); + Agc::Marker::Pop(buf, __func__); + return packet; } } diff --git a/core/libs/prx/libSceAgc/Command/src/Packet.cpp b/core/libs/prx/libSceAgc/Command/src/Packet.cpp index aebb7c0f4f..b32303b1c9 100644 --- a/core/libs/prx/libSceAgc/Command/src/Packet.cpp +++ b/core/libs/prx/libSceAgc/Command/src/Packet.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgc/Command/include/Packet.hpp" #include +#include #include #include #include @@ -22,7 +23,10 @@ void CheckBits(std::uint64_t value, std::uint64_t mask, const char* function) { } void CheckAddress(std::uint64_t address, std::uint32_t alignment, const char* function) { - Require(address != 0 && (address & (alignment - 1u)) == 0, function, "null or misaligned address"); + if (address != 0 && (address & (alignment - 1u)) == 0) return; + char message[64]; + std::snprintf(message, sizeof(message), "null or misaligned address 0x%llx", static_cast(address)); + Require(false, function, message); } std::uint32_t Header(std::uint32_t opcode, std::uint32_t count, std::uint32_t flags) { diff --git a/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp b/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp index e7c71b3b68..bdc1ac7c4f 100644 --- a/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp +++ b/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp @@ -9,7 +9,7 @@ extern "C" { std::uint32_t* APS5_VABI sceAgcDcbSetIndexBuffer(CommandBuffer* buf, std::uint64_t indexAddress) { - Agc::Command::CheckAddress(indexAddress, 2, __func__); + if (indexAddress != 0) Agc::Command::CheckAddress(indexAddress, 2, __func__); return Agc::Command::Emit(buf, 0x26u, {static_cast(indexAddress), static_cast(indexAddress >> 32u)}, __func__); } diff --git a/core/libs/prx/libSceAgc/DcbState/include/Marker.hpp b/core/libs/prx/libSceAgc/DcbState/include/Marker.hpp index d974b008ac..c0ccfdea02 100644 --- a/core/libs/prx/libSceAgc/DcbState/include/Marker.hpp +++ b/core/libs/prx/libSceAgc/DcbState/include/Marker.hpp @@ -1,4 +1,14 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGC_DCBSTATE_INCLUDE_MARKER_HPP #define CORE_LIBS_PRX_LIBSCEAGC_DCBSTATE_INCLUDE_MARKER_HPP +#include +#include "SceTypes.hpp" + +namespace Agc::Marker { + +std::uint32_t* Push(CommandBuffer* buf, const char* str, const char* function); +std::uint32_t* Pop(CommandBuffer* buf, const char* function); + +} + #endif diff --git a/core/libs/prx/libSceAgc/DcbState/src/Marker.cpp b/core/libs/prx/libSceAgc/DcbState/src/Marker.cpp index f457dcc3fe..7e88856ccc 100644 --- a/core/libs/prx/libSceAgc/DcbState/src/Marker.cpp +++ b/core/libs/prx/libSceAgc/DcbState/src/Marker.cpp @@ -15,7 +15,11 @@ constexpr std::uint32_t OpcodeNop = 0x10; constexpr std::uint32_t CustomPushMarker = 0x0b; constexpr std::uint32_t CustomPopMarker = 0x0c; -std::uint32_t* PushMarker(CommandBuffer* buf, const char* str, const char* function) { +} + +namespace Agc::Marker { + +std::uint32_t* Push(CommandBuffer* buf, const char* str, const char* function) { Agc::Command::Require(buf != nullptr, function, "null command buffer"); const char* text = str ? str : ""; const std::size_t bytes = std::strlen(text) + 1; @@ -28,7 +32,7 @@ std::uint32_t* PushMarker(CommandBuffer* buf, const char* str, const char* funct return packet; } -std::uint32_t* PopMarker(CommandBuffer* buf, const char* function) { +std::uint32_t* Pop(CommandBuffer* buf, const char* function) { Agc::Command::Require(buf != nullptr, function, "null command buffer"); auto* packet = Agc::Command::Allocate(buf, 2, function); packet[0] = Agc::Command::Header(OpcodeNop, 2, CustomPopMarker << 2); @@ -42,18 +46,18 @@ extern "C" { uint32_t* APS5_VABI sceAgcDcbSetMarker(CommandBuffer* buf, const char* str, uint32_t color) { (void)color; - auto* packet = PushMarker(buf, str, __func__); - PopMarker(buf, __func__); + auto* packet = Agc::Marker::Push(buf, str, __func__); + Agc::Marker::Pop(buf, __func__); return packet; } uint32_t* APS5_VABI sceAgcDcbPopMarker(CommandBuffer* buf) { - return PopMarker(buf, __func__); + return Agc::Marker::Pop(buf, __func__); } uint32_t* APS5_VABI sceAgcDcbPushMarker(CommandBuffer* buf, const char* str, uint32_t color) { (void)color; - return PushMarker(buf, str, __func__); + return Agc::Marker::Push(buf, str, __func__); } } diff --git a/core/libs/prx/libSceAgc/tests/Commands.cpp b/core/libs/prx/libSceAgc/tests/Commands.cpp index d936c998b7..821a95ce70 100644 --- a/core/libs/prx/libSceAgc/tests/Commands.cpp +++ b/core/libs/prx/libSceAgc/tests/Commands.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include extern "C" std::uint32_t* APS5_VABI sceAgcDcbResetQueue(CommandBuffer* buf, std::uint32_t op, std::uint32_t state); @@ -20,6 +21,13 @@ extern "C" int APS5_VABI sceAgcWaitRegMemPatchReference(std::uint32_t* cmd, std: extern "C" int APS5_VABI sceAgcGetDataPacketPayloadAddressUnk(std::uint32_t** addr, std::uint32_t* cmd, int type); extern "C" std::uint32_t* APS5_VABI sceAgcCbSetShRegisterRangeDirect(CommandBuffer* buf, std::uint32_t offset, const std::uint32_t* values, std::uint32_t numValues); extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateAnotherOp(CommandBuffer* buf, std::uint32_t operation); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbPushMarker(CommandBuffer* buf, const char* str, std::uint32_t color); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbPopMarker(CommandBuffer* buf); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetMarker(CommandBuffer* buf, const char* str, std::uint32_t color); +extern "C" std::uint32_t* APS5_VABI sceAgcAcbPushMarker(CommandBuffer* buf, const char* str, std::uint32_t color); +extern "C" std::uint32_t* APS5_VABI sceAgcAcbPopMarker(CommandBuffer* buf); +extern "C" std::uint32_t* APS5_VABI sceAgcAcbSetMarker(CommandBuffer* buf, const char* str, std::uint32_t color); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetIndexBuffer(CommandBuffer* buf, std::uint64_t indexAddress); namespace { @@ -116,6 +124,46 @@ void testIndexedIndirectDraws() { check(storage.words == before, "invalid indexed indirect draw modified packet memory"); } +void testMarkers() { + Storage dcb; + Storage acb; + const auto* dcbPush = sceAgcDcbPushMarker(&dcb.buffer, "frame", 0xff0000u); + const auto* acbPush = sceAgcAcbPushMarker(&acb.buffer, "frame", 0x00ff00u); + check(acbPush == acb.words.data() && acbPush[0] == Agc::Command::Header(0x10, 3, 0x0bu << 2u), "ACB push marker header mismatch"); + check(std::strcmp(reinterpret_cast(acbPush + 1), "frame") == 0, "ACB push marker text mismatch"); + const auto* acbPop = sceAgcAcbPopMarker(&acb.buffer); + check(acbPop == acb.words.data() + 3 && acbPop[0] == Agc::Command::Header(0x10, 2, 0x0cu << 2u) && acbPop[1] == 0, "ACB pop marker mismatch"); + sceAgcDcbPopMarker(&dcb.buffer); + const auto* acbSet = sceAgcAcbSetMarker(&acb.buffer, nullptr, 0); + const auto* dcbSet = sceAgcDcbSetMarker(&dcb.buffer, nullptr, 0); + check(acbSet == acb.words.data() + 5 && dcbSet == dcb.words.data() + 5, "set marker did not return its push packet"); + check(acbSet[0] == Agc::Command::Header(0x10, 2, 0x0bu << 2u) && acbSet[1] == 0 && acbSet[2] == Agc::Command::Header(0x10, 2, 0x0cu << 2u), "ACB set marker is not a push and pop pair"); + check(dcbPush == dcb.words.data() && dcb.words == acb.words, "ACB and DCB markers differ"); + check(acb.buffer.cursor_up == acb.words.data() + 9, "incorrect ACB marker cursor advance"); + expectFailure([] { sceAgcAcbPushMarker(nullptr, "frame", 0); }); + expectFailure([] { sceAgcAcbPopMarker(nullptr); }); + expectFailure([] { sceAgcAcbSetMarker(nullptr, "frame", 0); }); +} + +void testIndexBuffer() { + Storage storage; + alignas(4) std::uint16_t indices[2]{}; + const auto address = reinterpret_cast(indices); + const auto* bound = sceAgcDcbSetIndexBuffer(&storage.buffer, address); + check(bound[1] == static_cast(address) && bound[2] == static_cast(address >> 32u), "index buffer address mismatch"); + const auto* unbound = sceAgcDcbSetIndexBuffer(&storage.buffer, 0); + check(unbound == bound + 3 && unbound[0] == bound[0] && unbound[1] == 0 && unbound[2] == 0, "index buffer was not unbound"); + const auto before = storage.words; + try { + sceAgcDcbSetIndexBuffer(&storage.buffer, 0x1001); + } catch (const std::runtime_error& error) { + check(std::string(error.what()).find("0x1001") != std::string::npos, "misaligned index buffer error omits the address"); + check(storage.words == before, "misaligned index buffer modified packet memory"); + return; + } + throw std::runtime_error("misaligned index buffer was accepted"); +} + void testContextState() { const std::array, 4> sizes{{{5}, {5, 8, 9, 3, 2}, {3, 5, 8, 9, 2}, {5, 8, 9, 3, 2, 5}}}; const std::array, 4> reservations{{{5}, {22, 3, 2}, {3, 22, 2}, {22, 3, 2, 5}}}; @@ -293,6 +341,8 @@ int main() { try { testPackets(); testIndexedIndirectDraws(); + testMarkers(); + testIndexBuffer(); testContextState(); testFlip(); testRegisters();