diff --git a/.github/workflows/web-tests.yml b/.github/workflows/web-tests.yml index 41602ab6fe..8de6e8d91f 100644 --- a/.github/workflows/web-tests.yml +++ b/.github/workflows/web-tests.yml @@ -31,7 +31,9 @@ jobs: - name: Check browser module syntax run: | node --check demo/web/app.mjs + node --check demo/web/archive.mjs + node --check demo/web/batch.mjs node --check demo/web/output-directory.mjs node --check demo/web/worker.mjs - - name: Test folder output recovery - run: node --test demo/web/tests/output-directory.test.mjs + - name: Test browser converter helpers + run: node --test demo/web/tests/batch.test.mjs demo/web/tests/output-directory.test.mjs diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp index f567a93e2e..29a81c9231 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp @@ -20,7 +20,7 @@ void Driver::countIndirect(int path, double readMs) { lastReport = now; std::uint64_t cpu = 0; for (int i = 1; i < IndirectPaths; ++i) cpu += counts[i]; - std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu, device memory LDS %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), static_cast(counts[IndirectWorkgroupMemory]), readWaitedMs / 1000); + std::fprintf(stderr, "[indirect] (cumulative): gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu, device memory LDS %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), static_cast(counts[IndirectWorkgroupMemory]), readWaitedMs / 1000); } int Driver::indirectArgumentsPath(std::span packet, std::uint64_t arguments) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 76f71dc937..abfd695545 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -823,6 +823,7 @@ void BuildRegisteredAbiKey(const QueueState& state, const VulkanDevice& device, struct RegisteredPreparation { std::size_t codeOffset = 0; + bool registration = false; ShaderRecompiler::SwappcInfo swappc{}; std::vector userData; std::array memory{}; @@ -868,6 +869,7 @@ std::unique_ptr PlanRegistered(const ShaderSnapshot& snap auto plan = std::make_unique(); plan->codeOffset = codeOffset; ShaderRecompiler::RdnaProgram decoded; + plan->registration = registration; try { decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); } catch (const ShaderRecompiler::UnsupportedInstructionError& error) { @@ -984,6 +986,11 @@ std::vector PrepareRegistered(RegisteredPreparation& pla request.layout.pushConstantSizeBytes = 0; append(); } + if (plan.registration) { + for (const auto& entry : entries) { + if (entry.handle != nullptr) ShaderRecompiler::ReleaseUndrawnResourcePlan(*entry.handle); + } + } return entries; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index e77cfdfc1c..88c98dc3e4 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -2344,7 +2344,7 @@ bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::spa const auto now = std::chrono::steady_clock::now(); if (now - lastReport > std::chrono::seconds(10)) { lastReport = now; - AgcDriver::ProfilePrint_nid_no_patch("[fill-sync] fills: %llu synced (%llu with a completion label pending) waited %.0f ms, %llu ordered by barrier, %llu with no pending write; phases ms: decide %.0f record %.0f notes %.0f; %llu uniform, %llu pattern (%llu doubling steps in place; pattern buffers reused %llu, made %llu)\n", static_cast(synced), static_cast(labelSynced), syncedMs, static_cast(ordered), static_cast(fills - synced - ordered), decideMs, recordMs, notesMs, static_cast(uniformFills), static_cast(patternFills), static_cast(doublingSteps), static_cast(patternBuffersReused), static_cast(patternBuffersMade)); + AgcDriver::ProfilePrint_nid_no_patch("[fill-sync] fills (cumulative): %llu synced (%llu with a completion label pending) waited %.0f ms, %llu ordered by barrier, %llu with no pending write; phases ms: decide %.0f record %.0f notes %.0f; %llu uniform, %llu pattern (%llu doubling steps in place; pattern buffers reused %llu, made %llu)\n", static_cast(synced), static_cast(labelSynced), syncedMs, static_cast(ordered), static_cast(fills - synced - ordered), decideMs, recordMs, notesMs, static_cast(uniformFills), static_cast(patternFills), static_cast(doublingSteps), static_cast(patternBuffersReused), static_cast(patternBuffersMade)); } } auto phaseStart = fillStart; @@ -3081,7 +3081,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { } if (profile && std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) { lastReport = std::chrono::steady_clock::now(); - AgcDriver::ProfilePrint_nid_no_patch("[flip] %llu presents from the resident image (%llu refreshed first), through guest memory: %llu not pending, %llu unsuitable; %llu GPU frame dumps\n", static_cast(residentPresents), static_cast(refreshedPresents), static_cast(notPending), static_cast(unsuitable), static_cast(gpuDumps)); + AgcDriver::ProfilePrint_nid_no_patch("[flip] (cumulative): %llu presents from the resident image (%llu refreshed first), through guest memory: %llu not pending, %llu unsuitable; %llu GPU frame dumps\n", static_cast(residentPresents), static_cast(refreshedPresents), static_cast(notPending), static_cast(unsuitable), static_cast(gpuDumps)); } CaptureTrace::Log("present dump=%d address=%llx width=%u height=%u resident=%d generation=%llu", dumpFrame ? state->nextDumpIndex : -1, static_cast(buffer.address), buffer.width, buffer.height, resident != nullptr, static_cast(resident ? resident->Generation() : 0)); if (passthrough && resident == nullptr) ensureHdrScaler(DisplayTexelFormat(buffer.pixelFormat)); @@ -5011,8 +5011,8 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } if (profile && std::chrono::steady_clock::now() - d.cacheReport > std::chrono::seconds(10)) { d.cacheReport = std::chrono::steady_clock::now(); - AgcDriver::ProfilePrint_nid_no_patch("[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast(d.cacheHits), static_cast(d.cacheMisses), static_cast(d.cacheInvalidated), state->resourceCache.Size(), static_cast(d.templateRefreshed), d.templateRevalidateMs, static_cast(d.templateSameWords), static_cast(ResourceCache::Finds()), static_cast(ResourceCache::Touches())); - AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call: %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast(d.procLookups), d.profiledDispatches != 0 ? static_cast(d.procLookups) / static_cast(d.profiledDispatches) : 0.0, static_cast(d.preBarriersRecorded), static_cast(d.preBarriersSkipped), static_cast(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast(computePipelinesCreatedLocked.load(std::memory_order_relaxed))); + AgcDriver::ProfilePrint_nid_no_patch("[rescache] (cumulative): %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast(d.cacheHits), static_cast(d.cacheMisses), static_cast(d.cacheInvalidated), state->resourceCache.Size(), static_cast(d.templateRefreshed), d.templateRevalidateMs, static_cast(d.templateSameWords), static_cast(ResourceCache::Finds()), static_cast(ResourceCache::Touches())); + AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call (cumulative): %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast(d.procLookups), d.profiledDispatches != 0 ? static_cast(d.procLookups) / static_cast(d.profiledDispatches) : 0.0, static_cast(d.preBarriersRecorded), static_cast(d.preBarriersSkipped), static_cast(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast(computePipelinesCreatedLocked.load(std::memory_order_relaxed))); reportRecipes(); reportDispatchMove(); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 3101ae4f90..5994cf4860 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -62,7 +62,7 @@ public: // The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large. static std::size_t Capacity(std::size_t bytes, VkMemoryPropertyFlags properties); static VkBufferUsageFlags Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties); - VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation); + VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional hostType = std::nullopt, std::uint32_t* allocatedType = nullptr); std::optional Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); void Put(const BufferAllocation& allocation) noexcept; static bool SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom); @@ -156,6 +156,7 @@ private: }; std::shared_ptr GetBufferPool(const Context& context); +VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index d5f3cd7fee..21a08c9361 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -304,15 +305,25 @@ struct Context { const auto budget = reported.heapBudget[heap]; const auto usage = reported.heapUsage[heap]; if (budget == 0 || (usage < budget && bytes <= budget - usage)) return type; + const auto host = HostMemoryType(mask); + if (!host) return type; + static std::atomic placements{0}, placedBytes{0}; + const auto count = placements.fetch_add(1) + 1; + const auto total = placedBytes.fetch_add(bytes) + bytes; + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast(usage >> 20u), static_cast(budget >> 20u), static_cast(bytes), *host, static_cast(count), static_cast(total >> 20u)); + return *host; + } + + std::optional HostMemoryType(std::uint32_t mask) const { for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) { - if ((mask & (1u << i)) == 0 || (memory.memoryHeaps[memory.memoryTypes[i].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) continue; - static std::atomic placements{0}, placedBytes{0}; - const auto count = placements.fetch_add(1) + 1; - const auto total = placedBytes.fetch_add(bytes) + bytes; - if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast(usage >> 20u), static_cast(budget >> 20u), static_cast(bytes), i, static_cast(count), static_cast(total >> 20u)); - return i; + if ((mask & (1u << i)) != 0 && (memory.memoryHeaps[memory.memoryTypes[i].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return i; } - return type; + return std::nullopt; + } + + std::optional RefusedDeviceMemoryFallback(std::uint32_t mask, std::uint32_t refused) const { + if (refused >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[refused].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt; + return HostMemoryType(mask); } }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 7beb86bef3..e2f7fc6dc7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include +#include #include #include #include @@ -51,7 +52,8 @@ BufferPool::~BufferPool() { for (auto& [memory, block] : slabBlocks) freeBlock(*block); } -VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation) { +VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional hostType, std::uint32_t* allocatedType) { + if (allocatedType != nullptr) *allocatedType = allocation.memoryTypeIndex; VkDeviceMemory memory = VK_NULL_HANDLE; auto result = allocateMemory(device, &allocation, nullptr, &memory); const auto initialResult = result; @@ -75,6 +77,21 @@ VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation ++attempts; } } + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && hostType.has_value() && *hostType != allocation.memoryTypeIndex) { + auto host = allocation; + host.memoryTypeIndex = *hostType; + memory = VK_NULL_HANDLE; + result = allocateMemory(device, &host, nullptr, &memory); + ++attempts; + if (result == VK_SUCCESS) { + if (allocatedType != nullptr) *allocatedType = *hostType; + static std::atomic placements{0}, placedBytes{0}; + const auto count = placements.fetch_add(1, std::memory_order_relaxed) + 1; + const auto total = placedBytes.fetch_add(allocation.allocationSize, std::memory_order_relaxed) + allocation.allocationSize; + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[bufferpool] %s: memory type %u is out of device memory for %llu bytes after reclaiming %llu and releasing %llu; placed in host memory type %u until it is released; %llu such allocations, %llu MiB\n", operation, allocation.memoryTypeIndex, static_cast(allocation.allocationSize), static_cast(reclaimed), static_cast(released), *hostType, static_cast(count), static_cast(total >> 20u)); + return memory; + } + } if (result == VK_SUCCESS) { if (attempts > 1) std::fprintf(stderr, "[bufferpool] %s recovered: allocation=%llu reclaimed=%llu released=%llu initialResult=%d attempts=%u\n", operation, static_cast(allocation.allocationSize), static_cast(reclaimed), static_cast(released), static_cast(initialResult), attempts); return memory; @@ -370,4 +387,8 @@ std::shared_ptr GetBufferPool(const Context& context) { return context.bufferPool; } +VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation) { + return GetBufferPool(context)->AllocateMemory(allocation, operation, context.RefusedDeviceMemoryFallback(mask, allocation.memoryTypeIndex)); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index 0ac8d9b104..4d82b509aa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" @@ -256,7 +257,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory depth target"); + memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth"); if (target.samples != 1u) owned = std::make_shared(this->context, image, memory); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; @@ -698,7 +699,7 @@ private: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &stencilCopy.memory), "vkAllocateMemory stencil copy"); + stencilCopy.memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory stencil copy"); Check(context.Function("vkBindImageMemory")(context.device, stencilCopy.image, stencilCopy.memory, 0), "vkBindImageMemory stencil copy"); stencilCopy.staging = std::make_unique(context, static_cast(target.extent.width) * target.extent.height, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); } catch (...) { @@ -846,7 +847,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory depth plane copy"); + memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth plane copy"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth plane copy"); stagingSize = static_cast(sliceBytes()) * layers; staging = std::make_unique(context, stagingSize, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp index 4ca7a22552..14123d9136 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" @@ -121,7 +122,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &created.memory), "vkAllocateMemory multisampled target"); + created.memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory multisampled target"); Check(context.Function("vkBindImageMemory")(context.device, created.image, created.memory, 0), "vkBindImageMemory multisampled target"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = created.image; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index 8e5b642c4e..678f501634 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -178,8 +178,10 @@ DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsa allocation.allocationSize = ClassAllocationBytes(requirements, capacity); allocationBytes = allocation.allocationSize; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, allocation.allocationSize, "device buffer"); - hostPlaced = allocation.memoryTypeIndex != context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - memory = cache->AllocateMemory(allocation, "vkAllocateMemory device buffer"); + const auto deviceType = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + std::uint32_t allocatedType = allocation.memoryTypeIndex; + memory = cache->AllocateMemory(allocation, "vkAllocateMemory device buffer", context.RefusedDeviceMemoryFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex), &allocatedType); + hostPlaced = allocatedType != deviceType; Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory device"); ready = true; } catch (...) { @@ -292,7 +294,7 @@ RenderTarget::RenderTarget(const Context& context, const ColorTarget& target, bo VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory render target"); + memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory render target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = image; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index b1b6cbf115..9336ef3d6b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -2638,10 +2638,10 @@ void countRevalidate(bool fast, bool ok, std::chrono::steady_clock::time_point s const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); auto last = profile.lastReport.load(); if (nowMs - last < 10000 || !profile.lastReport.compare_exchange_strong(last, nowMs)) return; - const auto byReason = [](const auto& counts, const auto& names) { + const auto byReason = [](auto& counts, const auto& names) { std::string reasons; for (std::size_t i = 0; i < names.size(); ++i) { - const auto count = counts[i].load(std::memory_order_relaxed); + const auto count = counts[i].exchange(0, std::memory_order_relaxed); if (count == 0) continue; reasons += " " + std::string(names[i]) + " " + std::to_string(count); } @@ -2650,7 +2650,7 @@ void countRevalidate(bool fast, bool ok, std::chrono::steady_clock::time_point s const auto reasons = byReason(profile.fastFails, FastFailNames); const auto fullReasons = byReason(profile.fullByReason, FastFailNames); const auto fallbacks = byReason(profile.ownFallbacks, OwnRefreshFallbackNames); - AgcDriver::ProfilePrint_nid_no_patch("[rescache] revalidate %llu calls %.1f ms: fast %llu, full %llu, failed %llu; fast-fail by reason:%s; full walks: T1 refreshed %llu (%llu storage images, %llu view sources, %llu views served by their own pending source, %llu overlaps left by the refresh accepted), full by reason:%s, T1 fallback by reason:%s, proofs verified %llu; fast-fail pending image: own 0 (the query excepts the own object), foreign %llu, snapshot %llu, T1 eligible %llu; keys proven: %llu sampled, %llu storage; epoch gate: %llu registry scans skipped, %llu import loops skipped\n", static_cast(profile.calls.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.fast.load()), static_cast(profile.full.load()), static_cast(profile.failed.load()), reasons.c_str(), static_cast(profile.ownRefreshed.load()), static_cast(profile.ownStorageRefreshes.load()), static_cast(profile.ownViewRefreshes.load()), static_cast(profile.ownSourceViews.load()), static_cast(profile.refreshedOverlaps.load()), fullReasons.c_str(), fallbacks.c_str(), static_cast(profile.proofsVerified.load()), static_cast(profile.pendingForeign.load()), static_cast(profile.pendingSnapshot.load()), static_cast(profile.pendingT1Eligible.load()), static_cast(profile.keysProven.load()), static_cast(profile.storageKeysProven.load()), static_cast(profile.serialSkips.load()), static_cast(profile.importSkips.load())); + AgcDriver::ProfilePrint_nid_no_patch("[rescache] revalidate (10 s): %llu calls %.1f ms: fast %llu, full %llu, failed %llu; fast-fail by reason:%s; full walks: T1 refreshed %llu (%llu storage images, %llu view sources, %llu views served by their own pending source, %llu overlaps left by the refresh accepted), full by reason:%s, T1 fallback by reason:%s, proofs verified %llu; fast-fail pending image: own 0 (the query excepts the own object), foreign %llu, snapshot %llu, T1 eligible %llu; keys proven: %llu sampled, %llu storage; epoch gate: %llu registry scans skipped, %llu import loops skipped\n", static_cast(profile.calls.exchange(0, std::memory_order_relaxed)), profile.nanoseconds.exchange(0, std::memory_order_relaxed) / 1e6, static_cast(profile.fast.exchange(0, std::memory_order_relaxed)), static_cast(profile.full.exchange(0, std::memory_order_relaxed)), static_cast(profile.failed.exchange(0, std::memory_order_relaxed)), reasons.c_str(), static_cast(profile.ownRefreshed.exchange(0, std::memory_order_relaxed)), static_cast(profile.ownStorageRefreshes.exchange(0, std::memory_order_relaxed)), static_cast(profile.ownViewRefreshes.exchange(0, std::memory_order_relaxed)), static_cast(profile.ownSourceViews.exchange(0, std::memory_order_relaxed)), static_cast(profile.refreshedOverlaps.exchange(0, std::memory_order_relaxed)), fullReasons.c_str(), fallbacks.c_str(), static_cast(profile.proofsVerified.exchange(0, std::memory_order_relaxed)), static_cast(profile.pendingForeign.exchange(0, std::memory_order_relaxed)), static_cast(profile.pendingSnapshot.exchange(0, std::memory_order_relaxed)), static_cast(profile.pendingT1Eligible.exchange(0, std::memory_order_relaxed)), static_cast(profile.keysProven.exchange(0, std::memory_order_relaxed)), static_cast(profile.storageKeysProven.exchange(0, std::memory_order_relaxed)), static_cast(profile.serialSkips.exchange(0, std::memory_order_relaxed)), static_cast(profile.importSkips.exchange(0, std::memory_order_relaxed))); } // APS5_NO_EPOCH_REVALIDATE=1: the per-element registry, cache and stamp checks and the per-region @@ -5178,7 +5178,7 @@ void ShaderResources::MarkGpuWrites(Recorder& recorder) { if (nowMs - last < 10000 || !counters.lastReport.compare_exchange_strong(last, nowMs)) return; const auto elements = counters.elements.load(); const auto readOnly = counters.readOnly.load(); - AgcDriver::ProfilePrint_nid_no_patch("[buffers] descriptor elements bound: %llu total, %llu written, %llu read-only; %llu pending-write notes skipped\n", static_cast(elements), static_cast(elements - readOnly), static_cast(readOnly), static_cast(counters.notesSkipped.load())); + AgcDriver::ProfilePrint_nid_no_patch("[buffers] descriptor elements bound (cumulative): %llu total, %llu written, %llu read-only; %llu pending-write notes skipped\n", static_cast(elements), static_cast(elements - readOnly), static_cast(readOnly), static_cast(counters.notesSkipped.load())); } void ShaderResources::CheckFaults() const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 82f7339415..86fc376680 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -265,7 +265,7 @@ ExpandImage CreateExpandImage(const Context& context, const GuestTextureResource VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture"); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &result.owned->memory), "vkAllocateMemory sRGB copy"); + result.owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory sRGB copy"); Check(context.Function("vkBindImageMemory")(context.device, result.image, result.owned->memory, 0), "vkBindImageMemory sRGB copy"); result.bytes = requirements.size; return result; @@ -347,7 +347,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture"); - owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory texture"); + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory texture"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory"); const VkImage uploadImage = image; @@ -623,7 +623,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory repacked copy"); + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory repacked copy"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory repacked copy"); const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -681,7 +681,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory block copy"); + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory block copy"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory block copy"); const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -738,7 +738,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory depth copy"); + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth copy"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory depth copy"); const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -1402,7 +1402,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, allocation.allocationSize = requirements.size; memoryBytes = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "storage texture"); - memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory storage texture"); + memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory storage texture"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage"); } catch (...) { release(); @@ -1832,7 +1832,7 @@ VkImageView StorageTexture::AttachmentProxyView() { VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &proxyMemory), "vkAllocateMemory attachment proxy"); + proxyMemory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory attachment proxy"); Check(context.Function("vkBindImageMemory")(context.device, proxyImage, proxyMemory, 0), "vkBindImageMemory attachment proxy"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = proxyImage; diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp index 7863a8ce05..50356ef052 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -7,10 +7,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -175,8 +177,7 @@ VKAPI_ATTR void VKAPI_CALL mockMemoryProperties2(VkPhysicalDevice, VkPhysicalDev reported->heapUsage[0] = reportedUsage; } -void HostPlacement() { - mock = MockDevice{}; +Context budgetContext() { auto context = mockContext(); context.physical = reinterpret_cast(std::uintptr_t{1}); context.memoryProperties2 = mockMemoryProperties2; @@ -185,6 +186,12 @@ void HostPlacement() { context.memory.memoryHeaps[1] = {16384 * MiB, 0}; context.memory.memoryTypes[0].heapIndex = 0; context.memory.memoryTypes[1].heapIndex = 1; + return context; +} + +void HostPlacement() { + mock = MockDevice{}; + auto context = budgetContext(); constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; reportedBudget = 1620 * MiB; reportedUsage = 1600 * MiB; @@ -207,6 +214,57 @@ void HostPlacement() { Expect(MemoryOf(again.Handle()) == device, "a retained device-local buffer was not reused"); } +void RefusedDevicePlacement() { + mock = MockDevice{}; + auto context = budgetContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + reportedBudget = 1620 * MiB; + reportedUsage = 200 * MiB; + mock.results = {VK_ERROR_OUT_OF_DEVICE_MEMORY}; + VkDeviceMemory placed = VK_NULL_HANDLE; + { + DeviceBuffer refused(context, 9 * MiB, usage); + placed = MemoryOf(refused.Handle()); + Expect(mock.memoryTypes.at(placed) == 1, "a device buffer whose device-local allocation ran out of device memory within the reported budget was not placed in the host memory type"); + } + Expect(!mock.memoryBytes.contains(placed) && mock.liveBuffers.empty(), "a device buffer placed in host memory after a refused device-local allocation was kept in the device pool after its release"); + mock.results = {VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}; + context.memory.memoryHeaps[1].flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; + bool threw = false; + try { + DeviceBuffer refused(context, 9 * MiB, usage); + } catch (const std::runtime_error&) { + threw = true; + } + Expect(threw, "a refused device-local allocation without a host memory type did not fail"); + Expect(mock.liveBuffers.empty(), "a failed device buffer left its VkBuffer alive"); +} + +void AttachmentReclaim() { + mock = MockDevice{}; + auto context = budgetContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + reportedBudget = 1620 * MiB; + reportedUsage = 200 * MiB; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = 8 * MiB; + allocation.memoryTypeIndex = 0; + const auto refuse = [&](std::initializer_list results) { + { DeviceBuffer idle(context, 4 * MiB, usage); } + Expect(mock.memoryBytes.size() == 1, "a released device buffer was not kept in the device pool"); + const auto pooled = mock.memoryBytes.begin()->first; + mock.results = results; + const auto memory = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory attachment"); + Expect(!mock.memoryBytes.contains(pooled), "an attachment refused device memory did not reclaim the pool before retrying"); + const auto type = mock.memoryTypes.at(memory); + mockFreeMemory(VK_NULL_HANDLE, memory, nullptr); + return type; + }; + Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 0, "an attachment was placed in host memory although reclaiming the pool made room in device memory"); + Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 1, "an attachment still refused device memory after reclaiming the pool was not placed in host memory"); + Expect(mock.liveBytes == 0, "an attachment allocation leaked memory"); +} + void SizeClassesAndDirections() { mock = MockDevice{}; auto context = mockContext(); @@ -693,6 +751,31 @@ void CacheReclaimRecovery() { Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "the failed cache release leaked a resource"); } +void ImageCacheReclaim() { + mock = MockDevice{}; + reclaimMock = ReclaimMock{}; + auto context = budgetContext(); + context.cacheReclaim = mockCacheReclaim; + reportedBudget = 1620 * MiB; + reportedUsage = 200 * MiB; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = 8 * MiB; + allocation.memoryTypeIndex = 0; + const auto refuse = [&](std::initializer_list results) { + ExternalAllocation(4 * MiB); + mock.results = results; + const auto requests = reclaimMock.requests; + const auto memory = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory texture"); + Expect(reclaimMock.requests == requests + 1 && reclaimMock.external == VK_NULL_HANDLE, "a texture refused device memory did not release the texture caches before retrying"); + const auto type = mock.memoryTypes.at(memory); + mockFreeMemory(VK_NULL_HANDLE, memory, nullptr); + return type; + }; + Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 0, "a texture was placed in host memory although releasing the texture caches made room in device memory"); + Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 1, "a texture still refused device memory after releasing the texture caches was not placed in host memory"); + Expect(mock.liveBytes == 0, "a texture allocation leaked memory"); +} + void SuccessfulAllocationKeepsPool() { mock = MockDevice{}; { @@ -788,7 +871,10 @@ int main(int argc, char** argv) { SuccessfulAllocationKeepsPool(); PartialSlabDoesNotRetry(); CacheReclaimRecovery(); + ImageCacheReclaim(); HostPlacement(); + RefusedDevicePlacement(); + AttachmentReclaim(); } catch (const std::exception& error) { std::fprintf(stderr, "FAIL: %s\n", error.what()); return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp b/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp index 2ed8ec1ba9..a5ff02af5d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp @@ -53,6 +53,16 @@ void Choose(const char* what, bool budgetExtension, std::uint64_t budget, std::u Expect(type == expected, std::string(what) + ": chose memory type " + std::to_string(type) + ", expected " + std::to_string(expected)); } +void RefusedFallbackTests() { + const auto context = mockContext(true); + const std::uint32_t both = (1u << HostType) | (1u << DeviceType); + const auto refused = context.RefusedDeviceMemoryFallback(both, DeviceType); + Expect(refused.has_value() && *refused == HostType, "a device-local type out of device memory did not fall back to the host type"); + Expect(!context.RefusedDeviceMemoryFallback(both, HostType).has_value(), "a host type out of memory was given a fallback"); + Expect(!context.RefusedDeviceMemoryFallback(1u << DeviceType, DeviceType).has_value(), "a fallback was chosen outside the allowed type mask"); + Expect(!context.RefusedDeviceMemoryFallback(both, 7).has_value(), "an unknown memory type was given a fallback"); +} + void BufferPlacementTests() { constexpr auto coherent = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; Context context = mockContext(true); @@ -106,6 +116,7 @@ int main() { Choose("usage already over the budget", true, 2952 * MiB, 3633 * MiB, both, HostType); Choose("no host type allowed", true, 3000 * MiB, 3000 * MiB, 1u << DeviceType, DeviceType); BufferPlacementTests(); + RefusedFallbackTests(); } catch (const std::exception& error) { std::fprintf(stderr, "FAIL: %s\n", error.what()); return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 6c9257ac77..106b9781a3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1960,6 +1960,69 @@ void verifyPlanRoundTrip() { std::cout << "plan round trip: " << fixture.name << ", " << plan->valueStorage.Size() << " values, " << ShaderDiskCache::EncodePlanEntry(key, *plan).size() << " bytes per entry, " << succeeded << " of 1000 runtimes materialized\n"; ++index; } + auto releaseFixture = fixtures.front(); + releaseFixture.waveSize = 32u; + auto registeredRequest = releaseFixture.Request(true); + const auto registeredHandle = PrepareShader(registeredRequest); + const auto registeredPlan = GetResourcePlan(registeredRequest); + ShaderDiskCache::Flush(); + ReleaseUndrawnResourcePlan(*registeredHandle); + const auto beforeReload = ShaderDiskCache::Totals(); + const auto reloadedPlan = GetResourcePlan(registeredRequest); + require(ShaderDiskCache::Totals().planHits == beforeReload.planHits + 1u, "a released registration plan was not reloaded from the disk cache"); + require(samePlan(*registeredPlan, *reloadedPlan), "a released registration plan changed when reloaded"); + + auto reloadFixture = fixtures.front(); + reloadFixture.waveSize = 32u; + reloadFixture.code.insert(reloadFixture.code.begin(), 0xbf800004u); + const auto reloadRequest = reloadFixture.Request(true); + const auto reloadHandle = PrepareShader(reloadRequest); + static_cast(GetResourcePlan(reloadRequest)); + ShaderDiskCache::Flush(); + std::vector reloadKey; + ShaderDiskCache::BuildPlanKey(reloadRequest, 32u, reloadKey); + const auto reloadPath = ShaderDiskCache::EntryDirectory() / ShaderDiskCache::EntryName(reloadKey); + require(std::filesystem::is_regular_file(reloadPath), "the reload regression plan was not persisted"); + const auto preservedPath = reloadPath.string() + ".reload-preserved"; + std::filesystem::rename(reloadPath, preservedPath); + ReleaseUndrawnResourcePlan(*reloadHandle); + static_cast(GetResourcePlan(reloadRequest)); + auto secondLayout = reloadRequest; + secondLayout.layout.pushConstantSizeBytes = 0u; + ShaderPreparationContext preparation; + static_cast(PrepareShader(secondLayout, &preparation)); + require(preparation.ResourcePrograms() == 1u, "a plan reload retained full IR after an artifact for that layout was already available"); + + auto relocatedFixture = fixtures.front(); + relocatedFixture.waveSize = 32u; + relocatedFixture.code.insert(relocatedFixture.code.begin(), 0xbf800005u); + const auto relocatedRequest = relocatedFixture.Request(true); + const auto relocatedHandle = PrepareShader(relocatedRequest); + static_cast(GetResourcePlan(relocatedRequest)); + ShaderDiskCache::Flush(); + std::vector relocatedKey; + ShaderDiskCache::BuildPlanKey(relocatedRequest, 32u, relocatedKey); + const auto relocatedPath = ShaderDiskCache::EntryDirectory() / ShaderDiskCache::EntryName(relocatedKey); + std::filesystem::rename(relocatedPath, relocatedPath.string() + ".reload-preserved"); + ReleaseUndrawnResourcePlan(*relocatedHandle); + auto drawn = relocatedRequest; + drawn.layout.pushConstantOffsetBytes = 16u; + drawn.layout.pushConstantSizeBytes = 112u; + const auto relocatedInvocation = PreparedShaderInvocation::TryCreate(drawn, relocatedHandle); + require(relocatedInvocation.has_value(), "the prepared artifact refused a relocated push layout"); + RequestMemoryView drawnMemory(drawn.context.memory); + static_cast(relocatedInvocation->Materialize(*relocatedInvocation->Capture(drawnMemory.MakeRuntime(drawn.context.userData, drawn.shader.codeAddress)))); + auto relocatedProbe = relocatedRequest; + relocatedProbe.layout.pushConstantSizeBytes = 0u; + ShaderPreparationContext relocatedPreparation; + static_cast(PrepareShader(relocatedProbe, &relocatedPreparation)); + require(relocatedPreparation.ResourcePrograms() == 1u, "a plan reload retained full IR after the prepared artifact served a relocated push layout"); + + RequestMemoryView registeredMemory(registeredRequest.context.memory); + const auto runtimeCapture = CaptureResources(registeredRequest, registeredMemory.MakeRuntime(registeredRequest.context.userData, registeredRequest.shader.codeAddress), *registeredHandle); + const auto beforeKeep = ShaderDiskCache::Totals(); + ReleaseUndrawnResourcePlan(*registeredHandle); + require(GetResourcePlan(registeredRequest) == runtimeCapture->plan && ShaderDiskCache::Totals().planHits == beforeKeep.planHits, "a runtime-captured plan was released"); const auto covers = [&](const std::function& predicate) { return std::any_of(plans.begin(), plans.end(), [&](const std::shared_ptr& plan) { return predicate(*plan); }); }; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DepthClearHle.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DepthClearHle.cpp index 93f7886211..f3c8208133 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DepthClearHle.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DepthClearHle.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/GuestAllocations.hpp" @@ -98,12 +99,26 @@ std::vector Pm4Packet(std::uint32_t opcode, std::initializer_list return words; } +Graphics::Recorder* testRecorder = nullptr; +Graphics::Recorder* driverRecorder = nullptr; + +void UseTestDevice(AgcDriver::VulkanDevice& device) { + testRecorder = AgcDriver::VulkanDeviceTestAccess::ContextOf(device).recorder; + Require(testRecorder != nullptr, "depth clear: the test device has no recorder"); + testRecorder->Activate(); +} + void Submit(std::vector> packets) { std::vector words; for (const auto& packet : packets) words.insert(words.end(), packet.begin(), packet.end()); Packet dcb{words.data(), static_cast(words.size()), 0, {}}; + if (driverRecorder != nullptr) driverRecorder->Activate(); Require(sceAgcDriverSubmitDcb(&dcb) == 0, "depth clear: DCB submission failed"); AgcDriverWaitIdle_nid_postfix(); + if (driverRecorder == nullptr) { + driverRecorder = Graphics::Recorder::Active(); + Require(driverRecorder != nullptr && driverRecorder != testRecorder, "depth clear: the driver's device did not activate its own recorder"); + } } std::array TextureDescriptor(std::uint64_t address, std::uint32_t format, std::uint32_t tile = TileLinear, std::uint32_t type = Type2DArray) { @@ -284,6 +299,7 @@ void RequireDepthTexels(AgcDriver::VulkanDevice& device, const GuestBlock& plane } Graphics::DepthClearStatistics ClearSequence(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "mode " + std::to_string(mode) + ": "; @@ -341,6 +357,7 @@ Graphics::DepthClearStatistics ClearSequence(AgcDriver::VulkanDevice& device, co } void IntegerViewCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "integer view, mode " + std::to_string(mode) + ": "; @@ -362,6 +379,7 @@ void IntegerViewCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, in } void DriverMatchCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "driver submit, mode " + std::to_string(mode) + ": "; @@ -379,6 +397,7 @@ void DriverMatchCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, in } void SixteenBitCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "16-bit plane, mode " + std::to_string(mode) + ": "; @@ -443,6 +462,7 @@ void SixteenBitCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int } void StencilCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "depth and stencil, mode " + std::to_string(mode) + ": "; @@ -464,6 +484,40 @@ void StencilCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mo else Require(difference.clears == 1u, label + "the clear was not taken (" + std::to_string(difference.clears) + " clears, " + std::to_string(Declined(difference)) + " declined)"); } +void PlaneFillCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); + Graphics::SetDepthPlaneClearForTest(mode != 0); + const auto before = Graphics::DepthClearCounts(); + const std::string label = "plane fill, mode " + std::to_string(mode) + ": "; + auto& code = NewBlock(); + std::memcpy(code.Data(), ClearKernel.data(), sizeof(ClearKernel)); + auto& plane = NewBlock(); + auto& color = NewBlock(); + const Surface surface{plane.Address(), 0, 0, false, VK_FORMAT_D32_SFLOAT}; + const auto planeBytes = static_cast(Graphics::DepthSliceBytes({Width, Height}, 4u)); + const auto drawLess = [&] { return Draw(device, shaders, surface, {.depthCompare = VK_COMPARE_OP_LESS}); }; + const auto drawGreater = [&] { return Draw(device, shaders, surface, {.depthCompare = VK_COMPARE_OP_GREATER}); }; + Require(Draw(device, shaders, surface, {.depthWrite = true, .depthCompare = VK_COMPARE_OP_ALWAYS}) == Covered, label + "the producer did not cover the plane"); + Require(drawLess() == 0u, label + "the plane does not hold the 0.5 the producer wrote"); + + Graphics::NoteDepthMetadataFill(plane.Address(), planeBytes, std::bit_cast(0.125f)); + Require(drawGreater() == Covered, label + "a plane fill of 0.125 did not clear the depth"); + Graphics::NoteDepthMetadataFill(plane.Address(), planeBytes, std::bit_cast(1.0f)); + SetColor(color, 0.25f); + RunClear(device, code, plane, color, Format32Float, 8, 4); + Require(drawLess() == 0u, label + "a clear to 0.25 after a plane fill of 1.0 left the fill in the depth"); + Require(drawGreater() == Covered, label + "a clear to 0.25 after a plane fill of 1.0 did not leave 0.25 everywhere"); + + SetColor(color, 1.0f); + RunClear(device, code, plane, color, Format32Float, 8, 4); + Graphics::NoteDepthMetadataFill(plane.Address(), planeBytes, std::bit_cast(0.125f)); + Require(drawLess() == 0u, label + "a plane fill of 0.125 after a clear to 1.0 left the clear in the depth"); + Require(drawGreater() == Covered, label + "a plane fill of 0.125 after a clear to 1.0 did not leave 0.125 everywhere"); + const auto difference = Difference(Graphics::DepthClearCounts(), before); + if (mode == 0) Require(difference.matched == 0u && difference.clears == 0u, label + "the lever counted with the switch off"); + else Require(difference.clears == 2u, label + "the lever took " + std::to_string(difference.clears) + " clears, expected 2"); +} + } int main() { @@ -502,6 +556,7 @@ int main() { for (const int mode : {0, 1}) run(mode == 0 ? "integer view, switch off" : "integer view, lever", [&] { IntegerViewCase(*device, shaders, mode); }); for (const int mode : {0, 1}) run(mode == 0 ? "16-bit plane, switch off" : "16-bit plane, lever", [&] { SixteenBitCase(*device, shaders, mode); }); for (const int mode : {0, 1}) run(mode == 0 ? "depth and stencil, switch off" : "depth and stencil, lever", [&] { StencilCase(*device, shaders, mode); }); + for (const int mode : {0, 1}) run(mode == 0 ? "plane fill, switch off" : "plane fill, lever", [&] { PlaneFillCase(*device, shaders, mode); }); for (const int mode : {0, 1}) run(mode == 0 ? "driver submit, switch off" : "driver submit, lever", [&] { DriverMatchCase(*device, shaders, mode); }); Graphics::SetDepthPlaneClearForTest(true); LibcRunShutdown_nid_postfix(); diff --git a/core/libs/prx/libScePad/include/InputMapping.hpp b/core/libs/prx/libScePad/include/InputMapping.hpp index 70e875fed8..e0a4636b90 100644 --- a/core/libs/prx/libScePad/include/InputMapping.hpp +++ b/core/libs/prx/libScePad/include/InputMapping.hpp @@ -43,6 +43,7 @@ inline constexpr std::array InputMapping{ InputBinding{SDL_SCANCODE_C, MouseButton::None, InputControl::Button, PadButton::Circle}, InputBinding{SDL_SCANCODE_Q, MouseButton::None, InputControl::Button, PadButton::L1}, InputBinding{SDL_SCANCODE_E, MouseButton::None, InputControl::Button, PadButton::R1}, + InputBinding{SDL_SCANCODE_R, MouseButton::None, InputControl::Button, PadButton::L2}, InputBinding{SDL_SCANCODE_LALT, MouseButton::None, InputControl::Button, PadButton::R1}, InputBinding{SDL_SCANCODE_RALT, MouseButton::None, InputControl::Button, PadButton::R1}, InputBinding{SDL_SCANCODE_LSHIFT, MouseButton::None, InputControl::Button, PadButton::L3}, @@ -63,9 +64,11 @@ inline constexpr std::array InputMapping{ InputBinding{SDL_SCANCODE_G, MouseButton::None, InputControl::RightStickDown}, InputBinding{SDL_SCANCODE_BACKSPACE, MouseButton::None, InputControl::TouchLeft}, InputBinding{SDL_SCANCODE_TAB, MouseButton::None, InputControl::TouchRight}, + InputBinding{SDL_SCANCODE_F7, MouseButton::None, InputControl::ToggleMouse}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Middle, InputControl::ToggleMouse}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Left, InputControl::Button, PadButton::Square}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Right, InputControl::Button, PadButton::R2}, + InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::X1, InputControl::Button, PadButton::L2}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::None, InputControl::Button, PadButton::Up, 1}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::None, InputControl::Button, PadButton::Down, -1} }; diff --git a/core/libs/prx/libScePad/include/KeyboardMouse.hpp b/core/libs/prx/libScePad/include/KeyboardMouse.hpp index 229dd48da9..5703ff6656 100644 --- a/core/libs/prx/libScePad/include/KeyboardMouse.hpp +++ b/core/libs/prx/libScePad/include/KeyboardMouse.hpp @@ -14,6 +14,10 @@ namespace Pad { inline constexpr double MouseFullTiltCountsPerSecond = 3000.0; inline constexpr double MouseMinimumStepSeconds = 0.0005; +inline constexpr double MouseRateWindowSeconds = 0.25; +inline constexpr double MouseRateGapSeconds = 0.05; +inline constexpr double MouseSmoothingReports = 3.0; +inline constexpr double MouseSmoothingLimitMs = 30.0; class MouseStick { public: @@ -22,16 +26,26 @@ public: void Advance(double seconds); void Reset(); double Speed() const; + double ReportRate() const; + double SmoothingMs() const; std::array Deflection() const; private: + void measureReportRate(double step, bool moved); + InputSettings settings; double pendingX = 0.0; double pendingY = 0.0; double pendingSeconds = 0.0; + double pendingCounts = 0.0; + int pendingReports = 0; double speedX = 0.0; double speedY = 0.0; double idleSeconds = 0.0; + double rateSeconds = 0.0; + double rateCounts = 0.0; + int rateReports = 0; + double reportRate = 0.0; }; struct InputRequests { diff --git a/core/libs/prx/libSceVideoOut/src/KeyboardMouse.cpp b/core/libs/prx/libSceVideoOut/src/KeyboardMouse.cpp index 3f39dfa80f..d568400394 100644 --- a/core/libs/prx/libSceVideoOut/src/KeyboardMouse.cpp +++ b/core/libs/prx/libSceVideoOut/src/KeyboardMouse.cpp @@ -9,6 +9,32 @@ Pad::MouseStick::MouseStick(const InputSettings& settings) : settings(settings) void Pad::MouseStick::Move(int deltaX, int deltaY) { pendingX += deltaX; pendingY += deltaY; + pendingCounts += std::hypot(deltaX, deltaY); + ++pendingReports; +} + +void Pad::MouseStick::measureReportRate(double step, bool moved) { + if (moved) { + rateSeconds += std::min(idleSeconds + step, MouseRateGapSeconds); + rateCounts += pendingCounts; + rateReports += pendingReports; + } + pendingCounts = 0.0; + pendingReports = 0; + if (rateSeconds < MouseRateWindowSeconds) return; + if (rateReports >= 10 && rateCounts >= 2.0 * rateReports) reportRate = rateReports / rateSeconds; + rateSeconds = 0.0; + rateCounts = 0.0; + rateReports = 0; +} + +double Pad::MouseStick::ReportRate() const { + return reportRate; +} + +double Pad::MouseStick::SmoothingMs() const { + const double reports = reportRate > 0.0 ? std::min(MouseSmoothingReports * 1000.0 / reportRate, MouseSmoothingLimitMs) : 0.0; + return settings.mouseSmoothingMs > 0.0 ? std::max(settings.mouseSmoothingMs, reports) : 0.0; } void Pad::MouseStick::Advance(double seconds) { @@ -23,13 +49,15 @@ void Pad::MouseStick::Advance(double seconds) { const double targetY = pendingY * scale * settings.mouseVerticalScale; pendingX = 0.0; pendingY = 0.0; + measureReportRate(step, moved); idleSeconds = moved ? 0.0 : idleSeconds + step; if (idleSeconds * 1000.0 >= settings.mouseDecayMs) { speedX = 0.0; speedY = 0.0; return; } - const double keep = settings.mouseSmoothingMs > 0.0 ? std::exp(-step * 1000.0 / settings.mouseSmoothingMs) : 0.0; + const double smoothing = SmoothingMs(); + const double keep = smoothing > 0.0 ? std::exp(-step * 1000.0 / smoothing) : 0.0; speedX = targetX + (speedX - targetX) * keep; speedY = targetY + (speedY - targetY) * keep; const double speed = std::hypot(speedX, speedY); @@ -43,6 +71,11 @@ void Pad::MouseStick::Reset() { pendingX = 0.0; pendingY = 0.0; pendingSeconds = 0.0; + pendingCounts = 0.0; + pendingReports = 0; + rateSeconds = 0.0; + rateCounts = 0.0; + rateReports = 0; speedX = 0.0; speedY = 0.0; idleSeconds = 0.0; diff --git a/core/libs/prx/libSceVideoOut/tests/InputMapping.cpp b/core/libs/prx/libSceVideoOut/tests/InputMapping.cpp index 0d85e5f293..8a180743e4 100644 --- a/core/libs/prx/libSceVideoOut/tests/InputMapping.cpp +++ b/core/libs/prx/libSceVideoOut/tests/InputMapping.cpp @@ -164,6 +164,25 @@ void nonAsciiPaths() { require(message.find(utf8Name) != std::string::npos, "a missing file names the path in UTF-8"); } +std::size_t toggleMouseBindings(const Pad::InputConfiguration& configuration, SDL_Scancode key, Pad::MouseButton button) { + std::size_t count = 0; + for (const auto& binding : configuration.bindings) { + if (binding.control == Pad::InputControl::ToggleMouse && binding.key == key && binding.mouseButton == button) ++count; + } + return count; +} + +void toggleMouseDefaults() { + useConfig("Cross = KEY:F\n"); + auto configuration = Pad::LoadInputMapping(); + require(toggleMouseBindings(configuration, SDL_SCANCODE_F7, Pad::MouseButton::None) == 1, "F7 toggles the mouse by default"); + require(toggleMouseBindings(configuration, SDL_SCANCODE_UNKNOWN, Pad::MouseButton::Middle) == 1, "the middle button toggles the mouse by default"); + useConfig("ToggleMouse = MOUSE:Right\n"); + configuration = Pad::LoadInputMapping(); + require(toggleMouseBindings(configuration, SDL_SCANCODE_F7, Pad::MouseButton::None) == 0, "a ToggleMouse line replaces F7"); + require(toggleMouseBindings(configuration, SDL_SCANCODE_UNKNOWN, Pad::MouseButton::Right) == 1, "a ToggleMouse line adds its own binding"); +} + void deadzoneOffMatchesRawSticks() { for (const std::int16_t axis : {std::int16_t{-32768}, std::int16_t{-20000}, std::int16_t{-1}, std::int16_t{0}, std::int16_t{1}, std::int16_t{1234}, std::int16_t{32767}}) { const auto stick = Pad::StickWithDeadzone(axis, axis, 0); @@ -201,6 +220,7 @@ int main() try { mouseLookDefault(); mouseLookParse(); mouseLookReject(); + toggleMouseDefaults(); nonAsciiPaths(); deadzoneOffMatchesRawSticks(); deadzoneShapesSticks(); diff --git a/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp b/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp index 3bb0b15aa7..da82928b86 100644 --- a/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp +++ b/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -182,6 +183,62 @@ void testSmoothing() { check(early > 0.5 * 0.55 && early < 0.5 * 0.72, "smoothing reaches about 63 percent after one time constant"); } +void testSlowReportingMouse() { + const double speed = 0.5 * Pad::MouseFullTiltCountsPerSecond; + const double expected = expectedDeflection(defaults().settings, 0.5); + for (const double reportHz : {125.0, 250.0, 1000.0}) { + std::vector events{PadSimulation::MouseModeAt(0.0, true)}; + PadSimulation::AppendMotion(events, 0.0, 2000.0, speed, 0.0, reportHz); + for (const double poll : {16.7, 8.3, 6.9, 1.0}) { + const auto samples = Simulate(defaults(), events, poll, 2000.0); + double low = 1.0; + double high = 0.0; + for (const auto& sample : samples) { + if (sample.ms < 1000.0) continue; + low = std::min(low, axis(sample.state.sticks[2])); + high = std::max(high, axis(sample.state.sticks[2])); + } + check(high - low < 0.125, "a " + std::to_string(reportHz) + " Hz mouse at " + std::to_string(poll) + " ms polls wavers by " + std::to_string(high - low)); + check(std::abs((high + low) / 2.0 - expected) < 0.05, "a " + std::to_string(reportHz) + " Hz mouse at " + std::to_string(poll) + " ms polls stays around the expected tilt"); + } + } +} + +void testReportRateSmoothing() { + for (const double reportHz : {125.0, 250.0, 1000.0}) { + Pad::MouseStick mouse(defaults().settings); + check(mouse.SmoothingMs() == defaults().settings.mouseSmoothingMs, "smoothing starts at the setting"); + const double periodSeconds = 1.0 / reportHz; + for (int report = 0; report < static_cast(reportHz); ++report) { + mouse.Move(6, 0); + mouse.Advance(periodSeconds); + } + check(std::abs(mouse.ReportRate() - reportHz) < 0.1 * reportHz, "measured " + std::to_string(mouse.ReportRate()) + " reports per second for a " + std::to_string(reportHz) + " Hz mouse"); + const double smoothing = mouse.SmoothingMs(); + check(std::abs(smoothing - std::max(10.0, 3000.0 / reportHz)) < 0.15 * smoothing, "smoothing " + std::to_string(smoothing) + " ms for a " + std::to_string(reportHz) + " Hz mouse"); + } + Pad::MouseStick slow(defaults().settings); + for (int step = 0; step < 40; ++step) { + slow.Move(30, 0); + slow.Advance(0.025); + } + check(std::abs(slow.ReportRate() - 40.0) < 2.0 && slow.SmoothingMs() == Pad::MouseSmoothingLimitMs, "smoothing for very slow reports stops at 30 ms"); + auto off = defaults().settings; + off.mouseSmoothingMs = 0.0; + Pad::MouseStick unsmoothed(off); + for (int report = 0; report < 125; ++report) { + unsmoothed.Move(6, 0); + unsmoothed.Advance(0.008); + } + check(unsmoothed.SmoothingMs() == 0.0, "a smoothing of 0 stays off for a slow mouse"); + Pad::MouseStick single(defaults().settings); + for (int report = 0; report < 125; ++report) { + single.Move(1, 0); + single.Advance(0.008); + } + check(single.ReportRate() == 0.0 && single.SmoothingMs() == defaults().settings.mouseSmoothingMs, "single-count reports are not taken for a slow mouse"); +} + std::vector waveMotion(double base, double swing) { constexpr double Pi = 3.14159265358979323846; std::vector events{PadSimulation::MouseModeAt(0.0, true)}; @@ -283,12 +340,31 @@ void testToggleRequests() { const auto now = Pad::KeyboardMouse::Clock::now(); check(keyboardMouse.HandleEvent(PadSimulation::ButtonEvent(SDL_BUTTON_MIDDLE, true), now).toggleMouse, "middle click asks to toggle the mouse"); check(!keyboardMouse.HandleEvent(PadSimulation::ButtonEvent(SDL_BUTTON_MIDDLE, false), now).toggleMouse, "releasing it asks for nothing"); + check(keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F7, true), now).toggleMouse, "F7 asks to toggle the mouse"); + check(!keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F7, false), now).toggleMouse, "releasing F7 asks for nothing"); check(keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F11, true), now).toggleFullscreen, "F11 asks to toggle fullscreen"); const auto overlay = keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F10, true), now); check(overlay.togglePerformanceOverlay && !overlay.toggleFullscreen, "F10 asks to toggle the performance overlay"); check(!keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F10, false), now).togglePerformanceOverlay, "releasing F10 asks for nothing"); } +void testDefaultL2() { + const auto l2 = static_cast(Pad::PadButton::L2); + const auto r2 = static_cast(Pad::PadButton::R2); + const auto now = Pad::KeyboardMouse::Clock::now(); + for (const auto& [name, press, release] : { + std::tuple{"R", KeyEvent(SDL_SCANCODE_R, true), KeyEvent(SDL_SCANCODE_R, false)}, + std::tuple{"mouse button 4", PadSimulation::ButtonEvent(SDL_BUTTON_X1, true), PadSimulation::ButtonEvent(SDL_BUTTON_X1, false)}}) { + Pad::KeyboardMouse keyboardMouse(defaults()); + keyboardMouse.HandleEvent(press, now); + const auto held = keyboardMouse.Compose(PadInputState{}); + check((held.buttons & l2) != 0 && held.analogButtonsL2 == 255, std::string(name) + " presses L2 fully"); + check((held.buttons & r2) == 0 && held.analogButtonsR2 == 0, std::string(name) + " leaves R2 alone"); + keyboardMouse.HandleEvent(release, now); + const auto released = keyboardMouse.Compose(PadInputState{}); + check((released.buttons & l2) == 0 && released.analogButtonsL2 == 0, std::string(name) + " released lets go of L2"); + } +} void testTitleReadsMouse() { const auto square = static_cast(Pad::PadButton::Square); @@ -344,6 +420,8 @@ int main() { {"decay", testDecay}, {"max speed clamp", testMaxSpeedClamp}, {"smoothing", testSmoothing}, + {"slow reporting mouse", testSlowReportingMouse}, + {"report rate smoothing", testReportRateSmoothing}, {"poll rate independence", testPollRateIndependence}, {"vertical scale", testVerticalScale}, {"mouse mode off", testMouseModeOff}, @@ -352,6 +430,7 @@ int main() { {"wheel and keys on focus loss", testWheelFocusLoss}, {"focus loss releases the mouse", testFocusLossReleasesMouse}, {"toggle requests", testToggleRequests}, + {"default L2", testDefaultL2}, {"title reads the mouse", testTitleReadsMouse}, }; for (const auto& [name, test] : tests) { diff --git a/core/libs/prx/libc/src/General.cpp b/core/libs/prx/libc/src/General.cpp index bd7f31df31..97fc58606b 100644 --- a/core/libs/prx/libc/src/General.cpp +++ b/core/libs/prx/libc/src/General.cpp @@ -353,9 +353,25 @@ int SetWorkingDirectory(WorkingDirectory& state, const std::filesystem::path& re const auto canonical = std::filesystem::canonical(resolved, error); if (error) return DirectoryFailure(error); if (!std::filesystem::is_directory(canonical, error)) return error ? DirectoryFailure(error) : 20; - const auto relative = canonical.lexically_relative(state.root); + auto relative = canonical.lexically_relative(state.root); +#ifdef _WIN32 + if (relative.empty() || *relative.begin() == "..") { + auto ancestor = canonical; + while (!std::filesystem::equivalent(ancestor, state.root, error)) { + if (error) return DirectoryFailure(error); + const auto parent = ancestor.parent_path(); + if (parent == ancestor) return 45; + ancestor = parent; + } + relative = canonical.lexically_relative(ancestor); + } +#endif if (relative.empty() || *relative.begin() == "..") return 45; +#ifdef _WIN32 + state.current = relative == "." ? state.root : state.root / relative; +#else state.current = canonical; +#endif return 0; } } diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index 713a7b8462..725628abbe 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -306,6 +306,8 @@ double APS5_VABI acos_nid_postfix(double x) { return std::acos(x); } double APS5_VABI sqrt_nid_postfix(double x) { return std::sqrt(x); } float APS5_VABI sqrtf_nid_postfix(float x) { return std::sqrt(x); } double APS5_VABI fabs_nid_postfix(double x) { return std::fabs(x); } +float APS5_VABI fabsf_nid_postfix(float x) { return std::fabs(x); } +long double APS5_VABI fabsl_nid_postfix(long double x) { return std::fabs(x); } int APS5_VABI abs_nid_postfix(int x) { return x < 0 ? static_cast(0u - static_cast(x)) : x; } double APS5_VABI trunc_nid_postfix(double x) { return std::trunc(x); } double APS5_VABI ceil_nid_postfix(double x) { return std::ceil(x); } diff --git a/core/libs/prx/libkernel/File/include/DirectoryDescriptor.hpp b/core/libs/prx/libkernel/File/include/DirectoryDescriptor.hpp index a740c9ac0a..f6ab25f51a 100644 --- a/core/libs/prx/libkernel/File/include/DirectoryDescriptor.hpp +++ b/core/libs/prx/libkernel/File/include/DirectoryDescriptor.hpp @@ -11,6 +11,7 @@ extern "C" int OpenInDirectory_nid_no_patch(int directory, const char* path, int namespace File { int OpenDirectoryDescriptor(const std::filesystem::path& path); +bool IsDirectoryDescriptor(int fd); std::optional DirectoryDescriptorPath(int fd); void ForgetDirectoryDescriptor(int fd); void DuplicateDirectoryDescriptor(int from, int to); diff --git a/core/libs/prx/libkernel/File/include/File.hpp b/core/libs/prx/libkernel/File/include/File.hpp index d759d1e3f9..6c8fbfadc2 100644 --- a/core/libs/prx/libkernel/File/include/File.hpp +++ b/core/libs/prx/libkernel/File/include/File.hpp @@ -3,7 +3,9 @@ #include #include +#include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" @@ -27,6 +29,9 @@ std::int64_t NativePositioned_nid_no_patch(int descriptor, void* buf, std::size_ bool DescriptorIsOpen_nid_no_patch(int descriptor); namespace File { +#ifdef _WIN32 +std::optional WindowsDescriptorPath_nid_no_patch(int descriptor); +#endif int HostStandardError(); [[nodiscard]] std::unique_lock LockPositionedTransfer(int descriptor); void SharePositionedTransfer(int from, int to); diff --git a/core/libs/prx/libkernel/File/src/DescriptorControl.cpp b/core/libs/prx/libkernel/File/src/DescriptorControl.cpp index b20bde40a6..82db339af5 100644 --- a/core/libs/prx/libkernel/File/src/DescriptorControl.cpp +++ b/core/libs/prx/libkernel/File/src/DescriptorControl.cpp @@ -141,7 +141,7 @@ const char* KindName(Kind kind) { #ifdef _WIN32 std::optional Classify(int fd, const char* function) { if (fd < 0 || !DescriptorIsOpen_nid_no_patch(fd)) return std::nullopt; - if (File::DirectoryDescriptorPath(fd)) return Kind::Directory; + if (File::IsDirectoryDescriptor(fd)) return Kind::Directory; if (File::IsRandomDevice(fd)) return Kind::RandomDevice; if (File::IsArchivedFile(fd)) return Kind::ArchivedFile; switch (::GetFileType(WindowsDescriptorHandle_nid_no_patch(fd))) { diff --git a/core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp b/core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp index 05868357b8..aedc308062 100644 --- a/core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp +++ b/core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp @@ -17,10 +17,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -32,7 +34,6 @@ constexpr std::uint8_t GuestRegularType = 8; struct DirectoryState { std::mutex mutex; - std::filesystem::path path; std::vector> entries; std::size_t cursor = 0; bool loaded = false; @@ -48,13 +49,15 @@ std::shared_ptr Find(int fd) { return found == g_directories.end() ? nullptr : found->second; } -void Load(DirectoryState& state) { +void Load(DirectoryState& state, int fd) { if (state.loaded) return; + const auto path = File::WindowsDescriptorPath_nid_no_patch(fd); + if (!path) throw std::system_error(errno, std::generic_category(), "directory descriptor path"); state.loaded = true; state.entries.emplace_back(".", GuestDirectoryType); state.entries.emplace_back("..", GuestDirectoryType); std::error_code error; - for (const auto& entry : std::filesystem::directory_iterator(state.path, error)) { + for (const auto& entry : std::filesystem::directory_iterator(*path, error)) { std::error_code typeError; state.entries.emplace_back(entry.path().filename().string(), entry.is_directory(typeError) ? GuestDirectoryType : GuestRegularType); } @@ -65,20 +68,37 @@ void Load(DirectoryState& state) { namespace File { int OpenDirectoryDescriptor(const std::filesystem::path& path) { - const int fd = ::_open("NUL", _O_RDONLY | _O_BINARY); - if (fd < 0) return -1; - std::lock_guard lock(g_mutex); const auto state = std::make_shared(); - state->path = path; - g_directories[fd] = state; - g_directoryCount = g_directories.size(); + SECURITY_ATTRIBUTES security{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE}; + const auto handle = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &security, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); + if (handle == INVALID_HANDLE_VALUE) { + errno = WindowsOpenError_nid_no_patch(GetLastError()); + return -1; + } + const int fd = ::_open_osfhandle(reinterpret_cast(handle), _O_RDONLY | _O_BINARY); + if (fd < 0) { + CloseHandle(handle); + return -1; + } + try { + std::lock_guard lock(g_mutex); + g_directories[fd] = state; + g_directoryCount = g_directories.size(); + } catch (...) { + ::_close(fd); + throw; + } return fd; } +bool IsDirectoryDescriptor(int fd) { + return g_directoryCount != 0 && Find(fd) != nullptr; +} + std::optional DirectoryDescriptorPath(int fd) { - const auto state = Find(fd); - if (!state) return std::nullopt; - return state->path; + if (!IsDirectoryDescriptor(fd)) return std::nullopt; + return WindowsDescriptorPath_nid_no_patch(fd); } void ForgetDirectoryDescriptor(int fd) { @@ -123,7 +143,7 @@ std::optional SeekDirectoryDescriptor(int fd, std::int64_t offset, state.cursor = 0; return std::int64_t{0}; } - Load(state); + Load(state, fd); state.cursor = static_cast(target); return target; } @@ -133,7 +153,7 @@ int ReadDirectoryDescriptor(int fd, char* buf, int nbytes) { if (!found) return SCE_KERNEL_ERROR_ENOTDIR; auto& state = *found; std::lock_guard lock(state.mutex); - Load(state); + Load(state, fd); std::size_t used = 0; while (state.cursor < state.entries.size()) { const auto& [name, type] = state.entries[state.cursor]; @@ -188,7 +208,11 @@ extern "C" const char* GetDirectoryDescriptorPath_nid_no_patch(int fd, int* erro } const auto directory = File::DirectoryDescriptorPath(fd); if (!directory) { +#ifdef _WIN32 + *error = File::IsDirectoryDescriptor(fd) ? errno : ENOTDIR; +#else *error = ENOTDIR; +#endif return nullptr; } struct PathTag {}; diff --git a/core/libs/prx/libkernel/File/src/NativeStat.cpp b/core/libs/prx/libkernel/File/src/NativeStat.cpp index c83090182b..d303b68d50 100644 --- a/core/libs/prx/libkernel/File/src/NativeStat.cpp +++ b/core/libs/prx/libkernel/File/src/NativeStat.cpp @@ -1,5 +1,4 @@ #include "prx/libkernel/File/include/NativeStat.hpp" -#include "prx/libkernel/File/include/DirectoryDescriptor.hpp" #include "prx/libkernel/File/include/ArchivedFile.hpp" #include "prx/libc/include/General.hpp" @@ -43,7 +42,6 @@ static int DoStat(const std::filesystem::path& p, NativeStat* st) { return result; } static int DoFstat(int fd, NativeStat* st) { - if (const auto directory = File::DirectoryDescriptorPath(fd)) return DoStat(*directory, st); if (_fstat64(fd, st) != 0) return -1; return ReadHandleInfo(reinterpret_cast(::_get_osfhandle(fd)), st); } diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp index 9f8639670c..26774d8739 100644 --- a/core/libs/prx/libkernel/File/src/Open.cpp +++ b/core/libs/prx/libkernel/File/src/Open.cpp @@ -58,13 +58,13 @@ static int NativeRead(int fd, void* buf, std::size_t n) { if (n > static_cast(std::numeric_limits::max())) { throw std::runtime_error("sceKernelRead: nbytes exceeds platform limit"); } - const auto positioned = File::LockPositionedTransfer(fd); - const int result = ::_read(fd, buf, static_cast(n)); - if (result < 0 && _doserrno == ERROR_NO_DATA) errno = EAGAIN; - if (result == 0 && File::DirectoryDescriptorPath(fd)) { + if (File::IsDirectoryDescriptor(fd)) { errno = EISDIR; return -1; } + const auto positioned = File::LockPositionedTransfer(fd); + const int result = ::_read(fd, buf, static_cast(n)); + if (result < 0 && _doserrno == ERROR_NO_DATA) errno = EAGAIN; return result; } static int NativeWrite(int fd, const void* buf, std::size_t n) { diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 531c800fbf..7d5742e1d5 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -87,8 +87,7 @@ static int NativeChmod(const std::filesystem::path& path, int mode) { RecordWrittenPath_nid_no_patch(path); return ::_wchmod(path.wstring().c_str(), mode); } -static std::optional NativeDescriptorPath(int descriptor) { - if (auto directory = File::DirectoryDescriptorPath(descriptor)) return directory; +std::optional File::WindowsDescriptorPath_nid_no_patch(int descriptor) { const auto handle = reinterpret_cast(::_get_osfhandle(descriptor)); if (handle == INVALID_HANDLE_VALUE) { errno = EBADF; @@ -100,20 +99,27 @@ static std::optional NativeDescriptorPath(int descriptor) path.resize(length); length = ::GetFinalPathNameByHandleW(handle, path.data(), length, FILE_NAME_NORMALIZED); } - if (length == 0 || length >= path.size()) { + if (length == 0) { + errno = WindowsOpenError_nid_no_patch(GetLastError()); + return std::nullopt; + } + if (length >= path.size()) { errno = EINVAL; return std::nullopt; } path.resize(length); + if (path.starts_with(LR"(\\?\UNC\)")) path.replace(0, 8, LR"(\\)"); + else if (path.starts_with(LR"(\\?\)") && path.size() > 6 && path[5] == L':' && path[6] == L'\\') path.erase(0, 4); return path; } static int NativeFchmod(int descriptor, int mode) { - const auto path = NativeDescriptorPath(descriptor); + const auto path = File::WindowsDescriptorPath_nid_no_patch(descriptor); return path ? NativeChmod(*path, mode) : -1; } static int NativeFtruncate(int descriptor, std::int64_t length) { const int result = File::TruncateSharedMemory_nid_no_patch(descriptor, length); if (result != -2) return result; + if (File::IsDirectoryDescriptor(descriptor)) return EINVAL; const auto handle = reinterpret_cast(WithoutParameterHandler([descriptor] { return ::_get_osfhandle(descriptor); })); if (handle == INVALID_HANDLE_VALUE) return EBADF; if (::GetFileType(handle) != FILE_TYPE_DISK) return EINVAL; @@ -156,7 +162,7 @@ static int NativeFutimes(int descriptor, const KernelTimeval* times) { const int result = SetTimes(handle, times); CloseHandle(handle); if (result == 0) { - if (const auto path = NativeDescriptorPath(descriptor)) RecordWrittenPath_nid_no_patch(*path); + if (const auto path = File::WindowsDescriptorPath_nid_no_patch(descriptor)) RecordWrittenPath_nid_no_patch(*path); } return result; } @@ -215,9 +221,13 @@ static std::int64_t NativePositionedTransfer(int descriptor, void* buf, std::siz errno = EBADF; return -1; } + if (File::IsDirectoryDescriptor(descriptor)) { + errno = write ? EBADF : EISDIR; + return -1; + } LARGE_INTEGER position{}; if (::GetFileType(handle) != FILE_TYPE_DISK || !::SetFilePointerEx(handle, LARGE_INTEGER{}, &position, FILE_CURRENT)) { - errno = !File::DirectoryDescriptorPath(descriptor) ? ESPIPE : write ? EBADF : EISDIR; + errno = ESPIPE; return -1; } OVERLAPPED overlapped{}; @@ -430,7 +440,7 @@ static std::int64_t PosixResult(std::int64_t result) { #ifdef _WIN32 static int NativeFstatfs(int descriptor, KernelStatfs* fs) { - if (!File::DirectoryDescriptorPath(descriptor)) { + if (!File::IsDirectoryDescriptor(descriptor)) { const auto handle = reinterpret_cast(::_get_osfhandle(descriptor)); if (handle == INVALID_HANDLE_VALUE) return GUEST_EBADF; const DWORD type = ::GetFileType(handle); @@ -438,7 +448,7 @@ static int NativeFstatfs(int descriptor, KernelStatfs* fs) { if (type != FILE_TYPE_DISK) throw std::runtime_error("_fstatfs: descriptor " + std::to_string(descriptor) + " is not a file on a Windows volume"); } - const auto path = NativeDescriptorPath(descriptor); + const auto path = File::WindowsDescriptorPath_nid_no_patch(descriptor); if (!path) throw std::runtime_error("_fstatfs: no path for descriptor " + std::to_string(descriptor) + ", errno=" + std::to_string(errno)); std::wstring root(path->native().size() + 2, L'\0'); if (!::GetVolumePathNameW(path->c_str(), root.data(), static_cast(root.size()))) @@ -1163,7 +1173,7 @@ static std::int64_t TransferIovecs(int d, const KernelIovec* iov, int iovcnt, co const auto result = offset != nullptr ? NativePositionedTransfer(d, &none, 0, *offset, write) : NativeTransferBuffer(d, &none, 0, write); return result < 0 ? SceErrorFromErrno(errno) : 0; } - if (!write && offset == nullptr && File::DirectoryDescriptorPath(d)) return SceErrorFromErrno(GUEST_EISDIR); + if (!write && offset == nullptr && File::IsDirectoryDescriptor(d)) return SceErrorFromErrno(GUEST_EISDIR); if (write && offset == nullptr && iovcnt > 1 && total <= static_cast(std::numeric_limits::max()) && File::Appends(d)) { std::vector gathered; gathered.reserve(total); diff --git a/core/libs/tests/GuestKernelSys.cpp b/core/libs/tests/GuestKernelSys.cpp index 9041bbc22a..b31fc66caf 100644 --- a/core/libs/tests/GuestKernelSys.cpp +++ b/core/libs/tests/GuestKernelSys.cpp @@ -1,5 +1,7 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libkernel/File/include/FileFlags.hpp" +#include "SceTypes.hpp" +#include #include #include #include @@ -9,6 +11,7 @@ #include #include #include +#include #include #ifdef _WIN32 #include @@ -34,6 +37,13 @@ int APS5_VABI chroot_nid_postfix(const char*); char* APS5_VABI getcwd_nid_postfix(char*, std::size_t); int APS5_VABI sceKernelOpen(const char*, int, std::uint16_t); int APS5_VABI sceKernelClose(int); +int APS5_VABI sceKernelRename(const char*, const char*); +int APS5_VABI sceKernelFstat(int, FileStat*); +int APS5_VABI sceKernelGetdents(int, char*, int); +std::int64_t APS5_VABI sceKernelLseek(int, std::int64_t, int); +std::int64_t APS5_VABI sceKernelPread(int, void*, std::size_t, std::int64_t); +int APS5_VABI dup_nid_postfix(int); +int APS5_VABI dup2_nid_postfix(int, int); int APS5_VABI setsid_nid_postfix(void); int APS5_VABI link_nid_postfix(const char*, const char*); int APS5_VABI symlink_nid_postfix(const char*, const char*); @@ -110,7 +120,93 @@ void CheckPathconf(const std::filesystem::path& root, const std::string& file) { Require(pathconf_nid_postfix(longPath.c_str(), 4) == -1 && Error() == 63); } +void DirectoryRenameTests(const std::filesystem::path& parent) { + const auto fixture = parent / "directory-identity"; + const auto original = fixture / "original"; + const auto moved = fixture / "moved"; + const auto twice = fixture / "twice"; + Require(std::filesystem::create_directory(fixture)); + Require(std::filesystem::create_directory(original)); + { std::ofstream(original / "marker").put('o'); } + { std::ofstream(original / "old-only").put('a'); } + const auto readMarker = [](int directory) { + const int file = openat_nid_postfix(directory, "marker", SCE_KERNEL_O_RDONLY); + Require(file >= 0); + char marker = 0; + Require(sceKernelRead(file, &marker, 1) == 1); + Require(sceKernelClose(file) == 0); + return marker; + }; + const auto names = [](int directory) { + std::set result; + std::array buffer{}; + for (;;) { + const int bytes = sceKernelGetdents(directory, buffer.data(), buffer.size()); + Require(bytes >= 0); + if (bytes == 0) return result; + for (int offset = 0; offset < bytes;) { + std::uint16_t length = 0; + std::memcpy(&length, buffer.data() + offset + 4, sizeof(length)); + Require(length >= 12 && offset + length <= bytes); + result.emplace(buffer.data() + offset + 8); + offset += length; + } + } + }; + const int first = sceKernelOpen(original.string().c_str(), SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0); + Require(first >= 0 && readMarker(first) == 'o'); + FileStat before{}; + Require(sceKernelFstat(first, &before) == 0 && (before.st_mode & 0170000) == 0040000); + const int duplicate = dup_nid_postfix(first); + const int target = sceKernelOpen((original / "marker").string().c_str(), SCE_KERNEL_O_RDONLY, 0); + Require(duplicate >= 0 && target >= 0); + Require(dup2_nid_postfix(first, target) == target); + Require(sceKernelClose(first) == 0); + Require(sceKernelRename(original.string().c_str(), moved.string().c_str()) == 0); + Require(std::filesystem::create_directory(original)); + { std::ofstream(original / "marker").put('r'); } + { std::ofstream(original / "new-only").put('b'); } + const char marker = readMarker(duplicate); + std::fprintf(stderr, "directory marker after rename/replacement: %c, expected o\n", marker); + Require(marker == 'o' && readMarker(target) == 'o'); + const std::set expected{".", "..", "marker", "old-only"}; + Require(names(duplicate) == expected); + Require(names(target).empty()); + FileStat after{}; + Require(sceKernelFstat(target, &after) == 0 && after.st_dev == before.st_dev && after.st_ino == before.st_ino); + Require(sceKernelLseek(target, 0, 0) == 0); + char directoryByte = 0; + Require(sceKernelRead(target, &directoryByte, 1) == static_cast(0x80020015u)); + Require(sceKernelPread(target, &directoryByte, 1, 0) == static_cast(0x80020015u)); + char previous[4096]; + Require(getcwd_nid_postfix(previous, sizeof(previous)) == previous); + Require(fchdir_nid_postfix(target) == 0 && readMarker(-100) == 'o'); + Require(chdir_nid_postfix(previous) == 0); + Require(mkdirat_nid_postfix(target, "created", 0755) == 0); + Require(std::filesystem::is_directory(moved / "created") && !std::filesystem::exists(original / "created")); + Require(unlinkat_nid_postfix(duplicate, "created", 0x800) == 0); + Require(sceKernelRename(moved.string().c_str(), twice.string().c_str()) == 0); + Require(std::filesystem::create_directory(moved)); + { std::ofstream(moved / "different").put('c'); } + Require(sceKernelLseek(target, 0, 0) == 0 && names(target) == expected); + Require(sceKernelClose(duplicate) == 0); + Require(readMarker(target) == 'o'); + Require(unlinkat_nid_postfix(target, "marker", 0) == 0); + Require(unlinkat_nid_postfix(target, "old-only", 0) == 0); + Require(std::filesystem::remove(twice)); + Require(sceKernelFstat(target, &after) == 0 && after.st_dev == before.st_dev && after.st_ino == before.st_ino && (after.st_mode & 0170000) == 0040000); + Require(sceKernelClose(target) == 0); + Require(std::filesystem::remove(original / "marker") && std::filesystem::remove(original / "new-only")); + Require(std::filesystem::remove(moved / "different")); + Require(std::filesystem::remove(original) && std::filesystem::remove(moved) && std::filesystem::remove(fixture)); +} + int main() { +#ifdef _WIN32 + auto hostDirectorySpelling = std::filesystem::current_path().native(); + CharUpperBuffW(hostDirectorySpelling.data(), static_cast(hostDirectorySpelling.size())); + Require(SetCurrentDirectoryW(hostDirectorySpelling.c_str()) != 0); +#endif Require(setsid_nid_postfix() == -1 && Error() == 1); Require(umask_nid_postfix(0) == 022); Require(umask_nid_postfix(07777) == 0); @@ -118,6 +214,7 @@ int main() { const auto root = std::filesystem::path("anyps5-kernel-sys-test-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); Require(std::filesystem::create_directory(root)); + DirectoryRenameTests(root); const auto file = (root / "file.txt").string(); std::ofstream(file).put('x'); CheckPathconf(root, file); diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index 4c4e0e76df..b8f2c3f50d 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -72,6 +72,8 @@ float APS5_VABI hypotf_nid_postfix(float, float); double APS5_VABI hypot_nid_postfix(double, double); double APS5_VABI fdim_nid_postfix(double, double); float APS5_VABI fdimf_nid_postfix(float, float); +float APS5_VABI fabsf_nid_postfix(float); +long double APS5_VABI fabsl_nid_postfix(long double); float APS5_VABI tanf_nid_postfix(float); int APS5_VABI rand_nid_postfix(); void APS5_VABI srand_nid_postfix(unsigned int); @@ -192,6 +194,11 @@ static void CheckExactOperations() { Require(fabs_nid_postfix(-3.5) == 3.5 && fabs_nid_postfix(-INFINITY) == INFINITY); Require(std::bit_cast(fabs_nid_postfix(-0.)) == 0); Require(std::isnan(fabs_nid_postfix(-nan)) && !std::signbit(fabs_nid_postfix(-nan))); + Require(fabsf_nid_postfix(-3.5f) == 3.5f && fabsf_nid_postfix(-INFINITY) == INFINITY); + Require(std::bit_cast(fabsf_nid_postfix(-0.f)) == 0); + Require(std::isnan(fabsf_nid_postfix(-static_cast(nan))) && !std::signbit(fabsf_nid_postfix(-static_cast(nan)))); + Require(fabsl_nid_postfix(-3.5L) == 3.5L && fabsl_nid_postfix(-INFINITY) == INFINITY); + Require(std::isnan(fabsl_nid_postfix(-static_cast(nan))) && !std::signbit(fabsl_nid_postfix(-static_cast(nan)))); Require(trunc_nid_postfix(2.7) == 2. && trunc_nid_postfix(-2.7) == -2.); Require(std::bit_cast(trunc_nid_postfix(-0.5)) == 0x8000000000000000ull); Require(trunc_nid_postfix(4503599627370497.) == 4503599627370497. && trunc_nid_postfix(-1e300) == -1e300); diff --git a/core/libs/tests/GuestNetCtl.cpp b/core/libs/tests/GuestNetCtl.cpp index 0e5a4f6d26..00998c3d8a 100644 --- a/core/libs/tests/GuestNetCtl.cpp +++ b/core/libs/tests/GuestNetCtl.cpp @@ -23,6 +23,8 @@ int APS5_VABI sceNetCtlCheckCallback(void); int APS5_VABI sceNetCtlGetInfo(int, NetCtlInfo*); int APS5_VABI sceNetCtlGetNatInfo(NetCtlNatInfo*); int APS5_VABI sceNetCtlGetState(int*); +int APS5_VABI sceNetCtlGetStateV6(int*); +int APS5_VABI sceNetCtlGetResult(int, int*); int APS5_VABI sceNetCtlRegisterCallback(NetCtlCallback, void*, int*); int APS5_VABI sceNetCtlRegisterCallbackV6(NetCtlCallback, void*, int*); void APS5_VABI sceNetCtlTerm(void); @@ -189,6 +191,13 @@ int main() { #endif CheckCallbacks(); + Require(sceNetCtlGetStateV6(nullptr) == kErrInvalidAddr); + int stateV6 = -1; + Require(sceNetCtlGetStateV6(&stateV6) == 0); + Require(stateV6 == 0 || stateV6 == kStateIpObtained); + Require(sceNetCtlGetResult(0, nullptr) == kErrInvalidAddr); + int errorCode = -1; + Require(sceNetCtlGetResult(0, &errorCode) == 0 && errorCode == 0); NetCtlInfo info{}; Require(sceNetCtlGetInfo(0, &info) == kErrInvalidCode); Require(sceNetCtlGetInfo(22, &info) == kErrInvalidCode); diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index e4bcac3e90..a5ccd31294 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -251,6 +251,7 @@ function(add_shader_recompiler target runtimeTarget) ResolveSource ResourceMaterializer::EvaluateNanoseconds ResourceMaterializer::MaterializeNanoseconds + ReleaseUndrawnResourcePlan ResourceMaterializer::SpecializationNanoseconds RuntimeAbi::RequireVersion RuntimeAbi::HeapCapacity diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index f83da69525..7067a1fc6c 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -81,6 +81,7 @@ void BuildPreparedShaderKey(const RecompileRequest& request, std::vector RegistrationResult(const RecompileRequest& request, const SourceHandle& handle); [[nodiscard]] std::shared_ptr MaterializeShader(const RecompileRequest& request, const ResourceCapture& capture, const SourceHandle& handle); [[nodiscard]] std::shared_ptr ResolveSource(const RecompileRequest& request); +void ReleaseUndrawnResourcePlan(const SourceHandle& handle); [[nodiscard]] std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime, const SourceHandle& handle); } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index d08bc8a148..7583ddba0d 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -326,6 +326,7 @@ struct SourceEntry : std::enable_shared_from_this { std::vector code; std::vector linkedCallFallbacks; std::shared_ptr plan; + bool runtimeCaptured = false; // A plan build that threw (an unsupported resource chain or control flow) is remembered and // rethrown: the front end ran every pass before failing, ~13 ms per dispatch of a shader the // title issues every frame (0x1048947300 at the intro video). APS5_NO_FAILURE_MEMO=1 rebuilds. @@ -396,7 +397,9 @@ bool diskPlans() { return enabled && ShaderDiskCache::Enabled() && !DebugProbeActive(); } -std::shared_ptr getSource(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr, bool reclaim = true) { +bool sameLayout(const BindingLayout& left, const BindingLayout& right); + +std::shared_ptr getSource(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr, bool reclaim = true, const SourceHandle* preparedHandle = nullptr) { static PrimedSharedMutex mutex; // Entries whose code hashes alike share a bucket; the code comparison picks the right one. static std::unordered_map, std::vector>, SourceKeyHash> sources; @@ -439,6 +442,8 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr { std::lock_guard lock(source->mutex); if (source->plan == nullptr) { + const bool hasPreparedArtifact = preparedHandle != nullptr && preparedHandle->artifact != nullptr; + if (hasPreparedArtifact && (preparedHandle->source != source || !MatchesPreparedShader(request, *preparedHandle))) throw std::runtime_error("ShaderRecompiler: prepared artifact does not match the resource plan source"); if (FailureMemo() && source->planFailure) std::rethrow_exception(source->planFailure); const bool disk = diskPlans(); std::vector planKey; @@ -454,7 +459,9 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr try { ResourceProgram resource(request, preparation, reclaim); source->plan = std::move(resource.plan); - source->program = std::move(resource.program); + if (!hasPreparedArtifact && std::ranges::none_of(source->variants, [&](const auto& variant) { return sameLayout(variant->layout, request.layout); })) { + source->program = std::move(resource.program); + } } catch (...) { if (FailureMemo()) source->planFailure = std::current_exception(); throw; @@ -466,6 +473,25 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr return source; } +std::shared_ptr acquirePlan(const RecompileRequest& request, const std::shared_ptr& source, bool runtimeCapture, const SourceHandle* preparedHandle = nullptr) { + for (;;) { + { + std::lock_guard lock(source->mutex); + if (source->plan != nullptr) { + if (runtimeCapture) source->runtimeCaptured = true; + return source->plan; + } + } + if (getSource(request, nullptr, true, preparedHandle) != source) throw std::runtime_error("ShaderRecompiler: resource plan source changed while reloading"); + } +} + +void releaseUndrawnPlan(const SourceHandle& handle) { + if (handle.source == nullptr || !diskPlans()) return; + std::lock_guard lock(handle.source->mutex); + if (!handle.source->runtimeCaptured) handle.source->plan.reset(); +} + std::array partialThreads(const RecompileRequest& request) { return request.context.compute ? request.context.compute->partialThreads : std::array{}; } @@ -1124,7 +1150,7 @@ RecompileResult RecompileImpl(const RecompileRequest& request) { return materializeResult(variant, request, snapshot); } const auto source = getSource(request); - materializer.Materialize(*source->plan, runtime, snapshot); + materializer.Materialize(*acquirePlan(request, source, true), runtime, snapshot); return materializeVariant(*source, request, snapshot); } @@ -1334,10 +1360,17 @@ auto recompileReporting(const RecompileRequest& request, Impl&& impl) -> decltyp std::shared_ptr GetResourcePlan(const RecompileRequest& request) { static_cast(RequestInputInfo(request)); - if (request.useCache) return getSource(request)->plan; + if (request.useCache) { + const auto source = getSource(request); + return acquirePlan(request, source, false); + } return makeResourcePlan(request); } +void ReleaseUndrawnResourcePlan(const SourceHandle& handle) { + releaseUndrawnPlan(handle); +} + namespace { // The capture's materialization; with pure flat slots in the plan the walk's read addresses are @@ -1367,7 +1400,7 @@ std::shared_ptr CaptureResources(const RecompileRequest& auto capture = std::make_shared(); if (request.useCache) { capture->source = getSource(request); - capture->plan = capture->source->plan; + capture->plan = acquirePlan(request, capture->source, true); } else { capture->plan = makeResourcePlan(request); } @@ -1451,13 +1484,13 @@ std::optional PreparedShaderInvocation::TryCreate(cons std::shared_ptr PreparedShaderInvocation::Capture(const SrtRuntime& runtime) const { auto capture = std::make_shared(); capture->source = handle->source; - capture->plan = handle->source->plan; + capture->plan = acquirePlan(request, handle->source, true, handle.get()); materializeCapture(*capture, runtime); return capture; } std::shared_ptr PreparedShaderInvocation::Materialize(const ResourceCapture& capture) const { - if (capture.source != handle->source || capture.plan != handle->source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + if (capture.source != handle->source) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle->source, handle->artifact, request, capture.snapshot, true, nullptr); auto result = std::make_shared(materializeResult(*handle->artifact, request, capture.snapshot)); result->cacheHit = true; @@ -1485,7 +1518,7 @@ std::shared_ptr RegistrationResult(const RecompileRequest std::shared_ptr MaterializeShader(const RecompileRequest& request, const ResourceCapture& capture, const SourceHandle& handle) { if (!MatchesPreparedShader(request, handle)) throw std::runtime_error("ShaderRecompiler: prepared artifact does not match the static ABI"); - if (capture.source != handle.source || capture.plan != handle.source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + if (capture.source != handle.source) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle.source, handle.artifact, request, capture.snapshot, true, nullptr); auto result = std::make_shared(materializeResult(*handle.artifact, request, capture.snapshot)); result->cacheHit = true; @@ -1501,7 +1534,7 @@ std::shared_ptr CaptureResources(const RecompileRequest& if (request.shader.stage != ShaderStage::Compute && request.shader.stage != ShaderStage::Fragment) static_cast(RequestInputInfo(request)); auto capture = std::make_shared(); capture->source = handle.source; - capture->plan = handle.source->plan; + capture->plan = acquirePlan(request, handle.source, true, &handle); if (profile) capture->sourceNanoseconds = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); materializeCapture(*capture, runtime); return capture; diff --git a/demo/web/app.css b/demo/web/app.css index c5258a1102..b7c45cab14 100644 --- a/demo/web/app.css +++ b/demo/web/app.css @@ -4,3 +4,7 @@ *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:15px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif}button,input,select{font:inherit}button,a,select,input{touch-action:manipulation}button{cursor:pointer}button:disabled{cursor:not-allowed;opacity:.45}a{color:var(--accent);text-underline-offset:3px}button:focus-visible,a:focus-visible,select:focus-visible,input:focus-visible{outline:2px solid var(--accent);outline-offset:4px}code,pre,.eyebrow,.fine{font-family:ui-monospace,"Cascadia Mono",Consolas,monospace}code{font-size:.9em}h1,h2,h3,p{margin-top:0}[hidden]{display:none!important}header{border-bottom:1px solid var(--line);background:var(--surface)}.bar{max-width:1120px;margin:auto;min-height:68px;padding:12px 24px;display:flex;align-items:center;justify-content:space-between;gap:24px}.brand{display:flex;align-items:center;gap:10px;text-decoration:none;color:var(--text);font-weight:700;font-size:17px;white-space:nowrap}.brand>span:last-child{font-weight:400;color:var(--muted);font-size:14px}.mark{display:flex;gap:3px;align-items:center;margin-right:3px}.mark i{display:block;width:4px;height:20px;background:var(--text)}.mark i:last-child{opacity:.3}nav{display:flex;align-items:center;gap:24px;font-size:13px}nav a{color:var(--muted);text-decoration:none}nav a:hover{color:var(--text)}main{max-width:1024px;margin:auto;padding:58px 24px 36px}.intro{max-width:660px;margin-bottom:30px}.eyebrow{color:var(--muted);letter-spacing:.12em;font-size:11px;margin-bottom:12px}h1{font-size:clamp(29px,4vw,43px);line-height:1.15;letter-spacing:-.035em;font-weight:650;margin-bottom:18px}.converter{border:1px solid var(--line);border-radius:10px;background:var(--surface);overflow:hidden}.panel-heading{display:flex;justify-content:space-between;align-items:center;gap:16px;border-bottom:1px solid var(--line);padding:19px 26px}.panel-heading h2{font-size:15px;margin:0;font-weight:600}.private{font-size:12px;color:var(--muted)}.private b{color:var(--good);font-size:9px;margin-right:5px}.drop{margin:24px;border:1px dashed var(--line);border-radius:6px;text-align:center;padding:28px 16px 17px;background:var(--bg)}.drop.dragging{border-color:var(--accent);background:var(--sunken)}.archive-icon{display:inline-flex;align-items:center;justify-content:center;width:40px;height:45px;border:1px solid var(--line);border-radius:4px;font-size:25px;color:var(--muted);margin-bottom:15px}.drop h3{font-size:19px;margin-bottom:8px;font-weight:600}.drop>p{color:var(--muted);font-size:13px;margin-bottom:20px}.button{display:inline-flex;align-items:center;justify-content:center;gap:18px;border:1px solid transparent;border-radius:6px;background:var(--accent);color:var(--on-accent);text-decoration:none;font-weight:600;font-size:13px;padding:11px 20px;white-space:nowrap}.button:hover:not(:disabled){filter:brightness(.94)}.drop .fine{margin:15px 0 0;font-size:10px}.fine{font-size:11px;line-height:1.6;color:var(--muted);margin-bottom:0}.plain{border:0;background:none;color:var(--muted);padding:8px;font-size:13px}.plain:hover{color:var(--text)}#theme{font-size:19px;padding:3px}.settings{display:grid;grid-template-columns:1fr 1fr;gap:36px;padding:0 26px 20px}fieldset{border:0;margin:0;padding:0;min-width:0}legend,.save-settings>label{display:block;font-size:12px;font-weight:600;margin-bottom:10px}.choices{display:flex;gap:10px}.choices label{flex:1;display:flex;gap:9px;align-items:center;border:1px solid var(--line);border-radius:6px;padding:10px;font-size:13px;cursor:pointer}.choices label:has(input:checked){border-color:var(--accent)}.choices span{font-size:11px;color:var(--muted);margin-left:auto}input{accent-color:var(--accent)}input[type=radio],input[type=checkbox]{margin:0;width:15px;height:15px;flex-shrink:0}select{width:100%;padding:10px;border:1px solid var(--line);border-radius:6px;background:var(--surface);color:var(--text);font-size:13px}.save-settings .fine{margin-top:8px}.options{padding:0 26px 23px;font-size:12px;color:var(--muted)}.options label{display:flex;gap:9px;align-items:center}.action-row{border-top:1px solid var(--line);padding:17px 26px;display:flex;gap:12px;align-items:center}.action-row>p{margin:0 auto 0 0;font-size:12px}.muted{color:var(--muted)}.selection{display:flex;justify-content:space-between;align-items:center;padding:25px 26px;gap:16px}.selection h3{font-size:18px;margin-bottom:6px;overflow-wrap:anywhere}.selection p{margin-bottom:0;font-size:12px}.selection .eyebrow{font-size:10px;margin-bottom:6px}.warnings,.notice{padding:16px 20px;background:var(--warning-bg);color:var(--warning);font-size:13px;border:1px solid var(--line);border-radius:6px}.warnings{margin:0 26px 24px}.warnings ul{padding-left:18px;margin:0 0 12px}.warnings label{display:flex;gap:8px;align-items:center}.notice{margin-bottom:20px}.status{border-top:1px solid var(--line);padding:22px 26px}.status-line{display:flex;justify-content:space-between;font-size:13px;gap:16px}progress{appearance:none;width:100%;height:5px;margin:14px 0 8px;border:0;border-radius:3px;overflow:hidden;background:var(--line)}progress::-webkit-progress-bar{background:var(--line)}progress::-webkit-progress-value{background:var(--accent)}progress::-moz-progress-bar{background:var(--accent)}#current-file{overflow-wrap:anywhere;min-height:18px}details{margin-top:18px;font-size:11px;color:var(--muted)}summary{cursor:pointer}pre{max-height:180px;overflow:auto;white-space:pre-wrap;word-break:break-word;font-size:11px}.error{margin:0 26px 24px;padding:15px;background:var(--error-bg);color:var(--error);border-radius:6px;white-space:pre-wrap;overflow-wrap:anywhere}.done{padding:24px 26px;border-top:1px solid var(--line)}.done h3{font-size:20px;color:var(--good);margin-bottom:8px}.done p{font-size:13px;color:var(--muted)}.done .fine{margin:15px 0 0}.compatibility{margin:12px 0 0;color:var(--muted);font-size:11px;line-height:1.8}.limits{border-top:1px solid var(--line);padding-top:22px;margin:30px 0 0;color:var(--muted);font-size:11px;line-height:1.8}footer{max-width:1024px;margin:auto;border-top:1px solid var(--line);padding:22px 24px;display:flex;justify-content:space-between;font-size:11px;color:var(--muted)}footer div{display:flex;gap:20px}footer a{color:var(--muted);text-decoration:none}@media(max-width:680px){.bar{flex-wrap:wrap;gap:12px;padding:14px 20px}nav{gap:17px;font-size:12px}main{padding:34px 16px 26px}.panel-heading{padding:16px}.private{font-size:10px}.drop{margin:16px}.settings{grid-template-columns:1fr;padding:0 16px 18px;gap:22px}.options{padding:0 16px 20px}.action-row{padding:16px;flex-wrap:wrap}.action-row>p{width:100%}.selection{padding:20px 16px}.warnings{margin:0 16px 20px}.status,.done{padding:20px 16px}.error{margin:0 16px 20px}footer{flex-wrap:wrap;gap:12px}footer div{flex-wrap:wrap;gap:14px}} .input-actions{display:flex;justify-content:center;flex-wrap:wrap;gap:12px} +.app-list{display:grid;gap:8px;max-height:220px;overflow:auto;padding-left:22px;margin:14px 0 0;font-size:12px;color:var(--muted)} +.app-list li{padding:8px 0;border-top:1px solid var(--line)} +.app-list strong,.app-list span{display:block;overflow-wrap:anywhere} +.app-list strong{color:var(--text)} diff --git a/demo/web/app.mjs b/demo/web/app.mjs index bb48086a67..8e05b091e0 100644 --- a/demo/web/app.mjs +++ b/demo/web/app.mjs @@ -1,10 +1,12 @@ import { abortAndDiscardOutput, OutputDirectoryTransaction } from "./output-directory.mjs"; +import { batchMemoryEstimate } from "./batch.mjs"; const $ = id => document.getElementById(id); const stages = ["Converting executable", "Converting PRX modules", "Scanning syscalls", "Scanning instructions", "Converting instructions", "Analysing imports", "Building output image", "Writing output"]; let worker; -let file; -let summary; +let selectedInputs = []; +let summaries = []; +let inspectionComplete = false; let working = false; let failing = false; let sink; @@ -26,27 +28,55 @@ function bytes(value) { return `${(value / 1024 ** unit).toFixed(1)} ${["B", "KiB", "MiB", "GiB"][unit]}`; } +function sum(values) { + let total = 0; + for (const value of values) { + if (!Number.isSafeInteger(value) || value < 0 || total > Number.MAX_SAFE_INTEGER - value) return null; + total += value; + } + return total; +} + function update() { const target = document.querySelector('input[name="target"]:checked').value; - const folderBlocked = target === "windows" && $("save").value === "disk"; + const multi = selectedInputs.length > 1; + const inspected = inspectionComplete && summaries.length === selectedInputs.length; + const folderBlocked = $("save").value === "disk" && (target === "windows" || multi); const folderOption = $("save").querySelector('[value="disk"]'); - folderOption.disabled = !canStream || target === "windows"; - folderOption.textContent = target === "windows" ? "Save to folder · unavailable for Windows (DLL restrictions)" : "Save to folder · uncompressed"; + folderOption.disabled = !canStream || target === "windows" || multi; + folderOption.textContent = target === "windows" ? "Save to folder · unavailable for Windows (DLL restrictions)" : multi ? "Save to folder · unavailable for batch output" : "Save to folder · uncompressed"; const libraryBytes = release?.platforms[target]?.expandedBytes; - const selectedBytes = summary ? target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes : null; - const conversionTooLarge = selectedBytes !== null && selectedBytes > 256 * 1024 * 1024; - const iconTooLarge = target === "windows" && Boolean(summary?.windowsIconTooLarge); + const selectedBytes = inspected ? sum(summaries.map(summary => target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes)) : null; + const conversionTooLarge = inspected && summaries.some(summary => (target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes) > 256 * 1024 * 1024); + const iconTooLarge = target === "windows" && inspected && summaries.some(summary => summary.windowsIconTooLarge); const conversionBlocked = conversionTooLarge || iconTooLarge; - const estimate = summary && libraryBytes !== undefined ? Math.ceil(summary.totalBytes * 1.02) + selectedBytes * 4 + libraryBytes + summary.files * 1024 + 8 * 1024 * 1024 : null; - const tooLarge = estimate !== null && estimate > 256 * 1024 * 1024; + let estimate = null; + let estimateOverflow = false; + if (inspected && libraryBytes !== undefined) { + try { estimate = batchMemoryEstimate(summaries, target, libraryBytes); } + catch (error) { + if (!(error instanceof Error) || error.message !== "ZIP contains an unsupported file size") throw error; + estimateOverflow = true; + } + } + const tooLarge = estimateOverflow || (estimate !== null && estimate > 256 * 1024 * 1024); $("save").querySelector('[value="memory"]').disabled = Boolean(tooLarge); const memoryBlocked = $("save").value === "memory" && tooLarge; - if (summary) $("archive-info").textContent = `${summary.files} files · ${bytes(summary.totalBytes)} expanded · ${summary.modules} modules · ${bytes(selectedBytes)} ${target} conversion input` + (summary.root ? ` · Root: ${summary.root}` : ""); - $("save-note").textContent = $("save").value === "disk" ? "Creates a subfolder with the executable, libs and app0. No ZIP compression. Incomplete output folders are removed after cancellation or failure. Browsers may block DLL and other restricted files; use ZIP for those apps." : $("save").value === "zip" ? "Writes a compressed ZIP directly to disk without holding the whole archive in memory." : "Up to 256 MiB including expanded files and output overhead."; + if (inspected) { + const totalFiles = sum(summaries.map(summary => summary.files)); + const totalBytes = sum(summaries.map(summary => summary.totalBytes)); + const modules = sum(summaries.map(summary => summary.modules)); + const sizeText = value => value === null ? "beyond the supported size range" : bytes(value); + $("archive-info").textContent = multi + ? `${selectedInputs.length} apps · ${sizeText(totalFiles)} files · ${sizeText(totalBytes)} expanded · ${sizeText(modules)} modules · ${sizeText(selectedBytes)} ${target} conversion input` + : `${summaries[0].files} files · ${bytes(summaries[0].totalBytes)} expanded · ${summaries[0].modules} modules · ${bytes(target === "windows" ? summaries[0].windowsSelectedBytes : summaries[0].selectedBytes)} ${target} conversion input` + (summaries[0].root ? ` · Root: ${summaries[0].root}` : ""); + } + $("save-note").textContent = multi ? $("save").value === "memory" ? "Creates one ZIP with each app in its own folder. The complete bundle must fit in the 256 MiB memory-download budget." : "Writes one compressed ZIP containing each app in its own folder, without holding the whole bundle in memory." : $("save").value === "disk" ? "Creates a subfolder with the executable, libs and app0. No ZIP compression. Incomplete output folders are removed after cancellation or failure. Browsers may block DLL and other restricted files; use ZIP for those apps." : $("save").value === "zip" ? "Writes a compressed ZIP directly to disk without holding the whole archive in memory." : "Up to 256 MiB including expanded files and output overhead."; $("browser-note").hidden = supported && canStream && canSaveZip && !memoryBlocked && !folderBlocked && !conversionBlocked; - $("browser-note").textContent = !supported ? "This browser or connection does not support the converter. Use a current browser over HTTPS, or use the native relinker." : folderBlocked ? "Browsers block direct DLL creation. Windows output requires ZIP. Choose Save ZIP to disk or Download in memory." : iconTooLarge ? "The Windows icon exceeds 16 MiB. Choose Linux or use the native relinker." : conversionTooLarge ? `This ${target} conversion input exceeds 256 MiB. Use the native relinker for this app.` : memoryBlocked ? "This archive exceeds the memory-download limit. Choose Save ZIP to disk. If that option is unavailable, open this page in desktop Chrome or Edge." : "Some direct disk saving options are unavailable in this browser or context. Small apps can still be downloaded in memory. For larger apps, use desktop Chrome or Edge over HTTPS."; - $("convert").disabled = !supported || working || !summary || !release || conversionBlocked || memoryBlocked || folderBlocked || (summary.warnings.length > 0 && !$("acknowledge").checked); - $("ready-note").textContent = summary ? (folderBlocked ? "Choose a ZIP save mode for Windows." : iconTooLarge ? "The Windows icon exceeds 16 MiB. Choose Linux or use the native relinker." : conversionTooLarge ? `The ${target} conversion input exceeds 256 MiB. Use the native relinker.` : memoryBlocked ? "A direct disk save is required for this archive." : release ? `Libraries: ${release.tag} · ${$("save").value === "disk" ? "Uncompressed folder" : "ZIP compression level 9"}` : "Loading release information…") : "Choose a ZIP or folder to get started."; + $("browser-note").textContent = !supported ? "This browser or connection does not support the converter. Use a current browser over HTTPS, or use the native relinker." : folderBlocked && multi ? "Batch conversion creates one ZIP bundle. Choose Save ZIP to disk or Download in memory." : folderBlocked ? "Browsers block direct DLL creation. Windows output requires ZIP. Choose Save ZIP to disk or Download in memory." : iconTooLarge ? "A Windows icon exceeds 16 MiB. Choose Linux or use the native relinker." : conversionTooLarge ? `A ${target} conversion input exceeds 256 MiB. Use the native relinker for that app.` : estimateOverflow ? "The combined output exceeds the browser's supported size range. Choose Save ZIP to disk." : memoryBlocked ? "This output exceeds the memory-download limit. Choose Save ZIP to disk. If that option is unavailable, open this page in desktop Chrome or Edge." : "Some direct disk saving options are unavailable in this browser or context. Small apps can still be downloaded in memory. For larger apps, use desktop Chrome or Edge over HTTPS."; + const hasWarnings = inspected && summaries.some(summary => summary.warnings.length > 0); + $("convert").disabled = !supported || working || !inspected || !release || conversionBlocked || memoryBlocked || folderBlocked || (hasWarnings && !$("acknowledge").checked); + $("ready-note").textContent = inspected ? (folderBlocked && multi ? "Choose a ZIP save mode for the app bundle." : folderBlocked ? "Choose a ZIP save mode for Windows." : iconTooLarge ? "A Windows icon exceeds 16 MiB. Choose Linux or use the native relinker." : conversionTooLarge ? `A ${target} conversion input exceeds 256 MiB. Use the native relinker for that app.` : memoryBlocked ? "A direct disk save is required for this output." : release ? `Libraries: ${release.tag} · ${multi ? "ZIP bundle" : $("save").value === "disk" ? "Uncompressed folder" : "ZIP compression level 9"}` : "Loading release information…") : "Choose one folder or one or more ZIP files to get started."; } function lock(value) { @@ -59,7 +89,9 @@ async function fail(message) { worker?.terminate(); worker = null; working = true; - summary = null; + selectedInputs = []; + summaries = []; + inspectionComplete = false; const transaction = outputTransaction; outputTransaction = null; outputDirectory = null; @@ -75,6 +107,9 @@ async function fail(message) { $("status").hidden = true; $("cancel").hidden = true; $("drop").hidden = false; + $("selection").hidden = true; + $("warnings").hidden = true; + $("app-list").replaceChildren(); $("file").value = ""; working = false; failing = false; @@ -86,13 +121,14 @@ async function onMessage({ data }) { try { if (data.type === "progress") { if (data.message) { - logs.push(data.message.trim()); + logs.push(`${data.appName ? `${data.appName}: ` : ""}${data.message.trim()}`); logs = logs.slice(-150); $("log").textContent = logs.join("\n"); return; } - $("phase").textContent = data.phase ?? stages[data.stage]; - $("current-file").textContent = data.input ?? ""; + const phase = data.phase ?? stages[data.stage]; + $("phase").textContent = data.appCount > 1 ? `App ${data.appIndex + 1}/${data.appCount} · ${phase}` : phase; + $("current-file").textContent = [data.appName, data.input].filter(Boolean).join(" · "); if (Number.isFinite(data.total) && data.total > 0) { const percent = Math.min(100, Math.floor(data.completed / data.total * 100)); $("progress").value = percent; @@ -102,13 +138,34 @@ async function onMessage({ data }) { $("percent").textContent = ""; } } else if (data.type === "inspected") { - summary = data.summary; + summaries[data.index] = data.summary; + } else if (data.type === "inspection-complete") { + inspectionComplete = true; working = false; $("selection").hidden = false; $("drop").hidden = true; - $("archive-name").textContent = file.name; - $("warning-list").replaceChildren(...summary.warnings.map(message => { const item = document.createElement("li"); item.textContent = message; return item; })); - $("warnings").hidden = !summary.warnings.length; + $("archive-name").textContent = selectedInputs.length > 1 ? `${selectedInputs.length} app ZIPs` : selectedInputs[0].name; + $("app-list").replaceChildren(...selectedInputs.map((input, index) => { + const item = document.createElement("li"); + const name = document.createElement("strong"); + name.textContent = input.name; + const details = document.createElement("span"); + const summary = summaries[index]; + details.textContent = `${summary.files} files · ${bytes(summary.totalBytes)} expanded · ${summary.modules} modules`; + item.append(name, details); + return item; + })); + $("app-list").hidden = selectedInputs.length < 2; + const warningItems = []; + for (let index = 0; index < summaries.length; index++) { + for (const message of summaries[index].warnings) { + const item = document.createElement("li"); + item.textContent = selectedInputs.length > 1 ? `${selectedInputs[index].name}: ${message}` : message; + warningItems.push(item); + } + } + $("warning-list").replaceChildren(...warningItems); + $("warnings").hidden = warningItems.length === 0; $("status").hidden = true; $("cancel").hidden = true; lock(false); @@ -144,7 +201,7 @@ async function onMessage({ data }) { $("done").hidden = false; $("status").hidden = true; $("cancel").hidden = true; - $("result-info").textContent = `${bytes(data.size)} ${data.mode === "disk" ? "files" : "ZIP"} · ${data.release} system libraries · ${data.outputName}`; + $("result-info").textContent = `${data.apps > 1 ? `${data.apps} apps · ` : ""}${bytes(data.size)} ${data.mode === "disk" ? "files" : "ZIP"} · ${data.release} system libraries · ${data.apps > 1 ? "combined app bundle" : data.outputName}`; if (data.blob) { downloadUrl = URL.createObjectURL(data.blob); $("download").href = downloadUrl; @@ -156,11 +213,14 @@ async function onMessage({ data }) { $("result-info").textContent += ` · Saved in ${outputTransaction.name}/`; if (document.querySelector('input[name="target"]:checked').value === "linux") $("result-info").textContent += ` · On Linux, run chmod +x "${data.outputName}" before launching.`; } + $("result-note").textContent = data.apps > 1 ? "Extract the ZIP; each app is in its own folder with its executable, libs and app0 files. On Linux, run chmod +x */*.elf before launching." : "Extract the entire ZIP before launching the executable."; $("result-note").hidden = data.mode === "disk"; outputDirectory = null; outputTransaction?.commit(); outputTransaction = null; - summary = null; + selectedInputs = []; + summaries = []; + inspectionComplete = false; worker.terminate(); worker = null; lock(false); @@ -169,13 +229,17 @@ async function onMessage({ data }) { } catch (error) { await fail(error.message); } } -async function selectFile(selected, files) { - if (working || !supported || !selected) return; - if (!files && !selected.name.toLowerCase().endsWith(".zip")) { await fail("Choose a ZIP archive."); return; } +async function selectInputs(inputs) { + if (working || !supported || !inputs.length) return; + if (inputs.some(input => !input.files && !input.name.toLowerCase().endsWith(".zip")) || (inputs.length > 1 && inputs.some(input => input.files))) { + await fail("Batch conversion accepts ZIP archives only. Choose one folder or select ZIP files."); + return; + } worker?.terminate(); if (downloadUrl) { URL.revokeObjectURL(downloadUrl); downloadUrl = null; } - summary = null; - file = selected; + selectedInputs = inputs; + summaries = []; + inspectionComplete = false; working = true; logs = []; $("log").textContent = ""; @@ -184,7 +248,7 @@ async function selectFile(selected, files) { $("status").hidden = false; $("cancel").hidden = false; $("phase").textContent = "Inspecting files"; - $("current-file").textContent = file.name; + $("current-file").textContent = inputs.length > 1 ? `Inspecting ${inputs.length} ZIP archives` : inputs[0].name; $("percent").textContent = ""; $("progress").removeAttribute("value"); lock(true); @@ -192,7 +256,7 @@ async function selectFile(selected, files) { worker = new Worker(new URL("./worker.mjs", import.meta.url), { type: "module" }); worker.onmessage = onMessage; worker.onerror = event => fail(event.message || "The converter worker stopped. Try a smaller app or use the native relinker."); - worker.postMessage(files ? { type: "inspect", files } : { type: "inspect", file }); + worker.postMessage({ type: "inspect", inputs }); } $("choose").onclick = () => { $("file").value = ""; $("file").click(); }; @@ -211,12 +275,21 @@ $("change").onclick = () => { $("choose-folder").onclick = () => { $("folder").value = ""; $("folder").click(); }; $("folder").onchange = () => { const files = [...$("folder").files].map(file => ({ path: file.webkitRelativePath, file })); - if (files.length) selectFile({ name: files[0].path.split("/")[0] }, files); + if (files.length) selectInputs([{ name: files[0].path.split("/")[0], files }]); +}; +$("file").onchange = () => { + const inputs = [...$("file").files].map(file => ({ name: file.name, file })); + $("file").value = ""; + selectInputs(inputs); }; -$("file").onchange = () => selectFile($("file").files[0]); for (const type of ["dragenter", "dragover"]) $("drop").addEventListener(type, event => { event.preventDefault(); if (!working) $("drop").classList.add("dragging"); }); $("drop").ondragleave = () => $("drop").classList.remove("dragging"); -$("drop").ondrop = event => { event.preventDefault(); $("drop").classList.remove("dragging"); if (working) return; if (event.dataTransfer.files.length !== 1) { fail("Drop one ZIP archive at a time."); return; } selectFile(event.dataTransfer.files[0]); }; +$("drop").ondrop = event => { + event.preventDefault(); + $("drop").classList.remove("dragging"); + if (working) return; + selectInputs([...event.dataTransfer.files].map(file => ({ name: file.name, file }))); +}; for (const id of ["save", "target", "acknowledge"]) $(id).addEventListener("change", update); $("cancel").onclick = () => fail("Cancelled. Choose the ZIP or folder again to restart."); $("convert").onclick = async () => { @@ -225,19 +298,21 @@ $("convert").onclick = async () => { lock(true); update(); try { + const target = document.querySelector('input[name="target"]:checked').value; if ($("save").value === "zip") { - const handle = await window.showSaveFilePicker({ suggestedName: file.name.replace(/\.zip$/iu, "") + "-" + document.querySelector('input[name="target"]:checked').value + ".zip", types: [{ description: "ZIP archive", accept: { "application/zip": [".zip"] } }] }); + const name = selectedInputs.length > 1 ? `anyps5-${target}-apps.zip` : `${selectedInputs[0].name.replace(/\.zip$/iu, "")}-${target}.zip`; + const handle = await window.showSaveFilePicker({ suggestedName: name, types: [{ description: "ZIP archive", accept: { "application/zip": [".zip"] } }] }); sink = await handle.createWritable(); } else if ($("save").value === "disk") { const parent = await window.showDirectoryPicker({ mode: "readwrite" }); - const name = file.name.replace(/\.zip$/iu, "") + "-" + document.querySelector('input[name="target"]:checked').value; + const name = selectedInputs[0].name.replace(/\.zip$/iu, "") + "-" + target; outputTransaction = await OutputDirectoryTransaction.create(parent, name); outputDirectory = outputTransaction.directory; } $("status").hidden = false; $("cancel").hidden = false; $("error").hidden = true; - worker.postMessage({ type: "convert", target: document.querySelector('input[name="target"]:checked').value, toIntel: $("intel").checked, mode: $("save").value }); + worker.postMessage({ type: "convert", target, toIntel: $("intel").checked, mode: $("save").value }); } catch (error) { if (error.name === "AbortError") { working = false; lock(false); update(); } else await fail(error.message); diff --git a/demo/web/archive.mjs b/demo/web/archive.mjs index eca9ca9778..1e4d3319d2 100644 --- a/demo/web/archive.mjs +++ b/demo/web/archive.mjs @@ -1,8 +1,9 @@ import { BlobReader, Uint8ArrayReader, ZipReader, ZipWriter, configure } from "https://cdn.jsdelivr.net/npm/@zip.js/zip.js@2.7.57/lib/zip-no-worker.js/+esm"; +import { batchMemoryEstimate, batchOutputDirectory, MEMORY_LIMIT, memoryEstimate } from "./batch.mjs"; configure({ useWebWorkers: false, useCompressionStream: false, chunkSize: 65536 }); -export const MEMORY_LIMIT = 256 * 1024 * 1024; +export { batchMemoryEstimate, batchOutputDirectory, memoryEstimate, MEMORY_LIMIT }; export const CONVERSION_LIMIT = 256 * 1024 * 1024; export const ICON_LIMIT = 16 * 1024 * 1024; const MAX_ENTRIES = 100000; @@ -243,8 +244,8 @@ export class Archive { async close() { await this.reader.close(); } } -export function memoryEstimate(archiveBytes, selectedBytes, libraryBytes, entries) { - return safeSize(Math.ceil(archiveBytes * 1.02) + selectedBytes * 4 + libraryBytes + entries * 1024 + 8 * 1024 * 1024); +export function createPackageWriter(destination) { + return new ZipWriter(destination, { level: 9, zip64: true, bufferedWrite: false, useWebWorkers: false, useCompressionStream: false, preventClose: true }); } async function appendEntry(writer, name, entry, report) { @@ -321,13 +322,19 @@ class DirectoryWriter { async close() {} } -export async function packageArchive(archive, converted, outputName, libraryFile, destination, report, directory = false) { +export async function packageArchive(archive, converted, outputName, libraryFile, destination, report, directory = false, options = {}) { const libraries = new ZipReader(new BoundedBlobReader(libraryFile)); - const writer = directory ? new DirectoryWriter(destination) : new ZipWriter(destination, { level: 9, zip64: true, bufferedWrite: false, useWebWorkers: false, useCompressionStream: false, preventClose: true }); + const ownsWriter = !options.writer; + const writer = options.writer ?? (directory ? new DirectoryWriter(destination) : createPackageWriter(destination)); + const outputPath = name => { + const path = options.prefix ? `${options.prefix}${name}` : name; + validPath(path); + return path; + }; const names = new Set(); const claim = name => { - validPath(name); - const key = name.replace(/\/$/u, "").toLowerCase(); + const path = outputPath(name); + const key = path.replace(/\/$/u, "").toLowerCase(); if (names.has(key)) throw new Error(`Output path collision: ${name}`); names.add(key); }; @@ -363,18 +370,18 @@ export async function packageArchive(archive, converted, outputName, libraryFile } } const binary = { versionMadeBy: 0x31e, externalFileAttributes: (0o100755 << 16) >>> 0 }; - await writer.add(outputName, new Uint8ArrayReader(converted.executable), binary); - if (converted.diagnostics) await writer.add("windows-diagnostics-imports.txt", new Uint8ArrayReader(new TextEncoder().encode(converted.diagnostics))); - for (const module of converted.modules) await writer.add(module.name, new Uint8ArrayReader(module.bytes), binary); + await writer.add(outputPath(outputName), new Uint8ArrayReader(converted.executable), binary); + if (converted.diagnostics) await writer.add(outputPath("windows-diagnostics-imports.txt"), new Uint8ArrayReader(new TextEncoder().encode(converted.diagnostics))); + for (const module of converted.modules) await writer.add(outputPath(module.name), new Uint8ArrayReader(module.bytes), binary); for (const entry of libraryEntries) { - if (entry.directory) await writer.add(entry.filename, undefined, { directory: true }); - else await appendEntry(writer, entry.filename, entry, report); + if (entry.directory) await writer.add(outputPath(entry.filename), undefined, { directory: true }); + else await appendEntry(writer, outputPath(entry.filename), entry, report); } for (const { entry, name } of originalFiles) { - if (entry.directory) await writer.add("app0/" + name, undefined, { directory: true }); - else await appendEntry(writer, "app0/" + name, entry, report); + if (entry.directory) await writer.add(outputPath("app0/" + name), undefined, { directory: true }); + else await appendEntry(writer, outputPath("app0/" + name), entry, report); } - await writer.close(); + if (ownsWriter) await writer.close(); } finally { await libraries.close(); } diff --git a/demo/web/batch.mjs b/demo/web/batch.mjs new file mode 100644 index 0000000000..5058dbea4f --- /dev/null +++ b/demo/web/batch.mjs @@ -0,0 +1,27 @@ +export const MEMORY_LIMIT = 256 * 1024 * 1024; + +function safeSize(value) { + if (!Number.isSafeInteger(value) || value < 0) throw new Error("ZIP contains an unsupported file size"); + return value; +} + +export function memoryEstimate(archiveBytes, selectedBytes, libraryBytes, entries) { + return safeSize(Math.ceil(archiveBytes * 1.02) + selectedBytes * 4 + libraryBytes + entries * 1024 + 8 * 1024 * 1024); +} + +export function batchMemoryEstimate(summaries, target, libraryBytes) { + if (target !== "linux" && target !== "windows") throw new Error(`Unsupported conversion target: ${target}`); + return summaries.reduce((total, summary) => safeSize(total + memoryEstimate( + summary.totalBytes, + target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes, + libraryBytes, + summary.files + )), 0); +} + +export function batchOutputDirectory(name, index) { + if (!Number.isSafeInteger(index) || index < 0) throw new Error("Invalid app index"); + const stem = String(name).replace(/\.zip$/iu, "").normalize("NFKC") + .replace(/[^A-Za-z0-9._-]+/gu, "-").replace(/^[.-]+/u, "").slice(0, 80).replace(/[. -]+$/u, ""); + return `${String(index + 1).padStart(2, "0")}-${stem || "app"}`; +} diff --git a/demo/web/index.html b/demo/web/index.html index 793792ab62..e0d468febc 100644 --- a/demo/web/index.html +++ b/demo/web/index.html @@ -11,10 +11,10 @@

LOCAL CONVERSION

PS5 apps to Windows/Linux converter

-

Convert an app

Files stay on your device
-

Choose a folder or drop a ZIP

One executable ELF, sce_sys/ and sce_module/.
Other app files can stay in the archive.

ZIP & ZIP64 · Decrypted ELF files only

- - +

Convert apps

Files stay on your device
+

Choose a folder or drop ZIP files

Each app needs one executable ELF, sce_sys/ and sce_module/.
Other app files can stay in the archive.

ZIP & ZIP64 · Decrypted ELF files only · Multiple ZIPs become one bundle

+ +
Convert for

Choose a ZIP or folder to get started.

@@ -22,7 +22,7 @@
-

Memory downloads have a 256 MiB budget, checked against expanded files and output overhead. Larger results require desktop Chrome or Edge with direct disk saving. Executable and modules are limited to 256 MiB combined; the icon is also limited to 16 MiB and counts toward the Windows conversion limit. Use the native relinker for larger conversion inputs.

+

Memory downloads have a 256 MiB budget, checked against expanded files and output overhead. For multiple ZIPs, the limit applies to the combined bundle; larger batches require Save ZIP to disk in desktop Chrome or Edge. Executable and modules are limited to 256 MiB per app; the icon is also limited to 16 MiB and counts toward the Windows conversion limit. Use the native relinker for larger conversion inputs.

Conversion does not guarantee that an app runs. Check the community compatibility list for known blockers and reports.

diff --git a/demo/web/tests/archive.test.mjs b/demo/web/tests/archive.test.mjs index c176ff2d38..d0c2ca7c77 100644 --- a/demo/web/tests/archive.test.mjs +++ b/demo/web/tests/archive.test.mjs @@ -111,6 +111,29 @@ for (const target of ["windows", "linux"]) { } } +const bundleArchive = new Archive(source); +await bundleArchive.inspect(() => {}); +const bundleInput = await bundleArchive.conversionInput("linux", false, () => {}); +const bundleConversion = native.Convert(bundleInput, () => {}); +assert.equal(bundleConversion.ok, true, bundleConversion.error); +const bundleLibraries = await zip([["libs/system.prx", new Uint8Array([1, 2, 3])]]); +const bundleWriter = new ZipWriter(new BlobWriter("application/zip"), { zip64: true, level: 9, useWebWorkers: false, useCompressionStream: false }); +await packageArchive(bundleArchive, bundleConversion, bundleInput.outputName, bundleLibraries, null, () => {}, false, { prefix: "01-first/", writer: bundleWriter }); +await packageArchive(bundleArchive, bundleConversion, bundleInput.outputName, bundleLibraries, null, () => {}, false, { prefix: "02-second/", writer: bundleWriter }); +const bundle = await bundleWriter.close(); +const bundleReader = new ZipReader(new BlobReader(bundle)); +const bundleEntries = await bundleReader.getEntries(); +const bundleNames = new Set(bundleEntries.map(entry => entry.filename)); +assert(bundleNames.has(`01-first/${bundleInput.outputName}`)); +assert(bundleNames.has("01-first/libs/system.prx")); +assert(bundleNames.has("01-first/app0/content/resource.bin")); +assert(bundleNames.has(`02-second/${bundleInput.outputName}`)); +assert(bundleNames.has("02-second/libs/system.prx")); +assert(bundleNames.has("02-second/app0/content/resource.bin")); +await bundleReader.close(); +await bundleArchive.close(); +console.log("Multiple app packages append under separate roots in a single ZIP"); + for (const [entries, error] of [ [[["one", executable], ["two", executable]], /exactly one/u], [[["readme", new Uint8Array([1])]], /exactly one/u], @@ -210,6 +233,28 @@ try { await assert.rejects(worker.downloadLibraries("windows"), /Cannot verify the latest release \(403\)/u); latestResponse = new Response(JSON.stringify({ tag_name: "v2" })); await assert.rejects(worker.downloadLibraries("windows"), /A new release is being published/u); + + latestResponse = new Response(null, { status: 429 }); + await self.onmessage({ data: { type: "inspect", inputs: [ + { name: "First App.zip", file: source }, + { name: "First App.zip", file: source } + ] } }); + assert.equal(workerMessages.filter(message => message.type === "inspected").length, 2); + assert(workerMessages.some(message => message.type === "inspection-complete" && message.count === 2)); + await self.onmessage({ data: { type: "convert", target: "windows", toIntel: false, mode: "memory" } }); + const batchComplete = workerMessages.find(message => message.type === "complete"); + assert(batchComplete, workerMessages.find(message => message.type === "error")?.message); + assert.equal(batchComplete.apps, 2); + const batchReader = new ZipReader(new BlobReader(batchComplete.blob)); + const batchNames = new Set((await batchReader.getEntries()).map(entry => entry.filename)); + assert(batchNames.has("01-First-App/program.exe")); + assert(batchNames.has("01-First-App/libs/system.prx")); + assert(batchNames.has("01-First-App/app0/content/resource.bin")); + assert(batchNames.has("02-First-App/program.exe")); + assert(batchNames.has("02-First-App/libs/system.prx")); + assert(batchNames.has("02-First-App/app0/content/resource.bin")); + await batchReader.close(); + workerMessages.length = 0; } finally { globalThis.fetch = originalFetch; workerMessages.length = 0; @@ -218,6 +263,15 @@ await self.onmessage({ data: { type: "convert", target: "windows", mode: "disk" assert.equal(workerMessages.length, 1); assert.equal(workerMessages[0].type, "error"); assert.match(workerMessages[0].message, /browsers block DLL creation/u); +workerMessages.length = 0; +await self.onmessage({ data: { type: "inspect", inputs: [ + { name: "valid.zip", file: source }, + { name: "invalid.zip", file: new Blob(["not a ZIP"]) } +] } }); +assert.equal(workerMessages.at(-1).type, "error"); +assert.match(workerMessages.at(-1).message, /invalid\.zip/u); +assert.equal(workerMessages.some(message => message.type === "inspection-complete"), false); delete globalThis.self; console.log("Rate-limited release checks use the verified site manifest; other release-check failures remain errors"); console.log("Windows folder output rejected before conversion or file writes"); +console.log("Multiple ZIP inputs stream into separate app roots; invalid inputs name the failing archive"); diff --git a/demo/web/tests/batch.test.mjs b/demo/web/tests/batch.test.mjs new file mode 100644 index 0000000000..b19827d72b --- /dev/null +++ b/demo/web/tests/batch.test.mjs @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { batchMemoryEstimate, batchOutputDirectory, memoryEstimate, MEMORY_LIMIT } from "../batch.mjs"; + +test("estimates all ZIP outputs against one shared memory limit", () => { + const summaries = [ + { totalBytes: 1024, selectedBytes: 128, windowsSelectedBytes: 256, files: 2 }, + { totalBytes: 2048, selectedBytes: 256, windowsSelectedBytes: 512, files: 3 } + ]; + + assert.equal(batchMemoryEstimate(summaries, "linux", 1024), summaries.reduce( + (total, summary) => total + memoryEstimate(summary.totalBytes, summary.selectedBytes, 1024, summary.files), + 0 + )); + assert.equal(batchMemoryEstimate(summaries, "windows", 1024), summaries.reduce( + (total, summary) => total + memoryEstimate(summary.totalBytes, summary.windowsSelectedBytes, 1024, summary.files), + 0 + )); + assert.throws(() => batchMemoryEstimate([], "macos", 0), /Unsupported conversion target/u); + assert.throws(() => batchMemoryEstimate([{ totalBytes: Number.MAX_SAFE_INTEGER, selectedBytes: 0, windowsSelectedBytes: 0, files: 0 }], "linux", 0), /unsupported file size/u); + assert.equal(MEMORY_LIMIT, 256 * 1024 * 1024); +}); + +test("creates safe and distinct ZIP folder names for duplicate app filenames", () => { + assert.equal(batchOutputDirectory("First App.zip", 0), "01-First-App"); + assert.equal(batchOutputDirectory("First App.zip", 1), "02-First-App"); + assert.equal(batchOutputDirectory("CON.zip", 0), "01-CON"); + assert.equal(batchOutputDirectory("...zip", 0), "01-app"); + assert.throws(() => batchOutputDirectory("app.zip", -1), /Invalid app index/u); +}); diff --git a/demo/web/worker.mjs b/demo/web/worker.mjs index fcd816ffe8..51d63e03e0 100644 --- a/demo/web/worker.mjs +++ b/demo/web/worker.mjs @@ -1,6 +1,9 @@ -import { Archive, CONVERSION_LIMIT, MEMORY_LIMIT, memoryEstimate, packageArchive } from "./archive.mjs"; +import { batchMemoryEstimate, batchOutputDirectory, MEMORY_LIMIT } from "./batch.mjs"; +import { Archive, CONVERSION_LIMIT, createPackageWriter, packageArchive } from "./archive.mjs"; let archive; +let inputs = []; +let summaries = []; let busy = false; let pendingWrite; let lastReport = 0; @@ -15,6 +18,18 @@ function report(event) { self.postMessage({ type: "progress", ...event }); } +function reportFor(item, index) { + let started = false; + return event => { + if (!started) { + lastReport = 0; + lastPhase = ""; + started = true; + } + report({ ...event, appIndex: index, appCount: inputs.length, appName: item.name }); + }; +} + export async function downloadLibraries(target) { report({ phase: "Downloading system libraries" }); const response = await fetch("./release.json", { cache: "no-cache" }); @@ -59,6 +74,25 @@ export async function downloadLibraries(target) { return { file: new Blob([bytes]), release, expandedBytes: asset.expandedBytes }; } +async function convertInput(item, index, native, libraries, data, destination, directory, writer) { + const current = archive ?? new Archive(item.files ?? item.file); + archive = null; + try { + const itemReport = reportFor(item, index); + if (inputs.length > 1) await current.inspect(itemReport); + const input = await current.conversionInput(data.target, data.toIntel, itemReport); + const converted = native.Convert(input, itemReport); + if (!converted.ok) throw new Error(converted.error); + const options = inputs.length > 1 ? { prefix: `${batchOutputDirectory(item.name, index)}/`, writer } : {}; + await packageArchive(current, converted, input.outputName, libraries.file, destination, itemReport, directory, options); + return input.outputName; + } catch (error) { + throw new Error(`${item.name}: ${error instanceof Error ? error.message : String(error)}`); + } finally { + await current.close(); + } +} + self.onmessage = async ({ data }) => { if (data.type === "written") { if (!pendingWrite) return; @@ -72,27 +106,48 @@ self.onmessage = async ({ data }) => { try { if (data.type === "inspect") { if (archive) await archive.close(); - archive = new Archive(data.files ?? data.file); - const summary = await archive.inspect(report); - self.postMessage({ type: "inspected", summary }); + archive = null; + inputs = data.inputs ?? [{ name: data.file?.name ?? "selected app", file: data.file, files: data.files }]; + if (!inputs.length) throw new Error("Choose at least one ZIP archive."); + summaries = []; + for (let index = 0; index < inputs.length; index++) { + const item = inputs[index]; + const current = new Archive(item.files ?? item.file); + let keepOpen = false; + try { + const summary = await current.inspect(reportFor(item, index)); + summaries.push(summary); + if (inputs.length === 1) { + archive = current; + keepOpen = true; + } + self.postMessage({ type: "inspected", index, count: inputs.length, name: item.name, summary }); + } catch (error) { + throw new Error(`${item.name}: ${error instanceof Error ? error.message : String(error)}`); + } finally { + if (!keepOpen) await current.close(); + } + } + self.postMessage({ type: "inspection-complete", count: inputs.length }); } else if (data.type === "convert") { + if (data.target !== "linux" && data.target !== "windows") throw new Error(`Unsupported conversion target: ${data.target}`); + if (!["memory", "zip", "disk"].includes(data.mode)) throw new Error("Unknown save mode"); if (data.target === "windows" && data.mode === "disk") throw new Error("Windows folder output is unavailable because browsers block DLL creation. Choose a ZIP save mode."); - if (!archive) throw new Error("Select a ZIP or folder first"); - const selectedBytes = archive.selectedBytesForTarget(data.target); - if (selectedBytes > CONVERSION_LIMIT) throw new Error(`Selected ${data.target} conversion input exceeds the 256 MiB conversion limit. Use the native relinker for this app.`); + if (!inputs.length || summaries.length !== inputs.length) throw new Error("Inspect the selected apps before converting."); + if (inputs.length > 1 && data.mode === "disk") throw new Error("Batch conversion requires a ZIP save mode."); + for (const summary of summaries) { + const selectedBytes = data.target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes; + if (data.target === "windows" && summary.windowsIconTooLarge) throw new Error("A Windows icon exceeds 16 MiB. Choose Linux or use the native relinker."); + if (selectedBytes > CONVERSION_LIMIT) throw new Error(`Selected ${data.target} conversion input exceeds the 256 MiB conversion limit. Use the native relinker for this app.`); + } const libraries = await downloadLibraries(data.target); - const estimate = memoryEstimate(archive.totalBytes, selectedBytes, libraries.expandedBytes, archive.entries.length); - if (data.mode === "memory" && estimate > MEMORY_LIMIT) throw new Error("This result exceeds the 256 MiB memory-download budget. Use Save ZIP to disk in desktop Chrome or Edge."); + if (data.mode === "memory" && batchMemoryEstimate(summaries, data.target, libraries.expandedBytes) > MEMORY_LIMIT) throw new Error("This result exceeds the 256 MiB memory-download budget. Use Save ZIP to disk in desktop Chrome or Edge."); report({ phase: "Loading converter" }); const { default: createRelinker } = await import("./native/relinker_wasm.mjs"); const native = await createRelinker(); - let input = await archive.conversionInput(data.target, data.toIntel, report); - const outputName = input.outputName; - const converted = native.Convert(input, report); - input = null; - if (!converted.ok) throw new Error(converted.error); const chunks = []; let size = 0; + let outputName = "anyps5-apps"; const send = (event, transfer = []) => new Promise(resolve => { pendingWrite = resolve; self.postMessage(event, transfer); @@ -112,8 +167,8 @@ self.onmessage = async ({ data }) => { }); } }; - await packageArchive(archive, converted, outputName, libraries.file, destination, report, true); - } else if (data.mode === "memory" || data.mode === "zip") { + outputName = await convertInput(inputs[0], 0, native, libraries, data, destination, true); + } else { const destination = new WritableStream({ async write(chunk) { size += chunk.byteLength; @@ -126,12 +181,17 @@ self.onmessage = async ({ data }) => { chunks.push(chunk.slice()); } }); - await packageArchive(archive, converted, outputName, libraries.file, destination, report); + const writer = inputs.length > 1 ? createPackageWriter(destination) : null; + for (let index = 0; index < inputs.length; index++) { + const name = await convertInput(inputs[index], index, native, libraries, data, destination, false, writer); + if (inputs.length === 1) outputName = name; + } + if (writer) await writer.close(); await destination.close(); - } else throw new Error("Unknown save mode"); - self.postMessage({ type: "complete", mode: data.mode, blob: data.mode === "memory" ? new Blob(chunks, { type: "application/zip" }) : null, outputName, size, release: libraries.release.tag }); - await archive.close(); - archive = null; + } + self.postMessage({ type: "complete", mode: data.mode, blob: data.mode === "memory" ? new Blob(chunks, { type: "application/zip" }) : null, outputName, size, release: libraries.release.tag, apps: inputs.length }); + inputs = []; + summaries = []; } else throw new Error("Unknown worker request"); } catch (error) { self.postMessage({ type: "error", message: error instanceof Error ? error.message : String(error) }); diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index a735fffdad..03372da27e 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -209,6 +209,24 @@ Set `APS5_TIME_SCALE=` to scale `sceKernelGetProcessTime`, `sceKernelGet `APS5_PAD_INPUT_FILE` command durations and reload polling use `sceKernelGetProcessTime`, so they follow the scale. At `0.02`, a 1-second injected press lasts about 50 seconds of real time, and a new command can take up to 5 seconds to be noticed. +## Driver profile lines + +With `APS5_PROFILE_DRAW` set (any value), the driver prints a profile line on stderr at most every 10 s. A line either counts the last 10 s (its counters are read and cleared when it is printed; the text says `(10 s)` or `over 10 s`) or counts from the start of the process (the text says `(cumulative)`). A rate is the difference of two cumulative lines divided by the time between them. + +| Line | Counts | +|---|---| +| `[rescache] revalidate` | last 10 s | +| `[texture-cache]` | last 10 s | +| `[pipecache]` | last 10 s | +| `[rescache]` (hits, misses, entries) | from the start | +| `[vk]` | from the start | +| `[indirect]` (gpu-side, cpu-side) | from the start | +| `[flip]` | from the start | +| `[fill-sync]` | from the start | +| `[buffers] descriptor elements bound` | from the start | + +`[gputime]` is described above and needs `APS5_PROFILE_GPU`. Other profile lines state their window in their text or are not listed here because their counters were not checked. + ## Driver thread priority Set `APS5_THREAD_PRIORITY=high` to raise the AGC driver's queue workers and the video output presenter above the title's threads. On Linux the threads get nice `APS5_THREAD_NICE` (default -10) through rtkit; on Windows they get `THREAD_PRIORITY_HIGHEST`. `APS5_THREAD_PRIORITY=rt` asks rtkit for real-time priority `APS5_THREAD_RT` (default 10, at most rtkit's `MaxRealtimePriority`) under rtkit's policy, `SCHED_RR` by default; on Windows it sets `THREAD_PRIORITY_TIME_CRITICAL`. rtkit only grants it when the process's hard `RLIMIT_RTTIME` is at most its `RTTimeUSecMax` (200000 us by default), so a hard limit above `APS5_THREAD_RTTIME_US` (default 200000, capped at `RTTimeUSecMax`) is lowered to it and the soft limit to at most half of it. The hard limit cannot be raised again and stays lowered for the whole process even if rtkit then refuses; see [technical debt](TechnicalDebt.md#functional). There is no fallback between the modes: when rtkit or `SetThreadPriority` refuses, a queue worker reports the refusal as a driver failure, which the next submission, wait or shutdown rethrows, and the presenter's start throws it. Each raised thread logs its priority once as `[thread-priority]`. The setting is `off` by default; a value other than `off`, `high` or `rt` is reported the same way as a refusal. diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 38c7df505e..6f94c272d1 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -76,6 +76,7 @@ - [Covered-centroid raster state](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) accepts PA_SC_AA_CONFIG.COVERED_CENTROID_IS_CENTER only when the pixel program is skipped or SPI_PS_INPUT_ENA and SPI_PS_INPUT_ADDR have no active perspective or linear centroid input. Bit 29 follows [PS5_Mesa GFX10.3 registers](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/registers/gfx103.json); [RADV](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/vulkan/radv_cmd_buffer.c) sets it on GFX10.3 and later. Active centroid interpolation and unspecified input registers still throw. Console covered-pixel centroid results were not measured. - [Compute LDS beyond the device's shared-memory limit](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) lives in a coherent storage buffer with one slice of `LDS bytes / 4 + 1` dwords, rounded up to an even count so that 64-bit atomics stay 8-byte aligned, per workgroup (the dword after the LDS is the atomic lock on a device without `shaderBufferInt64Atomics`), shared by all such dispatches and grown in 1 MiB steps up to `maxStorageBufferRange`; a dispatch whose slices do not fit throws. It is slower than shared memory and its cost was not measured. An indirect dispatch of such a program always reads its group counts on the CPU. Its results are pinned against the in-limit semantics in `agc_driver_lds_beyond_device_limit`, not measured on hardware. - [AGC buffer allocations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) retry host/device out-of-memory once after reclaiming idle pool buffers and fully unused slab blocks. Live buffers and blocks with live slots are retained. Live images and other retained resources have no global memory bound; allocation failure still throws when reclaim cannot satisfy the request. +- [Host placement after a device-memory refusal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) (libSceAgcDriver): when the driver refuses a device-local allocation with `VK_ERROR_OUT_OF_DEVICE_MEMORY` after the pool's trim and the texture-cache release, device buffers and images (textures, storage textures, block, repacked, depth, depth plane and stencil copies, render, depth and multisampled targets and attachment proxies) are allocated in a host memory type their requirements allow. A host-placed device buffer leaves the pool on release. A host-placed image keeps that placement until it is destroyed and is never moved back to device memory, so every access to it crosses PCIe for its lifetime. The placements are counted in a rate-limited `[bufferpool]` line with the cumulative MiB, not the resident bytes. The 1x1 denormal clear probe still allocates device memory directly and throws on a refusal. - [Pipeline journal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp) (libSceAgcDriver): the journal is append-only and never compacted, so a module a later run no longer builds pipelines from stays in the file until it is deleted, and the whole file is read into memory while the replay runs (its modules reference that buffer until the last replayed pipeline was built). Only the first `APS5_PIPELINE_JOURNAL_ENTRIES` pipelines recorded are replayed (a record is never rewritten, so a pipeline past the bound stays past it), and the pipeline store evicts the least recently used entry past `APS5_PIPELINE_STORE_ENTRIES` (1024 each): a title with more configurations rebuilds the rest at the draw as before. A compute pipeline is replayed over the set layout `DescribeSetLayouts` derives from the result's bindings, which a dispatch that finds it resident checks against its `ShaderResources` layout only under `APS5_VERIFY_SET_LAYOUTS=1`. Recording and replay are best effort: a record whose write fails is lost, an entry whose build fails is skipped, and both are counted in the `[pipeline-journal]` line rather than stopping the run. A replayed pipeline that the title never asks for again costs its build time and its driver memory (its optimized relink is deferred to its first draw, so a run whose first frames bind many replayed pipelines queues their relinks then); nothing ages recipes out of the file. Two runs sharing the file take turns under its lock file, but each only knows the modules it loaded or wrote itself, so a module the other run appended meanwhile may be appended again (the loader keeps the first copy). Records a run appends after another run's load are replayed by that run only after its next start. The file header's format number is the only guard against a change of the recipe codec, `CanonicalPipelineState` or the store key that keeps the structures' sizes (the recompiler's source version does not cover the driver's sources). - [Graphics pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp) (libSceAgcDriver): a pipeline is fast-linked at the draw and relinked with `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` on a background thread, and binds the optimized one from the next draw on. Draws recorded before that use the fast link, whose vertex stage keeps the outputs the fragment stage does not read: on RADV (Radeon 660M, Mesa 26.2.4) 346 of 483 Astro's Playroom vertex shaders paired with a fragment stage that reads none of them used more VGPRs (median 8 more, up to 2.5 times), and on an RTX 4090 13 of 15 PPSA36082 vertex shaders up to 68 instead of 38 (#2242). The optimized link compiles to the same code as a whole pipeline - [Prebuilt pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver): each registered vertex or pixel shader in registration form gets its pre-rasterization or fragment-shader library built on one background thread, with a fixed state the draw may not use: triangle list, counter-clockwise front face, no depth clamp, depth bias or conservative rasterization, first provoking vertex, one sample. Libraries are kept until the device is torn down; on Astro's Playroom 1948 of them added 0.44 GB of peak memory on RADV. diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 82e511ac9a..b1131fcb99 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -114,7 +114,7 @@ - [EVFILT_VNODE, sceKernelAddFileEvent and sceKernelDeleteFileEvent](../../../core/libs/prx/libkernel/Equeue/src/FileEvent.cpp) (libkernel) watch the host file or directory behind the descriptor and turn host changes into FreeBSD 11 vnode hints, which [`filt_vfsvnode`](https://github.com/freebsd/freebsd-src/blob/34bd34750187bae02a0086402c78c031c59dd8b3/sys/kern/vfs_subr.c#L5032) adds whole when they share a bit with the requested `fflags`: a write that grows a file is `NOTE_WRITE | NOTE_EXTEND`, another write `NOTE_WRITE`, a shrinking truncate or an attribute change `NOTE_ATTRIB`, a higher link count `NOTE_LINK`, a lower one or a pending delete `NOTE_DELETE`, a new path `NOTE_RENAME`; in a directory, an entry created, removed or renamed is `NOTE_WRITE`, with `NOTE_LINK` for a subdirectory, and on Linux an entry moved from or to another directory adds `NOTE_EXTEND`; inotify queues the two halves of a rename one after the other, so a watcher woken by the first can read it alone, and a half read without the other is classified only after a `getdents64` on the watched directory, which takes the directory lock that `rename` holds until both halves are queued ([`do_renameat2`](https://github.com/torvalds/linux/blob/v6.8/fs/namei.c#L4975) around `vfs_rename` and its `fsnotify_move`, [`iterate_dir`](https://github.com/torvalds/linux/blob/v6.8/fs/readdir.c#L103)), an ordering taken from the kernel source, not from inotify(7), which promises none; a directory registered with `NOTE_EXTEND` on Windows is accepted, as `vfs_kqfilter` does not check it, but never receives it, because the directory handle cannot tell a move between directories from a removal plus an addition. FreeBSD's hints depend on the filesystem and the console's filesystems are not known: a growing `ftruncate` reports `NOTE_WRITE | NOTE_EXTEND` where `vop_setattr_post` reports `NOTE_ATTRIB`, and on Windows `utimes` on a file reports `NOTE_WRITE`. The three Windows notifications are told apart in the error, which names the `parent`, `entries` or `subdirectories` watch, the path it was given and `GetLastError`, but a `FindFirstChangeNotification` that fails still throws rather than returning an error to the guest: the EVFILT_VNODE attach path is [`vfs_kqfilter`](https://github.com/freebsd/freebsd-src/blob/34bd34750187bae02a0086402c78c031c59dd8b3/sys/kern/vfs_subr.c#L4936), which returns `EINVAL`, `ENOMEM` or 0 after `fget` has turned a descriptor that is not open into `EBADF`, and consults no directory permission, so no FreeBSD errno stands for a host that refuses to notify, and what the console does on such a host is not measured. Only the `parent` watch can fail this way in practice: `entries` and `subdirectories` need the `FILE_LIST_DIRECTORY` that opening the watched directory already took. The parent handle produces no hint of its own, as `RunHostWatch` only calls `FindNextChangeNotification` on it, so a later change may leave the watch on its 50 ms poll instead of throwing; that is not done here. Windows uses `FindFirstChangeNotification` on the parent directory (and on a watched directory itself) and compares `GetFileInformationByHandleEx` snapshots at least every 50 ms, because a write through an open handle is reported only when the handle is flushed or closed, so file hints can arrive 50 ms late, and a file deleted while watched, which the POSIX delete of Windows 10 1903 moves into `$Extend\$Deleted` until its last handle closes, reports `NOTE_DELETE` without `NOTE_RENAME` and keeps its handle without a parent directory watch, as that directory cannot be watched; Linux uses inotify on the file and is not run. `NOTE_REVOKE`, `NOTE_OPEN`, `NOTE_CLOSE`, `NOTE_CLOSE_WRITE`, `NOTE_READ`, unknown bits and descriptors other than files, directories, pipes and sockets throw. `sceKernelAddFileEvent(eq, fd, watch, udata)` and `sceKernelDeleteFileEvent(eq, fd)` take the [OpenOrbis](https://github.com/OpenOrbis/OpenOrbis-PS4-Toolchain/blob/0a1aaf9d/include/orbis/libkernel.h#L35) and [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0/src/SharpProspero/Interop/Kernel/KernelEqueue.cs#L164) signatures; `watch` is taken as the `fflags` and the event is added with `EV_CLEAR`, neither confirmed on a title (feat(libkernel): implement kevent EV_DISABLE, EV_DISPATCH and EV_FORCEONESHOT) (fix(libkernel): follow kqueue_register for EV_FORCEONESHOT and modify flags) - [sem_timedwait](../../../core/libs/prx/libkernel/Pthread/Posix/Sem.cpp) (libkernel) consumes an available count before checking the timeout and waits indefinitely for a null timeout, following FreeBSD 11's `sem_timedwait`. Console behavior for null and invalid timeouts when a count is available is unverified. - [tcsendbreak, tcdrain, tcflush, tcflow, tcgetpgrp, tcsetpgrp](../../../core/libs/prx/libkernel/File/src/Terminal.cpp) (libkernel) follow FreeBSD 11.4's [`termios.c`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/gen/termios.c): `tcflush` answers `EINVAL` for a queue other than `TCIFLUSH`, `TCOFLUSH` or `TCIOFLUSH` and `tcflow` for an action outside `TCOOFF`..`TCION` before the descriptor is looked at, and every other call fails as its terminal ioctl does on a descriptor that is not a terminal, with the errors `tcgetattr` gives, so `tcsendbreak` never reaches its 0.4 s pause. FreeBSD's `pipe_ioctl` answers `TIOCGPGRP` and `TIOCSPGRP` on a pipe or FIFO with the descriptor's SIGIO owner, which is not modelled, so `tcgetpgrp` and `tcsetpgrp` throw on a host pipe or FIFO above descriptor 2; descriptors 0 to 2 are taken as non-terminals that are not pipes, whatever the host attaches to them, so a pipe the guest creates on a freed descriptor 0 to 2 gets `ENOTTY` there. The PS5 `libkernel` is assumed to use this FreeBSD code, and `tcdrain` is not a cancellation point here -- [Windows file timestamps](../../../core/libs/prx/libkernel/File/include/WindowsFileTime.hpp) use the host filesystem's timestamp range and resolution. `utimes`, `futimes`, and `lutimes` reject timestamps outside nonnegative signed 64-bit Windows file-time ticks with `EINVAL`, including exactly 1601-01-01 00:00:00 UTC, whose zero value means "leave unchanged" to `SetFileTime`. Guest directory descriptors still resolve through their recorded path. Filesystems with coarser timestamp resolution cannot retain every guest microsecond +- [Windows file timestamps](../../../core/libs/prx/libkernel/File/include/WindowsFileTime.hpp) use the host filesystem's timestamp range and resolution. `utimes`, `futimes`, and `lutimes` reject timestamps outside nonnegative signed 64-bit Windows file-time ticks with `EINVAL`, including exactly 1601-01-01 00:00:00 UTC, whose zero value means "leave unchanged" to `SetFileTime`. Directory timestamp updates resolve the current path of the opened handle. Filesystems with coarser timestamp resolution cannot retain every guest microsecond - [EVFILT_VNODE on macOS](../../../core/libs/prx/libkernel/Equeue/src/FileEvent.cpp) (libkernel) registers `EVFILT_VNODE` on a duplicate of the descriptor in a host kqueue, whose notes have FreeBSD's values, and drops the `NOTE_LINK` that XNU adds to `NOTE_DELETE` when the watched file or directory is removed, since FreeBSD's `vop_remove_post` and `vop_rmdir_post` report `NOTE_DELETE` alone. As on Windows, a directory registered with `NOTE_EXTEND` is accepted but never receives it: XNU reports a move between directories as `NOTE_WRITE` alone, with `NOTE_LINK` for a subdirectory - [Windows file identity](../../../core/libs/prx/libkernel/File/src/NativeStat.cpp) (libkernel) - `stat` and `fstat` report the volume serial number as `st_dev` and the 64-bit file index folded into the guest's 32-bit `st_ino` (never 0), so two different files can in principle share an `st_ino`; the fold is not taken from a reference. `st_nlink` is clamped to 65535, the width of the guest field. The call keeps the `_wstat64` values when `GetFileInformationByHandle` fails - [lstat of a Windows symbolic link](../../../core/libs/prx/libkernel/File/src/NativeStat.cpp) (libkernel) reports a link whose target is relative, as `symlink` creates them, with the link's own times, `st_dev` and `st_ino`, mode `S_IFLNK | 0777` and the length of the target `readlink` returns as `st_size`. FreeBSD gives a new link `0777` less the creating process's umask, which the host does not keep, so the permission bits are not the console's. A link with an absolute target is reported the same way, using its print name, or its substitute name when the print name is empty, without a `\??\` or `\\?\` prefix. A directory junction is reported the same way when the name left after that prefix is a drive path (`X:\...`): the print name when that name is a drive path, otherwise the substitute name. A mounted volume, whose substitute name is `\??\Volume{GUID}\`, and any other reparse point throw, as for `readlink` @@ -122,7 +122,7 @@ - On Linux, [direct memory](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) lives in one sparse memfd at file offset = physical address, so the [GPU imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) can share one udmabuf buffer object per 64 MiB of it. Released direct memory is not punched out of the file, because a live import of its chunk pins the pages and the GPU would keep reading the old ones: its pages stay committed (the process keeps the high-water mark of the direct memory it used) and are zeroed when the range is allocated again. - [system](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) runs no command. It follows FreeBSD 11's `lib/libc/stdlib/system.c` for a sandbox without `/bin/sh`: a null command returns 1 unconditionally, as that version does, and any command returns the wait status of a child that could not exec the shell and called `_exit(127)` (`0x7f00`). Whether the console's application sandbox has no shell, and whether `vfork` succeeds there (FreeBSD returns -1 when it fails), is unverified - [sceKernelWriteThrottlingStatus](../../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed; returns `EFAULT` (`0x8002000e`) when passed a null status buffer pointer. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown -- [Windows directory descriptors](../../../core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp) (libkernel) take one snapshot of the directory, on the first `getdents`, `getdirentries` or non-zero `lseek` (`SEEK_SET` or `SEEK_CUR`), and keep it until it is discarded; `lseek` to offset 0 only discards the snapshot, and the next read lists the directory again, so every rewind costs O(entries) on the next read. `lseek` takes the offset as an entry index and `SEEK_CUR` with offset 0 only reports the index. `SEEK_END` throws, where the NUL descriptor used to return 0, because FreeBSD defines it relative to the directory size, which is not modelled. A negative result returns `EINVAL` and a `SEEK_CUR` offset that overflows returns `EOVERFLOW`, as FreeBSD `vn_seek` reports them. Names whose UTF-8 encoding exceeds the one-byte `d_namlen` limit are not representable in this guest directory-record layout: `getdents` returns `ENAMETOOLONG` without consuming the entry. Repeated calls therefore report the same error and entries after that name cannot be reached through this descriptor; FreeBSD's `getdents` behavior for host filenames that exceed its `MAXNAMLEN` is not verified. `getdirentries` still reports 0 through `basep` instead of the position of the block it read, so only 0 and values returned by `lseek` are valid offsets; FreeBSD offsets are filesystem cookies +- [Windows directory descriptors](../../../core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp) (libkernel) take one snapshot of the directory, on the first `getdents`, `getdirentries` or non-zero `lseek` (`SEEK_SET` or `SEEK_CUR`), and keep it until it is discarded; `lseek` to offset 0 only discards the snapshot, and the next read lists the directory again, so every rewind costs O(entries) on the next read. `lseek` takes the offset as an entry index and `SEEK_CUR` with offset 0 only reports the index. `SEEK_END` throws because FreeBSD defines it relative to the directory size, which is not modelled. A negative result returns `EINVAL` and a `SEEK_CUR` offset that overflows returns `EOVERFLOW`, as FreeBSD `vn_seek` reports them. Names whose UTF-8 encoding exceeds the one-byte `d_namlen` limit are not representable in this guest directory-record layout: `getdents` returns `ENAMETOOLONG` without consuming the entry. Repeated calls therefore report the same error and entries after that name cannot be reached through this descriptor; FreeBSD's `getdents` behavior for host filenames that exceed its `MAXNAMLEN` is not verified. `getdirentries` still reports 0 through `basep` instead of the position of the block it read, so only 0 and values returned by `lseek` are valid offsets; FreeBSD offsets are filesystem cookies. Descriptors retain the opened directory's native handle across renames. Enumeration reloads and `*at` operations resolve its current path, so a concurrent rename between resolution and the operation is not atomic. Enumeration of a removed directory and network-share path queries have not been verified - [/dev/random and /dev/urandom](../../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) (libkernel) opened for reading return a descriptor backed by the host null device whose reads are filled with host OS entropy (`getentropy`, `rand_s` on Windows); both paths behave the same, never block, and positioned reads ignore the offset (FreeBSD `random(4)`). Only the exact paths are recognised, `fstat`, `lseek` and `mmap` see the null device, and opening them for writing throws - [pthread_attr_getinheritsched, pthread_attr_setinheritsched, pthread_attr_getscope, pthread_attr_setscope, pthread_attr_getstackaddr](../../../core/libs/prx/libkernel/Pthread/Posix/Attr.cpp) (libkernel) follow FreeBSD 11 libthr ([thr_attr.c](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/lib/libthr/thread/thr_attr.c), [thr_create.c](https://github.com/freebsd/freebsd-src/blob/5ed7eb0d97ba4436218e810f61bd059acba984c2/lib/libthr/thread/thr_create.c)), not measured on the console: the scope defaults to `PTHREAD_SCOPE_SYSTEM` (2), only it and `PTHREAD_SCOPE_PROCESS` (0) are accepted and other values return `EINVAL`, and an inherit value other than `PTHREAD_INHERIT_SCHED` (4) or `PTHREAD_EXPLICIT_SCHED` (0) returns `ENOTSUP`. `pthread_attr_getstackaddr` returns the configured stack address pointer (null by default). A null output of `pthread_attr_getinheritsched` throws, since libthr writes through it unchecked. A thread keeps the scope it was created with, taken from the creating thread under `PTHREAD_INHERIT_SCHED`, and `pthread_attr_get_np` and `scePthreadAttrGet` report it, but every guest thread is a host thread at system scope, so process scope changes nothing - [Shared memory](../../../core/libs/prx/libkernel/File/src/SharedMemory.cpp) (libkernel) on Linux implements anonymous and named objects, bounded descriptor writes and seeks, and `shm_unlink` with effective read/write permission checks from FreeBSD 11. Named paths up to 1023 bytes use `/anyps5-shm-` plus SHA-1; collisions are not detected, and the namespace is shared across processes in the host mount namespace. Named creation applies the guest umask without changing the host mask. Linux procfs and `/dev/shm` are required. Operations use an advisory directory lock; external host operations bypass it, unrelated objects serialize, and descriptor operations require two spare host descriptors and compare device/inode with the initial descriptor identity on both the pinned-object and ordinary-file fallback paths. [Guest mappings](../../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp) support shared and private mappings of these descriptors through the allocation tracker and pin a duplicate descriptor during validation and mapping; mapping requires one spare host descriptor; file-backed mappings use the write-watch scan fallback because aliases and descriptor writes bypass per-mapping tracking. Named objects and file-backed mappings on Windows remain unsupported. Invalid non-null name pointers, console behaviour and title compatibility remain unverified. diff --git a/docs/user/INPUT_MAPPING.md b/docs/user/INPUT_MAPPING.md index ea37776a37..a6cd6ec1bc 100644 --- a/docs/user/INPUT_MAPPING.md +++ b/docs/user/INPUT_MAPPING.md @@ -30,6 +30,7 @@ Without an `anyps5-input.ini`, or for actions the file does not mention, these b | `Square` | Left mouse button | | `L1` | Q | | `R1` | E, Left Alt, Right Alt | +| `L2` | R, Mouse button 4 (X1) | | `R2` | Right mouse button | | `L3` | Left Shift, Right Shift | | `R3` | Left Ctrl, Right Ctrl | @@ -38,11 +39,9 @@ Without an `anyps5-input.ini`, or for actions the file does not mention, these b | `LeftStickLeft`, `LeftStickRight`, `LeftStickUp`, `LeftStickDown` | A, D, W, S | | `RightStickLeft`, `RightStickRight`, `RightStickUp`, `RightStickDown` | F, H, T, G | | `TouchLeft`, `TouchRight` | Backspace, Tab | -| `ToggleMouse` | Middle mouse button | +| `ToggleMouse` | Middle mouse button, F7 | | `ToggleFullscreen` | F11 | -`L2` has no built-in keyboard or mouse binding. - For example, this changes Cross to F or Space, moves the left stick to IJKL, uses the mouse buttons for Square and R2, and keeps all other built-in bindings: ```ini @@ -67,7 +66,7 @@ The same file also accepts these settings, each at most once, in the form `Setti | `MouseCurve` | Number from 0.2 to 5, the response exponent. Above 1 gives finer control at low speed; below 1 reaches large tilts sooner. | `1` | | `MouseDeadzoneOffset` | Number from 0 to 0.9, the smallest tilt any movement produces as a fraction of full tilt. Set it just above the game's own deadzone: raise it if slow movement doesn't move the camera, lower it if small movements jump. | `0.2` | | `MouseMaxSpeed` | Number from 0.1 to 10, the highest tracked speed as a multiple of the full-tilt speed. Above 1, a fast flick keeps the stick at full tilt a little longer after the mouse stops. | `1` | -| `MouseSmoothingMs` | Number from 0 to 500, the smoothing time constant in milliseconds. 0 follows every mouse report directly. | `10` | +| `MouseSmoothingMs` | Number from 0 to 500, the smoothing time constant in milliseconds. A mouse that reports fewer than 300 times per second, such as a 125 or 250 Hz mouse, is smoothed over at least three of its report intervals, up to 30 ms; the report rate is measured from the movement itself. 0 follows every mouse report directly and turns this off too. | `10` | | `MouseDecayMs` | Number from 1 to 1000, the milliseconds without movement after which the stick is centered. | `30` | ```ini @@ -78,7 +77,7 @@ MouseSensitivity = 2.5 ## Mouse as the right stick -`ToggleMouse` (the middle mouse button by default) captures the mouse and drives the right stick from how fast the mouse moves. Capture uses raw relative input, so the pointer speed and acceleration set in the operating system don't apply. Press it again to release the mouse. Capture also ends, and every held key is let go, when the window loses focus. +`ToggleMouse` (the middle mouse button or F7 by default; a mouse without a middle button can use F7) captures the mouse and drives the right stick from how fast the mouse moves. Capture uses raw relative input, so the pointer speed and acceleration set in the operating system don't apply. Press it again to release the mouse. Capture also ends, and every held key is let go, when the window loses focus. The stick tilts in the direction the mouse moves, and further the faster it moves. Speed is measured in counts per second over the time between updates, so the same hand speed gives the same tilt at any frame rate. Any movement tilts the stick at least `MouseDeadzoneOffset`, so slow movement isn't lost inside the game's own deadzone. When the mouse stops, the stick eases back and is centered `MouseDecayMs` after the last movement. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 5c1719fe3a..3d7c301910 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -6,6 +6,10 @@ The online converter can save Linux output as an uncompressed folder in supported browsers. It does not overwrite an existing destination folder. If conversion fails or is cancelled, it aborts the active file and attempts to remove the newly created output folder. If the browser prevents cleanup, the error names the folder that may contain incomplete files. +### Browser batch conversion + +Select or drop multiple app ZIP archives to convert them into one ZIP bundle. The converter inspects each app separately and puts its executable, libraries and `app0/` files in a numbered folder, so archives with the same filename remain separate. Batch output uses ZIP mode; saving to an uncompressed folder remains available for one Linux app at a time. The 256 MiB memory-download limit applies to the whole bundle, so use **Save ZIP to disk** for larger batches. If a batch is cancelled or any app fails, the converter aborts the incomplete bundle; any cleanup failure is reported instead of offering it as a completed result. + ## Input and conversion Use a clean ELF executable. Place its bundled ELF modules in `sce_module/`, `sce_modules/`, or `prx/` beside the input executable, or under the parent directory selected with `--sce-module-path `. The option accepts absolute paths and paths relative to the current working directory; do not include the `sce_module` directory itself. Only files directly inside the module directories are processed. SELF/fSELF containers are skipped with a warning; unwrap them before relinking if they need to be included. `tools/unwrap_self.py ` does that for a decrypted container: it places the segments at the offsets of the SELF segment records and restores `p_offset % p_align == p_vaddr % p_align`, and it refuses a container whose load segments all look encrypted. The command-line relinker skips an ELF module as a backup copy, with a message, when its file name extends the name of another ELF module in the same directory and both have the same SONAME, such as a dumper's `libX.prx.esbak` or `libX.prxbak` beside an ELF `libX.prx`; a single backup beside a SELF container is still relinked. `prx/` can coexist with either `sce_module/` or `sce_modules/`. Both `sce_module/` and `sce_modules/` present is an error. All three absent is an error unless an explicit `--module-dir` is supplied.