chore: sync current main for PR 9802

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -337,6 +337,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderResources.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderAddressResources.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/GuestBufferMemory.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ResidentMirror.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/BdaResources.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/BufferAddress.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderValidation.cpp
@@ -2560,12 +2561,13 @@ if(BUILD_TESTING)
     add_test(NAME agc_heap_mirror_unwatch_no_space_cache COMMAND agc_driver_graphics_tests --heap-mirror-unwatch)
     add_test(NAME agc_sparse_heap_mirror_unwatch COMMAND agc_driver_graphics_tests --sparse-heap-mirror-unwatch)
     add_test(NAME agc_sparse_heap_mirror_unwatch_no_space_cache COMMAND agc_driver_graphics_tests --sparse-heap-mirror-unwatch)
+    add_test(NAME agc_sparse_heap_mirror_budget COMMAND agc_driver_graphics_tests --sparse-heap-mirror-budget)
     add_test(NAME agc_heap_mirror_run_unwatch COMMAND agc_driver_graphics_tests --heap-mirror-run-unwatch)
     add_test(NAME agc_heap_mirror_run_unwatch_no_space_cache COMMAND agc_driver_graphics_tests --heap-mirror-run-unwatch)
     set_tests_properties(agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch_no_space_cache PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_SPACE_CACHE=1")
-    set_tests_properties(agc_heap_mirror_unwatch agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch agc_heap_mirror_run_unwatch_no_space_cache PROPERTIES TIMEOUT 30)
+    set_tests_properties(agc_heap_mirror_unwatch agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch agc_heap_mirror_run_unwatch_no_space_cache agc_sparse_heap_mirror_budget PROPERTIES TIMEOUT 30)
     if(WIN32)
-        set_tests_properties(agc_heap_mirror_unwatch agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch agc_heap_mirror_run_unwatch_no_space_cache PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_heap_mirror_unwatch agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch agc_heap_mirror_run_unwatch_no_space_cache agc_sparse_heap_mirror_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     else()
         add_test(NAME agc_heap_mirror_unwatched_import COMMAND agc_driver_graphics_tests --heap-mirror-unwatched-import)
         set_tests_properties(agc_heap_mirror_unwatched_import PROPERTIES TIMEOUT 30)

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -16,6 +16,7 @@
 #include <functional>
 #include <memory>
 #include <optional>
+#include <span>
 #include <string>
 #include <utility>
 
@@ -35,6 +36,18 @@ struct RecordedDispatch;
 // The module, layout and pipeline a compute dispatch binds (VulkanDevice::createComputePipeline).
 struct ComputePipelineObjects;
 
+struct DeviceQueuePlan {
+    std::uint32_t mainCount = 1;
+    bool settle = false;
+    std::uint32_t settleFamily = 0;
+    std::uint32_t settleIndex = 0;
+    bool sparseBinding = false;
+    std::uint32_t sparseFamily = VK_QUEUE_FAMILY_IGNORED;
+    std::uint32_t sparseIndex = 0;
+};
+
+DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding);
+
 class VulkanDevice {
 public:
     explicit VulkanDevice(const PresentationWindow* window = nullptr);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -10,6 +10,7 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "Optimization/ResourceProgram.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
 #include <atomic>
 #include <array>
 #include <algorithm>
@@ -20,6 +21,7 @@
 #include <cstdlib>
 #include <cstdio>
 #include <cstring>
+#include <unordered_map>
 
 namespace AgcDriver::DriverDetail {
 
@@ -191,6 +193,17 @@ DrawFinisher::Pending heldWrites(const DrawTail& tail) {
     return held;
 }
 
+bool EmptyIndirectSkip() {
+    static const bool enabled = std::getenv("APS5_NO_EMPTY_INDIRECT_SKIP") == nullptr;
+    return enabled;
+}
+
+struct CpuIndirectProgramsStorage {};
+
+std::unordered_map<std::uint64_t, bool>& CpuIndirectPrograms() {
+    return HostThreadLocal<std::unordered_map<std::uint64_t, bool>, CpuIndirectProgramsStorage>();
+}
+
 }
 
 DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::string& rejected) {
@@ -386,6 +399,25 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     programs = decode->programs;
     for (const auto& patch : userPatches) programs[patch.program].userData[patch.index] = patch.value;
     if (const auto skipped = SkippedShader(); skipped != 0 && std::any_of(programs.begin(), programs.end(), [skipped](const DrawProgram& program) { return program.binary.codeAddress == skipped; })) return drawn();
+    const bool emptyIndirectSkip = EmptyIndirectSkip();
+    auto& cpuIndirectPrograms = CpuIndirectPrograms();
+    const auto recordDraws = [&](const Pm4::DrawArguments& arguments) { return arguments.count != 0 && arguments.instances != 0 && (!drawParameters.indexed || arguments.firstVertexOrIndex < drawParameters.indexCount); };
+    if (emptyIndirectSkip && drawParameters.indirect && !programs.empty()) {
+        const auto known = cpuIndirectPrograms.find(programs.front().binary.codeAddress);
+        if (known != cpuIndirectPrograms.end() && known->second) {
+            const auto& indirect = *drawParameters.indirect;
+            recordQueuedLabelsBeforeRead(submission.queue);
+            const auto readStart = std::chrono::steady_clock::now();
+            const auto count = std::min(indirect.countIndirect ? Pm4::ReadDrawCount(indirect) : indirect.count, indirect.count);
+            bool any = false;
+            for (std::uint32_t record = 0; record < count && !any; ++record) any = recordDraws(Pm4::ReadDrawArguments(indirect, record));
+            if (!any) {
+                Graphics::CountEmptyIndirectDraw(std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - readStart).count());
+                timing.Mark("empty_indirect_skipped");
+                return drawn();
+            }
+        }
+    }
     const auto setMeshIndexBuffer = [&](const Pm4::DrawParameters& parameters) {
         if (!graphics.stages.mesh) return;
         auto& words = programs.front().userData;
@@ -756,6 +788,7 @@ DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) {
         const auto& result = *programResults[i];
         if (i == 0 && drawParameters.indirect) {
             indirectCpu = ClassifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect);
+            if (EmptyIndirectSkip()) CpuIndirectPrograms()[programs.front().binary.codeAddress] = indirectCpu.has_value();
         } else if (i == 0) {
             fold(result, drawParameters);
         }

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -81,6 +81,44 @@
 #include <utility>
 
 namespace AgcDriver {
+
+DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding) {
+    DeviceQueuePlan plan;
+    const auto count = static_cast<std::uint32_t>(families.size());
+    if (settles) {
+        if (family < count && families[family].queueCount >= 2) {
+            plan.mainCount = 2;
+            plan.settle = true;
+            plan.settleFamily = family;
+            plan.settleIndex = 1;
+        } else {
+            const auto copies = [&](std::uint32_t index) { return index != family && families[index].queueCount != 0 && (families[index].queueFlags & (VK_QUEUE_TRANSFER_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) != 0; };
+            for (std::uint32_t i = 0; i < count && !plan.settle; ++i) {
+                if (copies(i) && (families[i].queueFlags & (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) == 0) plan.settle = true, plan.settleFamily = i;
+            }
+            for (std::uint32_t i = 0; i < count && !plan.settle; ++i) {
+                if (copies(i)) plan.settle = true, plan.settleFamily = i;
+            }
+        }
+    }
+    if (!sparseBinding) return plan;
+    const auto taken = [&](std::uint32_t index) { return plan.settle && index == plan.settleFamily ? plan.settleIndex + 1 : 0u; };
+    const auto binds = [&](std::uint32_t index) { return index != family && families[index].queueCount > taken(index) && (families[index].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0; };
+    for (std::uint32_t i = 0; i < count && plan.sparseFamily == VK_QUEUE_FAMILY_IGNORED; ++i) {
+        if (binds(i) && families[i].queueFlags == VK_QUEUE_SPARSE_BINDING_BIT) plan.sparseFamily = i;
+    }
+    for (std::uint32_t i = 0; i < count && plan.sparseFamily == VK_QUEUE_FAMILY_IGNORED; ++i) {
+        if (binds(i)) plan.sparseFamily = i;
+    }
+    if (plan.sparseFamily != VK_QUEUE_FAMILY_IGNORED) {
+        plan.sparseIndex = taken(plan.sparseFamily);
+        plan.sparseBinding = true;
+    } else {
+        plan.sparseBinding = family < count && (families[family].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0;
+    }
+    return plan;
+}
+
 namespace {
 
 void check(VkResult result, const char* operation) {
@@ -378,8 +416,11 @@ struct VulkanDevice::State {
     bool drawInputsInPlace = true;
     bool dmaBufImport = false;
     bool sparseBinding = false;
+    bool sparseResidencyBuffer = false;
+    bool sparseResidencyAliased = false;
     std::uint32_t queueFamily = 0;
     std::uint32_t sparseFamily = VK_QUEUE_FAMILY_IGNORED;
+    std::uint32_t sparseIndex = 0;
     VkQueue sparseQueue = VK_NULL_HANDLE;
     std::mutex sparseQueueMutex;
     VkQueue settleQueue = VK_NULL_HANDLE;
@@ -1614,35 +1655,18 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     queueInfo.queueCount = 1;
     queueInfo.pQueuePriorities = &priority;
     const float priorities[2]{1.0f, 1.0f};
-    bool settleQueue = false;
-    std::uint32_t settleFamily = family;
-    std::uint32_t settleIndex = 0;
-    std::vector<VkQueueFamilyProperties> queues;
 #ifdef _WIN32
     const bool settles = true;
 #else
     const bool settles = state->dmaBufImport;
 #endif
-    if (settles) {
+    std::vector<VkQueueFamilyProperties> queues;
+    {
         std::uint32_t families = 0;
         auto getFamilies = state->InstanceFunction<PFN_vkGetPhysicalDeviceQueueFamilyProperties>("vkGetPhysicalDeviceQueueFamilyProperties");
         getFamilies(selected, &families, nullptr);
         queues.resize(families);
         getFamilies(selected, &families, queues.data());
-        if (family < families && queues[family].queueCount >= 2) {
-            queueInfo.queueCount = 2;
-            queueInfo.pQueuePriorities = priorities;
-            settleQueue = true;
-            settleIndex = 1;
-        } else {
-            const auto copies = [&](std::uint32_t index) { return index != family && queues[index].queueCount != 0 && (queues[index].queueFlags & (VK_QUEUE_TRANSFER_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) != 0; };
-            for (std::uint32_t i = 0; i < families && !settleQueue; ++i) {
-                if (copies(i) && (queues[i].queueFlags & (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) == 0) settleFamily = i, settleQueue = true;
-            }
-            for (std::uint32_t i = 0; i < families && !settleQueue; ++i) {
-                if (copies(i)) settleFamily = i, settleQueue = true;
-            }
-        }
     }
     VkDeviceCreateInfo deviceInfo{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};
     deviceInfo.queueCreateInfoCount = 1;
@@ -1677,50 +1701,38 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     // Guest shaders routinely read past descriptor ranges; robust access turns that into zeros
     // instead of a GPU fault that loses the device.
     enabled.robustBufferAccess = available.robustBufferAccess;
+    const auto plan = PlanDeviceQueues(queues, family, settles, available.sparseBinding == VK_TRUE);
+    const bool settleQueue = plan.settle;
+    const auto settleFamily = plan.settleFamily;
+    const auto settleIndex = plan.settleIndex;
     VkDeviceQueueCreateInfo queueInfos[3]{queueInfo, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}};
-    if (available.sparseBinding && state->dmaBufImport) {
-        std::uint32_t families = 0;
-        auto getFamilies = state->InstanceFunction<PFN_vkGetPhysicalDeviceQueueFamilyProperties>("vkGetPhysicalDeviceQueueFamilyProperties");
-        getFamilies(selected, &families, nullptr);
-        std::vector<VkQueueFamilyProperties> queues(families);
-        getFamilies(selected, &families, queues.data());
-        const auto binds = [&](std::uint32_t index) { return index != family && queues[index].queueCount != 0 && (queues[index].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0; };
-        std::uint32_t sparseFamily = families;
-        for (std::uint32_t i = 0; i < families && sparseFamily == families; ++i) {
-            if (binds(i) && queues[i].queueFlags == VK_QUEUE_SPARSE_BINDING_BIT) sparseFamily = i;
+    queueInfos[0].queueCount = plan.mainCount;
+    queueInfos[0].pQueuePriorities = plan.mainCount == 2 ? priorities : &priority;
+    deviceInfo.queueCreateInfoCount = 1;
+    const auto addQueue = [&](std::uint32_t queueFamily, std::uint32_t index) {
+        for (std::uint32_t info = 1; info < deviceInfo.queueCreateInfoCount; ++info) {
+            if (queueInfos[info].queueFamilyIndex != queueFamily) continue;
+            queueInfos[info].queueCount = std::max(queueInfos[info].queueCount, index + 1);
+            queueInfos[info].pQueuePriorities = queueInfos[info].queueCount == 2 ? priorities : &priority;
+            return;
         }
-        for (std::uint32_t i = 0; i < families && sparseFamily == families; ++i) {
-            if (binds(i)) sparseFamily = i;
-        }
-        if (sparseFamily != families) {
-            queueInfos[1].queueFamilyIndex = sparseFamily;
-            queueInfos[1].queueCount = 1;
-            queueInfos[1].pQueuePriorities = &priority;
-            deviceInfo.queueCreateInfoCount = 2;
-            state->sparseFamily = sparseFamily;
-            state->sparseBinding = true;
-        } else {
-            state->sparseBinding = family < families && (queues[family].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0;
-        }
-        enabled.sparseBinding = state->sparseBinding ? VK_TRUE : VK_FALSE;
-    }
+        auto& added = queueInfos[deviceInfo.queueCreateInfoCount++];
+        added = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
+        added.queueFamilyIndex = queueFamily;
+        added.queueCount = index + 1;
+        added.pQueuePriorities = added.queueCount == 2 ? priorities : &priority;
+    };
+    if (plan.settle && plan.settleFamily != family) addQueue(plan.settleFamily, plan.settleIndex);
+    if (plan.sparseFamily != VK_QUEUE_FAMILY_IGNORED) addQueue(plan.sparseFamily, plan.sparseIndex);
     deviceInfo.pQueueCreateInfos = queueInfos;
-    if (settleQueue && settleFamily != family) {
-        if (deviceInfo.queueCreateInfoCount == 2 && queueInfos[1].queueFamilyIndex == settleFamily) {
-            settleQueue = queues[settleFamily].queueCount >= 2;
-            if (settleQueue) {
-                queueInfos[1].queueCount = 2;
-                queueInfos[1].pQueuePriorities = priorities;
-                settleIndex = 1;
-            }
-        } else {
-            auto& settleInfo = queueInfos[deviceInfo.queueCreateInfoCount++];
-            settleInfo = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
-            settleInfo.queueFamilyIndex = settleFamily;
-            settleInfo.queueCount = 1;
-            settleInfo.pQueuePriorities = &priority;
-        }
-    }
+    state->sparseFamily = plan.sparseFamily;
+    state->sparseIndex = plan.sparseIndex;
+    state->sparseBinding = plan.sparseBinding;
+    enabled.sparseBinding = state->sparseBinding ? VK_TRUE : VK_FALSE;
+    enabled.sparseResidencyBuffer = state->sparseBinding ? available.sparseResidencyBuffer : VK_FALSE;
+    enabled.sparseResidencyAliased = state->sparseBinding ? available.sparseResidencyAliased : VK_FALSE;
+    state->sparseResidencyBuffer = enabled.sparseResidencyBuffer == VK_TRUE;
+    state->sparseResidencyAliased = enabled.sparseResidencyAliased == VK_TRUE;
     // Indirect draw records with a non-zero first instance, and several records per call.
     enabled.drawIndirectFirstInstance = available.drawIndirectFirstInstance;
     enabled.multiDrawIndirect = available.multiDrawIndirect;
@@ -2042,7 +2054,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         state->settleFamily = settleFamily;
         state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, settleFamily, settleIndex, &state->settleQueue);
     }
-    if (state->sparseFamily != VK_QUEUE_FAMILY_IGNORED) state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, state->sparseFamily, 0, &state->sparseQueue);
+    if (state->sparseFamily != VK_QUEUE_FAMILY_IGNORED) state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, state->sparseFamily, state->sparseIndex, &state->sparseQueue);
     if (checkpoints) state->checkpointMarker = state->DeviceFunction<PFN_vkCmdSetCheckpointNV>("vkCmdSetCheckpointNV");
     if (bufferMarkers) state->CreateMarkerBuffer();
     if (deviceFault || checkpoints || bufferMarkers) {
@@ -3849,6 +3861,8 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.individuallyFreeDescriptorSets = state->individuallyFreeDescriptorSets;
     context.dmaBufImport = state->dmaBufImport;
     context.sparseBinding = state->sparseBinding;
+    context.sparseResidencyBuffer = state->sparseResidencyBuffer;
+    context.sparseResidencyAliased = state->sparseResidencyAliased;
     context.queueFamily = state->queueFamily;
     context.sparseFamily = state->sparseFamily;
     context.sparseQueue = state->sparseQueue;

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -182,6 +182,8 @@ struct Context {
     bool drawInputsInPlace = true;
     bool dmaBufImport = false;
     bool sparseBinding = false;
+    bool sparseResidencyBuffer = false;
+    bool sparseResidencyAliased = false;
     std::uint32_t queueFamily = 0;
     std::uint32_t sparseFamily = VK_QUEUE_FAMILY_IGNORED;
     VkQueue sparseQueue = VK_NULL_HANDLE;

core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp

@@ -94,6 +94,8 @@ bool DrawRecipes();
 enum class IndirectDrawPath : std::uint8_t { Gpu, NotFolded, FetchUnknown, NonVertexPath, IndxOffset, DrawIndex, VertexRange, FeatureGap, PendingImage, PendingLabelOrCopy, NotImported, Disabled, IndexRange, Count };
 void CountIndirectDraw(IndirectDrawPath path, double readMs, bool rewritten = false);
 std::array<std::uint64_t, static_cast<std::size_t>(IndirectDrawPath::Count)> IndirectDrawCounts();
+void CountEmptyIndirectDraw(double readMs);
+std::uint64_t EmptyIndirectDrawCount();
 
 struct DrawPassCounts {
     std::uint64_t draws = 0;

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -248,6 +248,11 @@ struct MirrorStats {
 };
 MirrorStats MirrorCounters();
 std::uint64_t SnapshotMismatches();
+struct HeapMirrorHold {
+    std::uint64_t mirrors = 0;
+    std::uint64_t bytes = 0;
+};
+HeapMirrorHold HeapMirrorsWithin(std::uint64_t begin, std::uint64_t end);
 void ClearSnapshotCopies(VkDevice device);
 std::vector<std::uint64_t> AddressTableKey(std::span<const std::pair<std::uint64_t, std::uint64_t>> writes, std::span<const ShaderRecompiler::BdaAbi::Range> own);
 void ClearImageMirrors(VkDevice device);
@@ -492,6 +497,7 @@ private:
     std::vector<std::pair<std::uint64_t, std::uint64_t>> writes;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> readsOverBase;
     std::vector<std::pair<std::uint64_t, std::vector<std::byte>>> heapReferences;
+    std::vector<std::pair<std::uint64_t, std::shared_ptr<Buffer>>> heapReadbacks;
     // UploadPrepare ran (regions are frozen); `uploaded` once UploadFinish ran.
     bool prepared = false;
     bool uploaded = false;

core/libs/prx/libSceAgcDriver/Graphics/include/ResidentMirror.hpp

@@ -0,0 +1,77 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RESIDENTMIRROR_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RESIDENTMIRROR_HPP
+
+#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <span>
+#include <utility>
+#include <vector>
+
+namespace AgcDriver::Graphics {
+
+class Buffer;
+class ResidentMemory;
+
+using GuestRuns = std::vector<std::pair<std::uint64_t, std::uint64_t>>;
+
+GuestRuns GuestNonzeroBlocks(const GuestRuns& candidates);
+bool GuestBytesZero(std::uint64_t begin, std::uint64_t end);
+void SetResidentAddressSpaceLimit(std::uint64_t bytes);
+
+class ResidentBuffer : public std::enable_shared_from_this<ResidentBuffer> {
+public:
+    static constexpr std::uint64_t BlockBytes = 65536;
+
+    static std::shared_ptr<ResidentBuffer> Create(const Context& context, std::uint64_t base, std::uint64_t bytes, const GuestRuns& backed);
+    ~ResidentBuffer();
+    ResidentBuffer(const ResidentBuffer&) = delete;
+    ResidentBuffer& operator=(const ResidentBuffer&) = delete;
+
+    VkBuffer Handle() const { return buffer; }
+    VkDeviceAddress DeviceAddress() const { return address + lead; }
+    std::uint64_t Origin() const { return origin; }
+    std::uint64_t BoundBytes() const { return bound * BlockBytes; }
+    std::uint32_t MemoryType() const { return type; }
+    std::byte* Block(std::uint64_t guest) const;
+    std::uint64_t UnboundBytes(const GuestRuns& runs) const;
+    void Bind(const GuestRuns& runs);
+    void MarkChanged(std::uint64_t begin, std::uint64_t end);
+    void Read(std::uint64_t guest, std::span<std::byte> out) const;
+    void Store(std::uint64_t guest, std::span<const std::byte> bytes);
+    std::shared_ptr<Buffer> RecordReadback(VkCommandBuffer commands, std::uint64_t guest, std::uint64_t bytes) const;
+    void ReadDevice(std::uint64_t guest, std::span<std::byte> out);
+    static void FlushChanged(const Context& context);
+
+private:
+    struct Slot {
+        void* slab = nullptr;
+        std::uint32_t index = 0;
+        std::byte* host = nullptr;
+    };
+    ResidentBuffer(const Context& context, std::uint64_t base, std::uint64_t bytes, const GuestRuns& backed);
+    std::size_t blockIndex(std::uint64_t guest) const;
+    void submit(const std::vector<VkSparseMemoryBind>& binds);
+    void flush();
+    std::shared_ptr<Buffer> readbackBatch(std::uint64_t guest, std::uint64_t bytes);
+    void release() noexcept;
+
+    Context context;
+    std::shared_ptr<ResidentMemory> memory;
+    VkBuffer buffer = VK_NULL_HANDLE;
+    VkDeviceAddress address = 0;
+    std::uint32_t type = 0;
+    std::uint64_t origin = 0;
+    std::uint64_t lead = 0;
+    std::uint64_t size = 0;
+    std::uint64_t bound = 0;
+    std::vector<Slot> slots;
+    std::mutex changedMutex;
+    GuestRuns changed;
+};
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -468,6 +468,7 @@ struct DrawProfile {
     // with a constant, and the time of the CPU record reads (their syncs).
     std::array<std::uint64_t, IndirectPathCount> indirect{};
     std::uint64_t indirectRewritten = 0;
+    std::uint64_t indirectEmpty = 0;
     double indirectReadUs = 0;
     // Draw packets that drew nothing, by DrawSkip, and their time.
     std::array<std::uint64_t, static_cast<std::size_t>(DrawSkip::Count)> skips{};
@@ -565,12 +566,12 @@ void reportDraw(const std::array<double, PhaseCount>& us, const ShaderResources:
     if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast<std::size_t>(n), "; hook waits inside draws %.1f ms (max %.1f; the draws' own syncs, %.1f ms, not counted)", profile.hookWaitUs / 1000.0, profile.maxHookWaitUs / 1000.0, profile.ownSyncUs / 1000.0);
     std::uint64_t indirectCpu = 0;
     for (std::size_t i = 1; i < IndirectPathCount; ++i) indirectCpu += profile.indirect[i];
-    if ((profile.indirect[0] != 0 || indirectCpu != 0) && room()) {
+    if ((profile.indirect[0] != 0 || indirectCpu != 0 || profile.indirectEmpty != 0) && room()) {
         n += std::snprintf(line + n, sizeof(line) - static_cast<std::size_t>(n), "; indirect: gpu-side %llu (%llu rewritten), cpu-side %llu (", static_cast<unsigned long long>(profile.indirect[0]), static_cast<unsigned long long>(profile.indirectRewritten), static_cast<unsigned long long>(indirectCpu));
         for (std::size_t i = 1; i < IndirectPathCount && room(); ++i) {
             if (profile.indirect[i] != 0) n += std::snprintf(line + n, sizeof(line) - static_cast<std::size_t>(n), " %s %llu", IndirectDrawPathNames[i], static_cast<unsigned long long>(profile.indirect[i]));
         }
-        if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast<std::size_t>(n), "), argument reads %.1f ms", profile.indirectReadUs / 1000.0);
+        if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast<std::size_t>(n), "), empty skipped before their stages %llu, argument reads %.1f ms", static_cast<unsigned long long>(profile.indirectEmpty), profile.indirectReadUs / 1000.0);
     }
     if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast<std::size_t>(n), "; skipped packets:");
     for (std::size_t i = 0; i < profile.skips.size() && room(); ++i) {
@@ -901,6 +902,21 @@ std::array<std::uint64_t, IndirectPathCount> IndirectDrawCounts() {
     return stats.indirect;
 }
 
+void CountEmptyIndirectDraw(double readMs) {
+    static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+    if (!profile) return;
+    auto& stats = Profile();
+    std::lock_guard lock(stats.mutex);
+    ++stats.indirectEmpty;
+    stats.indirectReadUs += readMs * 1000.0;
+}
+
+std::uint64_t EmptyIndirectDrawCount() {
+    auto& stats = Profile();
+    std::lock_guard lock(stats.mutex);
+    return stats.indirectEmpty;
+}
+
 std::array<DrawPassCounts, 2> DrawPassTotals() {
     auto& stats = Profile();
     std::lock_guard lock(stats.mutex);

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -7,6 +7,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ResidentMirror.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
@@ -74,6 +75,9 @@ struct ImageMirror {
     std::vector<std::uint64_t> generations;
     std::weak_ptr<const GuestAllocations::Range> range;
     std::shared_ptr<Buffer> buffer;
+    std::shared_ptr<ResidentBuffer> resident;
+    std::uint64_t accounted = 0;
+    bool listed = false;
     // Writable ranges only: the guest bytes the mirror holds, compared on refresh and at write-back.
     std::vector<std::byte> shadow;
     // Identity for the life of this mirror (see ImageMirrorSerial); the top bit keeps mirror serials
@@ -1655,7 +1659,11 @@ const HostImport* importJoined(const Context& context, HostImports& state, std::
     auto run = joinedRun(lease, begin, end);
     if (run.ranges.empty()) return nullptr;
     if (const auto failed = state.failedJoins.find(run.base); failed != state.failedJoins.end() && sameRun(failed->second, run)) return nullptr;
-    return importRunAt(context, state, std::move(run), true, begin, lease, end, true, ImportRequester::Join);
+    const auto existing = state.imports.find(run.base);
+    const bool existed = existing != state.imports.end() && sameRun(existing->second, run);
+    const auto* entry = importRunAt(context, state, std::move(run), true, begin, lease, end, true, ImportRequester::Join);
+    if (entry != nullptr && !existed) ++state.epoch;
+    return entry;
 }
 
 const HostImport* importRunAt(const Context& context, HostImports& state, ImportRun run, bool joined, std::uint64_t address, const GuestAllocations::Lease& lease, std::uint64_t end, bool evict, ImportRequester requester) {
@@ -2122,25 +2130,104 @@ std::uint64_t heapMirrorBudget() {
     return bytes;
 }
 
-std::string heapMirrorError(const GuestAllocations::Range& range, std::uint64_t held, std::string_view reason) {
+std::string heapMirrorError(std::uint64_t address, std::uint64_t bytes, std::uint64_t held, std::string_view reason) {
     constexpr std::string_view prefix = "AGC graphics: ";
     if (reason.starts_with(prefix)) reason.remove_prefix(prefix.size());
     char where[64];
-    std::snprintf(where, sizeof(where), "heap mirror of 0x%llx+0x%llx: ", static_cast<unsigned long long>(range.address), static_cast<unsigned long long>(range.bytes));
+    std::snprintf(where, sizeof(where), "heap mirror of 0x%llx+0x%llx: ", static_cast<unsigned long long>(address), static_cast<unsigned long long>(bytes));
     char holding[96];
     std::snprintf(holding, sizeof(holding), "; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)", static_cast<unsigned long long>(held >> 20u), static_cast<unsigned long long>(heapMirrorBudget() >> 20u));
     return std::string(prefix) + where + std::string(reason) + holding;
 }
 
-std::uint64_t heapMirrorsBeside(const GuestAllocations::Range& range) {
-    auto& state = Mirrors();
-    std::lock_guard lock(state.mutex);
-    std::uint64_t held = state.heapBytes;
-    if (const auto found = state.entries.find(range.address); found != state.entries.end() && found->second->heap) held -= found->second->bytes;
-    return held;
+void sweepMirrors();
+
+void unlistMirror(ImageMirrors& state, ImageMirror& mirror) {
+    if (!mirror.listed) return;
+    state.heapBytes -= mirror.accounted;
+    mirror.listed = false;
 }
 
-void sweepMirrors();
+std::uint64_t blockBytes(const GuestRuns& runs) {
+    constexpr std::uint64_t block = ResidentBuffer::BlockBytes;
+    std::uint64_t blocks = 0;
+    std::uint64_t last = 0;
+    bool any = false;
+    for (const auto& [first, end] : runs) {
+        if (first >= end) continue;
+        auto index = first / block;
+        if (any && index == last) ++index;
+        if (index <= (end - 1) / block) blocks += (end - 1) / block - index + 1;
+        last = (end - 1) / block;
+        any = true;
+    }
+    return blocks * block;
+}
+
+void reserveHeapBytes(std::uint64_t address, std::uint64_t bytes, std::uint64_t need) {
+    auto& state = Mirrors();
+    for (int attempt = 0;; ++attempt) {
+        {
+            std::lock_guard lock(state.mutex);
+            std::uint64_t held = state.heapBytes;
+            if (const auto found = state.entries.find(address); found != state.entries.end() && found->second->heap && found->second->listed) held -= found->second->accounted;
+            if (held + need <= heapMirrorBudget()) {
+                state.heapBytes += need;
+                return;
+            }
+            if (attempt != 0) throw std::runtime_error(heapMirrorError(address, bytes, held, "past the budget"));
+        }
+        sweepMirrors();
+    }
+}
+
+void growResident(ImageMirror& mirror, const GuestRuns& runs) {
+    if (mirror.resident == nullptr) return;
+    const auto need = mirror.resident->UnboundBytes(runs);
+    if (need == 0) return;
+    auto& state = Mirrors();
+    bool counted = false;
+    for (int attempt = 0; !counted; ++attempt) {
+        {
+            std::lock_guard lock(state.mutex);
+            if (!mirror.listed) break;
+            if (state.heapBytes + need <= heapMirrorBudget()) {
+                state.heapBytes += need;
+                mirror.accounted += need;
+                counted = true;
+                break;
+            }
+            if (attempt != 0) throw std::runtime_error(heapMirrorError(mirror.base, mirror.reserved, state.heapBytes, "past the budget"));
+        }
+        sweepMirrors();
+    }
+    try {
+        mirror.resident->Bind(runs);
+    } catch (const std::runtime_error& error) {
+        std::uint64_t held = 0;
+        {
+            std::lock_guard lock(state.mutex);
+            if (counted && mirror.listed) {
+                state.heapBytes -= need;
+                mirror.accounted -= need;
+            }
+            held = state.heapBytes;
+        }
+        throw std::runtime_error(heapMirrorError(mirror.base, mirror.reserved, held, error.what()));
+    }
+}
+
+std::uint64_t mirrorStart(const ImageMirror& mirror) {
+    return mirror.resident != nullptr ? mirror.resident->Origin() : mirror.base;
+}
+
+VkBuffer mirrorHandle(const ImageMirror& mirror) {
+    return mirror.resident != nullptr ? mirror.resident->Handle() : mirror.buffer->Handle();
+}
+
+VkDeviceAddress mirrorAddress(const ImageMirror& mirror) {
+    return mirror.resident != nullptr ? mirror.resident->DeviceAddress() : mirror.buffer->DeviceAddress();
+}
 
 bool sameRange(const std::weak_ptr<const GuestAllocations::Range>& mirrored, const std::shared_ptr<const GuestAllocations::Range>& range) {
     return !mirrored.owner_before(range) && !range.owner_before(mirrored);
@@ -2162,6 +2249,16 @@ constexpr std::uint8_t SparseZero = 2;
 bool compareBlock(const RefreshBlock& block) {
     const auto offset = static_cast<std::size_t>(block.address - block.mirror->base);
     const auto* guest = reinterpret_cast<const std::byte*>(block.address);
+    if (block.mirror->resident != nullptr) {
+        auto* target = block.mirror->resident->Block(block.address);
+        if (block.fill == SparseZero) {
+            if (target != nullptr) std::memset(target, 0, block.length);
+            return true;
+        }
+        if (target != nullptr) std::memcpy(target, guest, block.length);
+        if (block.fill == 0) block.mirror->generations[static_cast<std::size_t>(block.address / 65536 - block.mirror->base / 65536)] = block.generation;
+        return target != nullptr;
+    }
     if (block.fill != 0) {
         auto* target = block.mirror->buffer->Bytes().data() + offset;
         if (block.fill == SparseZero) std::memset(target, 0, block.length);
@@ -2313,9 +2410,33 @@ void appendBlocks(std::vector<RefreshBlock>& blocks, ImageMirror& mirror, std::u
 }
 
 // Compares the gathered blocks (of one or many mirrors) and counts them.
+void bindHeapBlocks(std::span<const RefreshBlock> blocks) {
+    std::vector<std::pair<ImageMirror*, GuestRuns>> wanted;
+    for (const auto& block : blocks) {
+        if (block.mirror->resident == nullptr || block.fill == SparseZero) continue;
+        if (block.mirror->resident->Block(block.address) == nullptr && GuestBytesZero(block.address, block.address + block.length)) continue;
+        if (wanted.empty() || wanted.back().first != block.mirror) {
+            const auto found = std::find_if(wanted.begin(), wanted.end(), [&](const auto& entry) { return entry.first == block.mirror; });
+            if (found == wanted.end()) wanted.emplace_back(block.mirror, GuestRuns{});
+            else std::rotate(found, std::next(found), wanted.end());
+        }
+        auto& runs = wanted.back().second;
+        if (!runs.empty() && runs.back().second == block.address) runs.back().second = block.address + block.length;
+        else runs.emplace_back(block.address, block.address + block.length);
+    }
+    for (auto& [mirror, runs] : wanted) {
+        std::sort(runs.begin(), runs.end());
+        growResident(*mirror, runs);
+    }
+}
+
 void compareBlocks(std::span<const RefreshBlock> blocks) {
     if (blocks.empty()) return;
+    bindHeapBlocks(blocks);
     const auto copied = RefreshPool::Get().Run(blocks);
+    for (const auto& block : blocks) {
+        if (block.mirror->resident != nullptr && block.mirror->resident->Block(block.address) != nullptr) block.mirror->resident->MarkChanged(block.address, block.address + block.length);
+    }
     auto& state = Mirrors();
     state.blocksCompared += blocks.size();
     state.blocksCopied += copied;
@@ -2542,6 +2663,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
     {
         std::lock_guard lock(state.mutex);
         if (state.device != context.device) {
+            for (auto& [base, entry] : state.entries) entry->listed = false;
             state.entries.clear();
             state.failed.clear();
             state.heapBytes = 0;
@@ -2574,6 +2696,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
     mirror->writable = range->writable;
     mirror->heap = heap;
     mirror->range = range;
+    GuestRuns backed;
     if (heap) {
         std::uint64_t generation = 0;
         const auto serial = GuestMemory::UnwatchSerial();
@@ -2591,7 +2714,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
             std::lock_guard lock(state.mutex);
             ++state.heapUnwatched;
             if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) {
-                state.heapBytes -= found->second->bytes;
+                unlistMirror(state, *found->second);
                 stale = std::move(found->second);
                 state.entries.erase(found);
                 ++state.version;
@@ -2599,15 +2722,64 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
             return nullptr;
         }
         mirror->watchedSerial.store(serial, std::memory_order_relaxed);
-        if (heapMirrorsBeside(*range) + mirror->bytes > heapMirrorBudget()) {
-            sweepMirrors();
-            if (const auto held = heapMirrorsBeside(*range); held + mirror->bytes > heapMirrorBudget()) throw std::runtime_error(heapMirrorError(*range, held, "past the budget"));
+        if (context.sparseBinding && context.sparseResidencyBuffer && context.sparseResidencyAliased) backed = GuestNonzeroBlocks(sparse ? clipPieces(pieces, mirror->base, mirror->base + mirror->bytes) : GuestRuns{{mirror->base, mirror->base + mirror->bytes}});
+        auto need = blockBytes(backed);
+        reserveHeapBytes(range->address, range->bytes, need);
+        try {
+            mirror->resident = ResidentBuffer::Create(context, mirror->base, mirror->bytes, backed);
+        } catch (const std::runtime_error& error) {
+            std::uint64_t held = 0;
+            {
+                std::lock_guard lock(state.mutex);
+                state.heapBytes -= need;
+                held = state.heapBytes;
+            }
+            throw std::runtime_error(heapMirrorError(range->address, range->bytes, held, error.what()));
         }
+        if (mirror->resident == nullptr) {
+            {
+                std::lock_guard lock(state.mutex);
+                state.heapBytes -= need;
+            }
+            need = mirror->bytes;
+            reserveHeapBytes(range->address, range->bytes, need);
+        }
+        mirror->accounted = need;
+    }
+    if (mirror->resident != nullptr) {
+        {
+            const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::MirrorRefresh);
+            for (const auto& [first, last] : backed) {
+                for (auto at = first; at < last;) {
+                    const auto next = std::min(last, (at / ResidentBuffer::BlockBytes + 1) * ResidentBuffer::BlockBytes);
+                    GuestMemory::Read(at, std::span<std::byte>(mirror->resident->Block(at), static_cast<std::size_t>(next - at)));
+                    at = next;
+                }
+                mirror->resident->MarkChanged(first / ResidentBuffer::BlockBytes * ResidentBuffer::BlockBytes, (last + ResidentBuffer::BlockBytes - 1) / ResidentBuffer::BlockBytes * ResidentBuffer::BlockBytes);
+            }
+        }
+        mirror->committed = std::move(pieces);
+        std::lock_guard lock(state.mutex);
+        ++state.rebuilds;
+        mirror->serial = (1ull << 63u) | ++state.serials;
+        if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) unlistMirror(state, *found->second);
+        mirror->listed = true;
+        state.entries[range->address] = mirror;
+        ++state.version;
+        return mirror;
     }
     try {
         mirror->buffer = std::make_shared<Buffer>(context, static_cast<std::size_t>(mirror->bytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
     } catch (const std::runtime_error& error) {
-        if (heap) throw std::runtime_error(heapMirrorError(*range, heapMirrorsBeside(*range), error.what()));
+        if (heap) {
+            std::uint64_t held = 0;
+            {
+                std::lock_guard lock(state.mutex);
+                state.heapBytes -= mirror->accounted;
+                held = state.heapBytes;
+            }
+            throw std::runtime_error(heapMirrorError(range->address, range->bytes, held, error.what()));
+        }
         std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast<unsigned long long>(range->address), static_cast<unsigned long long>(range->bytes), error.what());
         std::lock_guard lock(state.mutex);
         state.failed.insert(range->address);
@@ -2634,8 +2806,8 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
     ++state.rebuilds;
     mirror->serial = (1ull << 63u) | ++state.serials;
     // A replaced mirror lives on in the regions of recorded work that bound it.
-    if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) state.heapBytes -= found->second->bytes;
-    if (heap) state.heapBytes += mirror->bytes;
+    if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) unlistMirror(state, *found->second);
+    mirror->listed = heap;
     state.entries[range->address] = mirror;
     ++state.version;
     return mirror;
@@ -2671,7 +2843,7 @@ void sweepMirrors() {
             ++it;
             continue;
         }
-        if (it->second->heap) state.heapBytes -= it->second->bytes;
+        if (it->second->heap) unlistMirror(state, *it->second);
         it = state.entries.erase(it);
         ++state.version;
     }
@@ -2688,9 +2860,9 @@ void releaseImportedMirrors(const Context& context, const std::vector<std::uint6
     for (const auto base : imported) {
         const auto found = state.entries.find(base);
         if (found == state.entries.end() || !found->second->heap) continue;
-        state.heapBytes -= found->second->bytes;
+        state.heapImportedBytes += found->second->accounted;
+        unlistMirror(state, *found->second);
         ++state.heapImported;
-        state.heapImportedBytes += found->second->bytes;
         released.push_back(std::move(found->second));
         state.entries.erase(found);
     }
@@ -3013,6 +3185,18 @@ std::uint64_t SnapshotMismatches() {
     return Snapshots().mismatched.load(std::memory_order_relaxed);
 }
 
+HeapMirrorHold HeapMirrorsWithin(std::uint64_t begin, std::uint64_t end) {
+    auto& state = Mirrors();
+    std::lock_guard lock(state.mutex);
+    HeapMirrorHold hold;
+    for (auto it = state.entries.lower_bound(begin); it != state.entries.end() && it->first < end; ++it) {
+        if (!it->second->heap || it->second->range.expired()) continue;
+        ++hold.mirrors;
+        if (it->second->listed) hold.bytes += it->second->accounted;
+    }
+    return hold;
+}
+
 std::vector<std::uint64_t> AddressTableKey(std::span<const std::pair<std::uint64_t, std::uint64_t>> writes, std::span<const ShaderRecompiler::BdaAbi::Range> own) {
     std::vector<std::uint64_t> key;
     key.reserve(writes.size() * 2u + 1u + own.size() * 4u);
@@ -3041,6 +3225,7 @@ void ClearImageMirrors(VkDevice device) {
         std::lock_guard lock(state.mutex);
         if (state.device != device) return;
         entries.swap(state.entries);
+        for (auto& [base, entry] : entries) entry->listed = false;
         state.failed.clear();
         state.heapBytes = 0;
         state.device = VK_NULL_HANDLE;
@@ -3983,7 +4168,7 @@ bool GuestBufferMemory::bindSparseMirror(Region& region, const GuestAllocations:
         refreshHeapMirrors(whole, blocks);
     }
     compareBlocks(blocks);
-    if (mirror == nullptr || !mirror->heap || region.begin < mirror->base || (region.begin - mirror->base) % context.limits.minStorageBufferOffsetAlignment != 0) return false;
+    if (mirror == nullptr || !mirror->heap || region.begin < mirror->base || (region.begin - mirrorStart(*mirror)) % context.limits.minStorageBufferOffsetAlignment != 0) return false;
     region.mirror = std::move(mirror);
     if (!servesView(region, std::max(region.lastView, region.begin))) {
         region.mirror = nullptr;
@@ -4689,6 +4874,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) {
     if (heapsEnd > heapsBegin) StorageTexture::PublishShadowsOnly(heapsBegin, static_cast<std::size_t>(heapsEnd - heapsBegin), PublishScope::Whole, "address-based heaps");
     if (!gpuCopies.empty()) recordGpuCopies(gpuCopies, addressable);
     takeHeapReferences();
+    ResidentBuffer::FlushChanged(context);
     if (!profile) return;
     const auto loopUs = microsecondsSince(loopStart);
     readUs.fetch_add(refreshUs, std::memory_order_relaxed);
@@ -4923,16 +5109,36 @@ void GuestBufferMemory::RecordCopyBacks(Recorder& recorder) {
     VkCommandBuffer commands = VK_NULL_HANDLE;
     auto timing = Recorder::NoTiming;
     std::uint64_t copiedBytes = 0;
+    if (!writes.empty()) {
+        auto sorted = writes;
+        std::sort(sorted.begin(), sorted.end());
+        for (const auto& range : sorted) {
+            if (!merged.empty() && range.first < merged.back().second) merged.back().second = std::max(merged.back().second, range.second);
+            else merged.push_back(range);
+        }
+    }
+    for (const auto& [begin, end] : merged) {
+        const auto* found = owner(begin, end);
+        if (found == nullptr || found->mirror == nullptr || !found->mirror->heap || !found->mirror->writable || found->mirror->resident == nullptr) continue;
+        const auto& region = *found;
+        {
+            const auto from = begin;
+            const auto to = std::min(end, servedEnd(region));
+            if (from >= to || std::any_of(heapReadbacks.begin(), heapReadbacks.end(), [&](const auto& entry) { return entry.first == from; })) continue;
+            if (!recording) {
+                commands = recorder.Commands();
+                timing = recorder.BeginGpuTiming(StagingCopyBackKey);
+                RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
+                Recorder::CountBarriers(Recorder::CommandClass::StagingOut);
+                recording = true;
+            }
+            heapReadbacks.emplace_back(from, region.mirror->resident->RecordReadback(commands, from, to - from));
+            recorder.Keep(heapReadbacks.back().second);
+            copiedBytes += to - from;
+        }
+    }
     for (auto& region : regions) {
         if (!region.gpuCopy || region.copiedBack) continue;
-        if (merged.empty() && !writes.empty()) {
-            auto sorted = writes;
-            std::sort(sorted.begin(), sorted.end());
-            for (const auto& range : sorted) {
-                if (!merged.empty() && range.first < merged.back().second) merged.back().second = std::max(merged.back().second, range.second);
-                else merged.push_back(range);
-            }
-        }
         for (const auto& [begin, end] : merged) {
             const auto from = std::max(begin, region.begin);
             const auto to = std::min(end, region.end);
@@ -4989,7 +5195,7 @@ VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std:
         return std::string(text);
     };
     if (!(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr))) Require(false, describe("exceeds"));
-    const auto base = region.direct != nullptr ? region.direct->base - region.direct->lead : region.mirror != nullptr ? region.mirror->base : region.begin;
+    const auto base = region.direct != nullptr ? region.direct->base - region.direct->lead : region.mirror != nullptr ? mirrorStart(*region.mirror) : region.begin;
     const auto offset = address - base;
     Require(context.limits.minStorageBufferOffsetAlignment != 0, "no storage buffer offset alignment");
     Require(base % 4 == 0, "a guest buffer view in a GPU owner that does not start at a DWORD boundary is not implemented");
@@ -5003,7 +5209,7 @@ VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std:
     }
     if (address - adjustment + range > end) Require(false, describe("rounded up to its view range exceeds"));
     Require(range <= context.limits.maxStorageBufferRange, "guest buffer view exceeds descriptor range limit");
-    const auto handle = region.direct != nullptr ? region.direct->buffer : region.mirror != nullptr ? region.mirror->buffer->Handle() : region.buffer->Handle();
+    const auto handle = region.direct != nullptr ? region.direct->buffer : region.mirror != nullptr ? mirrorHandle(*region.mirror) : region.buffer->Handle();
     return {handle, offset - adjustment, range};
 }
 
@@ -5016,7 +5222,7 @@ std::uint64_t GuestBufferMemory::ViewBytes(std::uint64_t bytes, std::uint32_t ad
 
 ShaderRecompiler::BdaAbi::Range GuestBufferMemory::addressRange(const Region& region) {
     Require(region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr, "incomplete guest GPU upload");
-    const auto address = region.direct != nullptr ? region.direct->address + (region.begin - region.direct->base) : region.mirror != nullptr ? region.mirror->buffer->DeviceAddress() + (region.begin - region.mirror->base) : region.buffer->DeviceAddress();
+    const auto address = region.direct != nullptr ? region.direct->address + (region.begin - region.direct->base) : region.mirror != nullptr ? mirrorAddress(*region.mirror) + (region.begin - region.mirror->base) : region.buffer->DeviceAddress();
     Require(region.end - region.begin <= std::numeric_limits<std::uint64_t>::max() - address, "GPU address range overflow");
     const auto permissions = ShaderRecompiler::BdaAbi::Read | (region.direct != nullptr && region.writable ? ShaderRecompiler::BdaAbi::Write : 0u);
     return {region.begin, region.direct != nullptr ? inPlaceEnd(region) : servedEnd(region), address, permissions, 0};
@@ -5279,15 +5485,19 @@ std::vector<std::pair<std::uint64_t, std::uint64_t>> mergeWrites(std::vector<std
 
 void GuestBufferMemory::takeHeapReferences() {
     heapReferences.clear();
+    heapReadbacks.clear();
     for (const auto& [begin, end] : mergeWrites(writes)) {
         const auto* found = owner(begin, end);
         if (found == nullptr || found->mirror == nullptr || !found->mirror->heap || !found->mirror->writable) continue;
         Require(end <= found->end, "written range exceeds its heap mirror");
-        const auto& mirror = *found->mirror;
+        auto& mirror = *found->mirror;
         const auto served = std::min(end, servedEnd(*found));
         if (begin >= served) continue;
-        const auto bytes = mirror.buffer->Bytes().subspan(static_cast<std::size_t>(begin - mirror.base), static_cast<std::size_t>(served - begin));
-        heapReferences.emplace_back(begin, std::vector<std::byte>(bytes.begin(), bytes.end()));
+        growResident(mirror, {{begin, served}});
+        std::vector<std::byte> bytes(static_cast<std::size_t>(served - begin));
+        if (mirror.resident != nullptr) mirror.resident->Read(begin, bytes);
+        else std::memcpy(bytes.data(), mirror.buffer->Bytes().data() + (begin - mirror.base), bytes.size());
+        heapReferences.emplace_back(begin, std::move(bytes));
     }
 }
 
@@ -5317,14 +5527,28 @@ void GuestBufferMemory::WriteBack() {
             const auto served = std::min(end, servedEnd(region));
             if (mirror.heap && begin >= served) continue;
             const auto length = static_cast<std::size_t>((mirror.heap ? served : end) - begin);
-            const auto current = mirror.buffer->Bytes().subspan(offset, length);
             if (mirror.heap) {
                 const auto reference = std::find_if(heapReferences.begin(), heapReferences.end(), [&](const auto& entry) { return entry.first == begin; });
                 Require(reference != heapReferences.end() && reference->second.size() == length, "heap mirror write-back has no reference");
+                if (mirror.resident == nullptr) {
+                    const auto current = mirror.buffer->Bytes().subspan(offset, length);
+                    if (mirror.sparse) storeSparse(mirror, begin, current, reference->second);
+                    else storeChanged(begin, current, reference->second);
+                    continue;
+                }
+                std::vector<std::byte> current(length);
+                if (const auto readback = std::find_if(heapReadbacks.begin(), heapReadbacks.end(), [&](const auto& entry) { return entry.first == begin; }); readback != heapReadbacks.end()) {
+                    Require(readback->second->Bytes().size() == length, "heap mirror readback does not match its write-back");
+                    std::memcpy(current.data(), readback->second->Bytes().data(), length);
+                } else {
+                    mirror.resident->ReadDevice(begin, current);
+                }
+                mirror.resident->Store(begin, current);
                 if (mirror.sparse) storeSparse(mirror, begin, current, reference->second);
                 else storeChanged(begin, current, reference->second);
                 continue;
             }
+            const auto current = mirror.buffer->Bytes().subspan(offset, length);
             GuestMemory::WriteChanged(begin, current, std::span<const std::byte>(mirror.shadow).subspan(offset, length));
             std::memcpy(mirror.shadow.data() + offset, current.data(), length);
             continue;
@@ -5375,6 +5599,7 @@ void GuestBufferMemory::WriteBack() {
     lease.clear();
     space.reset();
     heapReferences.clear();
+    heapReadbacks.clear();
 }
 
 std::optional<std::vector<std::pair<std::uint64_t, std::uint64_t>>> GuestBufferMemory::DirectRegions() const {
@@ -5471,6 +5696,7 @@ std::vector<std::pair<std::uint64_t, std::uint64_t>> GuestBufferMemory::DeviceRe
 }
 
 void GuestBufferMemory::RecordStagingCopies(Recorder& recorder) {
+    ResidentBuffer::FlushChanged(context);
     if (!uploaded || committed) return;
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     std::vector<Region*> copies;

core/libs/prx/libSceAgcDriver/Graphics/src/ResidentMirror.cpp

@@ -0,0 +1,659 @@
+#include "prx/libSceAgcDriver/Graphics/include/ResidentMirror.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#ifndef _WIN32
+#include <fcntl.h>
+#include <sys/types.h>
+#include <unistd.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cerrno>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <mutex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <tuple>
+
+namespace AgcDriver::Graphics {
+
+namespace {
+
+constexpr std::uint64_t BlockSize = ResidentBuffer::BlockBytes;
+constexpr std::uint64_t SlabBytes = std::uint64_t{16} << 20u;
+constexpr std::uint32_t SlabBlocks = static_cast<std::uint32_t>(SlabBytes / BlockSize);
+constexpr std::uint64_t ZeroBytes = SlabBytes;
+constexpr VkAccessFlags TransferredAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT;
+
+[[noreturn]] void fail(const char* step, VkResult result) {
+    char text[160];
+    std::snprintf(text, sizeof(text), "%s failed (%d)", step, static_cast<int>(result));
+    throw std::runtime_error(text);
+}
+
+struct TransientBuffer {
+    TransientBuffer(const Context& context, VkBuffer buffer) : context(context), buffer(buffer) {}
+    ~TransientBuffer() { context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, buffer, nullptr); }
+    TransientBuffer(const TransientBuffer&) = delete;
+    TransientBuffer& operator=(const TransientBuffer&) = delete;
+    Context context;
+    VkBuffer buffer;
+};
+
+template<typename Work>
+void recordTransfer(const Context& context, Work&& work) {
+    GuestMemory::AssertGpuLockHeld("heap mirror transfer");
+    std::vector<std::shared_ptr<void>> kept;
+    if (auto* recorder = Recorder::Active(); recorder != nullptr) {
+        const auto commands = recorder->Commands();
+        RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
+        work(commands, kept);
+        RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, TransferredAccess);
+        for (auto& object : kept) recorder->Keep(std::move(object));
+        recorder->MarkCovered(TransferredAccess);
+        return;
+    }
+    CommandBatch batch(context);
+    RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
+    work(batch.Handle(), kept);
+    RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, TransferredAccess);
+    batch.SubmitAndWait();
+}
+
+}
+
+class ResidentMemory {
+public:
+    struct Slab {
+        VkDeviceMemory memory = VK_NULL_HANDLE;
+        std::unique_ptr<std::byte[]> host;
+        std::vector<std::uint32_t> free;
+    };
+
+    ResidentMemory(const Context& context, std::uint32_t type) : context(context), type(type) {
+        zero = allocate(ZeroBytes, "vkAllocateMemory of the heap mirrors' zero block");
+    }
+
+    ~ResidentMemory() {
+        const auto freeMemory = context.Function<PFN_vkFreeMemory>("vkFreeMemory");
+        for (auto& slab : slabs) freeMemory(context.device, slab->memory, nullptr);
+        freeMemory(context.device, zero, nullptr);
+    }
+
+    ResidentMemory(const ResidentMemory&) = delete;
+    ResidentMemory& operator=(const ResidentMemory&) = delete;
+
+    VkDeviceMemory Zero() const { return zero; }
+    bool Zeroed() const { return zeroed.load(std::memory_order_acquire); }
+
+    void FillZero(VkCommandBuffer commands, std::vector<std::shared_ptr<void>>& kept) {
+        if (Zeroed()) return;
+        VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
+        info.size = ZeroBytes;
+        info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
+        VkBuffer buffer = VK_NULL_HANDLE;
+        if (const auto result = context.Function<PFN_vkCreateBuffer>("vkCreateBuffer")(context.device, &info, nullptr, &buffer); result != VK_SUCCESS) fail("vkCreateBuffer of the heap mirrors' zero fill", result);
+        auto transient = std::make_shared<TransientBuffer>(context, buffer);
+        VkMemoryRequirements requirements{};
+        context.Function<PFN_vkGetBufferMemoryRequirements>("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements);
+        if ((requirements.memoryTypeBits & (1u << type)) == 0 || requirements.size > ZeroBytes) throw std::runtime_error("the heap mirrors' zero block memory cannot back a transfer buffer to fill it");
+        if (const auto result = context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory")(context.device, buffer, zero, 0); result != VK_SUCCESS) fail("vkBindBufferMemory of the heap mirrors' zero fill", result);
+        context.Function<PFN_vkCmdFillBuffer>("vkCmdFillBuffer")(commands, buffer, 0, VK_WHOLE_SIZE, 0);
+        kept.push_back(std::move(transient));
+        zeroed.store(true, std::memory_order_release);
+    }
+
+    std::tuple<Slab*, std::uint32_t> Take() {
+        std::lock_guard lock(mutex);
+        for (auto& slab : slabs) {
+            if (slab->free.empty()) continue;
+            const auto index = slab->free.back();
+            slab->free.pop_back();
+            return {slab.get(), index};
+        }
+        auto slab = std::make_unique<Slab>();
+        slab->memory = allocate(SlabBytes, "vkAllocateMemory of a heap mirror slab");
+        slab->host = std::unique_ptr<std::byte[]>(new std::byte[SlabBytes]);
+        for (std::uint32_t index = SlabBlocks; index-- > 1;) slab->free.push_back(index);
+        slabs.push_back(std::move(slab));
+        return {slabs.back().get(), 0u};
+    }
+
+    void Put(Slab* slab, std::uint32_t index) noexcept {
+        std::lock_guard lock(mutex);
+        slab->free.push_back(index);
+        if (slab->free.size() != SlabBlocks) return;
+        const auto found = std::find_if(slabs.begin(), slabs.end(), [&](const auto& held) { return held.get() == slab; });
+        context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, slab->memory, nullptr);
+        slabs.erase(found);
+    }
+
+private:
+    VkDeviceMemory allocate(std::uint64_t bytes, const char* operation) {
+        const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0};
+        VkMemoryAllocateInfo info{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, context.bufferDeviceAddress ? &flags : nullptr};
+        info.allocationSize = bytes;
+        info.memoryTypeIndex = type;
+        if (context.bufferPool != nullptr) return context.bufferPool->AllocateMemory(info, operation);
+        VkDeviceMemory memory = VK_NULL_HANDLE;
+        if (const auto result = context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &info, nullptr, &memory); result != VK_SUCCESS) fail(operation, result);
+        return memory;
+    }
+
+    Context context;
+    std::uint32_t type;
+    VkDeviceMemory zero = VK_NULL_HANDLE;
+    std::atomic<bool> zeroed{false};
+    std::mutex mutex;
+    std::vector<std::unique_ptr<Slab>> slabs;
+};
+
+namespace {
+
+struct ResidentDevices {
+    std::mutex mutex;
+    std::map<std::tuple<VkDevice, std::uint32_t>, std::weak_ptr<ResidentMemory>> memories;
+    std::map<VkDevice, std::uint64_t> addressSpace;
+    std::uint64_t limit = 0;
+    std::mutex pendingMutex;
+    std::vector<std::weak_ptr<ResidentBuffer>> pending;
+};
+
+ResidentDevices& Devices() {
+    static ResidentDevices devices;
+    return devices;
+}
+
+std::shared_ptr<ResidentMemory> residentMemory(const Context& context, std::uint32_t type) {
+    auto& devices = Devices();
+    std::lock_guard lock(devices.mutex);
+    auto& slot = devices.memories[{context.device, type}];
+    if (auto held = slot.lock()) return held;
+    auto made = std::make_shared<ResidentMemory>(context, type);
+    slot = made;
+    return made;
+}
+
+struct Candidate {
+    std::uint64_t begin = 0;
+    std::uint64_t end = 0;
+    bool readable = true;
+};
+
+bool allZero(std::uint64_t begin, std::uint64_t end) {
+    const auto* at = reinterpret_cast<const unsigned char*>(begin);
+    const auto* stop = reinterpret_cast<const unsigned char*>(end);
+    while (at < stop && reinterpret_cast<std::uintptr_t>(at) % sizeof(std::uint64_t) != 0) {
+        if (*at++ != 0) return false;
+    }
+    while (stop - at >= static_cast<std::ptrdiff_t>(sizeof(std::uint64_t))) {
+        std::uint64_t word = 0;
+        std::memcpy(&word, at, sizeof(word));
+        if (word != 0) return false;
+        at += sizeof(word);
+    }
+    while (at < stop) {
+        if (*at++ != 0) return false;
+    }
+    return true;
+}
+
+#ifndef _WIN32
+struct Mapping {
+    std::uint64_t begin = 0;
+    std::uint64_t end = 0;
+    bool readable = false;
+    bool privateAnonymous = false;
+};
+
+std::vector<Mapping> mappings(std::uint64_t begin, std::uint64_t end) {
+    std::ifstream maps("/proc/self/maps");
+    Require(static_cast<bool>(maps), "cannot read /proc/self/maps for a heap mirror's pages");
+    std::vector<Mapping> found;
+    std::string line;
+    while (std::getline(maps, line)) {
+        std::istringstream fields(line);
+        std::string span, permissions, offset, device;
+        unsigned long long inode = 0;
+        fields >> span >> permissions >> offset >> device >> inode;
+        const auto dash = span.find('-');
+        Require(dash != std::string::npos && permissions.size() == 4, "unexpected /proc/self/maps line: " + line);
+        const auto first = std::stoull(span.substr(0, dash), nullptr, 16);
+        const auto last = std::stoull(span.substr(dash + 1), nullptr, 16);
+        if (last <= begin || first >= end) continue;
+        found.push_back({std::max<std::uint64_t>(first, begin), std::min<std::uint64_t>(last, end), permissions[0] == 'r', inode == 0 && permissions[3] == 'p'});
+    }
+    return found;
+}
+
+void fileData(std::vector<Candidate>& maybe, const Mapping& mapping, int file, std::uint64_t offset) {
+    const auto end = offset + (mapping.end - mapping.begin);
+    auto at = static_cast<off_t>(offset);
+    while (static_cast<std::uint64_t>(at) < end) {
+        const auto data = lseek(file, at, SEEK_DATA);
+        if (data < 0) {
+            Require(errno == ENXIO, "SEEK_DATA on a heap mirror's shared backing failed");
+            return;
+        }
+        if (static_cast<std::uint64_t>(data) >= end) return;
+        const auto hole = lseek(file, data, SEEK_HOLE);
+        Require(hole >= 0, "SEEK_HOLE on a heap mirror's shared backing failed");
+        const auto stop = std::min<std::uint64_t>(static_cast<std::uint64_t>(hole), end);
+        maybe.push_back({mapping.begin + (static_cast<std::uint64_t>(data) - offset), mapping.begin + (stop - offset), mapping.readable});
+        at = static_cast<off_t>(stop);
+    }
+}
+
+void populatedPages(std::vector<Candidate>& maybe, const Mapping& mapping) {
+    static const auto page = static_cast<std::uint64_t>(sysconf(_SC_PAGESIZE));
+    const int file = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC);
+    Require(file >= 0, "cannot open /proc/self/pagemap for a heap mirror's pages");
+    constexpr std::size_t Entries = 4096;
+    std::array<std::uint64_t, Entries> entries{};
+    const auto first = mapping.begin / page;
+    const auto last = (mapping.end + page - 1) / page;
+    for (auto at = first; at < last;) {
+        const auto count = static_cast<std::size_t>(std::min<std::uint64_t>(Entries, last - at));
+        const auto read = pread(file, entries.data(), count * sizeof(std::uint64_t), static_cast<off_t>(at * sizeof(std::uint64_t)));
+        if (read != static_cast<ssize_t>(count * sizeof(std::uint64_t))) {
+            close(file);
+            throw std::runtime_error("reading /proc/self/pagemap for a heap mirror's pages failed");
+        }
+        for (std::size_t index = 0; index < count; ++index) {
+            constexpr std::uint64_t present = std::uint64_t{1} << 63u;
+            constexpr std::uint64_t swapped = std::uint64_t{1} << 62u;
+            if ((entries[index] & (present | swapped)) == 0) continue;
+            const auto address = (at + index) * page;
+            const auto from = std::max(address, mapping.begin);
+            const auto to = std::min(address + page, mapping.end);
+            if (!maybe.empty() && maybe.back().end == from && maybe.back().readable == mapping.readable) maybe.back().end = to;
+            else maybe.push_back({from, to, mapping.readable});
+        }
+        at += count;
+    }
+    close(file);
+}
+#endif
+
+std::vector<Candidate> possiblyNonzero(std::uint64_t begin, std::uint64_t end) {
+    std::vector<Candidate> maybe;
+#ifdef _WIN32
+    maybe.push_back({begin, end, true});
+#else
+    for (const auto& mapping : mappings(begin, end)) {
+        int file = -1;
+        std::uint64_t offset = 0;
+        if (GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast<std::uintptr_t>(mapping.begin), static_cast<std::size_t>(mapping.end - mapping.begin), &file, &offset)) {
+            try {
+                fileData(maybe, mapping, file, offset);
+            } catch (...) {
+                close(file);
+                throw;
+            }
+            close(file);
+        } else if (mapping.privateAnonymous) {
+            populatedPages(maybe, mapping);
+        } else {
+            maybe.push_back({mapping.begin, mapping.end, mapping.readable});
+        }
+    }
+#endif
+    return maybe;
+}
+
+}
+
+bool GuestBytesZero(std::uint64_t begin, std::uint64_t end) {
+    return allZero(begin, end);
+}
+
+GuestRuns GuestNonzeroBlocks(const GuestRuns& candidates) {
+    GuestRuns runs;
+    for (const auto& [first, last] : candidates) {
+        if (first >= last) continue;
+        const auto maybe = possiblyNonzero(first, last);
+        std::size_t at = 0;
+        for (auto block = first / BlockSize * BlockSize; block < last; block += BlockSize) {
+            const auto from = std::max(first, block);
+            const auto to = std::min(last, block + BlockSize);
+            while (at < maybe.size() && maybe[at].end <= from) ++at;
+            bool nonzero = false;
+            for (auto next = at; !nonzero && next < maybe.size() && maybe[next].begin < to; ++next) {
+                const auto low = std::max(from, maybe[next].begin);
+                const auto high = std::min(to, maybe[next].end);
+                if (low < high) nonzero = !maybe[next].readable || !allZero(low, high);
+            }
+            if (!nonzero) continue;
+            if (!runs.empty() && runs.back().second == from) runs.back().second = to;
+            else runs.emplace_back(from, to);
+        }
+    }
+    return runs;
+}
+
+void SetResidentAddressSpaceLimit(std::uint64_t bytes) {
+    auto& devices = Devices();
+    std::lock_guard lock(devices.mutex);
+    devices.limit = bytes;
+}
+
+std::shared_ptr<ResidentBuffer> ResidentBuffer::Create(const Context& context, std::uint64_t base, std::uint64_t bytes, const GuestRuns& backed) {
+    if (!context.sparseBinding || !context.sparseResidencyBuffer || !context.sparseResidencyAliased) return nullptr;
+    const auto first = base / BlockSize * BlockSize;
+    const auto size = (base - first + bytes + BlockSize - 1) / BlockSize * BlockSize;
+    {
+        auto& devices = Devices();
+        std::lock_guard lock(devices.mutex);
+        const auto limit = devices.limit != 0 ? std::min(devices.limit, context.limits.sparseAddressSpaceSize) : context.limits.sparseAddressSpaceSize;
+        auto& used = devices.addressSpace[context.device];
+        if (size > limit || used > limit - size) return nullptr;
+        used += size;
+    }
+    return std::shared_ptr<ResidentBuffer>(new ResidentBuffer(context, base, bytes, backed));
+}
+
+ResidentBuffer::ResidentBuffer(const Context& context, std::uint64_t base, std::uint64_t bytes, const GuestRuns& backed) : context(context) {
+    origin = base / BlockSize * BlockSize;
+    lead = base - origin;
+    size = (lead + bytes + BlockSize - 1) / BlockSize * BlockSize;
+    try {
+        VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
+        info.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_ALIASED_BIT;
+        info.size = size;
+        info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
+        const std::array<std::uint32_t, 2> families{context.queueFamily, context.sparseFamily};
+        if (context.sparseQueue != VK_NULL_HANDLE && context.sparseFamily != context.queueFamily) {
+            info.sharingMode = VK_SHARING_MODE_CONCURRENT;
+            info.queueFamilyIndexCount = static_cast<std::uint32_t>(families.size());
+            info.pQueueFamilyIndices = families.data();
+        }
+        if (const auto result = context.Function<PFN_vkCreateBuffer>("vkCreateBuffer")(context.device, &info, nullptr, &buffer); result != VK_SUCCESS) fail("vkCreateBuffer of a resident heap mirror", result);
+        VkMemoryRequirements requirements{};
+        context.Function<PFN_vkGetBufferMemoryRequirements>("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements);
+        if (requirements.alignment == 0 || requirements.alignment > BlockSize || BlockSize % requirements.alignment != 0) {
+            char text[160];
+            std::snprintf(text, sizeof(text), "the device's sparse buffer block of %llu bytes does not divide a heap mirror's 64 KiB block", static_cast<unsigned long long>(requirements.alignment));
+            throw std::runtime_error(text);
+        }
+        type = context.DeviceMemoryType(requirements.memoryTypeBits, SlabBytes, "a resident heap mirror slab");
+        memory = residentMemory(context, type);
+        slots.resize(static_cast<std::size_t>(size / BlockSize));
+        std::vector<bool> wanted(slots.size(), false);
+        for (const auto& [first, last] : backed) {
+            if (first >= last) continue;
+            for (auto index = blockIndex(first); index <= blockIndex(last - 1); ++index) wanted[index] = true;
+        }
+        std::vector<VkSparseMemoryBind> binds;
+        for (std::size_t index = 0; index < slots.size();) {
+            if (wanted[index]) {
+                auto [slab, at] = memory->Take();
+                slots[index] = {slab, at, slab->host.get() + at * BlockSize};
+                std::memset(slots[index].host, 0, static_cast<std::size_t>(BlockSize));
+                binds.push_back({index * BlockSize, BlockSize, slab->memory, at * BlockSize, 0});
+                ++bound;
+                ++index;
+                continue;
+            }
+            auto last = index;
+            while (last < slots.size() && !wanted[last] && (last - index + 1) * BlockSize <= ZeroBytes) ++last;
+            binds.push_back({index * BlockSize, (last - index) * BlockSize, memory->Zero(), 0, 0});
+            index = last;
+        }
+        submit(binds);
+        const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, buffer};
+        address = context.Function<PFN_vkGetBufferDeviceAddressKHR>("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo);
+        if (address == 0) fail("vkGetBufferDeviceAddressKHR of a resident heap mirror", VK_ERROR_UNKNOWN);
+    } catch (...) {
+        release();
+        throw;
+    }
+}
+
+ResidentBuffer::~ResidentBuffer() {
+    release();
+}
+
+void ResidentBuffer::release() noexcept {
+    if (buffer != VK_NULL_HANDLE) context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, buffer, nullptr);
+    buffer = VK_NULL_HANDLE;
+    for (auto& slot : slots) {
+        if (slot.host != nullptr) memory->Put(static_cast<ResidentMemory::Slab*>(slot.slab), slot.index);
+        slot = {};
+    }
+    auto& devices = Devices();
+    std::lock_guard lock(devices.mutex);
+    if (const auto found = devices.addressSpace.find(context.device); found != devices.addressSpace.end()) found->second -= std::min(found->second, size);
+    size = 0;
+}
+
+std::size_t ResidentBuffer::blockIndex(std::uint64_t guest) const {
+    Require(guest >= origin && guest - origin < size, "guest address outside its resident heap mirror");
+    return static_cast<std::size_t>((guest - origin) / BlockSize);
+}
+
+std::byte* ResidentBuffer::Block(std::uint64_t guest) const {
+    const auto& slot = slots[blockIndex(guest)];
+    return slot.host == nullptr ? nullptr : slot.host + (guest - origin) % BlockSize;
+}
+
+std::uint64_t ResidentBuffer::UnboundBytes(const GuestRuns& runs) const {
+    std::vector<std::size_t> seen;
+    for (const auto& [first, last] : runs) {
+        if (first >= last) continue;
+        for (auto index = blockIndex(first); index <= blockIndex(last - 1); ++index) {
+            if (slots[index].host == nullptr) seen.push_back(index);
+        }
+    }
+    std::sort(seen.begin(), seen.end());
+    return static_cast<std::uint64_t>(std::unique(seen.begin(), seen.end()) - seen.begin()) * BlockSize;
+}
+
+void ResidentBuffer::Bind(const GuestRuns& runs) {
+    std::vector<VkSparseMemoryBind> binds;
+    std::vector<std::size_t> made;
+    for (const auto& [first, last] : runs) {
+        if (first >= last) continue;
+        for (auto index = blockIndex(first); index <= blockIndex(last - 1); ++index) {
+            if (slots[index].host != nullptr) continue;
+            auto [slab, at] = memory->Take();
+            slots[index] = {slab, at, slab->host.get() + at * BlockSize};
+            std::memset(slots[index].host, 0, static_cast<std::size_t>(BlockSize));
+            binds.push_back({index * BlockSize, BlockSize, slab->memory, at * BlockSize, 0});
+            made.push_back(index);
+        }
+    }
+    if (binds.empty()) return;
+    if (auto* recorder = Recorder::Active(); recorder != nullptr) recorder->Sync();
+    try {
+        submit(binds);
+    } catch (...) {
+        for (const auto index : made) {
+            memory->Put(static_cast<ResidentMemory::Slab*>(slots[index].slab), slots[index].index);
+            slots[index] = {};
+        }
+        throw;
+    }
+    bound += made.size();
+    for (const auto index : made) MarkChanged(origin + index * BlockSize, origin + (index + 1) * BlockSize);
+}
+
+void ResidentBuffer::MarkChanged(std::uint64_t begin, std::uint64_t end) {
+    if (begin >= end) return;
+    bool first = false;
+    {
+        std::lock_guard lock(changedMutex);
+        first = changed.empty();
+        if (!changed.empty() && changed.back().second == begin) changed.back().second = end;
+        else changed.emplace_back(begin, end);
+    }
+    if (!first) return;
+    auto& devices = Devices();
+    std::lock_guard lock(devices.pendingMutex);
+    devices.pending.push_back(weak_from_this());
+}
+
+void ResidentBuffer::Read(std::uint64_t guest, std::span<std::byte> out) const {
+    std::size_t done = 0;
+    while (done < out.size()) {
+        const auto at = guest + done;
+        const auto within = static_cast<std::size_t>((at - origin) % BlockSize);
+        const auto length = std::min<std::size_t>(out.size() - done, static_cast<std::size_t>(BlockSize) - within);
+        if (const auto* source = Block(at); source != nullptr) std::memcpy(out.data() + done, source, length);
+        else std::memset(out.data() + done, 0, length);
+        done += length;
+    }
+}
+
+void ResidentBuffer::Store(std::uint64_t guest, std::span<const std::byte> bytes) {
+    std::size_t done = 0;
+    while (done < bytes.size()) {
+        const auto at = guest + done;
+        const auto within = static_cast<std::size_t>((at - origin) % BlockSize);
+        const auto length = std::min<std::size_t>(bytes.size() - done, static_cast<std::size_t>(BlockSize) - within);
+        if (auto* target = Block(at); target != nullptr) std::memcpy(target, bytes.data() + done, length);
+        else Require(GuestBytesZero(reinterpret_cast<std::uintptr_t>(bytes.data() + done), reinterpret_cast<std::uintptr_t>(bytes.data() + done + length)), "the GPU wrote a heap mirror block that holds no memory");
+        done += length;
+    }
+}
+
+std::shared_ptr<Buffer> ResidentBuffer::RecordReadback(VkCommandBuffer commands, std::uint64_t guest, std::uint64_t bytes) const {
+    Require(guest >= origin && bytes <= size && guest - origin <= size - bytes, "heap mirror readback outside its buffer");
+    auto staging = std::make_shared<Buffer>(context, static_cast<std::size_t>(bytes), VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+    CopyBuffer(context, commands, buffer, guest - origin, staging->Handle(), 0, bytes);
+    return staging;
+}
+
+void ResidentBuffer::ReadDevice(std::uint64_t guest, std::span<std::byte> out) {
+    GuestMemory::AssertGpuLockHeld("heap mirror device read");
+    Require(!Recorder::InCompletion(), "heap mirror device read inside a GPU completion: its write-back needs a recorded readback");
+    auto staging = readbackBatch(guest, out.size());
+    std::memcpy(out.data(), staging->Bytes().data(), out.size());
+}
+
+std::shared_ptr<Buffer> ResidentBuffer::readbackBatch(std::uint64_t guest, std::uint64_t bytes) {
+    CommandBatch batch(context);
+    RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
+    auto staging = RecordReadback(batch.Handle(), guest, bytes);
+    RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT);
+    batch.SubmitAndWait();
+    return staging;
+}
+
+void ResidentBuffer::flush() {
+    GuestRuns runs;
+    {
+        std::lock_guard lock(changedMutex);
+        runs.swap(changed);
+    }
+    if (runs.empty()) return;
+    try {
+        std::sort(runs.begin(), runs.end());
+        std::vector<VkBufferCopy> regions;
+        std::uint64_t total = 0;
+        std::uint64_t cursor = 0;
+        for (const auto& [first, last] : runs) {
+            for (auto at = std::max(first, cursor); at < last;) {
+                const auto next = std::min(last, (at / BlockSize + 1) * BlockSize);
+                if (slots[blockIndex(at)].host != nullptr) {
+                    if (!regions.empty() && regions.back().dstOffset + regions.back().size == at - origin) regions.back().size += next - at;
+                    else regions.push_back({total, at - origin, next - at});
+                    total += next - at;
+                }
+                at = next;
+            }
+            cursor = std::max(cursor, last);
+        }
+        if (regions.empty()) return;
+        auto staging = std::make_shared<Buffer>(context, static_cast<std::size_t>(total), VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+        for (const auto& region : regions) {
+            for (VkDeviceSize done = 0; done < region.size;) {
+                const auto at = origin + region.dstOffset + done;
+                const auto length = std::min<VkDeviceSize>(region.size - done, BlockSize - (at - origin) % BlockSize);
+                std::memcpy(staging->Bytes().data() + region.srcOffset + done, Block(at), static_cast<std::size_t>(length));
+                done += length;
+            }
+        }
+        recordTransfer(context, [&](VkCommandBuffer commands, std::vector<std::shared_ptr<void>>& kept) {
+            context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(commands, staging->Handle(), buffer, static_cast<std::uint32_t>(regions.size()), regions.data());
+            kept.push_back(staging);
+            kept.push_back(shared_from_this());
+        });
+    } catch (...) {
+        {
+            std::lock_guard lock(changedMutex);
+            changed.insert(changed.begin(), runs.begin(), runs.end());
+        }
+        auto& devices = Devices();
+        std::lock_guard lock(devices.pendingMutex);
+        devices.pending.push_back(weak_from_this());
+        throw;
+    }
+}
+
+void ResidentBuffer::FlushChanged(const Context& context) {
+    auto& devices = Devices();
+    std::vector<std::shared_ptr<ResidentMemory>> unzeroed;
+    {
+        std::lock_guard lock(devices.mutex);
+        for (const auto& [key, held] : devices.memories) {
+            if (std::get<0>(key) != context.device) continue;
+            if (auto memory = held.lock(); memory != nullptr && !memory->Zeroed()) unzeroed.push_back(std::move(memory));
+        }
+    }
+    for (auto& memory : unzeroed) recordTransfer(context, [&](VkCommandBuffer commands, std::vector<std::shared_ptr<void>>& kept) { memory->FillZero(commands, kept); });
+    std::vector<std::weak_ptr<ResidentBuffer>> waiting;
+    {
+        std::lock_guard lock(devices.pendingMutex);
+        waiting.swap(devices.pending);
+    }
+    for (std::size_t index = 0; index < waiting.size(); ++index) {
+        const auto held = waiting[index].lock();
+        if (held == nullptr) continue;
+        if (held->context.device != context.device) {
+            std::lock_guard lock(devices.pendingMutex);
+            devices.pending.push_back(waiting[index]);
+            continue;
+        }
+        try {
+            held->flush();
+        } catch (...) {
+            std::lock_guard lock(devices.pendingMutex);
+            devices.pending.insert(devices.pending.end(), waiting.begin() + static_cast<std::ptrdiff_t>(index) + 1, waiting.end());
+            throw;
+        }
+    }
+}
+
+void ResidentBuffer::submit(const std::vector<VkSparseMemoryBind>& binds) {
+    const bool ownQueue = context.sparseQueue != VK_NULL_HANDLE && context.sparseQueueMutex != nullptr;
+    const VkSparseBufferMemoryBindInfo bufferBinds{buffer, static_cast<std::uint32_t>(binds.size()), binds.data()};
+    VkBindSparseInfo bind{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO};
+    bind.bufferBindCount = 1;
+    bind.pBufferBinds = &bufferBinds;
+    const VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
+    VkFence fence = VK_NULL_HANDLE;
+    if (const auto result = context.Function<PFN_vkCreateFence>("vkCreateFence")(context.device, &fenceInfo, nullptr, &fence); result != VK_SUCCESS) fail("vkCreateFence of a resident heap mirror bind", result);
+    VkResult submitted = VK_SUCCESS;
+    VkResult waited = VK_SUCCESS;
+    {
+        std::unique_lock<std::mutex> queueLock;
+        std::unique_lock<GuestMemory::GpuMutexType> gpuLock;
+        if (ownQueue) queueLock = std::unique_lock(*context.sparseQueueMutex);
+        else gpuLock = std::unique_lock(GuestMemory::GpuMutex());
+        submitted = context.Function<PFN_vkQueueBindSparse>("vkQueueBindSparse")(ownQueue ? context.sparseQueue : context.queue, 1, &bind, fence);
+        if (submitted == VK_SUCCESS) waited = context.Function<PFN_vkWaitForFences>("vkWaitForFences")(context.device, 1, &fence, VK_TRUE, UINT64_MAX);
+    }
+    context.Function<PFN_vkDestroyFence>("vkDestroyFence")(context.device, fence, nullptr);
+    if (submitted != VK_SUCCESS) fail("vkQueueBindSparse of a resident heap mirror", submitted);
+    if (waited != VK_SUCCESS) fail("vkWaitForFences of a resident heap mirror bind", waited);
+}
+
+}

core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp

@@ -3131,6 +3131,63 @@ void RunSparseHeapMirrorUnwatchTests(const Context& context, const BdaTestAccess
     std::cout << "an address-based build keeps and refreshes a sparse heap's mirror once its range has left the write watch\n";
 }
 
+void RunSparseHeapMirrorBudgetTests(const Context& context, const BdaTestAccess& access) {
+    constexpr std::size_t block = 65536;
+    constexpr std::size_t reserved = 128 * block;
+    constexpr std::size_t committed = 80 * block;
+#ifdef _WIN32
+    if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
+        std::cout << "guest arena unavailable or not write-watched: sparse heaps past the heap mirror budget not tested\n";
+        return;
+    }
+    auto* guest = static_cast<std::byte*>(GuestArena::GuestArenaAllocate_nid_postfix(reserved, block));
+    Require(guest != nullptr, "cannot allocate the sparse heap in the guest arena");
+    GuestArena::GuestArenaCommit_nid_postfix(guest, committed, PAGE_READWRITE, block);
+#else
+    if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
+        std::cout << "write watch unavailable: sparse heaps past the heap mirror budget not tested\n";
+        return;
+    }
+    auto* guest = reserveSparse(reserved, committed);
+    GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(guest, reserved);
+#endif
+    std::memset(guest, 0x5a, committed);
+    const auto base = reinterpret_cast<std::uintptr_t>(guest);
+    const auto registry = [&](bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest, reserved, true, true, true);
+        else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest);
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    registry(true);
+    const auto before = MirrorCounters();
+    {
+        GuestBufferMemory memory(context);
+        memory.AddWritable(base, reserved);
+        memory.Upload(false);
+        std::uint32_t adjustment = 0;
+        const auto view = memory.Descriptor(base, reserved, adjustment);
+        const auto bytes = access.bytes(view.buffer);
+        Require(adjustment == 0 && view.range >= committed && bytes[view.offset + committed - 1] == std::byte{0x5a}, "a sparse heap past the heap mirror budget was not copied up to its last committed byte");
+        bytes[view.offset + 8] = std::byte{0x77};
+        memory.WriteBack();
+    }
+    Require(guest[8] == std::byte{0x77}, "a store to a sparse heap copied past the heap mirror budget was not written back");
+    const auto after = MirrorCounters();
+    Require(after.sparseBinds == before.sparseBinds && after.heapMirrors == before.heapMirrors && after.heapBytes == before.heapBytes, "a sparse heap past the heap mirror budget was mirrored or left heap mirror bytes held");
+    registry(false);
+    ClearImageMirrors(context.device);
+    ClearSnapshotCopies(context.device);
+#ifdef _WIN32
+    GuestArena::GuestArenaReset_nid_postfix(guest, reserved);
+    GuestArena::GuestArenaRelease_nid_postfix(guest, reserved);
+#else
+    GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(guest, reserved);
+    releaseSparse(guest, reserved);
+#endif
+    std::cout << "a sparse heap whose mirror would pass the heap mirror budget is copied\n";
+}
+
 void RunHeapMirrorRunUnwatchTests(const Context& context, const BdaTestAccess& access) {
     constexpr std::size_t block = 65536;
     constexpr std::size_t heapBytes = 32 * block;

core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp

@@ -19,6 +19,7 @@ void RunBdaResourceTests(const AgcDriver::Graphics::Context& context, const BdaT
 bool RunSparseBufferHoleTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access);
 void RunHeapMirrorUnwatchTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access, bool throughImport);
 void RunSparseHeapMirrorUnwatchTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access);
+void RunSparseHeapMirrorBudgetTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access);
 void RunHeapMirrorRunUnwatchTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access);
 void RunGuestAllocationTests();
 void RunGuestLeaseWaitTests();

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -11239,6 +11239,23 @@ int main(int argc, char** argv) {
             Require(mock.live == 0, "the sparse heap mirror unwatch test leaked Vulkan objects");
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--sparse-heap-mirror-budget") {
+            {
+                mock = MockVulkan{};
+                auto context = mockContext();
+                context.bufferDeviceAddress = true;
+                context.limits.maxStorageBufferRange = 1u << 27;
+                RunSparseHeapMirrorBudgetTests(context, {
+                    [](VkBuffer buffer) -> std::span<std::byte> { return mockBufferMemory(buffer); },
+                    {},
+                    {},
+                    {},
+                    {}
+                });
+            }
+            Require(mock.live == 0, "the sparse heap mirror budget test leaked Vulkan objects");
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--heap-mirror-run-unwatch") {
             {
                 mock = MockVulkan{};

core/libs/prx/libSceAgcDriver/tests/execution/DeviceQueuePlan.cpp

@@ -0,0 +1,71 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+
+constexpr VkQueueFlags Graphics = VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT;
+constexpr VkQueueFlags Sparse = VK_QUEUE_SPARSE_BINDING_BIT;
+constexpr std::uint32_t Own = VK_QUEUE_FAMILY_IGNORED;
+
+struct Case {
+    const char* name;
+    std::vector<VkQueueFamilyProperties> families;
+    bool settles;
+    std::uint32_t mainCount;
+    bool settle;
+    std::uint32_t settleFamily;
+    std::uint32_t settleIndex;
+    bool sparseBinding;
+    std::uint32_t sparseFamily;
+    std::uint32_t sparseIndex;
+};
+
+VkQueueFamilyProperties family(VkQueueFlags flags, std::uint32_t queues) {
+    VkQueueFamilyProperties properties{};
+    properties.queueFlags = flags;
+    properties.queueCount = queues;
+    return properties;
+}
+
+std::string describe(const AgcDriver::DeviceQueuePlan& plan) {
+    return "main " + std::to_string(plan.mainCount) + " queues, settle " + (plan.settle ? std::to_string(plan.settleFamily) + ":" + std::to_string(plan.settleIndex) : std::string("none")) + ", sparse " + (!plan.sparseBinding ? std::string("none") : plan.sparseFamily == Own ? std::string("on the main queue") : std::to_string(plan.sparseFamily) + ":" + std::to_string(plan.sparseIndex));
+}
+
+void check(const Case& test) {
+    const auto plan = AgcDriver::PlanDeviceQueues(test.families, 0, test.settles, true);
+    const auto dense = AgcDriver::PlanDeviceQueues(test.families, 0, test.settles, false);
+    Require(plan.mainCount == dense.mainCount && plan.settle == dense.settle && plan.settleFamily == dense.settleFamily && plan.settleIndex == dense.settleIndex, std::string("device queue plan, ") + test.name + ": sparse binding changed the queues: " + describe(dense) + " without it, " + describe(plan) + " with it");
+    Require(!dense.sparseBinding && dense.sparseFamily == Own, std::string("device queue plan, ") + test.name + ": a device without sparse binding got a sparse queue: " + describe(dense));
+    const bool expected = plan.mainCount == test.mainCount && plan.settle == test.settle && (!test.settle || (plan.settleFamily == test.settleFamily && plan.settleIndex == test.settleIndex)) && plan.sparseBinding == test.sparseBinding && plan.sparseFamily == test.sparseFamily && (test.sparseFamily == Own || plan.sparseIndex == test.sparseIndex);
+    Require(expected, std::string("device queue plan, ") + test.name + ": got " + describe(plan));
+}
+
+}
+
+int main() {
+    try {
+        const std::vector<Case> cases{
+            {"one family with one queue", {family(Graphics | Sparse, 1)}, true, 1, false, 0, 0, true, Own, 0},
+            {"one family with one queue, no settles", {family(Graphics | Sparse, 1)}, false, 1, false, 0, 0, true, Own, 0},
+            {"settle family with one queue that also binds", {family(Graphics | Sparse, 1), family(VK_QUEUE_TRANSFER_BIT | Sparse, 1)}, true, 1, true, 1, 0, true, Own, 0},
+            {"settle family with two queues that also binds", {family(Graphics, 1), family(VK_QUEUE_TRANSFER_BIT | Sparse, 2)}, true, 1, true, 1, 0, true, 1, 1},
+            {"settle on the main family", {family(Graphics | Sparse, 2), family(VK_QUEUE_TRANSFER_BIT | Sparse, 1)}, true, 2, true, 0, 1, true, 1, 0},
+            {"sparse-only family preferred", {family(Graphics | Sparse, 16), family(VK_QUEUE_TRANSFER_BIT | Sparse, 2), family(Sparse, 1)}, true, 2, true, 0, 1, true, 2, 0},
+            {"no settles, a second family binds", {family(Graphics | Sparse, 1), family(VK_QUEUE_TRANSFER_BIT | Sparse, 1)}, false, 1, false, 0, 0, true, 1, 0},
+            {"no queue binds", {family(Graphics, 1), family(VK_QUEUE_TRANSFER_BIT, 1)}, true, 1, true, 1, 0, false, Own, 0},
+            {"only the settle queue binds", {family(Graphics, 1), family(VK_QUEUE_TRANSFER_BIT | Sparse, 1)}, true, 1, true, 1, 0, false, Own, 0},
+        };
+        for (const auto& test : cases) check(test);
+        std::puts("device queue plan tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/EmptyIndirectSkip.cpp

@@ -0,0 +1,162 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <exception>
+#include <initializer_list>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+
+constexpr std::byte Kept{0x40};
+constexpr std::uint32_t BaseVertexLocation = 0x8c;
+
+alignas(256) constexpr std::array<std::uint32_t, 20> VertexCode{
+    0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+    0x7e060208u, 0x7e0802f2u, 0x7e0a02ffu, 0x44800000u, 0xf80000d1u, 0x00000005u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u,
+};
+alignas(256) constexpr std::array<std::uint32_t, 8> PixelCode{0x7e000200u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u};
+alignas(256) std::array<std::byte, 1024> FirstMemory{};
+alignas(256) std::array<std::byte, 1024> NoInstancesMemory{};
+alignas(256) std::array<std::byte, 1024> NoVerticesMemory{};
+alignas(256) std::array<std::byte, 1024> LastMemory{};
+alignas(256) std::array<std::uint32_t, 16> Records{3, 1, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 0};
+
+struct Header {
+    Shader shader{};
+    ShaderUserData userData{};
+};
+
+alignas(8) Header VertexHeader{};
+alignas(8) Header PixelHeader{};
+
+void Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18u;
+    header.shader.header_size = sizeof(Header);
+    header.shader.shader_size = static_cast<std::uint32_t>(bytes);
+    header.shader.code = code;
+    header.shader.type = type;
+    header.shader.user_data = &header.userData;
+    AgcDriverRegisterShader_nid_postfix(&header.shader);
+}
+
+void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::initializer_list<std::uint32_t> payload) {
+    commands.push_back(0xc0000000u | (static_cast<std::uint32_t>(payload.size() - 1) << 16u) | (opcode << 8u));
+    commands.insert(commands.end(), payload);
+}
+
+std::vector<std::uint32_t> DrawFour() {
+    const auto vertex = reinterpret_cast<std::uintptr_t>(VertexCode.data());
+    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const auto records = reinterpret_cast<std::uintptr_t>(Records.data());
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x207, 0x01200000}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, 0x28028},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, (63u << 14u) | 3u},
+        {0x3b8, 0x9000000}, {0x1e0, 0},
+        {0xc, 0}, {0xd, 0x40040},
+        {0x81, 0x80000000}, {0x82, 0x40040},
+        {0x90, 0x80000000}, {0x91, 0x40040},
+        {0x94, 0x80000000}, {0x95, 0x40040},
+        {0x10f, std::bit_cast<std::uint32_t>(32.0f)}, {0x110, std::bit_cast<std::uint32_t>(32.0f)},
+        {0x111, std::bit_cast<std::uint32_t>(-2.0f)}, {0x112, std::bit_cast<std::uint32_t>(2.0f)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)},
+        {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0},
+    };
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> shader{
+        {0xc8, static_cast<std::uint32_t>(vertex >> 8u)}, {0xc9, static_cast<std::uint32_t>(vertex >> 40u)}, {0x8b, 1u << 1u},
+        {0x008, static_cast<std::uint32_t>(pixel >> 8u)}, {0x009, static_cast<std::uint32_t>(pixel >> 40u)}, {0x00b, 1u << 1u}, {0x00c, std::bit_cast<std::uint32_t>(1.0f)},
+    };
+    std::vector<std::uint32_t> commands;
+    Append(commands, 0x79, {0x242, 4});
+    Append(commands, 0x79, {0x24a, 0});
+    for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value});
+    for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value});
+    Append(commands, 0x11, {1, static_cast<std::uint32_t>(records), static_cast<std::uint32_t>(records >> 32u)});
+    const auto draw = [&](std::array<std::byte, 1024>& memory, std::uint32_t recordOffset) {
+        const auto color = reinterpret_cast<std::uintptr_t>(memory.data());
+        Append(commands, 0x69, {0x318, static_cast<std::uint32_t>(color >> 8u)});
+        Append(commands, 0x69, {0x390, static_cast<std::uint32_t>(color >> 40u)});
+        Append(commands, 0x76, {BaseVertexLocation, 0});
+        Append(commands, 0x24, {recordOffset, BaseVertexLocation, 0, 2});
+    };
+    draw(FirstMemory, 0);
+    draw(NoInstancesMemory, 16);
+    draw(NoVerticesMemory, 32);
+    draw(LastMemory, 48);
+    return commands;
+}
+
+std::size_t Covered(const std::array<std::byte, 1024>& memory) {
+    return static_cast<std::size_t>(std::count_if(memory.begin(), memory.end(), [](std::byte value) { return value != Kept; }));
+}
+
+void EmptyIndirectDrawsSkipTheirStages() {
+    FirstMemory.fill(Kept);
+    NoInstancesMemory.fill(Kept);
+    NoVerticesMemory.fill(Kept);
+    LastMemory.fill(Kept);
+    Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2);
+    Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
+    auto commands = DrawFour();
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw submission failed");
+    AgcDriverWaitIdle_nid_postfix();
+    const auto first = Covered(FirstMemory);
+    Require(first != 0, "the first indirect draw, whose record draws 3 vertices, covered nothing");
+    Require(Covered(NoInstancesMemory) == 0, "the indirect draw whose record has no instances covered " + std::to_string(Covered(NoInstancesMemory)) + " bytes, expected none");
+    Require(Covered(NoVerticesMemory) == 0, "the indirect draw whose record has no vertices covered " + std::to_string(Covered(NoVerticesMemory)) + " bytes, expected none");
+    for (std::size_t index = 0; index < FirstMemory.size(); ++index) {
+        Require(LastMemory[index] == FirstMemory[index], "byte " + std::to_string(index) + " of the indirect draw after the empty ones is " + std::to_string(static_cast<unsigned>(LastMemory[index])) + ", the first indirect draw wrote " + std::to_string(static_cast<unsigned>(FirstMemory[index])));
+    }
+    const auto empty = AgcDriver::Graphics::EmptyIndirectDrawCount();
+    Require(empty == 2, std::to_string(empty) + " of the 2 indirect draws whose records draw nothing were skipped before their stages were built");
+    const auto counts = AgcDriver::Graphics::IndirectDrawCounts();
+    std::uint64_t cpu = 0;
+    for (std::size_t path = 1; path < counts.size(); ++path) cpu += counts[path];
+    Require(counts[0] == 0 && cpu == 2, "the drawing indirect draws took the GPU-side path " + std::to_string(counts[0]) + " times and the CPU path " + std::to_string(cpu) + " times, expected the CPU path twice");
+}
+
+}
+
+int main() {
+    try {
+#ifdef _WIN32
+        _putenv_s("APS5_PROFILE_DRAW", "1");
+#else
+        setenv("APS5_PROFILE_DRAW", "1", 1);
+#endif
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+        }
+        EmptyIndirectDrawsSkipTheirStages();
+        AgcDriverShutdown_nid_postfix();
+        std::puts("CPU-path indirect draws whose records draw nothing were skipped before their stages were built");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        AgcDriverShutdown_nid_postfix();
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/ResidentHeapMirror.cpp

@@ -0,0 +1,351 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ResidentMirror.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint64_t Block = 65536;
+constexpr std::uint64_t HeapBytes = 8u << 20u;
+constexpr std::uint64_t JoinedBytes = 16u << 20u;
+constexpr std::uint64_t ThreadStride = 16384;
+constexpr std::uint64_t StoreBlock = 3 * Block;
+constexpr std::uint32_t LateThread = 5;
+constexpr std::uint64_t ZeroedBlock = 2 * Block;
+
+alignas(256) constexpr std::array<std::uint32_t, 10> Code{
+    0x3404008e, 0xdc388000, 0x04000002, 0x34060082, 0xbf8c3f70, 0xe0701000, 0x80010403, 0xe0701000, 0x80020303, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> CopyCode{0x34060082, 0xe0301000, 0x80000403, 0xbf8c3f70, 0xe0701000, 0x80010403, 0xbf810000};
+
+class OutputBlock {
+public:
+    OutputBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint32_t*>(VirtualAlloc(nullptr, Block, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint32_t*>(std::aligned_alloc(Block, Block));
+#endif
+        Require(block != nullptr, "resident heap mirror: cannot allocate the output block");
+        GuestAllocations::Mutation().Add(block, Block, true, true, true);
+    }
+
+    ~OutputBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    }
+
+    OutputBlock(const OutputBlock&) = delete;
+    OutputBlock& operator=(const OutputBlock&) = delete;
+
+    std::uint32_t* Data() const { return block; }
+
+private:
+    std::uint32_t* block = nullptr;
+};
+
+class GuestHeap {
+public:
+    explicit GuestHeap(bool writable, std::uint64_t ranges = 1, std::uint64_t rangeBytes = HeapBytes, void* at = nullptr) : ranges(ranges), rangeBytes(rangeBytes) {
+        const auto bytes = rangeBytes * ranges;
+#ifdef _WIN32
+        raw = at == nullptr ? GuestArena::GuestArenaAllocate_nid_postfix(bytes, Block)
+                            : GuestArena::GuestArenaAllocateAtOrAbove_nid_postfix(reinterpret_cast<std::uintptr_t>(at), bytes, Block);
+        Require(raw != nullptr && (at == nullptr || raw == at), "resident heap mirror: cannot allocate the guest heap");
+        GuestArena::GuestArenaCommit_nid_postfix(raw, bytes, PAGE_READWRITE, bytes);
+        base = static_cast<std::uint8_t*>(raw);
+#else
+        raw = mmap(at, bytes + Block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | (at != nullptr ? MAP_FIXED_NOREPLACE : 0), -1, 0);
+        Require(raw != MAP_FAILED && (at == nullptr || raw == at), "resident heap mirror: cannot map the guest heap");
+        base = reinterpret_cast<std::uint8_t*>((reinterpret_cast<std::uintptr_t>(raw) + Block - 1) / Block * Block);
+#endif
+        GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(base, bytes);
+        GuestAllocations::Mutation mutation;
+        for (std::uint64_t range = 0; range < ranges; ++range) mutation.Add(base + range * rangeBytes, rangeBytes, true, writable, true);
+    }
+
+    ~GuestHeap() {
+        {
+            GuestAllocations::Mutation mutation;
+            for (std::uint64_t range = 0; range < ranges; ++range) mutation.Remove(base + range * rangeBytes);
+        }
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(base, rangeBytes * ranges);
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(raw, rangeBytes * ranges);
+        GuestArena::GuestArenaRelease_nid_postfix(raw, rangeBytes * ranges);
+#else
+        munmap(raw, rangeBytes * ranges + Block);
+#endif
+    }
+
+    GuestHeap(const GuestHeap&) = delete;
+    GuestHeap& operator=(const GuestHeap&) = delete;
+
+    std::uint8_t* Data() const { return base; }
+    void* Raw() const { return raw; }
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(base); }
+
+    void Store(std::uint64_t offset, std::uint32_t value) const { std::memcpy(base + offset, &value, sizeof(value)); }
+
+    std::uint32_t Load(std::uint64_t offset) const {
+        std::uint32_t value = 0;
+        std::memcpy(&value, base + offset, sizeof(value));
+        return value;
+    }
+
+private:
+    std::uint64_t ranges = 1;
+    std::uint64_t rangeBytes = HeapBytes;
+    void* raw = nullptr;
+    std::uint8_t* base = nullptr;
+};
+
+struct ImportBudget {
+    explicit ImportBudget(std::uint64_t bytes) { AgcDriver::Graphics::SetHostImportBudget(bytes); }
+    ~ImportBudget() { AgcDriver::Graphics::SetHostImportBudget(0); }
+    ImportBudget(const ImportBudget&) = delete;
+    ImportBudget& operator=(const ImportBudget&) = delete;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(std::uint64_t address, std::uint32_t bytes) {
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::string Hex(std::uint64_t value) {
+    char text[24];
+    std::snprintf(text, sizeof(text), "0x%llx", static_cast<unsigned long long>(value));
+    return text;
+}
+
+std::uint32_t Pattern(std::uint32_t thread) {
+    return 0x5a000000u + thread * 0x00010203u + 1u;
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData) {
+    const std::array<ShaderRecompiler::MemoryRegion, 1> regions{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, regions},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint64_t read, std::uint64_t written, const OutputBlock& output) {
+    std::fill(output.Data(), output.Data() + Threads, 0xdeadbeefu);
+    std::vector<std::uint32_t> userData(12, 0u);
+    userData[0] = static_cast<std::uint32_t>(read);
+    userData[1] = static_cast<std::uint32_t>(read >> 32u);
+    const auto out = BufferDescriptor(reinterpret_cast<std::uintptr_t>(output.Data()), static_cast<std::uint32_t>(Threads * sizeof(std::uint32_t)));
+    const auto store = BufferDescriptor(written + StoreBlock, static_cast<std::uint32_t>(Threads * sizeof(std::uint32_t)));
+    std::copy(out.begin(), out.end(), userData.begin() + 4);
+    std::copy(store.begin(), store.end(), userData.begin() + 8);
+    Run(device, Code, userData);
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, const GuestHeap& read, const GuestHeap& written, const OutputBlock& output) {
+    Dispatch(device, read.Address(), written.Address(), output);
+}
+
+void Copy(AgcDriver::VulkanDevice& device, std::uint64_t source, const OutputBlock& output) {
+    std::fill(output.Data(), output.Data() + Threads, 0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto in = BufferDescriptor(source, static_cast<std::uint32_t>(Threads * sizeof(std::uint32_t)));
+    const auto out = BufferDescriptor(reinterpret_cast<std::uintptr_t>(output.Data()), static_cast<std::uint32_t>(Threads * sizeof(std::uint32_t)));
+    std::copy(in.begin(), in.end(), userData.begin());
+    std::copy(out.begin(), out.end(), userData.begin() + 4);
+    Run(device, CopyCode, userData);
+}
+
+void CheckReads(const OutputBlock& output, const std::string& step, bool late) {
+    for (std::uint32_t thread = 0; thread < Threads; ++thread) {
+        const auto expected = thread * ThreadStride < Block ? Pattern(thread) : late && thread == LateThread ? Pattern(thread) : 0u;
+        Require(output.Data()[thread] == expected, "resident heap mirror: " + step + ": thread " + std::to_string(thread) + " read " + Hex(output.Data()[thread]) + " at heap offset " + Hex(thread * ThreadStride) + ", expected " + Hex(expected));
+    }
+}
+
+void CheckStores(const GuestHeap& written, const std::string& step) {
+    for (std::uint32_t thread = 0; thread < Threads; ++thread) {
+        const auto value = written.Load(StoreBlock + thread * sizeof(std::uint32_t));
+        Require(value == thread * 4u, "resident heap mirror: " + step + ": the GPU store of thread " + std::to_string(thread) + " into a never-written heap block reached the guest as " + Hex(value) + ", expected " + Hex(thread * 4u));
+    }
+}
+
+void CheckHeld(std::uint64_t before, std::uint64_t written, const std::string& step) {
+    const auto held = AgcDriver::Graphics::MirrorCounters().heapBytes - before;
+    Require(held == written, "resident heap mirror: " + step + ": the heap mirrors of two " + std::to_string(HeapBytes >> 10u) + " KiB heaps hold " + std::to_string(held >> 10u) + " KiB of device memory; the guest and the GPU wrote " + std::to_string(written >> 10u) + " KiB of them");
+}
+
+void ResidentTests(AgcDriver::VulkanDevice& device) {
+    GuestHeap read(false);
+    GuestHeap written(true);
+    const OutputBlock output;
+    for (std::uint32_t thread = 0; thread * ThreadStride < Block; ++thread) read.Store(thread * ThreadStride, Pattern(thread));
+    written.Store(16, 0x77u);
+    const ImportBudget budget(AgcDriver::Graphics::HostImportLiveBytes() + Block);
+    const auto before = AgcDriver::Graphics::MirrorCounters();
+    Dispatch(device, read, written, output);
+    const auto made = AgcDriver::Graphics::MirrorCounters();
+    Require(made.heapMirrors == before.heapMirrors + 2, "resident heap mirror: the two heaps were not heap mirrored (" + std::to_string(made.heapMirrors - before.heapMirrors) + " mirrors made)");
+    CheckHeld(before.heapBytes, 3 * Block, "first dispatch");
+    CheckReads(output, "first dispatch", false);
+    CheckStores(written, "first dispatch");
+    Require(written.Load(16) == 0x77u, "resident heap mirror: the written heap's guest bytes changed");
+    read.Store(LateThread * ThreadStride, Pattern(LateThread));
+    std::memset(read.Data() + ZeroedBlock, 0, Block);
+    Dispatch(device, read, written, output);
+    CheckHeld(before.heapBytes, 4 * Block, "after a guest write into a never-written block and zeros written into another");
+    CheckReads(output, "after a guest write into a never-written block", true);
+    const auto steady = AgcDriver::Graphics::MirrorCounters();
+    Dispatch(device, read, written, output);
+    const auto again = AgcDriver::Graphics::MirrorCounters();
+    Require(again.rebuilds == steady.rebuilds && again.heapBytes == steady.heapBytes, "resident heap mirror: an unchanged dispatch rebuilt or grew the heap mirrors");
+    CheckReads(output, "unchanged dispatch", true);
+    CheckStores(written, "unchanged dispatch");
+}
+
+struct AddressSpaceLimit {
+    explicit AddressSpaceLimit(std::uint64_t bytes) { AgcDriver::Graphics::SetResidentAddressSpaceLimit(bytes); }
+    ~AddressSpaceLimit() { AgcDriver::Graphics::SetResidentAddressSpaceLimit(0); }
+    AddressSpaceLimit(const AddressSpaceLimit&) = delete;
+    AddressSpaceLimit& operator=(const AddressSpaceLimit&) = delete;
+};
+
+void CopiedTests(AgcDriver::VulkanDevice& device) {
+    const AddressSpaceLimit limit(1);
+    GuestHeap read(false);
+    GuestHeap written(true);
+    const OutputBlock output;
+    for (std::uint32_t thread = 0; thread * ThreadStride < Block; ++thread) read.Store(thread * ThreadStride, Pattern(thread));
+    written.Store(16, 0x77u);
+    const ImportBudget budget(AgcDriver::Graphics::HostImportLiveBytes() + Block);
+    const auto before = AgcDriver::Graphics::MirrorCounters();
+    Dispatch(device, read, written, output);
+    const auto made = AgcDriver::Graphics::MirrorCounters();
+    Require(made.heapMirrors == before.heapMirrors + 2, "copied heap mirror: the two heaps were not heap mirrored without sparse address space (" + std::to_string(made.heapMirrors - before.heapMirrors) + " mirrors made)");
+    Require(made.heapBytes - before.heapBytes == 2 * HeapBytes, "copied heap mirror: two " + std::to_string(HeapBytes >> 10u) + " KiB heaps without sparse address space hold " + std::to_string((made.heapBytes - before.heapBytes) >> 10u) + " KiB, expected whole copies");
+    CheckReads(output, "copied, first dispatch", false);
+    CheckStores(written, "copied, first dispatch");
+    Require(written.Load(16) == 0x77u, "copied heap mirror: the written heap's guest bytes changed");
+    read.Store(LateThread * ThreadStride, Pattern(LateThread));
+    Dispatch(device, read, written, output);
+    CheckReads(output, "copied, after a guest write", true);
+    CheckStores(written, "copied, after a guest write");
+}
+
+void ReusedAddressTests(AgcDriver::VulkanDevice& device) {
+    GuestHeap written(true);
+    const OutputBlock output;
+    const ImportBudget budget(AgcDriver::Graphics::HostImportLiveBytes() + Block);
+    void* at = nullptr;
+    std::uint64_t address = 0;
+    {
+        GuestHeap first(false);
+        for (std::uint32_t thread = 0; thread * ThreadStride < Block; ++thread) first.Store(thread * ThreadStride, Pattern(thread));
+        Dispatch(device, first, written, output);
+        CheckReads(output, "the first heap", false);
+        at = first.Raw();
+        address = first.Address();
+    }
+    GuestHeap second(false, 1, HeapBytes, at);
+    Require(second.Address() == address, "resident heap mirror: the second heap did not take the first heap's address");
+    for (std::uint32_t thread = 0; thread * ThreadStride < Block; ++thread) second.Store(thread * ThreadStride, ~Pattern(thread));
+    Dispatch(device, second, written, output);
+    for (std::uint32_t thread = 0; thread < Threads; ++thread) {
+        const auto expected = thread * ThreadStride < Block ? ~Pattern(thread) : 0u;
+        Require(output.Data()[thread] == expected, "resident heap mirror: a heap mapped where a mirrored heap was freed: thread " + std::to_string(thread) + " read " + Hex(output.Data()[thread]) + ", expected " + Hex(expected));
+    }
+    const auto held = AgcDriver::Graphics::HeapMirrorsWithin(address, address + HeapBytes);
+    Require(held.mirrors == 1 && held.bytes == Block, "resident heap mirror: a heap mapped where a mirrored heap was freed holds " + std::to_string(held.mirrors) + " live heap mirrors (" + std::to_string(held.bytes >> 10u) + " KiB), expected 1 (64 KiB)");
+}
+
+void JoinedImportTests(AgcDriver::VulkanDevice& device) {
+    GuestHeap pair(true, 2, JoinedBytes);
+    const OutputBlock output;
+    const auto begin = pair.Address();
+    const auto end = begin + 2 * JoinedBytes;
+    for (std::uint32_t thread = 0; thread * ThreadStride < Block; ++thread) pair.Store(thread * ThreadStride, Pattern(thread));
+    pair.Store(JoinedBytes - sizeof(std::uint32_t), 0x41u);
+    pair.Store(JoinedBytes, 0x42u);
+    const auto before = AgcDriver::Graphics::HeapMirrorsWithin(begin, end);
+    {
+        const ImportBudget budget(AgcDriver::Graphics::HostImportLiveBytes() + Block);
+        Dispatch(device, begin, begin + JoinedBytes, output);
+        CheckReads(output, "two adjacent heaps", false);
+    }
+    const auto mirrored = AgcDriver::Graphics::HeapMirrorsWithin(begin, end);
+    Require(before.mirrors == 0 && mirrored.mirrors == 2 && mirrored.bytes == 4 * Block, "resident heap mirror: two adjacent heaps hold " + std::to_string(before.mirrors) + " heap mirrors before the first dispatch and " + std::to_string(mirrored.mirrors) + " (" + std::to_string(mirrored.bytes >> 10u) + " KiB) after it, expected 0 and 2 (256 KiB); " + std::to_string(AgcDriver::Graphics::HostImportLiveBytes() >> 10u) + " KiB host imports live");
+    const auto live = AgcDriver::Graphics::HostImportLiveBytes();
+    Copy(device, begin + JoinedBytes - Threads * sizeof(std::uint32_t) / 2, output);
+    Require(output.Data()[Threads / 2 - 1] == 0x41u && output.Data()[Threads / 2] == 0x42u, "resident heap mirror: a view across two adjacent heaps read " + Hex(output.Data()[Threads / 2 - 1]) + " " + Hex(output.Data()[Threads / 2]) + ", expected 0x41 0x42");
+    const auto joined = AgcDriver::Graphics::HostImportLiveBytes() - live;
+    Require(joined >= 2 * JoinedBytes, "resident heap mirror: a view across two adjacent heaps made no joined host import (" + std::to_string(joined >> 10u) + " KiB imported)");
+    Dispatch(device, begin, begin + JoinedBytes, output);
+    CheckReads(output, "after the joined import", false);
+    const auto released = AgcDriver::Graphics::HeapMirrorsWithin(begin, end);
+    Require(released.mirrors == 0 && released.bytes == 0, "resident heap mirror: a joined host import over two heap-mirrored heaps left " + std::to_string(released.mirrors) + " of their " + std::to_string(mirrored.mirrors) + " heap mirrors and " + std::to_string(released.bytes >> 10u) + " of " + std::to_string(mirrored.bytes >> 10u) + " KiB held");
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+#ifdef _WIN32
+        const bool watched = GuestArena::GuestArenaWriteWatched_nid_postfix();
+#else
+        const bool watched = GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix();
+#endif
+        if (!watched) {
+            std::puts("skipped, no guest write watch: heap mirrors need one");
+            return VulkanTestSkipped;
+        }
+        ResidentTests(*device);
+        ReusedAddressTests(*device);
+        JoinedImportTests(*device);
+        CopiedTests(*device);
+        std::puts("resident heap mirror tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceLibreSsl/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_library(libSceLibreSsl SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Digest.cpp)
 target_include_directories(libSceLibreSsl PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(libSceLibreSsl PRIVATE mbedcrypto)
 configure_windows_unwind(libSceLibreSsl)
 
 if(BUILD_TESTING)
@@ -13,4 +14,15 @@ if(BUILD_TESTING)
         set_tests_properties(guest_libressl_digest PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceLibreSsl>")
     endif()
+
+    add_test_executable(guest_libressl_des_tests ${LIBS_INCLUDE_DIR}/tests/GuestLibreSslDes.cpp)
+    target_include_directories(guest_libressl_des_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_libressl_des_tests PRIVATE libSceLibreSsl libc)
+    configure_windows_unwind(guest_libressl_des_tests)
+    add_test(NAME guest_libressl_des COMMAND guest_libressl_des_tests)
+    set_tests_properties(guest_libressl_des PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_libressl_des PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceLibreSsl>")
+    endif()
 endif()

core/libs/prx/libSceLibreSsl/Export.cpp

@@ -1,7 +1,17 @@
 #include <cstddef>
 #include <cstdint>
+#include <cstring>
+#include <stdexcept>
+#include <string>
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
+#include "mbedtls/des.h"
+
+namespace {
+
+constexpr std::size_t DesScheduleBytes = 128;
+
+}
 
 extern "C" {
 
@@ -301,19 +311,28 @@ int APS5_VABI CRYPTO_set_mem_functions_nid_postfix(void) {
     return 0;
 }
 
-int APS5_VABI DES_ecb_encrypt_nid_postfix(void) {
-    NotImplemented_nid_no_patch(__func__);
+void APS5_VABI DES_ecb_encrypt_nid_postfix(const unsigned char* input, unsigned char* output, const unsigned char* schedule, int enc) {
+    if (input == nullptr || output == nullptr || schedule == nullptr) APS5_INVALID_ARG_EX;
+    mbedtls_des_context context;
+    mbedtls_des_init(&context);
+    unsigned char block[MBEDTLS_DES_KEY_SIZE];
+    std::memcpy(block, input, sizeof(block));
+    int status = enc ? mbedtls_des_setkey_enc(&context, schedule) : mbedtls_des_setkey_dec(&context, schedule);
+    if (status == 0) status = mbedtls_des_crypt_ecb(&context, block, output);
+    mbedtls_des_free(&context);
+    if (status != 0) throw std::runtime_error("DES_ecb_encrypt: mbedtls failed");
+}
+
+int APS5_VABI DES_set_key_nid_postfix(const unsigned char* key, unsigned char* schedule) {
+    if (key == nullptr || schedule == nullptr) APS5_INVALID_ARG_EX;
+    std::memset(schedule, 0, DesScheduleBytes);
+    std::memcpy(schedule, key, MBEDTLS_DES_KEY_SIZE);
     return 0;
 }
 
-int APS5_VABI DES_set_key_nid_postfix(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
-
-int APS5_VABI DES_set_odd_parity_nid_postfix(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+void APS5_VABI DES_set_odd_parity_nid_postfix(unsigned char* key) {
+    if (key == nullptr) APS5_INVALID_ARG_EX;
+    mbedtls_des_key_set_parity(key);
 }
 
 int APS5_VABI DTLSv1_client_method_nid_postfix(void) {

core/libs/tests/GuestLibreSslDes.cpp

@@ -0,0 +1,75 @@
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <stdexcept>
+
+using DesBlock = std::array<unsigned char, 8>;
+using DesSchedule = std::array<unsigned char, 128>;
+
+extern "C" {
+void APS5_VABI DES_ecb_encrypt_nid_postfix(const unsigned char*, unsigned char*, const unsigned char*, int);
+int APS5_VABI DES_set_key_nid_postfix(const unsigned char*, unsigned char*);
+void APS5_VABI DES_set_odd_parity_nid_postfix(unsigned char*);
+}
+
+static void Require(bool value) { if (!value) std::abort(); }
+
+template<typename TAction>
+static void RequireThrows(TAction action) {
+    try { action(); } catch (const std::invalid_argument&) { return; }
+    std::abort();
+}
+
+static DesBlock Block(std::uint64_t value) {
+    DesBlock block{};
+    for (int index = 0; index < 8; ++index) block[index] = static_cast<unsigned char>(value >> (56 - 8 * index));
+    return block;
+}
+
+static void CheckVector(std::uint64_t key, std::uint64_t plain, std::uint64_t cipher) {
+    const auto keyBlock = Block(key);
+    DesSchedule schedule{};
+    schedule.fill(0xaa);
+    Require(DES_set_key_nid_postfix(keyBlock.data(), schedule.data()) == 0);
+    const auto plainBlock = Block(plain);
+    DesBlock output{};
+    DES_ecb_encrypt_nid_postfix(plainBlock.data(), output.data(), schedule.data(), 1);
+    Require(output == Block(cipher));
+    DES_ecb_encrypt_nid_postfix(output.data(), output.data(), schedule.data(), 0);
+    Require(output == plainBlock);
+    DesBlock again{};
+    DES_ecb_encrypt_nid_postfix(plainBlock.data(), again.data(), schedule.data(), 7);
+    Require(again == Block(cipher));
+}
+
+static void CheckParity() {
+    DesBlock key{0x00, 0x01, 0x02, 0x03, 0x12, 0x13, 0xfe, 0xff};
+    DES_set_odd_parity_nid_postfix(key.data());
+    Require(key == (DesBlock{0x01, 0x01, 0x02, 0x02, 0x13, 0x13, 0xfe, 0xfe}));
+    for (int value = 0; value < 256; ++value) {
+        DesBlock single{};
+        single.fill(static_cast<unsigned char>(value));
+        DES_set_odd_parity_nid_postfix(single.data());
+        Require((single[0] & 0xfe) == (value & 0xfe) && std::popcount(single[0]) % 2 == 1);
+    }
+}
+
+int main() {
+    CheckVector(0x133457799bbcdff1, 0x0123456789abcdef, 0x85e813540f0ab405);
+    CheckVector(0x0e329232ea6d0d73, 0x8787878787878787, 0x0000000000000000);
+    CheckVector(0x0101010101010101, 0x8000000000000000, 0x95f8a5e5dd31d900);
+    CheckParity();
+    DesSchedule schedule{};
+    const auto weak = Block(0x0101010101010101);
+    Require(DES_set_key_nid_postfix(weak.data(), schedule.data()) == 0);
+    const auto badParity = Block(0x0000000000000000);
+    Require(DES_set_key_nid_postfix(badParity.data(), schedule.data()) == 0);
+    RequireThrows([&] { DES_set_key_nid_postfix(nullptr, schedule.data()); });
+    RequireThrows([&] { DES_ecb_encrypt_nid_postfix(weak.data(), schedule.data(), nullptr, 1); });
+    RequireThrows([] { DES_set_odd_parity_nid_postfix(nullptr); });
+    return 0;
+}

docs/dev/ENVIRONMENT.md

@@ -224,6 +224,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_NO_DRAW_RESOURCE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | builds every draw's shader resources instead of reusing a cached identical build |
 | `APS5_NO_DRAW_SRT_ENTRIES` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` |  | disables the draw cache's SRT entries, so every draw decodes its programs |
 | `APS5_NO_DRAW_STAGE_DATA_HITS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` |  | disables draw-cache hits that differ only in a stage's ShaderData words |
+| `APS5_NO_EMPTY_INDIRECT_SKIP` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp` |  | builds the stages of an indirect draw whose front program last took the CPU path even when none of its records draws anything, instead of reading its records first and skipping it |
 | `APS5_NO_EPOCH_COLLECT_MAP` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp` |  | drops the per-epoch map of completed collects, leaving only the fixed-size thread memo to answer a repeated collect |
 | `APS5_NO_EPOCH_REVALIDATE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` |  | runs every Revalidate's per-element and per-region checks instead of the epoch gate |
 | `APS5_NO_FAST_REVALIDATE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` |  | always repeats the resource lookups instead of using the fast proof |

docs/dev/debt/libSceLibreSsl.md

@@ -2,5 +2,6 @@
 
 ### Unknown function info
 
-- [libSceLibreSsl pending exports](../../../core/libs/prx/libSceLibreSsl/Export.cpp) - 377 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceLibreSsl.c) and were cross-checked against the same module in PS5 firmware 13.60. Their signatures and behaviour are unverified; the declarations use the no-parameter unknown-signature convention and throw through `NotImplemented_nid_no_patch`.
+- [libSceLibreSsl pending exports](../../../core/libs/prx/libSceLibreSsl/Export.cpp) - 374 named callable exports are listed in the [PS5 3.20 export table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceLibreSsl.c) and were cross-checked against the same module in PS5 firmware 13.60. Their signatures and behaviour are unverified; the declarations use the no-parameter unknown-signature convention and throw through `NotImplemented_nid_no_patch`.
 - [MD4, MD5, SHA1, SHA224, SHA256, SHA384 and SHA512 Init/Update/Final and SHA256](../../../core/libs/prx/libSceLibreSsl/src/Digest.cpp) (libSceLibreSsl) - signatures and context layouts follow the [LibreSSL headers](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/sha/sha.h), and buffering and `Final` follow its [sources](https://github.com/libressl/openbsd/tree/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/sha); the module's LibreSSL version is unknown and nothing is checked on the console. Behaviour that differs between LibreSSL versions throws: `SHA256` with a null digest buffer (older versions return a static buffer) and `SHA512_Final` with the SHA-512/224 or SHA-512/256 digest length. Null contexts, null input with a nonzero length, null digest buffers in the 32-bit `Final` functions and a buffered byte count that is not below the block size, which LibreSSL does not check, throw
+- [DES_set_key, DES_ecb_encrypt, DES_set_odd_parity](../../../core/libs/prx/libSceLibreSsl/Export.cpp) (libSceLibreSsl) - the 128-byte `DES_key_schedule` holds the 8-byte key followed by zeros instead of LibreSSL's precomputed subkeys, so a title reading its contents sees different bytes; `DES_ecb_encrypt` derives the subkeys through mbedtls on each call. Not compared with the console. Results follow LibreSSL [des_key.c](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/des/des_key.c) and [des.c](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/des/des.c): `DES_set_key` skips the parity and weak-key checks and returns 0, as it does while `DES_check_key` is 0 (that variable is not in the PS5 export inventory), a nonzero `enc` encrypts, and `DES_set_odd_parity` sets each byte's low bit for odd parity. Null pointers throw where LibreSSL faults