Merge main into fix/agc-sample-location-grid-exact

#4650 took request serialization version 35 for its depth remap flag, so this branch's sample order moves to 36. The serialization tests drop #4650's trailing byte from every older version they build, as main's do, along with the iteration fields and the sample order.

CMakeLists.txt

@@ -113,6 +113,10 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ShaderCacheVersion.cmake ${CMAKE_COMMAND})
         set_tests_properties(shader_cache_native_paths PROPERTIES TIMEOUT 20)
+        add_test(NAME shader_cache_runtime_sources COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND})
+        set_tests_properties(shader_cache_runtime_sources PROPERTIES TIMEOUT 180)
         add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
         add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE}

core/libs/prx/libSceAgc/DcbFlow/include/Control.hpp

@@ -1,4 +1,12 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGC_DCBFLOW_INCLUDE_CONTROL_HPP
 #define CORE_LIBS_PRX_LIBSCEAGC_DCBFLOW_INCLUDE_CONTROL_HPP
 
+#include "SceTypes.hpp"
+#include <cstdint>
+
+extern "C" {
+std::uint32_t* APS5_VABI sceAgcDcbAtomicGds(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData);
+std::uint32_t* APS5_VABI sceAgcDcbAtomicGds_0900(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData);
+}
+
 #endif

core/libs/prx/libSceAgc/DcbFlow/include/Memory.hpp

@@ -2,7 +2,11 @@
 #define CORE_LIBS_PRX_LIBSCEAGC_DCBFLOW_INCLUDE_MEMORY_HPP
 
 #include "SceTypes.hpp"
+#include <cstdint>
 
-extern "C" std::uint32_t* APS5_VABI sceAgcDcbAcquireMem(CommandBuffer* buf, std::uint8_t engine, std::uint32_t cbDbOp, std::uint32_t gcrControl, const volatile void* base, std::uint64_t sizeBytes, std::uint32_t pollCycles);
+extern "C" {
+std::uint32_t* APS5_VABI sceAgcDcbAcquireMem(CommandBuffer* buf, std::uint8_t engine, std::uint32_t cbDbOp, std::uint32_t gcrControl, const volatile void* base, std::uint64_t sizeBytes, std::uint32_t pollCycles);
+std::uint32_t APS5_VABI sceAgcDcbAtomicGdsGetSize();
+}
 
 #endif

core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp

@@ -12,10 +12,19 @@
 
 extern "C" {
 
-// unknown signature
-void* APS5_VABI sceAgcDcbAtomicGds_0900(void) {
- NotImplemented_nid_no_patch(__func__);
- return nullptr;
+std::uint32_t* APS5_VABI sceAgcDcbAtomicGds_0900(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData) {
+    const auto operation = (control & 0xffu) | ((control & 0x200u) << 7u) |
+        ((control & 0x20u) << 13u) | ((control & 0x40u) << 14u) |
+        ((control & 0x20u) == 0 && mode == 1 ? 0x20000u : 0u);
+    return Agc::Command::Emit(buf, 0x1du, {
+        operation, ((gdsOp & 1u) << 8u) | (src & 0x3fu), offset, index,
+        mask & 0x00ff00ffu, data, static_cast<std::uint32_t>(compareData),
+        static_cast<std::uint32_t>(compareData >> 32u), static_cast<std::uint32_t>(extraData),
+        static_cast<std::uint32_t>(extraData >> 32u)}, __func__);
+}
+
+std::uint32_t* APS5_VABI sceAgcDcbAtomicGds(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData) {
+    return sceAgcDcbAtomicGds_0900(buf, control, mode, gdsOp, src, offset, index, mask, data, compareData, extraData);
 }
 
 

core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp

@@ -53,7 +53,7 @@ std::uint32_t APS5_VABI sceAgcDcbAtomicMemGetSize() {
 }
 
 std::uint32_t APS5_VABI sceAgcDcbAtomicGdsGetSize() {
-    return 36;
+    return 44;
 }
 
 std::uint32_t* APS5_VABI sceAgcDcbWriteData(CommandBuffer* buf, std::uint8_t dst, std::uint8_t cachePolicy, std::uint64_t address, const void* data, std::uint32_t numDwords, std::uint8_t increment, std::uint8_t writeConfirm) {

core/libs/prx/libSceAgc/DcbState/include/ContextState.hpp

@@ -1,4 +1,17 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGC_DCBSTATE_INCLUDE_CONTEXTSTATE_HPP
 #define CORE_LIBS_PRX_LIBSCEAGC_DCBSTATE_INCLUDE_CONTEXTSTATE_HPP
 
+#include "SceTypes.hpp"
+#include <cstdint>
+
+struct CommandBuffer;
+
+extern "C" {
+
+uint32_t* APS5_VABI sceAgcDcbContextStateOp(CommandBuffer* buf, uint32_t operation);
+uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, uint32_t operation);
+uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(uint32_t operation);
+
+}
+
 #endif

core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp

@@ -9,10 +9,7 @@
 extern "C" {
 
 uint32_t* APS5_VABI sceAgcDcbContextStateOp(CommandBuffer* buf, uint32_t operation) {
- (void)buf;
- (void)operation;
- NotImplemented_nid_no_patch(__func__);
- return nullptr;
+    return sceAgcDcbContextStateOp_0100(buf, operation);
 }
 
 uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(uint32_t operation) {

core/libs/prx/libSceAgc/tests/ContextStateSize.cpp

@@ -8,6 +8,7 @@
 #include <stdexcept>
 
 extern "C" std::uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(std::uint32_t operation);
+extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateOp(CommandBuffer* buf, std::uint32_t operation);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, std::uint32_t operation);
 
 namespace {
@@ -37,6 +38,15 @@ void testSizes() {
         check(sceAgcDcbContextStateOp_0100(&buffer, operation) == words.data(), "context state op did not start at the cursor");
         const auto written = static_cast<std::uint64_t>(buffer.cursor_up - words.data()) * sizeof(std::uint32_t);
         check(written == sizes[operation], "context state size differs from the written packets");
+
+        std::array<std::uint32_t, 64> unversionedWords{};
+        CommandBuffer unversionedBuffer{unversionedWords.data(), unversionedWords.data() + unversionedWords.size(),
+                                        unversionedWords.data(), unversionedWords.data() + unversionedWords.size(),
+                                        nullptr, nullptr, 0};
+        check(sceAgcDcbContextStateOp(&unversionedBuffer, operation) == unversionedWords.data(), "context state op did not start at the cursor");
+        const auto unversionedWritten = static_cast<std::uint64_t>(unversionedBuffer.cursor_up - unversionedWords.data()) * sizeof(std::uint32_t);
+        check(unversionedWritten == sizes[operation], "context state size differs from the written packets");
+        check(words == unversionedWords, "unversioned context state op mismatch with versioned op");
     }
 }
 

core/libs/prx/libSceAgc/tests/EndOfShaderActionSize.cpp

@@ -2,6 +2,7 @@
 #include "prx/libc/include/general/VabiMacros.hpp"
 #include "prx/libc/include/Shutdown.hpp"
 
+#include <algorithm>
 #include <array>
 #include <cstdint>
 #include <cstdio>
@@ -11,6 +12,9 @@ extern "C" {
 std::uint32_t APS5_VABI sceAgcDcbQueueEndOfShaderActionGetSize();
 std::uint32_t APS5_VABI sceAgcAcbQueueEndOfShaderActionGetSize();
 std::uint32_t* APS5_VABI sceAgcCbReleaseMem(CommandBuffer*, std::uint8_t, std::uint16_t, std::uint8_t, std::uint8_t, const volatile Label*, std::uint8_t, std::uint64_t, std::uint16_t, std::uint16_t, std::uint8_t, std::uint32_t);
+std::uint32_t APS5_VABI sceAgcDcbAtomicGdsGetSize();
+std::uint32_t* APS5_VABI sceAgcDcbAtomicGds_0900(CommandBuffer*, std::uint32_t, std::uint32_t, std::uint32_t, std::uint32_t, std::uint16_t, std::uint16_t, std::uint32_t, std::uint32_t, std::uint64_t, std::uint64_t);
+std::uint32_t* APS5_VABI sceAgcDcbAtomicGds(CommandBuffer*, std::uint32_t, std::uint32_t, std::uint32_t, std::uint32_t, std::uint16_t, std::uint16_t, std::uint32_t, std::uint32_t, std::uint64_t, std::uint64_t);
 }
 
 namespace {
@@ -19,19 +23,43 @@ void check(bool condition, const char* message) {
     if (!condition) throw std::runtime_error(message);
 }
 
+void testEndOfShaderActionSize() {
+    std::array<std::uint32_t, 16> words{};
+    CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+    const auto* packet = sceAgcCbReleaseMem(&buffer, 0x2f, 0, 0, 0, nullptr, 0, 0, 0, 0, 0, 0);
+    const auto releaseBytes = static_cast<std::uint32_t>((buffer.cursor_up - packet) * sizeof(std::uint32_t));
+    check(((packet[0] >> 8u) & 0xffu) == 0x49u && releaseBytes == 32, "end-of-shader RELEASE_MEM size mismatch");
+    check(sceAgcDcbQueueEndOfShaderActionGetSize() == releaseBytes, "DCB end-of-shader action size differs from its RELEASE_MEM");
+    check(sceAgcDcbQueueEndOfShaderActionGetSize() == sceAgcAcbQueueEndOfShaderActionGetSize(), "DCB and ACB end-of-shader action sizes differ");
+}
+
+void testAtomicGds() {
+    constexpr std::uint32_t expectedBytes = 11u * sizeof(std::uint32_t);
+    check(sceAgcDcbAtomicGdsGetSize() == expectedBytes, "DCB atomic GDS size does not match 11-dword packet");
+
+    std::array<std::uint32_t, 32> words{};
+    CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+    const auto* packet = sceAgcDcbAtomicGds_0900(&buffer, 0x261u, 1, 3, 7, 0x100, 0x200, 0x00ff00ffu, 0x12345678u, 0x1122334455667788ull, 0x99aabbccddeeff00ull);
+    check(packet != nullptr && packet == words.data(), "DCB atomic GDS returned unexpected packet pointer");
+    check(buffer.cursor_up == words.data() + 11, "DCB atomic GDS cursor advance does not match 11 dwords");
+    check(packet[0] == 0xc0091d00u, "DCB atomic GDS header opcode mismatch");
+
+    std::array<std::uint32_t, 32> unversionedWords{};
+    CommandBuffer unversionedBuffer{unversionedWords.data(), unversionedWords.data() + unversionedWords.size(), unversionedWords.data(), unversionedWords.data() + unversionedWords.size(), nullptr, nullptr, 0};
+    const auto* unversionedPacket = sceAgcDcbAtomicGds(&unversionedBuffer, 0x261u, 1, 3, 7, 0x100, 0x200, 0x00ff00ffu, 0x12345678u, 0x1122334455667788ull, 0x99aabbccddeeff00ull);
+    check(unversionedPacket != nullptr && unversionedPacket == unversionedWords.data(), "unversioned DCB atomic GDS returned unexpected packet pointer");
+    check(unversionedBuffer.cursor_up == unversionedWords.data() + 11, "unversioned DCB atomic GDS cursor advance does not match 11 dwords");
+    check(std::equal(words.begin(), words.begin() + 11, unversionedWords.begin()), "versioned and unversioned DCB atomic GDS emitted different packets");
+}
+
 }
 
 int main() {
     try {
-        std::array<std::uint32_t, 16> words{};
-        CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
-        const auto* packet = sceAgcCbReleaseMem(&buffer, 0x2f, 0, 0, 0, nullptr, 0, 0, 0, 0, 0, 0);
-        const auto releaseBytes = static_cast<std::uint32_t>((buffer.cursor_up - packet) * sizeof(std::uint32_t));
-        check(((packet[0] >> 8u) & 0xffu) == 0x49u && releaseBytes == 32, "end-of-shader RELEASE_MEM size mismatch");
-        check(sceAgcDcbQueueEndOfShaderActionGetSize() == releaseBytes, "DCB end-of-shader action size differs from its RELEASE_MEM");
-        check(sceAgcDcbQueueEndOfShaderActionGetSize() == sceAgcAcbQueueEndOfShaderActionGetSize(), "DCB and ACB end-of-shader action sizes differ");
+        testEndOfShaderActionSize();
+        testAtomicGds();
         LibcRunShutdown_nid_postfix();
-        std::puts("AGC end-of-shader action size tests passed");
+        std::puts("AGC end-of-shader action and atomic GDS size tests passed");
         return 0;
     } catch (const std::exception& error) {
         std::fprintf(stderr, "%s\n", error.what());

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -1212,8 +1212,12 @@ configure_windows_unwind(agc_driver_recorder_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_driver_recorder_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
     add_test(NAME agc_recorder_device_selection COMMAND agc_driver_recorder_tests --device-selection-only)
+    add_test(NAME agc_gpu_timing_complete COMMAND agc_driver_recorder_tests --gpu-timing-complete-only)
+    add_test(NAME agc_gpu_timing_incomplete COMMAND agc_driver_recorder_tests --gpu-timing-incomplete-only)
+    set_tests_properties(agc_gpu_timing_complete agc_gpu_timing_incomplete PROPERTIES
+        ENVIRONMENT "APS5_PROFILE_GPU=1" TIMEOUT 15 SKIP_RETURN_CODE 77)
     if(WIN32)
-        set_tests_properties(agc_recorder_device_selection PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_recorder_device_selection agc_gpu_timing_complete agc_gpu_timing_incomplete PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_pipeline_cache COMMAND agc_driver_recorder_tests --pipeline-cache-only)
@@ -1285,6 +1289,12 @@ if(BUILD_TESTING)
         set_tests_properties(agc_storage_baseline_settled PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_read_written_staging COMMAND agc_driver_recorder_tests --read-written-staging-only)
+    set_tests_properties(agc_read_written_staging PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_read_written_staging PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_storage_written_buffer_order COMMAND agc_driver_recorder_tests --written-buffer-order-only)
     set_tests_properties(agc_storage_written_buffer_order PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     if(WIN32)
@@ -1297,6 +1307,12 @@ if(BUILD_TESTING)
         set_tests_properties(agc_storage_split_writeback PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_storage_write_back COMMAND agc_driver_recorder_tests --storage-write-back-only)
+    set_tests_properties(agc_storage_write_back PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120 ENVIRONMENT "APS5_PIN_WAIT_MS=3000")
+    if(WIN32)
+        set_tests_properties(agc_storage_write_back PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_copied_direct_ordering COMMAND agc_driver_recorder_tests --copied-direct-only)
     set_tests_properties(agc_copied_direct_ordering PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     if(WIN32)
@@ -1589,6 +1605,15 @@ target_link_libraries(agc_driver_graphics_tests PRIVATE
         libkernel
 )
 
+if(BUILD_TESTING)
+    add_test(NAME agc_command_batch_fence_failure COMMAND agc_driver_graphics_tests --command-batch-fence-failure)
+    set_tests_properties(agc_command_batch_fence_failure PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(agc_command_batch_fence_failure PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 set_target_properties(agc_driver_graphics_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_driver_graphics_tests)
 
@@ -1707,6 +1732,10 @@ target_include_directories(agc_driver_alpha_to_coverage_tests PRIVATE $<TARGET_P
 
 target_include_directories(agc_driver_sample_multisample_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_image_table_cache_bounds_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
+target_include_directories(agc_driver_draw_cache_relocation_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 if(BUILD_TESTING)
     add_test(NAME agc_driver_store_pending_identity COMMAND agc_driver_store_over_untracked_tests --pending-identity)
     set_tests_properties(agc_driver_store_pending_identity PROPERTIES
@@ -1769,6 +1798,8 @@ target_include_directories(agc_driver_color1555_target_tests PRIVATE $<TARGET_PR
 
 target_include_directories(agc_driver_color565_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_depth_range_remap_draw_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 foreach(stage IN ITEMS vert frag)
     set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage})
     set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv)
@@ -1870,6 +1901,15 @@ if(BUILD_TESTING)
     set_tests_properties(agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_srt_read_patching COMMAND agc_shader_memory_tests --srt-read-patching)
     set_tests_properties(agc_shader_srt_read_patching PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_shader_table_snapshot_reuse COMMAND agc_shader_memory_tests --table-snapshot-reuse)
+    set_tests_properties(agc_shader_table_snapshot_reuse PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups)
+    set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60)
+    add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws)
+    set_tests_properties(agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60)
+    if(WIN32)
+        set_tests_properties(agc_shader_table_snapshot_reuse agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_shader_type_zero_image_materialization COMMAND agc_shader_memory_tests --type-zero-image-materialization)
     set_tests_properties(agc_shader_type_zero_image_materialization PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_frontend_pair COMMAND agc_shader_memory_tests --frontend-pair)
@@ -2086,6 +2126,12 @@ if(BUILD_TESTING)
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_tests_properties(agc_driver_provoking_vertex PROPERTIES TIMEOUT 30)
+    add_test(NAME agc_driver_dword_pattern_fill_translated COMMAND agc_driver_dword_pattern_fill_tests)
+    set_tests_properties(agc_driver_dword_pattern_fill_translated PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_FILL_HLE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_dword_pattern_fill_translated PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_uint8x4_color_target_no_resident COMMAND agc_driver_uint8x4_color_target_tests)
     set_tests_properties(agc_driver_uint8x4_color_target_no_resident PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_RESIDENT_TARGETS=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
     if(WIN32)
@@ -2270,9 +2316,12 @@ if(BUILD_TESTING)
             endif()
         endforeach()
     endforeach()
-    foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync lazy lazy_sync)
+    foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync abi_deferred_compute abi_deferred_pixel abi_deferred_stages abi_deferred_rectangle abi_deferred_undecodable_state abi_deferred_registration_failure abi_deferred_vertex_inputs abi_deferred_rectangle_pairs lazy lazy_sync)
         set(preparedTest "agc_driver_prepared_shaders_${preparedMode}")
-        if(preparedMode STREQUAL "indirect")
+        if(preparedMode MATCHES "^abi_deferred_")
+            string(REPLACE "_" "-" preparedArgument ${preparedMode})
+            add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --${preparedArgument})
+        elseif(preparedMode STREQUAL "indirect")
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --indirect)
         elseif(preparedMode STREQUAL "deferred")
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred)
@@ -2289,7 +2338,9 @@ if(BUILD_TESTING)
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests)
         endif()
         set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
-        if(preparedMode STREQUAL "uncached")
+        if(preparedMode MATCHES "^abi_deferred_")
+            set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_NO_FAILURE_MEMO=1")
+        elseif(preparedMode STREQUAL "uncached")
             set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_NO_DISPATCH_CACHE=1")
         elseif(preparedMode STREQUAL "registered_workgroup_sync")
             set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_SYNC_SHADER_PREPARE=1")
@@ -2840,6 +2891,20 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_depth_range_remap_tests EXCLUDE_FROM_ALL tests/DepthRangeRemap.cpp)
+target_link_libraries(agc_depth_range_remap_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_depth_range_remap_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_depth_range_remap_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_depth_range_remap_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_depth_range_remap COMMAND agc_depth_range_remap_tests)
+    set_tests_properties(agc_depth_range_remap PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1")
+    if(WIN32)
+        set_tests_properties(agc_depth_range_remap PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_position_invariance_tests EXCLUDE_FROM_ALL tests/PositionInvariance.cpp)
 target_link_libraries(agc_position_invariance_tests PRIVATE shader_recompiler_agc_driver libc)
 set_target_properties(agc_position_invariance_tests PROPERTIES CXX_EXTENSIONS OFF)

core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp

@@ -390,9 +390,7 @@ inline void InstallAsyncLog_nid_no_patch() {
     static_cast<void>(AsyncLog::Get());
 }
 
-inline void FlushAsyncLog_nid_no_patch() {
-    AsyncLog::Get().Flush();
-}
+void FlushAsyncLog_nid_no_patch();
 
 inline void FlushAsyncLogAtEnd_nid_no_patch() {
     AsyncLog::FlushAtEnd();

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp

@@ -27,6 +27,7 @@ struct DispatchVariant {
 
     std::vector<std::pair<std::uint64_t, std::uint64_t>> runs;
     std::vector<std::uint32_t> words;
+    std::vector<SharedWords> shared;
     std::uint64_t forgetSerial = 0;
     std::shared_ptr<const ShaderRecompiler::RecompileResult> compiled;
     std::array<std::uint32_t, 4> meshIndexDescriptor{};

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -53,7 +53,7 @@ struct PreparedGraphicsStage {
     std::shared_ptr<const ShaderRecompiler::RecompileResult> registration;
 };
 
-std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target);
+std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared = false);
 
 struct DrawRecipeRecord {
     std::vector<std::weak_ptr<const DispatchVariant>> stages;
@@ -118,6 +118,7 @@ struct StageCapture {
     std::shared_ptr<const ShaderRecompiler::RecompileResult> compiled;
     std::shared_ptr<const ShaderRecompiler::ResourceCapture> capture;
     std::vector<ShaderRecompiler::MemoryRegion> regions;
+    std::vector<std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> supplied;
     std::uint64_t forgetSerial = 0;
     std::uint32_t pushOffset = 0;
     ShaderRecompiler::SrtUserDataUse userDataUse;

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -53,7 +53,8 @@ public:
     std::uint64_t DispatchesAdoptedAhead() const;
     void CheckFailure();
     void ReportFailure(std::exception_ptr error);
-    void Submit(const Packet* packet, std::uint32_t queue);
+    void Submit(const Packet* packet, std::uint32_t queue, bool cpuFlip = false);
+    void SubmitFlip(const FlipInfo& info);
     void SuspendPoint();
     std::uint32_t ReserveEopFlip(const FlipInfo& info);
     void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr<IVideoOutput>& output);
@@ -68,6 +69,7 @@ public:
     void SettleShaderPreparations(std::span<const Shader* const> stages);
     static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters);
     static std::optional<Graphics::IndirectDrawPath> ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect);
+    static void tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords, std::span<const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> supplied = {});
     static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request, std::string_view suffix = {});
     DrawEntryCounters DrawCacheCounters();
     struct DispatchCacheFootprint {
@@ -76,6 +78,7 @@ public:
     };
     DispatchCacheFootprint DispatchCacheSize();
     static std::uint64_t DispatchCacheBytes();
+    static ShaderMemory::PendingWrite QueryPendingSpan(std::uint64_t address, std::size_t bytes, std::span<std::byte> known);
     std::shared_ptr<const ShaderRegistry> RegisteredShaders();
 
 private:
@@ -151,15 +154,14 @@ private:
     static std::uint64_t drawIdentityKey(const QueueState& queue, const ShaderRegistry& registry);
     static bool drawRelocationEnabled();
     static std::uint64_t relocatedContentKey(const RelocatableUserWord& word, std::uint32_t value, bool& readable);
-    static void tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords);
     static void applyShaderDataPatches(DispatchVariant& variant);
     void learnPendingRelocation();
     void relocateDrawEntry(const QueueState& queue, std::uint64_t alias, std::shared_ptr<DrawEntry>& entry, std::shared_ptr<const DrawDecode>& decode, std::vector<DrawUserPatch>& patches);
     void reportDrawRelocation();
     static bool sameVertexInfo(const ShaderRecompiler::ShaderVertexStageInfo& a, const ShaderRecompiler::ShaderVertexStageInfo& b);
     static bool sameDecode(const DrawDecode& a, const DrawDecode& b);
-    std::shared_ptr<DrawDecode> decodeDraw(const QueueState& queue, const Submission& submission);
-    void resolveDrawDecode(const QueueState& queue, const Submission& submission, std::shared_ptr<const DrawDecode>& decode, bool registerKey, std::uint64_t drawKey, bool profile);
+    std::shared_ptr<DrawDecode> decodeDraw(const QueueState& queue, const Submission& submission, const Graphics::Context& context);
+    void resolveDrawDecode(const QueueState& queue, const Submission& submission, const Graphics::Context& context, std::shared_ptr<const DrawDecode>& decode, bool registerKey, std::uint64_t drawKey, bool profile);
     void lookupDraw(const Submission& submission, const std::shared_ptr<VulkanDevice>& localDevice, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector<DrawProgram>& programs, const std::vector<ShaderRecompiler::ProgramRole>& roles, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, bool useDrawEntries, bool registerKey, bool profile, std::uint64_t& drawKey, std::shared_ptr<DrawEntry>& entry, std::vector<std::shared_ptr<DispatchVariant>>& matched, std::vector<std::vector<ShaderRecompiler::MemoryRegion>>& matchedRegions, bool& drawHit, bool& verifyHit, DrawPhaseTiming& phaseTiming, std::array<double, DrawDriverPhaseCount>& phaseMs, const std::shared_ptr<DrawEntry>& dataEntry, DrawDataCandidates& dataCandidates, std::vector<StageDataHit>& stageData);
     std::shared_ptr<const ShaderRecompiler::RecompileResult> materializeDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, const std::vector<DrawProgram>& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, std::vector<ShaderRecompiler::MemoryRegion>& memory, const std::vector<ShaderRecompiler::LinkedProgram>& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr<VulkanDevice>& localDevice, ShaderMemory& shaderMemory, std::vector<StageCapture>& stageCaptures, std::vector<bool>& recompiled, bool drawHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, const std::vector<std::vector<ShaderRecompiler::MemoryRegion>>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array<double, DrawDriverPhaseCount>& phaseMs, std::string& rejected);
     static void recordStageDataPositions(DispatchVariant& variant, const ShaderRecompiler::ResourceCapture& capture, const std::vector<Graphics::DecodeRead>& decodeReads);
@@ -208,6 +210,7 @@ private:
     static std::uint64_t hookWaits();
     static ShaderMemory::HookWaitCounter hookWaitCounter();
     static bool validateSkipEnabled();
+    static bool vouchedValidation();
     static bool validateLegacy();
     static bool gateRetry();
     static bool validateSkipVerify();
@@ -242,10 +245,10 @@ private:
     void traceCapture(const char* what, std::uint64_t program, std::uint32_t queue, std::span<const ShaderRecompiler::MemoryRegion> regions, double waitedMs);
     void reportValidation(ValidateCounters& counters);
     bool captureStable(std::span<const ShaderRecompiler::MemoryRegion> captured);
-    bool validateCaptured(std::uint64_t program, std::uint32_t queue, std::span<const ShaderRecompiler::MemoryRegion> captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped = nullptr, std::optional<SampledReadScope>* sampling = nullptr, const DataMask* data = nullptr);
+    bool validateCaptured(std::uint64_t program, std::uint32_t queue, std::span<const ShaderRecompiler::MemoryRegion> captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped = nullptr, std::optional<SampledReadScope>* sampling = nullptr, const DataMask* data = nullptr, std::span<const VouchedRegions> vouched = {});
     static void appendEntryRegions(const DispatchVariant& variant, std::vector<ShaderRecompiler::MemoryRegion>& regions, const std::vector<std::uint32_t>* words = nullptr);
     static std::shared_ptr<const ShaderRecompiler::RecompileResult> patchDataWords(const DispatchVariant& variant, const std::vector<std::uint32_t>& words);
-    bool syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span<const ShaderRecompiler::MemoryRegion> regions, std::uint64_t& synced, PendingView& pending, std::optional<SampledReadScope>& sampling);
+    bool syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span<const ShaderRecompiler::MemoryRegion> regions, std::uint64_t& synced, PendingView& pending, std::optional<SampledReadScope>& sampling, std::span<const VouchedRegions> skipped = {});
     EntryOutcome validateVariant(std::uint64_t program, std::uint32_t queue, const DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, std::uint64_t& imagesFlushed, std::uint64_t& runsSynced, std::optional<SampledReadScope>& sampling, std::vector<std::pair<std::uint32_t, std::uint32_t>>* live = nullptr);
     static int waitTimeoutMs();
     static bool pollReapAll();
@@ -328,7 +331,7 @@ private:
     std::condition_variable changed;
 
     std::map<std::uint32_t, QueueWorker> workers;
-    std::uint64_t frameSerial = 0;
+    std::atomic<std::uint64_t> frameSerial{0};
     std::shared_ptr<FrameTiming> pendingFrameTiming;
 
     std::atomic<std::uint64_t> flipsCounted{0};

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -211,7 +211,7 @@ void AwaitLibraryPrebuilds();
 void CloseRegistrationPreparation();
 unsigned RegistrationPreparationThreads();
 
-void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target);
+void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared = false);
 
 ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId);
 ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint64_t vertexId, std::uint64_t fragmentId, const ShaderRecompiler::SpirvTarget& target);

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -196,6 +196,7 @@ unsigned long long CodeOffset(const void* address);
 // calls FlushGpuWrites itself.
 void SetFlushHook(void (*hook)(std::uint64_t address, std::size_t bytes));
 void SetReadGuard(void (*guard)(std::uint64_t address, std::size_t bytes));
+void GuardRead(std::uint64_t address, std::size_t bytes);
 void FlushGpuWrites(std::uint64_t address, std::size_t bytes);
 // Moves with every ForgetPages call (the libc invalidator: memory unmapped or re-registered): odd
 // while the call stores its page states, moved past it after, so a reader that loads an even

core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp

@@ -82,7 +82,7 @@ struct DrawRecipe {
     std::array<std::byte, Graphics::PipelinePushConstantBytes> pushBytes{};
     VkShaderStageFlags pushStages = 0;
     std::optional<Graphics::State> masked;
-    std::set<std::uint32_t> fragmentOutputs;
+    std::uint32_t fragmentOutputs = 0;
     VkPipelineStageFlags shaderStages = 0;
     bool hasDepth = false;
 };

core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp

@@ -16,6 +16,8 @@
 namespace ShaderRecompiler {
 struct SourceHandle;
 class PreparedShaderInvocation;
+struct SrtWriteStamp;
+struct SrtGuestWords;
 }
 
 namespace AgcDriver {
@@ -56,7 +58,7 @@ public:
     // by the driver: the [capture] line charges the waits made inside a capture to it.
     using WaitedMsProvider = double (*)();
     static void SetWaitedMsProvider(WaitedMsProvider provider);
-    explicit ShaderMemory(std::span<const ShaderRecompiler::MemoryRegion> initial, PendingWriteQuery pendingWrite = nullptr, PendingWriteObserver observe = nullptr, HookWaitCounter hookWaits = nullptr);
+    explicit ShaderMemory(std::span<const ShaderRecompiler::MemoryRegion> initial, PendingWriteQuery pendingWrite = nullptr, PendingWriteObserver observe = nullptr, HookWaitCounter hookWaits = nullptr, PendingWriteQuery pendingSpan = nullptr);
     ~ShaderMemory();
     ShaderMemory(const ShaderMemory&) = delete;
     ShaderMemory& operator=(const ShaderMemory&) = delete;
@@ -74,6 +76,16 @@ public:
     [[nodiscard]] std::vector<ShaderRecompiler::MemoryRegion> TakeRecentRegions();
     // The driver's source handle memo outcomes, for the [capture] line (APS5_PROFILE_DRAW).
     static void CountHandleMemo(bool hit);
+    struct SupplyCounts {
+        std::uint64_t shared, copied, spilled;
+    };
+    static SupplyCounts Supplies();
+    [[nodiscard]] std::vector<std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> SuppliedBlocks() const;
+    struct VouchCounts {
+        std::uint64_t calls, blocks, refused, words;
+    };
+    static VouchCounts Vouches();
+    static bool Vouched(PendingWriteQuery pending, std::uint64_t address, std::uint64_t bytes, const ShaderRecompiler::SrtWriteStamp& stamp);
 
 private:
     std::shared_ptr<const ShaderRecompiler::ResourceCapture> capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle, const ShaderRecompiler::PreparedShaderInvocation* invocation);
@@ -83,6 +95,10 @@ private:
     class WordMask {
     public:
         [[nodiscard]] bool Test(std::size_t index) const { return (bits[index / 64u] >> (index % 64u) & 1u) != 0u; }
+        [[nodiscard]] bool TestBlock(std::size_t index) const {
+            const auto block = ((std::uint64_t{1} << BlockWords) - 1u) << (index % 64u / BlockWords * BlockWords);
+            return (bits[index / 64u] & block) == block;
+        }
         void Set(std::size_t index) { bits[index / 64u] |= std::uint64_t{1} << (index % 64u); }
         void SetAll() { bits.fill(~std::uint64_t{0}); }
         void SetBlock(std::size_t index) { bits[index / 64u] |= ((std::uint64_t{1} << BlockWords) - 1u) << (index % 64u / BlockWords * BlockWords); }
@@ -107,22 +123,41 @@ private:
         bool lazy = false;
     };
 
+    struct Supplied {
+        std::shared_ptr<const ShaderRecompiler::SrtGuestWords> words;
+        std::uint64_t begin = 0;
+        std::uint64_t end = 0;
+        bool recent = true;
+    };
+
     static bool read(void* context, std::uint64_t address, std::uint32_t* value);
     static bool readWords(void* context, std::uint64_t address, std::uint32_t* values, std::uint64_t count);
     static bool accessible(void* context, std::uint64_t address, std::uint64_t bytes);
     static bool pending(void* context, std::uint64_t address, std::uint64_t bytes);
+    static ShaderRecompiler::SrtWriteStamp stampWrites(void* context, std::uint64_t address, std::uint64_t bytes);
+    static bool unchangedSince(void* context, std::uint64_t address, std::uint64_t bytes, const ShaderRecompiler::SrtWriteStamp& stamp);
+    static bool supplyWords(void* context, const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>& words);
     Page& page(std::uint64_t base);
+    [[nodiscard]] std::size_t pageAtOrAfter(std::uint64_t base) const;
+    Page& addPage(std::size_t position, std::uint64_t base);
     [[nodiscard]] bool overlapsInitial(std::uint64_t begin, std::uint64_t end) const;
+    bool place(std::uint64_t address, const std::uint32_t* values, std::uint64_t count, bool overwrite, bool recent);
+    void spill(std::size_t index);
+    void spillTouching(std::uint64_t begin, std::uint64_t end);
+    [[nodiscard]] std::vector<ShaderRecompiler::MemoryRegion> collect(bool recentOnly) const;
 
     // Regions given at construction (the registered shader's code and header), referenced as given:
     // the caller keeps them alive for as long as the capture is used.
-    std::map<std::uint64_t, std::span<const std::byte>> initial;
+    std::vector<std::pair<std::uint64_t, std::span<const std::byte>>> initial;
     std::vector<std::pair<std::uint64_t, Page*>> pages;
     std::uint64_t lastBase = ~std::uint64_t{0};
     Page* lastPage = nullptr;
+    std::vector<Supplied> supplied;
+    std::vector<std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> spilled;
     PendingWriteQuery pendingWrite = nullptr;
     PendingWriteObserver observe = nullptr;
     HookWaitCounter hookWaits = nullptr;
+    PendingWriteQuery pendingSpan = nullptr;
 };
 
 // The positions, among a dispatch-cache variant's stored words, of the pure flat-SRT leaves a
@@ -140,6 +175,25 @@ struct DataWordPositionCounts {
 };
 DataWordPositionCounts DataWordPositions(std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint32_t, std::uint64_t>> leaves, std::span<const std::uint64_t> otherReads, std::span<const std::uint32_t> words, std::span<const std::uint32_t> flattenedSrt, std::vector<std::uint32_t>& positions, std::vector<std::uint32_t>& slots);
 
+struct SharedWords {
+    std::size_t first = 0;
+    std::size_t count = 0;
+    std::size_t offset = 0;
+    std::weak_ptr<const ShaderRecompiler::SrtGuestWords> block;
+    std::size_t region = 0;
+    std::size_t regions = 0;
+};
+void AppendSharedWords(std::vector<SharedWords>& shared, std::size_t region, std::size_t first, std::span<const std::byte> bytes, std::span<const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> blocks);
+bool SameCapturedWords(std::span<const std::uint32_t> left, std::span<const SharedWords> leftShared, std::span<const std::uint32_t> right, std::span<const SharedWords> rightShared, std::size_t* compared = nullptr);
+struct VouchedRegions {
+    std::size_t begin = 0;
+    std::size_t end = 0;
+    std::size_t words = 0;
+    std::uint64_t low = 0;
+    std::uint64_t high = 0;
+};
+std::size_t VouchSharedRegions(std::span<const SharedWords> shared, std::size_t regions, ShaderMemory::PendingWriteQuery pending, std::vector<VouchedRegions>& vouched);
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp

@@ -54,6 +54,7 @@ public:
 }
 
 extern "C" std::uint32_t AgcDriverReserveEopFlip_nid_postfix(const AgcDriver::FlipInfo& info);
+extern "C" void AgcDriverSubmitFlip_nid_postfix(const AgcDriver::FlipInfo& info);
 
 extern "C" void AgcDriverRegisterVideoOutput_nid_postfix(std::uint32_t handle, const std::shared_ptr<AgcDriver::IVideoOutput>& output);
 extern "C" void AgcDriverUnregisterVideoOutput_nid_postfix(std::uint32_t handle, const std::shared_ptr<AgcDriver::IVideoOutput>& output);

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -332,9 +332,7 @@ public:
     void EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span<const Graphics::CompiledShader> shaders, std::span<const Graphics::GuestMemorySnapshot> snapshots = {});
     bool PrebuildGraphicsLibraries(const Graphics::ShaderStages& stages, std::span<const Graphics::CompiledShader> shaders);
 
-private:
-    VulkanDevice(const PresentationWindow* window, bool compileOnly);
-    friend struct VulkanDeviceTestAccess;
+public:
     // The device's Graphics::Context: a copy of the one built at setup (its instance functions
     // resolved then, its function table filled then), or with APS5_NO_CONTEXT_CACHE=1 built anew.
     Graphics::Context graphicsContext() const;
@@ -342,6 +340,10 @@ private:
     std::uint32_t computeSubgroupSize(const ShaderRecompiler::RecompileResult& shader) const;
     VkPipelineShaderStageCreateFlags computeStageFlags(const ShaderRecompiler::RecompileResult& shader) const;
     std::shared_ptr<ComputePipelineObjects> createComputePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span<const std::uint32_t> layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress);
+
+private:
+    VulkanDevice(const PresentationWindow* window, bool compileOnly);
+    friend struct VulkanDeviceTestAccess;
     ShaderRecompiler::SpirvTarget buildTarget() const;
     Graphics::Context buildContext() const;
     // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups.

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp

@@ -12,7 +12,7 @@ void Driver::verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffse
         std::fflush(stderr);
         std::abort();
     };
-    auto verifyMemory = std::make_shared<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter());
+    auto verifyMemory = std::make_shared<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan);
     auto invocation = InvocationFor(snapshot, codeOffset, request);
     const auto capture = CaptureInvocation(*verifyMemory, snapshot, codeOffset, request, invocation);
     const auto regions = verifyMemory->Regions();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp

@@ -326,7 +326,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
         require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader");
         captureMs += phaseTiming.Elapsed();
     } else if (!fromAhead) {
-        shaderMemory = std::make_shared<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter());
+        shaderMemory = std::make_shared<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan);
         std::uint64_t forgetAtCapture = 0;
 
         static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCache.cpp

@@ -60,7 +60,7 @@ Driver::DispatchCacheFootprint Driver::DispatchCacheSize() {
 }
 
 std::uint64_t Driver::variantBytes(const DispatchVariant& variant) {
-    return sizeof(DispatchVariant) + variant.words.size() * sizeof(std::uint32_t) + variant.runs.size() * sizeof(std::pair<std::uint64_t, std::uint64_t>) + variant.captured.size() * sizeof(ShaderRecompiler::MemoryRegion) + (variant.vertexInfo != nullptr ? sizeof(ShaderRecompiler::ShaderVertexStageInfo) : 0);
+    return sizeof(DispatchVariant) + variant.words.size() * sizeof(std::uint32_t) + variant.shared.size() * sizeof(SharedWords) + variant.runs.size() * sizeof(std::pair<std::uint64_t, std::uint64_t>) + variant.captured.size() * sizeof(ShaderRecompiler::MemoryRegion) + (variant.vertexInfo != nullptr ? sizeof(ShaderRecompiler::ShaderVertexStageInfo) : 0);
 }
 
 void Driver::accountVariant(const DispatchVariant& variant, bool added) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp

@@ -22,15 +22,23 @@ void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDis
         fresh->forgetSerial = forgetAtCapture;
 
         std::vector<ShaderRecompiler::MemoryRegion> srtRegions;
+        std::size_t capturedWords = 0;
         for (const auto& region : captured) {
             const bool registered = std::any_of(memory.begin(), memory.end(), [&](const auto& known) { return region.guestAddress >= known.guestAddress && region.guestAddress < known.guestAddress + known.bytes.size(); });
             if (registered) continue;
             srtRegions.push_back(region);
+            capturedWords += region.bytes.size() / sizeof(std::uint32_t);
+        }
+        const auto supplied = shaderMemory->SuppliedBlocks();
+        fresh->runs.reserve(srtRegions.size());
+        fresh->words.reserve(capturedWords);
+        for (const auto& region : srtRegions) {
             fresh->runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size());
             const auto count = region.bytes.size() / sizeof(std::uint32_t);
             const auto offset = fresh->words.size();
             fresh->words.resize(offset + count);
             std::memcpy(fresh->words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t));
+            AppendSharedWords(fresh->shared, fresh->runs.size() - 1u, offset, region.bytes, supplied);
         }
 
         DataWordPositionCounts dataCounts;
@@ -91,7 +99,7 @@ void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDis
                 dispatchOrder.push_front(key);
                 created->order = dispatchOrder.begin();
                 dispatchCache.emplace(key, std::move(created));
-            } else if (found->second != missedEntry && !found->second->variants.empty() && found->second->variants.front()->runs == fresh->runs && found->second->variants.front()->words == fresh->words) {
+            } else if (found->second != missedEntry && !found->second->variants.empty() && found->second->variants.front()->runs == fresh->runs && SameCapturedWords(found->second->variants.front()->words, found->second->variants.front()->shared, fresh->words, fresh->shared)) {
 
                 attachVariant = found->second->variants.front();
             } else {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp

@@ -1,6 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include <algorithm>
 #include <cstdlib>
@@ -28,10 +29,15 @@ std::shared_ptr<const ShaderRecompiler::RecompileResult> Driver::patchDataWords(
     return patched;
 }
 
-bool Driver::syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span<const ShaderRecompiler::MemoryRegion> regions, std::uint64_t& synced, PendingView& pending, std::optional<SampledReadScope>& sampling) {
+bool Driver::syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span<const ShaderRecompiler::MemoryRegion> regions, std::uint64_t& synced, PendingView& pending, std::optional<SampledReadScope>& sampling, std::span<const VouchedRegions> skipped) {
     using Policy = ShaderMemory::PendingWrite;
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
-    for (const auto& region : regions) {
+    for (std::size_t index = 0, range = 0; index < regions.size(); ++index) {
+        if (range < skipped.size() && index == skipped[range].begin) {
+            index = skipped[range++].end - 1u;
+            continue;
+        }
+        const auto& region = regions[index];
         if (!pending.Overlaps(region.guestAddress, region.bytes.size())) continue;
         if (!sampling) sampling.emplace(siteSampling() ? evidenceValidations : evidenceReads);
         std::uint64_t ValidateCounters::*reason = &ValidateCounters::syncedOff;
@@ -78,6 +84,11 @@ bool Driver::syncPendingRuns(std::uint64_t program, std::uint32_t queue, const S
     return true;
 }
 
+bool Driver::vouchedValidation() {
+    static const bool enabled = std::getenv("APS5_NO_VOUCHED_VALIDATION") == nullptr;
+    return enabled;
+}
+
 EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue, const DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, std::uint64_t& imagesFlushed, std::uint64_t& runsSynced, std::optional<SampledReadScope>& sampling, std::vector<std::pair<std::uint32_t, std::uint32_t>>* live) {
     if (live != nullptr) live->clear();
     if (DrawPipeline::Active()) {
@@ -86,14 +97,34 @@ EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue,
     }
     const bool masked = live != nullptr && !variant.dataPositions.empty();
     const DataMask mask{variant.dataPositions, live};
+    thread_local std::vector<VouchedRegions>* vouchedSlot = nullptr;
+    auto& vouched = ShaderRecompiler::ThreadOwned(vouchedSlot);
+    vouched.clear();
+    if (!variant.shared.empty() && vouchedValidation() && !validateLegacy() && variant.runs.size() == regions.size()) VouchSharedRegions(variant.shared, regions.size(), &QueryPendingSpan, vouched);
     PendingView pending;
     pending.Load();
-    if (!syncPendingRuns(program, queue, *variant.compiled, regions, runsSynced, pending, sampling)) return EntryOutcome::Differing;
+    std::erase_if(vouched, [&](const VouchedRegions& range) { return pending.Overlaps(range.low, static_cast<std::size_t>(range.high - range.low)); });
+    thread_local std::vector<std::pair<std::uint64_t, std::uint64_t>>* checkedSlot = nullptr;
+    auto& checked = ShaderRecompiler::ThreadOwned(checkedSlot);
+    checked.clear();
+    if (!vouched.empty()) {
+        for (std::size_t index = 0, range = 0; index < variant.runs.size(); ++index) {
+            if (range < vouched.size() && index == vouched[range].begin) {
+                checked.emplace_back(vouched[range].low, vouched[range].high);
+                index = vouched[range++].end - 1u;
+                continue;
+            }
+            checked.push_back(variant.runs[index]);
+        }
+    }
+    const std::span<const std::pair<std::uint64_t, std::uint64_t>> runs = vouched.empty() ? std::span<const std::pair<std::uint64_t, std::uint64_t>>(variant.runs) : std::span<const std::pair<std::uint64_t, std::uint64_t>>(checked);
+    if (!syncPendingRuns(program, queue, *variant.compiled, regions, runsSynced, pending, sampling, vouched)) return EntryOutcome::Differing;
     auto publish = Graphics::Recorder::PublishGeneration();
     auto pendingSerial = Graphics::StorageTexture::PendingSerial();
     auto forget = GuestMemory::ForgetSerial();
 
-    if (Graphics::StorageTexture::AnyPendingOverlaps(variant.runs) || Graphics::AnyShadowedOverlaps(variant.runs)) {
+    if (Graphics::StorageTexture::AnyPendingOverlaps(runs) || Graphics::AnyShadowedOverlaps(runs)) {
+        vouched.clear();
         for (const auto& region : regions) {
             if (Graphics::StorageTexture::FlushPending(region.guestAddress, region.bytes.size(), nullptr, "dispatch-cache entry")) ++imagesFlushed;
         }
@@ -106,7 +137,7 @@ EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue,
     }
     if ((forget & 1) != 0) return EntryOutcome::ForgetMoved;
     for (const auto& label : deferredLabels().labels) {
-        for (const auto& [begin, end] : variant.runs) {
+        for (const auto& [begin, end] : runs) {
             if (label.address < end && begin < label.address + label.size) return EntryOutcome::QueuedLabel;
         }
     }
@@ -118,7 +149,7 @@ EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue,
 
     pending.Load();
     bool unmapped = false;
-    if (!validateCaptured(program, queue, regions, *variant.compiled, true, pending, &unmapped, &sampling, masked ? &mask : nullptr)) return unmapped ? EntryOutcome::Inaccessible : EntryOutcome::Differing;
+    if (!validateCaptured(program, queue, regions, *variant.compiled, true, pending, &unmapped, &sampling, masked ? &mask : nullptr, vouched)) return unmapped ? EntryOutcome::Inaccessible : EntryOutcome::Differing;
     if (Graphics::Recorder::PublishGeneration() != publish) return EntryOutcome::PublishMoved;
     if (Graphics::StorageTexture::PendingSerial() != pendingSerial) return EntryOutcome::PendingMoved;
     if (GuestMemory::ForgetSerial() != forget) return EntryOutcome::ForgetMoved;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -8,6 +8,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "Optimization/ResourceProgram.hpp"
 #include <algorithm>
+#include <bit>
 #include <cstdlib>
 #include <cstdio>
 #include <cstring>
@@ -162,7 +163,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
             if (program.merged) InitializeMerged(queue, program, program.mergedPointer, program.mergedPointerRequired, false);
         }
         if (verifyDrawDataHits()) {
-            const auto full = decodeDraw(queue, submission);
+            const auto full = decodeDraw(queue, submission, localDevice->graphicsContext());
             if (!sameDecode(*refreshed, *full)) {
                 std::size_t differing = 0;
                 while (differing < full->programs.size() && differing < refreshed->programs.size() && full->programs[differing].userData == refreshed->programs[differing].userData) ++differing;
@@ -173,15 +174,18 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         }
         decode = std::move(refreshed);
     }
-    resolveDrawDecode(queue, submission, decode, registerKey, drawKey, profile);
+    resolveDrawDecode(queue, submission, localDevice->graphicsContext(), decode, registerKey, drawKey, profile);
     const auto& graphics = decode->state;
     auto pixel = decode->pixel;
-    std::vector<DrawProgram> programs = decode->programs;
-    for (const auto& patch : userPatches) programs[patch.program].userData[patch.index] = patch.value;
+    const bool writesPrograms = graphics.stages.mesh || drawParameters.indirect || !userPatches.empty();
+    std::vector<DrawProgram> writablePrograms;
+    if (writesPrograms) writablePrograms = decode->programs;
+    for (const auto& patch : userPatches) writablePrograms[patch.program].userData[patch.index] = patch.value;
+    const std::vector<DrawProgram>& programs = writesPrograms ? writablePrograms : decode->programs;
     if (const auto skipped = SkippedShader(); skipped != 0 && std::any_of(programs.begin(), programs.end(), [skipped](const DrawProgram& program) { return program.binary.codeAddress == skipped; })) return drawn();
     const auto setMeshIndexBuffer = [&](const Pm4::DrawParameters& parameters) {
         if (!graphics.stages.mesh) return;
-        auto& words = programs.front().userData;
+        auto& words = writablePrograms.front().userData;
         require(programs.front().firstUserSgpr == 0 && words.size() >= ShaderRecompiler::MeshIndexBufferUserWord + 4, "mesh program lacks the hidden user words");
         const auto descriptor = Graphics::MeshIndexBufferDescriptor(parameters);
         std::copy(descriptor.begin(), descriptor.end(), words.begin() + ShaderRecompiler::MeshIndexBufferUserWord);
@@ -222,7 +226,9 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         indirect.drawIndexSgpr = sgprOf(indirect.drawIndexLocation);
     }
     std::vector<ShaderRecompiler::MemoryRegion> memory;
+    memory.reserve(2 * programs.size() + 32);
     std::vector<ShaderRecompiler::LinkedProgram> linked;
+    linked.reserve(programs.size());
     for (std::size_t i = 0; i < programs.size(); ++i) {
         const auto& program = programs[i];
         for (const auto& region : program.memory) {
@@ -241,12 +247,15 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         decodeReads[i].clear();
         vertexInfos[i] = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, &decodeReads[i], false, 8u - program.firstUserSgpr);
         if (vertexInfos[i]) vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, readRegister(queue.context, 0x207), program.tailContinuation.get()) | graphics.layerExports;
+        if (vertexInfos[i]) {
+            vertexInfos[i]->depthRemapMode = graphics.depthRemap.mode;
+        }
     };
     if (!registerKey) {
         for (std::size_t i = 0; i < programs.size(); ++i) decodeVertexInfo(i);
         phaseTiming.Phase(DrawRowDecode);
     }
-    auto shaderMemory = std::make_unique<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter());
+    auto shaderMemory = std::make_unique<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan);
     std::vector<ShaderRecompiler::RecompileResult> results;
     std::vector<std::shared_ptr<const ShaderRecompiler::RecompileResult>> compiledStages;
     compiledStages.reserve(programs.size());
@@ -321,7 +330,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
 
     const auto freshRuns = [&](std::size_t i, std::vector<std::pair<std::uint64_t, std::uint64_t>>& runs, std::vector<std::uint32_t>& words) {
         std::vector<ShaderRecompiler::MemoryRegion> regions(stageCaptures[i].regions.begin(), stageCaptures[i].regions.end());
-        for (const auto& read : decodeReads[i]) regions.push_back({read.address, std::as_bytes(std::span(read.bytes))});
+        for (const auto& read : decodeReads[i]) regions.push_back({read.address, read.Bytes()});
         std::stable_sort(regions.begin(), regions.end(), [](const ShaderRecompiler::MemoryRegion& a, const ShaderRecompiler::MemoryRegion& b) { return a.guestAddress < b.guestAddress; });
         for (const auto& region : regions) {
             runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size());
@@ -423,7 +432,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     timing.Mark("shader_compile_and_link");
 
     for (const auto& reads : decodeReads) {
-        for (const auto& read : reads) memory.push_back({read.address, std::as_bytes(std::span(read.bytes))});
+        for (const auto& read : reads) memory.push_back({read.address, read.Bytes()});
     }
 
     if (pipelined && !deferredLabels().labels.empty()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Labels);
@@ -470,6 +479,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     std::vector<Graphics::GuestMemorySnapshot> snapshots;
     const auto snapshot = [&] {
         snapshots.clear();
+        snapshots.reserve(memory.size());
         for (const auto& region : memory) snapshots.push_back({region.guestAddress, region.bytes});
     };
     snapshot();
@@ -519,7 +529,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
             std::set<std::size_t> patched;
             const auto patch = [&](const std::optional<std::pair<std::size_t, std::size_t>>& word, std::uint32_t value) {
                 if (!word) return;
-                programs[word->first].userData[word->second] = value;
+                writablePrograms[word->first].userData[word->second] = value;
                 patched.insert(word->first);
             };
             patch(baseVertexWord, indirect.recordBytes == 20 ? arguments.vertexOffset : arguments.firstVertexOrIndex);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp

@@ -140,7 +140,7 @@ void Driver::insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<Disp
     for (std::size_t i = 0; i < fresh.size(); ++i) {
         if (fresh[i] == nullptr) continue;
         auto& variants = replacement->stages[i];
-        const auto present = std::find_if(variants.begin(), variants.end(), [&](const std::shared_ptr<DispatchVariant>& kept) { return kept->pushOffset == fresh[i]->pushOffset && kept->meshIndexDescriptor == fresh[i]->meshIndexDescriptor && kept->runs == fresh[i]->runs && kept->words == fresh[i]->words; });
+        const auto present = std::find_if(variants.begin(), variants.end(), [&](const std::shared_ptr<DispatchVariant>& kept) { return kept->pushOffset == fresh[i]->pushOffset && kept->meshIndexDescriptor == fresh[i]->meshIndexDescriptor && kept->runs == fresh[i]->runs && SameCapturedWords(kept->words, kept->shared, fresh[i]->words, fresh[i]->shared); });
         if (present != variants.end()) {
             ++counters.present;
             fresh[i] = *present;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp

@@ -23,7 +23,7 @@ std::shared_ptr<const ShaderRecompiler::RecompileResult> Driver::materializeDraw
         program.binary,
         {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, vertexInfos[i], memory, RegisteredFloatMode(*program.snapshot)},
         localDevice->Target(),
-        {0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : graphics.stages.tessellation && pushOffset < Graphics::PipelinePushSlotBytes ? ShaderRecompiler::TessellationLevelPushOffsetBytes : Graphics::PipelinePushSlotBytes) - pushOffset % Graphics::PipelinePushSlotBytes},
+        {0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : graphics.stages.tessellation && pushOffset < Graphics::PipelinePushSlotBytes ? ShaderRecompiler::TessellationLevelPushOffsetBytes : graphics.depthRemap.mode != 0u ? ShaderRecompiler::DepthRemapPushOffsetBytes : Graphics::PipelinePushSlotBytes) - pushOffset % Graphics::PipelinePushSlotBytes},
         ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}}
     };
     const auto waitedBefore = traceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0;
@@ -56,6 +56,7 @@ std::shared_ptr<const ShaderRecompiler::RecompileResult> Driver::materializeDraw
     }();
 
     stageCapture.regions = shaderMemory.TakeRecentRegions();
+    stageCapture.supplied = shaderMemory.SuppliedBlocks();
     stageCapture.userDataUse = capture->userDataUse;
     recompiled[i] = true;
     memory = shaderMemory.Regions();
@@ -120,7 +121,7 @@ void Driver::recordStageDataPositions(DispatchVariant& variant, const ShaderReco
     for (const auto& [begin, end] : variant.runs) {
         for (const auto& read : decodeReads) {
             const auto from = std::max(begin, read.address);
-            const auto to = std::min(end, read.address + read.bytes.size());
+            const auto to = std::min(end, read.address + read.size);
             if (from >= to) continue;
             for (auto word = (from - begin) / sizeof(std::uint32_t); word < (to - begin + sizeof(std::uint32_t) - 1) / sizeof(std::uint32_t); ++word) decoded[offset + static_cast<std::size_t>(word)] = true;
         }
@@ -159,9 +160,9 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
             if (vertexInfos[i]) variant->vertexInfo = std::make_shared<const ShaderRecompiler::ShaderVertexStageInfo>(*vertexInfos[i]);
 
             std::vector<ShaderRecompiler::MemoryRegion> regions(stageCapture.regions.begin(), stageCapture.regions.end());
-            for (const auto& read : decodeReads[i]) regions.push_back({read.address, std::as_bytes(std::span(read.bytes))});
+            for (const auto& read : decodeReads[i]) regions.push_back({read.address, read.Bytes()});
             std::stable_sort(regions.begin(), regions.end(), [](const ShaderRecompiler::MemoryRegion& a, const ShaderRecompiler::MemoryRegion& b) { return a.guestAddress < b.guestAddress; });
-            tagVariantRuns(*variant, regions, stageCapture.userDataUse, decodeReads[i].empty() && (!vertexInfos[i] || vertexInfos[i]->resourcesNum == 0), programs[i].hiddenUserWords);
+            tagVariantRuns(*variant, regions, stageCapture.userDataUse, decodeReads[i].empty() && (!vertexInfos[i] || vertexInfos[i]->resourcesNum == 0), programs[i].hiddenUserWords, stageCapture.supplied);
             if (drawStageDataHits() && stageCapture.capture != nullptr && !stageCapture.capture->readTrace.leaves.empty()) recordStageDataPositions(*variant, *stageCapture.capture, decodeReads[i]);
             if (traceDrawCache() && entry != nullptr && matched[i] == nullptr && i < entry->stages.size()) {
                 const auto& variants = entry->stages[i];

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp

@@ -22,6 +22,7 @@ void ReadUserWords(const QueueState& queue, DrawProgram& program, bool staticAbi
     const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u);
     require(userCount <= 32, "graphics user SGPR count exceeds the register bank");
     program.userData.clear();
+    program.userData.reserve(userCount + 8u);
     for (std::uint32_t i = 0; i < userCount; ++i) {
         Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, program.userDataBase + i);
         program.userData.push_back(staticAbi ? ReadGraphicsRegister(queue.shader, program.userDataBase + i) : readUserData(queue.shader, program.userDataBase + i));
@@ -121,9 +122,10 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
     }
 }
 
-std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Submission& submission) {
+std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Submission& submission, const Graphics::Context& context) {
     auto product = std::make_shared<DrawDecode>();
     product->state = Graphics::DecodeState(queue, VertexWorkgroupRequired(queue, *submission.shaders));
+    Graphics::ApplyDepthRemap(context, product->state);
     DecodeGraphicsPrograms(*product, queue, *submission.shaders, false, true);
     product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue));
     product->pixel.targetExportPacking = Graphics::ExportPackings(product->state);
@@ -149,11 +151,11 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
     return product;
 }
 
-void Driver::resolveDrawDecode(const QueueState& queue, const Submission& submission, std::shared_ptr<const DrawDecode>& decode, bool registerKey, std::uint64_t drawKey, bool profile) {
+void Driver::resolveDrawDecode(const QueueState& queue, const Submission& submission, const Graphics::Context& context, std::shared_ptr<const DrawDecode>& decode, bool registerKey, std::uint64_t drawKey, bool profile) {
     if (decode == nullptr || verifyDrawRecipe()) {
         std::vector<Graphics::RegisterRead> readLog;
         const Graphics::RegisterReadLogScope logScope(verifyDrawRecipe() && registerKey ? &readLog : nullptr);
-        auto fresh = decodeDraw(queue, submission);
+        auto fresh = decodeDraw(queue, submission, context);
         if (verifyDrawRecipe() && registerKey) {
             std::uint64_t facadeMismatches = 0;
             for (const auto read : readLog) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp

@@ -214,6 +214,7 @@ void Driver::traceDrawKeyLookup(const std::array<std::uint64_t, 5>& components,
 
 bool Driver::sameVertexInfo(const ShaderRecompiler::ShaderVertexStageInfo& a, const ShaderRecompiler::ShaderVertexStageInfo& b) {
     if (a.resourcesNum != b.resourcesNum || a.fetchAttribReg != b.fetchAttribReg || a.fetchBufferReg != b.fetchBufferReg || a.fetchEmbedded != b.fetchEmbedded || a.paClVsOutCntl != b.paClVsOutCntl) return false;
+    if (a.depthRemapMode != b.depthRemapMode) return false;
     for (std::uint32_t i = 0; i < a.resourcesNum && i < a.resources.size(); ++i) {
         if (a.resources[i].fields != b.resources[i].fields) return false;
         const auto& x = a.resourcesDst[i];

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
+#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp"
 #include <algorithm>
 #include <array>
 #include <bit>
@@ -55,7 +56,7 @@ std::uint64_t Driver::relocatedContentKey(const RelocatableUserWord& word, std::
     return hash;
 }
 
-void Driver::tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords) {
+void Driver::tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords, std::span<const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> supplied) {
     std::uint64_t relocatable = eligible && drawRelocationEnabled() ? use.asBase & ~use.asValue : 0;
     if (hiddenWords != 0) relocatable &= hiddenWords >= 64 ? 0 : ~((std::uint64_t {1} << hiddenWords) - 1);
     std::vector<std::pair<std::uint64_t, std::int32_t>> reads;
@@ -83,12 +84,14 @@ void Driver::tagVariantRuns(DispatchVariant& variant, std::span<const ShaderReco
         return (relocatable >> found->second & 1u) != 0 ? static_cast<std::int8_t>(found->second) : std::int8_t {-1};
     };
     variant.relocatableWords = relocatable;
-    for (const auto& region : regions) {
+    for (std::size_t index = 0; index < regions.size(); ++index) {
+        const auto& region = regions[index];
         const auto count = region.bytes.size() / sizeof(std::uint32_t);
         const auto offset = variant.words.size();
         variant.words.resize(offset + count);
         std::memcpy(variant.words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t));
         if (relocatable == 0) {
+            AppendSharedWords(variant.shared, variant.runs.size(), offset, region.bytes, supplied);
             variant.runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size());
             continue;
         }
@@ -97,6 +100,7 @@ void Driver::tagVariantRuns(DispatchVariant& variant, std::span<const ShaderReco
         for (std::size_t word = 1; word <= count; ++word) {
             const auto next = word < count ? origin(region.guestAddress + word * sizeof(std::uint32_t)) : std::int8_t {-2};
             if (next == current) continue;
+            AppendSharedWords(variant.shared, variant.runs.size(), offset + start, region.bytes.subspan(start * sizeof(std::uint32_t), (word - start) * sizeof(std::uint32_t)), supplied);
             variant.runs.emplace_back(region.guestAddress + start * sizeof(std::uint32_t), region.guestAddress + word * sizeof(std::uint32_t));
             variant.runBases.push_back(current);
             start = word;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp

@@ -99,6 +99,12 @@ extern "C" std::uint32_t AgcDriverReserveEopFlip_nid_postfix(const AgcDriver::Fl
     return AgcDriver::DriverDetail::Driver::Get().ReserveEopFlip(info);
 }
 
+extern "C" void AgcDriverSubmitFlip_nid_postfix(const AgcDriver::FlipInfo& info) try {
+    AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info);
+} catch (const ProcessShutdown&) {
+    LibcAwaitExit_nid_postfix();
+}
+
 extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix() {
     return AgcDriver::DriverDetail::Driver::Get().DispatchesAdoptedAhead();
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -277,6 +277,10 @@ bool& Driver::onWorkerThread() {
 
 namespace AgcDriver {
 
+void FlushAsyncLog_nid_no_patch() {
+    AsyncLog::Get().Flush();
+}
+
 std::unique_lock<std::mutex> HoldAsyncLog() {
     return AsyncLog::Get().Hold();
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp

@@ -23,6 +23,27 @@ bool matchesBufferPatternFillKernel(std::span<const std::uint32_t> code) {
     return code.size() >= kernel.size() && std::equal(kernel.begin(), kernel.end(), code.begin());
 }
 
+bool matchesDwordPatternFillKernel(std::span<const std::uint32_t> code) {
+    static constexpr std::array<std::uint32_t, 69> kernel{
+        0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
+        0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
+        0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
+        0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u, 0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u,
+        0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
+        0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
+        0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
+        0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
+        0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u};
+    static constexpr std::size_t comparesAt = 27;
+    static constexpr std::array<std::uint32_t, 3> comparesSwapped{0x7d8c02f9u, 0x06068c02u, 0x7d860609u};
+    if (code.size() < kernel.size()) return false;
+    const auto before = std::span(kernel).first(comparesAt);
+    const auto after = std::span(kernel).subspan(comparesAt + comparesSwapped.size());
+    const auto compares = code.subspan(comparesAt, comparesSwapped.size());
+    if (!std::equal(before.begin(), before.end(), code.begin()) || !std::equal(after.begin(), after.end(), code.begin() + comparesAt + comparesSwapped.size())) return false;
+    return std::equal(compares.begin(), compares.end(), kernel.begin() + comparesAt) || std::equal(compares.begin(), compares.end(), comparesSwapped.begin());
+}
+
 }
 
 bool Driver::matchesFillKernel(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) {
@@ -45,17 +66,7 @@ bool Driver::matchesFillKernel(std::span<const std::uint32_t> code, const std::v
 std::optional<DwordPatternFill> MatchDwordPatternFill(std::span<const std::uint32_t> packet, std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) {
     static const bool enabled = std::getenv("APS5_NO_FILL_HLE") == nullptr;
     if (!enabled || userData.size() != 10 || packet.size() < 5 || !compute.groupIdEnable[0] || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1) return std::nullopt;
-    static constexpr std::array<std::uint32_t, 69> kernel{
-        0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
-        0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
-        0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
-        0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u, 0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u,
-        0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
-        0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
-        0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
-        0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
-        0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u};
-    if (code.size() < kernel.size() || !std::equal(kernel.begin(), kernel.end(), code.begin())) return std::nullopt;
+    if (!matchesDwordPatternFillKernel(code)) return std::nullopt;
     const auto stride = (userData[1] >> 16u) & 0x3fffu;
     const bool swizzled = ((userData[1] >> 31u) & 1u) != 0;
     const bool addTid = ((userData[3] >> 23u) & 1u) != 0;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/CaptureValidation.cpp

@@ -57,13 +57,23 @@ bool Driver::captureStable(std::span<const ShaderRecompiler::MemoryRegion> captu
     return stable;
 }
 
-bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::span<const ShaderRecompiler::MemoryRegion> captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped, std::optional<SampledReadScope>* sampling, const DataMask* data) {
+bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::span<const ShaderRecompiler::MemoryRegion> captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped, std::optional<SampledReadScope>* sampling, const DataMask* data, std::span<const VouchedRegions> vouched) {
     using Policy = ShaderMemory::PendingWrite;
 
     thread_local std::vector<std::size_t>* pendingSlot = nullptr;
     auto& pending = ShaderRecompiler::ThreadOwned(pendingSlot);
     pending.clear();
-    for (std::size_t i = 0; i < captured.size(); ++i) {
+    thread_local std::vector<VouchedRegions>* skippedSlot = nullptr;
+    auto& skipped = ShaderRecompiler::ThreadOwned(skippedSlot);
+    skipped.clear();
+    for (const auto& range : vouched) {
+        if (range.begin < range.end && range.end <= captured.size() && (skipped.empty() || skipped.back().end <= range.begin) && !view.Overlaps(range.low, static_cast<std::size_t>(range.high - range.low))) skipped.push_back(range);
+    }
+    for (std::size_t i = 0, range = 0; i < captured.size(); ++i) {
+        if (range < skipped.size() && i == skipped[range].begin) {
+            i = skipped[range++].end - 1u;
+            continue;
+        }
         const auto& region = captured[i];
         if (view.Overlaps(region.guestAddress, region.bytes.size())) pending.push_back(i);
     }
@@ -107,7 +117,12 @@ bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::s
         observable.clear();
         reachedPending = 0;
         if (data != nullptr) data->live->clear();
-        for (std::size_t i = 0; i < captured.size(); ++i) {
+        for (std::size_t i = 0, range = 0; i < captured.size(); ++i) {
+            if (range < skipped.size() && i == skipped[range].begin) {
+                first += skipped[range].words;
+                i = skipped[range++].end - 1u;
+                continue;
+            }
             const auto& region = captured[i];
             const bool isPending = next < pending.size() && pending[next] == i;
             if (isPending) ++next;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp

@@ -215,4 +215,15 @@ ShaderMemory::PendingWrite Driver::queryPendingWrite(std::uint64_t address, std:
     return Get().classifyPendingWrite(address, bytes, reason, pending, known);
 }
 
+ShaderMemory::PendingWrite Driver::QueryPendingSpan(std::uint64_t address, std::size_t bytes, std::span<std::byte> known) {
+    if (DrawPipeline::Active()) {
+        auto& pipeline = DrawPipeline::Queue0();
+        if (pipeline.Busy() && pipeline.Overlaps(address, bytes)) return ShaderMemory::PendingWrite::Sync;
+    }
+    std::uint64_t ValidateCounters::*reason = nullptr;
+    PendingView pending;
+    pending.Load();
+    return Get().classifyPendingWrite(address, bytes, reason, pending, known);
+}
+
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp

@@ -8,6 +8,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Execution/include/SubmissionCapture.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp"
+#include "prx/libSceAgcDriver/Eq/include/Event.hpp"
 #include "ThreadOwned.hpp"
 #include "prx/libc/include/HostThreadLocal.hpp"
 #include <bit>
@@ -37,6 +38,11 @@ bool EopRelease(std::span<const std::uint32_t> words) {
     return words.size() >= 8 && (words[0] >> 30u) == 3u && Pm4::PacketWords(words[0]) == 8u && ((words[0] >> 8u) & 0xffu) == 0x49u && ((words[2] >> 24u) & 7u) == 2u;
 }
 
+bool& submitsQueue0() {
+    struct SubmitsQueue0Storage {};
+    return HostThreadLocal<bool, SubmitsQueue0Storage>();
+}
+
 }
 
 std::deque<const std::uint32_t*>& Driver::releasedTails() {
@@ -316,12 +322,22 @@ void Driver::executeRewindTail(const Submission& stalled) {
     execute(tail);
 }
 
-void Driver::Submit(const Packet* packet, std::uint32_t queue) {
+void Driver::SubmitFlip(const FlipInfo& info) {
+    const auto argument = static_cast<std::uint64_t>(info.argument);
+    std::array<std::uint32_t, FlipPacketWords> words{FlipPacketHeader, info.handle, static_cast<std::uint32_t>(info.index), info.mode, static_cast<std::uint32_t>(argument), static_cast<std::uint32_t>(argument >> 32u)};
+    const Packet packet{words.data(), static_cast<std::uint32_t>(words.size()), 0, {}};
+    Submit(&packet, 0, true);
+}
+
+void Driver::Submit(const Packet* packet, std::uint32_t queue, bool cpuFlip) {
     const auto receivedAt = APS5_ENABLE_TIMING_LOG ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
     CheckFailure();
     require(queue == 0 || (queue >= 0x20 && queue < SubmissionQueueEnd), "unsupported compute queue");
     static const bool ring = ReadSubmissionRingSettings();
     if (ring) PrepareSubmissionRing(queue);
+    static const bool flipsInOrder = std::getenv("APS5_CPU_FLIP_IN_ORDER") != nullptr || std::getenv("APS5_SYNC_FLIP") != nullptr;
+    const bool unorderedFlip = cpuFlip && !flipsInOrder && !APS5_ENABLE_TIMING_LOG && !submitsQueue0();
+    if (queue == 0 && !cpuFlip) submitsQueue0() = true;
     GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet));
     const auto descriptor = *packet;
     require(descriptor.flags == 0, "nonzero submission flags are not implemented");
@@ -351,11 +367,22 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) {
     auto* const capture = SubmissionCapture::Active();
     auto captured = capture != nullptr ? SubmissionCapture::Prepare(queue, SubmissionKind::Submit, submission.commands, submission.conditionalEnds) : CapturedSubmission{};
     {
-        std::lock_guard lock(mutex);
+        std::unique_lock lock(mutex);
         rethrowFailure();
         checkStopping();
         require(accepted != std::numeric_limits<std::uint64_t>::max(), "submission serial overflow");
         reserveOutputs(submission);
+        if (unorderedFlip && capture == nullptr && accepted != 0 && flipsAhead.load(std::memory_order_acquire) == 0) {
+            const auto now = FrameTiming::Clock::now();
+            auto frame = std::make_shared<FrameTiming>(++frameSerial);
+            frame->IncludeSubmission(accepted, now, now, now, true);
+            frame->SetFlip(accepted, 0, now, now);
+            ++flipsCounted;
+            submission.flips.begin()->second->GpuReady(frame);
+            lock.unlock();
+            AgcDriverDeliverEopInterrupt(0);
+            return;
+        }
         submission.shaders = shaders;
         submission.serial = accepted + 1;
 
@@ -384,6 +411,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) {
 void Driver::SuspendPoint() {
     const auto receivedAt = APS5_ENABLE_TIMING_LOG ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
     require(!onWorkerThread(), "worker cannot suspend itself");
+    submitsQueue0() = true;
     auto* const capture = SubmissionCapture::Active();
     auto captured = capture != nullptr ? SubmissionCapture::Prepare(0, SubmissionKind::Suspend, {}, {}) : CapturedSubmission{};
     std::unique_lock lock(mutex);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -365,6 +365,7 @@ void ShaderPreparationTransaction::Commit() {
             destination.rectangleProgress.swap(change.prepared.rectangleProgress);
             destination.fragments.swap(change.prepared.fragments);
             std::swap(destination.rectangleRequested, change.prepared.rectangleRequested);
+            destination.deferred = destination.deferred || change.prepared.deferred;
         }
     }
     committed = true;
@@ -1025,6 +1026,29 @@ std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snap
     return plan != nullptr ? PrepareRegistered(*plan) : std::vector<PreparedShaders::Entry>{};
 }
 
+void DeferPipelineStage(PreparedShaderState& prepared, const ShaderSnapshot& snapshot, const char* reason) {
+    if (!prepared.deferred) APS5_LOG_ERR("Shader 0x%llx could not be prepared when the title created its pipeline (%s); preparing it when it is used", static_cast<unsigned long long>(snapshot.codeAddress), reason);
+    prepared.deferred = true;
+}
+
+std::optional<std::string> FailedRegistration(const ShaderSnapshot& snapshot) {
+    std::unique_lock lock(snapshot.prepared->mutex);
+    try {
+        AwaitRegisteredPreparation(*snapshot.prepared, lock);
+    } catch (const std::exception& error) {
+        return error.what();
+    }
+    return std::nullopt;
+}
+
+bool DeferFailedRegistration(const ShaderSnapshot& snapshot) {
+    const auto failure = FailedRegistration(snapshot);
+    if (!failure) return false;
+    std::lock_guard lock(snapshot.prepared->mutex);
+    DeferPipelineStage(*snapshot.prepared, snapshot, failure->c_str());
+    return true;
+}
+
 #ifdef __GLIBC__
 void trimFreedMemory() {
     static std::atomic<std::int64_t> last{0};
@@ -1280,7 +1304,7 @@ unsigned RegistrationPreparationThreads() {
     return RegistrationPreparePool::LiveWorkers();
 }
 
-std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) {
+std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared) {
     ShaderPreparationTransaction transaction;
     require(decoded.programs.size() == decoded.roles.size(), "graphics ABI program roles are incomplete");
     for (const auto& program : decoded.programs) require(program.snapshot != nullptr, "graphics ABI has no registered shader snapshot");
@@ -1307,18 +1331,29 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
     std::vector<PreparedGraphicsStage> prepared;
     std::uint32_t pushOffset = 0;
     const bool fixedSlots = target.fixedPushSlots;
-    const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : decoded.state.stages.tessellation ? ShaderRecompiler::TessellationLevelPushOffsetBytes : Graphics::PipelinePushSlotBytes;
+    const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : decoded.state.stages.tessellation ? ShaderRecompiler::TessellationLevelPushOffsetBytes : decoded.state.depthRemap.mode != 0u ? ShaderRecompiler::DepthRemapPushOffsetBytes : Graphics::PipelinePushSlotBytes;
     for (std::size_t index = 0; index < decoded.programs.size(); ++index) {
         if (decoded.roles[index] == ShaderRecompiler::ProgramRole::GeometryBack) continue;
         const auto& program = decoded.programs[index];
         const bool fragment = program.binary.stage == ShaderRecompiler::ShaderStage::Fragment;
         const auto wave = fragment ? decoded.state.stages.fragmentWaveSize : decoded.state.stages.vertexWaveSize;
-        std::optional<ShaderRecompiler::ShaderVertexStageInfo> vertex;
-        if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true, 8u - program.firstUserSgpr);
-        if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl, program.tailContinuation.get()) | decoded.state.layerExports;
         pushOffset = Graphics::StagePushOffset(pushOffset, program.binary.stage, fixedSlots);
         const auto offset = pushOffset;
         const auto room = (fixedSlots && fragment ? Graphics::PipelinePushSlotBytes : capacity) - pushOffset % Graphics::PipelinePushSlotBytes;
+        if (deferUnprepared && DeferFailedRegistration(*program.snapshot)) {
+            prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr});
+            continue;
+        }
+        std::optional<ShaderRecompiler::ShaderVertexStageInfo> vertex;
+        try {
+            if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true, 8u - program.firstUserSgpr);
+        } catch (const std::exception& error) {
+            if (!deferUnprepared) throw;
+            DeferPipelineStage(transaction.Edit(*program.snapshot), *program.snapshot, error.what());
+            prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr});
+            continue;
+        }
+        if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl, program.tailContinuation.get()) | decoded.state.layerExports;
         ShaderRecompiler::RecompileRequest request{program.binary, {wave, program.firstUserSgpr, program.userData, {}, fragment ? std::optional(decoded.pixel) : std::nullopt, vertex, memory, RegisteredFloatMode(*program.snapshot)}, target, {0, 0, offset, room}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, decoded.state.stages.mesh, decoded.state.stages.tessellation, {}}};
         std::vector<std::uint64_t> key;
         ShaderRecompiler::BuildPreparedShaderKey(request, key);
@@ -1331,7 +1366,14 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
         }
         if (handle == nullptr) {
             PerformanceTimer timing("Shader.PrepareArtifact");
-            handle = PrepareShaderWithDiagnostics(request);
+            try {
+                handle = PrepareShaderWithDiagnostics(request);
+            } catch (const std::exception& error) {
+                if (!deferUnprepared) throw;
+                DeferPipelineStage(transaction.Edit(*program.snapshot), *program.snapshot, error.what());
+                prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr});
+                continue;
+            }
         }
         const auto bytes = handle->artifact->bindings.pushConstantSizeBytes;
         require(bytes <= room, "prepared graphics stages exceed the push constant block");
@@ -1368,6 +1410,7 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
     QueueState state{};
     std::shared_ptr<const ShaderRegistry> registry;
     std::shared_ptr<const ShaderSnapshot> owner;
+    std::vector<std::shared_ptr<const ShaderSnapshot>> selectedStages;
     {
         std::lock_guard lock(mutex);
         registry = shaders;
@@ -1377,7 +1420,7 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
             require(registry != nullptr && registry->contains(address), "graphics ABI refers to an unregistered shader");
             const auto selected = RegisteredHeader(*registry, shader);
             require(selected != nullptr, "graphics ABI refers to a replaced shader header");
-            if (owner == nullptr) owner = selected;
+            selectedStages.push_back(selected);
             const auto& snapshot = *selected;
             require(snapshot.registeredState != nullptr, "registered shader state is missing");
             const auto registered = DecodeRegisteredState(snapshot, {});
@@ -1395,6 +1438,13 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
     for (const auto reg : primitive) state.userConfig.insert_or_assign(reg.offset, reg.value);
     const auto localDevice = device.Load();
     require(localDevice != nullptr, "graphics ABI device is missing");
+    for (const auto& selected : selectedStages) {
+        if (!DeferFailedRegistration(*selected) && owner == nullptr) owner = selected;
+    }
+    if (owner == nullptr) {
+        transaction.Commit();
+        return;
+    }
     struct GraphicsAbiKeyStorage {};
     auto& key = HostThreadLocal<std::vector<std::uint64_t>, GraphicsAbiKeyStorage>();
     BuildRegisteredAbiKey(state, *localDevice, key);
@@ -1404,23 +1454,30 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
         for (const auto& stage : abi.stages) {
             const auto snapshot = stage.lock();
             require(snapshot != nullptr, "prepared graphics ABI lost a registered stage");
-            if (snapshot->type != 1) ResolvePreparedGraphics(*snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target());
+            if (snapshot->type != 1) ResolvePreparedGraphics(*snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target(), true);
         }
         transaction.Commit();
         return;
     }
     DrawDecode decoded{};
-    decoded.state.stages = Graphics::DecodeShaderStages(state, VertexWorkgroupRequired(state, *registry));
-    DecodeGraphicsPrograms(decoded, state, *registry, true, false);
-    auto prepared = PrepareGraphicsStages(decoded, localDevice->Target());
+    try {
+        decoded.state.stages = Graphics::DecodeShaderStages(state, VertexWorkgroupRequired(state, *registry));
+        DecodeGraphicsPrograms(decoded, state, *registry, true, false);
+    } catch (const std::exception& error) {
+        for (const auto& selected : selectedStages) DeferPipelineStage(transaction.Edit(*selected), *selected, error.what());
+        transaction.Commit();
+        return;
+    }
+    auto prepared = PrepareGraphicsStages(decoded, localDevice->Target(), true);
     for (auto& stage : prepared) {
+        if (stage.entry.handle == nullptr) continue;
         const auto& entries = transaction.Read(*stage.snapshot).entries;
         const auto duplicate = std::ranges::any_of(entries, [&](const auto& existing) { return existing.codeOffset == stage.entry.codeOffset && existing.handle->artifact == stage.entry.handle->artifact; });
         if (!duplicate) transaction.Edit(*stage.snapshot).entries.push_back(std::move(stage.entry));
     }
     for (const auto& stage : prepared) {
         if (stage.snapshot->type == 1) continue;
-        ResolvePreparedGraphics(*stage.snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target());
+        ResolvePreparedGraphics(*stage.snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target(), true);
     }
     if (decoded.state.stages.path == Graphics::ShaderPath::Vertex) {
         for (const auto& stage : prepared) {
@@ -1465,14 +1522,24 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
     for (const auto reg : primitive) state.userConfig[reg.offset] = reg.value;
     const auto localDevice = device.Load();
     require(localDevice != nullptr, "shader registration device is missing");
+    if (DeferFailedRegistration(*snapshot)) {
+        transaction.Commit();
+        return;
+    }
     struct RegisteredAbiKeyStorage {};
     auto& key = HostThreadLocal<std::vector<std::uint64_t>, RegisteredAbiKeyStorage>();
     BuildRegisteredAbiKey(state, *localDevice, key);
+    std::optional<std::string> deferral;
     const auto prepare = [&] {
-        const auto plan = PlanRegistered(*snapshot, *localDevice, state, false);
-        if (plan == nullptr) return std::vector<PreparedShaders::Entry>{};
-        plan->prebuildDevice = localDevice;
-        return PrepareRegistered(*plan);
+        try {
+            const auto plan = PlanRegistered(*snapshot, *localDevice, state, false);
+            if (plan == nullptr) return std::vector<PreparedShaders::Entry>{};
+            plan->prebuildDevice = localDevice;
+            return PrepareRegistered(*plan);
+        } catch (const std::exception& error) {
+            deferral = error.what();
+            return std::vector<PreparedShaders::Entry>{};
+        }
     };
     const bool committedAbi = CommittedAbi(*snapshot, key);
     auto entries = committedAbi ? std::vector<PreparedShaders::Entry>{} : prepare();
@@ -1487,6 +1554,7 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
         const auto duplicate = std::ranges::any_of(prepared.entries, [&](const auto& existing) { return existing.handle->artifact == entry.handle->artifact; });
         if (!duplicate) prepared.entries.push_back(std::move(entry));
     }
+    if (deferral) DeferPipelineStage(prepared, *snapshot, deferral->c_str());
     prepared.registeredAbis.push_back(key);
     transaction.Commit();
 }
@@ -1512,7 +1580,7 @@ void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.ResolveGraphicsAbi");
     CheckFailure();
-    if (primitiveType != 0 && primitiveType != 7 && primitiveType != 17) return;
+    if ((primitiveType != 0 && primitiveType != 7 && primitiveType != 17) || pixel == nullptr) return;
     ShaderPreparationTransaction transaction;
     require(vertex != nullptr && pixel != nullptr, "rectangle ABI requires vertex and fragment shaders");
     std::shared_ptr<const ShaderSnapshot> front;
@@ -1532,7 +1600,7 @@ void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::
     require(front != fragment, "rectangle stages refer to the same shader");
     const auto localDevice = device.Load();
     require(localDevice != nullptr, "shader registration device is missing");
-    ResolvePreparedGraphics(*front, fragment, primitiveType, localDevice->Target());
+    ResolvePreparedGraphics(*front, fragment, primitiveType, localDevice->Target(), true);
     transaction.Commit();
 }
 
@@ -1546,15 +1614,19 @@ void Driver::SettleShaderPreparations(std::span<const Shader* const> stages) {
             if (auto snapshot = RegisteredHeader(*shaders, shader)) snapshots.push_back(std::move(snapshot));
         }
     }
-    for (const auto& snapshot : snapshots) {
-        std::unique_lock lock(snapshot->prepared->mutex);
-        AwaitRegisteredPreparation(*snapshot->prepared, lock);
-    }
+    for (const auto& snapshot : snapshots) static_cast<void>(FailedRegistration(*snapshot));
 }
 
-void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target) {
+void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared) {
     ShaderPreparationTransaction transaction;
     require(fragment.get() != &front, "rectangle stages refer to the same shader");
+    if (deferUnprepared) {
+        const bool frontFailed = DeferFailedRegistration(front);
+        if ((fragment != nullptr && DeferFailedRegistration(*fragment)) || frontFailed) {
+            transaction.Commit();
+            return;
+        }
+    }
     std::vector<std::shared_ptr<const ShaderSnapshot>> fragments;
     {
         const auto& current = transaction.Read(front);
@@ -1580,27 +1652,37 @@ void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<
     for (const auto& pixel : fragments) {
         const auto& current = transaction.Read(front);
         const auto& pixelPrepared = transaction.Read(*pixel);
-        require(!current.entries.empty() && !pixelPrepared.entries.empty(), "rectangle ABI has missing stage artifacts");
+        if ((current.entries.empty() && current.deferred) || (pixelPrepared.entries.empty() && pixelPrepared.deferred)) continue;
+        if (current.entries.empty() || pixelPrepared.entries.empty()) {
+            require(deferUnprepared || pixel != fragment, "rectangle ABI has missing stage artifacts");
+            continue;
+        }
         const auto progress = std::ranges::find_if(current.rectangleProgress, [&](const auto& entry) { return entry.fragment.lock() == pixel; });
         const auto vertexCount = progress == current.rectangleProgress.end() ? 0u : progress->vertexCount;
         const auto fragmentCount = progress == current.rectangleProgress.end() ? 0u : progress->fragmentCount;
         if (vertexCount == current.entries.size() && fragmentCount == pixelPrepared.entries.size()) continue;
         auto& prepared = transaction.Edit(front);
         std::vector<PreparedShaders::Rectangle> rectangles;
-        for (std::size_t vertexIndex = 0; vertexIndex < prepared.entries.size(); ++vertexIndex) {
-            const auto& vertexEntry = prepared.entries[vertexIndex];
-            for (std::size_t fragmentIndex = vertexIndex < vertexCount ? fragmentCount : 0u; fragmentIndex < pixelPrepared.entries.size(); ++fragmentIndex) {
-                const auto& fragmentEntry = pixelPrepared.entries[fragmentIndex];
-                const auto& vertexArtifact = ShaderRecompiler::GetPreparedArtifact(*vertexEntry.handle);
-                const auto& fragmentArtifact = ShaderRecompiler::GetPreparedArtifact(*fragmentEntry.handle);
-                const auto exists = [&](const auto& entry) { return entry.vertexId == vertexArtifact.variantId && entry.fragmentId == fragmentArtifact.variantId; };
-                if (std::ranges::any_of(prepared.rectangles, exists) || std::ranges::any_of(rectangles, exists)) continue;
-                ShaderRecompiler::RecompileResult vertexResult;
-                ShaderRecompiler::RecompileResult fragmentResult;
-                static_cast<ShaderRecompiler::CompiledShaderArtifact&>(vertexResult) = vertexArtifact;
-                static_cast<ShaderRecompiler::CompiledShaderArtifact&>(fragmentResult) = fragmentArtifact;
-                rectangles.push_back({vertexArtifact.variantId, fragmentArtifact.variantId, std::make_shared<const ShaderRecompiler::RectListShaders>(ShaderRecompiler::BuildRectListShaders(vertexResult, fragmentResult, target))});
+        try {
+            for (std::size_t vertexIndex = 0; vertexIndex < prepared.entries.size(); ++vertexIndex) {
+                const auto& vertexEntry = prepared.entries[vertexIndex];
+                for (std::size_t fragmentIndex = vertexIndex < vertexCount ? fragmentCount : 0u; fragmentIndex < pixelPrepared.entries.size(); ++fragmentIndex) {
+                    const auto& fragmentEntry = pixelPrepared.entries[fragmentIndex];
+                    const auto& vertexArtifact = ShaderRecompiler::GetPreparedArtifact(*vertexEntry.handle);
+                    const auto& fragmentArtifact = ShaderRecompiler::GetPreparedArtifact(*fragmentEntry.handle);
+                    const auto exists = [&](const auto& entry) { return entry.vertexId == vertexArtifact.variantId && entry.fragmentId == fragmentArtifact.variantId; };
+                    if (std::ranges::any_of(prepared.rectangles, exists) || std::ranges::any_of(rectangles, exists)) continue;
+                    ShaderRecompiler::RecompileResult vertexResult;
+                    ShaderRecompiler::RecompileResult fragmentResult;
+                    static_cast<ShaderRecompiler::CompiledShaderArtifact&>(vertexResult) = vertexArtifact;
+                    static_cast<ShaderRecompiler::CompiledShaderArtifact&>(fragmentResult) = fragmentArtifact;
+                    rectangles.push_back({vertexArtifact.variantId, fragmentArtifact.variantId, std::make_shared<const ShaderRecompiler::RectListShaders>(ShaderRecompiler::BuildRectListShaders(vertexResult, fragmentResult, target))});
+                }
             }
+        } catch (const std::exception& error) {
+            if (!deferUnprepared && pixel == fragment) throw;
+            if (deferUnprepared) DeferPipelineStage(transaction.Edit(*pixel), *pixel, error.what());
+            continue;
         }
         prepared.rectangles.insert(prepared.rectangles.end(), std::make_move_iterator(rectangles.begin()), std::make_move_iterator(rectangles.end()));
         std::erase_if(prepared.rectangleProgress, [&](const auto& entry) { return entry.fragment.expired() || entry.fragment.lock() == pixel; });

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -2234,13 +2234,17 @@ void SetReadGuard(void (*guard)(std::uint64_t address, std::size_t bytes)) {
     GuestArena::GuestArenaSetHostReadHook_nid_no_patch(&PrepareHostRead);
 }
 
+void GuardRead(std::uint64_t address, std::size_t bytes) {
+    if (const auto guard = readGuard.load(std::memory_order_acquire)) guard(address, bytes);
+}
+
 void FlushGpuWrites(std::uint64_t address, std::size_t bytes) {
     if (const auto hook = flushHook.load(std::memory_order_acquire)) hook(address, bytes);
 }
 
 void Read(std::uint64_t address, std::span<std::byte> destination, std::size_t alignment) {
     if (destination.empty()) return;
-    if (const auto guard = readGuard.load(std::memory_order_acquire)) guard(address, destination.size());
+    GuardRead(address, destination.size());
     FlushGpuWrites(address, destination.size());
     const TimedAccess timed(CounterRead, destination.size());
     CountReadCaller(__builtin_return_address(0));
@@ -2287,7 +2291,7 @@ void copyCommitted(std::uint64_t address, std::span<std::byte> destination) {
 
 void ReadCommitted(std::uint64_t address, std::span<std::byte> destination) {
     if (destination.empty()) return;
-    if (const auto guard = readGuard.load(std::memory_order_acquire)) guard(address, destination.size());
+    GuardRead(address, destination.size());
     FlushGpuWrites(address, destination.size());
     const TimedAccess timed(CounterRead, destination.size());
     CountReadCaller(__builtin_return_address(0));
@@ -2296,7 +2300,7 @@ void ReadCommitted(std::uint64_t address, std::span<std::byte> destination) {
 
 void ReadCommittedUnsynced(std::uint64_t address, std::span<std::byte> destination) {
     if (destination.empty()) return;
-    if (const auto guard = readGuard.load(std::memory_order_acquire)) guard(address, destination.size());
+    GuardRead(address, destination.size());
     const TimedAccess timed(CounterRead, destination.size());
     CountReadCaller(__builtin_return_address(0));
     copyCommitted(address, destination);

core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp

@@ -1,10 +1,12 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
 #include "prx/libc/include/HostThreadLocal.hpp"
 #include "Optimization/RequestMemoryView.hpp"
 #include "Optimization/ResourceMaterializer.hpp"
 #include "Optimization/ResourceProgram.hpp"
+#include "ThreadOwned.hpp"
 #include <algorithm>
 #include <atomic>
 #include <bit>
@@ -54,6 +56,13 @@ struct CaptureProfile {
     // The driver's per-shader source handle memo: captures served from it, and resolves it made.
     std::atomic<std::uint64_t> handleHits{0};
     std::atomic<std::uint64_t> handleMisses{0};
+    std::atomic<std::uint64_t> suppliedShared{0};
+    std::atomic<std::uint64_t> suppliedCopied{0};
+    std::atomic<std::uint64_t> suppliedSpilled{0};
+    std::atomic<std::uint64_t> vouchCalls{0};
+    std::atomic<std::uint64_t> vouchedBlocks{0};
+    std::atomic<std::uint64_t> refusedBlocks{0};
+    std::atomic<std::uint64_t> vouchedWords{0};
 };
 
 constexpr std::uint64_t WordWaitNanoseconds = 50000;
@@ -90,6 +99,29 @@ double WaitedMs() {
     return provider != nullptr ? provider() : 0.0;
 }
 
+using GuestRuns = std::vector<std::pair<std::uint64_t, std::uint32_t>>;
+
+std::uint64_t RunEnd(const std::pair<std::uint64_t, std::uint32_t>& run) {
+    return run.first + static_cast<std::uint64_t>(run.second) * sizeof(std::uint32_t);
+}
+
+bool RunsTouch(const GuestRuns& runs, std::uint64_t begin, std::uint64_t end) {
+    const auto after = std::upper_bound(runs.begin(), runs.end(), begin, [](std::uint64_t address, const std::pair<std::uint64_t, std::uint32_t>& run) { return address < run.first; });
+    if (after != runs.begin() && RunEnd(*std::prev(after)) > begin) return true;
+    return after != runs.end() && after->first < end;
+}
+
+bool RunsOverlap(const GuestRuns& left, const GuestRuns& right) {
+    std::size_t a = 0;
+    std::size_t b = 0;
+    while (a < left.size() && b < right.size()) {
+        if (left[a].first < RunEnd(right[b]) && right[b].first < RunEnd(left[a])) return true;
+        if (RunEnd(left[a]) <= RunEnd(right[b])) ++a;
+        else ++b;
+    }
+    return false;
+}
+
 template<typename TPage>
 std::vector<std::unique_ptr<TPage>>& pagePool() {
     struct PagePoolStorage {};
@@ -104,11 +136,11 @@ void ShaderMemory::SetWaitedMsProvider(WaitedMsProvider provider) {
     waitedMsProvider.store(provider, std::memory_order_release);
 }
 
-ShaderMemory::ShaderMemory(std::span<const ShaderRecompiler::MemoryRegion> regions, PendingWriteQuery pendingWrite, PendingWriteObserver observe, HookWaitCounter hookWaits) : pendingWrite(pendingWrite), observe(observe), hookWaits(hookWaits) {
+ShaderMemory::ShaderMemory(std::span<const ShaderRecompiler::MemoryRegion> regions, PendingWriteQuery pendingWrite, PendingWriteObserver observe, HookWaitCounter hookWaits, PendingWriteQuery pendingSpan) : pendingWrite(pendingWrite), observe(observe), hookWaits(hookWaits), pendingSpan(pendingSpan != nullptr ? pendingSpan : pendingWrite) {
     const ShaderRecompiler::RequestMemoryView validated(regions);
-    for (const auto& region : regions) {
-        initial.emplace(region.guestAddress, region.bytes);
-    }
+    initial.reserve(regions.size());
+    for (const auto& region : regions) initial.emplace_back(region.guestAddress, region.bytes);
+    std::sort(initial.begin(), initial.end(), [](const auto& left, const auto& right) { return left.first < right.first; });
 }
 
 ShaderMemory::~ShaderMemory() {
@@ -153,15 +185,11 @@ void ShaderMemory::WordMask::ForEachRun(TRun&& run) const {
     }
 }
 
-ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) {
-    if (base == lastBase) return *lastPage;
-    auto position = pages.begin();
-    while (position != pages.end() && position->first < base) ++position;
-    if (position != pages.end() && position->first == base) {
-        lastBase = base;
-        lastPage = position->second;
-        return *lastPage;
-    }
+std::size_t ShaderMemory::pageAtOrAfter(std::uint64_t base) const {
+    return static_cast<std::size_t>(std::partition_point(pages.begin(), pages.end(), [base](const std::pair<std::uint64_t, Page*>& entry) { return entry.first < base; }) - pages.begin());
+}
+
+ShaderMemory::Page& ShaderMemory::addPage(std::size_t position, std::uint64_t base) {
     auto& pool = pagePool<Page>();
     std::unique_ptr<Page> owned;
     if (pool.empty()) {
@@ -175,10 +203,21 @@ ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) {
     owned->recent.Reset();
     owned->wordwise = false;
     owned->lazy = false;
-    auto& page = *owned;
-    pages.insert(position, {base, owned.release()});
+    pages.insert(pages.begin() + static_cast<std::ptrdiff_t>(position), {base, owned.get()});
     lastBase = base;
-    lastPage = &page;
+    lastPage = owned.release();
+    return *lastPage;
+}
+
+ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) {
+    if (base == lastBase) return *lastPage;
+    const auto position = pageAtOrAfter(base);
+    if (position != pages.size() && pages[position].first == base) {
+        lastBase = base;
+        lastPage = pages[position].second;
+        return *lastPage;
+    }
+    auto& page = addPage(position, base);
     if (CaptureProfiled()) ++CaptureTotals().pages;
     // A page is either mapped whole or read word by word where the guest mapped less than a page,
     // or where recorded GPU work writes into it (a whole-page read would wait for that work).
@@ -189,6 +228,7 @@ ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) {
             return page;
         }
         const auto started = CaptureProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
+        GuestMemory::GuardRead(base, PageBytes);
         GuestMemory::FlushGpuWrites(base, PageBytes);
         if (CaptureProfiled()) CaptureTotals().pageReadNanoseconds += NanosecondsSince(started);
         page.lazy = true;
@@ -203,7 +243,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
     }
     if (CaptureProfiled()) ++CaptureTotals().reads;
     if (!self.initial.empty() && address + sizeof(*value) > self.initial.begin()->first && address < InitialEnd(self.initial)) {
-        const auto next = self.initial.upper_bound(address);
+        const auto next = std::upper_bound(self.initial.begin(), self.initial.end(), address, [](std::uint64_t value, const auto& entry) { return value < entry.first; });
         if (next != self.initial.begin()) {
             const auto previous = std::prev(next);
             const auto offset = address - previous->first;
@@ -215,6 +255,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
         }
         if (next != self.initial.end() && next->first - address < sizeof(*value)) throw std::runtime_error("AGC driver: shader memory read overlaps a snapshot boundary");
     }
+    if (!self.supplied.empty()) self.spillTouching(address, address + sizeof(*value));
     auto& page = self.page(address & ~static_cast<std::uint64_t>(PageBytes - 1));
     const auto index = static_cast<std::size_t>((address % PageBytes) / sizeof(*value));
     if (!page.valid.Test(index) && page.lazy) {
@@ -249,6 +290,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
                 word = waited;
             }
         } else if (policy == PendingWrite::Raw || policy == PendingWrite::VerifyRaw) {
+            GuestMemory::GuardRead(address, sizeof(word));
             std::memcpy(&word, reinterpret_cast<const void*>(address), sizeof(word));
             ++CaptureTotals().wordsRaw;
             if (policy == PendingWrite::VerifyRaw) {
@@ -289,13 +331,14 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
 
 bool ShaderMemory::overlapsInitial(std::uint64_t begin, std::uint64_t end) const {
     if (initial.empty()) return false;
-    const auto after = initial.lower_bound(begin);
+    const auto after = std::lower_bound(initial.begin(), initial.end(), begin, [](const auto& entry, std::uint64_t value) { return entry.first < value; });
     if (after != initial.end() && after->first < end) return true;
     return after != initial.begin() && std::prev(after)->first + std::prev(after)->second.size() > begin;
 }
 
 bool ShaderMemory::readWords(void* context, std::uint64_t address, std::uint32_t* values, std::uint64_t count) {
     auto& self = *static_cast<ShaderMemory*>(context);
+    if (!self.supplied.empty() && count != 0) self.spillTouching(address, count > (std::numeric_limits<std::uint64_t>::max() - address) / sizeof(std::uint32_t) ? std::numeric_limits<std::uint64_t>::max() : address + count * sizeof(std::uint32_t));
     while (count != 0) {
         const auto base = address & ~static_cast<std::uint64_t>(PageBytes - 1);
         const auto index = static_cast<std::size_t>((address % PageBytes) / sizeof(std::uint32_t));
@@ -303,7 +346,22 @@ bool ShaderMemory::readWords(void* context, std::uint64_t address, std::uint32_t
         const auto bytes = static_cast<std::uint64_t>(words) * sizeof(std::uint32_t);
         Page* page = nullptr;
         if (address % sizeof(std::uint32_t) == 0 && address <= std::numeric_limits<std::uint64_t>::max() - bytes && !self.overlapsInitial(address, address + bytes)) page = &self.page(base);
-        if (page != nullptr && page->valid.All()) {
+        if (page != nullptr && page->lazy) {
+            const auto end = (index + words + BlockWords - 1u) / BlockWords * BlockWords;
+            auto first = index / BlockWords * BlockWords;
+            while (first < end) {
+                if (page->valid.TestBlock(first)) {
+                    first += BlockWords;
+                    continue;
+                }
+                auto last = first + BlockWords;
+                while (last < end && !page->valid.TestBlock(last)) last += BlockWords;
+                std::memcpy(&page->words[first], reinterpret_cast<const void*>(base + first * sizeof(std::uint32_t)), (last - first) * sizeof(std::uint32_t));
+                page->valid.SetRun(first, last - first);
+                first = last;
+            }
+        }
+        if (page != nullptr && (page->lazy || page->valid.All())) {
             std::memcpy(values, page->words.data() + index, bytes);
             page->read.SetRun(index, words);
             page->recent.SetRun(index, words);
@@ -326,7 +384,7 @@ bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_
     const auto end = address + bytes;
     while (address < end) {
         if (!self.initial.empty() && address < InitialEnd(self.initial)) {
-            const auto next = self.initial.upper_bound(address);
+            const auto next = std::upper_bound(self.initial.begin(), self.initial.end(), address, [](std::uint64_t value, const auto& entry) { return value < entry.first; });
             if (next != self.initial.begin()) {
                 const auto previous = std::prev(next);
                 if (address - previous->first < previous->second.size()) {
@@ -351,6 +409,149 @@ bool ShaderMemory::pending(void* context, std::uint64_t address, std::uint64_t b
     return policy == PendingWrite::Sync || policy == PendingWrite::VerifyRaw || policy == PendingWrite::VerifyKnownValue;
 }
 
+ShaderRecompiler::SrtWriteStamp ShaderMemory::stampWrites(void* context, std::uint64_t address, std::uint64_t bytes) {
+    const auto& self = *static_cast<const ShaderMemory*>(context);
+    if (bytes == 0 || address > std::numeric_limits<std::uint64_t>::max() - bytes || self.overlapsInitial(address, address + bytes)) return {};
+    const auto fetched = self.pageAtOrAfter(address & ~static_cast<std::uint64_t>(PageBytes - 1));
+    if (fetched != self.pages.size() && self.pages[fetched].first < address + bytes) return {};
+    GuestMemory::FlushGpuWrites(address, static_cast<std::size_t>(bytes));
+    const auto layout = GuestAllocations::GuestAllocationsGeneration_nid_postfix();
+    const auto generation = GuestMemory::CollectWrites(address, static_cast<std::size_t>(bytes));
+    if (generation == 0) return {};
+    if (self.pendingSpan != nullptr && self.pendingSpan(address, static_cast<std::size_t>(bytes), {}) != PendingWrite::None) return {};
+    return {generation, layout};
+}
+
+bool ShaderMemory::Vouched(PendingWriteQuery pending, std::uint64_t address, std::uint64_t bytes, const ShaderRecompiler::SrtWriteStamp& stamp) {
+    CaptureTotals().vouchCalls.fetch_add(1, std::memory_order_relaxed);
+    if (stamp.generation == 0 || bytes == 0 || address > std::numeric_limits<std::uint64_t>::max() - bytes || stamp.layout != GuestAllocations::GuestAllocationsGeneration_nid_postfix()) return false;
+    if (pending != nullptr && pending(address, static_cast<std::size_t>(bytes), {}) != PendingWrite::None) return false;
+    GuestMemory::FlushGpuWrites(address, static_cast<std::size_t>(bytes));
+    if (GuestMemory::CollectWrites(address, static_cast<std::size_t>(bytes)) == 0) return false;
+    return GuestMemory::UnchangedSince(address, static_cast<std::size_t>(bytes), stamp.generation);
+}
+
+bool ShaderMemory::unchangedSince(void* context, std::uint64_t address, std::uint64_t bytes, const ShaderRecompiler::SrtWriteStamp& stamp) {
+    return Vouched(static_cast<const ShaderMemory*>(context)->pendingSpan, address, bytes, stamp);
+}
+
+bool ShaderMemory::place(std::uint64_t address, const std::uint32_t* values, std::uint64_t count, bool overwrite, bool recent) {
+    while (count != 0) {
+        const auto base = address & ~static_cast<std::uint64_t>(PageBytes - 1);
+        const auto index = static_cast<std::size_t>((address % PageBytes) / sizeof(std::uint32_t));
+        const auto words = static_cast<std::size_t>(std::min<std::uint64_t>(count, PageWords - index));
+        const auto position = pageAtOrAfter(base);
+        Page* found = position != pages.size() && pages[position].first == base ? pages[position].second : nullptr;
+        if (found == nullptr) {
+            if (!overwrite && !GuestMemory::Accessible(reinterpret_cast<const void*>(base), PageBytes)) return false;
+            found = &addPage(position, base);
+        }
+        auto& page = *found;
+        if (page.wordwise && !overwrite) return false;
+        const auto head = index / BlockWords * BlockWords;
+        const auto tail = (index + words + BlockWords - 1) / BlockWords * BlockWords;
+        const bool headOpen = page.lazy && !page.valid.Test(head);
+        const bool tailOpen = page.lazy && !page.valid.Test(tail - 1);
+        if (overwrite || page.valid.None()) {
+            std::memcpy(page.words.data() + index, values, words * sizeof(std::uint32_t));
+        } else {
+            for (std::size_t word = 0; word < words; ++word) {
+                if (!page.valid.Test(index + word)) page.words[index + word] = values[word];
+            }
+        }
+        if (headOpen && head < index) std::memcpy(&page.words[head], reinterpret_cast<const void*>(base + head * sizeof(std::uint32_t)), (index - head) * sizeof(std::uint32_t));
+        if (tailOpen && index + words < tail) std::memcpy(&page.words[index + words], reinterpret_cast<const void*>(base + (index + words) * sizeof(std::uint32_t)), (tail - index - words) * sizeof(std::uint32_t));
+        page.valid.SetRun(page.lazy ? head : index, page.lazy ? tail - head : words);
+        page.read.SetRun(index, words);
+        if (recent) page.recent.SetRun(index, words);
+        address += static_cast<std::uint64_t>(words) * sizeof(std::uint32_t);
+        values += words;
+        count -= words;
+    }
+    return true;
+}
+
+void ShaderMemory::spill(std::size_t index) {
+    auto entry = std::move(supplied[index]);
+    supplied.erase(supplied.begin() + static_cast<std::ptrdiff_t>(index));
+    std::size_t at = 0;
+    for (const auto& [address, count] : entry.words->runs) {
+        static_cast<void>(place(address, entry.words->words.data() + at, count, true, entry.recent));
+        at += count;
+    }
+    CaptureTotals().suppliedSpilled += entry.words->words.size();
+    spilled.push_back(std::move(entry.words));
+}
+
+void ShaderMemory::spillTouching(std::uint64_t begin, std::uint64_t end) {
+    for (std::size_t index = supplied.size(); index-- > 0;) {
+        const auto& entry = supplied[index];
+        if (entry.begin < end && begin < entry.end && RunsTouch(entry.words->runs, begin, end)) spill(index);
+    }
+}
+
+bool ShaderMemory::supplyWords(void* context, const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>& words) {
+    auto& self = *static_cast<ShaderMemory*>(context);
+    if (words == nullptr || words->runs.empty()) return false;
+    std::uint64_t total = 0;
+    std::uint64_t end = 0;
+    if (words->ordered && words->low == words->runs.front().first && words->low < words->high) {
+        total = words->words.size();
+        end = words->high;
+    } else {
+        for (const auto& [address, count] : words->runs) {
+            const auto bytes = static_cast<std::uint64_t>(count) * sizeof(std::uint32_t);
+            if (count == 0 || address % sizeof(std::uint32_t) != 0 || address < end || address > std::numeric_limits<std::uint64_t>::max() - bytes) return false;
+            end = address + bytes;
+            total += count;
+        }
+    }
+    const auto begin = words->runs.front().first;
+    if (total != words->words.size() || self.overlapsInitial(begin, end)) return false;
+    auto& totals = CaptureTotals();
+    for (auto& entry : self.supplied) {
+        if (entry.words != words) continue;
+        entry.recent = true;
+        totals.suppliedShared += total;
+        return true;
+    }
+    const auto fetched = self.pageAtOrAfter(begin & ~static_cast<std::uint64_t>(PageBytes - 1));
+    const bool paged = fetched != self.pages.size() && self.pages[fetched].first < end;
+    const bool overlapping = std::any_of(self.supplied.begin(), self.supplied.end(), [&](const Supplied& entry) { return entry.begin < end && begin < entry.end && RunsOverlap(entry.words->runs, words->runs); });
+    if (paged || overlapping) {
+        for (std::size_t index = self.supplied.size(); index-- > 0;) {
+            const auto& entry = self.supplied[index];
+            if (entry.begin < end && begin < entry.end && RunsOverlap(entry.words->runs, words->runs)) self.spill(index);
+        }
+        std::size_t at = 0;
+        for (const auto& [address, count] : words->runs) {
+            if (!self.place(address, words->words.data() + at, count, false, true)) return false;
+            at += count;
+        }
+        totals.suppliedCopied += total;
+        return true;
+    }
+    const auto spanBegin = begin & ~static_cast<std::uint64_t>(PageBytes - 1);
+    const auto spanEnd = (end + PageBytes - 1) & ~static_cast<std::uint64_t>(PageBytes - 1);
+    if (!GuestMemory::Accessible(reinterpret_cast<const void*>(spanBegin), static_cast<std::size_t>(spanEnd - spanBegin))) {
+        std::uint64_t first = std::numeric_limits<std::uint64_t>::max();
+        std::uint64_t past = 0;
+        for (const auto& [address, count] : words->runs) {
+            const auto from = address & ~static_cast<std::uint64_t>(PageBytes - 1);
+            const auto to = (address + static_cast<std::uint64_t>(count) * sizeof(std::uint32_t) + PageBytes - 1) & ~static_cast<std::uint64_t>(PageBytes - 1);
+            if (from > past || first == std::numeric_limits<std::uint64_t>::max()) {
+                if (first != std::numeric_limits<std::uint64_t>::max() && !GuestMemory::Accessible(reinterpret_cast<const void*>(first), static_cast<std::size_t>(past - first))) return false;
+                first = from;
+            }
+            past = std::max(past, to);
+        }
+        if (!GuestMemory::Accessible(reinterpret_cast<const void*>(first), static_cast<std::size_t>(past - first))) return false;
+    }
+    self.supplied.push_back({words, begin, end, true});
+    totals.suppliedShared += total;
+    return true;
+}
+
 ShaderMemory::KnownValueCounts ShaderMemory::KnownValues() {
     auto& totals = CaptureTotals();
     return {totals.wordsKnown.load(std::memory_order_relaxed), totals.wordsKnownVerified.load(std::memory_order_relaxed), totals.wordsKnownMismatches.load(std::memory_order_relaxed)};
@@ -361,6 +562,24 @@ void ShaderMemory::CountHandleMemo(bool hit) {
     (hit ? CaptureTotals().handleHits : CaptureTotals().handleMisses).fetch_add(1, std::memory_order_relaxed);
 }
 
+ShaderMemory::VouchCounts ShaderMemory::Vouches() {
+    auto& totals = CaptureTotals();
+    return {totals.vouchCalls.load(std::memory_order_relaxed), totals.vouchedBlocks.load(std::memory_order_relaxed), totals.refusedBlocks.load(std::memory_order_relaxed), totals.vouchedWords.load(std::memory_order_relaxed)};
+}
+
+ShaderMemory::SupplyCounts ShaderMemory::Supplies() {
+    auto& totals = CaptureTotals();
+    return {totals.suppliedShared.load(std::memory_order_relaxed), totals.suppliedCopied.load(std::memory_order_relaxed), totals.suppliedSpilled.load(std::memory_order_relaxed)};
+}
+
+std::vector<std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> ShaderMemory::SuppliedBlocks() const {
+    std::vector<std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> blocks;
+    blocks.reserve(supplied.size() + spilled.size());
+    for (const auto& entry : supplied) blocks.push_back(entry.words);
+    blocks.insert(blocks.end(), spilled.begin(), spilled.end());
+    return blocks;
+}
+
 std::shared_ptr<const ShaderRecompiler::ResourceCapture> ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle) {
     return capture(request, handle, nullptr);
 }
@@ -386,6 +605,9 @@ std::shared_ptr<const ShaderRecompiler::ResourceCapture> ShaderMemory::capture(c
     runtime.accessible = &accessible;
     runtime.pendingWrite = &pending;
     runtime.readWords = &readWords;
+    runtime.stampWrites = &stampWrites;
+    runtime.unchangedSince = &unchangedSince;
+    runtime.supplyWords = &supplyWords;
     auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime);
     if (profile) {
         totals.captureNanoseconds += NanosecondsSince(started);
@@ -414,34 +636,159 @@ std::shared_ptr<const ShaderRecompiler::ResourceCapture> ShaderMemory::capture(c
     return capture;
 }
 
-std::vector<ShaderRecompiler::MemoryRegion> ShaderMemory::Regions() const {
+std::vector<ShaderRecompiler::MemoryRegion> ShaderMemory::collect(bool recentOnly) const {
+    struct Cursor {
+        const ShaderRecompiler::SrtGuestWords* words;
+        std::size_t run;
+        std::size_t at;
+    };
+    std::vector<Cursor> cursors;
+    std::size_t runs = 0;
+    for (const auto& entry : supplied) {
+        if (recentOnly && !entry.recent) continue;
+        cursors.push_back({entry.words.get(), 0u, 0u});
+        runs += entry.words->runs.size() + entry.words->words.size() / PageWords;
+    }
     std::vector<ShaderRecompiler::MemoryRegion> result;
-    result.reserve(initial.size() + pages.size());
-    auto next = initial.begin();
-    // Both maps are ordered by address and never overlap, so a merge keeps the result sorted.
+    result.reserve((recentOnly ? 0u : initial.size()) + pages.size() + runs);
+    auto next = recentOnly ? initial.end() : initial.begin();
+    const auto emitBefore = [&](std::uint64_t address) {
+        while (true) {
+            Cursor* lowest = nullptr;
+            for (auto& cursor : cursors) {
+                if (cursor.run < cursor.words->runs.size() && (lowest == nullptr || cursor.words->runs[cursor.run].first < lowest->words->runs[lowest->run].first)) lowest = &cursor;
+            }
+            const auto run = lowest != nullptr ? lowest->words->runs[lowest->run].first : std::numeric_limits<std::uint64_t>::max();
+            if (next != initial.end() && next->first < address && next->first < run) {
+                result.push_back({next->first, next->second});
+                ++next;
+            } else if (lowest != nullptr && run < address) {
+                const auto count = lowest->words->runs[lowest->run].second;
+                const auto words = std::span(lowest->words->words).subspan(lowest->at, count);
+                for (std::size_t done = 0; done < count;) {
+                    const auto at = run + done * sizeof(std::uint32_t);
+                    const auto piece = std::min<std::size_t>(count - done, (PageBytes - static_cast<std::size_t>(at % PageBytes)) / sizeof(std::uint32_t));
+                    result.push_back({at, std::as_bytes(words.subspan(done, piece))});
+                    done += piece;
+                }
+                lowest->at += count;
+                ++lowest->run;
+            } else {
+                return;
+            }
+        }
+    };
     for (const auto& [base, pagePointer] : pages) {
         const auto& page = *pagePointer;
-        while (next != initial.end() && next->first < base) {
-            result.push_back({next->first, next->second});
-            ++next;
-        }
-        page.read.ForEachRun([&](std::size_t first, std::size_t end) { result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, end - first))}); });
+        const auto& marks = recentOnly ? page.recent : page.read;
+        if (marks.None()) continue;
+        marks.ForEachRun([&](std::size_t first, std::size_t end) {
+            const auto address = base + first * sizeof(std::uint32_t);
+            emitBefore(address);
+            result.push_back({address, std::as_bytes(std::span(page.words).subspan(first, end - first))});
+        });
     }
-    for (; next != initial.end(); ++next) result.push_back({next->first, next->second});
+    emitBefore(std::numeric_limits<std::uint64_t>::max());
     return result;
 }
 
+std::vector<ShaderRecompiler::MemoryRegion> ShaderMemory::Regions() const {
+    return collect(false);
+}
+
 std::vector<ShaderRecompiler::MemoryRegion> ShaderMemory::TakeRecentRegions() {
-    std::vector<ShaderRecompiler::MemoryRegion> result;
-    for (auto& [base, pagePointer] : pages) {
-        auto& page = *pagePointer;
-        if (page.recent.None()) continue;
-        page.recent.ForEachRun([&](std::size_t first, std::size_t end) { result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, end - first))}); });
-        page.recent.Reset();
-    }
+    auto result = collect(true);
+    for (auto& [base, page] : pages) page->recent.Reset();
+    for (auto& entry : supplied) entry.recent = false;
     return result;
 }
 
+void AppendSharedWords(std::vector<SharedWords>& shared, std::size_t region, std::size_t first, std::span<const std::byte> bytes, std::span<const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> blocks) {
+    const auto address = reinterpret_cast<std::uintptr_t>(bytes.data());
+    const auto count = bytes.size() / sizeof(std::uint32_t);
+    if (count == 0) return;
+    for (const auto& block : blocks) {
+        if (block == nullptr || block->words.empty()) continue;
+        const auto begin = reinterpret_cast<std::uintptr_t>(block->words.data());
+        const auto end = begin + block->words.size() * sizeof(std::uint32_t);
+        if (address < begin || address >= end || end - address < count * sizeof(std::uint32_t) || (address - begin) % sizeof(std::uint32_t) != 0) continue;
+        const auto offset = static_cast<std::size_t>((address - begin) / sizeof(std::uint32_t));
+        if (!shared.empty()) {
+            auto& last = shared.back();
+            if (last.first + last.count == first && last.offset + last.count == offset && last.region + last.regions == region && !last.block.owner_before(block) && !block.owner_before(last.block)) {
+                last.count += count;
+                ++last.regions;
+                return;
+            }
+        }
+        shared.push_back({first, count, offset, block, region, 1u});
+        return;
+    }
+}
+
+bool SameCapturedWords(std::span<const std::uint32_t> left, std::span<const SharedWords> leftShared, std::span<const std::uint32_t> right, std::span<const SharedWords> rightShared, std::size_t* compared) {
+    if (compared != nullptr) *compared = 0;
+    if (left.size() != right.size()) return false;
+    std::size_t checked = 0;
+    const auto equal = [&](std::size_t from, std::size_t to) {
+        checked += to - from;
+        return std::equal(left.begin() + static_cast<std::ptrdiff_t>(from), left.begin() + static_cast<std::ptrdiff_t>(to), right.begin() + static_cast<std::ptrdiff_t>(from));
+    };
+    std::size_t at = 0;
+    std::size_t next = 0;
+    bool same = true;
+    for (const auto& run : leftShared) {
+        while (next < rightShared.size() && rightShared[next].first < run.first) ++next;
+        if (next == rightShared.size()) break;
+        const auto& other = rightShared[next];
+        const bool placed = other.first == run.first && other.count == run.count && other.offset == run.offset && run.first >= at && run.first <= left.size() && run.count <= left.size() - run.first;
+        if (!placed || run.block.expired() || run.block.owner_before(other.block) || other.block.owner_before(run.block)) continue;
+        if (!equal(at, run.first)) {
+            same = false;
+            break;
+        }
+        at = run.first + run.count;
+    }
+    same = same && equal(at, left.size());
+    if (compared != nullptr) *compared = checked;
+    return same;
+}
+
+std::size_t VouchSharedRegions(std::span<const SharedWords> shared, std::size_t regions, ShaderMemory::PendingWriteQuery pending, std::vector<VouchedRegions>& vouched) {
+    vouched.clear();
+    if (shared.empty() || regions == 0) return 0;
+    thread_local std::vector<std::pair<const ShaderRecompiler::SrtGuestWords*, bool>>* blocksSlot = nullptr;
+    auto& blocks = ShaderRecompiler::ThreadOwned(blocksSlot);
+    blocks.clear();
+    auto& totals = CaptureTotals();
+    std::size_t words = 0;
+    for (const auto& entry : shared) {
+        if (entry.regions == 0 || entry.region > regions || entry.regions > regions - entry.region || (!vouched.empty() && entry.region < vouched.back().end)) continue;
+        const auto block = entry.block.lock();
+        if (block == nullptr || block->low >= block->high) continue;
+        auto known = std::find_if(blocks.begin(), blocks.end(), [&](const auto& seen) { return seen.first == block.get(); });
+        if (known == blocks.end()) {
+            const bool good = ShaderMemory::Vouched(pending, block->low, block->high - block->low, ShaderRecompiler::LoadStamp(*block));
+            (good ? totals.vouchedBlocks : totals.refusedBlocks).fetch_add(1, std::memory_order_relaxed);
+            known = blocks.insert(blocks.end(), {block.get(), good});
+        }
+        if (!known->second) continue;
+        if (!vouched.empty() && vouched.back().end == entry.region) {
+            auto& last = vouched.back();
+            last.end += entry.regions;
+            last.words += entry.count;
+            last.low = std::min(last.low, block->low);
+            last.high = std::max(last.high, block->high);
+        } else {
+            vouched.push_back({entry.region, entry.region + entry.regions, entry.count, block->low, block->high});
+        }
+        words += entry.count;
+    }
+    blocks.clear();
+    totals.vouchedWords.fetch_add(words, std::memory_order_relaxed);
+    return words;
+}
+
 DataWordPositionCounts DataWordPositions(std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint32_t, std::uint64_t>> leaves, std::span<const std::uint64_t> otherReads, std::span<const std::uint32_t> words, std::span<const std::uint32_t> flattenedSrt, std::vector<std::uint32_t>& positions, std::vector<std::uint32_t>& slots) {
     DataWordPositionCounts counts;
     positions.clear();

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -2238,10 +2238,10 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
         // a title misbehaves with this path.
         if (drain) return 2;
         recordStore();
-        GuestMemory::MarkWritten(address, bytes.size());
         // Commands() opened the batch the store went into, so the completion is appended to that
         // batch and runs at its reap, after the write-backs of every batch before it.
         recorder.AfterCompletions(address, bytes, stamp, queue, true);
+        GuestMemory::MarkWritten(address, bytes.size());
         return 5;
     }
     recordStore();
@@ -2695,6 +2695,8 @@ VulkanDevice::CopyOutcome VulkanDevice::CopyBuffer(std::uint64_t destination, st
     recorder.EndGpuTiming(classTiming, bytes);
     recorder.MarkCovered(copiedAccess);
     recorder.NotePendingRead(source, bytes, Graphics::Recorder::ReadKind::CopySource);
+    GuestMemory::MarkWritten(destination, bytes);
+    recorder.NotePendingWrite(destination, bytes);
     // The destination's bytes after the transfer are the source's now when nothing recorded,
     // pending or still to land can change the source before the transfer reads it (a CPU store
     // to it in between is a hardware race too): the caller's ring entry carries them, stamped
@@ -3532,6 +3534,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
     target.float64DenormPreserve = state->float64DenormPreserve;
     target.float16DenormPreserve = state->float16DenormPreserve;
     target.spannedWaves = state->spannedWaves;
+    target.depthRangeRemap = !state->depthRangeUnrestricted;
     target.fixedPushSlots = state->graphicsPipelineLibrary;
     target.registrationModules = state->graphicsPipelineLibrary;
     target.robustBufferAccess2 = state->robustBufferAccess2;

core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp

@@ -20,6 +20,7 @@ SampleOrderTarget DepthSurfaceOrderTarget(const Context& context, const DepthTar
 DepthSampleLocations DepthSurfaceSampleLocations(const Context& context, const DepthTarget& target);
 bool SampleLocationsCompatibleDepth(const Context& context, VkFormat format, std::uint32_t samples);
 std::uint64_t DepthSurfacesSerial();
+std::uint64_t DepthSurfacesReleased();
 std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel);
 std::uint64_t MultisampledDepthBytes_nid_no_patch(VkExtent2D extent, std::uint32_t bytesPerTexel, std::uint32_t samples);
 void RequireNoResidentDepthRead_nid_no_patch(std::uint64_t address, std::size_t bytes);

core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp

@@ -29,7 +29,7 @@ struct DrawInputCopy {
     std::uint64_t registryGeneration = 0;
 };
 DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uint64_t address, std::size_t bytes, std::size_t alignment, Recorder::SnapshotUse use);
-DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes);
+DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes, std::size_t stride = 0);
 void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived);
 const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment, bool* behindGpuWrite = nullptr);
 struct InPlaceIndices {

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -77,6 +77,9 @@ void SetHostImportWindow(std::uint64_t bytes);
 void SetHostImportRangeWindow(std::uint64_t bytes);
 void SetHostImportBudget(std::uint64_t bytes);
 void SetHostImportDeferredSettles(bool enabled);
+// The larger of 6 GiB and half the physical memory, at most the largest host heap less the larger of 2 GiB and half
+// that heap: the other half stays for the buffers, mirrors and textures that share it.
+std::uint64_t HostImportBudget(const VkPhysicalDeviceMemoryProperties& memory, std::uint64_t physicalBytes);
 std::uint64_t HostImportLiveBytes();
 struct HostImportSettleCounts {
     std::uint64_t submitted = 0;

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -73,6 +73,7 @@ private:
         std::vector<bool> owned;
         VkExtent2D extent;
         std::uint32_t layers = 1;
+        std::uint64_t depthReleases = 0;
         std::shared_ptr<Framebuffer> framebuffer;
     };
     void release() noexcept;

core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp

@@ -176,6 +176,7 @@ public:
     static ReadStatistics ReadCounts();
     // Whether the OPEN (unsubmitted) batch writes the range: a wait on such a range must submit it.
     bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const;
+    std::uint64_t PendingWriterSerial(std::uint64_t address, std::size_t bytes) const;
     // What SyncThrough(address, bytes) would wait for, for the [hooksync] attribution: the newest
     // batch whose noted range overlaps the access (the open batch counts first, with the serial it
     // will get), whether that batch's fence has already signaled (its GPU work is done, so the
@@ -763,7 +764,7 @@ private:
     void removeLabels(Batch& batch);
     void release(Batch& batch) noexcept;
     static bool overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end);
-    static std::optional<ReadKind> readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end);
+    static std::optional<ReadKind> readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end, std::optional<ReadKind> only = std::nullopt);
     static std::vector<std::pair<std::uint64_t, std::uint64_t>> notedReads(const Batch& batch);
     bool signaled(const Batch& batch) const;
     // Rebuilds the lock-free snapshot of pending writes from open, inFlight and finishing.

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp

@@ -18,7 +18,9 @@ ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers&
 // the draw cache validates them by value with the stage's capture (design_cpu_final rule RD).
 struct DecodeRead {
     std::uint64_t address;
-    std::vector<std::byte> bytes;
+    std::array<std::byte, 16> data{};
+    std::uint32_t size = 0;
+    std::span<const std::byte> Bytes() const { return {data.data(), size}; }
 };
 ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span<const std::byte> header, std::uint64_t headerAddress, std::span<const std::uint32_t> userData, std::vector<DecodeRead>* reads = nullptr, bool staticAbi = false, std::uint32_t hiddenUserWords = 0);
 

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp

@@ -68,6 +68,13 @@ struct PipelineSetLayouts {
     std::span<const std::uint32_t> fragmentKey;
 };
 
+struct TableDescriptors {
+    std::uint32_t binding = 0;
+    std::vector<std::pair<std::uint32_t, std::uint32_t>> runs;
+    std::vector<VkDescriptorImageInfo> images;
+    std::shared_ptr<const std::vector<std::shared_ptr<Texture>>> textures;
+};
+
 class DescriptorCache {
 public:
     explicit DescriptorCache(const Context& context);
@@ -83,9 +90,13 @@ public:
         VkDescriptorPool pool = VK_NULL_HANDLE;
         std::size_t poolIndex = 0;
         bool updateAfterBind = false;
+        VkDescriptorSetLayout layout = VK_NULL_HANDLE;
+        std::vector<VkDescriptorPoolSize> sizes;
+        std::optional<std::uint32_t> variableCount;
+        std::shared_ptr<const TableDescriptors> table;
     };
     SetAllocation Allocate(VkDescriptorSetLayout layout, std::span<const VkDescriptorPoolSize> sizes, bool updateAfterBind = false, std::optional<std::uint32_t> variableCount = std::nullopt);
-    void Free(const SetAllocation& allocation) noexcept;
+    void Free(const SetAllocation& allocation, std::shared_ptr<const TableDescriptors> table = {}) noexcept;
     SetAllocation AllocateRewritten(VkDescriptorSetLayout layout, std::span<const VkDescriptorPoolSize> sizes, bool updateAfterBind = false, std::optional<std::uint32_t> variableCount = std::nullopt);
     void Recycle(VkDescriptorSetLayout layout, const SetAllocation& allocation) noexcept;
     // APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened.
@@ -95,6 +106,8 @@ public:
         std::uint64_t sets = 0;
         std::uint64_t pools = 0;
         std::uint64_t dedicatedPools = 0;
+        std::uint64_t keptSets = 0;
+        std::uint64_t reusedSets = 0;
     };
     Stats Counters() const;
 
@@ -115,6 +128,7 @@ private:
     std::vector<Pool> updateAfterBindPools;
     std::vector<VkDescriptorPool> dedicated;
     std::unordered_map<VkDescriptorSetLayout, std::vector<SetAllocation>> recycled;
+    std::vector<SetAllocation> kept;
     Stats stats;
 };
 
@@ -125,6 +139,9 @@ using ContentKeyDiff = std::array<bool, static_cast<std::size_t>(ContentKeyCompo
 struct ImageTableReuseCounts {
     std::uint64_t reused = 0;
     std::uint64_t resolved = 0;
+    std::uint64_t replayed = 0;
+    std::uint64_t writesKept = 0;
+    std::uint64_t writesMade = 0;
 };
 ImageTableReuseCounts ImageTableReuses();
 
@@ -181,7 +198,7 @@ public:
         std::size_t allocation;
         std::uint64_t address;
         std::size_t size;
-        std::vector<std::uint32_t> words;
+        std::span<const std::uint32_t> words;
     };
     std::shared_ptr<DrawBindings> PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved = {}, bool movedOnly = false) const;
     std::optional<std::vector<MovedBuffer>> MovedReadOnlyBuffers(std::span<const CompiledShader> shaders, Recorder& recorder) const;
@@ -405,7 +422,7 @@ private:
     // `written` is the element's DescriptorBinding::bufferWritten: a read-only element binds the
     // same way but is left out of the write set (no write-back, no pending-write note). `atomic`
     // is its bufferAtomic (see GuestBufferMemory::AddWritable).
-    std::size_t addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read);
+    std::size_t addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read, std::uint32_t extent);
     std::uint64_t guestBufferBytes(std::span<const std::uint32_t> words) const;
     std::size_t addDataBuffer(std::span<const std::uint32_t> words);
     // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache
@@ -540,6 +557,16 @@ private:
     bool splitSets = false;
     std::vector<std::uint32_t> fragmentLayoutKey;
     SetObjects fragment;
+    struct TableWrites {
+        bool recorded = false;
+        bool written = false;
+        std::uint32_t binding = 0;
+        std::size_t firstImage = 0;
+        std::size_t images = 0;
+        std::vector<std::pair<std::uint32_t, std::uint32_t>> runs;
+    };
+    std::array<TableWrites, 2> tableWrites;
+    std::shared_ptr<const TableDescriptors> keptTable(std::size_t set, std::shared_ptr<const std::vector<std::shared_ptr<Texture>>>& held) noexcept;
     std::uint32_t setOf(const Binding& binding) const;
     std::vector<Allocation> allocations;
     std::vector<VkDescriptorBufferInfo> descriptorBuffers;

core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp

@@ -73,7 +73,7 @@ inline VkShaderStageFlags PushConstantStages(std::span<const CompiledShader> sha
     VkShaderStageFlags result = 0;
     for (const auto& shader : shaders) {
         Require(shader.program != nullptr, "missing compiled shader");
-        if (!shader.program->pushConstants.empty() || shader.stage == ShaderRecompiler::ShaderStage::Mesh || shader.stage == ShaderRecompiler::ShaderStage::TessellationControl) result |= VulkanStage(shader.stage);
+        if (!shader.program->pushConstants.empty() || shader.program->readsDepthRemapPush || shader.stage == ShaderRecompiler::ShaderStage::Mesh || shader.stage == ShaderRecompiler::ShaderStage::TessellationControl) result |= VulkanStage(shader.stage);
     }
     return result;
 }

core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp

@@ -125,6 +125,16 @@ struct State {
     VkViewport viewport;
     bool negativeOneToOne;
     bool depthClamp = false;
+    float depthScale = 1;
+    float depthOffset = 0;
+    float depthClampMin = 0;
+    float depthClampMax = 1;
+    struct DepthRemap {
+        std::uint32_t mode = 0;
+        float scale = 1;
+        float offset = 0;
+        bool operator==(const DepthRemap&) const = default;
+    } depthRemap;
     bool alphaToCoverage = false;
     bool ditheredAlphaToCoverage = false;
     bool perSampleShading = false;
@@ -150,6 +160,7 @@ inline constexpr std::uint32_t PositionExportLayoutBits = 0xffffu | (1u << 21u)
 
 ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup = false);
 State DecodeState(const QueueState& queue, bool vertexWorkgroup = false);
+void ApplyDepthRemap(const Context& context, State& state);
 std::array<VkSampleLocationEXT, 32> SampleLocations(const State& state);
 std::array<std::uint8_t, 32> PixelSampleLocations(const State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);
 VkExtent2D SampleLocationGrid(const State& state);

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -362,6 +362,7 @@ public:
     // Keeps the image current with guest memory (see GuestMemory::CollectWrites).
     bool Refresh();
     static std::uint64_t SinglePassMoves();
+    static std::uint64_t SlabRetiles();
     static std::uint64_t RefreshesProved();
     static std::uint64_t UntrackedCompares();
     std::uint64_t GuestBytes() const;

core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp

@@ -475,6 +475,10 @@ public:
         return std::any_of(textures.begin(), textures.end(), held) || std::any_of(stencilCopy.textures.begin(), stencilCopy.textures.end(), held);
     }
 
+    bool SampledElsewhere() const {
+        return std::any_of(textures.begin(), textures.end(), [](const auto& entry) { return entry.second.use_count() > 1; });
+    }
+
     std::uint64_t written = ++depthWrites;
     bool retired = false;
 
@@ -1200,6 +1204,25 @@ bool takeClearedHtile(const DepthTarget& target) {
 const DepthSurface* drawnDepth = nullptr;
 
 std::atomic<std::uint64_t> surfacesSerial{0};
+std::atomic<std::uint64_t> surfacesReleased{0};
+
+void releaseRetired(const Context& context) {
+    auto* recorder = Recorder::Active();
+    if (recorder != nullptr && recorder->Device() != context.device) return;
+    auto& list = surfaces();
+    for (auto it = list.begin(); it != list.end();) {
+        const auto& surface = **it;
+        if (!surface.retired || surface.context.device != context.device || surface.target.samples != 1u || surface.SampledElsewhere()) {
+            ++it;
+            continue;
+        }
+        if (drawnDepth == &surface) drawnDepth = nullptr;
+        std::shared_ptr<DepthSurface> released = std::move(*it);
+        it = list.erase(it);
+        surfacesReleased.fetch_add(1, std::memory_order_release);
+        if (recorder != nullptr) recorder->Keep(std::move(released));
+    }
+}
 
 std::uint32_t expandedHtileWord(std::uint32_t word, bool stencilInHtile, VkImageAspectFlags cleared) {
     if (!stencilInHtile) return (cleared & VK_IMAGE_ASPECT_DEPTH_BIT) != 0 ? 0xfffc000fu : word;
@@ -1263,6 +1286,10 @@ std::uint64_t DepthSurfacesSerial() {
     return surfacesSerial.load(std::memory_order_acquire);
 }
 
+std::uint64_t DepthSurfacesReleased() {
+    return surfacesReleased.load(std::memory_order_acquire);
+}
+
 std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel) {
     const std::uint32_t blockWidth = bytesPerTexel == 4 ? 128u : 256u;
     const std::uint32_t blockHeight = bytesPerTexel == 1 ? 256u : 128u;
@@ -1390,6 +1417,7 @@ VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target)
         auto sampling = target.samples == 1u ? DepthSamplingState{} : prepareDepthSampling(context, target);
         creationLease = target.samples == 1u || !sampling.refusal.empty() ? GuestAllocations::Lease{} : GuestAllocations::GuestAllocationsAcquireAll_nid_postfix();
         lock.lock();
+        releaseRetired(context);
         surfacesSerial.fetch_add(1, std::memory_order_release);
         surfaces().push_back(std::make_unique<DepthSurface>(context, target, std::move(sampling)));
         auto& fresh = *surfaces().back();

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -15,6 +15,7 @@
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libc/include/General.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "Optimization/include/Optimization/ShaderStageInputInfo.hpp"
@@ -701,12 +702,19 @@ std::map<std::vector<std::uint64_t>, std::string>& validationFailures() {
     return failures;
 }
 
-std::set<std::uint32_t> CachedFragmentOutputs(const Context& context, std::span<const CompiledShader> shaders, const State& state, bool& memoized, bool& hit) {
+std::vector<std::uint64_t>& validationKeyScratch() {
+    thread_local std::vector<std::uint64_t>* keySlot = nullptr;
+    auto& key = ShaderRecompiler::ThreadOwned(keySlot);
+    key.clear();
+    return key;
+}
+
+std::uint32_t CachedFragmentOutputs(const Context& context, std::span<const CompiledShader> shaders, const State& state, bool& memoized, bool& hit) {
     memoized = false;
     hit = false;
-    std::vector<std::uint64_t> key;
+    auto& key = validationKeyScratch();
     const bool keyed = ValidationKey(context, shaders, state, key);
-    static std::map<std::vector<std::uint64_t>, std::set<std::uint32_t>> memo;
+    static std::map<std::vector<std::uint64_t>, std::uint32_t> memo;
     if (keyed) {
         memoized = true;
         std::lock_guard lock(validationMutex());
@@ -715,15 +723,18 @@ std::set<std::uint32_t> CachedFragmentOutputs(const Context& context, std::span<
             return found->second;
         }
     }
-    std::set<std::uint32_t> outputs;
+    std::uint32_t outputs = 0;
     try {
-        outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics, context.shaderClipDistance, context.shaderCullDistance, context.viewportIndexLayer, context.runtimeDescriptorArray, context.demoteToHelperInvocation, context.shaderTerminateInvocation);
+        for (const auto location : ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics, context.shaderClipDistance, context.shaderCullDistance, context.viewportIndexLayer, context.runtimeDescriptorArray, context.demoteToHelperInvocation, context.shaderTerminateInvocation)) {
+            Require(location < 32, "fragment output location exceeds the output mask");
+            outputs |= 1u << location;
+        }
     } catch (const std::exception& error) {
         if (keyed) {
             std::lock_guard lock(validationMutex());
             auto& failures = validationFailures();
             if (failures.size() >= 1024) failures.clear();
-            failures.emplace(std::move(key), error.what());
+            failures.emplace(key, error.what());
         }
         throw;
     }
@@ -731,7 +742,7 @@ std::set<std::uint32_t> CachedFragmentOutputs(const Context& context, std::span<
         std::lock_guard lock(validationMutex());
         // A handful of configurations recur; a runaway key space is dropped wholesale.
         if (memo.size() >= 1024) memo.clear();
-        memo.emplace(std::move(key), outputs);
+        memo.emplace(key, outputs);
     }
     return outputs;
 }
@@ -937,14 +948,20 @@ DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uin
     return copy;
 }
 
-DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes) {
+DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes, std::size_t stride) {
     Require(validBytes <= bytes, "the valid bytes of a vertex fetch exceed the fetch");
     GuestMemory::FlushGpuWrites(address, bytes);
     DrawInputCopy copy;
     copy.buffer = std::make_shared<Buffer>(context, bytes, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT);
     const auto span = copy.buffer->Bytes();
-    GuestMemory::Read(address, span.subspan(0, validBytes), 1);
-    std::fill(span.begin() + static_cast<std::ptrdiff_t>(validBytes), span.end(), std::byte{0});
+    auto readable = validBytes;
+    if (stride != 0 && !GuestMemory::Accessible(reinterpret_cast<const void*>(address), validBytes)) {
+        const auto ranges = GuestMemory::CommittedRanges(address, validBytes);
+        const std::size_t prefix = !ranges.empty() && ranges.front().first == address ? static_cast<std::size_t>(ranges.front().second - address) : 0;
+        if (validBytes - prefix < stride) readable = prefix;
+    }
+    GuestMemory::Read(address, span.subspan(0, readable), 1);
+    std::fill(span.begin() + static_cast<std::ptrdiff_t>(readable), span.end(), std::byte{0});
     return copy;
 }
 
@@ -1015,7 +1032,7 @@ struct DrawInputs {
     std::vector<VkDeviceSize> vertexOffsets;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> inPlaceReads;
     bool vertexInputBehindGpuWrite = false;
-    std::set<std::uint32_t> fragmentOutputs;
+    std::uint32_t fragmentOutputs = 0;
     VkPipelineStageFlags shaderStages = 0;
     std::uint32_t meshGroups = 0;
     std::vector<std::uint32_t> restartTable;
@@ -1173,6 +1190,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
         if (inputs.retainedVertexInput == nullptr) inputs.vertexInput = BuildVertexInputLayout(context, attributes);
     }
     inputs.vertexOffsets.assign(attributes.size(), 0);
+    inputs.vertexHandles.reserve(attributes.size());
     // An indexed draw's vertex offset moves every fetch: the copy must reach the last one.
     if (draw.indexed) {
         Require(draw.firstVertex <= std::numeric_limits<std::uint32_t>::max() - inputs.maxIndex, "indexed draw vertex range overflow");
@@ -1239,7 +1257,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
     for (const auto f : soloFetches) {
         const auto begin = fetches[f].begin;
         const auto bytes = static_cast<std::size_t>(fetches[f].end - begin);
-        auto copy = CopyZeroPaddedDrawInput(context, begin, bytes, fetchValid[f]);
+        auto copy = CopyZeroPaddedDrawInput(context, begin, bytes, fetchValid[f], fetches[f].stride);
         soloHandles[f] = copy.buffer->Handle();
         inputs.vertexBuffers.push_back(std::move(copy.buffer));
     }
@@ -1618,6 +1636,15 @@ struct RecordedDraw {
     std::optional<DepthSampleLocations> depthLocations;
 };
 
+// A remapped viewport depth range reads its scale and offset from the draw's push block (the vertex shader's
+// RemapPositionDepth), so draws that differ only in them share one pipeline; the draw's own state supplies them.
+void patchDepthRemapPush(std::array<std::byte, PipelinePushConstantBytes>& block, const State& state) {
+    if (state.depthRemap.mode == 0u) return;
+    static_assert(ShaderRecompiler::DepthRemapPushOffsetBytes + ShaderRecompiler::DepthRemapPushBytes <= PipelinePushSlotBytes);
+    const std::array<float, ShaderRecompiler::DepthRemapPushBytes / 4> words{state.depthRemap.scale, state.depthRemap.offset};
+    std::memcpy(block.data() + ShaderRecompiler::DepthRemapPushOffsetBytes, words.data(), sizeof(words));
+}
+
 void recordStencilReplay(const Context& context, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, const DrawInputs& inputs, std::span<const std::pair<std::uint32_t, std::shared_ptr<Pipeline>>> replay, std::span<const CompiledShader> shaders, const ShaderResources& resources, const std::array<std::byte, PipelinePushConstantBytes>* bytes = nullptr, VkShaderStageFlags stages = 0) {
     State selected = state;
     auto block = bytes != nullptr ? *bytes : AssemblePushConstants(shaders);
@@ -1625,6 +1652,7 @@ void recordStencilReplay(const Context& context, VkCommandBuffer commands, const
         resources.PatchPushConstants(block);
         stages = PushConstantStages(shaders);
     }
+    patchDepthRemapPush(block, state);
     const auto fragment = std::ranges::find_if(shaders, [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::Fragment && !shader.program->stencilReferences.empty(); });
     Require(fragment != shaders.end(), "stencil replay fragment stage is missing");
     const auto referenceOffset = fragment->pushConstantOffset + fragment->program->stencilReferenceDword * sizeof(std::uint32_t);
@@ -1662,6 +1690,7 @@ void pushDrawConstants(const Pipeline& pipeline, VkCommandBuffer commands, const
         resources.PatchPushConstants(block);
         stages = PushConstantStages(shaders);
     }
+    patchDepthRemapPush(block, state);
     if (state.stages.tessellation) {
         for (const auto& shader : shaders) Require(shader.program->pushConstants.empty() || shader.pushConstantOffset >= PipelinePushSlotBytes || shader.pushConstantOffset + shader.program->pushConstants.size() <= ShaderRecompiler::TessellationLevelPushOffsetBytes, "a stage's push constants overlap the tessellation levels");
         const std::array<float, ShaderRecompiler::TessellationLevelPushBytes / 4> levels{state.stages.minTessellationLevel, state.stages.maxTessellationLevel};
@@ -1949,10 +1978,10 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
 // The state with the outputs the pixel shader lacks masked: Vulkan leaves attachments a pixel
 // shader has no output for undefined, so their writes are masked (shaders that only store to
 // images or buffers keep their targets as they were). Empty when no mask must change.
-std::optional<State> maskedState(const State& state, const std::set<std::uint32_t>& fragmentOutputs) {
+std::optional<State> maskedState(const State& state, std::uint32_t fragmentOutputs) {
     std::optional<State> masked;
     for (std::size_t index = 0; index < state.blends.size(); ++index) {
-        if (fragmentOutputs.contains(static_cast<std::uint32_t>(index)) || state.blends[index].colorWriteMask == 0) continue;
+        if ((index < 32 && ((fragmentOutputs >> index) & 1u) != 0) || state.blends[index].colorWriteMask == 0) continue;
         if (!masked.has_value()) masked = state;
         masked->blends[index].colorWriteMask = 0;
     }
@@ -1987,7 +2016,7 @@ bool RecordDraws() {
 }
 
 std::optional<std::string> KnownValidationFailure(const Context& context, std::span<const CompiledShader> shaders, const State& state) {
-    std::vector<std::uint64_t> key;
+    auto& key = validationKeyScratch();
     if (!ValidationKey(context, shaders, state, key)) return std::nullopt;
     std::lock_guard lock(validationMutex());
     const auto& failures = validationFailures();

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -1259,6 +1259,12 @@ bool wouldRetireImport(const HostImports& state, const ImportRun& run) {
     return previous.base + previous.bytes > run.base;
 }
 
+bool sameRange(const ImportFailure& entry, const GuestAllocations::Lease& lease) {
+    const auto current = entry.ranges.size() == 1 ? leasedRangeOwner(lease, entry.base, entry.base + entry.bytes) : nullptr;
+    if (current != nullptr && entry.bytes < current->bytes) return !entry.ranges.front().owner_before(current) && !current.owner_before(entry.ranges.front());
+    return sameRun(entry, importRun(lease, entry.base));
+}
+
 bool sameRange(const HostImport& entry, const GuestAllocations::Lease& lease) {
     if (entry.ranges.size() == 1 && !entry.joined && entry.bytes < entry.ranges.front()->bytes) {
         const auto current = leasedRangeOwner(lease, entry.base, entry.base + entry.bytes);
@@ -1304,9 +1310,9 @@ std::uint64_t physicalMemoryBytes() {
 
 std::atomic<std::uint64_t> budgetOverride{0};
 
-// Pinned imports count against the driver's system memory heap, which imports and type-3 mirrors share; past it
-// ordinary host allocations fail. The budget is APS5_HOST_IMPORT_MIB, else the larger of 6 GiB and half the physical
-// memory, at most the largest host heap less 2 GiB.
+// Pinned imports count against the driver's system memory heap, which imports, type-3 mirrors, buffers and textures
+// placed in host memory share; past it ordinary host allocations fail. The budget is APS5_HOST_IMPORT_MIB, else
+// HostImportBudget.
 std::uint64_t importBudget(const Context& context) {
     if (const auto set = budgetOverride.load(std::memory_order_relaxed); set != 0) return set;
     static const std::uint64_t configured = []() -> std::uint64_t {
@@ -1314,13 +1320,7 @@ std::uint64_t importBudget(const Context& context) {
         return value != nullptr ? std::strtoull(value, nullptr, 10) << 20u : 0u;
     }();
     if (configured != 0) return configured;
-    std::uint64_t hostHeap = 0;
-    for (std::uint32_t index = 0; index < context.memory.memoryHeapCount; ++index) {
-        if ((context.memory.memoryHeaps[index].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) hostHeap = std::max<std::uint64_t>(hostHeap, context.memory.memoryHeaps[index].size);
-    }
-    const auto fromMemory = std::max<std::uint64_t>(6144ull << 20u, physicalMemoryBytes() / 2);
-    constexpr std::uint64_t reserve = 2048ull << 20u;
-    return hostHeap > reserve ? std::min(fromMemory, hostHeap - reserve) : fromMemory;
+    return HostImportBudget(context.memory, physicalMemoryBytes());
 }
 
 std::atomic<std::uint64_t> rangeImportWindow{64ull << 20u};
@@ -1683,7 +1683,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
         retireImport(context.device, state, it, lease);
         it = next;
     }
-    for (auto it = state.failed.begin(); it != state.failed.end();) it = sameRun(it->second, importRun(lease, it->first)) ? std::next(it) : state.failed.erase(it);
+    for (auto it = state.failed.begin(); it != state.failed.end();) it = sameRange(it->second, lease) ? std::next(it) : state.failed.erase(it);
     for (auto it = state.failedJoins.begin(); it != state.failedJoins.end();) it = sameRun(it->second, joinedRun(lease, it->second.base, it->second.base + it->second.bytes)) ? std::next(it) : state.failedJoins.erase(it);
 }
 
@@ -2982,6 +2982,16 @@ void SetHostImportDeferredSettles(bool enabled) {
 #endif
 }
 
+std::uint64_t HostImportBudget(const VkPhysicalDeviceMemoryProperties& memory, const std::uint64_t physicalBytes) {
+    std::uint64_t hostHeap = 0;
+    for (std::uint32_t index = 0; index < std::min<std::uint32_t>(memory.memoryHeapCount, VK_MAX_MEMORY_HEAPS); ++index) {
+        if ((memory.memoryHeaps[index].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) hostHeap = std::max<std::uint64_t>(hostHeap, memory.memoryHeaps[index].size);
+    }
+    const auto fromMemory = std::max<std::uint64_t>(6144ull << 20u, physicalBytes / 2);
+    const auto reserve = std::max<std::uint64_t>(2048ull << 20u, hostHeap / 2);
+    return hostHeap > reserve ? std::min(fromMemory, hostHeap - reserve) : fromMemory;
+}
+
 std::uint64_t HostImportLiveBytes() {
     auto& state = Imports();
     std::lock_guard lock(state.mutex);

core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp

@@ -645,18 +645,20 @@ std::shared_ptr<Framebuffer> Pipeline::AcquireFramebuffer(std::span<const VkImag
     Require(targets.size() == attachments && owners.size() == colorAttachments, "render targets do not match the pipeline's attachments");
     bool resident = true;
     for (const auto& owner : owners) resident = resident && owner != nullptr;
+    const auto depthReleases = attachments > colorAttachments ? DepthSurfacesReleased() : 0u;
     if (resident) {
         // Entries whose views are gone can never match again and go as soon as no recorded draw holds
         // them (Kept keeps its framebuffer until the batch completes).
-        std::erase_if(framebuffers, [](const CachedFramebuffer& entry) {
+        std::erase_if(framebuffers, [&](const CachedFramebuffer& entry) {
             if (entry.framebuffer.use_count() != 1) return false;
+            if (entry.depthReleases != depthReleases) return true;
             for (std::size_t i = 0; i < entry.owners.size(); ++i) {
                 if (entry.owned[i] && entry.owners[i].expired()) return true;
             }
             return false;
         });
         for (auto it = framebuffers.begin(); it != framebuffers.end(); ++it) {
-            if (it->extent.width != extent.width || it->extent.height != extent.height || it->layers != layers || !std::equal(it->views.begin(), it->views.end(), targets.begin(), targets.end())) continue;
+            if (it->extent.width != extent.width || it->extent.height != extent.height || it->layers != layers || it->depthReleases != depthReleases || !std::equal(it->views.begin(), it->views.end(), targets.begin(), targets.end())) continue;
             // View handles are recycled once a StorageTexture is destroyed, so the owners must be the
             // very objects the views were made for.
             bool same = true;
@@ -681,6 +683,7 @@ std::shared_ptr<Framebuffer> Pipeline::AcquireFramebuffer(std::span<const VkImag
     for (const auto& owner : owners) entry.owned.push_back(owner != nullptr);
     entry.extent = extent;
     entry.layers = layers;
+    entry.depthReleases = depthReleases;
     entry.framebuffer = framebuffer;
     framebuffers.push_back(std::move(entry));
     return framebuffer;

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp

@@ -35,7 +35,7 @@ static_assert(sizeof(VkStencilOpState) == 28, "VkStencilOpState changed: update
 static_assert(sizeof(VkVertexInputBindingDescription) == 12, "VkVertexInputBindingDescription changed: update the recipe codec");
 static_assert(sizeof(VkVertexInputAttributeDescription) == 16, "VkVertexInputAttributeDescription changed: update the recipe codec");
 #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__)
-static_assert(sizeof(State) == 752, "State changed: update CanonicalPipelineState and the recipe codec");
+static_assert(sizeof(State) == 784, "State changed: update CanonicalPipelineState and the recipe codec");
 static_assert(sizeof(ShaderStages) == 112, "ShaderStages changed: update the recipe codec");
 static_assert(sizeof(ShaderRecompiler::MeshConfiguration) == 40, "MeshConfiguration changed: update the recipe codec");
 static_assert(sizeof(ShaderRecompiler::TessellationConfiguration) == 40, "TessellationConfiguration changed: update the recipe codec");
@@ -44,7 +44,7 @@ static_assert(sizeof(ShaderRecompiler::TessellationConfiguration) == 40, "Tessel
 constexpr std::uint32_t FileMagic = 0x4c4a5041u;
 // Bumped when the recipe codec, CanonicalPipelineState or the store key change in a way the
 // layout guards above do not catch (a field renumbered or reinterpreted at the same size).
-constexpr std::uint32_t FileFormat = 3u;
+constexpr std::uint32_t FileFormat = 4u;
 constexpr std::uint32_t ModuleRecord = 1u;
 constexpr std::uint32_t GraphicsRecord = 2u;
 constexpr std::uint32_t ComputeRecord = 3u;
@@ -216,6 +216,7 @@ void encodeState(Writer& writer, const State& state) {
     writer.Bool(state.sampleLocationGrid);
     writer.Bool(state.perSampleShading);
     for (const auto mask : state.quadSampleMask) writer.Raw(mask);
+    writer.U32(state.depthRemap.mode);
 }
 
 void decodeState(Reader& reader, State& state) {
@@ -274,6 +275,7 @@ void decodeState(Reader& reader, State& state) {
     state.sampleLocationGrid = reader.Bool();
     state.perSampleShading = reader.Bool();
     for (auto& mask : state.quadSampleMask) mask = reader.Raw<std::uint16_t>();
+    state.depthRemap.mode = reader.U32();
 }
 
 void encodeRecipe(Writer& writer, const GraphicsPipelineRecipe& recipe) {
@@ -1037,6 +1039,7 @@ State CanonicalPipelineState(const State& state) {
     canonical.sampleLocationGrid = state.sampleLocationGrid;
     canonical.perSampleShading = state.perSampleShading;
     canonical.quadSampleMask = state.quadSampleMask;
+    canonical.depthRemap.mode = state.depthRemap.mode;
     return canonical;
 }
 

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -2856,6 +2856,17 @@ bool Recorder::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const
     return bytes != 0 && open != nullptr && overlaps(*open, address, address + bytes);
 }
 
+std::uint64_t Recorder::PendingWriterSerial(std::uint64_t address, std::size_t bytes) const {
+    if (bytes == 0) return 0;
+    const auto end = address + bytes;
+    const auto writes = [&](const Batch& batch) { return overlaps(batch, address, end) || readOverlap(batch, address, end, ReadKind::AddressBased).has_value(); };
+    if (open != nullptr && writes(*open)) return OpenSerial();
+    for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) {
+        if (writes(**it)) return (*it)->serial;
+    }
+    return 0;
+}
+
 bool Recorder::signaled(const Batch& batch) const {
     return batch.submitted && fenceStatus(batch.fence) == VK_SUCCESS;
 }
@@ -2954,8 +2965,9 @@ void Recorder::NotePendingReadList(std::shared_ptr<const ReadList> ranges, ReadK
     open->readLists.push_back({std::move(ranges), open->reads.size(), kind});
 }
 
-std::optional<Recorder::ReadKind> Recorder::readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end) {
+std::optional<Recorder::ReadKind> Recorder::readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end, std::optional<ReadKind> only) {
     const auto listOverlaps = [&](const Batch::ReadListNote& note) {
+        if (only.has_value() && note.kind != *only) return false;
         const auto& list = *note.ranges;
         const auto found = std::partition_point(list.begin(), list.end(), [&](const auto& range) { return range.second <= address; });
         return found != list.end() && found->first < end;
@@ -2966,7 +2978,7 @@ std::optional<Recorder::ReadKind> Recorder::readOverlap(const Batch& batch, std:
             if (listOverlaps(batch.readLists[next])) return batch.readLists[next].kind;
         }
         const auto& read = batch.reads[index];
-        if (address < read.end && read.begin < end) return read.kind;
+        if ((!only.has_value() || read.kind == *only) && address < read.end && read.begin < end) return read.kind;
     }
     for (; next < batch.readLists.size(); ++next) {
         if (listOverlaps(batch.readLists[next])) return batch.readLists[next].kind;

core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp

@@ -336,7 +336,9 @@ CommandBatch::CommandBatch(const Context& context) : context(context) {
         allocation.commandBufferCount = 1;
         Check(context.Function<PFN_vkAllocateCommandBuffers>("vkAllocateCommandBuffers")(context.device, &allocation, &commands), "vkAllocateCommandBuffers");
         VkFenceCreateInfo info{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
-        Check(context.Function<PFN_vkCreateFence>("vkCreateFence")(context.device, &info, nullptr, &fence), "vkCreateFence");
+        VkFence created = VK_NULL_HANDLE;
+        Check(context.Function<PFN_vkCreateFence>("vkCreateFence")(context.device, &info, nullptr, &created), "vkCreateFence");
+        fence = created;
         VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
         begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
         Check(context.Function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer");

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp

@@ -233,6 +233,14 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span<const st
     info.fetchEmbedded = true;
     info.fetchAttribReg = static_cast<std::uint32_t>(vertexAttribReg);
     info.fetchBufferReg = static_cast<std::uint32_t>(vertexBufferReg);
+    const auto record = [&](std::uint64_t address, std::span<const std::byte> bytes) {
+        if (reads == nullptr) return;
+        auto& read = reads->emplace_back();
+        read.address = address;
+        std::copy(bytes.begin(), bytes.end(), read.data.begin());
+        read.size = static_cast<std::uint32_t>(bytes.size());
+    };
+    if (reads != nullptr) reads->reserve(reads->size() + 2u * shader.num_input_semantics);
     for (std::uint32_t i = 0; i < shader.num_input_semantics; ++i) {
         const auto& semantic = semantics[i];
         if (semantic.static_vb_index == 1 || semantic.static_attribute == 1) throw std::runtime_error("AGC graphics: statically bound vertex attributes are not implemented");
@@ -246,7 +254,7 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span<const st
         std::array<std::byte, 4> attribWordBytes{};
         const auto attribWordAddress = attribTableAddr + static_cast<std::uint64_t>(semantic.semantic) * 4u;
         AgcDriver::GuestMemory::Read(attribWordAddress, attribWordBytes, 4);
-        if (reads != nullptr) reads->push_back({attribWordAddress, {attribWordBytes.begin(), attribWordBytes.end()}});
+        record(attribWordAddress, attribWordBytes);
         std::uint32_t attribWord;
         std::memcpy(&attribWord, attribWordBytes.data(), 4);
         const auto index = attribWord & 0x1fu;
@@ -257,7 +265,7 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span<const st
         std::array<std::byte, 16> sharpBytes{};
         const auto sharpAddress = bufferTableAddr + static_cast<std::uint64_t>(index) * 16u;
         AgcDriver::GuestMemory::Read(sharpAddress, sharpBytes, 4);
-        if (reads != nullptr) reads->push_back({sharpAddress, {sharpBytes.begin(), sharpBytes.end()}});
+        record(sharpAddress, sharpBytes);
         std::array<std::uint32_t, 4> sharp{};
         std::memcpy(sharp.data(), sharpBytes.data(), 16);
         if (info.resourcesNum >= ShaderRecompiler::ShaderVertexStageInfo::MaxResources) throw std::runtime_error("AGC graphics: vertex resource count exceeds the supported domain");

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -375,7 +375,10 @@ struct TextureCounters {
     // ShaderResources::refreshOwnObjects).
     std::atomic<std::uint64_t> ownRefreshes{0};
     std::atomic<std::uint64_t> tablesReused{0};
+    std::atomic<std::uint64_t> tablesReplayed{0};
     std::atomic<std::uint64_t> tablesResolved{0};
+    std::atomic<std::uint64_t> tableWritesKept{0};
+    std::atomic<std::uint64_t> tableWritesMade{0};
     // Snapshot hits over memory outside the write watch: kept by an unchanged UntrackedHint key, or
     // by a byte compare.
     std::atomic<std::uint64_t> untrackedKept{0};
@@ -1574,6 +1577,11 @@ bool TableReuseEnabled() {
     return !disabled;
 }
 
+bool TableReplay() {
+    static const bool disabled = std::getenv("APS5_NO_TABLE_REPLAY") != nullptr;
+    return !disabled;
+}
+
 struct TableState {
     std::uint64_t epoch = 0;
     std::uint64_t unwatched = 0;
@@ -1607,6 +1615,10 @@ struct ReusedTable {
     std::vector<ReusedTableElement> elements;
     std::vector<GuestMemory::UnchangedQuery> queries;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
+    std::vector<std::uint32_t> imageSamplers;
+    std::vector<std::uint32_t> samplers;
+    std::vector<std::string> bound;
+    bool replayable = false;
 };
 
 constexpr std::size_t ReusedTableCount = 4;
@@ -1820,7 +1832,8 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
                         const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element];
                         const bool atomic = element < binding.bufferAtomic.size() && binding.bufferAtomic[element];
                         const bool read = element < binding.bufferRead.size() && binding.bufferRead[element];
-                        const auto index = addGuestBuffer(std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * 4, 4), target, indexAddress, indexBytes, written, atomic, read);
+                        const std::uint32_t extent = element < binding.bufferExtent.size() ? binding.bufferExtent[element] : 0u;
+                        const auto index = addGuestBuffer(std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * 4, 4), target, indexAddress, indexBytes, written, atomic, read, extent);
                         const auto& push = shader.program->pushConstants;
                         if (!push.empty()) {
                             const auto position = shader.program->memoryOffsetDword * 4u + element;
@@ -2142,6 +2155,7 @@ void ShaderResources::buildComplete() {
             images.reserve(imageCount);
             auto& writes = descriptorWrites;
             writes.reserve(bindings.size());
+            std::vector<std::pair<std::size_t, std::size_t>> keptWrites;
             const auto absentImage = [this](std::uint32_t binding) {
                 constexpr auto bindingCount = static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Binding::Count);
                 const auto view = context.nullDescriptors ? VK_NULL_HANDLE : context.placeholderImages->View(binding % bindingCount);
@@ -2149,6 +2163,13 @@ void ShaderResources::buildComplete() {
             };
             for (const auto& binding : bindings) {
                 if (binding.flags != 0) {
+                    const auto set = setOf(binding);
+                    const bool sampled = binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
+                    const auto& allocation = set == 0 ? cacheAllocation : fragment.cacheAllocation;
+                    const bool recorded = sampled && set < tableWrites.size() && allocation.layout != VK_NULL_HANDLE;
+                    std::vector<std::pair<std::uint32_t, std::uint32_t>> runs;
+                    const auto firstWrite = writes.size();
+                    const auto firstImage = images.size();
                     for (std::size_t first = 0; first < binding.imageAllocations.size();) {
                         if (binding.imageAllocations[first] == std::numeric_limits<std::size_t>::max()) {
                             ++first;
@@ -2157,7 +2178,7 @@ void ShaderResources::buildComplete() {
                         auto last = first;
                         while (last < binding.imageAllocations.size() && binding.imageAllocations[last] != std::numeric_limits<std::size_t>::max()) ++last;
                         VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
-                        write.dstSet = setOf(binding) == 0 ? _set : fragment.set;
+                        write.dstSet = set == 0 ? _set : fragment.set;
                         write.dstBinding = binding.layout.binding;
                         write.dstArrayElement = static_cast<std::uint32_t>(first);
                         write.descriptorCount = static_cast<std::uint32_t>(last - first);
@@ -2169,9 +2190,20 @@ void ShaderResources::buildComplete() {
                             else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
                             else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()});
                         }
+                        if (recorded) runs.emplace_back(write.dstArrayElement, write.descriptorCount);
                         writes.push_back(write);
                         first = last;
                     }
+                    if (!recorded) continue;
+                    const auto& held = allocation.table;
+                    auto& counters = TextureCounts();
+                    if (held != nullptr && held->binding == binding.layout.binding && held->runs == runs && std::equal(held->images.begin(), held->images.end(), images.begin() + static_cast<std::ptrdiff_t>(firstImage), images.end(), [](const VkDescriptorImageInfo& a, const VkDescriptorImageInfo& b) { return a.sampler == b.sampler && a.imageView == b.imageView && a.imageLayout == b.imageLayout; })) {
+                        keptWrites.emplace_back(firstWrite, writes.size());
+                        counters.tableWritesKept.fetch_add(1, std::memory_order_relaxed);
+                    } else {
+                        counters.tableWritesMade.fetch_add(1, std::memory_order_relaxed);
+                    }
+                    tableWrites[set] = {true, false, binding.layout.binding, firstImage, images.size() - firstImage, std::move(runs)};
                     continue;
                 }
                 VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
@@ -2212,7 +2244,22 @@ void ShaderResources::buildComplete() {
                 }
                 writes.push_back(write);
             }
-            context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast<std::uint32_t>(writes.size()), writes.data(), 0, nullptr);
+            if (keptWrites.empty()) {
+                context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast<std::uint32_t>(writes.size()), writes.data(), 0, nullptr);
+            } else {
+                std::vector<VkWriteDescriptorSet> made;
+                made.reserve(writes.size());
+                std::size_t next = 0;
+                for (std::size_t index = 0; index < writes.size(); ++index) {
+                    if (next < keptWrites.size() && index == keptWrites[next].first) {
+                        index = keptWrites[next++].second - 1u;
+                        continue;
+                    }
+                    made.push_back(writes[index]);
+                }
+                if (!made.empty()) context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast<std::uint32_t>(made.size()), made.data(), 0, nullptr);
+            }
+            for (auto& table : tableWrites) table.written = table.recorded;
         }
         for (const auto& allocation : allocations) {
             if (allocation.adjustment == 0) continue;
@@ -3688,12 +3735,26 @@ ResourceCache& SharedResourceCache() {
     return *cache;
 }
 
-DescriptorCache::DescriptorCache(const Context& context) : context(context), destroyLayout(context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")), destroyPool(context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")), resetPool(context.Function<PFN_vkResetDescriptorPool>("vkResetDescriptorPool")), freeDescriptorSets(context.individuallyFreeDescriptorSets ? context.Function<PFN_vkFreeDescriptorSets>("vkFreeDescriptorSets") : nullptr) {}
+namespace {
+std::size_t KeptSets() {
+    static const std::size_t count = [] {
+        const char* text = std::getenv("APS5_KEPT_DESCRIPTOR_SETS");
+        return text != nullptr ? static_cast<std::size_t>(std::strtoull(text, nullptr, 10)) : std::size_t{8};
+    }();
+    return count;
+}
+
+}
+
+DescriptorCache::DescriptorCache(const Context& context) : context(context), destroyLayout(context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")), destroyPool(context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")), resetPool(context.Function<PFN_vkResetDescriptorPool>("vkResetDescriptorPool")), freeDescriptorSets(context.individuallyFreeDescriptorSets ? context.Function<PFN_vkFreeDescriptorSets>("vkFreeDescriptorSets") : nullptr) {
+    kept.reserve(KeptSets() + 1u);
+}
 
 DescriptorCache::~DescriptorCache() {
     for (const auto& pool : pools) destroyPool(context.device, pool.handle, nullptr);
     for (const auto& pool : updateAfterBindPools) destroyPool(context.device, pool.handle, nullptr);
     for (const auto pool : dedicated) destroyPool(context.device, pool, nullptr);
+    for (const auto& set : kept) destroyPool(context.device, set.pool, nullptr);
     for (const auto& [key, layout] : layouts) destroyLayout(context.device, layout, nullptr);
 }
 
@@ -3761,6 +3822,17 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l
     allocation.pSetLayouts = &layout;
     if (!fits) {
         dedicated.reserve(dedicated.size() + 1);
+        const auto same = std::find_if(kept.begin(), kept.end(), [&](const SetAllocation& set) {
+            return set.layout == layout && set.updateAfterBind == updateAfterBind && set.variableCount == variableCount && std::equal(set.sizes.begin(), set.sizes.end(), sizes.begin(), sizes.end(), [](const VkDescriptorPoolSize& a, const VkDescriptorPoolSize& b) { return a.type == b.type && a.descriptorCount == b.descriptorCount; });
+        });
+        if (same != kept.end()) {
+            auto reused = std::move(*same);
+            kept.erase(same);
+            dedicated.push_back(reused.pool);
+            ++stats.reusedSets;
+            ++stats.sets;
+            return reused;
+        }
         const auto pool = CreateSetPool(context, sizes, updateAfterBind);
         allocation.descriptorPool = pool;
         VkDescriptorSet set = VK_NULL_HANDLE;
@@ -3772,7 +3844,7 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l
         dedicated.push_back(pool);
         ++stats.dedicatedPools;
         ++stats.sets;
-        return {set, pool};
+        return {set, pool, 0, updateAfterBind, layout, std::vector<VkDescriptorPoolSize>(sizes.begin(), sizes.end()), variableCount, {}};
     }
     auto& chain = updateAfterBind ? updateAfterBindPools : pools;
     for (std::size_t index = chain.size(); index != 0; --index) {
@@ -3836,12 +3908,22 @@ void DescriptorCache::Recycle(VkDescriptorSetLayout layout, const SetAllocation&
     Free(allocation);
 }
 
-void DescriptorCache::Free(const SetAllocation& allocation) noexcept {
+void DescriptorCache::Free(const SetAllocation& allocation, std::shared_ptr<const TableDescriptors> table) noexcept {
     if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return;
     std::lock_guard lock(mutex);
     if (const auto found = std::find(dedicated.begin(), dedicated.end(), allocation.pool); found != dedicated.end()) {
         dedicated.erase(found);
-        destroyPool(context.device, allocation.pool, nullptr);
+        if (KeptSets() == 0u || allocation.layout == VK_NULL_HANDLE) {
+            destroyPool(context.device, allocation.pool, nullptr);
+            return;
+        }
+        if (kept.size() >= KeptSets()) {
+            destroyPool(context.device, kept.back().pool, nullptr);
+            kept.pop_back();
+        }
+        kept.insert(kept.begin(), allocation);
+        kept.front().table = std::move(table);
+        ++stats.keptSets;
         return;
     }
     auto& chain = allocation.updateAfterBind ? updateAfterBindPools : pools;
@@ -3881,7 +3963,7 @@ std::uint64_t ShaderResources::guestBufferBytes(std::span<const std::uint32_t> w
     return byteSize;
 }
 
-std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) {
+std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read, std::uint32_t extent) {
     Require(words.size() == 4, "buffer descriptor must contain four DWORDs");
     const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1] & ~0x40000000u, words[2], words[3]}};
     Require(descriptor.Type() == 0u, "buffer descriptor uses an unsupported type");
@@ -3899,7 +3981,8 @@ std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words
     static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr;
     Require(!written || !overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer");
     written = written || allWritten;
-    const bool swept = read && dispatchThreads != 0 && descriptor.Stride() != 0 && dispatchThreads * descriptor.Stride() * 2u >= byteSize;
+    const std::uint64_t recordBytes = descriptor.Stride() != 0 ? descriptor.Stride() : extent;
+    const bool swept = read && dispatchThreads != 0 && recordBytes != 0 && dispatchThreads * recordBytes * 2u >= byteSize;
     if (written) guestMemory.AddWritable(address, size, atomic, swept);
     else {
         guestMemory.AddReadable(address, size);
@@ -4541,7 +4624,20 @@ void ShaderResources::resolveImageTables() {
         const auto stageSamplers = std::span<const std::shared_ptr<Sampler>>(samplers).subspan(table.firstSampler, table.samplerCount);
         table.elementTextures.assign(binding.count, std::numeric_limits<std::size_t>::max());
         const auto capacityMessage = [&] { return "the image table exceeds the device's capacity of " + std::to_string(context.imageTableCapacity) + " images"; };
-        const auto bind = [&](std::uint32_t element, std::span<const std::uint32_t> words, std::shared_ptr<Texture> texture, const GuestTextureResource& resource, std::uint64_t guestBytes, bool firstLayer) {
+        std::vector<std::uint32_t> samplerKey;
+        for (std::uint32_t sampler = 0; sampler < table.samplerCount; ++sampler) {
+            const auto& source = samplerSources.at(table.firstSampler + sampler);
+            samplerKey.insert(samplerKey.end(), source.words.begin(), source.words.end());
+            samplerKey.push_back((source.compareEnable ? 1u : 0u) | (source.unnormalized ? 2u : 0u) | (source.integerBorder ? 4u : 0u));
+        }
+        const auto place = [&](std::uint32_t element, std::shared_ptr<Texture> texture, const GuestTextureResource& resource, std::uint64_t guestBytes, bool firstLayer) {
+            textures.push_back(std::move(texture));
+            textureFirstLayer.push_back(firstLayer);
+            describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), resource.dccAddress});
+            table.elementTextures[element] = textures.size() - 1u;
+            item->imageAllocations[static_cast<std::size_t>(table.elementBase) + element] = textures.size() - 1u;
+        };
+        const auto bind = [&](std::uint32_t element, std::span<const std::uint32_t> words, std::shared_ptr<Texture> texture, const GuestTextureResource& resource, std::uint64_t guestBytes, bool firstLayer) -> std::string {
             try {
                 RequireFilterMinmax(context, texture->ViewFormat(), binding.imageSamplers.at(element), stageSamplers);
                 RequireBorderSwizzle(resource.bcSwizzle, binding.imageSamplers.at(element), stageSamplers);
@@ -4558,18 +4654,17 @@ void ShaderResources::resolveImageTables() {
                 }
             } catch (const std::runtime_error& error) {
                 table.map[faults + element] = rejectTableElement(table, words, error.what());
-                return;
+                return error.what();
             }
-            textures.push_back(std::move(texture));
-            textureFirstLayer.push_back(firstLayer);
-            describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), resource.dccAddress});
-            table.elementTextures[element] = textures.size() - 1u;
-            item->imageAllocations[static_cast<std::size_t>(table.elementBase) + element] = textures.size() - 1u;
+            place(element, std::move(texture), resource, guestBytes, firstLayer);
+            return {};
         };
         const auto state = CurrentTableState();
         std::vector<std::shared_ptr<Texture>> cached;
         const bool storageTable = std::find(binding.imageWritten.begin(), binding.imageWritten.end(), true) != binding.imageWritten.end();
-        if (auto* reused = storageTable ? nullptr : FindReusedTable(context.device, binding, state); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) {
+        if (auto* reused = storageTable || item == nullptr ? nullptr : FindReusedTable(context.device, binding, state); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) {
+            const bool replay = TableReplay() && reused->replayable && reused->imageSamplers == binding.imageSamplers && reused->samplers == samplerKey;
+            if (!replay) reused->bound.assign(binding.count, {});
             auto next = cached.begin();
             for (std::uint32_t element = 0; element < binding.count; ++element) {
                 const auto words = std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * 8u, 8u);
@@ -4581,13 +4676,26 @@ void ShaderResources::resolveImageTables() {
                     table.map[faults + element] = rejectTableElement(table, words, entry.message);
                 } else {
                     CompletePendingCpuWrites(context, entry.resource.baseAddress, static_cast<std::size_t>(entry.guestBytes));
-                    bind(element, words, std::move(*next++), entry.resource, entry.guestBytes, entry.firstLayer);
+                    if (!replay) {
+                        reused->bound[element] = bind(element, words, std::move(*next++), entry.resource, entry.guestBytes, entry.firstLayer);
+                    } else if (!reused->bound[element].empty()) {
+                        ++next;
+                        table.map[faults + element] = rejectTableElement(table, words, reused->bound[element]);
+                    } else {
+                        place(element, std::move(*next++), entry.resource, entry.guestBytes, entry.firstLayer);
+                    }
                 }
             }
-            counters.tablesReused.fetch_add(1, std::memory_order_relaxed);
+            if (!replay) {
+                reused->imageSamplers = binding.imageSamplers;
+                reused->samplers = samplerKey;
+                reused->replayable = static_cast<std::uint64_t>(table.elementBase) + binding.count <= context.imageTableCapacity;
+            }
+            (replay ? counters.tablesReplayed : counters.tablesReused).fetch_add(1, std::memory_order_relaxed);
             continue;
         }
         ReusedTable fresh;
+        fresh.bound.assign(binding.count, {});
         bool reusable = state.epoch != 0;
         for (std::uint32_t element = 0; element < binding.count; ++element) {
             const auto words = std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * 8u, 8u);
@@ -4676,7 +4784,7 @@ void ShaderResources::resolveImageTables() {
                     fresh.ranges.emplace_back(resource.baseAddress, resource.baseAddress + guestBytes);
                 }
             }
-            bind(element, words, std::move(texture), resource, guestBytes, firstLayer);
+            fresh.bound[element] = bind(element, words, std::move(texture), resource, guestBytes, firstLayer);
         }
         counters.tablesResolved.fetch_add(1, std::memory_order_relaxed);
         if (reusable && CurrentTableState() == state) {
@@ -4685,6 +4793,9 @@ void ShaderResources::resolveImageTables() {
                 fresh.device = context.device;
                 fresh.state = state;
                 fresh.pendingSerial = serial;
+                fresh.imageSamplers = binding.imageSamplers;
+                fresh.samplers = std::move(samplerKey);
+                fresh.replayable = static_cast<std::uint64_t>(table.elementBase) + binding.count <= context.imageTableCapacity;
                 RememberTable(std::move(fresh), binding);
             }
         }
@@ -4694,7 +4805,7 @@ void ShaderResources::resolveImageTables() {
 
 ImageTableReuseCounts ImageTableReuses() {
     const auto& counters = TextureCounts();
-    return {counters.tablesReused.load(std::memory_order_relaxed), counters.tablesResolved.load(std::memory_order_relaxed)};
+    return {counters.tablesReused.load(std::memory_order_relaxed) + counters.tablesReplayed.load(std::memory_order_relaxed), counters.tablesResolved.load(std::memory_order_relaxed), counters.tablesReplayed.load(std::memory_order_relaxed), counters.tableWritesKept.load(std::memory_order_relaxed), counters.tableWritesMade.load(std::memory_order_relaxed)};
 }
 
 void ShaderResources::finishImageTables() {
@@ -4816,12 +4927,29 @@ ShaderResources::~ShaderResources() {
     release();
 }
 
+std::shared_ptr<const TableDescriptors> ShaderResources::keptTable(std::size_t set, std::shared_ptr<const std::vector<std::shared_ptr<Texture>>>& held) noexcept {
+    const auto& table = tableWrites[set];
+    if (!table.written || KeptSets() == 0u || table.firstImage + table.images > descriptorImages.size()) return nullptr;
+    try {
+        if (held == nullptr) held = std::make_shared<const std::vector<std::shared_ptr<Texture>>>(std::move(textures));
+        auto kept = std::make_shared<TableDescriptors>();
+        kept->binding = table.binding;
+        kept->runs = table.runs;
+        kept->images.assign(descriptorImages.begin() + static_cast<std::ptrdiff_t>(table.firstImage), descriptorImages.begin() + static_cast<std::ptrdiff_t>(table.firstImage + table.images));
+        kept->textures = held;
+        return kept;
+    } catch (...) {
+        return nullptr;
+    }
+}
+
 void ShaderResources::release() noexcept {
     // A set from the cache's pool chain goes back to it; a dedicated pool dies with its set.
-    if (cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(cacheAllocation);
+    std::shared_ptr<const std::vector<std::shared_ptr<Texture>>> held;
+    if (cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(cacheAllocation, keptTable(0, held));
     if (pool) context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")(context.device, pool, nullptr);
     if (_layout && ownsLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, _layout, nullptr);
-    if (fragment.cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(fragment.cacheAllocation);
+    if (fragment.cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(fragment.cacheAllocation, keptTable(1, held));
     if (fragment.pool) context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")(context.device, fragment.pool, nullptr);
     if (fragment.layout && fragment.ownsLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, fragment.layout, nullptr);
     cacheAllocation = {};
@@ -4829,6 +4957,7 @@ void ShaderResources::release() noexcept {
     _set = VK_NULL_HANDLE;
     _layout = VK_NULL_HANDLE;
     fragment = {};
+    tableWrites = {};
 }
 
 std::uint32_t ShaderResources::setOf(const Binding& binding) const {
@@ -4878,6 +5007,7 @@ std::optional<std::vector<ShaderResources::MovedBuffer>> ShaderResources::MovedR
                 const bool same = !item.dataWords.empty() ? item.dataWords == binding.guestDescriptor : !patched && std::memcmp(item.buffer->Bytes().data(), binding.guestDescriptor.data(), item.size) == 0;
                 if (same) continue;
                 if (item.dataWords.empty() && patched) return std::nullopt;
+                if (moved.empty()) moved.reserve(allocations.size());
                 moved.push_back({index, 0, item.size, binding.guestDescriptor});
                 continue;
             }
@@ -4903,6 +5033,7 @@ std::optional<std::vector<ShaderResources::MovedBuffer>> ShaderResources::MovedR
                 const auto bytes = static_cast<std::size_t>(GuestBufferMemory::ViewBytes(size, item.adjustment));
                 if (!GuestMemory::Accessible(reinterpret_cast<const void*>(begin), bytes)) return std::nullopt;
                 if (guestMemory.WritesOverlap(begin, bytes) || recorder.PendingWriteOverlaps(begin, bytes) || PendingStorageOverlaps(begin, bytes, nullptr) || AnyShadowedOverlaps(begin, bytes)) return std::nullopt;
+                if (moved.empty()) moved.reserve(allocations.size());
                 moved.push_back({index, address, static_cast<std::size_t>(size)});
             }
         }
@@ -4994,12 +5125,16 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
     selected.clear();
     std::array<std::uint64_t, 3> stamp{};
     bool stamped = false;
+    const auto keep = [&](DrawBindings::Snapshot snapshot) {
+        if (result->snapshots.empty()) result->snapshots.reserve(allocations.size());
+        result->snapshots.push_back(std::move(snapshot));
+    };
     for (std::size_t index = 0; index < allocations.size(); ++index) {
         const auto& item = allocations[index];
         const auto override = std::find_if(moved.begin(), moved.end(), [&](const MovedBuffer& entry) { return entry.allocation == index; });
         if (override != moved.end() && !override->words.empty()) {
             constexpr std::size_t MovedWordBufferSlots = 8;
-            const auto held = std::find_if(movedWordBuffers.begin(), movedWordBuffers.end(), [&](const MovedWordBuffer& entry) { return entry.allocation == index && entry.size == override->size && entry.words == override->words; });
+            const auto held = std::find_if(movedWordBuffers.begin(), movedWordBuffers.end(), [&](const MovedWordBuffer& entry) { return entry.allocation == index && entry.size == override->size && std::ranges::equal(entry.words, override->words); });
             std::shared_ptr<Buffer> buffer;
             if (held != movedWordBuffers.end()) {
                 buffer = held->buffer;
@@ -5009,12 +5144,12 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
                 for (const auto& patch : dataPatches) {
                     if (patch.allocation == index && patch.byte < override->size) buffer->Bytes()[patch.byte] = static_cast<std::byte>(patch.adjustment);
                 }
-                MovedWordBuffer entry{index, override->words, override->size, buffer};
+                MovedWordBuffer entry{index, {override->words.begin(), override->words.end()}, override->size, buffer};
                 if (movedWordBuffers.size() < MovedWordBufferSlots) movedWordBuffers.push_back(std::move(entry));
                 else movedWordBuffers[movedWordBufferNext++ % MovedWordBufferSlots] = std::move(entry);
             }
             selected.push_back(index);
-            result->snapshots.push_back({0, std::move(buffer), 0, override->size});
+            keep({0, std::move(buffer), 0, override->size});
             continue;
         }
         std::uint64_t address = item.address;
@@ -5066,7 +5201,7 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
             else if (!uploaded || stamp[1] != 0) recorder.NoteUnwrittenDrawSnapshot(begin, bytes, stamp, buffer, offset);
         }
         selected.push_back(index);
-        result->snapshots.push_back({begin, std::move(buffer), offset, bytes});
+        keep({begin, std::move(buffer), offset, bytes});
         totalSnapshotBytes += bytes;
         CaptureTrace::Log("draw-snapshot batch=%llu address=%llx bytes=%zu", static_cast<unsigned long long>(recorder.Submissions() + 1), static_cast<unsigned long long>(begin), bytes);
     }

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp

@@ -541,7 +541,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
                 }
             }
         } else if (variable.storage == spv::StorageClassPushConstant) {
-            Require(!push && (!shader.pushConstants.empty() || mesh || (control && !rectListControl)) && (type[0] & 0xffffu) == spv::OpTypeStruct, "invalid push constant interface");
+            Require(!push && (!shader.pushConstants.empty() || shader.readsDepthRemapPush || mesh || (control && !rectListControl)) && (type[0] & 0xffffu) == spv::OpTypeStruct, "invalid push constant interface");
             push = true;
             Require(type.size() == 3 && module.decorations[typeId].block, "push constant variable must be a Block struct with exactly one member");
             const auto offset = module.offsets.find({typeId, 0});
@@ -613,7 +613,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
         Require(shader.vertexAttributes.empty(), "vertex attribute metadata is invalid for this stage");
     }
     Require(descriptors.size() == shader.bindings.size(), "recompiler binding metadata contains undeclared resources");
-    Require(push == (!shader.pushConstants.empty() || mesh || (control && !rectListControl && !barycentricPatches)), "recompiler push constant metadata disagrees with SPIR-V");
+    Require(push == (!shader.pushConstants.empty() || shader.readsDepthRemapPush || mesh || (control && !rectListControl && !barycentricPatches)), "recompiler push constant metadata disagrees with SPIR-V");
     for (const auto id : module.interface) Require(module.variables.contains(id), "entry point interface contains an unknown variable");
     if (mesh) Require(module.primitiveIndices == state.stages.mesh->outputPrimitiveSize, "mesh shader does not export primitive indices");
     if (stage == Stage::Vertex || mesh || evaluation || geometry) Require(module.position, "vertex shader does not export position");

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -1108,6 +1108,32 @@ ReversedBlend TranslateReversedBlend(std::uint32_t blendControl, std::uint8_t co
     return {blend, routing->code};
 }
 
+void ApplyDepthRemap(const Context& context, State& state) {
+    const auto& viewport = state.viewport;
+    const bool inRange = viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1;
+    if (context.depthRangeUnrestricted || inRange) return;
+    Require(state.stages.path == ShaderPath::Vertex && !state.stages.mesh && !state.stages.tessellation, "viewport depth outside [0, 1] is remapped on the vertex path only");
+    Require(context.depthClamp, "viewport depth outside [0, 1] needs depth clamping to stand in for the console's clamp");
+    const float clampMin = std::clamp(state.depthClampMin, 0.0f, 1.0f);
+    const float clampMax = std::clamp(state.depthClampMax, 0.0f, 1.0f);
+    const bool floatDepth = state.depth.has_value() && (state.depth->format == VK_FORMAT_D32_SFLOAT || state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT);
+    if ((clampMin != state.depthClampMin || clampMax != state.depthClampMax) && floatDepth && (state.depthWrite || state.depthTest)) {
+        static bool reported = false;
+        if (!reported) {
+            reported = true;
+            std::fprintf(stderr, "[gpu] viewport depth clamp bounds [%g, %g] on a float depth target are clipped to [0, 1]: the device has no VK_EXT_depth_range_unrestricted\n", state.depthClampMin, state.depthClampMax);
+        }
+    }
+    const float range = clampMax - clampMin;
+    const float scale = range > 0.0f ? state.depthScale / range : state.depthScale;
+    const float offset = range > 0.0f ? (state.depthOffset - clampMin) / range : state.depthOffset;
+    state.depthRemap = {state.negativeOneToOne ? 2u : 1u, scale, offset};
+    state.viewport.minDepth = clampMin;
+    state.viewport.maxDepth = clampMax;
+    state.negativeOneToOne = false;
+    state.depthClamp = true;
+}
+
 State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
     if (auto reason = DepthMaintenanceRejection(queue); !reason.empty()) throw std::runtime_error(reason);
     const auto& cx = queue.context;
@@ -1342,6 +1368,10 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
         throw std::runtime_error(message.str());
     }
     Require(readFloat(cx, 0xb4) <= readFloat(cx, 0xb5), "inverted viewport depth clamp bounds");
+    result.depthScale = zs;
+    result.depthOffset = zo;
+    result.depthClampMin = readFloat(cx, 0xb4);
+    result.depthClampMax = readFloat(cx, 0xb5);
     result.viewport = collapsed ? VkViewport{xo, yo, 1, 1, minDepth, maxDepth} : VkViewport{xo - xs, yo - ys, 2 * xs, 2 * ys, minDepth, maxDepth};
     if (const auto renderControl = find(cx, 0x000); result.depth && renderControl != cx.end() && (renderControl->second & 1u) != 0) {
         result.viewport.minDepth = result.depth->clearDepth;

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -87,7 +87,7 @@ thread_local const char* uploadReason = "first";
 // selections widened to every pending unit (the hysteresis in writeBack), units dropped as dead
 // (DiscardPendingInside) and units an alias's own results superseded (markLayersPending); relaxed,
 // reported only.
-std::atomic<std::uint64_t> singlePassMoves{0};
+std::atomic<std::uint64_t> singlePassMoves{0}, slabRetiles{0};
 std::atomic<std::uint64_t> partialUploads{0}, partialUploadBytes{0}, partialWriteBacks{0}, partialWriteBackBytes{0}, emptyWriteBacks{0}, coalescedWriteBacks{0}, unitsDropped{0}, unitsSuperseded{0};
 // FlushPending listings skipped because no pending unit of the image lay inside the range, images
 // whose selected pending units were dropped because their memory is no longer registered (see
@@ -1003,7 +1003,9 @@ Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>&
         Require(source != nullptr && CanCopyFrom(*source, descriptor), "storage image does not match the sampled texture");
         const auto vkFormat = SampledTextureFormat(context, descriptor.format);
         const auto geometry = DescribeSurface(descriptor);
+#ifdef APS5_DEBUG_LOG
         AGC_LOG_DEFERRED(stdout, "Texture address=0x%llx %ux%u mips=%u viewed from storage image (vk=%d)", static_cast<unsigned long long>(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, static_cast<int>(vkFormat));
+#endif
         // Storage images stay in the general layout; the view samples them there.
         layout = VK_IMAGE_LAYOUT_GENERAL;
         const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u;
@@ -1356,8 +1358,10 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler,
         Require(mipLevel < descriptor.mipCount, "storage texture mip level is outside the texture");
         const auto vkFormat = StorageFormatFor(context, ResolveTextureFormat(descriptor.format));
         storageFormat = vkFormat;
+#ifdef APS5_DEBUG_LOG
         AGC_LOG_DEFERRED(stdout, "StorageTexture address=0x%llx %ux%u mips=%u mip=%u layers=%u base=%u dim=%d tile=%d format=%u vk=%d", static_cast<unsigned long long>(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, mipLevel,
                      descriptor.depthOrLastArray, descriptor.baseArray, static_cast<int>(descriptor.dimension), static_cast<int>(descriptor.tileMode), descriptor.format, static_cast<int>(vkFormat));
+#endif
         geometry = DescribeSurface(descriptor);
         mips = geometry.mips;
         arrayLayers = geometry.layers;
@@ -1418,28 +1422,48 @@ namespace {
 class PendingList {
 public:
     using iterator = std::vector<StorageTexture*>::iterator;
+    using Ranges = std::vector<std::pair<std::uint64_t, std::uint64_t>>;
     iterator begin() { return textures.begin(); }
     iterator end() { return textures.end(); }
     void push_back(StorageTexture* texture) {
         textures.push_back(texture);
-        ++version;
+        changed();
     }
     iterator erase(iterator it) {
-        ++version;
-        return textures.erase(it);
+        const auto next = textures.erase(it);
+        changed();
+        return next;
     }
     void remove(const StorageTexture* texture) {
-        if (std::erase(textures, texture) != 0) ++version;
+        if (std::erase(textures, texture) != 0) changed();
     }
     bool MayOverlap(std::uint64_t address, std::size_t bytes) {
-        if (textures.empty() || bytes == 0) return false;
         if (indexed != version) rebuild();
+        return !textures.empty() && Overlaps(ranges, address, bytes);
+    }
+    bool SurelyClear(std::uint64_t address, std::size_t bytes) const {
+        const auto current = published.load(std::memory_order_acquire);
+        return current != nullptr && current->version == changes.load(std::memory_order_acquire) && !Overlaps(current->ranges, address, bytes);
+    }
+
+private:
+    struct Snapshot {
+        std::uint64_t version = 0;
+        Ranges ranges;
+    };
+
+    static bool Overlaps(const Ranges& ranges, std::uint64_t address, std::size_t bytes) {
+        if (bytes == 0) return false;
         const auto end = bytes > std::numeric_limits<std::uint64_t>::max() - address ? std::numeric_limits<std::uint64_t>::max() : address + bytes;
         const auto it = std::upper_bound(ranges.begin(), ranges.end(), address, [](std::uint64_t value, const std::pair<std::uint64_t, std::uint64_t>& range) { return value < range.second; });
         return it != ranges.end() && it->first < end;
     }
 
-private:
+    void changed() {
+        ++version;
+        changes.store(version, std::memory_order_release);
+    }
+
     void rebuild() {
         ranges.clear();
         for (const auto* texture : textures) ranges.emplace_back(texture->Descriptor().baseAddress, texture->Descriptor().baseAddress + texture->GuestBytes());
@@ -1451,11 +1475,14 @@ private:
         }
         ranges.resize(merged);
         indexed = version;
+        published.store(std::make_shared<const Snapshot>(Snapshot{version, ranges}), std::memory_order_release);
     }
     std::vector<StorageTexture*> textures;
-    std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
+    Ranges ranges;
     std::uint64_t version = 0;
     std::uint64_t indexed = ~std::uint64_t{0};
+    std::atomic<std::uint64_t> changes{0};
+    std::atomic<std::shared_ptr<const Snapshot>> published;
 };
 
 // Storage images whose results have not reached guest memory yet.
@@ -2271,6 +2298,10 @@ std::uint64_t StorageTexture::SinglePassMoves() {
     return singlePassMoves.load(std::memory_order_relaxed);
 }
 
+std::uint64_t StorageTexture::SlabRetiles() {
+    return slabRetiles.load(std::memory_order_relaxed);
+}
+
 bool StorageTexture::otherPendingOverlaps() const {
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
@@ -3117,6 +3148,15 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
     std::vector<SlabPieces> slabPieces;
     std::vector<std::vector<VkBufferCopy>> importCopies(segments.size());
     std::vector<SeedUnit> seeds;
+    struct AddedPiece {
+        std::size_t pieces;
+        std::size_t copy;
+        std::uint64_t begin;
+        std::uint64_t end;
+        bool slab;
+        bool seeded;
+    };
+    std::vector<AddedPiece> added;
     const auto addPiece = [&](std::uint64_t scratchOffset, std::uint64_t guestBegin, std::uint64_t guestEnd) {
         while (guestBegin < guestEnd) {
             std::size_t segment = 0;
@@ -3132,9 +3172,11 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
                 for (const auto& [seedBegin, seedEnd] : destination->seedUnits) {
                     if (std::none_of(seeds.begin(), seeds.end(), [&](const SeedUnit& seed) { return seed.begin == seedBegin; })) seeds.push_back({seedBegin, seedEnd, destination->slab, &import});
                 }
+                added.push_back({slabPieces.size() - 1, slabPieces.back().copies.size() - 1, guestBegin - descriptor.baseAddress, pieceEnd - descriptor.baseAddress, true, !destination->seedUnits.empty()});
                 if (!shadowed.empty() && shadowed.back().end == guestBegin && shadowed.back().slab == destination->slab) shadowed.back().end = pieceEnd;
                 else shadowed.push_back({guestBegin, pieceEnd, destination->slab});
             } else {
+                added.push_back({0, 0, guestBegin - descriptor.baseAddress, pieceEnd - descriptor.baseAddress, false, false});
                 importCopies[segment].push_back({scratchOffset, HostImportOffset(import, guestBegin), pieceEnd - guestBegin});
                 if (!imported.empty() && imported.back().second == guestBegin) imported.back().second = pieceEnd;
                 else imported.emplace_back(guestBegin, pieceEnd);
@@ -3143,16 +3185,37 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
             guestBegin = pieceEnd;
         }
     };
+    struct SlabRetile {
+        std::size_t window;
+        std::size_t first;
+        std::size_t last;
+    };
+    std::vector<SlabRetile> slabRetiled;
+    const auto blockBytes = ThinBlockLayout(descriptor.tileMode, elementBytes)[0];
     for (std::size_t i = 0; i < windows.size(); ++i) {
         const auto& window = windows[i];
         // The tail levels follow each other, sharing the previous one's region.
         if (i != 0 && scratchPositions[i] == scratchPositions[i - 1]) continue;
+        const auto first = added.size();
+        std::size_t kept = 0;
+        bool whole = false;
         for (const auto& [from, to] : runs) {
             const auto begin = std::max(from, window.tiledBegin);
             const auto end = std::min(to, window.tiledEnd);
             if (begin >= end) continue;
+            ++kept;
             addPiece(scratchPositions[i] + (begin - window.tiledBegin), descriptor.baseAddress + begin, descriptor.baseAddress + end);
+            whole = begin == window.tiledBegin && end == window.tiledEnd;
         }
+        if (save != nullptr || geometry.thick || mips[window.level].tail || std::count(scratchPositions.begin(), scratchPositions.end(), scratchPositions[i]) != 1 || kept != 1 || !whole) continue;
+        const auto last = added.size();
+        const auto parts = std::span<const AddedPiece>(added).subspan(first, last - first);
+        const auto sliceBegin = window.tiledBegin - window.window.rangeBegin;
+        const auto intoSlabs = std::all_of(parts.begin(), parts.end(), [&](const AddedPiece& part) {
+            const auto& copy = slabPieces[part.pieces].copies[part.copy];
+            return part.slab && !part.seeded && copy.dstOffset % 4 == 0 && copy.size % 4 == 0 && (part.begin - sliceBegin) % blockBytes == 0 && (part.end - sliceBegin) % blockBytes == 0;
+        });
+        if (intoSlabs) slabRetiled.push_back({i, first, last});
     }
     std::sort(seeds.begin(), seeds.end(), [](const SeedUnit& a, const SeedUnit& b) { return a.begin < b.begin; });
     std::vector<CopiedBytes> copied;
@@ -3169,7 +3232,88 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
     if (save != nullptr) {
         for (const auto& piece : copied) saves.push_back({piece.scratch, piece.begin, piece.end - piece.begin});
     }
-    const auto padding = paddingSeeds(segments, copied);
+    auto padding = paddingSeeds(segments, copied);
+    struct DirectPiece {
+        VkBuffer slab;
+        VkDeviceSize offset;
+        DetileWindow window;
+        std::uint64_t linearOffset;
+    };
+    std::vector<DirectPiece> directPieces;
+    std::vector<std::pair<std::size_t, std::size_t>> directRanges(windows.size(), {0, 0});
+    struct HoleCopies {
+        VkBuffer source;
+        VkBuffer slab;
+        std::vector<VkBufferCopy> copies;
+    };
+    std::vector<HoleCopies> holeCopies;
+    if (!slabRetiled.empty()) {
+        std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
+        for (const auto& retiled : slabRetiled) ranges.emplace_back(windows[retiled.window].tiledBegin, windows[retiled.window].tiledEnd);
+        const auto holes = uncoveredBytes(ranges);
+        for (const auto& retiled : slabRetiled) {
+            const auto& window = windows[retiled.window];
+            const auto parts = std::span<const AddedPiece>(added).subspan(retiled.first, retiled.last - retiled.first);
+            const auto position = scratchPositions[retiled.window];
+            const auto length = window.tiledEnd - window.tiledBegin;
+            const auto inside = [&](const VkBufferCopy& hole) { return hole.size != 0 && hole.dstOffset >= position && hole.dstOffset + hole.size <= position + length; };
+            const auto partCopy = [&](const AddedPiece& part) -> VkBufferCopy& { return slabPieces[part.pieces].copies[part.copy]; };
+            const auto partSlab = [&](const AddedPiece& part) { return slabPieces[part.pieces].slab->buffer; };
+            std::uint64_t holeBytes = 0, sourcedBytes = 0;
+            for (const auto& [first, last] : holes) {
+                const auto begin = std::max(first, window.tiledBegin);
+                const auto end = std::min(last, window.tiledEnd);
+                if (begin < end) holeBytes += end - begin;
+            }
+            bool sameSlabMoved = false;
+            for (const auto& [source, copies] : padding.copies) {
+                for (const auto& hole : copies) {
+                    if (!inside(hole)) continue;
+                    sourcedBytes += hole.size;
+                    const auto holeBegin = window.tiledBegin + (hole.dstOffset - position);
+                    for (const auto& part : parts) {
+                        const auto from = std::max(holeBegin, part.begin);
+                        const auto to = std::min(holeBegin + hole.size, part.end);
+                        if (from < to && source == partSlab(part) && hole.srcOffset + (from - holeBegin) != partCopy(part).dstOffset + (from - part.begin)) sameSlabMoved = true;
+                    }
+                }
+            }
+            if (sourcedBytes != holeBytes || sameSlabMoved) continue;
+            for (auto& [source, copies] : padding.copies) {
+                for (auto& hole : copies) {
+                    if (!inside(hole)) continue;
+                    const auto holeBegin = window.tiledBegin + (hole.dstOffset - position);
+                    for (const auto& part : parts) {
+                        const auto from = std::max(holeBegin, part.begin);
+                        const auto to = std::min(holeBegin + hole.size, part.end);
+                        if (from >= to || source == partSlab(part)) continue;
+                        const auto slab = partSlab(part);
+                        auto entry = std::find_if(holeCopies.begin(), holeCopies.end(), [&](const HoleCopies& entry) { return entry.source == source && entry.slab == slab; });
+                        if (entry == holeCopies.end()) entry = holeCopies.insert(holeCopies.end(), {source, slab, {}});
+                        entry->copies.push_back({hole.srcOffset + (from - holeBegin), partCopy(part).dstOffset + (from - part.begin), to - from});
+                    }
+                    hole.size = 0;
+                }
+            }
+            directRanges[retiled.window] = {directPieces.size(), directPieces.size() + parts.size()};
+            for (const auto& part : parts) {
+                auto& copy = partCopy(part);
+                if (parts.size() == 1) {
+                    directPieces.push_back({partSlab(part), copy.dstOffset, window.window, 0});
+                } else {
+                    const std::pair<std::uint64_t, std::uint64_t> run{part.begin, part.end};
+                    const auto narrowed = sliceWindows(std::span(&run, 1));
+                    Require(narrowed.size() == 1, "a storage write-back slab piece is not one window");
+                    directPieces.push_back({partSlab(part), copy.dstOffset, narrowed.front().window, narrowed.front().window.linearBase - window.window.linearBase});
+                }
+                copy.size = 0;
+            }
+            slabRetiles.fetch_add(1, std::memory_order_relaxed);
+        }
+        for (auto& pieces : slabPieces) std::erase_if(pieces.copies, [](const VkBufferCopy& copy) { return copy.size == 0; });
+        for (auto& entry : padding.copies) std::erase_if(entry.second, [](const VkBufferCopy& copy) { return copy.size == 0; });
+        std::erase_if(padding.copies, [](const auto& entry) { return entry.second.empty(); });
+    }
     // A queued label or key store inside a seeded unit lands before the seed copies the import.
     auto flushBegin = firstStored;
     auto flushEnd = lastStored;
@@ -3179,7 +3323,8 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
     }
     const bool direct = singlePass();
     auto linear = direct ? nullptr : std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
-    auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+    const bool scratchUsed = std::any_of(directRanges.begin(), directRanges.end(), [](const auto& range) { return range.first == range.second; }) || std::any_of(slabPieces.begin(), slabPieces.end(), [](const SlabPieces& pieces) { return !pieces.copies.empty(); }) || std::any_of(importCopies.begin(), importCopies.end(), [](const auto& copies) { return !copies.empty(); }) || !saves.empty() || !padding.copies.empty() || !seeds.empty();
+    auto tiledScratch = scratchUsed ? std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT) : nullptr;
     detiler.BeginBatch();
     auto* recorder = Recorder::Active();
     std::unique_ptr<CommandBatch> batch;
@@ -3192,11 +3337,11 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
         commands = recorder->Commands(&covered);
         timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack);
         if (linear) recorder->Keep(linear, linear->Size());
-        recorder->Keep(tiledScratch, tiledScratch->Size());
+        if (tiledScratch) recorder->Keep(tiledScratch, tiledScratch->Size());
         for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab);
         for (const auto& slab : padding.slabs) recorder->Keep(slab);
         if (auto self = weak_from_this().lock()) keepStorageTexture(*recorder, std::move(self));
-        Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, direct ? 3 : 4);
+        Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, (direct ? 3 : 4) - (tiledScratch ? 0 : 1));
         if (Recorder::BarrierValidate()) {
             const std::pair<VkImage, bool> read{image, false};
             std::vector<std::pair<std::uint64_t, std::uint64_t>> seedReads;
@@ -3218,7 +3363,7 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
     toSource.image = image;
     toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
     if (!direct) context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource);
-    if (!seeds.empty() || !padding.copies.empty()) {
+    if (!seeds.empty() || !padding.copies.empty() || !holeCopies.empty()) {
         // The seeded units' import bytes (every earlier writer of them, host stores included)
         // precede the seed copies; the previous command's trailing barrier may have covered that.
         constexpr VkAccessFlags transferAccess = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
@@ -3237,6 +3382,7 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
         // The padding's current bytes, under the retile that follows (the import-ready barrier
         // orders the scratch writes).
         for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast<std::uint32_t>(copies.size()), copies.data());
+        for (const auto& [source, slab, copies] : holeCopies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, slab, static_cast<std::uint32_t>(copies.size()), copies.data());
     }
     const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT};
     if (direct) {
@@ -3249,7 +3395,12 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
             const auto& window = windows[i];
             auto retile = window.window;
             retile.pipeBankXor = descriptor.pipeBankXor;
-            detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile);
+            for (auto k = directRanges[i].first; k < directRanges[i].second; ++k) {
+                auto piece = directPieces[k].window;
+                piece.pipeBankXor = descriptor.pipeBankXor;
+                detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, directPieces[k].slab, directPieces[k].offset, elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, piece);
+            }
+            if (directRanges[i].first == directRanges[i].second) detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile);
         }
         singlePassMoves.fetch_add(1, std::memory_order_relaxed);
     } else {
@@ -3267,16 +3418,23 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
             const auto& window = windows[i];
             auto retile = window.window;
             retile.pipeBankXor = descriptor.pipeBankXor;
-            detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile);
+            for (auto k = directRanges[i].first; k < directRanges[i].second; ++k) {
+                auto piece = directPieces[k].window;
+                piece.pipeBankXor = descriptor.pipeBankXor;
+                detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i] + directPieces[k].linearOffset, directPieces[k].slab, directPieces[k].offset, mips[window.level], true, window.layer, false, piece);
+            }
+            if (directRanges[i].first == directRanges[i].second) detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile);
         }
     }
-    {
+    if (tiledScratch) {
         // The retiled scratch (and a seed's slab bytes, which the scratch copies overwrite in
         // part) precede the copies into the slabs and the import.
         const VkMemoryBarrier scratchDone{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT};
         context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &scratchDone, 0, nullptr, 0, nullptr);
         const auto copyBuffer = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer");
-        for (const auto& pieces : slabPieces) copyBuffer(commands, tiledScratch->Handle(), pieces.slab->buffer, static_cast<std::uint32_t>(pieces.copies.size()), pieces.copies.data());
+        for (const auto& pieces : slabPieces) {
+            if (!pieces.copies.empty()) copyBuffer(commands, tiledScratch->Handle(), pieces.slab->buffer, static_cast<std::uint32_t>(pieces.copies.size()), pieces.copies.data());
+        }
         for (std::size_t segment = 0; segment < segments.size(); ++segment) {
             if (!importCopies[segment].empty()) copyBuffer(commands, tiledScratch->Handle(), segments[segment].import->buffer, static_cast<std::uint32_t>(importCopies[segment].size()), importCopies[segment].data());
         }
@@ -3535,6 +3693,20 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons
         for (const auto& texture : flush) std::fprintf(stderr, "[flush] image 0x%llx+0x%llx for %s 0x%llx+0x%zx\n", static_cast<unsigned long long>(texture->descriptor.baseAddress), static_cast<unsigned long long>(texture->guestBytes), reason, static_cast<unsigned long long>(address), bytes);
     }
     std::lock_guard gpu(GuestMemory::GpuMutex());
+    if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion() && recorder->PendingWriterSerial(address, bytes) != 0) {
+        try {
+            recorder->Reap();
+            if (const auto writer = recorder->PendingWriterSerial(address, bytes); writer == recorder->OpenSerial()) {
+                Recorder::CountSync(recorder->OpenWriteOverlaps(address, bytes) ? 1 : 3);
+                recorder->Sync();
+            } else if (writer != 0) {
+                recorder->FinishUpTo(writer);
+            }
+        } catch (...) {
+            for (const auto& texture : flush) texture->reconcilePending();
+            throw;
+        }
+    }
     std::exception_ptr failure;
     const auto previousReason = std::exchange(flushReason, reason);
     for (const auto& texture : flush) {
@@ -3884,6 +4056,7 @@ std::vector<std::shared_ptr<StorageTexture>> StorageTexture::overlappingPending(
     // FlushPending's listing; the images stay alive across the caller's scan as there.
     std::vector<std::shared_ptr<StorageTexture>> overlapping;
     auto& pending = Pending();
+    if (pending.textures.SurelyClear(address, bytes)) return overlapping;
     std::lock_guard lock(pending.mutex);
     if (!pending.textures.MayOverlap(address, bytes)) return overlapping;
     const auto* exempt = refreshing;

core/libs/prx/libSceAgcDriver/tests/AsyncLog.cpp

@@ -24,6 +24,9 @@
 #else
 #include <sys/resource.h>
 #endif
+#ifdef __linux__
+#include <sys/prctl.h>
+#endif
 
 namespace {
 
@@ -248,6 +251,9 @@ constexpr int ChildLines = 50;
 #else
     const rlimit noCore{0, 0};
     setrlimit(RLIMIT_CORE, &noCore);
+#endif
+#ifdef __linux__
+    prctl(PR_SET_DUMPABLE, 0);
 #endif
     std::thread([] {
         std::this_thread::sleep_for(8s);

core/libs/prx/libSceAgcDriver/tests/DepthRangeRemap.cpp

@@ -0,0 +1,166 @@
+#include "Recompiler.hpp"
+#include "PipelineSpecialization.hpp"
+#include "Optimization/ResourceProgram.hpp"
+#include <spirv/unified1/spirv.hpp>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <iostream>
+#include <map>
+#include <set>
+#include <span>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace {
+
+struct ModuleFacts {
+    std::map<std::uint32_t, std::uint32_t> specializationDefaults;
+    std::uint32_t clipDistanceLength = 0;
+    bool clipDistanceBuiltIn = false;
+    std::size_t multiplies = 0;
+    bool pushConstantVariable = false;
+    std::set<std::uint32_t> pushDwordLoads;
+};
+
+ModuleFacts inspect(std::span<const std::uint32_t> words) {
+    ModuleFacts facts;
+    std::map<std::uint32_t, std::uint32_t> specIds;
+    std::map<std::uint32_t, std::uint32_t> specValues;
+    std::map<std::uint32_t, std::uint32_t> constants;
+    std::map<std::uint32_t, std::uint32_t> arrayLengths;
+    std::map<std::uint32_t, std::uint32_t> pointerTargets;
+    std::map<std::uint32_t, std::uint32_t> variableTypes;
+    std::set<std::uint32_t> clipVariables;
+    std::set<std::uint32_t> pushVariables;
+    std::vector<std::uint32_t> pushIndexIds;
+    for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
+        const auto op = static_cast<spv::Op>(words[at] & 0xffffu);
+        const auto count = words[at] >> 16u;
+        if (op == spv::OpDecorate && count == 4 && words[at + 2] == spv::DecorationSpecId) specIds[words[at + 1]] = words[at + 3];
+        if (op == spv::OpDecorate && count == 4 && words[at + 2] == spv::DecorationBuiltIn && words[at + 3] == spv::BuiltInClipDistance) clipVariables.insert(words[at + 1]);
+        if (op == spv::OpSpecConstant && count == 4) specValues[words[at + 2]] = words[at + 3];
+        if (op == spv::OpConstant && count == 4) constants[words[at + 2]] = words[at + 3];
+        if (op == spv::OpTypeArray && count == 4) arrayLengths[words[at + 1]] = words[at + 3];
+        if (op == spv::OpTypePointer && count == 4) pointerTargets[words[at + 1]] = words[at + 3];
+        if (op == spv::OpVariable && count >= 4) variableTypes[words[at + 2]] = words[at + 1];
+        if (op == spv::OpVariable && count >= 4 && words[at + 3] == spv::StorageClassPushConstant) pushVariables.insert(words[at + 2]);
+        if (op == spv::OpAccessChain && count == 6 && pushVariables.contains(words[at + 3])) pushIndexIds.push_back(words[at + 5]);
+        if (op == spv::OpFMul) ++facts.multiplies;
+    }
+    for (const auto& [id, specId] : specIds) {
+        if (const auto value = specValues.find(id); value != specValues.end()) facts.specializationDefaults[specId] = value->second;
+    }
+    facts.pushConstantVariable = !pushVariables.empty();
+    for (const auto id : pushIndexIds) {
+        if (const auto value = constants.find(id); value != constants.end()) facts.pushDwordLoads.insert(value->second);
+    }
+    for (const auto variable : clipVariables) {
+        facts.clipDistanceBuiltIn = true;
+        const auto pointer = variableTypes.find(variable);
+        if (pointer == variableTypes.end()) continue;
+        const auto array = pointerTargets.find(pointer->second);
+        if (array == pointerTargets.end()) continue;
+        const auto length = arrayLengths.find(array->second);
+        if (length == arrayLengths.end()) continue;
+        const auto constant = constants.find(length->second);
+        if (constant != constants.end()) facts.clipDistanceLength = constant->second;
+    }
+    return facts;
+}
+
+ModuleFacts inspect(const ShaderRecompiler::RecompileResult& result) {
+    return inspect(result.spirv.Words());
+}
+
+constexpr std::array<std::uint32_t, 3> PositionExport{0xf80008cfu, 0u, 0xbf810000u};
+
+ShaderRecompiler::RecompileRequest baseRequest(bool remap) {
+    using namespace ShaderRecompiler;
+    RecompileRequest request{};
+    request.shader = {ShaderStage::Vertex, 0x10000u, PositionExport, 0, {}};
+    request.context.waveSize = 64;
+    ShaderVertexStageInfo vertex{};
+    vertex.depthRemapMode = 2u;
+    request.context.vertex = vertex;
+    request.target.vulkanVersion = 0x00401000u;
+    request.target.spirvVersion = 0x00010300u;
+    request.target.subgroupSize = 64;
+    static constexpr std::array<std::uint32_t, 2> capabilities{spv::CapabilityShader, spv::CapabilityClipDistance};
+    request.target.supportedCapabilities = capabilities;
+    request.target.depthRangeRemap = remap;
+    request.layout.pushConstantSizeBytes = ShaderRecompiler::DepthRemapPushOffsetBytes;
+    request.useCache = false;
+    return request;
+}
+
+void require(bool condition, const char* what) {
+    if (!condition) throw std::runtime_error(what);
+}
+
+void preparedTransformTests() {
+    using namespace ShaderRecompiler;
+    auto request = baseRequest(true);
+    request.useCache = true;
+    const auto handle = PrepareShader(request);
+    const auto materialize = [&](std::uint32_t mode) {
+        auto moved = request;
+        moved.context.vertex->depthRemapMode = mode;
+        const auto invocation = PreparedShaderInvocation::TryCreate(moved, handle);
+        require(invocation.has_value(), "the prepared vertex shader did not match a request of its own mode");
+        return invocation->Materialize(*invocation->Capture(SrtRuntime{}));
+    };
+    const auto prepared = inspect(GetPreparedArtifact(*handle).spirv.Words());
+    require(prepared.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase) && !prepared.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 1u) && !prepared.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 2u), "only the depth remap mode may be a specialization constant of the prepared module");
+    const auto first = materialize(2u);
+    require(materialize(2u)->PipelineVariantId() == first->PipelineVariantId(), "the same depth remap mode did not reuse its materialized vertex shader");
+    auto other = request;
+    other.context.vertex->depthRemapMode = 1u;
+    require(!PreparedShaderInvocation::TryCreate(other, handle).has_value(), "a prepared vertex shader matched a request of another depth remap mode");
+}
+
+void pushedTransformTests() {
+    using namespace ShaderRecompiler;
+    const auto remapped = Recompile(baseRequest(true));
+    const auto plain = Recompile(baseRequest(false));
+    const auto facts = inspect(remapped);
+    require(!facts.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 1u) && !facts.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 2u), "the depth remap scale or offset is a specialization constant");
+    require(facts.pushConstantVariable && facts.pushDwordLoads.contains(DepthRemapPushOffsetBytes / 4u) && facts.pushDwordLoads.contains(DepthRemapPushOffsetBytes / 4u + 1u), "the remapped vertex shader does not read the scale and offset from the push block");
+    require(!inspect(plain).pushConstantVariable, "a vertex shader without a depth remap gained a push constant block");
+    auto crowded = baseRequest(true);
+    crowded.layout.pushConstantSizeBytes = DepthRemapPushOffsetBytes + 4u;
+    bool refused = false;
+    try {
+        static_cast<void>(Recompile(crowded));
+    } catch (const std::runtime_error&) {
+        refused = true;
+    }
+    require(refused, "a remapped vertex shader with push data over the depth remap words was accepted");
+}
+
+}
+
+int main() {
+    using namespace ShaderRecompiler;
+    try {
+        const auto remapped = inspect(Recompile(baseRequest(true)));
+        const auto plain = inspect(Recompile(baseRequest(false)));
+        require(remapped.clipDistanceBuiltIn && remapped.clipDistanceLength == plain.clipDistanceLength + 2u, "the remap did not add its two clip distances");
+        require(remapped.multiplies >= plain.multiplies + 2u, "the remap does not scale the clip z and offset it by w");
+        require(!plain.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase) && !remapped.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 3u), "unexpected specialization constants");
+        preparedTransformTests();
+        pushedTransformTests();
+        std::cout << "Depth range remap tests passed\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << "Depth range remap test failed: " << error.what() << '\n';
+        for (const bool remap : {true, false}) {
+            const auto facts = inspect(Recompile(baseRequest(remap)));
+            std::cerr << "  remap=" << remap << ": spec ids";
+            for (const auto& [id, value] : facts.specializationDefaults) std::cerr << ' ' << id << '=' << value;
+            std::cerr << "; clip builtin " << facts.clipDistanceBuiltIn << " length " << facts.clipDistanceLength << "; fmul " << facts.multiplies << '\n';
+        }
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include <algorithm>
 #include <array>
 #include <cstdint>
 #include <map>
@@ -47,6 +48,7 @@ VkImageUsageFlags lastImageUsage = 0;
 std::uint32_t imagesDestroyed = 0;
 std::vector<std::pair<VkImage, VkImageAspectFlags>> imageReads;
 std::vector<std::pair<VkImage, VkImageAspectFlags>> imageWrites;
+std::vector<VkImage> destroyedImages;
 
 VKAPI_ATTR VkResult VKAPI_CALL createImage(VkDevice, const VkImageCreateInfo* info, const VkAllocationCallbacks*, VkImage* image) {
     lastImageFormat = info->format;
@@ -108,7 +110,10 @@ VKAPI_ATTR VkResult VKAPI_CALL getFenceStatus(VkDevice, VkFence) {
 VKAPI_ATTR void VKAPI_CALL freeCommandBuffers(VkDevice, VkCommandPool, std::uint32_t, const VkCommandBuffer*) {}
 VKAPI_ATTR void VKAPI_CALL destroyFence(VkDevice, VkFence, const VkAllocationCallbacks*) {}
 VKAPI_ATTR void VKAPI_CALL destroyImageView(VkDevice, VkImageView, const VkAllocationCallbacks*) {}
-VKAPI_ATTR void VKAPI_CALL destroyImage(VkDevice, VkImage, const VkAllocationCallbacks*) { ++imagesDestroyed; }
+VKAPI_ATTR void VKAPI_CALL destroyImage(VkDevice, VkImage image, const VkAllocationCallbacks*) {
+    ++imagesDestroyed;
+    destroyedImages.push_back(image);
+}
 VKAPI_ATTR void VKAPI_CALL freeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*) {}
 VKAPI_ATTR void VKAPI_CALL cmdPipelineBarrier(VkCommandBuffer, VkPipelineStageFlags, VkPipelineStageFlags, VkDependencyFlags, std::uint32_t, const VkMemoryBarrier*, std::uint32_t, const VkBufferMemoryBarrier*, std::uint32_t, const VkImageMemoryBarrier*) {}
 VKAPI_ATTR void VKAPI_CALL cmdClearDepthStencilImage(VkCommandBuffer, VkImage image, VkImageLayout, const VkClearDepthStencilValue*, std::uint32_t, const VkImageSubresourceRange*) {
@@ -374,6 +379,44 @@ void TestMultisampleStencilAliases(Context context) {
 
 }
 
+void TestRetiredSurfaceRelease(const Context& context) {
+    ClearDepthSurfaces(context.device);
+    constexpr VkExtent2D extent{256, 256};
+    const auto bytes = DepthSliceBytes(extent, 4u);
+    const auto destroyed = [](VkImage image) { return std::find(destroyedImages.begin(), destroyedImages.end(), image) != destroyedImages.end(); };
+    constexpr std::uint64_t moving = 0x90000000;
+    std::vector<VkImage> images;
+    for (std::uint64_t frame = 0; frame < 4; ++frame) {
+        DepthSurfaceView(context, {moving + frame * 0x100000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0});
+        images.push_back(lastImage);
+        RetireDepthSurfaces(context.device, moving + frame * 0x100000u, bytes);
+    }
+    const auto released = DepthSurfacesReleased();
+    DepthSurfaceView(context, {moving + 4u * 0x100000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0});
+    Require(std::all_of(images.begin(), images.end(), destroyed), "a retired depth surface must free its image once a new surface is created");
+    Require(!destroyed(lastImage), "the new depth surface must keep its image");
+    Require(DepthSurfacesReleased() > released, "freeing retired depth surfaces must be counted for the framebuffer cache");
+
+    constexpr std::uint64_t sampled = 0x98000000;
+    DepthSurfaceView(context, {sampled, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0});
+    const auto sampledImage = lastImage;
+    auto view = lookup(context, {sampled, FormatR32Float, 256, 256});
+    Require(view != nullptr, "an R32F view of a D32 surface's own extent must sample its depth plane");
+    RetireDepthSurfaces(context.device, sampled, bytes);
+    DepthSurfaceView(context, {sampled + 0x100000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0});
+    Require(!destroyed(sampledImage), "a retired depth surface must keep its image while its sampled view is held");
+    view.reset();
+    DepthSurfaceView(context, {sampled + 0x200000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0});
+    Require(destroyed(sampledImage), "a retired depth surface must free its image once its sampled view is released");
+
+    constexpr std::uint64_t kept = 0x9c000000;
+    DepthSurfaceView(context, {kept, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0});
+    const auto keptImage = lastImage;
+    DepthSurfaceView(context, {kept + 0x100000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0});
+    Require(!destroyed(keptImage), "a depth surface that was not retired must keep its image");
+    ClearDepthSurfaces(context.device);
+}
+
 void RunDepthSurfaceReuseTests() {
     static int deviceTag = 0;
     Context context{};
@@ -480,6 +523,7 @@ void RunDepthSurfaceReuseTests() {
         Require(clearsAfter(sliceHtile) == std::vector<VkImage>{firstImage}, "an HTILE clear of the first slice must clear that slice alone");
         Require(clearsAfter(2u * sliceHtile) == std::vector<VkImage>{firstImage, secondImage}, "an HTILE clear over both slices must clear both");
     }
+    TestRetiredSurfaceRelease(context);
 
     ClearDepthSurfaces(context.device);
 }

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -2,6 +2,7 @@
 #include "GraphicsTests.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp"
@@ -10,6 +11,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
+#include "ShaderDiskCache.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
@@ -28,6 +30,11 @@
 #include "ImageTableAbi.hpp"
 #include "SpirvBackend/SpirvSpecialization.hpp"
 #include <spirv/unified1/spirv.hpp>
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
 #include <algorithm>
 #include <array>
 #include <atomic>
@@ -35,6 +42,7 @@
 #include <chrono>
 #include <condition_variable>
 #include <cstddef>
+#include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <exception>
@@ -3894,6 +3902,57 @@ void quadPixelMaskTests() {
     expectFailure([&] { AgcDriver::Graphics::ValidateSampleMask(context, varying, storing); }, "differs between the pixels of a quad");
 }
 
+void DepthRemapTests() {
+    auto queue = makeState();
+    queue.context[0x204] = 0;
+    queue.context[0x113] = std::bit_cast<std::uint32_t>(1.0f);
+    queue.context[0x114] = 0;
+    AgcDriver::Graphics::Context remapping{};
+    remapping.depthClamp = true;
+    auto state = AgcDriver::Graphics::DecodeState(queue);
+    Require(state.negativeOneToOne && state.viewport.minDepth == -1 && state.viewport.maxDepth == 1 && state.depthRemap.mode == 0, "a negative-one-to-one depth transform decoded with a remap");
+    AgcDriver::Graphics::ApplyDepthRemap(remapping, state);
+    Require(state.depthRemap.mode == 2 && state.depthRemap.scale == 1.0f && state.depthRemap.offset == 0.0f, "the remap did not take the viewport's depth scale and offset");
+    Require(state.viewport.minDepth == 0 && state.viewport.maxDepth == 1 && !state.negativeOneToOne && state.depthClamp, "the remapped viewport does not clamp at the console's bounds in zero-to-one clip space");
+    AgcDriver::Graphics::Context unrestricted{};
+    unrestricted.depthRangeUnrestricted = true;
+    auto kept = AgcDriver::Graphics::DecodeState(queue);
+    AgcDriver::Graphics::ApplyDepthRemap(unrestricted, kept);
+    Require(kept.depthRemap.mode == 0 && kept.viewport.minDepth == -1 && kept.negativeOneToOne && !kept.depthClamp, "an unrestricted device remapped the depth range");
+    AgcDriver::Graphics::Context noClamp{};
+    auto refused = AgcDriver::Graphics::DecodeState(queue);
+    expectFailure([&] { AgcDriver::Graphics::ApplyDepthRemap(noClamp, refused); }, "depth clamping");
+    queue.context[0x113] = std::bit_cast<std::uint32_t>(0.5f);
+    queue.context[0x114] = std::bit_cast<std::uint32_t>(0.5f);
+    auto inRange = AgcDriver::Graphics::DecodeState(queue);
+    AgcDriver::Graphics::ApplyDepthRemap(remapping, inRange);
+    Require(inRange.depthRemap.mode == 0 && inRange.negativeOneToOne && !inRange.depthClamp, "an in-range depth transform was remapped");
+    queue.context[0x204] = 0x80000u;
+    queue.context[0x113] = std::bit_cast<std::uint32_t>(2.0f);
+    queue.context[0x114] = std::bit_cast<std::uint32_t>(-0.5f);
+    queue.context[0xb4] = std::bit_cast<std::uint32_t>(0.25f);
+    queue.context[0xb5] = std::bit_cast<std::uint32_t>(0.75f);
+    auto bounded = AgcDriver::Graphics::DecodeState(queue);
+    AgcDriver::Graphics::ApplyDepthRemap(remapping, bounded);
+    Require(bounded.depthRemap.mode == 1 && bounded.depthRemap.scale == 4.0f && bounded.depthRemap.offset == -1.5f && bounded.viewport.minDepth == 0.25f && bounded.viewport.maxDepth == 0.75f, "a zero-to-one remap did not divide its transform by the clamp range it gives the viewport");
+    queue.context[0xb4] = std::bit_cast<std::uint32_t>(-1.0f);
+    queue.context[0xb5] = std::bit_cast<std::uint32_t>(2.0f);
+    auto clipped = AgcDriver::Graphics::DecodeState(queue);
+    AgcDriver::Graphics::ApplyDepthRemap(remapping, clipped);
+    Require(clipped.depthRemap.mode == 1 && clipped.viewport.minDepth == 0 && clipped.viewport.maxDepth == 1, "clamp bounds beyond [0, 1] were not clipped to the representable depth range");
+    Require(clipped.depthRemap.scale == 2.0f && clipped.depthRemap.offset == -0.5f, "clamp bounds clipped to [0, 1] changed the depth transform");
+    auto unorm = AgcDriver::Graphics::DecodeState(queue);
+    unorm.depth = AgcDriver::Graphics::DepthTarget{0x10000, 0, {64, 64}, VK_FORMAT_D16_UNORM, 0.0f, 0};
+    unorm.depthTest = unorm.depthWrite = true;
+    AgcDriver::Graphics::ApplyDepthRemap(remapping, unorm);
+    Require(unorm.viewport.minDepth == 0 && unorm.viewport.maxDepth == 1, "a normalized depth target refused clipped clamp bounds");
+    auto floating = AgcDriver::Graphics::DecodeState(queue);
+    floating.depth = AgcDriver::Graphics::DepthTarget{0x10000, 0, {64, 64}, VK_FORMAT_D32_SFLOAT, 0.0f, 0};
+    floating.depthWrite = true;
+    AgcDriver::Graphics::ApplyDepthRemap(remapping, floating);
+    Require(floating.depthRemap.mode == 1 && floating.viewport.minDepth == 0 && floating.viewport.maxDepth == 1 && floating.depthClamp, "a written float depth target refused clipped clamp bounds");
+}
+
 void DepthClipTests() {
     auto queue = makeState();
     const auto direct = AgcDriver::Graphics::DecodeState(queue);
@@ -4591,6 +4650,117 @@ AgcDriver::Graphics::Context mockContext() {
     return context;
 }
 
+struct CommandBatchFailureProbe {
+    VkResult allocation = VK_SUCCESS;
+    VkResult creation = VK_SUCCESS;
+    VkResult begin = VK_SUCCESS;
+    VkFence failedFence = VK_NULL_HANDLE;
+    std::uint32_t freed = 0;
+    std::uint32_t creates = 0;
+    std::uint32_t destroys = 0;
+    std::uint32_t invalidDestroys = 0;
+    std::uint32_t begins = 0;
+};
+
+CommandBatchFailureProbe batchProbe;
+
+VKAPI_ATTR VkResult VKAPI_CALL batchAllocateCommands(VkDevice device, const VkCommandBufferAllocateInfo* info, VkCommandBuffer* commands) {
+    if (batchProbe.allocation != VK_SUCCESS) {
+        std::fill_n(commands, info->commandBufferCount, VK_NULL_HANDLE);
+        return batchProbe.allocation;
+    }
+    return mockAllocateCommandBuffers(device, info, commands);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL batchCreateFence(VkDevice device, const VkFenceCreateInfo* info, const VkAllocationCallbacks* allocator, VkFence* fence) {
+    ++batchProbe.creates;
+    if (batchProbe.creation != VK_SUCCESS) {
+        *fence = batchProbe.failedFence;
+        return batchProbe.creation;
+    }
+    return mockCreateFence(device, info, allocator, fence);
+}
+
+VKAPI_ATTR void VKAPI_CALL batchFreeCommands(VkDevice device, VkCommandPool pool, std::uint32_t count, const VkCommandBuffer* commands) {
+    batchProbe.freed += count;
+    mockFreeCommandBuffers(device, pool, count, commands);
+}
+
+VKAPI_ATTR void VKAPI_CALL batchDestroyFence(VkDevice device, VkFence fence, const VkAllocationCallbacks* allocator) {
+    ++batchProbe.destroys;
+    if (fence != VK_NULL_HANDLE && fence == batchProbe.failedFence) {
+        ++batchProbe.invalidDestroys;
+        return;
+    }
+    mockDestroyFence(device, fence, allocator);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL batchBeginCommands(VkCommandBuffer commands, const VkCommandBufferBeginInfo* info) {
+    ++batchProbe.begins;
+    if (batchProbe.begin != VK_SUCCESS) return batchProbe.begin;
+    return mockBeginCommandBuffer(commands, info);
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL batchFailureProc(VkDevice device, const char* name) {
+    if (std::strcmp(name, "vkAllocateCommandBuffers") == 0) return reinterpret_cast<PFN_vkVoidFunction>(batchAllocateCommands);
+    if (std::strcmp(name, "vkCreateFence") == 0) return reinterpret_cast<PFN_vkVoidFunction>(batchCreateFence);
+    if (std::strcmp(name, "vkFreeCommandBuffers") == 0) return reinterpret_cast<PFN_vkVoidFunction>(batchFreeCommands);
+    if (std::strcmp(name, "vkDestroyFence") == 0) return reinterpret_cast<PFN_vkVoidFunction>(batchDestroyFence);
+    if (std::strcmp(name, "vkBeginCommandBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(batchBeginCommands);
+    return mockProc(device, name);
+}
+
+void commandBatchFenceFailureTests() {
+    using namespace AgcDriver::Graphics;
+    const std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    std::uint32_t invalidDestroys = 0;
+    for (const auto result : {VK_ERROR_OUT_OF_HOST_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) {
+        for (const bool nonnull : {false, true}) {
+            mock = MockVulkan{};
+            batchProbe = {};
+            batchProbe.creation = result;
+            batchProbe.failedFence = nonnull ? reinterpret_cast<VkFence>(std::uintptr_t{0xdeadbeef}) : VK_NULL_HANDLE;
+            auto context = mockContext();
+            context.deviceProc = batchFailureProc;
+            bool failed = false;
+            try {
+                CommandBatch batch(context);
+            } catch (const std::runtime_error& error) {
+                const std::string message = error.what();
+                failed = message.find("vkCreateFence") != std::string::npos && message.find("Vulkan result " + std::to_string(result)) != std::string::npos;
+            }
+            Require(failed && batchProbe.creates == 1 && batchProbe.begins == 0, "fence creation failure lost its error or continued construction");
+            Require(mock.live == 0 && batchProbe.freed == 1, "fence creation failure leaked the allocated command buffer");
+            invalidDestroys += batchProbe.invalidDestroys;
+            std::printf("command batch fence result=%d nonnull=%d invalid destroys=%u\n", result, nonnull, batchProbe.invalidDestroys);
+            const auto destroyed = batchProbe.destroys;
+            const auto invalid = batchProbe.invalidDestroys;
+            batchProbe.creation = VK_SUCCESS;
+            {
+                CommandBatch recovered(context);
+                Require(recovered.Handle() != VK_NULL_HANDLE, "command batch did not recover after fence creation failed");
+            }
+            Require(mock.live == 0 && batchProbe.freed == 2 && batchProbe.destroys == destroyed + 1 && batchProbe.invalidDestroys == invalid, "recovered command batch mishandled its owned objects");
+        }
+    }
+    for (const bool allocated : {false, true}) {
+        mock = MockVulkan{};
+        batchProbe = {};
+        batchProbe.allocation = allocated ? VK_SUCCESS : VK_ERROR_OUT_OF_HOST_MEMORY;
+        batchProbe.begin = VK_ERROR_OUT_OF_HOST_MEMORY;
+        auto context = mockContext();
+        context.deviceProc = batchFailureProc;
+        bool failed = false;
+        try {
+            CommandBatch batch(context);
+        } catch (const std::runtime_error& error) {
+            failed = std::string(error.what()).find(allocated ? "vkBeginCommandBuffer" : "vkAllocateCommandBuffers") != std::string::npos;
+        }
+        Require(failed && mock.live == 0 && batchProbe.freed == (allocated ? 1u : 0u) && batchProbe.destroys == (allocated ? 1u : 0u) && batchProbe.invalidDestroys == 0, "command batch constructor failure mishandled its owned objects");
+    }
+    Require(invalidDestroys == 0, "command batch destroyed undefined failed fence outputs");
+}
+
 void addressSpaceTeardownTests() {
 #ifdef _WIN32
     _putenv_s("APS5_NO_LEASE_MIRROR", "1");
@@ -5067,6 +5237,16 @@ void pushConstantTests() {
     fragment.pushConstants.clear();
     shaders[1].pushConstantOffset = 999;
     Require(AgcDriver::Graphics::PushConstantStages(shaders) == VK_SHADER_STAGE_VERTEX_BIT, "empty push constants contributed a stage");
+    {
+        ShaderRecompiler::RecompileResult reading;
+        reading.readsDepthRemapPush = true;
+        std::vector<std::byte> encoded;
+        ShaderRecompiler::ShaderDiskCache::EncodeResult(reading, encoded);
+        ShaderRecompiler::RecompileResult replayed;
+        Require(ShaderRecompiler::ShaderDiskCache::DecodeResult(encoded, replayed), "a result that reads the depth remap push words does not decode");
+        const std::array<AgcDriver::Graphics::CompiledShader, 1> remapped{{{ShaderRecompiler::ShaderStage::Vertex, &replayed, 0}}};
+        Require(replayed.pushConstants.empty() && AgcDriver::Graphics::PushConstantStages(remapped) == VK_SHADER_STAGE_VERTEX_BIT, "a journalled vertex shader that reads the depth remap push words lost its push stage");
+    }
     Require(AgcDriver::Graphics::AssemblePushConstants(shaders)[8] == std::byte{0}, "empty push constants were copied");
     shaders[1].program = nullptr;
     expectFailure([&] { AgcDriver::Graphics::AssemblePushConstants(shaders); }, "missing compiled shader");
@@ -5095,10 +5275,12 @@ void descriptorSnapshotTests() {
             const auto original = findWrite(0).buffers;
             const auto originalThird = findWrite(4).buffers;
             std::vector<ShaderResources::MovedBuffer> moved;
+            std::array<std::vector<std::uint32_t>, 2> words;
             for (std::size_t index = 0; index < 2; ++index) {
                 if ((selected & (1u << index)) == 0) continue;
                 const std::size_t bytes = index == 0 ? 24 : 12;
-                moved.push_back({index, 0, bytes, std::vector<std::uint32_t>(bytes / 4, static_cast<std::uint32_t>(0x12340000u + index))});
+                words[index].assign(bytes / 4, static_cast<std::uint32_t>(0x12340000u + index));
+                moved.push_back({index, 0, bytes, words[index]});
             }
             mock.writes.clear();
             const auto snapshots = resources.PrepareDrawBindings(recorder, moved);
@@ -5517,19 +5699,27 @@ void descriptorCacheTests() {
         Require(mock.poolSizes.size() == 2 && mock.poolSizes[0].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER && mock.poolSizes[0].descriptorCount == 4097 && mock.poolSizes[1].type == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE && mock.poolSizes[1].descriptorCount == 3, "the dedicated pool is not sized to its set");
         Require(cache.Counters().pools == 0 && cache.Counters().sets == 1, "an oversized set was counted in the chain pools");
         cache.Free(dedicated);
-        Require(mock.live == live, "freeing an oversized set did not destroy its pool");
+        Require(mock.live == live + 1 && cache.Counters().keptSets == 1, "freeing an oversized set did not keep its set and pool");
+        const auto again = cache.Allocate(large, oversized);
+        Require(again.set == dedicated.set && again.pool == dedicated.pool && mock.live == live + 1 && cache.Counters().reusedSets == 1 && cache.Counters().dedicatedPools == 1, "an oversized set of the kept set's layout and sizes did not take the kept set");
+        cache.Free(again);
+        const std::array<VkDescriptorPoolSize, 2> otherSizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4098}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 3}}};
+        const auto other = cache.Allocate(large, otherSizes);
+        Require(other.pool != dedicated.pool && mock.live == live + 2 && cache.Counters().dedicatedPools == 2, "an oversized set of other sizes took a kept set");
+        cache.Free(other);
+        Require(mock.live == live + 2 && cache.Counters().keptSets == 3, "freeing a second oversized set did not keep it");
         const std::array<VkDescriptorPoolSize, 1> fitting{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096}}};
         const auto chained = cache.Allocate(layout(4096), fitting);
         Require(chained.set != VK_NULL_HANDLE && mock.poolMaxSets == 1024 && mock.poolFlags == 0 && cache.Counters().pools == 1, "a set the chain pool holds left the chain");
         cache.Free(chained);
-        Require(mock.descriptorPools.at(chained.pool).resets == 1 && mock.live == live + 2, "a chain pool was not reset after its last set");
+        Require(mock.descriptorPools.at(chained.pool).resets == 1 && mock.live == live + 4, "a chain pool was not reset after its last set");
         const std::array<VkDescriptorPoolSize, 1> regular{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096}}};
         const auto first = cache.Allocate(layout(4096), regular);
         const auto oversizedMixed = cache.Allocate(large, oversized);
         const auto second = cache.Allocate(layout(4096), regular);
         Require(first.pool == chained.pool && second.pool == chained.pool && oversizedMixed.pool != chained.pool && mock.descriptorPools.at(chained.pool).resets == 1, "an oversized set disturbed the chain pool");
         cache.Free(oversizedMixed);
-        Require(mock.live == live + 2 && mock.descriptorPools.count(oversizedMixed.pool) == 0 && mock.descriptorPools.at(chained.pool).resets == 1, "freeing an oversized set between chain sets touched the chain pool");
+        Require(oversizedMixed.pool == dedicated.pool && mock.live == live + 4 && mock.descriptorPools.count(oversizedMixed.pool) == 1 && mock.descriptorPools.at(chained.pool).resets == 1, "freeing an oversized set between chain sets touched the chain pool");
         cache.Free(first);
         Require(mock.descriptorPools.at(chained.pool).resets == 1, "a chain pool reset while a set was live");
         cache.Free(second);
@@ -5540,6 +5730,15 @@ void descriptorCacheTests() {
         expectFailure([&] { cache.Allocate(large, storageImages); }, "descriptor set exceeds the device's per-set descriptor limit");
         const std::array<VkDescriptorPoolSize, 1> uniformBuffers{{{VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1}}};
         expectFailure([&] { cache.Allocate(large, uniformBuffers); }, "descriptor set uses an unsupported descriptor type 6");
+        const auto keptBefore = mock.live;
+        std::vector<AgcDriver::Graphics::DescriptorCache::SetAllocation> spread;
+        for (std::uint32_t index = 0; index < 10; ++index) {
+            const std::array<VkDescriptorPoolSize, 1> sized{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4100 + index}}};
+            spread.push_back(cache.Allocate(large, sized));
+        }
+        Require(mock.live == keptBefore + 10, "ten oversized sets of new sizes did not get ten pools");
+        for (const auto& set : spread) cache.Free(set);
+        Require(mock.live == keptBefore - 2 + 8, "freeing more oversized sets than the cache keeps did not destroy the oldest pools");
     }
     Require(mock.live == 0, "the descriptor cache leaked a pool or layout");
     mock = MockVulkan{};
@@ -5673,7 +5872,8 @@ void imageTableSnapshotAllocationTests() {
         const auto state = DecodeState(makeState());
         ShaderResources resources(context, vertex, fragment, state.color, 0, 0);
         const auto originalMap = findWrite(map.binding).buffers.at(0);
-        const std::array<ShaderResources::MovedBuffer, 1> moved{{{0, 0, 16, {11, 22, 33, 44}}}};
+        const std::array<std::uint32_t, 4> movedWords{11, 22, 33, 44};
+        const std::array<ShaderResources::MovedBuffer, 1> moved{{{0, 0, 16, movedWords}}};
         const auto sets = cache.Counters().sets;
         mock.writes.clear();
         mock.variableDescriptorCount.reset();
@@ -5809,6 +6009,31 @@ void samplerTableDeviceLimitTests(std::optional<std::uint32_t> selected = std::n
     }
 }
 
+void hostImportBudgetTests() {
+    using AgcDriver::Graphics::HostImportBudget;
+    constexpr std::uint64_t MiB = 1ull << 20u;
+    constexpr std::uint64_t GiB = 1ull << 30u;
+    VkPhysicalDeviceMemoryProperties memory{};
+    Require(HostImportBudget(memory, 0) == 6 * GiB, "a device without memory heaps does not keep the 6 GiB import budget");
+    Require(HostImportBudget(memory, 32 * GiB) == 16 * GiB, "a device without a host heap does not take half the physical memory");
+    memory.memoryHeapCount = 3;
+    memory.memoryHeaps[0] = {7920 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
+    memory.memoryHeaps[1] = {8128 * MiB, 0};
+    memory.memoryHeaps[2] = {256 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
+    Require(HostImportBudget(memory, 16 * GiB) == 4064 * MiB, "imports on a 16 GiB machine take more than half its 8128 MiB host heap");
+    memory.memoryHeaps[1].size = 16 * GiB;
+    Require(HostImportBudget(memory, 32 * GiB) == 8 * GiB, "imports on a 32 GiB machine take more than half its 16 GiB host heap");
+    memory.memoryHeaps[1].size = 32 * GiB;
+    Require(HostImportBudget(memory, 64 * GiB) == 16 * GiB, "imports on a 64 GiB machine take more than half its 32 GiB host heap");
+    memory.memoryHeaps[1].size = 3 * GiB;
+    Require(HostImportBudget(memory, 16 * GiB) == 1 * GiB, "a 3 GiB host heap does not keep its 2 GiB reserve");
+    memory.memoryHeaps[1].size = 8128 * MiB;
+    memory.memoryHeaps[0].size = 64 * GiB;
+    Require(HostImportBudget(memory, 16 * GiB) == 4064 * MiB, "a device-local heap counted as the host heap");
+    memory.memoryHeapCount = 1;
+    Require(HostImportBudget(memory, 16 * GiB) == 8 * GiB, "a host heap past memoryHeapCount limits the import budget");
+}
+
 void textureCacheBudgetTests() {
     using AgcDriver::Graphics::TextureCacheBudget;
     constexpr std::uint64_t GiB = 1ull << 30u;
@@ -9279,6 +9504,39 @@ void vertexZeroFillTests() {
     Require(std::all_of(bytes.begin() + static_cast<std::ptrdiff_t>(valid), bytes.end(), [](std::byte value) { return value == std::byte{0}; }), "a vertex fetch past the record count did not read as zeros");
 }
 
+void vertexOverhangTests() {
+    using AgcDriver::Graphics::CopyZeroPaddedDrawInput;
+    constexpr std::size_t page = 65536;
+    constexpr std::size_t stride = 44;
+    constexpr std::size_t overhang = 8;
+#ifdef _WIN32
+    auto* block = static_cast<std::byte*>(VirtualAlloc(nullptr, page * 2, MEM_RESERVE, PAGE_NOACCESS));
+    Require(block != nullptr && VirtualAlloc(block, page, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot reserve the guest pages of the overhang test");
+#else
+    void* mapped = mmap(nullptr, page * 2, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    Require(mapped != MAP_FAILED && mprotect(mapped, page, PROT_READ | PROT_WRITE) == 0, "cannot reserve the guest pages of the overhang test");
+    auto* block = static_cast<std::byte*>(mapped);
+#endif
+    constexpr std::size_t offset = 0x650;
+    const auto readable = page - offset;
+    std::fill(block + offset, block + page, std::byte{0x7a});
+    const auto address = reinterpret_cast<std::uint64_t>(block) + offset;
+    {
+        const auto copy = CopyZeroPaddedDrawInput(mockContext(), address, readable + overhang, readable + overhang, stride);
+        const auto bytes = copy.buffer->Bytes();
+        Require(bytes.size() == readable + overhang, "an overhanging vertex fetch changed its size");
+        Require(std::all_of(bytes.begin(), bytes.begin() + static_cast<std::ptrdiff_t>(readable), [](std::byte value) { return value == std::byte{0x7a}; }), "an overhanging vertex fetch did not copy the readable bytes");
+        Require(std::all_of(bytes.begin() + static_cast<std::ptrdiff_t>(readable), bytes.end(), [](std::byte value) { return value == std::byte{0}; }), "an overhanging vertex fetch did not zero its last record");
+    }
+    expectFailure([&] { CopyZeroPaddedDrawInput(mockContext(), address, readable + stride, readable + stride, stride); }, "is not readable");
+    expectFailure([&] { CopyZeroPaddedDrawInput(mockContext(), address, readable + overhang, readable + overhang, 0); }, "is not readable");
+#ifdef _WIN32
+    VirtualFree(block, 0, MEM_RELEASE);
+#else
+    munmap(block, page * 2);
+#endif
+}
+
 void vertexZeroFillMergeTests() {
     using AgcDriver::Graphics::PlanVertexCopies;
     using AgcDriver::Graphics::SoloZeroPaddedFetchIndices;
@@ -9733,6 +9991,10 @@ int main(int argc, char** argv) {
             stencilReferenceUnusedTests();
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--command-batch-fence-failure") {
+            commandBatchFenceFailureTests();
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--pipeline-journal") {
             pipelineJournalTests();
             std::cout << "pipeline journal recipe, record and replay tests passed\n";
@@ -9893,6 +10155,7 @@ int main(int argc, char** argv) {
         hardwareScreenOffsetTests();
         srgb8TargetTests();
         DepthClipTests();
+        DepthRemapTests();
         DepthStencilTests();
         OneDimensionalColorTests();
         htileDepthClearTests();
@@ -9971,6 +10234,7 @@ int main(int argc, char** argv) {
         samplerTableDeviceLimitTests();
         misalignedShaderDataTests();
         debugBranchTests();
+        hostImportBudgetTests();
         textureCacheBudgetTests();
         sampledTextureBudgetTests();
         storageTextureBudgetTests();
@@ -9985,6 +10249,7 @@ int main(int argc, char** argv) {
         storeAtLabelTests();
         nullVertexDescriptorTests();
         vertexZeroFillTests();
+        vertexOverhangTests();
         vertexZeroFillMergeTests();
         parallelCompareTests();
         pixelParameterSlotTests();

core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp

@@ -58,6 +58,8 @@ struct MockDevice {
     std::map<VkDeviceMemory, VkDeviceSize> memories;
     std::vector<Bind> binds;
     VkResult bindResult = VK_SUCCESS;
+    VkResult allocationResult = VK_SUCCESS;
+    std::uint64_t allocationAttempts = 0;
     std::uint64_t nextAddress = 0x500000000ull;
     VkDeviceSize sparseAlignment = Alignment;
 };
@@ -119,6 +121,8 @@ VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer bu
 }
 
 VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) {
+    ++mock.allocationAttempts;
+    if (mock.allocationResult != VK_SUCCESS) return mock.allocationResult;
     *memory = reinterpret_cast<VkDeviceMemory>(mock.next++);
     mock.memories[*memory] = info->allocationSize;
     return VK_SUCCESS;
@@ -340,6 +344,34 @@ void LargerSparseBlockIsDecidedBeforeImporting() {
     Expect(mock.memories.size() == memories && mock.binds.empty(), "a chunk was imported or bound for a range that does not take the sparse path");
 }
 
+void RetainsFailedWindowAcrossUnrelatedMutation() {
+    constexpr std::uint64_t window = 4ull << 20u;
+    SharedRange range(window * 2, 0, true);
+    const auto context = mockContext();
+    SetHostImportRangeWindow(window);
+    mock.allocationResult = VK_ERROR_INVALID_EXTERNAL_HANDLE;
+    const auto address = range.Address() + window;
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    const auto attempts = mock.allocationAttempts;
+    Expect(HostImportFor(context, address, Alignment) == nullptr, "the failed import window was accepted");
+    const auto failedAttempts = mock.allocationAttempts;
+    Expect(failedAttempts > attempts, "the import window did not reach allocation");
+    Expect(HostImportFor(context, address, Alignment) == nullptr && mock.allocationAttempts == failedAttempts, "an unchanged failed window was retried");
+    SharedRange unrelated(Alignment, 0, true);
+    Expect(HostImportFor(context, address, Alignment) == nullptr && mock.allocationAttempts == failedAttempts, "an unrelated registry change retried a failed import window");
+    Expect(HostImportFor(context, range.Address(), Alignment) == nullptr && mock.allocationAttempts > failedAttempts, "a different import window inherited the first window's failure");
+    const auto beforeRemap = mock.allocationAttempts;
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Remove(range.backing.base);
+        mutation.Add(range.backing.base, static_cast<std::size_t>(range.backing.bytes), true, true, true);
+    }
+    mock.allocationResult = VK_SUCCESS;
+    const auto* imported = HostImportFor(context, address, Alignment);
+    Expect(imported != nullptr && imported->base == address && imported->bytes == window && mock.allocationAttempts > beforeRemap, "a remapped failed import window was not retried successfully");
+    SetHostImportRangeWindow(64ull << 20u);
+}
+
 void FailureAfterTheDecisionThrows(bool sealed, VkResult bind, const char* step) {
     const auto bytes = RangeBytes();
     SharedRange range(bytes, 0, sealed);
@@ -382,6 +414,7 @@ int main() {
         LargerSparseBlockIsDecidedBeforeImporting();
         FailureAfterTheDecisionThrows(false, VK_SUCCESS, "UDMABUF_CREATE");
         FailureAfterTheDecisionThrows(true, VK_ERROR_DEVICE_LOST, "vkQueueBindSparse");
+        RetainsFailedWindowAcrossUnrelatedMutation();
     } catch (const std::exception& error) {
         std::fprintf(stderr, "%s\n", error.what());
         return 1;

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -38,6 +38,7 @@
 #include "VertexRange_vert_spv.h"
 #include "VertexRangeProducer_frag_spv.h"
 #include "prx/libSceAgcDriver/Graphics/include/ColorTargetTransfer.hpp"
+#include "BdaAbi.hpp"
 #include <SDL_loadso.h>
 #ifdef _WIN32
 #ifndef NOMINMAX
@@ -391,6 +392,14 @@ public:
     bool NullDescriptors() const { return nullDescriptors; }
     const Context& GetContext() const { return context; }
     bool Graphics() const { return graphics; }
+    bool Timestamps() const {
+        std::uint32_t count = 0;
+        const auto queues = function<PFN_vkGetPhysicalDeviceQueueFamilyProperties>("vkGetPhysicalDeviceQueueFamilyProperties");
+        queues(context.physical, &count, nullptr);
+        std::vector<VkQueueFamilyProperties> families(count);
+        queues(context.physical, &count, families.data());
+        return families.at(context.queueFamily).timestampValidBits != 0;
+    }
     VkPhysicalDeviceProperties Properties() const {
         VkPhysicalDeviceProperties properties{};
         function<PFN_vkGetPhysicalDeviceProperties>("vkGetPhysicalDeviceProperties")(context.physical, &properties);
@@ -1194,6 +1203,44 @@ void afterRecordedWorkTests(const Device& device, Recorder& recorder) {
     Require(seen == 1, "an action ran before the completion label recorded ahead of it");
 }
 
+int gpuTimingCompletionTests(const Device& device, bool incomplete) {
+    const auto& context = device.GetContext();
+    if (!device.Timestamps()) {
+        std::cout << "skipped, queue timestamp queries unavailable\n";
+        return 77;
+    }
+    Require(Recorder::GpuTimingEnabled(), "GPU timing completion test requires APS5_PROFILE_GPU");
+    std::unique_lock gpu(GpuMutex());
+    int completed = 0;
+    Recorder recorder(context);
+    recorder.Activate();
+    for (int round = 0; round < 3; ++round) {
+        const bool missingEnd = incomplete && round == 1;
+        const auto first = recorder.BeginGpuTiming(0x12340);
+        Require(first != Recorder::NoTiming, "first GPU timing range was not allocated");
+        if (!missingEnd) recorder.EndGpuTiming(first);
+        const auto second = recorder.BeginGpuTiming(0x12341);
+        Require(second != Recorder::NoTiming && second != first, "second GPU timing range was not allocated");
+        recorder.EndGpuTiming(second);
+        recorder.OnComplete([&] { ++completed; });
+        const auto serial = recorder.Submissions() + 1;
+        recorder.Submit();
+        recorder.Sync();
+        Require(completed == round + 1 && recorder.Idle(), "GPU timing prevented batch completion");
+        std::size_t missing = 0;
+        const auto batches = recorder.CompletedBatches(serial - 1, serial, missing);
+        Require(missing == 0 && batches.size() == 1 && batches.front().serial == serial, "GPU timing batch completion record is missing");
+        const auto& batch = batches.front();
+        if (missingEnd) {
+            Require(batch.gpuStartNs == 0 && batch.gpuEndNs == 0, "incomplete query batch published GPU timings");
+        } else {
+            Require(batch.gpuStartNs > 0 && batch.gpuEndNs >= batch.gpuStartNs, "complete query batch lost GPU timings");
+        }
+    }
+    std::cout << "GPU timing batch completion checked, incomplete=" << incomplete << '\n';
+    return 0;
+}
+
 void batchStampTests(Recorder& recorder) {
     Require(recorder.Idle(), "batch stamps: the recorder is busy");
     double previousEnd = 0;
@@ -2592,11 +2639,11 @@ void resourceReadTests(const Device& device, Recorder& recorder) {
     HostImportFor(context, address, bytes);
 }
 
-void readWrittenStagingTests(const Device& device, Recorder& recorder) {
+bool readWrittenStagingTests(const Device& device, Recorder& recorder) {
     const auto& context = device.GetContext();
     if (context.hostImportAlignment == 0) {
         std::cout << "host imports unavailable: read and written staging not tested\n";
-        return;
+        return false;
     }
     constexpr std::size_t bytes = 8u << 20u;
 #ifdef _WIN32
@@ -2613,7 +2660,7 @@ void readWrittenStagingTests(const Device& device, Recorder& recorder) {
     }
     if (HostImportFor(context, address, bytes) == nullptr) {
         std::cout << "host import of the test block refused: read and written staging not tested\n";
-        return;
+        return false;
     }
     const auto boundInPlace = [&](std::uint64_t element, std::size_t elementBytes, bool read) {
         GuestBufferMemory memory(context);
@@ -2630,10 +2677,10 @@ void readWrittenStagingTests(const Device& device, Recorder& recorder) {
     Require(!boundInPlace(address + 4096, 1u << 20u, false), "a written 1 MiB element was not staged");
     Require(boundInPlace(address + 4096, 4u << 20u, false), "a written-only 4 MiB element was staged past the written window");
     Require(!boundInPlace(address + 4096, 4u << 20u, true), "a read and written 4 MiB element was bound in place");
-    const auto buildInPlace = [&](std::uint64_t dispatchThreads, bool read) {
+    const auto buildInPlace = [&](std::uint64_t dispatchThreads, bool read, std::uint32_t stride, std::uint32_t extent) {
         const auto element = address + 4096;
-        constexpr std::uint32_t stride = 16;
-        constexpr std::uint32_t records = (4u << 20u) / stride;
+        constexpr std::uint32_t elementBytes = 4u << 20u;
+        const std::uint32_t records = stride != 0 ? elementBytes / stride : elementBytes;
         ShaderRecompiler::RecompileResult program;
         ShaderRecompiler::DescriptorBinding binding;
         binding.kind = ShaderRecompiler::DescriptorKind::StorageBuffer;
@@ -2644,25 +2691,30 @@ void readWrittenStagingTests(const Device& device, Recorder& recorder) {
         binding.guestDescriptor = {static_cast<std::uint32_t>(element), (static_cast<std::uint32_t>(element >> 32u) & 0xffffu) | (stride << 16u), records, 0x31000000u};
         binding.bufferWritten = {true};
         binding.bufferRead = {read};
+        binding.bufferExtent = {extent};
         program.bindings.push_back(binding);
         const CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &program, 0};
         ShaderResources resources(context, compute, {}, false, dispatchThreads);
         const auto reads = resources.InPlaceReads();
-        const bool direct = std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return range.first <= element && range.second >= element + records * stride; });
+        const bool direct = std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return range.first <= element && range.second >= element + elementBytes; });
         resources.MarkGpuWrites(recorder);
         recorder.Sync();
         return direct;
     };
-    Require(!buildInPlace((4u << 20u) / 16u, true), "a dispatch sweeping a read and written 4 MiB element bound it in place");
-    Require(buildInPlace(1024, true), "a dispatch touching 16 KiB of a read and written 4 MiB element staged all of it");
-    Require(buildInPlace(0, true), "a dispatch of unknown size staged a read and written 4 MiB element");
-    Require(buildInPlace((4u << 20u) / 16u, false), "a dispatch sweeping a written-only 4 MiB element staged it");
+    Require(!buildInPlace((4u << 20u) / 16u, true, 16, 0), "a dispatch sweeping a read and written 4 MiB element bound it in place");
+    Require(buildInPlace(1024, true, 16, 0), "a dispatch touching 16 KiB of a read and written 4 MiB element staged all of it");
+    Require(buildInPlace(0, true, 16, 0), "a dispatch of unknown size staged a read and written 4 MiB element");
+    Require(buildInPlace((4u << 20u) / 16u, false, 16, 0), "a dispatch sweeping a written-only 4 MiB element staged it");
+    Require(!buildInPlace((4u << 20u) / 32u, true, 0, 32), "a dispatch sweeping a read and written 4 MiB raw element bound it in place");
+    Require(buildInPlace(1024, true, 0, 32), "a dispatch touching 32 KiB of a read and written 4 MiB raw element staged all of it");
+    Require(buildInPlace((4u << 20u) / 32u, true, 0, 0), "a raw element without an access extent was staged");
     Require(std::all_of(static_cast<const std::byte*>(block), static_cast<const std::byte*>(block) + bytes, [](std::byte value) { return value == std::byte{0x11}; }), "a staged copy-back changed the guest bytes");
     {
         GuestAllocations::Mutation mutation;
         mutation.Remove(block);
     }
     HostImportFor(context, address, bytes);
+    return true;
 }
 
 bool sparseRunImportTests(const Device& device, bool copied) {
@@ -3081,7 +3133,8 @@ void drawDescriptorWriteTests(const Device& device, Recorder& recorder) {
         const auto original = updates.calls.front();
         Require(original.buffers.size() == 2 && original.images.size() == 2, "descriptor test lacks mixed bindings and an image array");
         for (std::size_t allocation = 0; allocation < 2; ++allocation) {
-            const ShaderResources::MovedBuffer moved{allocation, 0, 16, {5, 6, 7, 8}};
+            const std::array<std::uint32_t, 4> words{5, 6, 7, 8};
+            const ShaderResources::MovedBuffer moved{allocation, 0, 16, words};
             const auto draw = resources.PrepareDrawBindings(snapshots, std::span(&moved, 1));
             Require(draw != nullptr && draw->snapshots.size() == 1, "moved shader data did not create a draw snapshot");
             Require(updates.calls.size() == allocation + 2 && updates.copies == 0, "draw descriptors copied GPU memory or needed multiple updates");
@@ -3165,7 +3218,8 @@ void drawSnapshotTableSetTests(const Device& device, Recorder& recorder) {
         const ColorTarget target{};
         ShaderResources resources(context, vertex, fragment, target, 0, 0);
         Require(resources.SplitSets(), "a fragment image table did not get its own descriptor set");
-        const ShaderResources::MovedBuffer moved{0, 0, 16, {5, 6, 7, 8}};
+        const std::array<std::uint32_t, 4> movedWords{5, 6, 7, 8};
+        const ShaderResources::MovedBuffer moved{0, 0, 16, movedWords};
         const auto draw = resources.PrepareDrawBindings(snapshots, std::span(&moved, 1));
         Require(draw != nullptr && draw->snapshots.size() == 1, "moved shader data did not create a draw snapshot");
         const auto& written = updates.calls.back().written;
@@ -4806,17 +4860,21 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche
         std::uint32_t height;
         std::uint32_t mipCount;
         std::uint64_t offset;
+        bool direct = false;
     };
     constexpr Case cases[] = {
         {"64 KiB R_X 200x150", TextureTileMode::kR64KBX, 200, 150, 1, 0},
         {"64 KiB R_X 200x150, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 0},
-        {"64 KiB R_X 520x260, 6 mips", TextureTileMode::kR64KBX, 520, 260, 6, 0},
+        {"64 KiB R_X 520x260, 6 mips", TextureTileMode::kR64KBX, 520, 260, 6, 0, true},
         {"64 KiB R_X 200x150 off a 64 KiB boundary, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 4096},
         {"4 KiB S 100x70, 2 mips", TextureTileMode::kStandard4KB, 100, 70, 2, 0},
         {"linear 200x150", TextureTileMode::kLinear, 200, 150, 1, 0},
+        {"64 KiB R_X 512x512, 3 mips, no padding below the tail", TextureTileMode::kR64KBX, 512, 512, 3, 0, true},
+        {"64 KiB R_X 1000x700", TextureTileMode::kR64KBX, 1000, 700, 1, 0, true},
+        {"64 KiB R_X 1900x1100, across two 8 MiB slabs", TextureTileMode::kR64KBX, 1900, 1100, 1, 0, true},
     };
     const auto pattern = [](std::size_t i) { return static_cast<std::uint8_t>(i * 131u + 7u); };
-    constexpr std::array<std::uint8_t, 4> red{255, 0, 0, 255};
+    constexpr std::array<std::array<std::uint8_t, 4>, 2> colours{{{255, 0, 0, 255}, {0, 255, 0, 255}}};
     std::string failures;
     for (const auto& test : cases) {
         const std::string what = std::string(watched ? "(watched) " : "") + test.name;
@@ -4886,34 +4944,40 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche
         {
             auto image = StorageTexture::Create(context, detiler, resource, 0);
             const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, test.mipCount, 0, 1};
-            const auto commands = recorder.Commands();
-            recorder.Keep(image);
-            const VkClearColorValue value{{1.0f, 0.0f, 0.0f, 1.0f}};
-            RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
-            context.Function<PFN_vkCmdClearColorImage>("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range);
-            RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
-            image->MarkDirty();
-            image->WriteBack();
-            recorder.Submit();
-            device.WaitQueue();
-            recorder.Sync();
-            std::vector<std::byte> read(bytes);
-            if (watched) AgcDriver::GuestMemory::Read(blockAddress, read);
-            else std::memcpy(read.data(), memory, bytes);
-            std::size_t texels = 0, garbage = 0, outside = 0;
-            for (std::size_t i = 0; i < bytes; i += 4) {
-                const bool inside = i >= test.offset && i < test.offset + surfaceBytes;
-                bool isTexel = inside, isPattern = true;
-                for (std::size_t c = 0; c < 4; ++c) {
-                    const auto byte = std::to_integer<std::uint8_t>(read[i + c]);
-                    isTexel = isTexel && byte == red[c];
-                    isPattern = isPattern && byte == pattern(i + c);
+            for (std::size_t pass = 0; pass < (test.direct ? colours.size() : 1u); ++pass) {
+                const auto& colour = colours[pass];
+                const auto commands = recorder.Commands();
+                recorder.Keep(image);
+                const VkClearColorValue value{{colour[0] / 255.0f, colour[1] / 255.0f, colour[2] / 255.0f, colour[3] / 255.0f}};
+                RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
+                context.Function<PFN_vkCmdClearColorImage>("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range);
+                RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
+                image->MarkDirty();
+                const auto slabRetilesBefore = StorageTexture::SlabRetiles();
+                image->WriteBack();
+                recorder.Submit();
+                device.WaitQueue();
+                recorder.Sync();
+                const auto slabRetiled = StorageTexture::SlabRetiles() - slabRetilesBefore;
+                if (test.direct && watched && UnitShadowEnabled() && AgcDriver::GuestMemory::Watched(address, surfaceBytes) && slabRetiled == 0) failures += what + ": no window was retiled straight into its unit-shadow slab\n";
+                std::vector<std::byte> read(bytes);
+                if (watched) AgcDriver::GuestMemory::Read(blockAddress, read);
+                else std::memcpy(read.data(), memory, bytes);
+                std::size_t texels = 0, garbage = 0, outside = 0;
+                for (std::size_t i = 0; i < bytes; i += 4) {
+                    const bool inside = i >= test.offset && i < test.offset + surfaceBytes;
+                    bool isTexel = inside, isPattern = true;
+                    for (std::size_t c = 0; c < 4; ++c) {
+                        const auto byte = std::to_integer<std::uint8_t>(read[i + c]);
+                        isTexel = isTexel && byte == colour[c];
+                        isPattern = isPattern && byte == pattern(i + c);
+                    }
+                    if (isTexel) ++texels;
+                    else if (!isPattern) ++(inside ? garbage : outside);
+                }
+                if (garbage != 0 || outside != 0 || texels != texelCount) {
+                    failures += what + ", write-back " + std::to_string(pass + 1) + ": stored " + std::to_string(texels) + " texels of " + std::to_string(texelCount) + " and changed " + std::to_string(garbage) + " words no texel holds (" + std::to_string(outside) + " past the surface)\n";
                 }
-                if (isTexel) ++texels;
-                else if (!isPattern) ++(inside ? garbage : outside);
-            }
-            if (garbage != 0 || outside != 0 || texels != texelCount) {
-                failures += what + ": the write-back stored " + std::to_string(texels) + " texels of " + std::to_string(texelCount) + " and changed " + std::to_string(garbage) + " words no texel holds (" + std::to_string(outside) + " past the surface)\n";
             }
         }
         recorder.Sync();
@@ -7036,6 +7100,31 @@ void multisampleGuestAccessTests(const Device& device, Recorder& recorder) {
     refused([&] { AgcDriver::GuestMemory::ReadCommitted(address + 256u, texels); }, "a CPU read", "a committed CPU read");
     refused([&] { AgcDriver::GuestMemory::ReadCommittedUnsynced(address + 256u, texels); }, "a CPU read", "a committed CPU read without the flush hook");
     refused([&] { RequireNoMultisampleTarget(address, 64u, "a DMA copy"); }, "a DMA copy", "a DMA copy");
+    const auto captureSrt = [](std::uint64_t srt, AgcDriver::ShaderMemory::PendingWriteQuery pendingWrite) {
+        static constexpr std::array<std::uint32_t, 6> code{0xf4080100u, 0xfa000000u, 0xbf8cc07fu, 0xe0700000u, 0x80010000u, 0xbf810000u};
+        const std::array<std::uint32_t, 2> userData{static_cast<std::uint32_t>(srt), static_cast<std::uint32_t>(srt >> 32u)};
+        ShaderRecompiler::RecompileRequest request{};
+        request.shader = {ShaderRecompiler::ShaderStage::Compute, 0x24000u, code, 0, {}};
+        request.context.waveSize = 32u;
+        request.context.userDataBaseRegister = 0;
+        request.context.userData = userData;
+        request.context.compute = ShaderRecompiler::ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
+        request.target.vulkanVersion = 0x00401000u;
+        request.target.spirvVersion = 0x00010300u;
+        request.target.subgroupSize = 32;
+        request.target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version;
+        request.layout.pushConstantSizeBytes = 128;
+        AgcDriver::ShaderMemory memory({}, pendingWrite);
+        static_cast<void>(memory.Capture(request));
+    };
+    refused([&] { captureSrt(address + 256u, nullptr); }, "a CPU read", "a shader capture");
+    captureSrt(address + 4u * 65536u + 256u, nullptr);
+    refused([&] { captureSrt(address + 256u, nullptr); }, "a CPU read", "a shader capture on a pooled page");
+    const auto raw = [](std::uint64_t, std::size_t, std::span<std::byte>) { return AgcDriver::ShaderMemory::PendingWrite::Raw; };
+    const auto synced = [](std::uint64_t, std::size_t, std::span<std::byte>) { return AgcDriver::ShaderMemory::PendingWrite::Sync; };
+    refused([&] { captureSrt(address + 256u, raw); }, "a CPU read", "a shader capture's raw word read");
+    refused([&] { captureSrt(address + 256u, synced); }, "a CPU read", "a shader capture's synced word read");
+    captureSrt(address + 4u * 65536u + 256u, raw);
     ShaderRecompiler::RecompileResult program;
     ShaderRecompiler::DescriptorBinding binding;
     binding.kind = ShaderRecompiler::DescriptorKind::SampledImage;
@@ -8918,6 +9007,9 @@ int main(int argc, char** argv) {
         Device& device = *created;
         if (argc == 2 && std::string_view(argv[1]) == "--host-import-lifetime-only") return RunHostImportLifetimeTests(device.GetContext());
         if (argc == 2 && std::string_view(argv[1]) == "--host-import-mutation-race-only") return RunHostImportMutationRaceTests(device.GetContext());
+        if (argc == 2 && (std::string_view(argv[1]) == "--gpu-timing-complete-only" || std::string_view(argv[1]) == "--gpu-timing-incomplete-only")) {
+            return gpuTimingCompletionTests(device, std::string_view(argv[1]) == "--gpu-timing-incomplete-only");
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--import-watch-only") {
             importWatchTests(device);
             return 0;
@@ -9059,6 +9151,11 @@ int main(int argc, char** argv) {
             std::cout << "Completion label storage and ordering tests passed\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--read-written-staging-only") {
+            if (!readWrittenStagingTests(device, recorder)) return 77;
+            std::cout << "read and written staging tests passed\n";
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--written-buffer-order-only") {
             if (!writtenBufferOrderTests(device, recorder)) return 77;
             std::cout << "written buffer order tests passed\n";
@@ -9106,6 +9203,19 @@ int main(int argc, char** argv) {
             splitWriteBackTests(device, recorder);
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--storage-write-back-only") {
+            if (device.GetContext().hostImportAlignment == 0) {
+                std::cout << "skipped, host imports are unavailable\n";
+                return 77;
+            }
+            storageRefreshTests(device, recorder, false);
+            storageRefreshTests(device, recorder, true);
+            writeBackPaddingTests(device, recorder, false);
+            writeBackPaddingTests(device, recorder, true);
+            singlePassTests(device, recorder, false);
+            singlePassTests(device, recorder, true);
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--cmask-only") {
             cmaskPassTests(device, recorder);
             std::cout << "CMASK copied clear, write-back and failure tests passed\n";

core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp

@@ -84,7 +84,7 @@ void removeCacheDirectory(const std::filesystem::path& directory) {
 }
 
 bool sameBinding(const DescriptorBinding& left, const DescriptorBinding& right) {
-    return left.kind == right.kind && left.role == right.role && left.descriptorSet == right.descriptorSet && left.binding == right.binding && left.count == right.count && left.guestDescriptor == right.guestDescriptor && left.readOnly == right.readOnly && left.imageShape == right.imageShape && left.samplerDepthCompare == right.samplerDepthCompare && left.imageWritten == right.imageWritten && left.imageDepthCompare == right.imageDepthCompare && left.imageAtomic == right.imageAtomic && left.imageShapes == right.imageShapes && left.bufferAtomic == right.bufferAtomic && left.bufferWritten == right.bufferWritten && left.bufferRead == right.bufferRead && left.samplerUnnormalized == right.samplerUnnormalized && left.samplerIntegerBorder == right.samplerIntegerBorder && left.samplerCoordinatesIgnored == right.samplerCoordinatesIgnored && left.imageUnnormalized == right.imageUnnormalized && left.imageUnnormalizedBaseLevel == right.imageUnnormalizedBaseLevel && left.imageSamplers == right.imageSamplers;
+    return left.kind == right.kind && left.role == right.role && left.descriptorSet == right.descriptorSet && left.binding == right.binding && left.count == right.count && left.guestDescriptor == right.guestDescriptor && left.readOnly == right.readOnly && left.imageShape == right.imageShape && left.samplerDepthCompare == right.samplerDepthCompare && left.imageWritten == right.imageWritten && left.imageDepthCompare == right.imageDepthCompare && left.imageAtomic == right.imageAtomic && left.imageShapes == right.imageShapes && left.bufferAtomic == right.bufferAtomic && left.bufferWritten == right.bufferWritten && left.bufferRead == right.bufferRead && left.bufferExtent == right.bufferExtent && left.samplerUnnormalized == right.samplerUnnormalized && left.samplerIntegerBorder == right.samplerIntegerBorder && left.samplerCoordinatesIgnored == right.samplerCoordinatesIgnored && left.imageUnnormalized == right.imageUnnormalized && left.imageUnnormalizedBaseLevel == right.imageUnnormalizedBaseLevel && left.imageSamplers == right.imageSamplers;
 }
 
 bool sameBindings(const std::vector<DescriptorBinding>& left, const std::vector<DescriptorBinding>& right) {
@@ -125,6 +125,7 @@ void requireSameResult(const RecompileResult& left, const RecompileResult& right
     const std::string prefix = std::string(what) + ": ";
     require(sameBindings(left.bindings, right.bindings), prefix + "bindings differ");
     require(left.pushConstants == right.pushConstants, prefix + "push constants differ");
+    require(left.readsDepthRemapPush == right.readsDepthRemapPush, prefix + "depth remap push reads differ");
     require(left.specialization == right.specialization, prefix + "specialization constants differ");
     require(left.workgroupMemoryDwords == right.workgroupMemoryDwords, prefix + "workgroup memory stride differs");
     require(left.poisonedSrtReads == right.poisonedSrtReads, prefix + "poisoned SRT read counts differ");
@@ -181,6 +182,7 @@ DescriptorBinding sampleBinding(std::uint32_t seed) {
     binding.imageUnnormalizedBaseLevel = {false, true, seed % 3 == 0};
     binding.imageSamplers = {0x5u, 0u, 0x80000000u};
     binding.bufferRead = {true, false, false, true, true, false};
+    binding.bufferExtent = {32, 0, 16, 4096, 8, 0};
     return binding;
 }
 
@@ -195,6 +197,7 @@ RecompileResult sampleResult() {
     result.bindings[1].role = DescriptorRole::GuestBuffers;
     result.bindings[1].imageShape.reset();
     result.pushConstants = {std::byte{1}, std::byte{0xff}, std::byte{0}, std::byte{0x80}, std::byte{7}};
+    result.readsDepthRemapPush = true;
     result.bdaAbiVersion = 3;
     result.memoryOffsetDword = 7;
     result.hostSubgroupSize = 32;
@@ -2141,6 +2144,10 @@ std::filesystem::path planEntryPath(PlanFixture& fixture) {
 int runPlanLoadingProcess(std::string_view mode) {
     if (mode == "--load-plan-switch") setEnvironment("APS5_TRACE_IMAGE_TABLE", "1");
     if (mode == "--load-plan-off") setEnvironment("APS5_NO_PERF_DISK_PLANS", "1");
+    if (mode == "--load-plan-reuse-off") {
+        setEnvironment("APS5_NO_TABLE_SNAPSHOT_REUSE", "1");
+        setEnvironment("APS5_NO_TABLE_RESOLUTION_REUSE", "1");
+    }
     auto fixtures = planFixtures();
     const auto count = fixtures.size();
     std::vector<std::shared_ptr<const IrResourcePlan>> plans;
@@ -2156,6 +2163,8 @@ int runPlanLoadingProcess(std::string_view mode) {
     }
     if (mode == "--load-plan") {
         require(builds == 0u && totals.planHits == count && totals.planMisses == 0u && totals.planLoadFailures == 0u && totals.hits == count && totals.misses == 0u, "the second process ran the front end" + counts);
+    } else if (mode == "--load-plan-reuse-off") {
+        require(builds == 0u && totals.planHits == count && totals.planMisses == 0u && totals.planLoadFailures == 0u && totals.hits == count && totals.misses == 0u, "turning the image table reuse off changed the stored plan or variant keys" + counts);
     } else if (mode == "--load-plan-off") {
         require(builds == count && totals.planHits == 0u && totals.planMisses == 0u && totals.planWrites == 0u && totals.hits == count && totals.misses == 0u, "APS5_NO_PERF_DISK_PLANS=1 used the stored plans or missed the stored variants" + counts);
     } else {
@@ -2175,6 +2184,8 @@ int runPlanLoadingProcess(std::string_view mode) {
         const auto after = ShaderDiskCache::Totals();
         require(FrontEndBuilds() == 1u && after.planHits == count && after.planMisses == 1u, "a changed code word reused a stored plan");
         std::cout << "second process prepared " << count << " shaders from stored plans without running the front end\n";
+    } else if (mode == "--load-plan-reuse-off") {
+        std::cout << "with the image table reuse off, a process loaded all " << count << " stored plans and variants with the same output\n";
     } else if (mode == "--load-plan-off") {
         std::cout << "APS5_NO_PERF_DISK_PLANS=1 ran the front end for all " << count << " shaders with the same output\n";
     } else {
@@ -2218,6 +2229,7 @@ void verifyPlanAcrossProcesses(const char* self) {
     };
     run("--load-plan");
     run("--load-plan-off");
+    run("--load-plan-reuse-off");
     std::vector<std::byte> entry;
     const auto damaged = planEntryPath(fixtures.front());
     require(ReadWholeFile(damaged, entry), "cannot read a stored plan entry");

core/libs/prx/libSceAgcDriver/tests/execution/DepthRangeRemapDraw.cpp

@@ -0,0 +1,257 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "Recompiler.hpp"
+#include "Optimization/ResourceProgram.hpp"
+#include "DepthFastClearHarness.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <optional>
+#include <span>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace {
+
+namespace Harness = DepthFastClearHarness;
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+struct Transform {
+    bool negativeOneToOne;
+    float scale;
+    float offset;
+    float clampMin;
+    float clampMax;
+};
+
+struct Case {
+    Transform transform;
+    float z;
+    bool drawn;
+    float probe;
+    bool covered;
+    bool remapOnly;
+    const char* what;
+};
+
+constexpr Transform Plain{false, 1.0f, 0.0f, 0.0f, 1.0f};
+
+alignas(4096) std::array<std::array<float, Harness::Width * Harness::Height>, 32> Depths{};
+alignas(256) std::array<std::array<std::array<float, 4>, 3>, 128> VertexSets{};
+std::size_t vertexSets = 0;
+
+// A vertex shader that reads no user data: the fullscreen triangle comes from the vertex index (v5), and the
+// clip z is the literal at index 9. Its program carries no push data of its own.
+alignas(256) std::array<std::uint32_t, 14> FetchlessCode{
+    0x36020a81u, 0x34020282u, 0x4a0202c1u, 0x7e020b01u, 0x2c040a81u, 0x34040482u, 0x4a0404c1u, 0x7e040b02u,
+    0x7e0602ffu, 0u, 0x7e0802f2u, 0xf80008cfu, 0x04030201u, 0xbf810000u,
+};
+
+std::uint32_t Draw(AgcDriver::VulkanDevice& device, std::uint64_t depth, const Transform& transform, bool remap, float z, bool write, bool fetchless = false, bool targetRemap = false, bool prepared = false) {
+    Require(vertexSets < VertexSets.size(), "the test ran out of vertex sets");
+    auto& vertices = VertexSets[vertexSets++];
+    vertices = {{{-1.0f, -1.0f, z, 1.0f}, {3.0f, -1.0f, z, 1.0f}, {-1.0f, 3.0f, z, 1.0f}}};
+    Harness::Pixels.fill(std::byte{0});
+
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(Harness::Pixels.data()), {Harness::Width, Harness::Height}, VK_FORMAT_R8G8B8A8_UNORM, Harness::Pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.depth = AgcDriver::Graphics::DepthTarget{depth, 0, {Harness::Width, Harness::Height}, VK_FORMAT_D32_SFLOAT, 1.0f, 0};
+    state.depthTest = true;
+    state.depthWrite = write;
+    state.depthCompare = write ? VK_COMPARE_OP_ALWAYS : VK_COMPARE_OP_LESS;
+    state.renderExtent = {Harness::Width, Harness::Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    const float minDepth = transform.negativeOneToOne ? transform.offset - transform.scale : transform.offset;
+    state.viewport = {0, static_cast<float>(Harness::Height), static_cast<float>(Harness::Width), -static_cast<float>(Harness::Height), minDepth, transform.offset + transform.scale};
+    state.negativeOneToOne = transform.negativeOneToOne;
+    state.depthScale = transform.scale;
+    state.depthOffset = transform.offset;
+    state.depthClampMin = transform.clampMin;
+    state.depthClampMax = transform.clampMax;
+    state.scissor = {{0, 0}, {Harness::Width, Harness::Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15;
+    state.blends = {state.blend};
+    state.blendConstants = {};
+    auto context = device.graphicsContext();
+    context.depthRangeUnrestricted = !remap;
+    AgcDriver::Graphics::ApplyDepthRemap(context, state);
+    const bool inRange = minDepth >= 0.0f && transform.offset + transform.scale <= 1.0f;
+    Require((state.depthRemap.mode != 0u) == (remap && !inRange), "the draw did not take the depth path the test asked for");
+
+    auto target = device.Target();
+    target.depthRangeRemap = state.depthRemap.mode != 0u || targetRemap;
+    ShaderRecompiler::ShaderVertexStageInfo vertexInfo{};
+    vertexInfo.depthRemapMode = state.depthRemap.mode;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    if (fetchless) {
+        FetchlessCode[9] = std::bit_cast<std::uint32_t>(z);
+    } else {
+        const auto vertexBuffer = Harness::BufferDescriptor(vertices.data(), 16u, static_cast<std::uint32_t>(vertices.size()));
+        std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    }
+    const std::span<const std::uint32_t> vertexCode = fetchless ? std::span<const std::uint32_t>(FetchlessCode) : std::span<const std::uint32_t>(Harness::VertexCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(vertexCode.data()), std::as_bytes(vertexCode)}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(vertexCode.data()), vertexCode, 0, {}},
+        {32, 0, vertexUserData, std::nullopt, std::nullopt, vertexInfo, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = prepared;
+    std::shared_ptr<const ShaderRecompiler::RecompileResult> vertexHolder;
+    if (prepared) {
+        const auto handle = ShaderRecompiler::PrepareShader(vertex);
+        const auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(vertex, handle);
+        Require(invocation.has_value(), "the prepared vertex shader did not match its own request");
+        vertexHolder = invocation->Materialize(*invocation->Capture(ShaderRecompiler::SrtRuntime{std::span<const std::uint32_t>(vertexUserData)}));
+    } else {
+        vertexHolder = std::make_shared<const ShaderRecompiler::RecompileResult>(ShaderRecompiler::Recompile(vertex));
+    }
+    const auto& vertexResult = *vertexHolder;
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+    if (fetchless) Require(vertexPush == 0u, "the fetchless vertex shader carries push data of its own");
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = true;
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY);
+    pixel.posX = true;
+    pixel.posY = true;
+    pixel.targetOutputMode[0] = 9;
+    pixel.targetExportMapping.fill(0xe4u);
+    std::vector<std::uint32_t> pixelUserData(4, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(Harness::PixelCode.data()), std::as_bytes(std::span(Harness::PixelCode))}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(Harness::PixelCode.data()), Harness::PixelCode, 0, {}},
+        {32, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(vertices.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+    device.WaitIdle();
+    std::uint32_t covered = 0;
+    for (std::size_t index = 0; index < Harness::Pixels.size() / 4u; ++index) {
+        if (std::to_integer<std::uint8_t>(Harness::Pixels[index * 4u]) == 255u) ++covered;
+    }
+    return covered;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const auto context = device->graphicsContext();
+        if (!context.depthClamp) {
+            std::puts("skipped, the device has no depth clamping");
+            return VulkanTestSkipped;
+        }
+        constexpr float Step = 1.0f / 64.0f;
+        constexpr Transform minecraft{true, 1.0f, 0.0f, -1.0f, 1.0f};
+        constexpr Transform widened{false, 2.0f, -0.5f, 0.0f, 1.0f};
+        constexpr Transform bounded{false, 2.0f, -0.5f, 0.25f, 0.75f};
+        constexpr std::array<Case, 13> cases{{
+            {minecraft, 0.5f - Step, true, 0.5f, false, false, "negative-one-to-one depth just below 0.5"},
+            {minecraft, 0.5f + Step, true, 0.5f, true, false, "negative-one-to-one depth just above 0.5"},
+            {minecraft, -1.5f, false, 0.5f, true, false, "negative-one-to-one clip z in front of the near plane"},
+            {minecraft, 1.5f, false, 0.5f, true, false, "negative-one-to-one clip z behind the far plane"},
+            {widened, 0.5f - Step / 2.0f, true, 0.5f, false, false, "scaled zero-to-one depth just below 0.5"},
+            {widened, 0.5f + Step / 2.0f, true, 0.5f, true, false, "scaled zero-to-one depth just above 0.5"},
+            {widened, -0.125f, false, 0.5f, true, false, "zero-to-one clip z in front of the near plane"},
+            {widened, 1.125f, false, 0.5f, true, false, "zero-to-one clip z behind the far plane"},
+            {bounded, 0.4f, true, 0.35f, false, false, "depth 0.3 inside clamp bounds [0.25, 0.75], probed at 0.35"},
+            {bounded, 0.4f, true, 0.25f, true, false, "depth 0.3 inside clamp bounds [0.25, 0.75], probed at 0.25"},
+            {bounded, 0.6f, true, 0.65f, true, false, "depth 0.7 inside clamp bounds [0.25, 0.75], probed at 0.65"},
+            {bounded, 0.2f, true, 0.2f, true, true, "depth -0.1 clamped to 0.25, probed at 0.2"},
+            {bounded, 0.8f, true, 0.8f, false, true, "depth 1.1 clamped to 0.75, probed at 0.8"},
+        }};
+        std::size_t surfaces = 0;
+        std::size_t ran = 0;
+        for (const bool remap : {true, false}) {
+            if (!remap && !context.depthRangeUnrestricted) continue;
+            for (const auto& test : cases) {
+                if (!remap && (test.remapOnly || (test.transform.negativeOneToOne && !context.depthClipControl))) continue;
+                Require(surfaces < Depths.size(), "the test ran out of depth surfaces");
+                const auto depth = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Depths[surfaces++].data()));
+                const auto drawn = Draw(*device, depth, test.transform, remap, test.z, true);
+                const auto passed = Draw(*device, depth, Plain, false, test.probe, false);
+                const auto path = remap ? "remapped" : "unrestricted";
+                if (drawn != (test.drawn ? Harness::Covered : 0u) || passed != (test.covered ? Harness::Covered : 0u)) {
+                    char detail[256];
+                    std::snprintf(detail, sizeof(detail), "%s (%s path): the draw covered %u of %u pixels (expected %s) and the probe passed %u (expected %s)", test.what, path, drawn, Harness::Covered, test.drawn ? "all" : "none", passed, test.covered ? "all" : "none");
+                    throw std::runtime_error(detail);
+                }
+                ++ran;
+            }
+        }
+        // Scale and offset reach the vertex shader through the draw's push block, so two draws whose depth remap
+        // differs only in them build one pipeline between them: the second finds it in the store, and both
+        // store the depth they should.
+        {
+            constexpr Transform widerScale{false, 4.0f, -1.5f, 0.0f, 1.0f};
+            Require(surfaces + 2u <= Depths.size(), "the test ran out of depth surfaces");
+            const auto depthA = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Depths[surfaces++].data()));
+            const auto depthB = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Depths[surfaces++].data()));
+            const auto first = Draw(*device, depthA, widened, true, 0.5f + Step / 2.0f, true);
+            const auto before = AgcDriver::Graphics::PipelineStoreTotals();
+            const auto second = Draw(*device, depthB, widerScale, true, 0.5f + Step / 4.0f, true);
+            const auto after = AgcDriver::Graphics::PipelineStoreTotals();
+            Require(first == Harness::Covered && second == Harness::Covered, "a draw with a remapped depth did not cover the target");
+            Require(after.misses == before.misses && after.hits > before.hits, "a draw that differs only in the depth scale and offset built another pipeline");
+            for (const auto depth : {depthA, depthB}) {
+                Require(Draw(*device, depth, Plain, false, 0.5f, false) == Harness::Covered, "a depth remapped by a shared pipeline was stored below 0.5 + 1/64");
+                Require(Draw(*device, depth, Plain, false, 0.5f + 2.0f * Step, false) == 0u, "a depth remapped by a shared pipeline was stored above 0.5 + 1/64");
+            }
+            ++ran;
+        }
+        // A remapped vertex shader with no push data of its own, and a fragment shader with none: the pipeline's
+        // push stages must still include the vertex stage (a non-GPL layout names them), or the scale and offset
+        // reach no one.
+        {
+            Require(surfaces < Depths.size(), "the test ran out of depth surfaces");
+            const auto depth = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Depths[surfaces++].data()));
+            const auto drawn = Draw(*device, depth, widened, true, 0.5f + Step / 2.0f, true, true);
+            Require(drawn == Harness::Covered, "a fetchless remapped vertex shader did not cover the target");
+            Require(Draw(*device, depth, Plain, false, 0.5f, false) == Harness::Covered, "a fetchless remapped vertex shader stored a depth below 0.5 + 1/64, so its scale and offset did not arrive");
+            Require(Draw(*device, depth, Plain, false, 0.5f + 2.0f * Step, false) == 0u, "a fetchless remapped vertex shader stored a depth above 0.5 + 1/64");
+            ++ran;
+        }
+        // On a device without VK_EXT_depth_range_unrestricted the target remaps every vertex shader, so an ordinary
+        // in-range draw (mode 0) with no push data of its own still carries the remap's push block: it must validate
+        // and draw, from a direct build and from a prepared one.
+        for (const bool prepared : {false, true}) {
+            Require(surfaces < Depths.size(), "the test ran out of depth surfaces");
+            const auto depth = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Depths[surfaces++].data()));
+            const auto drawn = Draw(*device, depth, Plain, false, 0.5f + Step / 2.0f, true, true, true, prepared);
+            Require(drawn == Harness::Covered, prepared ? "a fetchless mode 0 vertex shader on a remapping target, prepared, did not cover the target" : "a fetchless mode 0 vertex shader on a remapping target did not cover the target");
+            Require(Draw(*device, depth, Plain, false, 0.5f, false) == Harness::Covered && Draw(*device, depth, Plain, false, 0.5f + 2.0f * Step, false) == 0u, "a fetchless mode 0 vertex shader on a remapping target stored the wrong depth");
+            ++ran;
+        }
+        std::printf("depth range remap draw tests passed (%zu cases, unrestricted path %s)\n", ran, context.depthRangeUnrestricted ? "compared" : "absent");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp

@@ -1,5 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp"
+#include "Optimization/SrtWalker.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
 #include "VulkanTestDevice.hpp"
 #include <array>
@@ -11,6 +13,8 @@
 #include <exception>
 #include <initializer_list>
 #include <iostream>
+#include <memory>
+#include <span>
 #include <string>
 #include <utility>
 #include <vector>
@@ -131,6 +135,27 @@ void MovedDataHitsTheCache(bool libraries) {
     Require(again.lookups == 10 && again.hits == 5, mode + ": a draw at an address seen before did not reuse the cached draw (" + std::to_string(again.hits) + " hits)");
 }
 
+void SplitRunsNameTheirSharedWords() {
+    alignas(16) static std::array<std::uint32_t, 4> data{1u, 2u, 3u, 4u};
+    auto table = std::make_shared<ShaderRecompiler::SrtGuestWords>();
+    table->words = {5u, 6u, 7u, 8u};
+    const auto dataAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data.data()));
+    const auto tableAddress = dataAddress + 0x1000u;
+    const std::array<ShaderRecompiler::MemoryRegion, 2> regions{{{dataAddress, std::as_bytes(std::span(data))}, {tableAddress, std::as_bytes(std::span(table->words))}}};
+    ShaderRecompiler::SrtUserDataUse use;
+    use.asBase = 0x3u;
+    use.reads = {{dataAddress, 0}, {dataAddress + 8u, 1}};
+    const std::array<std::shared_ptr<const ShaderRecompiler::SrtGuestWords>, 1> supplied{table};
+    AgcDriver::DriverDetail::DispatchVariant variant;
+    AgcDriver::DriverDetail::Driver::tagVariantRuns(variant, regions, use, true, 0, supplied);
+    Require(variant.runs.size() > regions.size(), "two user-data bases into one region did not split it into several runs");
+    Require(variant.shared.size() == 1u && variant.shared.front().regions == 1u, "the table region's words were not recorded as one shared run");
+    const auto& shared = variant.shared.front();
+    std::size_t first = 0;
+    for (std::size_t run = 0; run < shared.region && run < variant.runs.size(); ++run) first += static_cast<std::size_t>((variant.runs[run].second - variant.runs[run].first) / sizeof(std::uint32_t));
+    Require(shared.region < variant.runs.size() && variant.runs[shared.region].first == tableAddress && first == shared.first, "the shared words of a variant whose runs were split for relocation name run " + std::to_string(shared.region) + " of " + std::to_string(variant.runs.size()) + ", not the table's run " + std::to_string(variant.runs.size() - 1u));
+}
+
 void UnrelocatedAliasesStayOut(bool libraries) {
     Registered(SecondVertexHeader, SecondVertexCode.data(), sizeof(SecondVertexCode), 2);
     Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
@@ -205,6 +230,7 @@ void MovedForwardedWordReachesStageDataHit(bool libraries) {
 
 int main() {
     try {
+        SplitRunsNameTheirSharedWords();
         bool libraries = false;
         {
             const auto device = OpenVulkanTestDevice();

core/libs/prx/libSceAgcDriver/tests/execution/DwordPatternFill.cpp

@@ -26,6 +26,13 @@ alignas(256) constexpr std::array<std::uint32_t, 69> Code{
     0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
     0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
     0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u};
+constexpr std::size_t ComparesAt = 27;
+constexpr std::array<std::uint32_t, 69> WithCompares(std::array<std::uint32_t, 3> compares) {
+    auto code = Code;
+    for (std::size_t word = 0; word < compares.size(); ++word) code[ComparesAt + word] = compares[word];
+    return code;
+}
+alignas(256) constexpr std::array<std::uint32_t, 69> SwappedCode = WithCompares({0x7d8c02f9u, 0x06068c02u, 0x7d860609u});
 constexpr std::array<std::uint32_t, 4> Pattern{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
 constexpr std::uint32_t Untouched = 0xcafebabeu;
 alignas(256) std::array<std::uint32_t, 1028> Output{};
@@ -35,6 +42,35 @@ std::vector<std::uint32_t> UserData(void* output, std::uint32_t records, std::ui
     return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (4u << 16u), records, format, Pattern[0], Pattern[1], Pattern[2], Pattern[3], count, period};
 }
 
+void SwappedComparesTests() {
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{64, 1, 1}, 0, {true, false, false}, false, 1};
+    const std::array<std::uint32_t, 5> packet{0xc0031500u, 4, 1, 1, 0x41u};
+    for (const auto period : {1u, 2u, 4u}) {
+        const auto data = UserData(Output.data(), 1024, 256, period);
+        const auto listed = MatchDwordPatternFill(packet, Code, data, compute);
+        const auto swapped = MatchDwordPatternFill(packet, SwappedCode, data, compute);
+        Require(listed && swapped && swapped->base == listed->base && swapped->bytes == listed->bytes && swapped->pattern == listed->pattern, "the kernel with its two compares in the other order was not recognized as the same fill");
+    }
+    const auto data = UserData(Output.data(), 1024, 256, 4);
+    for (std::size_t word = 0; word < SwappedCode.size(); ++word) {
+        auto altered = SwappedCode;
+        altered[word] ^= 1u;
+        Require(!MatchDwordPatternFill(packet, altered, data, compute), "a different shader matched the dword pattern-fill kernel with swapped compares");
+    }
+    Require(!MatchDwordPatternFill(packet, std::span(SwappedCode).first(68), data, compute), "a truncated shader with swapped compares matched");
+    constexpr std::array<std::array<std::uint32_t, 3>, 5> otherOrders{{
+        {0x7d8c02f9u, 0x7d860609u, 0x06068c02u},
+        {0x06068c02u, 0x7d8c02f9u, 0x7d860609u},
+        {0x7d860609u, 0x06068c02u, 0x7d8c02f9u},
+        {0x7d860609u, 0x7d860609u, 0x7d8c02f9u},
+        {0x7d8c02f9u, 0x06068c02u, 0x7d8c02f9u},
+    }};
+    for (const auto& order : otherOrders) {
+        const auto code = WithCompares(order);
+        Require(!MatchDwordPatternFill(packet, code, data, compute), "a kernel whose compare words are not one of the two orders matched");
+    }
+}
+
 void MatchTests() {
     const ShaderRecompiler::ShaderComputeStageInfo compute{{64, 1, 1}, 0, {true, false, false}, false, 1};
     const std::array<std::uint32_t, 5> packet{0xc0031500u, 4, 1, 1, 0x41u};
@@ -74,7 +110,7 @@ void MatchTests() {
     Require(clamped && clamped->bytes == 256, "the fill exceeded the descriptor's records");
 }
 
-std::vector<std::uint32_t> Commands(const std::vector<std::uint32_t>& data, std::uint32_t groups, bool partial, std::uint32_t threads) {
+std::vector<std::uint32_t> Commands(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& data, std::uint32_t groups, bool partial, std::uint32_t threads) {
     std::vector<std::uint32_t> words;
     const auto registers = [&](std::uint32_t first, std::span<const std::uint32_t> values) {
         words.push_back(0xc0007600u | (static_cast<std::uint32_t>(values.size()) << 16u));
@@ -83,7 +119,7 @@ std::vector<std::uint32_t> Commands(const std::vector<std::uint32_t>& data, std:
     };
     const std::array<std::uint32_t, 3> shape{64, 1, 1};
     registers(0x207, shape);
-    const auto address = reinterpret_cast<std::uintptr_t>(Code.data());
+    const auto address = reinterpret_cast<std::uintptr_t>(code.data());
     const std::array<std::uint32_t, 2> program{static_cast<std::uint32_t>(address >> 8u), static_cast<std::uint32_t>(address >> 40u)};
     registers(0x20c, program);
     const std::array<std::uint32_t, 1> resource{(static_cast<std::uint32_t>(data.size()) << 1u) | (1u << 7u)};
@@ -111,18 +147,20 @@ void ExecuteTests() {
         {4, 1024, 256, 5, false, 0},
         {0, 1024, 256, 4, true, 70},
     }};
-    for (const auto& item : cases) {
-        Output.fill(Untouched);
-        const auto words = Commands(UserData(Output.data(), item.records, item.count, item.period), item.groups, item.partial, item.threads);
-        Packet packet{const_cast<std::uint32_t*>(words.data()), static_cast<std::uint32_t>(words.size()), 0, {}};
-        AgcDriver::Submit(&packet, 0);
-        AgcDriverWaitIdle_nid_postfix();
-        AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast<std::uintptr_t>(Output.data()), sizeof(Output));
-        const auto launched = item.partial ? item.threads : item.groups * 64u;
-        const auto filled = std::min({launched, item.records, item.count});
-        for (std::uint32_t i = 0; i < Output.size(); ++i) {
-            const auto expected = i < filled ? Pattern[std::min(i % item.period, 3u)] : Untouched;
-            Require(Output[i] == expected, "dword pattern fill mismatch at " + std::to_string(i) + " (period " + std::to_string(item.period) + ", count " + std::to_string(item.count) + ", records " + std::to_string(item.records) + "): got " + std::to_string(Output[i]) + ", expected " + std::to_string(expected));
+    for (const std::span<const std::uint32_t> code : {std::span<const std::uint32_t>(Code), std::span<const std::uint32_t>(SwappedCode)}) {
+        for (const auto& item : cases) {
+            Output.fill(Untouched);
+            const auto words = Commands(code, UserData(Output.data(), item.records, item.count, item.period), item.groups, item.partial, item.threads);
+            Packet packet{const_cast<std::uint32_t*>(words.data()), static_cast<std::uint32_t>(words.size()), 0, {}};
+            AgcDriver::Submit(&packet, 0);
+            AgcDriverWaitIdle_nid_postfix();
+            AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast<std::uintptr_t>(Output.data()), sizeof(Output));
+            const auto launched = item.partial ? item.threads : item.groups * 64u;
+            const auto filled = std::min({launched, item.records, item.count});
+            for (std::uint32_t i = 0; i < Output.size(); ++i) {
+                const auto expected = i < filled ? Pattern[std::min(i % item.period, 3u)] : Untouched;
+                Require(Output[i] == expected, "dword pattern fill mismatch at " + std::to_string(i) + " (" + (code.data() == SwappedCode.data() ? "swapped" : "listed") + " compares, period " + std::to_string(item.period) + ", count " + std::to_string(item.count) + ", records " + std::to_string(item.records) + "): got " + std::to_string(Output[i]) + ", expected " + std::to_string(expected));
+            }
         }
     }
 }
@@ -131,7 +169,10 @@ void ExecuteTests() {
 
 int main() {
     try {
-        if (std::getenv("APS5_NO_FILL_HLE") == nullptr) MatchTests();
+        if (std::getenv("APS5_NO_FILL_HLE") == nullptr) {
+            MatchTests();
+            SwappedComparesTests();
+        }
         {
             const auto device = OpenVulkanTestDevice();
             if (!device) return VulkanTestSkipped;

core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp

@@ -1,5 +1,10 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp"
+#include "Optimization/ResourceMaterializer.hpp"
+#include "Optimization/SrtWalker.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
@@ -16,6 +21,8 @@
 #endif
 #include <algorithm>
 #include <array>
+#include <atomic>
+#include <chrono>
 #include <cstddef>
 #include <cstdint>
 #include <cstdio>
@@ -23,7 +30,10 @@
 #include <cstring>
 #include <initializer_list>
 #include <iostream>
+#include <span>
 #include <string>
+#include <thread>
+#include <tuple>
 #include <vector>
 
 namespace {
@@ -47,6 +57,12 @@ constexpr std::size_t HeapOffset = OutputOffset + 4096;
 constexpr std::size_t TexturesOffset = HeapOffset + static_cast<std::size_t>(Records) * Stride;
 constexpr std::size_t BlockBytes = (TexturesOffset + static_cast<std::size_t>(Surfaces) * TextureBytes + 65535) & ~static_cast<std::size_t>(65535);
 
+constexpr std::uint32_t WideStride = 48;
+constexpr std::uint32_t WideRecords = 32768;
+constexpr std::size_t WideKernelOffset = 2048;
+constexpr std::size_t WideHeapOffset = 65536;
+constexpr std::array<std::size_t, 3> WideSlots{SrtSlots - 1u, SrtSlots - 2u, SrtSlots - 3u};
+constexpr std::array<std::uint32_t, 17> WideTableCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x9310b010u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u};
 constexpr std::array<std::uint32_t, 17> TableCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u};
 
 class GuestBlock {
@@ -97,10 +113,9 @@ std::vector<std::uint32_t> Pm4(std::uint32_t opcode, std::initializer_list<std::
     return words;
 }
 
-void Dispatch(const GuestBlock& block, std::size_t slot) {
-    const auto kernel = block.At(0);
+void Dispatch(const GuestBlock& block, std::size_t slot, std::size_t kernelOffset = 0, std::vector<std::uint32_t> words = {}) {
+    const auto kernel = block.At(kernelOffset);
     const auto srt = block.At(SrtOffset + slot * SrtBytes);
-    std::vector<std::uint32_t> words;
     for (const auto& packet : {Pm4(0x76, {0x20c, static_cast<std::uint32_t>(kernel >> 8u)}), Pm4(0x76, {0x20d, static_cast<std::uint32_t>(kernel >> 40u) & 0xffu}), Pm4(0x76, {0x207, 64}), Pm4(0x76, {0x208, 1}), Pm4(0x76, {0x209, 1}), Pm4(0x76, {0x213, 2u << 1u}), Pm4(0x76, {0x240, static_cast<std::uint32_t>(srt)}), Pm4(0x76, {0x241, static_cast<std::uint32_t>(srt >> 32u)}), Pm4(0x15, {1, 1, 1, 0x41})}) {
         words.insert(words.end(), packet.begin(), packet.end());
     }
@@ -150,6 +165,173 @@ void SrtMovingPerDispatch(const GuestBlock& block) {
     Require(most <= Driver::DispatchCacheBytes(), "image table cache bounds: dispatches reading a 2 MiB table through " + std::to_string(SrtSlots) + " SRTs kept " + std::to_string(most >> 20u) + " MiB of captured words in the dispatch cache, over its budget of " + std::to_string(Driver::DispatchCacheBytes() >> 20u) + " MiB");
 }
 
+std::array<std::uint32_t, 8> TextureWords(const GuestBlock& block, std::uint32_t element) {
+    const auto address = block.At(TexturesOffset + static_cast<std::size_t>(element / Aliases) * TextureBytes);
+    return {static_cast<std::uint32_t>(address >> 8u), static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (3u << 30u), 7u << 14u, 0xfacu | (Type2D << 28u), 0u, 0u, element % Aliases, 0u};
+}
+
+std::uint64_t WideRecord(const GuestBlock& block, std::uint32_t record) {
+    return block.At(WideHeapOffset + static_cast<std::size_t>(record) * WideStride);
+}
+
+void SetWideRecord(const GuestBlock& block, std::uint32_t record, std::uint32_t element) {
+    const auto words = TextureWords(block, element);
+    std::memcpy(block.Bytes(WideHeapOffset + static_cast<std::size_t>(record) * WideStride), words.data(), sizeof(words));
+}
+
+void VouchedLookups(const GuestBlock& block) {
+    using AgcDriver::ShaderMemory;
+    using ShaderRecompiler::ResourceMaterializer;
+    std::memcpy(block.Bytes(WideKernelOffset), WideTableCode.data(), sizeof(WideTableCode));
+    for (std::uint32_t record = 0; record < WideRecords; ++record) {
+        SetWideRecord(block, record, (1u + record % 15u) * Aliases);
+        std::memset(block.Bytes(WideHeapOffset + static_cast<std::size_t>(record) * WideStride + 32u), 0x5a, WideStride - 32u);
+    }
+    for (const auto slot : WideSlots) {
+        auto* srt = reinterpret_cast<std::uint32_t*>(block.Bytes(SrtOffset + slot * SrtBytes));
+        std::fill(srt, srt + SrtBytes / 4u, 0u);
+        const auto heap = V(block.At(WideHeapOffset), WideStride, WideRecords);
+        const auto output = V(block.At(OutputOffset), 0u, 4096u);
+        std::copy(heap.begin(), heap.end(), srt);
+        std::copy(output.begin(), output.end(), srt + 12);
+    }
+    struct Step {
+        std::uint64_t vouched = 0;
+        std::uint64_t refused = 0;
+        std::uint64_t columns = 0;
+        std::uint64_t reused = 0;
+    };
+    const auto dispatch = [&](std::size_t slot, std::vector<std::uint32_t> before = {}) {
+        const auto vouches = ShaderMemory::Vouches();
+        const auto reuse = ResourceMaterializer::TableReuse();
+        Dispatch(block, slot, WideKernelOffset, std::move(before));
+        const auto after = ShaderMemory::Vouches();
+        const auto reused = ResourceMaterializer::TableReuse();
+        return Step{after.blocks - vouches.blocks, after.refused - vouches.refused, reused.columns - reuse.columns, (reused.unwritten - reuse.unwritten) + (reused.compared - reuse.compared)};
+    };
+    const auto text = [](const Step& step) { return " (" + std::to_string(step.vouched) + " vouched, " + std::to_string(step.refused) + " refused, " + std::to_string(step.columns) + " columns read, " + std::to_string(step.reused) + " reused)"; };
+    const auto read = dispatch(WideSlots[0]);
+    Require(read.columns == 1u, "image table vouched lookups: the first dispatch did not read the strided table" + text(read));
+    const auto shared = dispatch(WideSlots[1]);
+    Require(shared.columns == 0u && shared.reused == 1u, "image table vouched lookups: a second dispatch over the unwritten table did not reuse its column" + text(shared));
+    const auto hit = dispatch(WideSlots[1]);
+    Require(hit.vouched >= 1u && hit.refused == 0u && hit.columns == 0u && hit.reused == 0u, "image table vouched lookups: a cached dispatch over an unwritten table was not validated by its table's vouch" + text(hit));
+
+    std::size_t vouchedSlot = WideSlots[1];
+    std::size_t otherSlot = WideSlots[0];
+    const auto recapture = [&](const std::string& what, std::vector<std::uint32_t> before = {}) {
+        const auto changed = dispatch(vouchedSlot, std::move(before));
+        Require(changed.refused >= 1u && changed.columns == 1u, "image table vouched lookups: " + what + " was hidden by the cached dispatch's vouch" + text(changed));
+        const auto other = dispatch(otherSlot);
+        Require(other.columns == 0u && other.reused == 1u, "image table vouched lookups: the other dispatch after " + what + " did not reuse the column read again" + text(other));
+        const auto again = dispatch(otherSlot);
+        Require(again.vouched >= 1u && again.refused == 0u && again.columns == 0u && again.reused == 0u, "image table vouched lookups: the dispatch after " + what + " was not vouched for again" + text(again));
+        std::swap(vouchedSlot, otherSlot);
+    };
+    SetWideRecord(block, 0u, 9u * Aliases);
+    recapture("a CPU store into a record");
+
+    const auto words = TextureWords(block, 10u * Aliases);
+    const auto target = WideRecord(block, 0u);
+    std::vector<std::uint32_t> cpWrite{0xc0000000u | ((3u + static_cast<std::uint32_t>(words.size()) - 1u) << 16u) | (0x37u << 8u), 0x00100200u, static_cast<std::uint32_t>(target), static_cast<std::uint32_t>(target >> 32u)};
+    cpWrite.insert(cpWrite.end(), words.begin(), words.end());
+    recapture("a GPU write into a record pending ahead of the dispatch", cpWrite);
+    Require(std::memcmp(block.Bytes(WideHeapOffset), words.data(), sizeof(words)) == 0, "image table vouched lookups: the GPU write did not land in the record");
+
+    const auto driverWords = TextureWords(block, 11u * Aliases);
+    AgcDriver::GuestMemory::Write(WideRecord(block, 0u), std::as_bytes(std::span(driverWords)), 4);
+    recapture("a driver store into a record");
+
+    *reinterpret_cast<std::uint32_t*>(block.Bytes(WideHeapOffset + 36u)) ^= 0x10u;
+    const auto gap = dispatch(vouchedSlot);
+    Require(gap.refused >= 1u && gap.columns == 0u && gap.reused == 0u, "image table vouched lookups: a store between two records did not refuse the vouch, or the table was captured again" + text(gap));
+    const auto compared = dispatch(WideSlots[2]);
+    Require(compared.columns == 0u && compared.reused == 1u, "image table vouched lookups: a new dispatch after a store between two records did not compare and keep the column" + text(compared));
+    const auto restamped = dispatch(vouchedSlot);
+    Require(restamped.vouched >= 1u && restamped.refused == 0u && restamped.columns == 0u, "image table vouched lookups: a dispatch sharing a column compared after a store between two records was not vouched for again" + text(restamped));
+    std::puts("image table vouched lookups passed");
+}
+
+struct QueuedDraw {
+    std::atomic<bool> release{false};
+
+    void Enqueue(std::uint64_t begin, std::uint64_t end) {
+        AgcDriver::DriverDetail::DrawPipeline::Queue0().Enqueue([this] {
+            while (!release.load()) std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }, {{begin, end}});
+    }
+};
+
+void VouchUnderQueuedDraws(const GuestBlock& block) {
+    using AgcDriver::DriverDetail::DrawPipeline;
+    using AgcDriver::DriverDetail::Driver;
+    Require(DrawPipeline::Depth() >= 3u, "image table vouch under queued draws: the draw pipeline is off");
+    for (std::uint32_t record = 0; record < WideRecords; ++record) SetWideRecord(block, record, (1u + record % 15u) * Aliases);
+    const auto low = WideRecord(block, 0u);
+    const auto bytes = static_cast<std::size_t>(WideRecord(block, WideRecords - 1u) + 32u - low);
+    static_cast<void>(Driver::Get());
+    const ShaderRecompiler::SrtWriteStamp stamp{AgcDriver::GuestMemory::CollectWrites(low, bytes), GuestAllocations::GuestAllocationsGeneration_nid_postfix()};
+    Require(stamp.generation != 0u, "image table vouch under queued draws: the table is not write-watched");
+    auto& pipeline = DrawPipeline::Queue0();
+    QueuedDraw elsewhere;
+    QueuedDraw between;
+    QueuedDraw into;
+    const auto releaseAll = [&] {
+        elsewhere.release = true;
+        between.release = true;
+        into.release = true;
+    };
+    DrawPipeline::Active() = true;
+    const auto vouch = [&](const std::string& what, bool expected) {
+        const auto outstanding = pipeline.Outstanding();
+        std::atomic<bool> finished{false};
+        std::atomic<bool> waited{false};
+        std::thread watchdog([&] {
+            const auto start = std::chrono::steady_clock::now();
+            while (!finished.load()) {
+                if (std::chrono::steady_clock::now() - start > std::chrono::seconds(5)) {
+                    waited = true;
+                    releaseAll();
+                    return;
+                }
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
+        });
+        const bool vouched = AgcDriver::ShaderMemory::Vouched(&Driver::QueryPendingSpan, low, bytes, stamp);
+        finished = true;
+        watchdog.join();
+        const bool queued = !waited.load() && pipeline.Outstanding() == outstanding;
+        if (!queued || vouched != expected) releaseAll();
+        Require(queued, "image table vouch under queued draws: " + what + ": the vouch waited for the queued draws");
+        Require(vouched == expected, "image table vouch under queued draws: " + what + (expected ? ": the table was not vouched for" : ": the table was still vouched for"));
+    };
+    const auto settle = [&](std::size_t outstanding) {
+        const auto start = std::chrono::steady_clock::now();
+        while (pipeline.Outstanding() != outstanding) {
+            if (std::chrono::steady_clock::now() - start > std::chrono::seconds(20)) {
+                releaseAll();
+                Require(false, "image table vouch under queued draws: the released draws were never committed");
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+    };
+    vouch("an idle pipeline", true);
+    elsewhere.Enqueue(block.At(OutputOffset), block.At(OutputOffset) + 4096u);
+    vouch("a queued draw writing outside the table", true);
+    between.Enqueue(low + 32u, low + WideStride);
+    vouch("a queued draw writing between two records", false);
+    into.Enqueue(WideRecord(block, 7u), WideRecord(block, 7u) + 32u);
+    elsewhere.release = true;
+    between.release = true;
+    settle(1u);
+    vouch("a queued draw writing into a record", false);
+    into.release = true;
+    pipeline.Drain(DrawPipeline::DrainReason::Capture);
+    vouch("committed draws that wrote nothing", true);
+    DrawPipeline::Active() = false;
+    std::puts("image table vouch under queued draws passed");
+}
+
 void SetBudgets() {
 #ifdef _WIN32
     _putenv_s("APS5_RESOURCE_CACHE_TABLE_ELEMENTS", "8192");
@@ -162,7 +344,7 @@ void SetBudgets() {
 
 }
 
-int main() {
+int main(int argc, char** argv) {
     SetBudgets();
     try {
         {
@@ -171,6 +353,16 @@ int main() {
         }
         GuestBlock block;
         Prepare(block);
+        if (argc == 2 && std::string(argv[1]) == "--vouched-lookups") {
+            VouchedLookups(block);
+            AgcDriverShutdown_nid_postfix();
+            return 0;
+        }
+        if (argc == 2 && std::string(argv[1]) == "--queued-draws") {
+            VouchUnderQueuedDraws(block);
+            AgcDriverShutdown_nid_postfix();
+            return 0;
+        }
         TablesChangingPerDispatch(block);
         SrtMovingPerDispatch(block);
         AgcDriverShutdown_nid_postfix();

core/libs/prx/libSceAgcDriver/tests/execution/ImageTableReuse.cpp

@@ -19,6 +19,7 @@
 #include <array>
 #include <bit>
 #include <cstdint>
+#include <cstdlib>
 #include <cstdio>
 #include <cstring>
 #include <iostream>
@@ -44,11 +45,7 @@ constexpr std::uint32_t Type2D = 9;
 constexpr std::size_t TexturesOffset = 65536;
 constexpr std::size_t BlockBytes = TexturesOffset + 2 * 65536;
 constexpr std::size_t UnimportableBytes = BlockBytes - 256;
-#ifdef _WIN32
-constexpr std::uint64_t CpuWriteResolutions = 2;
-#else
 constexpr std::uint64_t CpuWriteResolutions = 1;
-#endif
 
 alignas(256) std::array<std::uint32_t, Records> Keys{};
 alignas(256) std::array<std::uint32_t, Lanes * 4> Output{};
@@ -207,6 +204,18 @@ Step Since(const AgcDriver::Graphics::ImageTableReuseCounts& before) {
     return {now.reused - before.reused, now.resolved - before.resolved};
 }
 
+void ExpectWrites(const AgcDriver::Graphics::ImageTableReuseCounts& before, std::uint64_t kept, std::uint64_t made, const std::string& what) {
+    static const bool keptSets = std::getenv("APS5_KEPT_DESCRIPTOR_SETS") == nullptr || std::string(std::getenv("APS5_KEPT_DESCRIPTOR_SETS")) != "0";
+    const auto now = AgcDriver::Graphics::ImageTableReuses();
+    const auto keptNow = now.writesKept - before.writesKept;
+    const auto madeNow = now.writesMade - before.writesMade;
+    if (!keptSets) {
+        Require(keptNow == 0u && madeNow == kept + made, "image table reuse: " + what + ": table descriptors were kept with kept sets turned off");
+        return;
+    }
+    Require(keptNow == kept && madeNow == made, "image table reuse: " + what + ": " + std::to_string(keptNow) + " table descriptor writes kept and " + std::to_string(madeNow) + " made, expected " + std::to_string(kept) + " and " + std::to_string(made));
+}
+
 void Expect(const AgcDriver::Graphics::ImageTableReuseCounts& before, std::uint64_t reused, std::uint64_t resolved, const std::string& what) {
     const auto step = Since(before);
     Require(step.reused == reused && step.resolved == resolved, "image table reuse: " + what + ": " + std::to_string(step.reused) + " reused and " + std::to_string(step.resolved) + " resolved tables, expected " + std::to_string(reused) + " and " + std::to_string(resolved));
@@ -271,12 +280,15 @@ void RunTests(AgcDriver::VulkanDevice& device, const GuestBlock& block, const Gu
         return;
     }
     Expect(before, 2, 1, "three dispatches of one table in one epoch");
+    if (std::getenv("APS5_NO_TABLE_REPLAY") == nullptr) Require(AgcDriver::Graphics::ImageTableReuses().replayed - before.replayed == 2u, "image table reuse: three dispatches of one table in one epoch did not replay the bound table twice");
+    ExpectWrites(before, 3, 0, "three dispatches of one table in one epoch");
 
     before = AgcDriver::Graphics::ImageTableReuses();
     AgcDriver::Graphics::ClearCachedTextures(device.Device());
     Require(Dispatch(device, result, 7) == Row(block.Texels(7)), "image table reuse: the table after the texture cache was cleared sampled the wrong entry");
     Require(Dispatch(device, result, 9) == Row(block.Texels(9)), "image table reuse: the table resolved after the texture cache was cleared sampled the wrong entry");
     Expect(before, 1, 1, "a kept table whose textures the texture cache dropped");
+    ExpectWrites(before, 1, 1, "a kept table whose textures the texture cache dropped");
 
     before = AgcDriver::Graphics::ImageTableReuses();
     std::vector<float> stored(Width * Height);
@@ -303,6 +315,7 @@ void RunTests(AgcDriver::VulkanDevice& device, const GuestBlock& block, const Gu
     Require(Dispatch(device, moved, 19) == Row(block.Texels(Moved)), "image table reuse: the moved table sampled the wrong entry when reused");
     Require(Dispatch(device, result, 21) == Row(block.Texels(21)), "image table reuse: the original table sampled the wrong entry when reused");
     Expect(before, 2, 2, "two tables that differ in one record");
+    ExpectWrites(before, 0, 4, "two tables that differ in one record, alternating");
 
     SetOutput(block.Texels(13));
     const auto writer = Compile(device, block);

core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp

@@ -5,8 +5,11 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
-#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp"
+#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/AsyncLog.hpp"
 #include "../NarrowConstantStoreFixture.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Acb.hpp"
@@ -36,6 +39,9 @@
 #define NOMINMAX
 #endif
 #include <windows.h>
+#include <io.h>
+#else
+#include <unistd.h>
 #endif
 
 namespace {
@@ -720,7 +726,12 @@ void LazyRegistration() {
     header.registers[0].value = static_cast<std::uint32_t>(invalidAddress >> 8u);
     header.registers[1].value = static_cast<std::uint32_t>(invalidAddress >> 40u);
     AgcDriverRegisterShader_nid_postfix(&header.shader);
-    ExpectFailure([&] { AgcDriverResolveShaderAbi_nid_postfix(&header.shader, {}, {}); }, "s_code_end");
+    AgcDriverResolveShaderAbi_nid_postfix(&header.shader, {}, {});
+    {
+        AgcDriver::DriverDetail::ShaderPreparationTransaction transaction;
+        const auto& state = transaction.Read(*registered());
+        Require(state.deferred && state.entries.empty(), "the pipeline ABI of a lazily registered shader it cannot prepare did not leave it to its dispatch");
+    }
     commands[2] = header.registers[0].value;
     commands[5] = header.registers[1].value;
     sceAgcDriverSubmitAcb(0x20, &packet);
@@ -784,11 +795,429 @@ void DeferredUndecodableRegistration() {
     ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "unsupported MIMG opcode");
 }
 
+class StderrCapture {
+public:
+    StderrCapture() {
+        AgcDriver::FlushAsyncLog_nid_no_patch();
+        std::fflush(stderr);
+        file = std::tmpfile();
+        Require(file != nullptr, "cannot open a temporary file for stderr");
+#ifdef _WIN32
+        saved = _dup(_fileno(stderr));
+        _dup2(_fileno(file), _fileno(stderr));
+#else
+        saved = dup(fileno(stderr));
+        dup2(fileno(file), fileno(stderr));
+#endif
+    }
+    std::string Finish() {
+        AgcDriver::FlushAsyncLog_nid_no_patch();
+        std::fflush(stderr);
+#ifdef _WIN32
+        _dup2(saved, _fileno(stderr));
+        _close(saved);
+#else
+        dup2(saved, fileno(stderr));
+        close(saved);
+#endif
+        std::rewind(file);
+        std::string text;
+        char buffer[4096];
+        for (std::size_t read = 0; (read = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, read);
+        std::fclose(file);
+        std::fwrite(text.data(), 1, text.size(), stderr);
+        return text;
+    }
+
+private:
+    std::FILE* file = nullptr;
+    int saved = -1;
+};
+
+template<typename TAction>
+std::string Logged(TAction action) {
+    StderrCapture capture;
+    try {
+        action();
+    } catch (...) {
+        static_cast<void>(capture.Finish());
+        throw;
+    }
+    return capture.Finish();
+}
+
+std::size_t Deferrals(const std::string& log) {
+    constexpr std::string_view line = "could not be prepared when the title created its pipeline";
+    std::size_t count = 0;
+    for (auto found = log.find(line); found != std::string::npos; found = log.find(line, found + line.size())) ++count;
+    return count;
+}
+
+struct AbiHeader {
+    Shader shader{};
+    std::array<ShaderRegister, 8> registers{};
+    std::array<ShaderRegister, 8> context{};
+    ShaderSpecialRegs specials{};
+    ShaderUserData users{};
+};
+
+std::uint32_t ProgramLow(const void* code) {
+    return static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(code) >> 8u);
+}
+
+std::uint32_t ProgramHigh(const void* code) {
+    return static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(code) >> 40u);
+}
+
+template<std::size_t TCount>
+void DescribeAbi(AbiHeader& header, const std::array<std::uint32_t, TCount>& code, std::uint8_t type, std::initializer_list<ShaderRegister> registers, std::initializer_list<ShaderRegister> context) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18;
+    header.shader.header_size = sizeof(AbiHeader);
+    header.shader.shader_size = sizeof(code);
+    header.shader.code = code.data();
+    header.shader.type = type;
+    header.shader.user_data = &header.users;
+    std::copy(registers.begin(), registers.end(), header.registers.begin());
+    header.shader.sh_registers = header.registers.data();
+    header.shader.num_sh_registers = static_cast<std::uint8_t>(registers.size());
+    std::copy(context.begin(), context.end(), header.context.begin());
+    header.shader.cx_registers = header.context.data();
+    header.shader.num_cx_registers = static_cast<std::uint8_t>(context.size());
+}
+
+alignas(256) constexpr std::array<std::uint32_t, 64> UserDataCaller{
+    0x7e020280u, 0xbe940383u, 0xbe8e0300u, 0xbe8f0301u, 0xbe8e210eu, 0xbe8e0302u, 0xbe8f0303u, 0xbe8e210eu,
+    0x80948114u, 0xbf078014u, 0xbf85fff7u, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u, 0xbf9f0000u,
+    0x30306c73u, 0x00000040u, 0x00000008u};
+alignas(256) constexpr std::array<std::uint32_t, 2> AddFiveCallee{0x4a020285u, 0xbe80200eu};
+alignas(256) constexpr std::array<std::uint32_t, 2> AddSixtyFourCallee{0x4a0202c0u, 0xbe80200eu};
+alignas(256) std::array<std::uint32_t, 64> CallerOutput{};
+alignas(256) constexpr std::array<std::uint32_t, 64> WaterfallStoreCode{
+    0x7e020280u, 0x7e040280u, 0x7e080280u, 0x7e0a0280u, 0x7e0c0280u, 0x7e0e0280u, 0xbe86047eu, 0xf40c0200u,
+    0xfa000000u, 0xd4c10000u, 0x00020083u, 0xbf8cc07fu, 0xf0200f08u, 0x00020401u, 0x8afe007eu, 0xbf89fff7u,
+    0xbefe0406u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 64> WaterfallPixelCode{
+    0xc80a0002u, 0x7e000280u, 0x7e020280u, 0xbe86047eu, 0xf40c0200u, 0xfa000000u, 0xd4c10000u, 0x00020483u,
+    0xbf8cc07fu, 0xf0000f08u, 0x00020400u, 0x8afe007eu, 0xbf89fff7u, 0xbefe0406u, 0xbf8c3f70u, 0xf800180fu,
+    0x07060504u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 64> WaterfallVertexCode{
+    0x7e020280u, 0x7e040280u, 0xbe86037eu, 0xf40c0404u, 0xfa000000u, 0xd4c10008u, 0x00020083u, 0xbf8cc07fu,
+    0xf0000f08u, 0x00040401u, 0x8a7e087eu, 0xbf89fff7u, 0xbefe0306u, 0xbf8a0000u, 0xbf8c3f70u, 0xf80008cfu,
+    0x07060504u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 64> OtherWaterfallVertexCode = WaterfallVertexCode;
+alignas(256) constexpr std::array<std::uint32_t, 64> BarrierFreeWaterfallVertexCode{
+    0x7e020280u, 0x7e040280u, 0xbe86037eu, 0xf40c0404u, 0xfa000000u, 0xd4c10008u, 0x00020083u, 0xbf8cc07fu,
+    0xf0000f08u, 0x00040401u, 0x8a7e087eu, 0xbf89fff7u, 0xbefe0306u, 0xbf800000u, 0xbf8c3f70u, 0xf80008cfu,
+    0x07060504u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 11> FrontCode{0x7e0002ffu, 0, 0x7e0202ffu, 0, 0x7e0402ffu, 0, 0x7e0602ffu, 0x3f800000u, 0xf80008cfu, 0x03020100u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 64> UndecodablePixelCode{0xf1949f05u, 0x00060018u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 64> PassthroughPixelCode{0xc8020002u, 0xf800180fu, 0x00000000u, 0xbf810000u};
+
+void DescribeWaterfallPixel(AbiHeader& pixel) {
+    DescribeAbi(pixel, WaterfallPixelCode, 1, {{0x8, ProgramLow(WaterfallPixelCode.data())}, {0x9, ProgramHigh(WaterfallPixelCode.data())}, {0xa, 0}, {0xb, 2u << 1u}}, {{0x1b6, 1}, {0x1b3, 2}, {0x1b4, 2}, {0x191, 0}, {0x1c5, 4}});
+}
+
+void DescribeFront(AbiHeader& vertex) {
+    DescribeAbi(vertex, FrontCode, 2, {{0xc8, ProgramLow(FrontCode.data())}, {0xc9, ProgramHigh(FrontCode.data())}, {0x8b, 0}, {0x8a, 0}, {0xca, 0}, {0xcb, 0}, {0xcc, 0}}, {{0x2d5, 0x2000}});
+    vertex.shader.specials = &vertex.specials;
+    vertex.specials.dispatch_modifier = 0x8000;
+}
+
+void DescribeWaterfallVertex(AbiHeader& vertex, const std::array<std::uint32_t, 64>& code) {
+    DescribeAbi(vertex, code, 2, {{0xc8, ProgramLow(code.data())}, {0xc9, ProgramHigh(code.data())}, {0x8a, 0}, {0x8b, 2u << 1u}}, {{0x1ff, 64}, {0x2ab, 4}, {0x2ce, 0}});
+    vertex.shader.specials = &vertex.specials;
+    vertex.specials = {{0x25b, 0x8040}, {0x2d5, 0x00402000}, 0, {}, {}, {0x29b, 2}, {0x260, 0}};
+}
+
+struct FetchingHeader {
+    AbiHeader abi;
+    std::array<std::uint16_t, 11> directOffsets{};
+    ShaderSemantic semantic{};
+};
+
+void DescribeStaticFetch(FetchingHeader& vertex) {
+    vertex.abi.shader.header_size = sizeof(FetchingHeader);
+    for (auto& reg : vertex.abi.registers) {
+        if (reg.offset == 0x8b) reg.value = 4u << 1u;
+    }
+    vertex.directOffsets.fill(0xffffu);
+    vertex.directOffsets[8] = 0;
+    vertex.directOffsets[10] = 2;
+    vertex.abi.users.direct_resource_offset = vertex.directOffsets.data();
+    vertex.abi.users.direct_resource_count = static_cast<std::uint16_t>(vertex.directOffsets.size());
+    vertex.semantic.size_in_elements = 4;
+    vertex.semantic.static_vb_index = 1;
+    vertex.abi.shader.input_semantics = &vertex.semantic;
+    vertex.abi.shader.num_input_semantics = 1;
+}
+
+void AbiDeferredCompute() {
+    static AbiHeader caller;
+    DescribeAbi(caller, UserDataCaller, 0, {{0x20c, ProgramLow(UserDataCaller.data())}, {0x20d, ProgramHigh(UserDataCaller.data())}, {0x207, 64}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 8u << 1u}}, {});
+    caller.shader.specials = &caller.specials;
+    caller.specials.dispatch_modifier = 0x8000u;
+    AgcDriverRegisterShader_nid_postfix(&caller.shader);
+    const auto before = ShaderRecompiler::FrontEndBuilds();
+    AgcDriverResolveShaderAbi_nid_postfix(&caller.shader, {}, {});
+    const auto attempted = ShaderRecompiler::FrontEndBuilds();
+    AgcDriverResolveShaderAbi_nid_postfix(&caller.shader, {}, {});
+    Require(attempted > before && ShaderRecompiler::FrontEndBuilds() == attempted, "a compute static ABI deferred to its dispatch was prepared again for the same state");
+    const auto first = reinterpret_cast<std::uintptr_t>(AddFiveCallee.data());
+    const auto second = reinterpret_cast<std::uintptr_t>(AddSixtyFourCallee.data());
+    const auto output = reinterpret_cast<std::uintptr_t>(CallerOutput.data());
+    const std::array<std::uint32_t, 8> users{static_cast<std::uint32_t>(first), static_cast<std::uint32_t>(first >> 32u), static_cast<std::uint32_t>(second), static_cast<std::uint32_t>(second >> 32u),
+        static_cast<std::uint32_t>(output), static_cast<std::uint32_t>((output >> 32u) & 0xffffu), static_cast<std::uint32_t>(sizeof(CallerOutput)), 0x31016facu};
+    std::vector<std::uint32_t> commands;
+    for (std::size_t index = 0; index < caller.shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, caller.registers[index].offset, caller.registers[index].value});
+    for (std::uint32_t index = 0; index < users.size(); ++index) commands.insert(commands.end(), {0xc0017600u, 0x240u + index, users[index]});
+    commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x8041u});
+    CallerOutput.fill(0xcafebabeu);
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    sceAgcDriverSubmitAcb(0x20, &packet);
+    AgcDriverWaitIdle_nid_postfix();
+    AgcDriver::GuestMemory::FlushGpuWrites(output, sizeof(CallerOutput));
+    Require(std::ranges::all_of(CallerOutput, [](std::uint32_t lane) { return lane == 3u * (5u + 64u); }), "a compute shader its static ABI deferred did not run its user-data callees at its dispatch");
+    AgcDriverShutdown_nid_postfix();
+}
+
+void AbiDeferredRegistrationFailure() {
+    static AbiHeader store;
+    DescribeAbi(store, WaterfallStoreCode, 0, {{0x20c, ProgramLow(WaterfallStoreCode.data())}, {0x20d, ProgramHigh(WaterfallStoreCode.data())}, {0x207, 64}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 2u << 1u}}, {});
+    store.shader.specials = &store.specials;
+    static AbiHeader front;
+    DescribeFront(front);
+    static AbiHeader pixel;
+    DescribeWaterfallPixel(pixel);
+    pixel.context[0] = {0x1b6, 0};
+    static AbiHeader vertex;
+    DescribeWaterfallVertex(vertex, BarrierFreeWaterfallVertexCode);
+    static AbiHeader interpolated;
+    DescribeWaterfallPixel(interpolated);
+    for (auto* header : {&store, &front, &pixel, &vertex, &interpolated}) AgcDriverRegisterShader_nid_postfix(&header->shader);
+    const std::array<ShaderRegister, 1> primitive{{{0x242, 4}}};
+    const std::array<const Shader*, 2> frontStages{&front.shader, &pixel.shader};
+    const std::array<const Shader*, 2> vertexStages{&vertex.shader, &interpolated.shader};
+    const auto resolve = [&] {
+        AgcDriver::ShaderPreparationScope scope({&front.shader, &pixel.shader, &vertex.shader});
+        AgcDriverResolveShaderAbi_nid_postfix(&store.shader, {}, {});
+        AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {});
+        AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &pixel.shader, 17);
+        AgcDriverResolveGraphicsStagesAbi_nid_postfix(frontStages, {}, primitive);
+        AgcDriverResolveGraphicsStagesAbi_nid_postfix(vertexStages, {}, primitive);
+        AgcDriverResolveGraphicsAbi_nid_postfix(&vertex.shader, &interpolated.shader, 17);
+        scope.Commit();
+    };
+    auto log = Logged(resolve);
+    Require(Deferrals(log) == 3 && log.find("is not a valid runtime value") != std::string::npos, "stages whose registration preparation failed were not deferred once each, with the stored reason, when the title created their pipelines");
+    log = Logged(resolve);
+    Require(Deferrals(log) == 0, "stages whose registration preparation failed were logged again for the same pipelines");
+    std::vector<std::uint32_t> commands;
+    for (std::size_t index = 0; index < store.shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, store.registers[index].offset, store.registers[index].value});
+    commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x8041u});
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    sceAgcDriverSubmitAcb(0x20, &packet);
+    ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "is not a valid runtime value");
+    ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "is not a valid runtime value");
+}
+
+void AbiDeferredPixel() {
+    static AbiHeader pixel;
+    DescribeWaterfallPixel(pixel);
+    AgcDriverRegisterShader_nid_postfix(&pixel.shader);
+    const auto before = ShaderRecompiler::FrontEndBuilds();
+    auto log = Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {}); });
+    const auto attempted = ShaderRecompiler::FrontEndBuilds();
+    log += Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {}); });
+    Require(attempted > before && ShaderRecompiler::FrontEndBuilds() == attempted, "a pixel static ABI deferred to its draw was prepared again for the same state");
+    Require(Deferrals(log) == 1 && log.find("is not a valid runtime value") != std::string::npos, "a pixel shader its static ABI cannot prepare was not logged once with its reason");
+    {
+        AgcDriver::ShaderPreparationScope scope({&pixel.shader});
+        const std::array<ShaderRegister, 1> interpolant{{{0x191, 0x20}}};
+        log = Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, interpolant, {}); });
+        scope.Commit();
+    }
+    Require(ShaderRecompiler::FrontEndBuilds() > attempted && Deferrals(log) == 0, "a deferred pixel shader was not prepared once more for another state, or was logged again");
+    static AbiHeader front;
+    DescribeFront(front);
+    AgcDriverRegisterShader_nid_postfix(&front.shader);
+    std::vector<std::uint32_t> commands;
+    for (const auto* header : {&front, &pixel}) {
+        for (std::size_t index = 0; index < header->shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, header->registers[index].offset, header->registers[index].value});
+        for (std::size_t index = 0; index < header->shader.num_cx_registers; ++index) commands.insert(commands.end(), {0xc0016900u, header->context[index].offset, header->context[index].value});
+    }
+    commands.insert(commands.end(), {0xc0016900u, 0x8e, 0xfu, 0xc0016900u, 0x8f, 0xfu, 0xc0016900u, 0xd, 0x00100010u, 0xc0016900u, 0x204, 0x80000u, 0xc0016900u, 0x1c3, 4, 0xc0017900u, 0x242, 4, 0xc0012d00u, 3, 2});
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw of a deferred pixel shader was not submitted");
+    ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "is not a valid runtime value");
+    ExpectFailure([&] { AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {}); }, "is not a valid runtime value");
+    ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "is not a valid runtime value");
+}
+
+void AbiDeferredStages() {
+    static AbiHeader nested;
+    static AbiHeader direct;
+    DescribeWaterfallVertex(nested, WaterfallVertexCode);
+    DescribeWaterfallVertex(direct, OtherWaterfallVertexCode);
+    AgcDriverRegisterShader_nid_postfix(&nested.shader);
+    AgcDriverRegisterShader_nid_postfix(&direct.shader);
+    const std::array<ShaderRegister, 1> primitive{{{0x242, 4}}};
+    const auto before = ShaderRecompiler::FrontEndBuilds();
+    std::string log;
+    {
+        AgcDriver::ShaderPreparationScope scope({&nested.shader});
+        log = Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&nested.shader, {}, primitive); });
+        scope.Commit();
+    }
+    const auto nestedAttempt = ShaderRecompiler::FrontEndBuilds();
+    Require(nestedAttempt > before && Deferrals(log) == 1, "a vertex shader the static ABI resolves through its graphics stages was not deferred to its draw");
+    log = Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&nested.shader, {}, primitive); });
+    Require(ShaderRecompiler::FrontEndBuilds() == nestedAttempt && Deferrals(log) == 0, "a deferred vertex static ABI was prepared or logged again for the same state");
+    const std::array<const Shader*, 1> stages{&direct.shader};
+    log = Logged([&] { AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, {}, primitive); });
+    const auto directAttempt = ShaderRecompiler::FrontEndBuilds();
+    Require(directAttempt > nestedAttempt && Deferrals(log) == 1, "a graphics stage its ABI cannot prepare was not deferred to its draw");
+    log = Logged([&] { AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, {}, primitive); });
+    Require(ShaderRecompiler::FrontEndBuilds() == directAttempt && Deferrals(log) == 0, "a deferred graphics stage ABI was prepared or logged again for the same state");
+    AgcDriverShutdown_nid_postfix();
+}
+
+void AbiDeferredRectangle() {
+    static AbiHeader front;
+    static AbiHeader undecodable;
+    static AbiHeader pixel;
+    DescribeFront(front);
+    DescribeAbi(undecodable, UndecodablePixelCode, 1, {{0x8, ProgramLow(UndecodablePixelCode.data())}, {0x9, ProgramHigh(UndecodablePixelCode.data())}, {0xa, 0}, {0xb, 0}}, {});
+    DescribeWaterfallPixel(pixel);
+    for (auto* header : {&front, &undecodable, &pixel}) AgcDriverRegisterShader_nid_postfix(&header->shader);
+    AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &undecodable.shader, 17);
+    AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {});
+    {
+        AgcDriver::ShaderPreparationScope scope({&front.shader, &pixel.shader});
+        AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &pixel.shader, 17);
+        scope.Commit();
+    }
+    AgcDriverShutdown_nid_postfix();
+}
+
+void AbiDeferredUndecodableState() {
+    static AbiHeader front;
+    DescribeFront(front);
+    front.context[0] = {0x2ce, 3};
+    front.specials = {{0x25b, 0x8040}, {0x2d5, 0x2030}, 0x8000, {}, {}, {0x29b, 1}, {0x262, 0}};
+    AgcDriverRegisterShader_nid_postfix(&front.shader);
+    const std::array<const Shader*, 1> stages{&front.shader};
+    const std::array<ShaderRegister, 2> context{front.specials.vgt_shader_stages_en, front.specials.vgt_gs_out_prim_type};
+    const std::array<ShaderRegister, 3> primitive{front.specials.ge_cntl, front.specials.ge_user_vgpr_en, ShaderRegister{0x242, 4}};
+    const auto before = ShaderRecompiler::FrontEndBuilds();
+    auto log = Logged([&] { AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, context, primitive); });
+    Require(Deferrals(log) == 1 && log.find("VGT_SHADER_STAGES_EN=0x2030: ") != std::string::npos, "a graphics stage whose pipeline state does not decode was not deferred with the decode's reason");
+    log = Logged([&] { AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, context, primitive); });
+    Require(ShaderRecompiler::FrontEndBuilds() == before && Deferrals(log) == 0, "a graphics stage deferred for a pipeline state that does not decode was prepared or logged again");
+    std::vector<std::uint32_t> commands;
+    for (std::size_t index = 0; index < front.shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, front.registers[index].offset, front.registers[index].value});
+    commands.insert(commands.end(), {0xc0016900u, 0x2d5, 0x2000u, 0xc0017600u, 0x8, 0, 0xc0016900u, 0xd, 0x00100010u, 0xc0016900u, 0x204, 0x80000u, 0xc0016900u, 0x1c3, 4, 0xc0017900u, 0x242, 4, 0xc0012d00u, 3, 2});
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    log = Logged([&] {
+        Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw of a graphics stage deferred at its pipeline ABI was not submitted");
+        AgcDriverWaitIdle_nid_postfix();
+    });
+    Require(ShaderRecompiler::FrontEndBuilds() > before && log.find("has no artifact prepared at registration") == std::string::npos, "a graphics stage deferred at its pipeline ABI was not prepared at its first draw, or was reported there as missing a registration artifact");
+    AgcDriverShutdown_nid_postfix();
+}
+
+void AbiDeferredVertexInputs() {
+    static FetchingHeader fetching;
+    DescribeWaterfallVertex(fetching.abi, WaterfallVertexCode);
+    DescribeStaticFetch(fetching);
+    AgcDriverRegisterShader_nid_postfix(&fetching.abi.shader);
+    const std::array<const Shader*, 1> stages{&fetching.abi.shader};
+    const std::array<ShaderRegister, 1> primitive{{{0x242, 4}}};
+    const auto resolve = [&] {
+        AgcDriver::ShaderPreparationScope scope({&fetching.abi.shader});
+        AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, {}, primitive);
+        scope.Commit();
+    };
+    const auto before = ShaderRecompiler::FrontEndBuilds();
+    auto log = Logged(resolve);
+    Require(Deferrals(log) == 1 && log.find("statically bound vertex attributes are not implemented") != std::string::npos, "a graphics stage whose vertex inputs its pipeline ABI cannot decode was not deferred with the decode's reason");
+    log = Logged(resolve);
+    Require(ShaderRecompiler::FrontEndBuilds() == before && Deferrals(log) == 0, "a graphics stage deferred for vertex inputs that do not decode was prepared or logged again");
+    alignas(16) static std::array<std::uint32_t, 16> tables{};
+    const auto address = reinterpret_cast<std::uintptr_t>(tables.data());
+    std::vector<std::uint32_t> commands;
+    for (std::size_t index = 0; index < fetching.abi.shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, fetching.abi.registers[index].offset, fetching.abi.registers[index].value});
+    for (std::uint32_t index = 0; index < 4; ++index) commands.insert(commands.end(), {0xc0017600u, 0x8cu + index, static_cast<std::uint32_t>(index % 2u == 0u ? address : address >> 32u)});
+    for (std::size_t index = 0; index < fetching.abi.shader.num_cx_registers; ++index) commands.insert(commands.end(), {0xc0016900u, fetching.abi.context[index].offset, fetching.abi.context[index].value});
+    for (const auto reg : {fetching.abi.specials.vgt_shader_stages_en, fetching.abi.specials.vgt_gs_out_prim_type}) commands.insert(commands.end(), {0xc0016900u, reg.offset, reg.value});
+    for (const auto reg : {fetching.abi.specials.ge_cntl, fetching.abi.specials.ge_user_vgpr_en}) commands.insert(commands.end(), {0xc0017900u, reg.offset, reg.value});
+    commands.insert(commands.end(), {0xc0017600u, 0x8, 0, 0xc0016900u, 0xd, 0x00100010u, 0xc0016900u, 0x204, 0x80000u, 0xc0016900u, 0x1c3, 4, 0xc0017900u, 0x242, 4, 0xc0012d00u, 3, 2});
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw of a graphics stage deferred for its vertex inputs was not submitted");
+    ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "statically bound vertex attributes are not implemented");
+    ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "statically bound vertex attributes are not implemented");
+}
+
+void AbiDeferredRectanglePairs() {
+    static AbiHeader front;
+    DescribeFront(front);
+    static AbiHeader pixel;
+    DescribeAbi(pixel, PassthroughPixelCode, 1, {{0x8, ProgramLow(PassthroughPixelCode.data())}, {0x9, ProgramHigh(PassthroughPixelCode.data())}, {0xa, 0}, {0xb, 0}}, {{0x1b6, 1}, {0x1b3, 2}, {0x1b4, 2}, {0x191, 0x420}, {0x1c5, 4}});
+    static AbiHeader unresolved;
+    DescribeWaterfallPixel(unresolved);
+    for (auto* header : {&front, &pixel, &unresolved}) AgcDriverRegisterShader_nid_postfix(&header->shader);
+    const std::array<ShaderRegister, 1> passthrough{pixel.context[3]};
+    const auto resolve = [&] {
+        AgcDriver::ShaderPreparationScope scope({&front.shader, &pixel.shader});
+        AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, nullptr, 17);
+        AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &pixel.shader, 17);
+        AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &unresolved.shader, 17);
+        AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, passthrough, {});
+        AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &pixel.shader, 17);
+        scope.Commit();
+    };
+    auto log = Logged(resolve);
+    Require(Deferrals(log) == 1 && log.find("custom per-vertex interpolation is unsupported") != std::string::npos, "a rectangle pair its pipeline ABI cannot build was not deferred once with the builder's reason");
+    log = Logged(resolve);
+    Require(Deferrals(log) == 0, "a rectangle pair deferred at its pipeline ABI was logged again");
+    const auto draw = [&](bool withPixel) {
+        std::vector<std::uint32_t> commands;
+        for (const auto* header : {&front, &pixel}) {
+            if (header == &pixel && !withPixel) continue;
+            for (std::size_t index = 0; index < header->shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, header->registers[index].offset, header->registers[index].value});
+            for (std::size_t index = 0; index < header->shader.num_cx_registers; ++index) commands.insert(commands.end(), {0xc0016900u, header->context[index].offset, header->context[index].value});
+        }
+        if (!withPixel) commands.insert(commands.end(), {0xc0017600u, 0x8, 0});
+        commands.insert(commands.end(), {0xc0016900u, 0x8e, 0xfu, 0xc0016900u, 0x8f, 0xfu, 0xc0016900u, 0xd, 0x00100010u, 0xc0016900u, 0x204, 0x80000u, 0xc0016900u, 0x1c3, 4, 0xc0017900u, 0x242, 17, 0xc0012d00u, 3, 2});
+        Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+        Require(sceAgcDriverSubmitDcb(&packet) == 0, "a rect-list draw was not submitted");
+        AgcDriverWaitIdle_nid_postfix();
+    };
+    draw(false);
+    ExpectFailure([&] { draw(true); }, "custom per-vertex interpolation is unsupported");
+    ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "custom per-vertex interpolation is unsupported");
+}
+
 }
 
 int main(int argc, char** argv) {
     try {
-        Require(argc == 1 || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration" || std::string_view(argv[1]) == "--deferred" || std::string_view(argv[1]) == "--deferred-undecodable" || std::string_view(argv[1]) == "--registered-workgroup" || std::string_view(argv[1]) == "--lazy")), "invalid test arguments");
+        const std::string_view abiMode = argc == 2 ? argv[1] : "";
+        const bool abiDeferred = abiMode == "--abi-deferred-compute" || abiMode == "--abi-deferred-pixel" || abiMode == "--abi-deferred-stages" || abiMode == "--abi-deferred-rectangle" || abiMode == "--abi-deferred-undecodable-state" || abiMode == "--abi-deferred-registration-failure" || abiMode == "--abi-deferred-vertex-inputs" || abiMode == "--abi-deferred-rectangle-pairs";
+        Require(argc == 1 || abiDeferred || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration" || std::string_view(argv[1]) == "--deferred" || std::string_view(argv[1]) == "--deferred-undecodable" || std::string_view(argv[1]) == "--registered-workgroup" || std::string_view(argv[1]) == "--lazy")), "invalid test arguments");
+        if (abiDeferred) {
+            if (!OpenVulkanTestDevice()) return VulkanTestSkipped;
+            if (abiMode == "--abi-deferred-compute") AbiDeferredCompute();
+            else if (abiMode == "--abi-deferred-pixel") AbiDeferredPixel();
+            else if (abiMode == "--abi-deferred-stages") AbiDeferredStages();
+            else if (abiMode == "--abi-deferred-undecodable-state") AbiDeferredUndecodableState();
+            else if (abiMode == "--abi-deferred-registration-failure") AbiDeferredRegistrationFailure();
+            else if (abiMode == "--abi-deferred-vertex-inputs") AbiDeferredVertexInputs();
+            else if (abiMode == "--abi-deferred-rectangle-pairs") AbiDeferredRectanglePairs();
+            else AbiDeferredRectangle();
+            std::cout << "shader ABI deferral tests passed: " << abiMode << '\n';
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--lazy") {
             if (!OpenVulkanTestDevice()) return VulkanTestSkipped;
             LazyRegistration();

core/libs/prx/libSceAgcDriver/tests/execution/StorageFlushOrdering.cpp

@@ -0,0 +1,173 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <mutex>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::size_t BlockBytes = 1u << 20u;
+constexpr std::uint32_t Width = 256;
+constexpr std::uint32_t Height = 128;
+constexpr std::size_t ImageBytes = Width * Height * 4u;
+constexpr std::uint32_t Format32UInt = 20;
+constexpr std::uint32_t Type2D = 9;
+constexpr std::uint32_t Initial = 0x3f800000u;
+constexpr std::uint32_t Stored = 0x11111111u;
+constexpr std::uint32_t Overwritten = 0x22222222u;
+constexpr std::size_t OverwrittenDword = 0x100u / 4u;
+constexpr std::size_t FarDword = 100u * Width;
+
+alignas(256) constexpr std::array<std::uint32_t, 11> StoreImageCode{0x7e000280u, 0x7e020280u, 0x7e0402ffu, Stored, 0xf0200108u, 0x00010200u, 0x7e0202ffu, 0x00000064u, 0xf0200108u, 0x00010200u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 7> StoreGlobalCode{0x7e000208u, 0x7e020209u, 0x7e0402ffu, Overwritten, 0xdc708100u, 0x007d0200u, 0xbf810000u};
+
+class GuestBlock {
+public:
+    explicit GuestBlock(bool watched) : watched(watched) {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536));
+        if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes);
+#else
+        constexpr std::uintptr_t alignment = 65536;
+        void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (mapped != MAP_FAILED) {
+            const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
+            const auto aligned = (begin + alignment - 1) & ~(alignment - 1);
+            if (aligned != begin) munmap(mapped, aligned - begin);
+            if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast<void*>(aligned + BlockBytes), begin + alignment - aligned);
+            block = reinterpret_cast<std::uint8_t*>(aligned);
+            if (watched) GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes);
+        }
+#endif
+        Require(block != nullptr, "storage flush ordering: cannot allocate the guest block");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), BlockBytes), "storage flush ordering: the guest block is not write-watched");
+        std::fill(Words(), Words() + BlockBytes / 4u, Initial);
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true);
+        AgcDriver::GuestMemory::CollectWritesUncached(Address(), BlockBytes);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes);
+        GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes);
+#else
+        munmap(block, BlockBytes);
+        if (watched) GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint32_t* Words() const { return reinterpret_cast<std::uint32_t*>(block); }
+    std::uint64_t Address() const { return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(block)); }
+
+private:
+    bool watched;
+    std::uint8_t* block = nullptr;
+};
+
+ShaderRecompiler::RecompileResult Compile(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const ShaderRecompiler::SpirvTarget& target) {
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {64u, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        target,
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+std::vector<std::uint32_t> ImageUserData(std::uint64_t base) {
+    std::vector<std::uint32_t> userData(12, 0u);
+    userData[4] = static_cast<std::uint32_t>(base >> 8u);
+    userData[5] = static_cast<std::uint32_t>((base >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Width - 1u) & 3u) << 30u);
+    userData[6] = ((Width - 1u) >> 2u) | ((Height - 1u) << 14u);
+    userData[7] = 0xfacu | (Type2D << 28u);
+    return userData;
+}
+
+std::vector<std::uint32_t> GlobalUserData(std::uint64_t base) {
+    std::vector<std::uint32_t> userData(10, 0u);
+    userData[8] = static_cast<std::uint32_t>(base);
+    userData[9] = static_cast<std::uint32_t>(base >> 32u);
+    return userData;
+}
+
+bool AddressBasedStorePending(std::uint64_t image, bool open) {
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    const auto read = recorder != nullptr ? recorder->DescribePendingRead(image + ImageBytes, 4u) : std::nullopt;
+    return read.has_value() && read->kind == AgcDriver::Graphics::Recorder::ReadKind::AddressBased && read->open == open;
+}
+
+void Run(AgcDriver::VulkanDevice& device, const GuestBlock& guest, std::size_t offset, bool submit, const char* name) {
+    const auto* words = guest.Words() + offset / 4u;
+    const auto image = guest.Address() + offset;
+    const auto target = device.Target();
+    const auto storeImage = Compile(StoreImageCode, ImageUserData(image), target);
+    const auto storeGlobal = Compile(StoreGlobalCode, GlobalUserData(image), target);
+    device.Dispatch(storeImage, 1u, 1u, 1u, {}, reinterpret_cast<std::uintptr_t>(StoreImageCode.data()));
+    device.Dispatch(storeGlobal, 1u, 1u, 1u, {}, reinterpret_cast<std::uintptr_t>(StoreGlobalCode.data()));
+    if (submit) {
+        std::uint64_t serial = 0;
+        {
+            std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+            serial = device.SubmitAndEpoch();
+        }
+        device.WaitRecorded(serial);
+    }
+    const std::string what = std::string("storage flush ordering, ") + name + ": ";
+    Require(AddressBasedStorePending(image, !submit), what + "the address-based store was not pending in the expected batch at the flush");
+    Require(AgcDriver::Graphics::StorageTexture::FlushPending(image, ImageBytes, nullptr, "test"), what + "the image store left no results pending");
+    device.WaitIdle();
+    Require(words[FarDword] == Stored, what + "the pending image's texel away from the newer write was not stored");
+    Require(words[OverwrittenDword] == Overwritten, what + "the pending image's store put its older texels over a newer address-based store");
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const bool watched = AgcDriver::GuestMemory::WriteWatched();
+        GuestBlock guest(watched);
+        if (!watched) std::puts("guest memory has no write watch: the writer-in-flight case is skipped");
+        else if (!device->CanWaitUnlocked()) std::puts("the recorder cannot wait for the GPU without running completions: the writer-in-flight case is skipped");
+        else Run(*device, guest, 0, true, "writer in flight");
+        Run(*device, guest, BlockBytes / 2u, false, "writer in the open batch");
+        std::puts("storage flush ordering tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceIme/Export.cpp

@@ -1,3 +1,4 @@
+#include <atomic>
 #include <algorithm>
 #include <cstdint>
 #include <cstddef>
@@ -67,6 +68,7 @@ std::mutex g_keyboardMutex;
 std::set<int32_t> g_openKeyboards;
 std::mutex g_imeMutex;
 std::optional<Session> g_session;
+std::atomic<ImeHostTextSink> g_textSink{nullptr};
 
 void PanelSize(const Param& param, uint32_t* width, uint32_t* height) {
     const uint32_t scale = (param.option & OptionUseOver2K) != 0 ? 2 : 1;
@@ -284,13 +286,31 @@ int ValidateOpen(const Param* param, const ExtendedParam* extended) {
 extern "C" {
 
 void ImePublishInput_nid_postfix(const ImeHostInput& input) {
-    std::lock_guard lock(g_imeMutex);
-    if (g_session) Apply(*g_session, input);
+    {
+        std::lock_guard lock(g_imeMutex);
+        if (g_session) {
+            Apply(*g_session, input);
+            return;
+        }
+    }
+    const ImeHostTextSink sink = g_textSink.load();
+    if (sink == nullptr) return;
+    std::u16string units;
+    if (input.kind == ImeHostInputKind::Text) {
+        std::size_t size = 0;
+        while (size < sizeof(input.text) && input.text[size] != '\0') ++size;
+        units = DecodeUtf8(input.text, size);
+    }
+    sink(input.kind, units.data(), units.size());
 }
 
 bool ImeCapturesKeyboard_nid_postfix() {
     std::lock_guard lock(g_imeMutex);
-    return g_session.has_value();
+    return g_session.has_value() || g_textSink.load() != nullptr;
+}
+
+void ImeSetHostTextSink_nid_no_patch(ImeHostTextSink sink) {
+    g_textSink = sink;
 }
 
 int APS5_VABI sceImeOpen_nid_postfix(const Param* param, const ExtendedParam* extended) {

core/libs/prx/libSceIme/include/ImeInput.hpp

@@ -1,6 +1,7 @@
 #ifndef CORE_LIBS_PRX_LIBSCEIME_INCLUDE_IMEINPUT_HPP
 #define CORE_LIBS_PRX_LIBSCEIME_INCLUDE_IMEINPUT_HPP
 
+#include <cstddef>
 #include <cstdint>
 
 enum class ImeHostInputKind : std::uint32_t {
@@ -23,4 +24,8 @@ struct ImeHostInput {
 extern "C" void ImePublishInput_nid_postfix(const ImeHostInput& input);
 extern "C" bool ImeCapturesKeyboard_nid_postfix();
 
+using ImeHostTextSink = void (*)(ImeHostInputKind kind, const char16_t* text, std::size_t length);
+
+extern "C" void ImeSetHostTextSink_nid_no_patch(ImeHostTextSink sink);
+
 #endif

core/libs/prx/libSceImeDialog/CMakeLists.txt

@@ -0,0 +1,17 @@
+add_library(libSceImeDialog SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp)
+target_include_directories(libSceImeDialog PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(libSceImeDialog PRIVATE libSceIme)
+configure_windows_unwind(libSceImeDialog)
+
+if(BUILD_TESTING)
+    add_test_executable(guest_ime_dialog_tests ${LIBS_INCLUDE_DIR}/tests/GuestImeDialog.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/../libSceVideoOut/src/ImeTextInput.cpp)
+    target_include_directories(guest_ime_dialog_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_SDL2_INCLUDE_DIR})
+    target_link_libraries(guest_ime_dialog_tests PRIVATE libSceImeDialog libSceIme libc)
+    configure_windows_unwind(guest_ime_dialog_tests)
+    add_test(NAME guest_ime_dialog COMMAND guest_ime_dialog_tests)
+    if(WIN32)
+        set_tests_properties(guest_ime_dialog PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceImeDialog>")
+    endif()
+endif()

core/libs/prx/libSceImeDialog/Export.cpp

@@ -1,9 +1,11 @@
-#include <atomic>
+#include <algorithm>
 #include <cstdint>
 #include <cstddef>
-#include <cstdio>
+#include <mutex>
+#include <string>
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
+#include "prx/libSceIme/include/ImeInput.hpp"
 
 namespace {
 
@@ -11,21 +13,105 @@ constexpr int StatusNone = 0;
 constexpr int StatusRunning = 1;
 constexpr int StatusFinished = 2;
 constexpr std::uint32_t EndStatusOk = 0;
+constexpr std::uint32_t EndStatusUserCanceled = 1;
 constexpr std::uint32_t EndStatusAborted = 2;
+constexpr std::uint32_t MaxTextLength = 2048;
+constexpr int ErrorBusy = static_cast<int>(0x80bc0001u);
 constexpr int ErrorNotOpened = static_cast<int>(0x80bc0002u);
+constexpr int ErrorInvalidMaxTextLength = static_cast<int>(0x80bc0016u);
+constexpr int ErrorInvalidInputTextBuffer = static_cast<int>(0x80bc0017u);
+constexpr int ErrorInvalidAddress = static_cast<int>(0x80bc0031u);
 
-std::atomic<int> status{StatusNone};
-std::atomic<int> polls{0};
-std::atomic<std::uint32_t> endStatus{EndStatusOk};
+struct Dialog {
+    int status = StatusNone;
+    std::uint32_t endStatus = EndStatusOk;
+    char16_t* buffer = nullptr;
+    std::uint32_t maxLength = 0;
+    std::u16string text;
+    std::size_t caret = 0;
+};
+
+std::mutex g_mutex;
+Dialog g_dialog;
+
+bool IsHighSurrogate(char16_t unit) { return unit >= 0xd800 && unit <= 0xdbff; }
+bool IsLowSurrogate(char16_t unit) { return unit >= 0xdc00 && unit <= 0xdfff; }
+
+std::size_t UnitsBefore(const Dialog& dialog) {
+    if (dialog.caret == 0) return 0;
+    return dialog.caret >= 2 && IsLowSurrogate(dialog.text[dialog.caret - 1]) && IsHighSurrogate(dialog.text[dialog.caret - 2]) ? 2 : 1;
+}
+
+std::size_t UnitsAfter(const Dialog& dialog) {
+    if (dialog.caret >= dialog.text.size()) return 0;
+    return dialog.caret + 1 < dialog.text.size() && IsHighSurrogate(dialog.text[dialog.caret]) && IsLowSurrogate(dialog.text[dialog.caret + 1]) ? 2 : 1;
+}
+
+void Finish(std::uint32_t endStatus) {
+    g_dialog.endStatus = endStatus;
+    g_dialog.status = StatusFinished;
+    ImeSetHostTextSink_nid_no_patch(nullptr);
+}
+
+void Insert(Dialog& dialog, const char16_t* units, std::size_t length) {
+    std::u16string accepted;
+    for (std::size_t i = 0; i < length; ++i) {
+        if (units[i] != u'\n' && units[i] != u'\r') accepted.push_back(units[i]);
+    }
+    std::size_t count = std::min<std::size_t>(accepted.size(), dialog.maxLength - dialog.text.size());
+    if (count > 0 && count < accepted.size() && IsHighSurrogate(accepted[count - 1])) --count;
+    dialog.text.insert(dialog.caret, accepted, 0, count);
+    dialog.caret += count;
+}
+
+void Receive(ImeHostInputKind kind, const char16_t* units, std::size_t length) {
+    std::lock_guard lock(g_mutex);
+    Dialog& dialog = g_dialog;
+    if (dialog.status != StatusRunning) return;
+    switch (kind) {
+        case ImeHostInputKind::Text:
+            Insert(dialog, units, length);
+            return;
+        case ImeHostInputKind::Backspace: {
+            const std::size_t count = UnitsBefore(dialog);
+            dialog.caret -= count;
+            dialog.text.erase(dialog.caret, count);
+            return;
+        }
+        case ImeHostInputKind::Delete:
+            dialog.text.erase(dialog.caret, UnitsAfter(dialog));
+            return;
+        case ImeHostInputKind::Left:
+            dialog.caret -= UnitsBefore(dialog);
+            return;
+        case ImeHostInputKind::Right:
+            dialog.caret += UnitsAfter(dialog);
+            return;
+        case ImeHostInputKind::Home:
+            dialog.caret = 0;
+            return;
+        case ImeHostInputKind::End:
+            dialog.caret = dialog.text.size();
+            return;
+        case ImeHostInputKind::Enter:
+            std::copy(dialog.text.begin(), dialog.text.end(), dialog.buffer);
+            dialog.buffer[dialog.text.size()] = u'\0';
+            Finish(EndStatusOk);
+            return;
+        case ImeHostInputKind::Escape:
+            Finish(EndStatusUserCanceled);
+            return;
+    }
+}
 
 }
 
 extern "C" {
 
 int APS5_VABI sceImeDialogAbort(void) {
-    if (status.load() != StatusRunning) return ErrorNotOpened;
-    endStatus = EndStatusAborted;
-    status = StatusFinished;
+    std::lock_guard lock(g_mutex);
+    if (g_dialog.status != StatusRunning) return ErrorNotOpened;
+    Finish(EndStatusAborted);
     return 0;
 }
 
@@ -50,47 +136,42 @@ int APS5_VABI sceImeDialogGetPanelSizeExtended(const Param* param, const Extende
 }
 
 int APS5_VABI sceImeDialogGetResult(Result* result) {
-    std::fprintf(stderr, "[ime] GetResult (status %d)\n", status.load());
-    if (result == nullptr || status.load() != StatusFinished) return ErrorNotOpened;
+    std::lock_guard lock(g_mutex);
+    if (result == nullptr || g_dialog.status != StatusFinished) return ErrorNotOpened;
     *result = Result{};
-    result->endstatus = endStatus.load();
+    result->endstatus = g_dialog.endStatus;
     return 0;
 }
 
 int APS5_VABI sceImeDialogGetStatus(void) {
-    if (status.load() == StatusRunning && ++polls > 1) status = StatusFinished;
-    static std::atomic<int> traced{0};
-    const int count = traced.fetch_add(1);
-    if (count < 8 || count % 2000 == 0) std::fprintf(stderr, "[ime] GetStatus #%d -> %d\n", count, status.load());
-    return status.load();
+    std::lock_guard lock(g_mutex);
+    return g_dialog.status;
 }
 
 int APS5_VABI sceImeDialogInit(const Param* param, const ExtendedParam* extended) {
     (void)extended;
-    static std::atomic<int> reported{0};
-    bool filled = false;
-    if (param != nullptr && param->input_text_buffer != nullptr && param->max_text_length != 0 && param->input_text_buffer[0] == u'\0') {
-        static constexpr char16_t defaultText[] = u"Slayer";
-        std::uint32_t length = 0;
-        while (defaultText[length] != u'\0' && length < param->max_text_length) {
-            param->input_text_buffer[length] = defaultText[length];
-            ++length;
-        }
-        param->input_text_buffer[length] = u'\0';
-        filled = true;
-    }
-    std::fprintf(stderr, "[ime] Init #%d\n", reported.load());
-    if (reported.fetch_add(1) == 0) std::fprintf(stderr, "[ime] dialog requested (max text length %u): finishing at once with %s\n", param != nullptr ? param->max_text_length : 0u, filled ? "a default text" : "the text unchanged");
-    polls = 0;
-    endStatus = EndStatusOk;
-    status = StatusRunning;
+    std::lock_guard lock(g_mutex);
+    if (g_dialog.status != StatusNone) return ErrorBusy;
+    if (param == nullptr) return ErrorInvalidAddress;
+    if (param->max_text_length == 0 || param->max_text_length > MaxTextLength) return ErrorInvalidMaxTextLength;
+    if (param->input_text_buffer == nullptr) return ErrorInvalidInputTextBuffer;
+    g_dialog = Dialog{};
+    g_dialog.buffer = param->input_text_buffer;
+    g_dialog.maxLength = param->max_text_length;
+    std::size_t length = 0;
+    while (length < param->max_text_length && param->input_text_buffer[length] != u'\0') ++length;
+    g_dialog.text.assign(param->input_text_buffer, length);
+    g_dialog.caret = length;
+    g_dialog.status = StatusRunning;
+    ImeSetHostTextSink_nid_no_patch(&Receive);
     return 0;
 }
 
 int APS5_VABI sceImeDialogTerm(void) {
-    std::fprintf(stderr, "[ime] Term (status %d)\n", status.load());
-    if (status.load() == StatusNone) return ErrorNotOpened;
-    status = StatusNone;
+    std::lock_guard lock(g_mutex);
+    if (g_dialog.status == StatusNone) return ErrorNotOpened;
+    ImeSetHostTextSink_nid_no_patch(nullptr);
+    g_dialog = Dialog{};
     return 0;
 }
 

core/libs/prx/libSceNet/Export.cpp

@@ -887,6 +887,14 @@ int APS5_VABI sceNetAccept(int s, void* addr, uint32_t* addrlen) {
         std::lock_guard lk(g_mutex);
         g_socks.emplace(descriptor, connection);
     }
+    if (listener.nonblock) {
+        const int enabled = 1;
+        if (sceNetSetsockopt(descriptor, NET_SOL_SOCKET, NET_SO_NBIO, &enabled, sizeof(enabled)) != 0) {
+            const int error = *errno_slot();
+            sceNetSocketClose(descriptor);
+            return fail(error);
+        }
+    }
     if (addr && !native_to_guest_address(peer, addr, addrlen)) {
         sceNetSocketClose(descriptor);
         return fail(NET_EAFNOSUPPORT);

core/libs/prx/libSceRegMgr/CMakeLists.txt

@@ -0,0 +1,15 @@
+add_library(libSceRegMgr SHARED EXCLUDE_FROM_ALL Export.cpp)
+target_include_directories(libSceRegMgr PRIVATE ${LIBS_INCLUDE_DIR})
+configure_windows_unwind(libSceRegMgr)
+
+if(BUILD_TESTING)
+    add_test_executable(guest_regmgr_tests ${LIBS_INCLUDE_DIR}/tests/GuestRegMgr.cpp)
+    target_include_directories(guest_regmgr_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_regmgr_tests PRIVATE libSceRegMgr libc)
+    configure_windows_unwind(guest_regmgr_tests)
+    add_test(NAME guest_regmgr COMMAND guest_regmgr_tests)
+    if(WIN32)
+        set_tests_properties(guest_regmgr PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceRegMgr>")
+    endif()
+endif()

core/libs/prx/libSceRegMgr/Export.cpp

@@ -0,0 +1,14 @@
+#include <cstdint>
+#include "prx/libc/include/General.hpp"
+
+extern "C" {
+
+std::int32_t APS5_VABI sceRegMgrNonSysGetInt(std::int64_t key, std::int32_t unk, std::int32_t* data) {
+    (void)key;
+    (void)unk;
+    (void)data;
+    NotImplemented_nid_no_patch(__func__);
+    return 0;
+}
+
+}

core/libs/prx/libSceSsl/CMakeLists.txt

@@ -85,9 +85,14 @@ if(BUILD_TESTING)
         add_test(NAME guest_ssl_transport_${table} COMMAND guest_ssl_transport_tests ${table})
         set_tests_properties(guest_ssl_transport_${table} PROPERTIES TIMEOUT 30)
     endforeach()
+    add_test_executable(guest_ssl_ps5_readers_tests ${LIBS_INCLUDE_DIR}/tests/GuestSslPs5Readers.cpp)
+    target_include_directories(guest_ssl_ps5_readers_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_ssl_ps5_readers_tests PRIVATE libSceSsl libc)
+    configure_windows_unwind(guest_ssl_ps5_readers_tests)
+    add_test(NAME guest_ssl_ps5_readers COMMAND guest_ssl_ps5_readers_tests)
     if(WIN32)
         target_link_libraries(guest_ssl_transport_tests PRIVATE bcrypt ws2_32)
-        set_tests_properties(guest_ssl guest_ssl_load guest_ssl_transport_kernel guest_ssl_transport_net PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(guest_ssl guest_ssl_load guest_ssl_ps5_readers guest_ssl_transport_kernel guest_ssl_transport_net PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSsl>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNet>")
     endif()
 endif()

core/libs/prx/libSceSsl/Export.cpp

@@ -11,12 +11,14 @@
 #include "prx/libkernel/Socket/include/SocketPoll.hpp"
 
 #include <mbedtls/asn1.h>
+#include <mbedtls/base64.h>
 #include <mbedtls/ctr_drbg.h>
 #include <mbedtls/entropy.h>
 #include <mbedtls/error.h>
 #include <mbedtls/md.h>
 #include <mbedtls/net_sockets.h>
 #include <mbedtls/oid.h>
+#include <mbedtls/pem.h>
 #include <mbedtls/pk.h>
 #include <mbedtls/platform.h>
 #include <mbedtls/platform_util.h>
@@ -1053,8 +1055,26 @@ int APS5_VABI sceSslGetFingerprint(int sslCtxId, void* sslCert, int hashType, un
     return 0;
 }
 
-int APS5_VABI sceSslGetPem(void) {
-    NotImplemented_nid_no_patch(__func__);
+int APS5_VABI sceSslGetPem(int sslCtxId, void* sslCert, char* buf, std::size_t* bufLen) {
+    if (!bufLen) return ErrorInvalidArg;
+    std::lock_guard lock(State().mutex);
+    auto& context = GetContext(sslCtxId);
+    const auto* crt = RequireCertificate(__func__, sslCtxId, sslCert);
+    PoolScope scope(context.pool);
+    std::size_t required = 0;
+    const auto encode = [&](unsigned char* output, std::size_t capacity, std::size_t* written) {
+        return mbedtls_pem_write_buffer("-----BEGIN CERTIFICATE-----\n", "-----END CERTIFICATE-----\n", crt->raw.p, crt->raw.len, output, capacity, written);
+    };
+    const int sizing = encode(nullptr, 0, &required);
+    if (sizing != MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL) CheckBackend(sizing, "SSL PEM sizing failed");
+    if (!buf) {
+        *bufLen = required;
+        return 0;
+    }
+    if (*bufLen < required) throw std::length_error("SSL PEM buffer is too small");
+    std::size_t written = 0;
+    CheckBackend(encode(reinterpret_cast<unsigned char*>(buf), required, &written), "SSL PEM encoding failed");
+    *bufLen = written;
     return 0;
 }
 

core/libs/prx/libSceVideoOut/CMakeLists.txt

@@ -106,7 +106,7 @@ if(BUILD_TESTING)
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         endif()
     endforeach()
-    foreach(flipMode IN ITEMS eop eop_main eop_aftergpu)
+    foreach(flipMode IN ITEMS eop eop_main eop_aftergpu cpuflip)
         add_test(NAME video_out_flip_${flipMode} COMMAND video_out_flip_tests ${flipMode})
         set_tests_properties(video_out_flip_${flipMode} PROPERTIES TIMEOUT 30 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0 ENVIRONMENT "SDL_VIDEODRIVER=${ANYPS5_VIDEO_OUT_TEST_DRIVER}")
         if(WIN32)
@@ -114,6 +114,12 @@ if(BUILD_TESTING)
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         endif()
     endforeach()
+    add_test(NAME video_out_flip_cpuflip_in_order COMMAND video_out_flip_tests cpuflip)
+    set_tests_properties(video_out_flip_cpuflip_in_order PROPERTIES TIMEOUT 30 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0 ENVIRONMENT "SDL_VIDEODRIVER=${ANYPS5_VIDEO_OUT_TEST_DRIVER};APS5_CPU_FLIP_IN_ORDER=1")
+    if(WIN32)
+        set_tests_properties(video_out_flip_cpuflip_in_order PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
 endif()
 
 if(BUILD_TESTING AND NOT WIN32)

core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp

@@ -28,7 +28,6 @@
 #include "prx/libSceAgcDriver/Execution/include/Presentation.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
-#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
 #include "prx/libc/include/PreciseWait.hpp"
 #include "prx/libc/include/Shutdown.hpp"
 
@@ -440,10 +439,7 @@ int VideoOutDriver::SubmitFlip(int handle, int index, int flipMode, int64_t flip
     // A title that does not pace on flipPendingNum can run ahead of the presenter now that the queue
     // worker no longer waits per flip; a full queue is the documented error, not a failure.
     if (flipQueue->reservations.load() >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) return VIDEO_OUT_ERROR_FLIP_QUEUE_FULL;
-    std::array<uint32_t, AgcDriver::FlipPacketWords> words{AgcDriver::FlipPacketHeader, static_cast<uint32_t>(handle), static_cast<uint32_t>(index), static_cast<uint32_t>(flipMode), static_cast<uint32_t>(static_cast<uint64_t>(flipArg)), static_cast<uint32_t>(static_cast<uint64_t>(flipArg) >> 32u)};
-    Packet packet{words.data(), static_cast<uint32_t>(words.size()), 0, {}};
-    const auto result = sceAgcDriverSubmitDcb(&packet);
-    require(result == 0, "driver rejected flip submission");
+    AgcDriverSubmitFlip_nid_postfix(AgcDriver::FlipInfo{static_cast<std::uint32_t>(handle), index, static_cast<std::uint32_t>(flipMode), flipArg});
     return 0;
 }
 

core/libs/prx/libSceVideoOut/tests/Flip.cpp

@@ -225,6 +225,57 @@ void testDeferredBuffer() {
     LibcRunShutdown_nid_postfix();
 }
 
+void testCpuFlipOrder() {
+    const bool inOrder = std::getenv("APS5_CPU_FLIP_IN_ORDER") != nullptr;
+    const int handle = sceVideoOutOpen(255, 0, 0, nullptr);
+    auto cfg = VideoOutDriver::Get().GetConfig(handle);
+    std::vector<std::byte> allocation(65536 + 65535);
+    const auto storage = alignedBuffer(allocation);
+    VideoOutBuffers buffer{storage.data(), nullptr, {nullptr, nullptr}};
+    VideoOutBufferAttribute2 attribute{};
+    sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 64, 64, 0, 0, 0);
+    check(sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr) == 0, "buffer registration failed");
+    const auto flipsWithin = [&](int count, std::chrono::milliseconds limit) {
+        std::unique_lock lock(cfg->mutex);
+        const bool done = cfg->vblankCond.wait_for(lock, limit, [&] { return cfg->failure || cfg->flipStatus.count == count; });
+        if (cfg->failure) std::rethrow_exception(cfg->failure);
+        return done;
+    };
+    const auto shownArgument = [&] {
+        std::lock_guard lock(cfg->mutex);
+        return cfg->flipStatus.flipArg;
+    };
+    auto first = std::make_shared<Gate>();
+    AgcDriverRegisterVideoOutput_nid_postfix(7, first);
+    std::array<uint32_t, 6> firstHeld{0xc004105c, 7, 0xfffffffeu, 1, 0, 0};
+    Packet firstPacket{firstHeld.data(), 6, 0, {}};
+    check(sceAgcDriverSubmitDcb(&firstPacket) == 0, "the first gated submission was not accepted");
+    std::thread flipThread([&] { check(sceVideoOutSubmitFlip(handle, 0, 1, 5) == 0, "the flip thread's flip was not accepted"); });
+    flipThread.join();
+    const bool ahead = flipsWithin(1, std::chrono::milliseconds(inOrder ? 1000 : 8000));
+    if (inOrder) check(!ahead, "with APS5_CPU_FLIP_IN_ORDER a flip completed while a queue-0 submission ahead of it was held");
+    else check(ahead, "a flip from a thread that never submitted queue-0 work waited for a held queue-0 submission");
+    first->Release();
+    check(flipsWithin(1, std::chrono::seconds(30)), "the flip thread's flip did not complete");
+    check(shownArgument() == 5, "the flip thread's flip presented the wrong request");
+    auto second = std::make_shared<Gate>();
+    AgcDriverRegisterVideoOutput_nid_postfix(8, second);
+    std::array<uint32_t, 6> secondHeld{0xc004105c, 8, 0xfffffffeu, 1, 0, 0};
+    Packet secondPacket{secondHeld.data(), 6, 0, {}};
+    check(sceAgcDriverSubmitDcb(&secondPacket) == 0, "the second gated submission was not accepted");
+    check(sceVideoOutSubmitFlip(handle, 0, 1, 6) == 0, "the submitting thread's flip was not accepted");
+    check(!flipsWithin(2, std::chrono::milliseconds(1000)), "a flip from the thread that submitted queue-0 work completed ahead of that work");
+    second->Release();
+    check(flipsWithin(2, std::chrono::seconds(30)), "the submitting thread's flip did not complete");
+    check(shownArgument() == 6, "the submitting thread's flip presented the wrong request");
+    AgcDriverWaitIdle_nid_postfix();
+    sceVideoOutUnregisterBuffers(handle, 0);
+    sceVideoOutClose(handle);
+    AgcDriverUnregisterVideoOutput_nid_postfix(7, first);
+    AgcDriverUnregisterVideoOutput_nid_postfix(8, second);
+    LibcRunShutdown_nid_postfix();
+}
+
 void testUnregisteredBuffer() {
     const int handle = sceVideoOutOpen(255, 0, 0, nullptr);
     auto cfg = VideoOutDriver::Get().GetConfig(handle);
@@ -975,6 +1026,7 @@ int main(int argc, char** argv) {
         else if (argc == 2 && std::string(argv[1]) == "deferred") testDeferredBuffer();
         else if (argc == 2 && std::string(argv[1]) == "unregistered") testUnregisteredBuffer();
         else if (argc == 2 && std::string(argv[1]) == "ps4") testPs4Buffers();
+        else if (argc == 2 && std::string(argv[1]) == "cpuflip") testCpuFlipOrder();
         else testLifetime(argc == 2 && std::string(argv[1]) == "reopen");
         std::puts("VideoOut flip tests passed");
         return 0;

core/libs/prx/libc/CMakeLists.txt

@@ -45,6 +45,7 @@ target_sources(libc PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/src/FloatBuiltins.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/HostEntropy.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/Backtrace.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/DinkumwareAtomics.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/Time.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/AtomicBuiltins.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/FileAndHeap.cpp
@@ -402,6 +403,15 @@ if(BUILD_TESTING)
         set_tests_properties(guest_environment PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test_executable(guest_dinkumware_atomics_tests ${LIBS_INCLUDE_DIR}/tests/GuestDinkumwareAtomics.cpp)
+    target_include_directories(guest_dinkumware_atomics_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_dinkumware_atomics_tests PRIVATE libc)
+    configure_windows_unwind(guest_dinkumware_atomics_tests)
+    add_test(NAME guest_dinkumware_atomics COMMAND guest_dinkumware_atomics_tests)
+    if(WIN32)
+        set_tests_properties(guest_dinkumware_atomics PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test_executable(guest_hostname_tests ${LIBS_INCLUDE_DIR}/tests/GuestHostname.cpp)
     target_include_directories(guest_hostname_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_hostname_tests PRIVATE libc)

core/libs/prx/libc/include/FilesystemError.hpp

@@ -23,7 +23,7 @@ inline int FilesystemError(const std::error_code& error) {
     return 5;
 }
 
-std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, bool replacesFile);
+std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, bool replacesExisting);
 
 inline int ResolvePathError_nid_no_patch(const char* path, std::filesystem::path& resolved, bool module = false, bool archived = false) {
     try {

core/libs/prx/libc/src/DinkumwareAtomics.cpp

@@ -0,0 +1,175 @@
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+#include "prx/libc/include/General.hpp"
+
+namespace {
+
+constexpr int MemoryOrderSeqCst = 5;
+
+template<typename TValue>
+void Store(volatile TValue* target, TValue value) {
+    __atomic_store_n(target, value, __ATOMIC_SEQ_CST);
+}
+
+template<typename TValue>
+TValue Load(volatile TValue* target) {
+    return __atomic_load_n(target, __ATOMIC_SEQ_CST);
+}
+
+bool IsValidOrder(int order) {
+    return order >= 0 && order <= MemoryOrderSeqCst;
+}
+
+template<typename TValue>
+TValue Exchange(volatile TValue* target, TValue value) {
+    return __atomic_exchange_n(target, value, __ATOMIC_SEQ_CST);
+}
+
+template<typename TValue>
+int CompareExchange(volatile TValue* target, TValue* expected, TValue desired) {
+    return __atomic_compare_exchange_n(target, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ? 1 : 0;
+}
+
+void LockFlag(volatile unsigned char* flag) {
+    while (__atomic_exchange_n(flag, static_cast<unsigned char>(1), __ATOMIC_ACQUIRE) != 0) {
+    }
+}
+
+void UnlockFlag(volatile unsigned char* flag) {
+    __atomic_store_n(flag, static_cast<unsigned char>(0), __ATOMIC_SEQ_CST);
+}
+
+void CopyBytes(volatile void* destination, const volatile void* source, std::size_t size) {
+    std::memcpy(const_cast<void*>(destination), const_cast<const void*>(source), size);
+}
+
+int LockedCompareExchange(volatile unsigned char* flag, std::size_t size, volatile void* target, void* expected, const void* desired) {
+    LockFlag(flag);
+    void* object = const_cast<void*>(target);
+    const bool equal = std::memcmp(object, expected, size) == 0;
+    if (equal) {
+        std::memcpy(object, desired, size);
+    } else {
+        std::memcpy(expected, object, size);
+    }
+    UnlockFlag(flag);
+    return equal ? 1 : 0;
+}
+
+}
+
+extern "C" {
+
+void APS5_VABI _Atomic_store_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { Store(target, value); }
+void APS5_VABI _Atomic_store_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { Store(target, value); }
+void APS5_VABI _Atomic_store_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { Store(target, value); }
+void APS5_VABI _Atomic_store_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int) { Store(target, value); }
+
+std::uint8_t APS5_VABI _Atomic_load_1_nid_postfix(volatile std::uint8_t* target, int) { return Load(target); }
+std::uint16_t APS5_VABI _Atomic_load_2_nid_postfix(volatile std::uint16_t* target, int) { return Load(target); }
+std::uint64_t APS5_VABI _Atomic_load_8_nid_postfix(volatile std::uint64_t* target, int order) {
+    if (order == 3 || order == 4 || !IsValidOrder(order)) return 0;
+    return Load(target);
+}
+
+std::uint8_t APS5_VABI _Atomic_exchange_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return Exchange(target, value); }
+std::uint16_t APS5_VABI _Atomic_exchange_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return Exchange(target, value); }
+std::uint32_t APS5_VABI _Atomic_exchange_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { return Exchange(target, value); }
+std::uint64_t APS5_VABI _Atomic_exchange_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? Exchange(target, value) : 0; }
+
+int APS5_VABI _Atomic_compare_exchange_strong_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t* expected, std::uint8_t desired, int, int) {
+    return CompareExchange(target, expected, desired);
+}
+
+int APS5_VABI _Atomic_compare_exchange_strong_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t* expected, std::uint16_t desired, int, int) {
+    return CompareExchange(target, expected, desired);
+}
+
+int APS5_VABI _Atomic_compare_exchange_strong_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t* expected, std::uint32_t desired, int, int) {
+    return CompareExchange(target, expected, desired);
+}
+
+int APS5_VABI _Atomic_compare_exchange_strong_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t* expected, std::uint64_t desired, int, int) {
+    return CompareExchange(target, expected, desired);
+}
+
+int APS5_VABI _Atomic_compare_exchange_weak_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t* expected, std::uint8_t desired, int, int) {
+    return CompareExchange(target, expected, desired);
+}
+
+int APS5_VABI _Atomic_compare_exchange_weak_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t* expected, std::uint16_t desired, int, int) {
+    return CompareExchange(target, expected, desired);
+}
+
+int APS5_VABI _Atomic_compare_exchange_weak_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t* expected, std::uint64_t desired, int, int) {
+    return CompareExchange(target, expected, desired);
+}
+
+std::uint8_t APS5_VABI _Atomic_fetch_add_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_add(target, value, __ATOMIC_SEQ_CST); }
+std::uint16_t APS5_VABI _Atomic_fetch_add_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_add(target, value, __ATOMIC_SEQ_CST); }
+std::uint64_t APS5_VABI _Atomic_fetch_add_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_add(target, value, __ATOMIC_SEQ_CST) : 0; }
+std::uint8_t APS5_VABI _Atomic_fetch_sub_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_sub(target, value, __ATOMIC_SEQ_CST); }
+std::uint16_t APS5_VABI _Atomic_fetch_sub_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_sub(target, value, __ATOMIC_SEQ_CST); }
+std::uint64_t APS5_VABI _Atomic_fetch_sub_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_sub(target, value, __ATOMIC_SEQ_CST) : 0; }
+std::uint8_t APS5_VABI _Atomic_fetch_and_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); }
+std::uint16_t APS5_VABI _Atomic_fetch_and_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); }
+std::uint32_t APS5_VABI _Atomic_fetch_and_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); }
+std::uint64_t APS5_VABI _Atomic_fetch_and_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST) : 0; }
+std::uint8_t APS5_VABI _Atomic_fetch_or_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); }
+std::uint16_t APS5_VABI _Atomic_fetch_or_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); }
+std::uint32_t APS5_VABI _Atomic_fetch_or_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); }
+std::uint64_t APS5_VABI _Atomic_fetch_or_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST) : 0; }
+std::uint8_t APS5_VABI _Atomic_fetch_xor_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); }
+std::uint16_t APS5_VABI _Atomic_fetch_xor_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); }
+std::uint32_t APS5_VABI _Atomic_fetch_xor_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); }
+std::uint64_t APS5_VABI _Atomic_fetch_xor_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST) : 0; }
+
+int APS5_VABI _Atomic_is_lock_free_1_nid_postfix() { return 1; }
+int APS5_VABI _Atomic_is_lock_free_2_nid_postfix() { return 1; }
+int APS5_VABI _Atomic_is_lock_free_4_nid_postfix() { return 1; }
+int APS5_VABI _Atomic_is_lock_free_8_nid_postfix() { return 1; }
+
+int APS5_VABI _Atomic_flag_test_and_set_nid_postfix(volatile unsigned char* flag, int) {
+    return __atomic_exchange_n(flag, static_cast<unsigned char>(1), __ATOMIC_SEQ_CST);
+}
+
+void APS5_VABI _Atomic_flag_clear_nid_postfix(volatile unsigned char* flag, int) {
+    __atomic_store_n(flag, static_cast<unsigned char>(0), __ATOMIC_SEQ_CST);
+}
+
+void APS5_VABI _Atomic_thread_fence_nid_postfix(int order) {
+    if (order == MemoryOrderSeqCst) __atomic_thread_fence(__ATOMIC_SEQ_CST);
+}
+
+void APS5_VABI _Atomic_signal_fence_nid_postfix(int) {
+}
+
+void APS5_VABI _Atomic_copy_nid_postfix(volatile unsigned char* flag, std::size_t size, volatile void* target, const volatile void* source, int) {
+    LockFlag(flag);
+    CopyBytes(target, source, size);
+    UnlockFlag(flag);
+}
+
+void APS5_VABI _Atomic_exchange_nid_postfix(volatile unsigned char* flag, std::size_t size, volatile void* target, volatile void* value, int) {
+    LockFlag(flag);
+    auto* object = static_cast<unsigned char*>(const_cast<void*>(target));
+    auto* other = static_cast<unsigned char*>(const_cast<void*>(value));
+    for (std::size_t i = 0; i < size; ++i) {
+        const unsigned char byte = object[i];
+        object[i] = other[i];
+        other[i] = byte;
+    }
+    UnlockFlag(flag);
+}
+
+int APS5_VABI _Atomic_compare_exchange_strong_nid_postfix(volatile unsigned char* flag, std::size_t size, volatile void* target, void* expected, const void* desired, int, int) {
+    return LockedCompareExchange(flag, size, target, expected, desired);
+}
+
+int APS5_VABI _Atomic_compare_exchange_weak_nid_postfix(volatile unsigned char* flag, std::size_t size, volatile void* target, void* expected, const void* desired, int, int) {
+    return LockedCompareExchange(flag, size, target, expected, desired);
+}
+
+}

core/libs/prx/libc/src/Rename.cpp

@@ -10,11 +10,11 @@
 #include <windows.h>
 #endif
 
-std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, [[maybe_unused]] bool replacesFile) {
+std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, [[maybe_unused]] bool replacesExisting) {
     namespace fs = std::filesystem;
     std::error_code error;
 #ifdef _WIN32
-    if (replacesFile) {
+    if (replacesExisting) {
         const auto target = fs::absolute(destination, error);
         if (error) return error;
         struct RenameInfo { DWORD flags; HANDLE root; DWORD nameBytes; WCHAR name[1]; };
@@ -28,7 +28,7 @@ std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, c
         info->flags = replaceExisting | posixSemantics | (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_READONLY) ? ignoreReadOnly : 0);
         info->nameBytes = static_cast<DWORD>(nameBytes);
         std::memcpy(info->name, target.c_str(), nameBytes);
-        const auto handle = CreateFileW(source.c_str(), DELETE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, nullptr);
+        const auto handle = CreateFileW(source.c_str(), DELETE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, nullptr);
         if (handle == INVALID_HANDLE_VALUE) return std::error_code(static_cast<int>(GetLastError()), std::system_category());
         constexpr auto renameInfoEx = static_cast<FILE_INFO_BY_HANDLE_CLASS>(22);
         if (!SetFileInformationByHandle(handle, renameInfoEx, info, static_cast<DWORD>(buffer.size()))) error = std::error_code(static_cast<int>(GetLastError()), std::system_category());
@@ -42,7 +42,6 @@ std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, c
 
 static int RenamePath_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination) {
     constexpr int GuestEnoent = 2;
-    constexpr int GuestEio = 5;
     constexpr int GuestEnotdir = 20;
     constexpr int GuestEisdir = 21;
     constexpr int GuestEinval = 22;
@@ -90,14 +89,7 @@ static int RenamePath_nid_no_patch(const std::filesystem::path& source, const st
             return error ? FilesystemError(error) : GuestEnotempty;
         }
     }
-#ifdef _WIN32
-    if (!sameFile && sourceIsDirectory && destinationIsDirectory) {
-        error.clear();
-        if (!fs::remove(destination, error)) return error ? FilesystemError(error) : GuestEio;
-    }
-#endif
-
-    error = RenameNative_nid_no_patch(source, destination, destinationExists && !sourceIsDirectory && !destinationIsDirectory);
+    error = RenameNative_nid_no_patch(source, destination, destinationExists && (!sourceIsDirectory || !sameFile));
     return error ? FilesystemError(error) : 0;
 }
 

core/libs/prx/libc/src/specifics/windows/CrashReport.cpp

@@ -381,6 +381,9 @@ LONG WINAPI ReportCrash(EXCEPTION_POINTERS* info) {
     if (record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && record->NumberParameters >= 2) {
         const char* kind = record->ExceptionInformation[0] == 0 ? "read" : record->ExceptionInformation[0] == 1 ? "write" : "execute";
         Report("  %s of 0x%016llx\n", kind, static_cast<unsigned long long>(record->ExceptionInformation[1]));
+        std::uint8_t trap[2];
+        if (record->ExceptionInformation[0] == 0 && record->ExceptionInformation[1] == ~0ull && ReadCode(context->Rip, trap, sizeof(trap)) == sizeof(trap) && trap[0] == 0xcd && trap[1] == 0x45)
+            Report("  guest abort() (int 0x45)\n");
     }
     Report("  rax %016llx rbx %016llx rcx %016llx rdx %016llx\n", context->Rax, context->Rbx, context->Rcx, context->Rdx);
     Report("  rsi %016llx rdi %016llx rbp %016llx rsp %016llx\n", context->Rsi, context->Rdi, context->Rbp, context->Rsp);

core/libs/prx/libkernel/Aio/src/Aio.cpp

@@ -134,9 +134,9 @@ int SubmitCommands(KernelAioRwRequest* req, std::int32_t size, bool write, std::
 }
 
 int SubmitCommandsMultiple(KernelAioRwRequest* req, std::int32_t size, bool write, std::int32_t* id, const char* name) {
+    if (req != nullptr && id != nullptr && size > AioRequestNumMax) throw std::runtime_error(std::string(name) + ": more than 128 requests per batch is not modelled");
     const int error = ValidateRequests(req, size, id);
     if (error != 0) return error;
-    if (size > AioRequestNumMax) throw std::runtime_error(std::string(name) + ": more than 128 requests per batch is not modelled");
     for (std::int32_t i = 0; i < size; ++i) {
         const std::int32_t queue = AllocateId();
         SetState(queue, RunRequest(req[i], write) ? AioCompleted : AioAborted);

core/libs/prx/libkernel/CMakeLists.txt

@@ -238,7 +238,7 @@ if(BUILD_TESTING)
     endif()
     add_test(NAME guest_dynamic_loader COMMAND guest_dynamic_loader_tests $<TARGET_FILE_NAME:guest_module_fixture>)
     set_tests_properties(guest_dynamic_loader PROPERTIES
-        WORKING_DIRECTORY "$<TARGET_FILE_DIR:guest_module_fixture>")
+        WORKING_DIRECTORY "$<TARGET_FILE_DIR:guest_module_fixture>" RESOURCE_LOCK guest_dynamic_loader_files)
     if(WIN32)
         set_tests_properties(guest_dynamic_loader PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
@@ -271,7 +271,7 @@ if(BUILD_TESTING)
         target_link_options(guest_dynamic_loader_fixed_base_tests PRIVATE -no-pie -fuse-ld=bfd)
         add_test(NAME guest_dynamic_loader_fixed_base COMMAND guest_dynamic_loader_fixed_base_tests $<TARGET_FILE_NAME:guest_module_fixture>)
         set_tests_properties(guest_dynamic_loader_fixed_base PROPERTIES
-            WORKING_DIRECTORY "$<TARGET_FILE_DIR:guest_module_fixture>")
+            WORKING_DIRECTORY "$<TARGET_FILE_DIR:guest_module_fixture>" RESOURCE_LOCK guest_dynamic_loader_files)
     endif()
     add_test_executable(guest_module_info_tests ${LIBS_INCLUDE_DIR}/tests/GuestModuleInfo.cpp)
     target_include_directories(guest_module_info_tests PRIVATE ${LIBS_INCLUDE_DIR})
@@ -587,6 +587,16 @@ if(BUILD_TESTING)
         set_tests_properties(guest_pthread_start PROPERTIES TIMEOUT 60 RUN_SERIAL TRUE ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    add_test_executable(guest_pthread_join_detach_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadJoinDetach.cpp)
+    target_include_directories(guest_pthread_join_detach_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_pthread_join_detach_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_pthread_join_detach_tests)
+    add_test(NAME guest_pthread_join_detach COMMAND guest_pthread_join_detach_tests)
+    set_tests_properties(guest_pthread_join_detach PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_join_detach PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
     add_test_executable(guest_pthread_attr_errors_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadAttrErrors.cpp)
     target_include_directories(guest_pthread_attr_errors_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_pthread_attr_errors_tests PRIVATE libkernel libc)

core/libs/prx/libkernel/DirectMemory/Export.cpp

@@ -123,7 +123,20 @@ int _mapFlexible(void** addr, size_t len, int prot, int flags) {
     const size_t configured = _configuredFlexibleSize();
     const size_t images = GuestImageSegmentBytes_nid_no_patch();
     std::lock_guard lock(g_flexibleLock);
-    const size_t used = images + _flexibleUsedLocked();
+    size_t replaced = 0;
+    constexpr int MapFixed = 0x10;
+    constexpr int MapNoOverwrite = 0x80;
+    if (addr != nullptr && *addr != nullptr && (flags & (MapFixed | MapNoOverwrite)) == MapFixed) {
+        const auto start = reinterpret_cast<uintptr_t>(*addr);
+        if (len <= std::numeric_limits<uintptr_t>::max() - start) {
+            const auto end = start + len;
+            for (const auto& [rangeStart, rangeLength] : g_flexibleRanges) {
+                const auto rangeEnd = rangeStart + rangeLength;
+                if (rangeStart < end && rangeEnd > start) replaced += std::min(end, rangeEnd) - std::max(start, rangeStart);
+            }
+        }
+    }
+    const size_t used = images + _flexibleUsedLocked() - replaced;
     if (len > configured - std::min(configured, used)) {
         std::fprintf(stderr, "[memory] flexible memory exhausted: request 0x%zx with 0x%zx of 0x%zx in use\n", len, used, configured);
         return SCE_KERNEL_ERROR_ENOMEM;

core/libs/prx/libkernel/File/src/Stdio.cpp

@@ -1323,24 +1323,8 @@ int64_t APS5_VABI pwritev_nid_postfix(int d, const KernelIovec* iov, int iovcnt,
 
 int APS5_VABI sceKernelRename(const char* from, const char* to) {
     if (!from || !to) return SceErrorFromErrno(GUEST_EFAULT);
-    if (*from == '\0' || *to == '\0') return SceErrorFromErrno(GUEST_ENOENT);
-    std::filesystem::path source;
-    if (const int error = ResolvePathError_nid_no_patch(from, source)) return SceKernelError(error);
-    std::error_code error;
-    const auto sourceStatus = std::filesystem::status(source, error);
-    if (!std::filesystem::exists(sourceStatus)) return SceErrorFromErrno(GUEST_ENOENT);
-    std::filesystem::path destination;
-    if (const int result = ResolvePathError_nid_no_patch(to, destination)) return SceKernelError(result);
-    bool replacesFile = false;
-#ifdef _WIN32
-    const auto destinationStatus = std::filesystem::symlink_status(destination, error);
-    replacesFile = !std::filesystem::is_directory(sourceStatus) && std::filesystem::exists(destinationStatus) && !std::filesystem::is_directory(destinationStatus);
-#endif
-    error = RenameNative_nid_no_patch(source, destination, replacesFile);
-    if (error) return SceErrorFromErrno(GUEST_EIO);
-    RecordWrittenPath_nid_no_patch(source);
-    RecordWrittenPath_nid_no_patch(destination);
-    return 0;
+    const int result = RenameGuestPath_nid_no_patch(from, to);
+    return result == 0 ? 0 : SceKernelError(errno);
 }
 
 int APS5_VABI sceKernelRmdir(const char* path) {

core/libs/prx/libkernel/Module/src/DynamicLoader.cpp

@@ -2,6 +2,7 @@
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "prx/libkernel/Module/ModuleArgs.hpp"
 #include <nid/NidCompute.hpp>
+#include <algorithm>
 #include <array>
 #include <condition_variable>
 #include <filesystem>
@@ -39,6 +40,9 @@ struct Module {
     void* native = nullptr;
     bool owned = true;
     bool global = false;
+#ifdef _WIN32
+    std::uintptr_t globalOrder = 0;
+#endif
     ~Module() {
         if (owned && native) {
 #ifdef _WIN32
@@ -193,7 +197,21 @@ void* APS5_VABI dlopen_nid_postfix(const char* path, int flags) {
 #endif
         std::lock_guard lock(modulesMutex);
         const auto handle = nextHandle++;
-        modules.emplace(handle, std::move(module));
+#ifdef _WIN32
+        module->globalOrder = module->global ? handle : 0;
+        for (const auto& [key, entry] : modules) {
+            if (entry->native == module->native && entry->globalOrder != 0 &&
+                (module->globalOrder == 0 || entry->globalOrder < module->globalOrder)) module->globalOrder = entry->globalOrder;
+        }
+#endif
+        modules.emplace(handle, module);
+#ifdef _WIN32
+        for (const auto& [key, entry] : modules) {
+            if (entry->native != module->native) continue;
+            entry->global = module->globalOrder != 0;
+            entry->globalOrder = module->globalOrder;
+        }
+#endif
         return reinterpret_cast<void*>(handle);
     } catch (const std::exception& error) { Error(error.what()); return nullptr; }
 }
@@ -205,6 +223,10 @@ void* APS5_VABI dlsym_nid_postfix(void* handle, const char* name) {
             std::lock_guard lock(modulesMutex);
             if (handle == nullptr || handle == reinterpret_cast<void*>(static_cast<std::intptr_t>(-2))) {
                 for (const auto& [key, module] : modules) if (module->global) search.push_back(module);
+#ifdef _WIN32
+                std::sort(search.begin(), search.end(), [](const auto& left, const auto& right) { return left->globalOrder < right->globalOrder; });
+                search.erase(std::unique(search.begin(), search.end(), [](const auto& left, const auto& right) { return left->native == right->native; }), search.end());
+#endif
             } else {
                 auto found = modules.find(reinterpret_cast<std::uintptr_t>(handle));
                 if (found == modules.end()) { Error("dlsym: invalid or unsupported module handle"); return nullptr; }